<commit_before>package filepathfilter

import (
	"fmt"
	"path/filepath"
	"regexp"
	"strings"
)

type Pattern interface {
	Match(filename string) bool
}

type Filter struct {
	include []Pattern
	exclude []Pattern
}

func NewFromPatterns(include, exclude []Pattern) *Filter {
	return &Filter{include: include, exclude: exclude}
}

func New(include, exclude []string) *Filter {
	return NewFromPatterns(convertToPatterns(include), convertToPatterns(exclude))
}

func (f *Filter) Allows(filename string) bool {
	if f == nil {
		return true
	}

	if len(f.include)+len(f.exclude) == 0 {
		return true
	}

	cleanedName := filepath.Clean(filename)

	if len(f.include) > 0 {
		matched := false
		for _, inc := range f.include {
			matched = inc.Match(cleanedName)
			if matched {
				break
			}
		}
		if !matched {
			return false
		}
	}

	if len(f.exclude) > 0 {
		for _, ex := range f.exclude {
			if ex.Match(cleanedName) {
				return false
			}
		}
	}

	return true
}

func NewPattern(rawpattern string) Pattern {
	cleanpattern := filepath.Clean(rawpattern)

	// Special case local dir, matches all (inc subpaths)
	if _, local := localDirSet[cleanpattern]; local {
		return noOpMatcher{}
	}

	sep := string(filepath.Separator)
	hasPathSep := strings.Contains(cleanpattern, sep)
	ext := filepath.Ext(cleanpattern)
	plen := len(cleanpattern)
	if plen > 1 && !hasPathSep && strings.HasPrefix(cleanpattern, "*") && cleanpattern[1:plen] == ext {
		return &simpleExtPattern{ext: ext}
	}

	// special case * when there are no path separators
	// filepath.Match never allows * to match a path separator, which is correct
	// for gitignore IF the pattern includes a path separator, but not otherwise
	// So *.txt should match in any subdir, as should test*, but sub/*.txt would
	// only match directly in the sub dir
	// Don't need to test cross-platform separators as both cleaned above
	if !hasPathSep && strings.Contains(cleanpattern, "*") {
		pattern := regexp.QuoteMeta(cleanpattern)
		regpattern := fmt.Sprintf("^%s$", strings.Replace(pattern, "\\*", ".*", -1))
		return &pathlessWildcardPattern{
			rawPattern: cleanpattern,
			wildcardRE: regexp.MustCompile(regpattern),
		}
	}

	// Also support ** with path separators
	if hasPathSep && strings.Contains(cleanpattern, "**") {
		pattern := regexp.QuoteMeta(cleanpattern)
		regpattern := fmt.Sprintf("^%s$", strings.Replace(pattern, "\\*\\*", ".*", -1))
		return &doubleWildcardPattern{
			rawPattern: cleanpattern,
			wildcardRE: regexp.MustCompile(regpattern),
		}
	}

	if hasPathSep && strings.HasPrefix(cleanpattern, sep) {
		rel := cleanpattern[1:len(cleanpattern)]
		prefix := rel
		if strings.HasSuffix(rel, sep) {
			rel = rel[0 : len(rel)-1]
		} else {
			prefix += sep
		}

		return &pathPrefixPattern{
			rawPattern: cleanpattern,
			relative:   rel,
			prefix:     prefix,
		}
	}

	return &pathPattern{
		rawPattern: cleanpattern,
		prefix:     cleanpattern + sep,
		suffix:     sep + cleanpattern,
		inner:      sep + cleanpattern + sep,
	}
}

func convertToPatterns(rawpatterns []string) []Pattern {
	patterns := make([]Pattern, len(rawpatterns))
	for i, raw := range rawpatterns {
		patterns[i] = NewPattern(raw)
	}
	return patterns
}

type pathPrefixPattern struct {
	rawPattern string
	relative   string
	prefix     string
}

// Match is a revised version of filepath.Match which makes it behave more
// like gitignore
func (p *pathPrefixPattern) Match(name string) bool {
	if name == p.relative || strings.HasPrefix(name, p.prefix) {
		return true
	}
	matched, _ := filepath.Match(p.rawPattern, name)
	return matched
}

// String returns a string representation of the underlying pattern for which
// this *pathPrefixPattern is matching.
func (p *pathPrefixPattern) String() string {
	return p.rawPattern
}

type pathPattern struct {
	rawPattern string
	prefix     string
	suffix     string
	inner      string
}

// Match is a revised version of filepath.Match which makes it behave more
// like gitignore
func (p *pathPattern) Match(name string) bool {
	if strings.HasPrefix(name, p.prefix) || strings.HasSuffix(name, p.suffix) || strings.Contains(name, p.inner) {
		return true
	}
	matched, _ := filepath.Match(p.rawPattern, name)
	return matched
}

// String returns a string representation of the underlying pattern for which
// this *pathPattern is matching.
func (p *pathPattern) String() string {
	return p.rawPattern
}

type simpleExtPattern struct {
	ext string
}

func (p *simpleExtPattern) Match(name string) bool {
	return strings.HasSuffix(name, p.ext)
}

// String returns a string representation of the underlying pattern for which
// this *simpleExtPattern is matching.
func (p *simpleExtPattern) String() string {
	return fmt.Sprintf("*%s", p.ext)
}

type pathlessWildcardPattern struct {
	rawPattern string
	wildcardRE *regexp.Regexp
}

// Match is a revised version of filepath.Match which makes it behave more
// like gitignore
func (p *pathlessWildcardPattern) Match(name string) bool {
	matched, _ := filepath.Match(p.rawPattern, name)
	// Match the whole of the base name but allow matching in folders if no path
	return matched || p.wildcardRE.MatchString(filepath.Base(name))
}

// String returns a string representation of the underlying pattern for which
// this *pathlessWildcardPattern is matching.
func (p *pathlessWildcardPattern) String() string {
	return p.rawPattern
}

type doubleWildcardPattern struct {
	rawPattern string
	wildcardRE *regexp.Regexp
}

// Match is a revised version of filepath.Match which makes it behave more
// like gitignore
func (p *doubleWildcardPattern) Match(name string) bool {
	matched, _ := filepath.Match(p.rawPattern, name)
	// Match the whole of the base name but allow matching in folders if no path
	return matched || p.wildcardRE.MatchString(name)
}

// String returns a string representation of the underlying pattern for which
// this *doubleWildcardPattern is matching.
func (p *doubleWildcardPattern) String() string {
	return p.rawPattern
}

type noOpMatcher struct {
}

func (n noOpMatcher) Match(name string) bool {
	return true
}

func (n noOpMatcher) String() string {
	return ""
}

var localDirSet = map[string]struct{}{
	".":   struct{}{},
	"./":  struct{}{},
	".\\": struct{}{},
}
<commit_msg>filepathfilter: add String() to Pattern interface<commit_after>package filepathfilter

import (
	"fmt"
	"path/filepath"
	"regexp"
	"strings"
)

type Pattern interface {
	Match(filename string) bool
	// String returns a string representation (see: regular expressions) of
	// the underlying pattern used to match filenames against this Pattern.
	String() string
}

type Filter struct {
	include []Pattern
	exclude []Pattern
}

func NewFromPatterns(include, exclude []Pattern) *Filter {
	return &Filter{include: include, exclude: exclude}
}

func New(include, exclude []string) *Filter {
	return NewFromPatterns(convertToPatterns(include), convertToPatterns(exclude))
}

func (f *Filter) Allows(filename string) bool {
	if f == nil {
		return true
	}

	if len(f.include)+len(f.exclude) == 0 {
		return true
	}

	cleanedName := filepath.Clean(filename)

	if len(f.include) > 0 {
		matched := false
		for _, inc := range f.include {
			matched = inc.Match(cleanedName)
			if matched {
				break
			}
		}
		if !matched {
			return false
		}
	}

	if len(f.exclude) > 0 {
		for _, ex := range f.exclude {
			if ex.Match(cleanedName) {
				return false
			}
		}
	}

	return true
}

func NewPattern(rawpattern string) Pattern {
	cleanpattern := filepath.Clean(rawpattern)

	// Special case local dir, matches all (inc subpaths)
	if _, local := localDirSet[cleanpattern]; local {
		return noOpMatcher{}
	}

	sep := string(filepath.Separator)
	hasPathSep := strings.Contains(cleanpattern, sep)
	ext := filepath.Ext(cleanpattern)
	plen := len(cleanpattern)
	if plen > 1 && !hasPathSep && strings.HasPrefix(cleanpattern, "*") && cleanpattern[1:plen] == ext {
		return &simpleExtPattern{ext: ext}
	}

	// special case * when there are no path separators
	// filepath.Match never allows * to match a path separator, which is correct
	// for gitignore IF the pattern includes a path separator, but not otherwise
	// So *.txt should match in any subdir, as should test*, but sub/*.txt would
	// only match directly in the sub dir
	// Don't need to test cross-platform separators as both cleaned above
	if !hasPathSep && strings.Contains(cleanpattern, "*") {
		pattern := regexp.QuoteMeta(cleanpattern)
		regpattern := fmt.Sprintf("^%s$", strings.Replace(pattern, "\\*", ".*", -1))
		return &pathlessWildcardPattern{
			rawPattern: cleanpattern,
			wildcardRE: regexp.MustCompile(regpattern),
		}
	}

	// Also support ** with path separators
	if hasPathSep && strings.Contains(cleanpattern, "**") {
		pattern := regexp.QuoteMeta(cleanpattern)
		regpattern := fmt.Sprintf("^%s$", strings.Replace(pattern, "\\*\\*", ".*", -1))
		return &doubleWildcardPattern{
			rawPattern: cleanpattern,
			wildcardRE: regexp.MustCompile(regpattern),
		}
	}

	if hasPathSep && strings.HasPrefix(cleanpattern, sep) {
		rel := cleanpattern[1:len(cleanpattern)]
		prefix := rel
		if strings.HasSuffix(rel, sep) {
			rel = rel[0 : len(rel)-1]
		} else {
			prefix += sep
		}

		return &pathPrefixPattern{
			rawPattern: cleanpattern,
			relative:   rel,
			prefix:     prefix,
		}
	}

	return &pathPattern{
		rawPattern: cleanpattern,
		prefix:     cleanpattern + sep,
		suffix:     sep + cleanpattern,
		inner:      sep + cleanpattern + sep,
	}
}

func convertToPatterns(rawpatterns []string) []Pattern {
	patterns := make([]Pattern, len(rawpatterns))
	for i, raw := range rawpatterns {
		patterns[i] = NewPattern(raw)
	}
	return patterns
}

type pathPrefixPattern struct {
	rawPattern string
	relative   string
	prefix     string
}

// Match is a revised version of filepath.Match which makes it behave more
// like gitignore
func (p *pathPrefixPattern) Match(name string) bool {
	if name == p.relative || strings.HasPrefix(name, p.prefix) {
		return true
	}
	matched, _ := filepath.Match(p.rawPattern, name)
	return matched
}

// String returns a string representation of the underlying pattern for which
// this *pathPrefixPattern is matching.
func (p *pathPrefixPattern) String() string {
	return p.rawPattern
}

type pathPattern struct {
	rawPattern string
	prefix     string
	suffix     string
	inner      string
}

// Match is a revised version of filepath.Match which makes it behave more
// like gitignore
func (p *pathPattern) Match(name string) bool {
	if strings.HasPrefix(name, p.prefix) || strings.HasSuffix(name, p.suffix) || strings.Contains(name, p.inner) {
		return true
	}
	matched, _ := filepath.Match(p.rawPattern, name)
	return matched
}

// String returns a string representation of the underlying pattern for which
// this *pathPattern is matching.
func (p *pathPattern) String() string {
	return p.rawPattern
}

type simpleExtPattern struct {
	ext string
}

func (p *simpleExtPattern) Match(name string) bool {
	return strings.HasSuffix(name, p.ext)
}

// String returns a string representation of the underlying pattern for which
// this *simpleExtPattern is matching.
func (p *simpleExtPattern) String() string {
	return fmt.Sprintf("*%s", p.ext)
}

type pathlessWildcardPattern struct {
	rawPattern string
	wildcardRE *regexp.Regexp
}

// Match is a revised version of filepath.Match which makes it behave more
// like gitignore
func (p *pathlessWildcardPattern) Match(name string) bool {
	matched, _ := filepath.Match(p.rawPattern, name)
	// Match the whole of the base name but allow matching in folders if no path
	return matched || p.wildcardRE.MatchString(filepath.Base(name))
}

// String returns a string representation of the underlying pattern for which
// this *pathlessWildcardPattern is matching.
func (p *pathlessWildcardPattern) String() string {
	return p.rawPattern
}

type doubleWildcardPattern struct {
	rawPattern string
	wildcardRE *regexp.Regexp
}

// Match is a revised version of filepath.Match which makes it behave more
// like gitignore
func (p *doubleWildcardPattern) Match(name string) bool {
	matched, _ := filepath.Match(p.rawPattern, name)
	// Match the whole of the base name but allow matching in folders if no path
	return matched || p.wildcardRE.MatchString(name)
}

// String returns a string representation of the underlying pattern for which
// this *doubleWildcardPattern is matching.
func (p *doubleWildcardPattern) String() string {
	return p.rawPattern
}

type noOpMatcher struct {
}

func (n noOpMatcher) Match(name string) bool {
	return true
}

func (n noOpMatcher) String() string {
	return ""
}

var localDirSet = map[string]struct{}{
	".":   struct{}{},
	"./":  struct{}{},
	".\\": struct{}{},
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"testing"

	"github.com/kylelemons/godebug/diff"
	"gopkg.in/yaml.v2"
)

// This file runs tests in the testdata directory, excluding those in testdata/single-file-tests

func TestCLISessions(t *testing.T) {
	godebug := compileGodebug(t)
	defer os.Remove(godebug)

	// Read the testdata directory
	fd, err := os.Open("testdata")
	if err != nil {
		t.Fatal(err)
	}
	defer fd.Close()
	names, err := fd.Readdirnames(-1)
	if err != nil {
		t.Fatal("Readdirnames:", err)
	}
	tests := make([]string, 0, len(names))
	for _, name := range names {
		if strings.HasSuffix(name, ".yaml") {
			tests = append(tests, name)
		}
	}

	// Run tests in parallel
	var wg sync.WaitGroup
	for _, test := range tests {
		for _, tt := range parseCases(t, filepath.Join("testdata", test)) {
			s := parseSessionFromBytes([]byte(tt.Transcript))
			for i := range tt.Invocations {
				wg.Add(1)
				go func(filename string, s *session, tt testCase, i int) {
					defer wg.Done()
					runTest(t, godebug, filename, tt, i, s)
				}(test, s, tt, i)
			}
		}
	}
	wg.Wait()
}

type testCase struct {
	Invocations []struct {
		Dir, Cmd string
	}
	Desc, Transcript string
	Creates          []string
	NonzeroExit      bool `yaml:"nonzero_exit"`
	Godebugwork      bool
}

func parseCases(t *testing.T, filename string) []testCase {
	b, err := ioutil.ReadFile(filename)
	if err != nil {
		t.Fatal(err)
	}
	bb := bytes.Split(b, []byte("\n---\n"))
	testCases := make([]testCase, len(bb))
	for i := range bb {
		if err = yaml.Unmarshal(bb[i], &testCases[i]); err != nil {
			fmt.Println(string(bb[i]))
			t.Fatal(err)
		}
	}
	return testCases
}

func runTest(t *testing.T, godebug, filename string, tt testCase, i int, session *session) {
	var buf bytes.Buffer
	command, dir := tt.Invocations[i].Cmd, tt.Invocations[i].Dir
	cmd := exec.Command(godebug, strings.Split(command, " ")[1:]...)
	cmd.Dir = filepath.FromSlash("testdata/test-filesystem/" + dir)
	cmd.Stdout = &buf
	cmd.Stderr = &buf
	cmd.Stdin = bytes.NewReader(session.input)
	setTestGopath(t, cmd)

	// Show multiple errors if they exist and format them nicely.
	var errs []string
	defer func() {
		if errs != nil {
			t.Errorf("File: %s\nDescription: %s\nWorking dir: %s\nCommand: %s\nFailures:\n\t%v",
				filename, tt.Desc, dir, command, strings.Join(errs, "\n\t"))
		}
	}()

	cmd.Env = append(cmd.Env, logFileEnvVar+"=true")
	err := cmd.Run()
	// Because we set `logFileEnvVar` above, godebug will print the
	// files it creates to stdout. Parse those lines and then pretend
	// they were not printed.
	createdFiles, output := recordCreatedFiles(buf.Bytes())

	switch err.(type) {
	case nil:
		if tt.NonzeroExit {
			errs = append(errs, "got exit code == 0, wanted a nonzero exit code.")
			return
		}
	case *exec.ExitError:
		if !tt.NonzeroExit {
			errs = append(errs, fmt.Sprintf("%q failed to run: %v\n%s", command, err, output))
			return
		}
	default:
		errs = append(errs, fmt.Sprintf("%q failed to run: %v\n%s", command, err, output))
		return
	}

	// Check that we created the files we expected and did not create
	// any files we did not expect.
	errs = append(errs, checkCreatedFiles(t, createdFiles, tt.Creates)...)

	if tt.Godebugwork {
		output, err = checkGodebugwork(t, session.fullSession, output)
		if err != nil {
			errs = append(errs, err.Error())
		}
	}

	got := interleaveCommands(session.input, output)
	if equivalent(got, session.fullSession) {
		return
	}
	errs = append(errs, fmt.Sprintf("golden transcript did not match actual transcript. Diff:\n\n%v", diff.Diff(string(session.fullSession), string(got))))
}

func checkGodebugwork(t *testing.T, transcript, output []byte) ([]byte, error) {
	if !bytes.HasPrefix(transcript, []byte("$TMP\n")) {
		return output, errors.New(`incorrect test: set "godebugwork: true" but did not prepend "$TMP\n" to the output`)
	}

	tmpDir := string(bytes.SplitN(output, newline, 2)[0])
	if !strings.HasPrefix(tmpDir, os.TempDir()) {
		return output, fmt.Errorf("got %q as first line of output, expected a temporary directory", tmpDir)
	}

	_, err := os.Stat(tmpDir)
	if os.IsNotExist(err) {
		return output, fmt.Errorf("godebug deleted the temporary directory %q when -godebugwork was passed", tmpDir)
	}

	if err != nil {
		return output, fmt.Errorf("failed to stat temporary directory %q:  %s", tmpDir, err)
	}

	output = append([]byte("$TMP\n"), output[len(tmpDir)+1:]...)
	if err = os.RemoveAll(tmpDir); err != nil {
		return output, fmt.Errorf("failed to remove temporary directory: %v", err)
	}

	return output, nil
}

func checkCreatedFiles(t *testing.T, g, w []string) (errs []string) {
	got, want := listToMap(g), listToMap(w)
	for f := range got {
		if !want[f] {
			errs = append(errs, "created a file we did not want: "+f)
		}
	}
	for f := range want {
		if !got[f] {
			errs = append(errs, "did not create a file we wanted: "+f)
		}
	}
	return errs
}

func recordCreatedFiles(b []byte) (files []string, rest []byte) {
	bb := bytes.Split(b, newline)

	for i := range bb {
		if bytes.HasPrefix(bb[i], []byte(logFilePrefix)) {
			files = append(files, string(bb[i][len(logFilePrefix):]))
		} else {
			rest = append(rest, bb[i]...)
			if i+1 < len(bb) {
				rest = append(rest, newline...)
			}
		}
	}
	return files, rest
}

func listToMap(list []string) map[string]bool {
	m := make(map[string]bool)
	for _, s := range list {
		m[s] = true
	}
	return m
}

// equivalent does a linewise comparison of a and b.
// Each line must be exactly equal or the want line must end in "//substr"
// and be a substring of the got line.
// Otherwise equivalent returns false.
func equivalent(got, want []byte) bool {
	var (
		gotLines  = bytes.Split(got, newline)
		wantLines = bytes.Split(want, newline)
		substr    = []byte("//substr")
		gg, ww    []byte
	)

	if len(gotLines) != len(wantLines) {
		return false
	}

	for i := range gotLines {
		gg, ww = gotLines[i], wantLines[i]
		if !(bytes.Equal(gg, ww) || bytes.HasSuffix(ww, substr) && bytes.Contains(gg, ww[:len(ww)-len(substr)])) {
			return false
		}
	}
	return true
}

func setTestGopath(t *testing.T, cmd *exec.Cmd) {
	cwd, err := os.Getwd()
	if err != nil {
		t.Fatal(err)
	}
	setGopath(cmd, filepath.Join(cwd, "testdata", "test-filesystem", "gopath"))
}
<commit_msg>small portability fix<commit_after>package main

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"testing"

	"github.com/kylelemons/godebug/diff"
	"gopkg.in/yaml.v2"
)

// This file runs tests in the testdata directory, excluding those in testdata/single-file-tests

func TestCLISessions(t *testing.T) {
	godebug := compileGodebug(t)
	defer os.Remove(godebug)

	// Read the testdata directory
	fd, err := os.Open("testdata")
	if err != nil {
		t.Fatal(err)
	}
	defer fd.Close()
	names, err := fd.Readdirnames(-1)
	if err != nil {
		t.Fatal("Readdirnames:", err)
	}
	tests := make([]string, 0, len(names))
	for _, name := range names {
		if strings.HasSuffix(name, ".yaml") {
			tests = append(tests, name)
		}
	}

	// Run tests in parallel
	var wg sync.WaitGroup
	for _, test := range tests {
		for _, tt := range parseCases(t, filepath.Join("testdata", test)) {
			s := parseSessionFromBytes([]byte(tt.Transcript))
			for i := range tt.Invocations {
				wg.Add(1)
				go func(filename string, s *session, tt testCase, i int) {
					defer wg.Done()
					runTest(t, godebug, filename, tt, i, s)
				}(test, s, tt, i)
			}
		}
	}
	wg.Wait()
}

type testCase struct {
	Invocations []struct {
		Dir, Cmd string
	}
	Desc, Transcript string
	Creates          []string
	NonzeroExit      bool `yaml:"nonzero_exit"`
	Godebugwork      bool
}

func parseCases(t *testing.T, filename string) []testCase {
	b, err := ioutil.ReadFile(filename)
	if err != nil {
		t.Fatal(err)
	}
	bb := bytes.Split(b, []byte("\n---\n"))
	testCases := make([]testCase, len(bb))
	for i := range bb {
		if err = yaml.Unmarshal(bb[i], &testCases[i]); err != nil {
			fmt.Println(string(bb[i]))
			t.Fatal(err)
		}
	}
	return testCases
}

func runTest(t *testing.T, godebug, filename string, tt testCase, i int, session *session) {
	var buf bytes.Buffer
	command, dir := tt.Invocations[i].Cmd, tt.Invocations[i].Dir
	cmd := exec.Command(godebug, strings.Split(command, " ")[1:]...)
	cmd.Dir = filepath.FromSlash("testdata/test-filesystem/" + dir)
	cmd.Stdout = &buf
	cmd.Stderr = &buf
	cmd.Stdin = bytes.NewReader(session.input)
	setTestGopath(t, cmd)

	// Show multiple errors if they exist and format them nicely.
	var errs []string
	defer func() {
		if errs != nil {
			t.Errorf("File: %s\nDescription: %s\nWorking dir: %s\nCommand: %s\nFailures:\n\t%v",
				filename, tt.Desc, dir, command, strings.Join(errs, "\n\t"))
		}
	}()

	cmd.Env = append(cmd.Env, logFileEnvVar+"=true")
	err := cmd.Run()
	// Because we set `logFileEnvVar` above, godebug will print the
	// files it creates to stdout. Parse those lines and then pretend
	// they were not printed.
	createdFiles, output := recordCreatedFiles(buf.Bytes())

	switch err.(type) {
	case nil:
		if tt.NonzeroExit {
			errs = append(errs, "got exit code == 0, wanted a nonzero exit code.")
			return
		}
	case *exec.ExitError:
		if !tt.NonzeroExit {
			errs = append(errs, fmt.Sprintf("%q failed to run: %v\n%s", command, err, output))
			return
		}
	default:
		errs = append(errs, fmt.Sprintf("%q failed to run: %v\n%s", command, err, output))
		return
	}

	// Check that we created the files we expected and did not create
	// any files we did not expect.
	errs = append(errs, checkCreatedFiles(t, createdFiles, tt.Creates)...)

	if tt.Godebugwork {
		output, err = checkGodebugwork(t, session.fullSession, output)
		if err != nil {
			errs = append(errs, err.Error())
		}
	}

	got := interleaveCommands(session.input, output)
	if equivalent(got, session.fullSession) {
		return
	}
	errs = append(errs, fmt.Sprintf("golden transcript did not match actual transcript. Diff:\n\n%v", diff.Diff(string(session.fullSession), string(got))))
}

func checkGodebugwork(t *testing.T, transcript, output []byte) ([]byte, error) {
	if !bytes.HasPrefix(transcript, []byte("$TMP\n")) {
		return output, errors.New(`incorrect test: set "godebugwork: true" but did not prepend "$TMP\n" to the output`)
	}

	tmpDir := string(bytes.SplitN(output, newline, 2)[0])
	if !strings.HasPrefix(tmpDir, os.TempDir()) {
		return output, fmt.Errorf("got %q as first line of output, expected a temporary directory", tmpDir)
	}

	_, err := os.Stat(tmpDir)
	if os.IsNotExist(err) {
		return output, fmt.Errorf("godebug deleted the temporary directory %q when -godebugwork was passed", tmpDir)
	}

	if err != nil {
		return output, fmt.Errorf("failed to stat temporary directory %q:  %s", tmpDir, err)
	}

	output = append([]byte("$TMP\n"), output[len(tmpDir)+1:]...)
	if err = os.RemoveAll(tmpDir); err != nil {
		return output, fmt.Errorf("failed to remove temporary directory: %v", err)
	}

	return output, nil
}

func checkCreatedFiles(t *testing.T, g, w []string) (errs []string) {
	for i, f := range w {
		w[i] = filepath.FromSlash(f)
	}
	got, want := listToMap(g), listToMap(w)
	for f := range got {
		if !want[f] {
			errs = append(errs, "created a file we did not want: "+f)
		}
	}
	for f := range want {
		if !got[f] {
			errs = append(errs, "did not create a file we wanted: "+f)
		}
	}
	return errs
}

func recordCreatedFiles(b []byte) (files []string, rest []byte) {
	bb := bytes.Split(b, newline)

	for i := range bb {
		if bytes.HasPrefix(bb[i], []byte(logFilePrefix)) {
			files = append(files, string(bb[i][len(logFilePrefix):]))
		} else {
			rest = append(rest, bb[i]...)
			if i+1 < len(bb) {
				rest = append(rest, newline...)
			}
		}
	}
	return files, rest
}

func listToMap(list []string) map[string]bool {
	m := make(map[string]bool)
	for _, s := range list {
		m[s] = true
	}
	return m
}

// equivalent does a linewise comparison of a and b.
// Each line must be exactly equal or the want line must end in "//substr"
// and be a substring of the got line.
// Otherwise equivalent returns false.
func equivalent(got, want []byte) bool {
	var (
		gotLines  = bytes.Split(got, newline)
		wantLines = bytes.Split(want, newline)
		substr    = []byte("//substr")
		gg, ww    []byte
	)

	if len(gotLines) != len(wantLines) {
		return false
	}

	for i := range gotLines {
		gg, ww = gotLines[i], wantLines[i]
		if !(bytes.Equal(gg, ww) || bytes.HasSuffix(ww, substr) && bytes.Contains(gg, ww[:len(ww)-len(substr)])) {
			return false
		}
	}
	return true
}

func setTestGopath(t *testing.T, cmd *exec.Cmd) {
	cwd, err := os.Getwd()
	if err != nil {
		t.Fatal(err)
	}
	setGopath(cmd, filepath.Join(cwd, "testdata", "test-filesystem", "gopath"))
}
<|endoftext|>


<commit_before>// Copyright © 2017 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package device

import (
	"reflect"
	"time"

	"github.com/TheThingsNetwork/ttn/core/types"
	"github.com/fatih/structs"
)

const currentDBVersion = "2.4.1"

// Options for the specified device
type Options struct {
	ActivationConstraints string `json:"activation_constraints,omitempty"` // Activation Constraints (public/local/private)
	DisableFCntCheck      bool   `json:"disable_fcnt_check,omitemtpy"`     // Disable Frame counter check (insecure)
	Uses32BitFCnt         bool   `json:"uses_32_bit_fcnt,omitemtpy"`       // Use 32-bit Frame counters
}

// Device contains the state of a device
type Device struct {
	old *Device

	DevEUI   types.DevEUI  `redis:"dev_eui"`
	AppEUI   types.AppEUI  `redis:"app_eui"`
	AppID    string        `redis:"app_id"`
	DevID    string        `redis:"dev_id"`
	DevAddr  types.DevAddr `redis:"dev_addr"`
	NwkSKey  types.NwkSKey `redis:"nwk_s_key"`
	FCntUp   uint32        `redis:"f_cnt_up"`
	FCntDown uint32        `redis:"f_cnt_down"`
	LastSeen time.Time     `redis:"last_seen"`
	Options  Options       `redis:"options"`
	ADR      ADRSettings   `redis:"adr,include"`

	CreatedAt time.Time    `redis:"created_at"`
	UpdatedAt time.Time    `redis:"updated_at"`
	ActivatedAt time.Time  `redis:"activated_at"` // Indicates whether the device was activated via OTAA method
}

// ADRSettings contains the (desired) settings for a device that uses ADR
type ADRSettings struct {
	Band   string `redis:"band"`
	Margin int    `redis:"margin"`

	// Indicates whether the NetworkServer should send a LinkADRReq when possible
	SentInitial      bool `redis:"sent_initial"`
	ConfirmedInitial bool `redis:"confirmed_initial"`
	SendReq          bool `redis:"send_req"`
	ExpectRes        bool `redis:"expect_res"`
	Failed           int  `redis:"failed"` // number of failed ADR attempts

	// Desired Settings:
	DataRate string `redis:"data_rate"`
	TxPower  int    `redis:"tx_power"`
	NbTrans  int    `redis:"nb_trans"`
}

// StartUpdate stores the state of the device
func (d *Device) StartUpdate() {
	old := *d
	d.old = &old
}

// DBVersion of the model
func (d *Device) DBVersion() string {
	return currentDBVersion
}

// ChangedFields returns the names of the changed fields since the last call to StartUpdate
func (d Device) ChangedFields() (changed []string) {
	new := structs.New(d)
	fields := new.Names()
	if d.old == nil {
		return fields
	}
	old := structs.New(*d.old)

	for _, field := range new.Fields() {
		if !field.IsExported() || field.Name() == "old" {
			continue
		}
		if !reflect.DeepEqual(field.Value(), old.Field(field.Name()).Value()) {
			changed = append(changed, field.Name())
			if field.Kind() == reflect.Struct {
				oldSubField := structs.New(old.Field(field.Name()).Value())
				for _, subField := range field.Fields() {
					if !subField.IsExported() {
						continue
					}
					if !reflect.DeepEqual(subField.Value(), oldSubField.Field(subField.Name()).Value()) {
						changed = append(changed, field.Name()+"."+subField.Name())
					}
				}
			}
		}
	}

	if len(changed) == 1 && changed[0] == "UpdatedAt" {
		return []string{}
	}

	return
}
<commit_msg>Format code<commit_after>// Copyright © 2017 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package device

import (
	"reflect"
	"time"

	"github.com/TheThingsNetwork/ttn/core/types"
	"github.com/fatih/structs"
)

const currentDBVersion = "2.4.1"

// Options for the specified device
type Options struct {
	ActivationConstraints string `json:"activation_constraints,omitempty"` // Activation Constraints (public/local/private)
	DisableFCntCheck      bool   `json:"disable_fcnt_check,omitemtpy"`     // Disable Frame counter check (insecure)
	Uses32BitFCnt         bool   `json:"uses_32_bit_fcnt,omitemtpy"`       // Use 32-bit Frame counters
}

// Device contains the state of a device
type Device struct {
	old *Device

	DevEUI   types.DevEUI  `redis:"dev_eui"`
	AppEUI   types.AppEUI  `redis:"app_eui"`
	AppID    string        `redis:"app_id"`
	DevID    string        `redis:"dev_id"`
	DevAddr  types.DevAddr `redis:"dev_addr"`
	NwkSKey  types.NwkSKey `redis:"nwk_s_key"`
	FCntUp   uint32        `redis:"f_cnt_up"`
	FCntDown uint32        `redis:"f_cnt_down"`
	LastSeen time.Time     `redis:"last_seen"`
	Options  Options       `redis:"options"`
	ADR      ADRSettings   `redis:"adr,include"`

	CreatedAt   time.Time `redis:"created_at"`
	UpdatedAt   time.Time `redis:"updated_at"`
	ActivatedAt time.Time `redis:"activated_at"` // Indicates whether the device was activated via OTAA method
}

// ADRSettings contains the (desired) settings for a device that uses ADR
type ADRSettings struct {
	Band   string `redis:"band"`
	Margin int    `redis:"margin"`

	// Indicates whether the NetworkServer should send a LinkADRReq when possible
	SentInitial      bool `redis:"sent_initial"`
	ConfirmedInitial bool `redis:"confirmed_initial"`
	SendReq          bool `redis:"send_req"`
	ExpectRes        bool `redis:"expect_res"`
	Failed           int  `redis:"failed"` // number of failed ADR attempts

	// Desired Settings:
	DataRate string `redis:"data_rate"`
	TxPower  int    `redis:"tx_power"`
	NbTrans  int    `redis:"nb_trans"`
}

// StartUpdate stores the state of the device
func (d *Device) StartUpdate() {
	old := *d
	d.old = &old
}

// DBVersion of the model
func (d *Device) DBVersion() string {
	return currentDBVersion
}

// ChangedFields returns the names of the changed fields since the last call to StartUpdate
func (d Device) ChangedFields() (changed []string) {
	new := structs.New(d)
	fields := new.Names()
	if d.old == nil {
		return fields
	}
	old := structs.New(*d.old)

	for _, field := range new.Fields() {
		if !field.IsExported() || field.Name() == "old" {
			continue
		}
		if !reflect.DeepEqual(field.Value(), old.Field(field.Name()).Value()) {
			changed = append(changed, field.Name())
			if field.Kind() == reflect.Struct {
				oldSubField := structs.New(old.Field(field.Name()).Value())
				for _, subField := range field.Fields() {
					if !subField.IsExported() {
						continue
					}
					if !reflect.DeepEqual(subField.Value(), oldSubField.Field(subField.Name()).Value()) {
						changed = append(changed, field.Name()+"."+subField.Name())
					}
				}
			}
		}
	}

	if len(changed) == 1 && changed[0] == "UpdatedAt" {
		return []string{}
	}

	return
}
<|endoftext|>


<commit_before>package client

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strings"
)

var (
	server     string
	jobstepID  string
	workspace  string
	runAsChild bool
	debug      bool
)

type ConfigCmd struct {
	ID            string
	Script        string
	Env           map[string]string
	Cwd           string
	Artifacts     []string
	CaptureOutput bool
	Type          struct {
		ID string
	}
}

type Config struct {
	Server    string
	JobstepID string
	Workspace string
	// TODO(dcramer): remove RunAsChild entirely once new CLI is written
	RunAsChild bool
	Debug      bool
	Source     struct {
		Revision struct {
			Sha string
		}
		Patch struct {
			ID string
		}
	}
	Repository struct {
		URL     string
		Backend struct {
			ID string
		}
	}
	Cmds []ConfigCmd `json:"commands"`
}

func fetchConfig(url string) (*Config, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		err := fmt.Errorf("Request to fetch config failed with status code: %d", resp.StatusCode)
		return nil, err
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	return LoadConfig(body)
}

func LoadConfig(content []byte) (*Config, error) {
	r := &Config{}
	err := json.Unmarshal(content, r)
	if err != nil {
		return nil, err
	}

	return r, nil
}

func GetConfig() (*Config, error) {
	var err error

	server = strings.TrimRight(server, "/")

	url := server + "/jobsteps/" + jobstepID + "/"
	conf, err := fetchConfig(url)
	if err != nil {
		return nil, err
	}

	if workspace == "" {
		cwd, err := os.Getwd()
		if err != nil {
			panic("Unable to find working directory")
		}
		workspace, err = filepath.Abs(cwd)
	} else {
		workspace, err = filepath.Abs(workspace)
	}
	if err != nil {
		panic("Unable to find working directory")
	}

	conf.Server = server
	conf.JobstepID = jobstepID
	conf.Workspace = workspace
	conf.RunAsChild = runAsChild
	conf.Debug = debug

	return conf, err
}

func init() {
	flag.StringVar(&server, "server", "", "URL to get config from")
	flag.StringVar(&jobstepID, "jobstep_id", "", "Job ID whose commands are to be executed")
	flag.StringVar(&workspace, "workspace", "", "Workspace to checkout source into")
	// TODO(dcramer): we need to define a spec for what the behavior of run-as-child
	// is, specifically what it will still report and what it wont
	// Currently this only disables jobstep status updates.
	flag.BoolVar(&runAsChild, "run_as_child", false, "Indicates that the client is running as a child of another wrapper")
	flag.BoolVar(&debug, "debug", false, "Indicates that the client is running in debug mode and should not report results upstream")
}
<commit_msg>Remove RunAsChild<commit_after>package client

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strings"
)

var (
	server     string
	jobstepID  string
	workspace  string
	runAsChild bool
	debug      bool
)

type ConfigCmd struct {
	ID            string
	Script        string
	Env           map[string]string
	Cwd           string
	Artifacts     []string
	CaptureOutput bool
	Type          struct {
		ID string
	}
}

type Config struct {
	Server    string
	JobstepID string
	Workspace string
	Debug      bool
	Source     struct {
		Revision struct {
			Sha string
		}
		Patch struct {
			ID string
		}
	}
	Repository struct {
		URL     string
		Backend struct {
			ID string
		}
	}
	Cmds []ConfigCmd `json:"commands"`
}

func fetchConfig(url string) (*Config, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		err := fmt.Errorf("Request to fetch config failed with status code: %d", resp.StatusCode)
		return nil, err
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	return LoadConfig(body)
}

func LoadConfig(content []byte) (*Config, error) {
	r := &Config{}
	err := json.Unmarshal(content, r)
	if err != nil {
		return nil, err
	}

	return r, nil
}

func GetConfig() (*Config, error) {
	var err error

	server = strings.TrimRight(server, "/")

	url := server + "/jobsteps/" + jobstepID + "/"
	conf, err := fetchConfig(url)
	if err != nil {
		return nil, err
	}

	if workspace == "" {
		cwd, err := os.Getwd()
		if err != nil {
			panic("Unable to find working directory")
		}
		workspace, err = filepath.Abs(cwd)
	} else {
		workspace, err = filepath.Abs(workspace)
	}
	if err != nil {
		panic("Unable to find working directory")
	}

	conf.Server = server
	conf.JobstepID = jobstepID
	conf.Workspace = workspace
	conf.Debug = debug

	return conf, err
}

func init() {
	flag.StringVar(&server, "server", "", "URL to get config from")
	flag.StringVar(&jobstepID, "jobstep_id", "", "Job ID whose commands are to be executed")
	flag.StringVar(&workspace, "workspace", "", "Workspace to checkout source into")
	flag.BoolVar(&debug, "debug", false, "Indicates that the client is running in debug mode and should not report results upstream")
}
<|endoftext|>


<commit_before>package etcd

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"path"
	"strings"
	"sync"
	"time"
)

// Errors introduced by handling requests
var (
	ErrRequestCancelled = errors.New("sending request is cancelled")
)

type RawRequest struct {
	Method       string
	RelativePath string
	Values       url.Values
	Cancel       <-chan bool
}

// NewRawRequest returns a new RawRequest
func NewRawRequest(method, relativePath string, values url.Values, cancel <-chan bool) *RawRequest {
	return &RawRequest{
		Method:       method,
		RelativePath: relativePath,
		Values:       values,
		Cancel:       cancel,
	}
}

// getCancelable issues a cancelable GET request
func (c *Client) getCancelable(key string, options Options,
	cancel <-chan bool) (*RawResponse, error) {
	logger.Debugf("get %s [%s]", key, c.cluster.pick())
	p := keyToPath(key)

	str, err := options.toParameters(VALID_GET_OPTIONS)
	if err != nil {
		return nil, err
	}
	p += str

	req := NewRawRequest("GET", p, nil, cancel)
	resp, err := c.SendRequest(req)

	if err != nil {
		return nil, err
	}

	return resp, nil
}

// get issues a GET request
func (c *Client) get(key string, options Options) (*RawResponse, error) {
	return c.getCancelable(key, options, nil)
}

// put issues a PUT request
func (c *Client) put(key string, value string, ttl uint64,
	options Options) (*RawResponse, error) {

	logger.Debugf("put %s, %s, ttl: %d, [%s]", key, value, ttl, c.cluster.pick())
	p := keyToPath(key)

	str, err := options.toParameters(VALID_PUT_OPTIONS)
	if err != nil {
		return nil, err
	}
	p += str

	req := NewRawRequest("PUT", p, buildValues(value, ttl), nil)
	resp, err := c.SendRequest(req)

	if err != nil {
		return nil, err
	}

	return resp, nil
}

// post issues a POST request
func (c *Client) post(key string, value string, ttl uint64) (*RawResponse, error) {
	logger.Debugf("post %s, %s, ttl: %d, [%s]", key, value, ttl, c.cluster.pick())
	p := keyToPath(key)

	req := NewRawRequest("POST", p, buildValues(value, ttl), nil)
	resp, err := c.SendRequest(req)

	if err != nil {
		return nil, err
	}

	return resp, nil
}

// delete issues a DELETE request
func (c *Client) delete(key string, options Options) (*RawResponse, error) {
	logger.Debugf("delete %s [%s]", key, c.cluster.pick())
	p := keyToPath(key)

	str, err := options.toParameters(VALID_DELETE_OPTIONS)
	if err != nil {
		return nil, err
	}
	p += str

	req := NewRawRequest("DELETE", p, nil, nil)
	resp, err := c.SendRequest(req)

	if err != nil {
		return nil, err
	}

	return resp, nil
}

// SendRequest sends a HTTP request and returns a Response as defined by etcd
func (c *Client) SendRequest(rr *RawRequest) (*RawResponse, error) {
	var req *http.Request
	var resp *http.Response
	var httpPath string
	var err error
	var respBody []byte

	var numReqs = 1

	checkRetry := c.CheckRetry
	if checkRetry == nil {
		checkRetry = DefaultCheckRetry
	}

	cancelled := make(chan bool, 1)
	reqLock := new(sync.Mutex)

	if rr.Cancel != nil {
		cancelRoutine := make(chan bool)
		defer close(cancelRoutine)

		go func() {
			select {
			case <-rr.Cancel:
				cancelled <- true
				logger.Debug("send.request is cancelled")
			case <-cancelRoutine:
				return
			}

			// Repeat canceling request until this thread is stopped
			// because we have no idea about whether it succeeds.
			for {
				reqLock.Lock()
				c.httpClient.Transport.(*http.Transport).CancelRequest(req)
				reqLock.Unlock()

				select {
				case <-time.After(100 * time.Millisecond):
				case <-cancelRoutine:
					return
				}
			}
		}()
	}

	// If we connect to a follower and consistency is required, retry until
	// we connect to a leader
	sleep := 25 * time.Millisecond
	maxSleep := time.Second

	for attempt := 0; ; attempt++ {
		if attempt > 0 {
			select {
			case <-cancelled:
				return nil, ErrRequestCancelled
			case <-time.After(sleep):
				sleep = sleep * 2
				if sleep > maxSleep {
					sleep = maxSleep
				}
			}
		}

		logger.Debug("Connecting to etcd: attempt ", attempt+1, " for ", rr.RelativePath)

		httpPath = c.getHttpPath(rr.RelativePath)

		// Return a cURL command if curlChan is set
		if c.cURLch != nil {
			command := fmt.Sprintf("curl -X %s %s", rr.Method, httpPath)
			for key, value := range rr.Values {
				command += fmt.Sprintf(" -d %s=%s", key, value[0])
			}
			if c.credentials != nil {
				command += fmt.Sprintf(" -u %s", c.credentials.username)
			}
			c.sendCURL(command)
		}

		logger.Debug("send.request.to ", httpPath, " | method ", rr.Method)

		req, err := func() (*http.Request, error) {
			reqLock.Lock()
			defer reqLock.Unlock()

			if rr.Values == nil {
				if req, err = http.NewRequest(rr.Method, httpPath, nil); err != nil {
					return nil, err
				}
			} else {
				body := strings.NewReader(rr.Values.Encode())
				if req, err = http.NewRequest(rr.Method, httpPath, body); err != nil {
					return nil, err
				}

				req.Header.Set("Content-Type",
					"application/x-www-form-urlencoded; param=value")
			}
			return req, nil
		}()

		if err != nil {
			return nil, err
		}

		if c.credentials != nil {
			req.SetBasicAuth(c.credentials.username, c.credentials.password)
		}

		resp, err = c.httpClient.Do(req)
		defer func() {
			if resp != nil {
				resp.Body.Close()
			}
		}()

		// If the request was cancelled, return ErrRequestCancelled directly
		select {
		case <-cancelled:
			return nil, ErrRequestCancelled
		default:
		}

		numReqs++

		// network error, change a machine!
		if err != nil {
			logger.Debug("network error: ", err.Error())
			lastResp := http.Response{}
			if checkErr := checkRetry(c.cluster, numReqs, lastResp, err); checkErr != nil {
				return nil, checkErr
			}

			c.cluster.failure()
			continue
		}

		// if there is no error, it should receive response
		logger.Debug("recv.response.from ", httpPath)

		if validHttpStatusCode[resp.StatusCode] {
			// try to read byte code and break the loop
			respBody, err = ioutil.ReadAll(resp.Body)
			if err == nil {
				logger.Debug("recv.success ", httpPath)
				break
			}
			// ReadAll error may be caused due to cancel request
			select {
			case <-cancelled:
				return nil, ErrRequestCancelled
			default:
			}

			if err == io.ErrUnexpectedEOF {
				// underlying connection was closed prematurely, probably by timeout
				// TODO: empty body or unexpectedEOF can cause http.Transport to get hosed;
				// this allows the client to detect that and take evasive action. Need
				// to revisit once code.google.com/p/go/issues/detail?id=8648 gets fixed.
				respBody = []byte{}
				break
			}
		}

		if checkErr := checkRetry(c.cluster, numReqs, *resp,
			errors.New("Unexpected HTTP status code")); checkErr != nil {
			return nil, checkErr
		}
		resp.Body.Close()
	}

	r := &RawResponse{
		StatusCode: resp.StatusCode,
		Body:       respBody,
		Header:     resp.Header,
	}

	return r, nil
}

// DefaultCheckRetry defines the retrying behaviour for bad HTTP requests
// If we have retried 2 * machine number, stop retrying.
// If status code is InternalServerError, sleep for 200ms.
func DefaultCheckRetry(cluster *Cluster, numReqs int, lastResp http.Response,
	err error) error {

	if isEmptyResponse(lastResp) {
		if !isConnectionError(err) {
			return err
		}
	} else if !shouldRetry(lastResp) {
		body := []byte("nil")
		if lastResp.Body != nil {
			if b, err := ioutil.ReadAll(lastResp.Body); err == nil {
				body = b
			}
		}
		errStr := fmt.Sprintf("unhandled http status [%s] with body [%s]", http.StatusText(lastResp.StatusCode), body)
		return newError(ErrCodeUnhandledHTTPStatus, errStr, 0)
	}

	if numReqs > 2*len(cluster.Machines) {
		errStr := fmt.Sprintf("failed to propose on members %v twice [last error: %v]", cluster.Machines, err)
		return newError(ErrCodeEtcdNotReachable, errStr, 0)
	}
	if shouldRetry(lastResp) {
		// sleep some time and expect leader election finish
		time.Sleep(time.Millisecond * 200)
	}

	logger.Warning("bad response status code", lastResp.StatusCode)
	return nil
}

func isEmptyResponse(r http.Response) bool { return r.StatusCode == 0 }

func isConnectionError(err error) bool {
	_, ok := err.(*net.OpError)
	return ok
}

// shouldRetry returns whether the reponse deserves retry.
func shouldRetry(r http.Response) bool {
	// TODO: only retry when the cluster is in leader election
	// We cannot do it exactly because etcd doesn't support it well.
	return r.StatusCode == http.StatusInternalServerError
}

func (c *Client) getHttpPath(s ...string) string {
	fullPath := c.cluster.pick() + "/" + version
	for _, seg := range s {
		fullPath = fullPath + "/" + seg
	}
	return fullPath
}

// buildValues builds a url.Values map according to the given value and ttl
func buildValues(value string, ttl uint64) url.Values {
	v := url.Values{}

	if value != "" {
		v.Set("value", value)
	}

	if ttl > 0 {
		v.Set("ttl", fmt.Sprintf("%v", ttl))
	}

	return v
}

// convert key string to http path exclude version, including URL escaping
// for example: key[foo] -> path[keys/foo]
// key[/%z] -> path[keys/%25z]
// key[/] -> path[keys/]
func keyToPath(key string) string {
	// URL-escape our key, except for slashes
	p := strings.Replace(url.QueryEscape(path.Join("keys", key)), "%2F", "/", -1)

	// corner case: if key is "/" or "//" ect
	// path join will clear the tailing "/"
	// we need to add it back
	if p == "keys" {
		p = "keys/"
	}

	return p
}
<commit_msg>request: follow http redirection<commit_after>package etcd

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"path"
	"strings"
	"sync"
	"time"
)

// Errors introduced by handling requests
var (
	ErrRequestCancelled = errors.New("sending request is cancelled")
)

type RawRequest struct {
	Method       string
	RelativePath string
	Values       url.Values
	Cancel       <-chan bool
}

// NewRawRequest returns a new RawRequest
func NewRawRequest(method, relativePath string, values url.Values, cancel <-chan bool) *RawRequest {
	return &RawRequest{
		Method:       method,
		RelativePath: relativePath,
		Values:       values,
		Cancel:       cancel,
	}
}

// getCancelable issues a cancelable GET request
func (c *Client) getCancelable(key string, options Options,
	cancel <-chan bool) (*RawResponse, error) {
	logger.Debugf("get %s [%s]", key, c.cluster.pick())
	p := keyToPath(key)

	str, err := options.toParameters(VALID_GET_OPTIONS)
	if err != nil {
		return nil, err
	}
	p += str

	req := NewRawRequest("GET", p, nil, cancel)
	resp, err := c.SendRequest(req)

	if err != nil {
		return nil, err
	}

	return resp, nil
}

// get issues a GET request
func (c *Client) get(key string, options Options) (*RawResponse, error) {
	return c.getCancelable(key, options, nil)
}

// put issues a PUT request
func (c *Client) put(key string, value string, ttl uint64,
	options Options) (*RawResponse, error) {

	logger.Debugf("put %s, %s, ttl: %d, [%s]", key, value, ttl, c.cluster.pick())
	p := keyToPath(key)

	str, err := options.toParameters(VALID_PUT_OPTIONS)
	if err != nil {
		return nil, err
	}
	p += str

	req := NewRawRequest("PUT", p, buildValues(value, ttl), nil)
	resp, err := c.SendRequest(req)

	if err != nil {
		return nil, err
	}

	return resp, nil
}

// post issues a POST request
func (c *Client) post(key string, value string, ttl uint64) (*RawResponse, error) {
	logger.Debugf("post %s, %s, ttl: %d, [%s]", key, value, ttl, c.cluster.pick())
	p := keyToPath(key)

	req := NewRawRequest("POST", p, buildValues(value, ttl), nil)
	resp, err := c.SendRequest(req)

	if err != nil {
		return nil, err
	}

	return resp, nil
}

// delete issues a DELETE request
func (c *Client) delete(key string, options Options) (*RawResponse, error) {
	logger.Debugf("delete %s [%s]", key, c.cluster.pick())
	p := keyToPath(key)

	str, err := options.toParameters(VALID_DELETE_OPTIONS)
	if err != nil {
		return nil, err
	}
	p += str

	req := NewRawRequest("DELETE", p, nil, nil)
	resp, err := c.SendRequest(req)

	if err != nil {
		return nil, err
	}

	return resp, nil
}

// SendRequest sends a HTTP request and returns a Response as defined by etcd
func (c *Client) SendRequest(rr *RawRequest) (*RawResponse, error) {
	var req *http.Request
	var resp *http.Response
	var httpPath string
	var err error
	var respBody []byte

	var numReqs = 1

	checkRetry := c.CheckRetry
	if checkRetry == nil {
		checkRetry = DefaultCheckRetry
	}

	cancelled := make(chan bool, 1)
	reqLock := new(sync.Mutex)

	if rr.Cancel != nil {
		cancelRoutine := make(chan bool)
		defer close(cancelRoutine)

		go func() {
			select {
			case <-rr.Cancel:
				cancelled <- true
				logger.Debug("send.request is cancelled")
			case <-cancelRoutine:
				return
			}

			// Repeat canceling request until this thread is stopped
			// because we have no idea about whether it succeeds.
			for {
				reqLock.Lock()
				c.httpClient.Transport.(*http.Transport).CancelRequest(req)
				reqLock.Unlock()

				select {
				case <-time.After(100 * time.Millisecond):
				case <-cancelRoutine:
					return
				}
			}
		}()
	}

	// If we connect to a follower and consistency is required, retry until
	// we connect to a leader
	sleep := 25 * time.Millisecond
	maxSleep := time.Second

	for attempt := 0; ; attempt++ {
		if attempt > 0 {
			select {
			case <-cancelled:
				return nil, ErrRequestCancelled
			case <-time.After(sleep):
				sleep = sleep * 2
				if sleep > maxSleep {
					sleep = maxSleep
				}
			}
		}

		logger.Debug("Connecting to etcd: attempt ", attempt+1, " for ", rr.RelativePath)

		// get httpPath if not set
		if httpPath == "" {
			httpPath = c.getHttpPath(rr.RelativePath)
		}

		// Return a cURL command if curlChan is set
		if c.cURLch != nil {
			command := fmt.Sprintf("curl -X %s %s", rr.Method, httpPath)
			for key, value := range rr.Values {
				command += fmt.Sprintf(" -d %s=%s", key, value[0])
			}
			if c.credentials != nil {
				command += fmt.Sprintf(" -u %s", c.credentials.username)
			}
			c.sendCURL(command)
		}

		logger.Debug("send.request.to ", httpPath, " | method ", rr.Method)

		req, err := func() (*http.Request, error) {
			reqLock.Lock()
			defer reqLock.Unlock()

			if rr.Values == nil {
				if req, err = http.NewRequest(rr.Method, httpPath, nil); err != nil {
					return nil, err
				}
			} else {
				body := strings.NewReader(rr.Values.Encode())
				if req, err = http.NewRequest(rr.Method, httpPath, body); err != nil {
					return nil, err
				}

				req.Header.Set("Content-Type",
					"application/x-www-form-urlencoded; param=value")
			}
			return req, nil
		}()

		if err != nil {
			return nil, err
		}

		if c.credentials != nil {
			req.SetBasicAuth(c.credentials.username, c.credentials.password)
		}

		resp, err = c.httpClient.Do(req)
		// clear previous httpPath
		httpPath = ""
		defer func() {
			if resp != nil {
				resp.Body.Close()
			}
		}()

		// If the request was cancelled, return ErrRequestCancelled directly
		select {
		case <-cancelled:
			return nil, ErrRequestCancelled
		default:
		}

		numReqs++

		// network error, change a machine!
		if err != nil {
			logger.Debug("network error: ", err.Error())
			lastResp := http.Response{}
			if checkErr := checkRetry(c.cluster, numReqs, lastResp, err); checkErr != nil {
				return nil, checkErr
			}

			c.cluster.failure()
			continue
		}

		// if there is no error, it should receive response
		logger.Debug("recv.response.from ", httpPath)

		if validHttpStatusCode[resp.StatusCode] {
			// try to read byte code and break the loop
			respBody, err = ioutil.ReadAll(resp.Body)
			if err == nil {
				logger.Debug("recv.success ", httpPath)
				break
			}
			// ReadAll error may be caused due to cancel request
			select {
			case <-cancelled:
				return nil, ErrRequestCancelled
			default:
			}

			if err == io.ErrUnexpectedEOF {
				// underlying connection was closed prematurely, probably by timeout
				// TODO: empty body or unexpectedEOF can cause http.Transport to get hosed;
				// this allows the client to detect that and take evasive action. Need
				// to revisit once code.google.com/p/go/issues/detail?id=8648 gets fixed.
				respBody = []byte{}
				break
			}
		}

		if resp.StatusCode == http.StatusTemporaryRedirect {
			u, err := resp.Location()

			if err != nil {
				logger.Warning(err)
			} else {
				// set httpPath for following redirection
				httpPath = u.String()
			}
			resp.Body.Close()
			continue
		}

		if checkErr := checkRetry(c.cluster, numReqs, *resp,
			errors.New("Unexpected HTTP status code")); checkErr != nil {
			return nil, checkErr
		}
		resp.Body.Close()
	}

	r := &RawResponse{
		StatusCode: resp.StatusCode,
		Body:       respBody,
		Header:     resp.Header,
	}

	return r, nil
}

// DefaultCheckRetry defines the retrying behaviour for bad HTTP requests
// If we have retried 2 * machine number, stop retrying.
// If status code is InternalServerError, sleep for 200ms.
func DefaultCheckRetry(cluster *Cluster, numReqs int, lastResp http.Response,
	err error) error {

	if isEmptyResponse(lastResp) {
		if !isConnectionError(err) {
			return err
		}
	} else if !shouldRetry(lastResp) {
		body := []byte("nil")
		if lastResp.Body != nil {
			if b, err := ioutil.ReadAll(lastResp.Body); err == nil {
				body = b
			}
		}
		errStr := fmt.Sprintf("unhandled http status [%s] with body [%s]", http.StatusText(lastResp.StatusCode), body)
		return newError(ErrCodeUnhandledHTTPStatus, errStr, 0)
	}

	if numReqs > 2*len(cluster.Machines) {
		errStr := fmt.Sprintf("failed to propose on members %v twice [last error: %v]", cluster.Machines, err)
		return newError(ErrCodeEtcdNotReachable, errStr, 0)
	}
	if shouldRetry(lastResp) {
		// sleep some time and expect leader election finish
		time.Sleep(time.Millisecond * 200)
	}

	logger.Warning("bad response status code", lastResp.StatusCode)
	return nil
}

func isEmptyResponse(r http.Response) bool { return r.StatusCode == 0 }

func isConnectionError(err error) bool {
	_, ok := err.(*net.OpError)
	return ok
}

// shouldRetry returns whether the reponse deserves retry.
func shouldRetry(r http.Response) bool {
	// TODO: only retry when the cluster is in leader election
	// We cannot do it exactly because etcd doesn't support it well.
	return r.StatusCode == http.StatusInternalServerError
}

func (c *Client) getHttpPath(s ...string) string {
	fullPath := c.cluster.pick() + "/" + version
	for _, seg := range s {
		fullPath = fullPath + "/" + seg
	}
	return fullPath
}

// buildValues builds a url.Values map according to the given value and ttl
func buildValues(value string, ttl uint64) url.Values {
	v := url.Values{}

	if value != "" {
		v.Set("value", value)
	}

	if ttl > 0 {
		v.Set("ttl", fmt.Sprintf("%v", ttl))
	}

	return v
}

// convert key string to http path exclude version, including URL escaping
// for example: key[foo] -> path[keys/foo]
// key[/%z] -> path[keys/%25z]
// key[/] -> path[keys/]
func keyToPath(key string) string {
	// URL-escape our key, except for slashes
	p := strings.Replace(url.QueryEscape(path.Join("keys", key)), "%2F", "/", -1)

	// corner case: if key is "/" or "//" ect
	// path join will clear the tailing "/"
	// we need to add it back
	if p == "keys" {
		p = "keys/"
	}

	return p
}
<|endoftext|>


<commit_before>package eventexporter

type FakeExporter struct {
	Events []*Event
}

func NewFakeExporter() *FakeExporter {
	return &FakeExporter{Events: []*Event{}}
}

func (l *FakeExporter) Send(event *Event) error {
	l.Events = append(l.Events, event)
	return nil
}

func (l *FakeExporter) Close() error {
	return nil
}
<commit_msg>fake: add name function<commit_after>package eventexporter

type FakeExporter struct {
	Events []*Event
}

func NewFakeExporter() *FakeExporter {
	return &FakeExporter{Events: []*Event{}}
}

func (l *FakeExporter) Send(event *Event) error {
	l.Events = append(l.Events, event)
	return nil
}

// Name returns the name of the exporter.
func (FakeExporter) Name() string { return "fake" }

func (l *FakeExporter) Close() error {
	return nil
}
<|endoftext|>


<commit_before>/*Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package event

import (
	"errors"
	"fmt"
	"github.com/skyrings/skyring-common/conf"
	"github.com/skyrings/skyring-common/db"
	"github.com/skyrings/skyring-common/models"
	"github.com/skyrings/skyring-common/tools/logger"
	"gopkg.in/mgo.v2/bson"
	"time"
)

func Persist_event(event models.Event, ctxt string) error {
	sessionCopy := db.GetDatastore().Copy()
	defer sessionCopy.Close()
	coll := sessionCopy.DB(conf.SystemConfig.DBConfig.Database).C(models.COLL_NAME_NODE_EVENTS)
	if err := coll.Insert(event); err != nil {
		logger.Get().Error("%s-Error adding the node event: %v", ctxt, err)
		return err
	}
	return nil
}

func ClearCorrespondingAlert(event models.AppEvent, ctxt string) (models.AlarmStatus, error) {
	var events []models.AppEvent
	sessionCopy := db.GetDatastore().Copy()
	defer sessionCopy.Close()
	coll := sessionCopy.DB(conf.SystemConfig.DBConfig.Database).C(models.COLL_NAME_APP_EVENTS)
	count := 0
	for count < 3 {
		if err := coll.Find(bson.M{
			"name":      event.Name,
			"entityid":  event.EntityId,
			"clusterid": event.ClusterId}).Sort("-timestamp").All(&events); err != nil {
			logger.Get().Error("%s-Error getting record from DB: %v", ctxt, err)
			return models.ALARM_STATUS_INDETERMINATE, err
		}
		if len(events) == 0 || events[0].Severity == models.ALARM_STATUS_CLEARED {
			if event.Severity != models.ALARM_STATUS_CLEARED {
				break
			}
			time.Sleep(180 * time.Second)
			count = count + 1
		} else {
			break
		}
	}
	if len(events) == 0 || events[0].Severity == models.ALARM_STATUS_CLEARED {
		return models.ALARM_STATUS_INDETERMINATE, errors.New(fmt.Sprintf("Corresponding alert not available for event: %s",
			event.EventId))
	} else {
		events[0].Acked = true
		events[0].SystemAckedTime = time.Now()
		events[0].AckedByEvent = event.EventId.String()
		events[0].SystemAckComment = fmt.Sprintf("This event is dismissed automatically,"+
			" as we have recieved a corresponding event: %s", event.EventId.String())

		if err := coll.Update(bson.M{"eventid": events[0].EventId},
			bson.M{"$set": events[0]}); err != nil {
			logger.Get().Warning(fmt.Sprintf("%s-Error updating record in DB for event:"+
				" %v. error: %v", ctxt, events[0].EventId.String(), err))
			return models.ALARM_STATUS_INDETERMINATE, errors.New(fmt.Sprintf("%s-Error updating"+
				" record in DB for event: %v. error: %v",
				ctxt, events[0].EventId.String(), err))
		}

	}
	return events[0].Severity, nil
}

func UpdateNodeAlarmCount(event models.AppEvent, clearedSeverity models.AlarmStatus, ctxt string) error {
	var node models.Node
	sessionCopy := db.GetDatastore().Copy()
	defer sessionCopy.Close()
	coll := sessionCopy.DB(conf.SystemConfig.DBConfig.Database).C(models.COLL_NAME_STORAGE_NODES)
	if err := coll.Find(bson.M{
		"nodeid": event.NodeId, "clusterid": event.ClusterId}).One(&node); err != nil {
		logger.Get().Error("%s-Error getting record from DB: %v", ctxt, err)
		return err
	}

	if event.Severity == models.ALARM_STATUS_MAJOR ||
		event.Severity == models.ALARM_STATUS_CRITICAL {
		node.AlmCritCount += 1
	} else if event.Severity == models.ALARM_STATUS_MINOR ||
		event.Severity == models.ALARM_STATUS_WARNING {
		node.AlmWarnCount += 1
	}

	if clearedSeverity == models.ALARM_STATUS_MAJOR ||
		clearedSeverity == models.ALARM_STATUS_CRITICAL {
		if node.AlmCritCount > 0 {
			node.AlmCritCount -= 1
		}
	} else if clearedSeverity == models.ALARM_STATUS_MINOR ||
		clearedSeverity == models.ALARM_STATUS_WARNING {
		if node.AlmWarnCount > 0 {
			node.AlmWarnCount -= 1
		}
	}

	if node.AlmCritCount > 0 {
		node.AlmStatus = models.ALARM_STATUS_CRITICAL
	} else if node.AlmWarnCount > 0 {
		node.AlmStatus = models.ALARM_STATUS_WARNING
	} else {
		node.AlmStatus = models.ALARM_STATUS_CLEARED
	}

	if err := coll.Update(bson.M{"nodeid": event.NodeId},
		bson.M{"$set": node}); err != nil {
		logger.Get().Error("%s-Error Updating the Alarm state/count: %v", ctxt, err)
		return err
	}
	return nil
}

func UpdateClusterAlarmCount(event models.AppEvent, clearedSeverity models.AlarmStatus, ctxt string) error {
	var cluster models.Cluster
	sessionCopy := db.GetDatastore().Copy()
	defer sessionCopy.Close()
	coll := sessionCopy.DB(conf.SystemConfig.DBConfig.Database).C(models.COLL_NAME_STORAGE_CLUSTERS)
	if err := coll.Find(bson.M{
		"clusterid": event.ClusterId}).One(&cluster); err != nil {
		logger.Get().Error("%s-Error getting record from DB: %v", ctxt, err)
		return err
	}

	if event.Severity == models.ALARM_STATUS_MAJOR ||
		event.Severity == models.ALARM_STATUS_CRITICAL {
		cluster.AlmCritCount += 1
	} else if event.Severity == models.ALARM_STATUS_MINOR ||
		event.Severity == models.ALARM_STATUS_WARNING {
		cluster.AlmWarnCount += 1
	}

	if clearedSeverity == models.ALARM_STATUS_MAJOR ||
		clearedSeverity == models.ALARM_STATUS_CRITICAL {
		if cluster.AlmCritCount > 0 {
			cluster.AlmCritCount -= 1
		}
	} else if clearedSeverity == models.ALARM_STATUS_MINOR ||
		clearedSeverity == models.ALARM_STATUS_WARNING {
		if cluster.AlmWarnCount > 0 {
			cluster.AlmWarnCount -= 1
		}
	}

	if cluster.AlmCritCount > 0 {
		cluster.AlmStatus = models.ALARM_STATUS_CRITICAL
	} else if cluster.AlmWarnCount > 0 {
		cluster.AlmStatus = models.ALARM_STATUS_WARNING
	} else {
		cluster.AlmStatus = models.ALARM_STATUS_CLEARED
	}

	if err := coll.Update(bson.M{"clusterid": event.ClusterId},
		bson.M{"$set": cluster}); err != nil {
		logger.Get().Error("%s-Error Updating the Alarm state/count: %v", ctxt, err)
		return err
	}
	return nil
}
<commit_msg>AlertAggregation: Added alert aggregator for pool and osds.<commit_after>/*Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package event

import (
	"errors"
	"fmt"
	"github.com/skyrings/skyring-common/conf"
	"github.com/skyrings/skyring-common/db"
	"github.com/skyrings/skyring-common/models"
	"github.com/skyrings/skyring-common/tools/logger"
	"gopkg.in/mgo.v2/bson"
	"time"
)

func Persist_event(event models.Event, ctxt string) error {
	sessionCopy := db.GetDatastore().Copy()
	defer sessionCopy.Close()
	coll := sessionCopy.DB(conf.SystemConfig.DBConfig.Database).C(models.COLL_NAME_NODE_EVENTS)
	if err := coll.Insert(event); err != nil {
		logger.Get().Error("%s-Error adding the node event: %v", ctxt, err)
		return err
	}
	return nil
}

func ClearCorrespondingAlert(event models.AppEvent, ctxt string) (models.AlarmStatus, error) {
	var events []models.AppEvent
	sessionCopy := db.GetDatastore().Copy()
	defer sessionCopy.Close()
	coll := sessionCopy.DB(conf.SystemConfig.DBConfig.Database).C(models.COLL_NAME_APP_EVENTS)
	count := 0
	for count < 3 {
		if err := coll.Find(bson.M{
			"name":      event.Name,
			"entityid":  event.EntityId,
			"clusterid": event.ClusterId}).Sort("-timestamp").All(&events); err != nil {
			logger.Get().Error("%s-Error getting record from DB: %v", ctxt, err)
			return models.ALARM_STATUS_INDETERMINATE, err
		}
		if len(events) == 0 || events[0].Severity == models.ALARM_STATUS_CLEARED {
			if event.Severity != models.ALARM_STATUS_CLEARED {
				break
			}
			time.Sleep(180 * time.Second)
			count = count + 1
		} else {
			break
		}
	}
	if len(events) == 0 || events[0].Severity == models.ALARM_STATUS_CLEARED {
		return models.ALARM_STATUS_INDETERMINATE, errors.New(fmt.Sprintf("Corresponding alert not available for event: %s",
			event.EventId))
	} else {
		events[0].Acked = true
		events[0].SystemAckedTime = time.Now()
		events[0].AckedByEvent = event.EventId.String()
		events[0].SystemAckComment = fmt.Sprintf("This event is dismissed automatically,"+
			" as we have recieved a corresponding event: %s", event.EventId.String())

		if err := coll.Update(bson.M{"eventid": events[0].EventId},
			bson.M{"$set": events[0]}); err != nil {
			logger.Get().Warning(fmt.Sprintf("%s-Error updating record in DB for event:"+
				" %v. error: %v", ctxt, events[0].EventId.String(), err))
			return models.ALARM_STATUS_INDETERMINATE, errors.New(fmt.Sprintf("%s-Error updating"+
				" record in DB for event: %v. error: %v",
				ctxt, events[0].EventId.String(), err))
		}

	}
	return events[0].Severity, nil
}

func getAlarmCountAndStatus(eventSeverity models.AlarmStatus,
	clearedSeverity models.AlarmStatus,
	alarmCritCount int,
	alarmWarnCount int) (models.AlarmStatus, int, int) {
	if eventSeverity == models.ALARM_STATUS_MAJOR ||
		eventSeverity == models.ALARM_STATUS_CRITICAL {
		alarmCritCount += 1
	} else if eventSeverity == models.ALARM_STATUS_MINOR ||
		eventSeverity == models.ALARM_STATUS_WARNING {
		alarmWarnCount += 1
	}

	if clearedSeverity == models.ALARM_STATUS_MAJOR ||
		clearedSeverity == models.ALARM_STATUS_CRITICAL {
		if alarmCritCount > 0 {
			alarmCritCount -= 1
		}
	} else if clearedSeverity == models.ALARM_STATUS_MINOR ||
		clearedSeverity == models.ALARM_STATUS_WARNING {
		if alarmWarnCount > 0 {
			alarmWarnCount -= 1
		}
	}
	var alarmStatus models.AlarmStatus
	if alarmCritCount > 0 {
		alarmStatus = models.ALARM_STATUS_CRITICAL
	} else if alarmWarnCount > 0 {
		alarmStatus = models.ALARM_STATUS_WARNING
	} else {
		alarmStatus = models.ALARM_STATUS_CLEARED
	}

	return alarmStatus, alarmWarnCount, alarmCritCount
}

func UpdateNodeAlarmCount(event models.AppEvent, clearedSeverity models.AlarmStatus, ctxt string) error {
	var node models.Node
	sessionCopy := db.GetDatastore().Copy()
	defer sessionCopy.Close()
	coll := sessionCopy.DB(conf.SystemConfig.DBConfig.Database).C(models.COLL_NAME_STORAGE_NODES)
	if err := coll.Find(bson.M{
		"nodeid": event.NodeId, "clusterid": event.ClusterId}).One(&node); err != nil {
		logger.Get().Error("%s-Error getting record from DB: %v", ctxt, err)
		return err
	}
	node.AlmStatus, node.AlmWarnCount, node.AlmCritCount = getAlarmCountAndStatus(event.Severity,
		clearedSeverity,
		node.AlmCritCount,
		node.AlmWarnCount)

	if err := coll.Update(bson.M{"nodeid": event.NodeId},
		bson.M{"$set": node}); err != nil {
		logger.Get().Error("%s-Error Updating the Alarm state/count: %v", ctxt, err)
		return err
	}
	return nil
}

func UpdateClusterAlarmCount(event models.AppEvent, clearedSeverity models.AlarmStatus, ctxt string) error {
	var cluster models.Cluster
	sessionCopy := db.GetDatastore().Copy()
	defer sessionCopy.Close()
	coll := sessionCopy.DB(conf.SystemConfig.DBConfig.Database).C(models.COLL_NAME_STORAGE_CLUSTERS)
	if err := coll.Find(bson.M{
		"clusterid": event.ClusterId}).One(&cluster); err != nil {
		logger.Get().Error("%s-Error getting record from DB: %v", ctxt, err)
		return err
	}

	cluster.AlmStatus, cluster.AlmWarnCount, cluster.AlmCritCount = getAlarmCountAndStatus(event.Severity,
		clearedSeverity,
		cluster.AlmCritCount,
		cluster.AlmWarnCount)

	if err := coll.Update(bson.M{"clusterid": event.ClusterId},
		bson.M{"$set": cluster}); err != nil {
		logger.Get().Error("%s-Error Updating the Alarm state/count: %v", ctxt, err)
		return err
	}
	return nil
}

func UpdateSluAlarmCount(event models.AppEvent, clearedSeverity models.AlarmStatus, ctxt string) error {
	var slu models.StorageLogicalUnit
	sessionCopy := db.GetDatastore().Copy()
	defer sessionCopy.Close()
	coll := sessionCopy.DB(conf.SystemConfig.DBConfig.Database).C(models.COLL_NAME_STORAGE_LOGICAL_UNITS)
	if err := coll.Find(bson.M{
		"clusterid": event.ClusterId,
		"sluid":     event.EntityId}).One(&slu); err != nil {
		logger.Get().Error("%s-Error getting record from DB: %v", ctxt, err)
		return err
	}

	slu.AlmStatus, slu.AlmWarnCount, slu.AlmCritCount = getAlarmCountAndStatus(event.Severity,
		clearedSeverity,
		slu.AlmCritCount,
		slu.AlmWarnCount)

	if err := coll.Update(bson.M{"clusterid": event.ClusterId, "sluid": event.EntityId},
		bson.M{"$set": slu}); err != nil {
		logger.Get().Error("%s-Error Updating the Alarm state/count: %v", ctxt, err)
		return err
	}
	return nil
}

func UpdateStorageAlarmCount(event models.AppEvent, clearedSeverity models.AlarmStatus, ctxt string) error {
	var storage models.StorageLogicalUnit
	sessionCopy := db.GetDatastore().Copy()
	defer sessionCopy.Close()
	coll := sessionCopy.DB(conf.SystemConfig.DBConfig.Database).C(models.COLL_NAME_STORAGE)
	if err := coll.Find(bson.M{
		"clusterid": event.ClusterId,
		"storageid": event.EntityId}).One(&storage); err != nil {
		logger.Get().Error("%s-Error getting record from DB: %v", ctxt, err)
		return err
	}

	storage.AlmStatus, storage.AlmWarnCount, storage.AlmCritCount = getAlarmCountAndStatus(event.Severity,
		clearedSeverity,
		storage.AlmCritCount,
		storage.AlmWarnCount)

	if err := coll.Update(bson.M{"clusterid": event.ClusterId, "storageid": event.EntityId},
		bson.M{"$set": storage}); err != nil {
		logger.Get().Error("%s-Error Updating the Alarm state/count: %v", ctxt, err)
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>// gorewind is an event store server written in Python that talks ZeroMQ.
// Copyright (C) 2013  Jens Rantil
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package server

import (
	"testing"
	"testing/quick"
	"bytes"
)

type comparatorTest struct {
	a string
	b string
	expectedResult int
}

func testComparater(t *testing.T, test comparatorTest, c EventStreamComparer) {
	testComparator(t, test, c)
	testSeparator(t, test, c)
	testSuccessor(t, test.a, c)
	testSuccessor(t, test.b, c)
}

func testSeparator(t *testing.T, test comparatorTest, c EventStreamComparer) {
	ba := []byte(test.a)
	bb := []byte(test.b)
	res := c.Separator(ba, bb)
	diff := c.Compare(ba, bb)
	if diff >= 0 && bytes.Compare(res, ba) != 0 {
		t.Errorf("Separator was modified when it shouldn't.")
		t.Errorf("a: %s", test.a)
		t.Errorf("b: %s", test.b)
	}
	if len(res) > len(ba) {
		t.Errorf("a was lengthened, not shortened.")
	}
	if c.Compare(ba, res) > 0 {
		t.Errorf("The Separator was less than 'a'.")
	}
	if c.Compare(bb, res) <= 0 {
		t.Errorf("The Separator was >= 'a'.")
	}
}

func testComparator(t *testing.T, test comparatorTest, comparer EventStreamComparer) {
	res := comparer.Compare([]byte(test.a), []byte(test.b))
	if res != test.expectedResult {
		t.Errorf("a: %s", test.a)
		t.Errorf("b: %s", test.b)
		t.Errorf("Output was %d. Expected: %d", res,
			test.expectedResult)
	}
}

func testSuccessor(t *testing.T, s string, comparer EventStreamComparer) {
	bs := []byte(s)
	shorter := comparer.Successor(bs)
	if len(shorter) > len(bs) {
		t.Errorf("Successor was longer: %s", s)
	}
	if comparer.Compare(shorter, bs) < 0 {
		t.Errorf("Successor was greater the its origin: %s", s)
	}
}

func TestComparator(t *testing.T) {
	comparer := EventStreamComparer{}
	tests := []comparatorTest{
		comparatorTest{"g:a", "g:a", 0},
		comparatorTest{"g:a", "g:b", -1},
		comparatorTest{"g:a", "h:a", -1},
		comparatorTest{"g:a:1", "g:a:1", 0},
		comparatorTest{"g:a:1", "h:a:1", -1},
		comparatorTest{"g:a:1", "h:a:2", -1},
		comparatorTest{"g:a:1", "h:a:11", -1},
		comparatorTest{"g:a:b:1", "g:a:b:11", -1},

		// Test cases where there are multiple colon keys in the
		// middle.
		comparatorTest{"g:a:b", "g:a:b", 0},
		comparatorTest{"g:a:b", "g:b:c", -1},
		comparatorTest{"g:a:b", "h:a:b", -1},
		comparatorTest{"g:a:b:1", "g:a:b:1", 0},
		comparatorTest{"g:a:q:1", "h:a:q:1", -1},
		comparatorTest{"g:a:b:1", "h:a:b:2", -1},
		comparatorTest{"g:a:b:1", "h:a:b:11", -1},
	}
	for _, test := range(tests) {
		testComparator(t, test, comparer)
		if test.expectedResult != 0 {
			// Testing the inverted case
			invertedTest := comparatorTest{
				test.b,
				test.a,
				-test.expectedResult,
			}
			testComparator(t, invertedTest, comparer)
		}
	}
}

type evStoreKeySerTest struct {
	origin string,
	group string,
	key string,
	keyId *big.Int,
}

func TestEventStoreKeySerialization(t *testing.T) {
	f := func(groupId, key []byte, keyId int) bool {
		// GroupId must not contain any :
		bytes.Replace(groupId, ":", "", -1)

		keyIdBytes := []byte(strconv.Itoa(keyId))
		data := bytes.Join([]byte{
			groupId,
			key,
			keyIdBytes,
		}, []byte(":"))

		parsedAndSerialized := newEventStoreKey(data).bytes()
		return bytes.Compare(parsedAndSerialized, bytes) == 0
	}
	if err := quick.Check(f, nil); err != nil {
		t.Error("QuickTest failed:", err)
	}

	tests := []evStoreKeySerTest{
		evStoreKeySerTest{
			"g:a",
			"g",
			"a",
			nil,
		},
		evStoreKeySerTest{
			"g:a:1"
			"g",
			"a",
			big.NewInt(1),
		},
		// Test cases where there are multiple colon keys in the
		// middle.
		evStoreKeySerTest{
			"g:a:b:1"
			"g",
			"a:b",
			big.NewInt(1),
		},
		evStoreKeySerTest{
			"g:a:b"
			"g",
			"a:b",
			nil
		},
		evStoreKeySerTest{
			"h:a:b:2"
			"h",
			"a:b",
			big.NewInt(2),
		},
		evStoreKeySerTest{
			"h:a:b:c1"
			"h",
			"a:b:c1",
			nil
		},
		evStoreKeySerTest{
			"h:a:b:11"
			"h",
			"a:b",
			big.NewInt(11),
		},
	}
	for _, test := range tests {
		key := newEventStoreKey([]byte(test.origin))
		if bytes.Compare(key.groupKey, []byte(test.group) != 0 {
			t.Error("Wrong groupkey for:", test.origin)
			t.Error("Expected:", test.group)
			t.Error("Got:     ", test.groupKey)
		}
		if bytes.Compare(key.key, []byte(test.key) != 0 {
			t.Error("Wrong key for:", test.origin)
			t.Error("Expected:", test.group)
			t.Error("Got:     ", test.groupKey)
		}
		switch {
		case test.keyId != nil && key.keyId == nil:
			t.Error("KeyId was nil, not expected.")
		case test.keyId == nil && key.keyId != nil:
			t.Error("KeyId was not nil, expected it to be.")
		case test.keyId != nil && key.keyId != nil:
			if test.keyId.Cmp(key.keyId) != 0 {
				t.Error("Wrong keyId.")
				t.Error("Expected:", test.keyId.String())
				t.Error("Was:     ", key.keyId.String())
			}
	}

}
<commit_msg>Fixing compile errors for tests<commit_after>// gorewind is an event store server written in Python that talks ZeroMQ.
// Copyright (C) 2013  Jens Rantil
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package server

import (
	"testing"
	"testing/quick"
	"bytes"
)

type comparatorTest struct {
	a string
	b string
	expectedResult int
}

func testComparater(t *testing.T, test comparatorTest, c eventStreamComparer) {
	testComparator(t, test, c)
	testSeparator(t, test, c)
	testSuccessor(t, test.a, c)
	testSuccessor(t, test.b, c)
}

func testSeparator(t *testing.T, test comparatorTest, c eventStreamComparer) {
	ba := []byte(test.a)
	bb := []byte(test.b)
	res := c.Separator(ba, bb)
	diff := c.Compare(ba, bb)
	if diff >= 0 && bytes.Compare(res, ba) != 0 {
		t.Errorf("Separator was modified when it shouldn't.")
		t.Errorf("a: %s", test.a)
		t.Errorf("b: %s", test.b)
	}
	if len(res) > len(ba) {
		t.Errorf("a was lengthened, not shortened.")
	}
	if c.Compare(ba, res) > 0 {
		t.Errorf("The Separator was less than 'a'.")
	}
	if c.Compare(bb, res) <= 0 {
		t.Errorf("The Separator was >= 'a'.")
	}
}

func testComparator(t *testing.T, test comparatorTest, comparer eventStreamComparer) {
	res := comparer.Compare([]byte(test.a), []byte(test.b))
	if res != test.expectedResult {
		t.Errorf("a: %s", test.a)
		t.Errorf("b: %s", test.b)
		t.Errorf("Output was %d. Expected: %d", res,
			test.expectedResult)
	}
}

func testSuccessor(t *testing.T, s string, comparer eventStreamComparer) {
	bs := []byte(s)
	shorter := comparer.Successor(bs)
	if len(shorter) > len(bs) {
		t.Errorf("Successor was longer: %s", s)
	}
	if comparer.Compare(shorter, bs) < 0 {
		t.Errorf("Successor was greater the its origin: %s", s)
	}
}

func TestComparator(t *testing.T) {
	comparer := eventStreamComparer{}
	tests := []comparatorTest{
		comparatorTest{"g:a", "g:a", 0},
		comparatorTest{"g:a", "g:b", -1},
		comparatorTest{"g:a", "h:a", -1},
		comparatorTest{"g:a:1", "g:a:1", 0},
		comparatorTest{"g:a:1", "h:a:1", -1},
		comparatorTest{"g:a:1", "h:a:2", -1},
		comparatorTest{"g:a:1", "h:a:11", -1},
		comparatorTest{"g:a:b:1", "g:a:b:11", -1},

		// Test cases where there are multiple colon keys in the
		// middle.
		comparatorTest{"g:a:b", "g:a:b", 0},
		comparatorTest{"g:a:b", "g:b:c", -1},
		comparatorTest{"g:a:b", "h:a:b", -1},
		comparatorTest{"g:a:b:1", "g:a:b:1", 0},
		comparatorTest{"g:a:q:1", "h:a:q:1", -1},
		comparatorTest{"g:a:b:1", "h:a:b:2", -1},
		comparatorTest{"g:a:b:1", "h:a:b:11", -1},
	}
	for _, test := range(tests) {
		testComparator(t, test, comparer)
		if test.expectedResult != 0 {
			// Testing the inverted case
			invertedTest := comparatorTest{
				test.b,
				test.a,
				-test.expectedResult,
			}
			testComparator(t, invertedTest, comparer)
		}
	}
}

type evStoreKeySerTest struct {
	origin string
	group string
	key string
	keyId byteCounter
}

func TestEventStoreKeySerialization(t *testing.T) {
	f := func(groupId, key, bKeyId []byte) bool {
		// GroupId must not contain any :
		bytes.Replace(groupId, []byte(":"), []byte(""), -1)

		ev := eventStoreKey{
			groupId,
			key,
			loadByteCounter(bKeyId),
		}

		parsed, err := newEventStoreKey(ev.toBytes())
		if err != nil {
			return false
		}
		return parsed.Compare(&ev) == 0
	}
	if err := quick.Check(f, nil); err != nil {
		t.Error("QuickTest failed:", err)
	}
}
<|endoftext|>


<commit_before>package felica

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"unicode/utf8"
	"unsafe"

	"github.com/moznion/go-unicode-east-asian-width"
	"launchpad.net/goyaml"
)

// C言語で使うためにデータにアクセスするポインタを取得する
func DataPtr(data *[]byte) unsafe.Pointer {
	raw := (*reflect.SliceHeader)(unsafe.Pointer(data)).Data

	return unsafe.Pointer(raw)
}

// ファイルを検索ディレクトリから探す
func search_file(fname string, dirs []string) (string, error) {
	for _, dir := range dirs {
		path := filepath.Join(dir, fname)

		if _, err := os.Stat(path); err == nil {
			return path, nil
		}
	}

	return "", errors.New("file not found")
}

// YAML を読込む
func LoadYAML(fname string) (map[interface{}]interface{}, error) {
	cmd, _ := filepath.EvalSymlinks(os.Args[0])
	bindir := filepath.Dir(cmd)
	moddir := filepath.Join(bindir, strings.TrimSuffix(fname, ".yml"))

	dirs := []string{".", bindir, moddir}
	path, err := search_file(fname, dirs)

	if err != nil {
		// ファイルが見つからなかった
		return nil, err
	}

	contents, err := ioutil.ReadFile(path)
	if err != nil {
		// 読込みに失敗
		return nil, err
	}

	m := make(map[interface{}]interface{})
	err = goyaml.Unmarshal(contents, &m)

	return m, err
}

// テーブルを検索して表示用の文字列を返す
func DispName(tables map[interface{}]interface{}, name string, value int, base int, opt_values ...int) interface{} {
	var v interface{}

	t := tables[name]

	if t != nil {
		v = t.(map[interface{}]interface{})[value]
	}

	if v == nil {
		if 0 < len(opt_values) {
			value = opt_values[0]
		}

		if base != 0 {
			f := fmt.Sprintf("%s%dX", "0x%0", base)
			v = fmt.Sprintf(f, value)
		} else {
			v = value
		}
	}

	return v
}

// 指定された表示文字になるように調整する
func DispString(str string, width int) string {
	s, rest := StringTruncate(str, width, "…")

	if 0 < rest {
		s = s + strings.Repeat(" ", rest)
	}

	return s
}

// 文字列の表示幅
func StringWidth(str string) (w int) {
	for _, r := range str {
		if eastasianwidth.IsFullwidth(r) {
			w += 2
		} else {
			w++
		}
	}

	return
}

// 文字列の表示幅で切り詰める
func StringTruncate(str string, width int, tail string) (s string, rest int) {
	tailLen := StringWidth(tail)
	tailRest := width
	tp := 0
	i := 0

	rest = width
	s = str

	for _, r := range str {
		n := 1
		if eastasianwidth.IsFullwidth(r) {
			n = 2
		}

		if tp == 0 && rest < n+tailLen {
			// tail を追加できる位置を覚えておく
			tp = i
			tailRest = rest
		}

		if rest < n {
			s = str[0:tp] + tail
			rest = tailRest - tailLen
			break
		}

		rest -= n
		i += utf8.RuneLen(r)
	}

	return
}
<commit_msg>外部パッケージ disp_width に置き換え<commit_after>package felica

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"unsafe"

	"github.com/gnue/go-disp_width"
	"launchpad.net/goyaml"
)

// C言語で使うためにデータにアクセスするポインタを取得する
func DataPtr(data *[]byte) unsafe.Pointer {
	raw := (*reflect.SliceHeader)(unsafe.Pointer(data)).Data

	return unsafe.Pointer(raw)
}

// ファイルを検索ディレクトリから探す
func search_file(fname string, dirs []string) (string, error) {
	for _, dir := range dirs {
		path := filepath.Join(dir, fname)

		if _, err := os.Stat(path); err == nil {
			return path, nil
		}
	}

	return "", errors.New("file not found")
}

// YAML を読込む
func LoadYAML(fname string) (map[interface{}]interface{}, error) {
	cmd, _ := filepath.EvalSymlinks(os.Args[0])
	bindir := filepath.Dir(cmd)
	moddir := filepath.Join(bindir, strings.TrimSuffix(fname, ".yml"))

	dirs := []string{".", bindir, moddir}
	path, err := search_file(fname, dirs)

	if err != nil {
		// ファイルが見つからなかった
		return nil, err
	}

	contents, err := ioutil.ReadFile(path)
	if err != nil {
		// 読込みに失敗
		return nil, err
	}

	m := make(map[interface{}]interface{})
	err = goyaml.Unmarshal(contents, &m)

	return m, err
}

// テーブルを検索して表示用の文字列を返す
func DispName(tables map[interface{}]interface{}, name string, value int, base int, opt_values ...int) interface{} {
	var v interface{}

	t := tables[name]

	if t != nil {
		v = t.(map[interface{}]interface{})[value]
	}

	if v == nil {
		if 0 < len(opt_values) {
			value = opt_values[0]
		}

		if base != 0 {
			f := fmt.Sprintf("%s%dX", "0x%0", base)
			v = fmt.Sprintf(f, value)
		} else {
			v = value
		}
	}

	return v
}

// 指定された表示文字になるように調整する
func DispString(str string, width int) string {
	s, rest := disp_width.Truncate(str, width, "…")

	if 0 < rest {
		s = s + strings.Repeat(" ", rest)
	}

	return s
}
<|endoftext|>