<commit_before>package revgrep

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
)

// Checker provides APIs to filter static analysis tools to specific commits,
// such as showing only issues since last commit.
type Checker struct {
	// Patch file (unified) to read to detect lines being changed, if nil revgrep
	// will attempt to detect the VCS and generate an appropriate patch. Auto
	// detection will search for uncommitted changes first, if none found, will
	// generate a patch from last committed change. File paths within patches
	// must be relative to current working directory.
	Patch io.Reader
	// NewFiles is a list of file names (with absolute paths) where the entire
	// contents of the file is new.
	NewFiles []string
	// Debug sets the debug writer for additional output.
	Debug io.Writer
	// RevisionFrom check revision starting at, leave blank for auto detection
	// ignored if patch is set.
	RevisionFrom string
	// RevisionTo checks revision finishing at, leave blank for auto detection
	// ignored if patch is set.
	RevisionTo string
	// Regexp to match path, line number, optional column number, and message.
	Regexp string
	// AbsPath is used to make an absolute path of an issue's filename to be
	// relative in order to match patch file. If not set, current working
	// directory is used.
	AbsPath string
}

// Issue contains metadata about an issue found.
type Issue struct {
	// File is the name of the file as it appeared from the patch.
	File string
	// LineNo is the line number of the file.
	LineNo int
	// ColNo is the column number or 0 if none could be parsed.
	ColNo int
	// HunkPos is position from file's first @@, for new files this will be the
	// line number.
	//
	// See also: https://developer.github.com/v3/pulls/comments/#create-a-comment
	HunkPos int
	// Issue text as it appeared from the tool.
	Issue string
	// Message is the issue without file name, line number and column number.
	Message string
}

// Check scans reader and writes any lines to writer that have been added in
// Checker.Patch.
//
// Returns issues written to writer when no error occurs.
//
// If no VCS could be found or other VCS errors occur, all issues are written
// to writer and an error is returned.
//
// File paths in reader must be relative to current working directory or
// absolute.
func (c Checker) Check(reader io.Reader, writer io.Writer) (issues []Issue, err error) {
	// Check if patch is supplied, if not, retrieve from VCS
	var (
		writeAll  bool
		returnErr error
	)
	if c.Patch == nil {
		c.Patch, c.NewFiles, err = GitPatch(c.RevisionFrom, c.RevisionTo)
		if err != nil {
			writeAll = true
			returnErr = fmt.Errorf("could not read git repo: %s", err)
		}
		if c.Patch == nil {
			writeAll = true
			returnErr = errors.New("no version control repository found")
		}
	}

	// file.go:lineNo:colNo:message
	// colNo is optional, strip spaces before message
	lineRE := regexp.MustCompile(`(.*?\.go):([0-9]+):([0-9]+)?:?\s*(.*)`)
	if c.Regexp != "" {
		lineRE, err = regexp.Compile(c.Regexp)
		if err != nil {
			return nil, fmt.Errorf("could not parse regexp: %v", err)
		}
	}

	// TODO consider lazy loading this, if there's nothing in stdin, no point
	// checking for recent changes
	linesChanged := c.linesChanged()
	c.debug(fmt.Sprintf("lines changed: %+v", linesChanged))

	absPath := c.AbsPath
	if absPath == "" {
		absPath, err = os.Getwd()
		if err != nil {
			returnErr = fmt.Errorf("could not get current working directory: %s", err)
		}
	}

	// Scan each line in reader and only write those lines if lines changed
	scanner := bufio.NewScanner(reader)
	for scanner.Scan() {
		line := lineRE.FindSubmatch(scanner.Bytes())
		if line == nil {
			c.debug("cannot parse file+line number:", scanner.Text())
			continue
		}

		if writeAll {
			fmt.Fprintln(writer, scanner.Text())
			continue
		}

		// Make absolute path names relative
		path := string(line[1])
		if rel, err := filepath.Rel(absPath, path); err == nil {
			c.debug("rewrote path from %q to %q", path, rel)
			path = rel
		}

		// Parse line number
		lno, err := strconv.ParseUint(string(line[2]), 10, 64)
		if err != nil {
			c.debug("cannot parse line number:", scanner.Text())
			continue
		}

		// Parse optional column number
		var cno uint64
		if len(line[3]) > 0 {
			cno, err = strconv.ParseUint(string(line[3]), 10, 64)
			if err != nil {
				c.debug("cannot parse column number:", scanner.Text())
				// Ignore this error and continue
			}
		}

		// Extract message
		msg := string(line[4])

		var (
			fpos    pos
			changed bool
		)
		if fchanges, ok := linesChanged[path]; ok {
			// found file, see if lines matched
			for _, pos := range fchanges {
				if pos.lineNo == int(lno) {
					fpos = pos
					changed = true
				}
			}
			if changed || fchanges == nil {
				// either file changed or it's a new file
				issue := Issue{
					File:    path,
					LineNo:  fpos.lineNo,
					ColNo:   int(cno),
					HunkPos: fpos.lineNo,
					Issue:   scanner.Text(),
					Message: msg,
				}
				if changed {
					// existing file changed
					issue.HunkPos = fpos.hunkPos
				}
				issues = append(issues, issue)
				fmt.Fprintln(writer, scanner.Text())
			}
		}
		if !changed {
			c.debug("unchanged:", scanner.Text())
		}
	}
	if err := scanner.Err(); err != nil {
		returnErr = fmt.Errorf("error reading standard input: %s", err)
	}
	return issues, returnErr
}

func (c Checker) debug(s ...interface{}) {
	if c.Debug != nil {
		fmt.Fprint(c.Debug, "DEBUG: ")
		fmt.Fprintln(c.Debug, s...)
	}
}

type pos struct {
	lineNo  int // line number
	hunkPos int // position relative to first @@ in file
}

// linesChanges returns a map of file names to line numbers being changed
func (c Checker) linesChanged() map[string][]pos {
	type state struct {
		file    string
		lineNo  int   // current line number within chunk
		hunkPos int   // current line count since first @@ in file
		changes []pos // position of changes
	}

	var (
		s       state
		changes = make(map[string][]pos)
	)

	for _, file := range c.NewFiles {
		changes[file] = nil
	}

	if c.Patch == nil {
		return changes
	}

	scanner := bufio.NewScanner(c.Patch)
	for scanner.Scan() {
		line := scanner.Text() // TODO scanner.Bytes()
		c.debug(line)
		s.lineNo++
		switch {
		case strings.HasPrefix(line, "+++ ") && len(line) > 4:
			if s.changes != nil {
				// record the last state
				changes[s.file] = s.changes
			}
			// 6 removes "+++ b/"
			s = state{file: line[6:]}
		case strings.HasPrefix(line, "@@ "):
			//      @@ -1 +2,4 @@
			// chdr ^^^^^^^^^^^^^
			// ahdr       ^^^^
			// cstart      ^
			chdr := strings.Split(line, " ")
			ahdr := strings.Split(chdr[2], ",")
			// [1:] to remove leading plus
			cstart, err := strconv.ParseUint(ahdr[0][1:], 10, 64)
			if err != nil {
				panic(err)
			}
			s.lineNo = int(cstart) - 1 // -1 as cstart is the next line number
		case strings.HasPrefix(line, " "):
			s.hunkPos++
		case strings.HasPrefix(line, "-"):
			s.hunkPos++
			s.lineNo--
		case strings.HasPrefix(line, "+"):
			s.hunkPos++
			s.changes = append(s.changes, pos{lineNo: s.lineNo, hunkPos: s.hunkPos})
		}

	}
	if err := scanner.Err(); err != nil {
		fmt.Fprintln(os.Stderr, "reading standard input:", err)
	}
	// record the last state
	changes[s.file] = s.changes

	return changes
}

// GitPatch returns a patch from a git repository, if no git repository was
// was found and no errors occurred, nil is returned, else an error is returned
// revisionFrom and revisionTo defines the git diff parameters, if left blank
// and there are unstaged changes or untracked files, only those will be returned
// else only check changes since HEAD~. If revisionFrom is set but revisionTo
// is not, untracked files will be included, to exclude untracked files set
// revisionTo to HEAD~. It's incorrect to specify revisionTo without a
// revisionFrom.
func GitPatch(revisionFrom, revisionTo string) (io.Reader, []string, error) {
	var patch bytes.Buffer

	// check if git repo exists
	if err := exec.Command("git", "status").Run(); err != nil {
		// don't return an error, we assume the error is not repo exists
		return nil, nil, nil
	}

	// make a patch for untracked files
	var newFiles []string
	ls, err := exec.Command("git", "ls-files", "-o").CombinedOutput()
	if err != nil {
		return nil, nil, fmt.Errorf("error executing git ls-files: %s", err)
	}
	for _, file := range bytes.Split(ls, []byte{'\n'}) {
		if len(file) == 0 || bytes.HasSuffix(file, []byte{'/'}) {
			// ls-files was sometimes showing directories when they were ignored
			// I couldn't create a test case for this as I couldn't reproduce correctly
			// for the moment, just exclude files with trailing /
			continue
		}
		newFiles = append(newFiles, string(file))
	}

	if revisionFrom != "" {
		cmd := exec.Command("git", "diff", revisionFrom)
		if revisionTo != "" {
			cmd.Args = append(cmd.Args, revisionTo)
		}
		cmd.Stdout = &patch
		if err := cmd.Run(); err != nil {
			return nil, nil, fmt.Errorf("error executing git diff %q %q: %s", revisionFrom, revisionTo, err)
		}

		if revisionTo == "" {
			return &patch, newFiles, nil
		}
		return &patch, nil, nil
	}

	// make a patch for unstaged changes
	// use --no-prefix to remove b/ given: +++ b/main.go
	cmd := exec.Command("git", "diff")
	cmd.Stdout = &patch
	if err := cmd.Run(); err != nil {
		return nil, nil, fmt.Errorf("error executing git diff: %s", err)
	}
	unstaged := patch.Len() > 0

	// If there's unstaged changes OR untracked changes (or both), then this is
	// a suitable patch
	if unstaged || newFiles != nil {
		return &patch, newFiles, nil
	}

	// check for changes in recent commit

	cmd = exec.Command("git", "diff", "HEAD~")
	cmd.Stdout = &patch
	if err := cmd.Run(); err != nil {
		return nil, nil, fmt.Errorf("error executing git diff HEAD~: %s", err)
	}

	return &patch, nil, nil
}
<commit_msg>Import debug<commit_after>package revgrep

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
)

// Checker provides APIs to filter static analysis tools to specific commits,
// such as showing only issues since last commit.
type Checker struct {
	// Patch file (unified) to read to detect lines being changed, if nil revgrep
	// will attempt to detect the VCS and generate an appropriate patch. Auto
	// detection will search for uncommitted changes first, if none found, will
	// generate a patch from last committed change. File paths within patches
	// must be relative to current working directory.
	Patch io.Reader
	// NewFiles is a list of file names (with absolute paths) where the entire
	// contents of the file is new.
	NewFiles []string
	// Debug sets the debug writer for additional output.
	Debug io.Writer
	// RevisionFrom check revision starting at, leave blank for auto detection
	// ignored if patch is set.
	RevisionFrom string
	// RevisionTo checks revision finishing at, leave blank for auto detection
	// ignored if patch is set.
	RevisionTo string
	// Regexp to match path, line number, optional column number, and message.
	Regexp string
	// AbsPath is used to make an absolute path of an issue's filename to be
	// relative in order to match patch file. If not set, current working
	// directory is used.
	AbsPath string
}

// Issue contains metadata about an issue found.
type Issue struct {
	// File is the name of the file as it appeared from the patch.
	File string
	// LineNo is the line number of the file.
	LineNo int
	// ColNo is the column number or 0 if none could be parsed.
	ColNo int
	// HunkPos is position from file's first @@, for new files this will be the
	// line number.
	//
	// See also: https://developer.github.com/v3/pulls/comments/#create-a-comment
	HunkPos int
	// Issue text as it appeared from the tool.
	Issue string
	// Message is the issue without file name, line number and column number.
	Message string
}

// Check scans reader and writes any lines to writer that have been added in
// Checker.Patch.
//
// Returns issues written to writer when no error occurs.
//
// If no VCS could be found or other VCS errors occur, all issues are written
// to writer and an error is returned.
//
// File paths in reader must be relative to current working directory or
// absolute.
func (c Checker) Check(reader io.Reader, writer io.Writer) (issues []Issue, err error) {
	// Check if patch is supplied, if not, retrieve from VCS
	var (
		writeAll  bool
		returnErr error
	)
	if c.Patch == nil {
		c.Patch, c.NewFiles, err = GitPatch(c.RevisionFrom, c.RevisionTo)
		if err != nil {
			writeAll = true
			returnErr = fmt.Errorf("could not read git repo: %s", err)
		}
		if c.Patch == nil {
			writeAll = true
			returnErr = errors.New("no version control repository found")
		}
	}

	// file.go:lineNo:colNo:message
	// colNo is optional, strip spaces before message
	lineRE := regexp.MustCompile(`(.*?\.go):([0-9]+):([0-9]+)?:?\s*(.*)`)
	if c.Regexp != "" {
		lineRE, err = regexp.Compile(c.Regexp)
		if err != nil {
			return nil, fmt.Errorf("could not parse regexp: %v", err)
		}
	}

	// TODO consider lazy loading this, if there's nothing in stdin, no point
	// checking for recent changes
	linesChanged := c.linesChanged()
	c.debugf("lines changed: %+v", linesChanged)

	absPath := c.AbsPath
	if absPath == "" {
		absPath, err = os.Getwd()
		if err != nil {
			returnErr = fmt.Errorf("could not get current working directory: %s", err)
		}
	}

	// Scan each line in reader and only write those lines if lines changed
	scanner := bufio.NewScanner(reader)
	for scanner.Scan() {
		line := lineRE.FindSubmatch(scanner.Bytes())
		if line == nil {
			c.debugf("cannot parse file+line number:", scanner.Text())
			continue
		}

		if writeAll {
			fmt.Fprintln(writer, scanner.Text())
			continue
		}

		// Make absolute path names relative
		path := string(line[1])
		if rel, err := filepath.Rel(absPath, path); err == nil {
			c.debugf("rewrote path from %q to %q (absPath: %q)", path, rel, absPath)
			path = rel
		}

		// Parse line number
		lno, err := strconv.ParseUint(string(line[2]), 10, 64)
		if err != nil {
			c.debugf("cannot parse line number: %q", scanner.Text())
			continue
		}

		// Parse optional column number
		var cno uint64
		if len(line[3]) > 0 {
			cno, err = strconv.ParseUint(string(line[3]), 10, 64)
			if err != nil {
				c.debugf("cannot parse column number: %q", scanner.Text())
				// Ignore this error and continue
			}
		}

		// Extract message
		msg := string(line[4])

		c.debugf("path: %q, lineNo: %v, colNo: %v, msg: %q", path, lno, cno, msg)

		var (
			fpos    pos
			changed bool
		)
		if fchanges, ok := linesChanged[path]; ok {
			// found file, see if lines matched
			for _, pos := range fchanges {
				if pos.lineNo == int(lno) {
					fpos = pos
					changed = true
				}
			}
			if changed || fchanges == nil {
				// either file changed or it's a new file
				issue := Issue{
					File:    path,
					LineNo:  fpos.lineNo,
					ColNo:   int(cno),
					HunkPos: fpos.lineNo,
					Issue:   scanner.Text(),
					Message: msg,
				}
				if changed {
					// existing file changed
					issue.HunkPos = fpos.hunkPos
				}
				issues = append(issues, issue)
				fmt.Fprintln(writer, scanner.Text())
			}
		}
		if !changed {
			c.debugf("unchanged:", scanner.Text())
		}
	}
	if err := scanner.Err(); err != nil {
		returnErr = fmt.Errorf("error reading standard input: %s", err)
	}
	return issues, returnErr
}

func (c Checker) debugf(format string, s ...interface{}) {
	if c.Debug != nil {
		fmt.Fprint(c.Debug, "DEBUG: ")
		fmt.Fprintf(c.Debug, format+"\n", s...)
	}
}

type pos struct {
	lineNo  int // line number
	hunkPos int // position relative to first @@ in file
}

// linesChanges returns a map of file names to line numbers being changed
func (c Checker) linesChanged() map[string][]pos {
	type state struct {
		file    string
		lineNo  int   // current line number within chunk
		hunkPos int   // current line count since first @@ in file
		changes []pos // position of changes
	}

	var (
		s       state
		changes = make(map[string][]pos)
	)

	for _, file := range c.NewFiles {
		changes[file] = nil
	}

	if c.Patch == nil {
		return changes
	}

	scanner := bufio.NewScanner(c.Patch)
	for scanner.Scan() {
		line := scanner.Text() // TODO scanner.Bytes()
		c.debugf(line)
		s.lineNo++
		switch {
		case strings.HasPrefix(line, "+++ ") && len(line) > 4:
			if s.changes != nil {
				// record the last state
				changes[s.file] = s.changes
			}
			// 6 removes "+++ b/"
			s = state{file: line[6:]}
		case strings.HasPrefix(line, "@@ "):
			//      @@ -1 +2,4 @@
			// chdr ^^^^^^^^^^^^^
			// ahdr       ^^^^
			// cstart      ^
			chdr := strings.Split(line, " ")
			ahdr := strings.Split(chdr[2], ",")
			// [1:] to remove leading plus
			cstart, err := strconv.ParseUint(ahdr[0][1:], 10, 64)
			if err != nil {
				panic(err)
			}
			s.lineNo = int(cstart) - 1 // -1 as cstart is the next line number
		case strings.HasPrefix(line, " "):
			s.hunkPos++
		case strings.HasPrefix(line, "-"):
			s.hunkPos++
			s.lineNo--
		case strings.HasPrefix(line, "+"):
			s.hunkPos++
			s.changes = append(s.changes, pos{lineNo: s.lineNo, hunkPos: s.hunkPos})
		}

	}
	if err := scanner.Err(); err != nil {
		fmt.Fprintln(os.Stderr, "reading standard input:", err)
	}
	// record the last state
	changes[s.file] = s.changes

	return changes
}

// GitPatch returns a patch from a git repository, if no git repository was
// was found and no errors occurred, nil is returned, else an error is returned
// revisionFrom and revisionTo defines the git diff parameters, if left blank
// and there are unstaged changes or untracked files, only those will be returned
// else only check changes since HEAD~. If revisionFrom is set but revisionTo
// is not, untracked files will be included, to exclude untracked files set
// revisionTo to HEAD~. It's incorrect to specify revisionTo without a
// revisionFrom.
func GitPatch(revisionFrom, revisionTo string) (io.Reader, []string, error) {
	var patch bytes.Buffer

	// check if git repo exists
	if err := exec.Command("git", "status").Run(); err != nil {
		// don't return an error, we assume the error is not repo exists
		return nil, nil, nil
	}

	// make a patch for untracked files
	var newFiles []string
	ls, err := exec.Command("git", "ls-files", "-o").CombinedOutput()
	if err != nil {
		return nil, nil, fmt.Errorf("error executing git ls-files: %s", err)
	}
	for _, file := range bytes.Split(ls, []byte{'\n'}) {
		if len(file) == 0 || bytes.HasSuffix(file, []byte{'/'}) {
			// ls-files was sometimes showing directories when they were ignored
			// I couldn't create a test case for this as I couldn't reproduce correctly
			// for the moment, just exclude files with trailing /
			continue
		}
		newFiles = append(newFiles, string(file))
	}

	if revisionFrom != "" {
		cmd := exec.Command("git", "diff", revisionFrom)
		if revisionTo != "" {
			cmd.Args = append(cmd.Args, revisionTo)
		}
		cmd.Stdout = &patch
		if err := cmd.Run(); err != nil {
			return nil, nil, fmt.Errorf("error executing git diff %q %q: %s", revisionFrom, revisionTo, err)
		}

		if revisionTo == "" {
			return &patch, newFiles, nil
		}
		return &patch, nil, nil
	}

	// make a patch for unstaged changes
	// use --no-prefix to remove b/ given: +++ b/main.go
	cmd := exec.Command("git", "diff")
	cmd.Stdout = &patch
	if err := cmd.Run(); err != nil {
		return nil, nil, fmt.Errorf("error executing git diff: %s", err)
	}
	unstaged := patch.Len() > 0

	// If there's unstaged changes OR untracked changes (or both), then this is
	// a suitable patch
	if unstaged || newFiles != nil {
		return &patch, newFiles, nil
	}

	// check for changes in recent commit

	cmd = exec.Command("git", "diff", "HEAD~")
	cmd.Stdout = &patch
	if err := cmd.Run(); err != nil {
		return nil, nil, fmt.Errorf("error executing git diff HEAD~: %s", err)
	}

	return &patch, nil, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2014 Google Inc. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// copied from k8s: plugin/pkg/scheduler/factory/factory.go
package scheduler

import "sync"
import "time"
import log "github.com/golang/glog"

type clock interface {
	Now() time.Time
}

type realClock struct{}

func (realClock) Now() time.Time {
	return time.Now()
}

type backoffEntry struct {
	backoff    time.Duration
	lastUpdate time.Time
}

type podBackoff struct {
	perPodBackoff map[string]*backoffEntry
	lock          sync.Mutex
	clock         clock
}

func (p *podBackoff) getEntry(podID string) *backoffEntry {
	p.lock.Lock()
	defer p.lock.Unlock()
	entry, ok := p.perPodBackoff[podID]
	if !ok {
		entry = &backoffEntry{backoff: 1 * time.Second}
		p.perPodBackoff[podID] = entry
	}
	entry.lastUpdate = p.clock.Now()
	return entry
}

func (p *podBackoff) getBackoff(podID string) time.Duration {
	entry := p.getEntry(podID)
	duration := entry.backoff
	entry.backoff *= 2
	if entry.backoff > 60*time.Second {
		entry.backoff = 60 * time.Second
	}
	log.V(3).Infof("Backing off %s for pod %s", duration.String(), podID)
	return duration
}

func (p *podBackoff) wait(podID string) {
	time.Sleep(p.getBackoff(podID))
}

func (p *podBackoff) gc() {
	p.lock.Lock()
	defer p.lock.Unlock()
	now := p.clock.Now()
	for podID, entry := range p.perPodBackoff {
		if now.Sub(entry.lastUpdate) > 60*time.Second {
			delete(p.perPodBackoff, podID)
		}
	}
}
<commit_msg>added TODO re: alt backoff strategies<commit_after>/*
Copyright 2014 Google Inc. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// copied from k8s: plugin/pkg/scheduler/factory/factory.go
package scheduler

import "sync"
import "time"
import log "github.com/golang/glog"

type clock interface {
	Now() time.Time
}

type realClock struct{}

func (realClock) Now() time.Time {
	return time.Now()
}

type backoffEntry struct {
	backoff    time.Duration
	lastUpdate time.Time
}

type podBackoff struct {
	perPodBackoff map[string]*backoffEntry
	lock          sync.Mutex
	clock         clock
}

func (p *podBackoff) getEntry(podID string) *backoffEntry {
	p.lock.Lock()
	defer p.lock.Unlock()
	entry, ok := p.perPodBackoff[podID]
	if !ok {
		entry = &backoffEntry{backoff: 1 * time.Second}
		p.perPodBackoff[podID] = entry
	}
	entry.lastUpdate = p.clock.Now()
	return entry
}

// TODO(jdef): allow for pluggable backoff sequences, strategies here. For
// example we may want to use a fibonacci backoff sequence for certain cluster
// sizes. Or, perhaps we'll allow per-pod backoff factors at some point (ala
// marathon).
func (p *podBackoff) getBackoff(podID string) time.Duration {
	entry := p.getEntry(podID)
	duration := entry.backoff
	entry.backoff *= 2
	if entry.backoff > 60*time.Second {
		entry.backoff = 60 * time.Second
	}
	log.V(3).Infof("Backing off %s for pod %s", duration.String(), podID)
	return duration
}

func (p *podBackoff) wait(podID string) {
	time.Sleep(p.getBackoff(podID))
}

func (p *podBackoff) gc() {
	p.lock.Lock()
	defer p.lock.Unlock()
	now := p.clock.Now()
	for podID, entry := range p.perPodBackoff {
		if now.Sub(entry.lastUpdate) > 60*time.Second {
			delete(p.perPodBackoff, podID)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"

	"github.com/bitfinexcom/bitfinex-api-go/v2"
	"github.com/bitfinexcom/bitfinex-api-go/v2/rest"
	"github.com/davecgh/go-spew/spew"
)

// Set BFX_API_KEY and BFX_API_SECRET:
//
// export BFX_API_KEY=<your-api-key>
// export BFX_API_SECRET=<your-api-secret>
//
// you can obtain it from https://www.bitfinex.com/api
//
// Below you can see different variations of using Order Multi ops

func main() {
	key := os.Getenv("BFX_API_KEY")
	secret := os.Getenv("BFX_API_SECRET")

	c := rest.
		NewClient().
		Credentials(key, secret)

	cancelOrdersMultiOp(c)
	cancelOrderMultiOp(c)
	orderMultiOp(c)
}

func cancelOrdersMultiOp(c *rest.Client) {
	resp, err := c.Orders.CancelOrdersMultiOp(rest.OrderIDs{1189452506, 1189452507})
	if err != nil {
		log.Fatalf("CancelOrdersMultiOp error: %s", err)
	}

	spew.Dump(resp)
}

func cancelOrderMultiOp(c *rest.Client) {
	resp, err := c.Orders.CancelOrderMultiOp(1189502586)
	if err != nil {
		log.Fatalf("CancelOrderMultiOp error: %s", err)
	}

	spew.Dump(resp)
}

func orderMultiOp(c *rest.Client) {
	pld := rest.OrderMultiArgs{
		Ops: [][]interface{}{
			{
				"on",
				bitfinex.OrderNewRequest{
					CID:    987,
					GID:    876,
					Type:   "EXCHANGE LIMIT",
					Symbol: "tBTCUSD",
					Price:  13,
					Amount: 0.001,
				},
			},
			{
				"oc",
				map[string]int{"id": 1189502430},
			},
			{
				"oc_multi",
				map[string][]int{"id": rest.OrderIDs{1189502431, 1189502432}},
			},
			{
				"ou",
				bitfinex.OrderUpdateRequest{
					ID:     1189502433,
					Price:  15,
					Amount: 0.002,
				},
			},
		},
	}

	resp, err := c.Orders.OrderMultiOp(pld)
	if err != nil {
		log.Fatalf("OrderMultiOp error: %s", err)
	}

	spew.Dump(resp)
}
<commit_msg>OrderNewMultiOp usage example<commit_after>package main

import (
	"log"
	"os"

	"github.com/bitfinexcom/bitfinex-api-go/v2"
	"github.com/bitfinexcom/bitfinex-api-go/v2/rest"
	"github.com/davecgh/go-spew/spew"
)

// Set BFX_API_KEY and BFX_API_SECRET:
//
// export BFX_API_KEY=<your-api-key>
// export BFX_API_SECRET=<your-api-secret>
//
// you can obtain it from https://www.bitfinex.com/api
//
// Below you can see different variations of using Order Multi ops

func main() {
	key := os.Getenv("BFX_API_KEY")
	secret := os.Getenv("BFX_API_SECRET")

	c := rest.
		NewClient().
		Credentials(key, secret)

	cancelOrdersMultiOp(c)
	cancelOrderMultiOp(c)
	orderNewMultiOp(c)
	orderMultiOp(c)
}

func cancelOrdersMultiOp(c *rest.Client) {
	resp, err := c.Orders.CancelOrdersMultiOp(rest.OrderIDs{1189452506, 1189452507})
	if err != nil {
		log.Fatalf("CancelOrdersMultiOp error: %s", err)
	}

	spew.Dump(resp)
}

func cancelOrderMultiOp(c *rest.Client) {
	resp, err := c.Orders.CancelOrderMultiOp(1189502586)
	if err != nil {
		log.Fatalf("CancelOrderMultiOp error: %s", err)
	}

	spew.Dump(resp)
}

func orderNewMultiOp(c *rest.Client) {
	o := bitfinex.OrderNewRequest{
		CID:    119,
		GID:    118,
		Type:   "EXCHANGE LIMIT",
		Symbol: "tBTCUSD",
		Price:  12,
		Amount: 0.002,
	}

	resp, err := c.Orders.OrderNewMultiOp(o)
	if err != nil {
		log.Fatalf("OrderNewMultiOp error: %s", err)
	}

	spew.Dump(resp)
}

func orderMultiOp(c *rest.Client) {
	pld := rest.OrderOps{
		{
			"on",
			bitfinex.OrderNewRequest{
				CID:    987,
				GID:    876,
				Type:   "EXCHANGE LIMIT",
				Symbol: "tBTCUSD",
				Price:  13,
				Amount: 0.001,
			},
		},
		{
			"oc",
			map[string]int{"id": 1189502587},
		},
		{
			"oc_multi",
			map[string][]int{"id": rest.OrderIDs{1189502588, 1189503341}},
		},
		{
			"ou",
			bitfinex.OrderUpdateRequest{
				ID:     1189503342,
				Price:  15,
				Amount: 0.002,
			},
		},
	}

	resp, err := c.Orders.OrderMultiOp(pld)
	if err != nil {
		log.Fatalf("OrderMultiOp error: %s", err)
	}

	spew.Dump(resp)
}
<|endoftext|>


<commit_before>package vpp

import (
	"github.com/ligato/cn-infra/core"
	"github.com/ligato/cn-infra/datasync"
	"github.com/ligato/cn-infra/datasync/resync"
	"github.com/ligato/cn-infra/flavors/etcdkafka"
	"github.com/ligato/cn-infra/flavors/local"
	"github.com/ligato/cn-infra/flavors/redis"
	"github.com/ligato/cn-infra/flavors/rpc"
	"github.com/ligato/vpp-agent/plugins/defaultplugins"
	"github.com/ligato/vpp-agent/plugins/govppmux"
	"github.com/ligato/vpp-agent/plugins/linuxplugin"
)

// Flavor glues together multiple plugins to translate ETCD configuration into VPP.
type Flavor struct {
	*local.FlavorLocal
	*rpc.FlavorRPC
	*etcdkafka.FlavorEtcdKafka
	*redis.FlavorRedis
	GoVPP     govppmux.GOVPPPlugin
	Linux     linuxplugin.Plugin
	VPP       defaultplugins.Plugin

	injected bool
}

// Inject sets object references
func (f *Flavor) Inject() error {
	if f.injected {
		return nil
	}
	f.injected = true

	if f.FlavorLocal == nil {
		f.FlavorLocal = &rpc.FlavorLocal{}
	}
	f.FlavorLocal.Inject()

	if f.FlavorEtcdKafka == nil {
		f.FlavorEtcdKafka = &etcdkafka.FlavorEtcdKafka{FlavorLocal: f.FlavorLocal}
	}
	f.FlavorEtcdKafka.Inject()

	if f.FlavorRPC == nil {
		f.FlavorRPC = &rpc.FlavorRPC{FlavorLocal: f.FlavorLocal}
	}
	f.FlavorRPC.Inject()

	if f.FlavorRedis == nil {
		f.FlavorRedis = &redis.FlavorRedis{FlavorLocal: f.FlavorLocal}
	}
	f.FlavorRedis.Inject()

	f.GoVPP.Deps.PluginInfraDeps = *f.FlavorEtcdKafka.FlavorLocal.InfraDeps("govpp")
	f.VPP.Deps.PluginInfraDeps = *f.FlavorEtcdKafka.FlavorLocal.InfraDeps("default-plugins")
	f.VPP.Deps.Publish = &f.FlavorEtcdKafka.ETCDDataSync
	f.VPP.Deps.PublishStatistics = datasync.CompositeKVProtoWriter{
		Adapters: []datasync.KeyProtoValWriter{&f.FlavorEtcdKafka.ETCDDataSync, &f.FlavorRedis.RedisDataSync},
	}
	f.VPP.Deps.Watch = &f.FlavorEtcdKafka.ETCDDataSync
	f.VPP.Deps.Kafka = &f.FlavorEtcdKafka.Kafka
	f.VPP.Deps.GoVppmux = &f.GoVPP
	f.VPP.Deps.Linux = &f.Linux
	f.VPP.Linux.Deps.Watcher = &f.FlavorEtcdKafka.ETCDDataSync

	return nil
}

// Plugins combines Generic Plugins and Standard VPP Plugins + (their ETCD Connector/Adapter with RESYNC)
func (f *Flavor) Plugins() []*core.NamedPlugin {
	f.Inject()
	return core.ListPluginsInFlavor(f)
}
<commit_msg>Cleanup FlavorLocal in vpp flavor<commit_after>package vpp

import (
	"github.com/ligato/cn-infra/core"
	"github.com/ligato/cn-infra/datasync"
	"github.com/ligato/cn-infra/datasync/resync"
	"github.com/ligato/cn-infra/flavors/etcdkafka"
	"github.com/ligato/cn-infra/flavors/local"
	"github.com/ligato/cn-infra/flavors/redis"
	"github.com/ligato/cn-infra/flavors/rpc"
	"github.com/ligato/vpp-agent/plugins/defaultplugins"
	"github.com/ligato/vpp-agent/plugins/govppmux"
	"github.com/ligato/vpp-agent/plugins/linuxplugin"
)

// Flavor glues together multiple plugins to translate ETCD configuration into VPP.
type Flavor struct {
	*local.FlavorLocal
	*rpc.FlavorRPC
	*etcdkafka.FlavorEtcdKafka
	*redis.FlavorRedis
	GoVPP     govppmux.GOVPPPlugin
	Linux     linuxplugin.Plugin
	VPP       defaultplugins.Plugin

	injected bool
}

// Inject sets object references
func (f *Flavor) Inject() error {
	if f.injected {
		return nil
	}
	f.injected = true

	if f.FlavorLocal == nil {
		f.FlavorLocal = &local.FlavorLocal{}
	}
	f.FlavorLocal.Inject()

	if f.FlavorEtcdKafka == nil {
		f.FlavorEtcdKafka = &etcdkafka.FlavorEtcdKafka{FlavorLocal: f.FlavorLocal}
	}
	f.FlavorEtcdKafka.Inject()

	if f.FlavorRPC == nil {
		f.FlavorRPC = &rpc.FlavorRPC{FlavorLocal: f.FlavorLocal}
	}
	f.FlavorRPC.Inject()

	if f.FlavorRedis == nil {
		f.FlavorRedis = &redis.FlavorRedis{FlavorLocal: f.FlavorLocal}
	}
	f.FlavorRedis.Inject()

	f.GoVPP.Deps.PluginInfraDeps = *f.FlavorEtcdKafka.FlavorLocal.InfraDeps("govpp")
	f.VPP.Deps.PluginInfraDeps = *f.FlavorEtcdKafka.FlavorLocal.InfraDeps("default-plugins")
	f.VPP.Deps.Publish = &f.FlavorEtcdKafka.ETCDDataSync
	f.VPP.Deps.PublishStatistics = datasync.CompositeKVProtoWriter{
		Adapters: []datasync.KeyProtoValWriter{&f.FlavorEtcdKafka.ETCDDataSync, &f.FlavorRedis.RedisDataSync},
	}
	f.VPP.Deps.Watch = &f.FlavorEtcdKafka.ETCDDataSync
	f.VPP.Deps.Kafka = &f.FlavorEtcdKafka.Kafka
	f.VPP.Deps.GoVppmux = &f.GoVPP
	f.VPP.Deps.Linux = &f.Linux
	f.VPP.Linux.Deps.Watcher = &f.FlavorEtcdKafka.ETCDDataSync

	return nil
}

// Plugins combines Generic Plugins and Standard VPP Plugins + (their ETCD Connector/Adapter with RESYNC)
func (f *Flavor) Plugins() []*core.NamedPlugin {
	f.Inject()
	return core.ListPluginsInFlavor(f)
}
<|endoftext|>


<commit_before>package speccy

import (
	"fmt"

	"github.com/jbert/zog"
	"github.com/veandco/go-sdl2/sdl"
)

const (
	screenWidth  = 256
	screenHeight = 192
	screenScale  = 4

	screenMemStart = 0x4000
)

type Screen struct {
	window   *sdl.Window
	renderer *sdl.Renderer
	mem      *zog.Memory
}

func NewScreen(mem *zog.Memory) (*Screen, error) {
	winTitle := "Speccy"
	window, err := sdl.CreateWindow(winTitle, sdl.WINDOWPOS_UNDEFINED, sdl.WINDOWPOS_UNDEFINED,
		screenWidth*screenScale, screenHeight*screenScale, sdl.WINDOW_SHOWN)
	if err != nil {
		return nil, fmt.Errorf("Failed to create window: %s\n", err)
	}

	renderer, err := sdl.CreateRenderer(window, -1, sdl.RENDERER_ACCELERATED)
	if err != nil {
		return nil, fmt.Errorf("Failed to create renderer: %s\n", err)
	}
	renderer.Clear()

	return &Screen{
		window:   window,
		renderer: renderer,
		mem:      mem,
	}, nil
}

func (s *Screen) Draw() {

	// Clear screen
	rect := sdl.Rect{0, 0, int32(screenWidth * screenScale), int32(screenHeight * screenScale)}
	s.renderer.SetDrawColor(255, 255, 255, 255)
	s.renderer.FillRect(&rect)

	// Draw each scanline
	for y := 0; y < screenHeight; y++ {
		s.drawScanline(y)
	}
	s.renderer.Present()
}

func (s *Screen) drawScanline(y int) {
	lineMemLen := screenWidth / 8

	sector := (y & 0xc0) >> 6
	sectorRow := (y & 0x38) >> 3
	charRow := (y & 0x07)
	addr := screenMemStart + (sector*64+charRow*8+sectorRow)*lineMemLen

	lineMem, err := s.mem.PeekBuf(uint16(addr), lineMemLen)
	if err != nil {
		panic(fmt.Errorf("Can't read screen memory at [%04X] len (%04X)", addr, lineMemLen))
	}
	for i, b := range lineMem {
		for bit := 0; bit < 8; bit++ {
			x := i*8 + bit
			if b&0x80 != 0 {
				s.renderer.SetDrawColor(0, 0, 0, 255)
			} else {
				s.renderer.SetDrawColor(255, 255, 255, 255)
			}
			//			fmt.Printf("x %d y %d i %d bit %d\n", x, y, i, b)
			rect := sdl.Rect{int32(x * screenScale), int32(y * screenScale), screenScale, screenScale}
			s.renderer.FillRect(&rect)
			//			s.renderer.DrawPoint(x, y)
			b <<= 1
		}
	}
}

func (s *Screen) Close() {
	s.renderer.Destroy()
	s.window.Destroy()
}
<commit_msg>Add colour attributes (and bright, no flash)<commit_after>package speccy

import (
	"fmt"

	"github.com/jbert/zog"
	"github.com/veandco/go-sdl2/sdl"
)

const (
	screenWidth  = 256
	screenHeight = 192
	screenScale  = 4

	screenMemStart = 0x4000
	colourMemStart = 0x5800
)

type Screen struct {
	window   *sdl.Window
	renderer *sdl.Renderer
	mem      *zog.Memory
}

func NewScreen(mem *zog.Memory) (*Screen, error) {
	winTitle := "Speccy"
	window, err := sdl.CreateWindow(winTitle, sdl.WINDOWPOS_UNDEFINED, sdl.WINDOWPOS_UNDEFINED,
		screenWidth*screenScale, screenHeight*screenScale, sdl.WINDOW_SHOWN)
	if err != nil {
		return nil, fmt.Errorf("Failed to create window: %s\n", err)
	}

	renderer, err := sdl.CreateRenderer(window, -1, sdl.RENDERER_ACCELERATED)
	if err != nil {
		return nil, fmt.Errorf("Failed to create renderer: %s\n", err)
	}
	renderer.Clear()

	return &Screen{
		window:   window,
		renderer: renderer,
		mem:      mem,
	}, nil
}

func (s *Screen) Draw() {

	// Clear screen
	rect := sdl.Rect{0, 0, int32(screenWidth * screenScale), int32(screenHeight * screenScale)}
	s.renderer.SetDrawColor(255, 255, 255, 255)
	s.renderer.FillRect(&rect)

	// Draw each scanline
	for y := 0; y < screenHeight; y++ {
		s.drawScanline(y)
	}
	s.renderer.Present()
}

func (s *Screen) drawScanline(y int) {
	lineMemLen := screenWidth / 8

	sector := (y & 0xc0) >> 6
	sectorRow := (y & 0x38) >> 3
	charRow := (y & 0x07)
	addr := screenMemStart + (sector*64+charRow*8+sectorRow)*lineMemLen

	colourRow := (sector << 3) + sectorRow
	colourAddr := colourMemStart + (256/8)*colourRow

	lineMem, err := s.mem.PeekBuf(uint16(addr), lineMemLen)
	if err != nil {
		panic(fmt.Errorf("Can't read screen memory at [%04X] len (%04X)", addr, lineMemLen))
	}
	colourMem, err := s.mem.PeekBuf(uint16(colourAddr), lineMemLen)
	if err != nil {
		panic(fmt.Errorf("Can't read screen memory at [%04X] len (%04X)", addr, lineMemLen))
	}

	for i, b := range lineMem {
		colourByte := colourMem[i]
		ink := colourByte & 0x07
		paper := (colourByte & 0x38) >> 3
		bright := (colourByte & 0x40) >> 6
		flash := (colourByte & 0x80) >> 7

		for bit := 0; bit < 8; bit++ {
			x := i*8 + bit
			if b&0x80 != 0 {
				s.SetDrawColour(true, ink, paper, bright, flash)
				//				s.renderer.SetDrawColor(0, 0, 0, 255)
			} else {
				s.SetDrawColour(false, ink, paper, bright, flash)
				//				s.renderer.SetDrawColor(255, 255, 255, 255)
			}
			//			fmt.Printf("x %d y %d i %d bit %d\n", x, y, i, b)
			rect := sdl.Rect{int32(x * screenScale), int32(y * screenScale), screenScale, screenScale}
			s.renderer.FillRect(&rect)
			//			s.renderer.DrawPoint(x, y)
			b <<= 1
		}
	}
}

// Bright versions, non-bright are reduced from ff to d7
var Colours = []sdl.Color{
	{0, 0, 0, 0},
	{0, 0, 1, 0},
	{1, 0, 0, 0},
	{1, 0, 1, 0},
	{0, 1, 0, 0},
	{0, 1, 1, 0},
	{1, 1, 0, 0},
	{1, 1, 1, 0},
}

func (s *Screen) SetDrawColour(wantInk bool, ink, paper, bright, flash byte) {
	index := paper
	if wantInk {
		index = ink
	}
	c := Colours[index]
	factor := byte(0xd7)
	if bright != 0 {
		factor = 0xff
	}
	s.renderer.SetDrawColor(c.R*factor, c.G*factor, c.B*factor, 255)
}

func (s *Screen) Close() {
	s.renderer.Destroy()
	s.window.Destroy()
}
<|endoftext|>


<commit_before>package formater_test

import (
	"fmt"
	"os"

	"github.com/muhqu/go-gherkin/formater"
	"github.com/muhqu/go-gherkin/nodes"
)

func ExampleGherkinPrettyFormater_FormatStep_stepWithTable() {
	fmt.Println(">")

	gfmt := &formater.GherkinPrettyFormater{}

	step := nodes.NewMutableStepNode("Given", "the following users:").
		WithTable(nodes.NewMutableTableNode().WithRows([][]string{
		{"username", "email"},
		{"Foobar", "foo@bar.org"},
		{"JohnDoe", "naked-john74@hotmail.com"},
	}))

	gfmt.FormatStep(step, os.Stdout)

	fmt.Println(">")
	// Output:
	// >
	//     Given the following users:
	//       | username | email                    |
	//       | Foobar   | foo@bar.org              |
	//       | JohnDoe  | naked-john74@hotmail.com |
	// >
}

func ExampleGherkinPrettyFormater_FormatStep_stepWithPyString() {
	fmt.Println(">")

	gfmt := &formater.GherkinPrettyFormater{}

	step := nodes.NewMutableStepNode("Given", "the following user relations:").
		WithPyString(nodes.NewMutablePyStringNode().WithLines([]string{
		"Jenny [follows] Mary, David",
		"Bill [knows] Mary, Jenny, David",
	}))

	gfmt.FormatStep(step, os.Stdout)

	fmt.Println(">")
	// Output:
	// >
	//     Given the following user relations:
	//       """
	//       Jenny [follows] Mary, David
	//       Bill [knows] Mary, Jenny, David
	//       """
	// >
}

func ExampleGherkinPrettyFormater_FormatStep_givenWhenThen() {
	fmt.Println(">")

	gfmt := &formater.GherkinPrettyFormater{CenterSteps: true}

	step := nodes.NewMutableStepNode("Given", "I have 2 banannas")
	gfmt.FormatStep(step, os.Stdout)

	step = nodes.NewMutableStepNode("When", "I eat 1 bananna")
	gfmt.FormatStep(step, os.Stdout)

	step = nodes.NewMutableStepNode("And", "I throw 1 bananna away")
	gfmt.FormatStep(step, os.Stdout)

	step = nodes.NewMutableStepNode("Then", "I should still have 2 banannas")
	gfmt.FormatStep(step, os.Stdout)

	fmt.Println(">")
	// Output:
	// >
	//     Given I have 2 banannas
	//      When I eat 1 bananna
	//       And I throw 1 bananna away
	//      Then I should still have 2 banannas
	// >
}

func ExampleGherkinPrettyFormater_FormatStep_givenWhenThenComments() {
	fmt.Println(">")

	gfmt := &formater.GherkinPrettyFormater{CenterSteps: true, AnsiColors: false}

	scenario := nodes.NewMutableScenarioNode("Awesome", nil)
	scenario.SetComment(nodes.NewCommentNode("scenario comment"))
	step := nodes.NewMutableStepNode("Given", "I have 2 banannas")
	step.SetComment(nodes.NewCommentNode("first step comment"))
	scenario.AddStep(step)

	step = nodes.NewMutableStepNode("When", "I eat 1 bananna")
	step.SetComment(nodes.NewCommentNode("2nd step comment"))
	scenario.AddStep(step)

	step = nodes.NewMutableStepNode("And", "I throw 1 bananna away")
	step.SetComment(nodes.NewCommentNode("3rd step comment"))
	scenario.AddStep(step)

	step = nodes.NewMutableStepNode("Then", "I should still have 2 banannas")
	step.SetComment(nodes.NewCommentNode("4th step comment"))
	scenario.AddStep(step)
	gfmt.FormatScenario(scenario, os.Stdout)

	fmt.Println(">")
	// Output:
	// >
	//   Scenario: Awesome                           # scenario comment
	//     Given I have 2 banannas                   # first step comment
	//      When I eat 1 bananna                     # 2nd step comment
	//       And I throw 1 bananna away              # 3rd step comment
	//      Then I should still have 2 banannas      # 4th step comment
	// >
}

func ExampleGherkinPrettyFormater_FormatStep_givenWhenThenFixAnds() {
	fmt.Println(">")

	gfmt := &formater.GherkinPrettyFormater{CenterSteps: true, FixAnd: true, AnsiColors: false}

	scenario := nodes.NewMutableScenarioNode("Awesome", nil)

	scenario.AddStep(nodes.NewMutableStepNode("Given", "I have 4 banannas"))
	scenario.AddStep(nodes.NewMutableStepNode("When", "I eat 1 bananna"))
	scenario.AddStep(nodes.NewMutableStepNode("When", "I throw 1 bananna away"))
	scenario.AddStep(nodes.NewMutableStepNode("Then", "I should still have 2 banannas"))
	scenario.AddStep(nodes.NewMutableStepNode("When", "I eat 1 bananna"))
	scenario.AddStep(nodes.NewMutableStepNode("When", "I buy 3 new banannas"))
	scenario.AddStep(nodes.NewMutableStepNode("Then", "I should have 4 banannas again"))
	gfmt.FormatScenario(scenario, os.Stdout)

	fmt.Println(">")
	// Output:
	// >
	//   Scenario: Awesome
	//     Given I have 4 banannas
	//      When I eat 1 bananna
	//       And I throw 1 bananna away
	//      Then I should still have 2 banannas
	//      When I eat 1 bananna
	//       And I buy 3 new banannas
	//      Then I should have 4 banannas again
	// >
}
<commit_msg>adjust other banana example too<commit_after>package formater_test

import (
	"fmt"
	"os"

	"github.com/muhqu/go-gherkin/formater"
	"github.com/muhqu/go-gherkin/nodes"
)

func ExampleGherkinPrettyFormater_FormatStep_stepWithTable() {
	fmt.Println(">")

	gfmt := &formater.GherkinPrettyFormater{}

	step := nodes.NewMutableStepNode("Given", "the following users:").
		WithTable(nodes.NewMutableTableNode().WithRows([][]string{
		{"username", "email"},
		{"Foobar", "foo@bar.org"},
		{"JohnDoe", "naked-john74@hotmail.com"},
	}))

	gfmt.FormatStep(step, os.Stdout)

	fmt.Println(">")
	// Output:
	// >
	//     Given the following users:
	//       | username | email                    |
	//       | Foobar   | foo@bar.org              |
	//       | JohnDoe  | naked-john74@hotmail.com |
	// >
}

func ExampleGherkinPrettyFormater_FormatStep_stepWithPyString() {
	fmt.Println(">")

	gfmt := &formater.GherkinPrettyFormater{}

	step := nodes.NewMutableStepNode("Given", "the following user relations:").
		WithPyString(nodes.NewMutablePyStringNode().WithLines([]string{
		"Jenny [follows] Mary, David",
		"Bill [knows] Mary, Jenny, David",
	}))

	gfmt.FormatStep(step, os.Stdout)

	fmt.Println(">")
	// Output:
	// >
	//     Given the following user relations:
	//       """
	//       Jenny [follows] Mary, David
	//       Bill [knows] Mary, Jenny, David
	//       """
	// >
}

func ExampleGherkinPrettyFormater_FormatStep_givenWhenThen() {
	fmt.Println(">")

	gfmt := &formater.GherkinPrettyFormater{CenterSteps: true}

	step := nodes.NewMutableStepNode("Given", "I have 2 banannas")
	gfmt.FormatStep(step, os.Stdout)

	step = nodes.NewMutableStepNode("When", "I eat 1 bananna")
	gfmt.FormatStep(step, os.Stdout)

	step = nodes.NewMutableStepNode("And", "I throw 1 bananna away")
	gfmt.FormatStep(step, os.Stdout)

	step = nodes.NewMutableStepNode("Then", "I should still have 2 banannas")
	gfmt.FormatStep(step, os.Stdout)

	fmt.Println(">")
	// Output:
	// >
	//     Given I have 2 banannas
	//      When I eat 1 bananna
	//       And I throw 1 bananna away
	//      Then I should still have 2 banannas
	// >
}

func ExampleGherkinPrettyFormater_FormatStep_givenWhenThenComments() {
	fmt.Println(">")

	gfmt := &formater.GherkinPrettyFormater{CenterSteps: true, AnsiColors: false}

	scenario := nodes.NewMutableScenarioNode("Awesome", nil)
	scenario.SetComment(nodes.NewCommentNode("scenario comment"))
	step := nodes.NewMutableStepNode("Given", "I have 4 banannas")
	step.SetComment(nodes.NewCommentNode("first step comment"))
	scenario.AddStep(step)

	step = nodes.NewMutableStepNode("When", "I eat 1 bananna")
	step.SetComment(nodes.NewCommentNode("2nd step comment"))
	scenario.AddStep(step)

	step = nodes.NewMutableStepNode("And", "I throw 1 bananna away")
	step.SetComment(nodes.NewCommentNode("3rd step comment"))
	scenario.AddStep(step)

	step = nodes.NewMutableStepNode("Then", "I should still have 2 banannas")
	step.SetComment(nodes.NewCommentNode("4th step comment"))
	scenario.AddStep(step)
	gfmt.FormatScenario(scenario, os.Stdout)

	fmt.Println(">")
	// Output:
	// >
	//   Scenario: Awesome                           # scenario comment
	//     Given I have 4 banannas                   # first step comment
	//      When I eat 1 bananna                     # 2nd step comment
	//       And I throw 1 bananna away              # 3rd step comment
	//      Then I should still have 2 banannas      # 4th step comment
	// >
}

func ExampleGherkinPrettyFormater_FormatStep_givenWhenThenFixAnds() {
	fmt.Println(">")

	gfmt := &formater.GherkinPrettyFormater{CenterSteps: true, FixAnd: true, AnsiColors: false}

	scenario := nodes.NewMutableScenarioNode("Awesome", nil)

	scenario.AddStep(nodes.NewMutableStepNode("Given", "I have 4 banannas"))
	scenario.AddStep(nodes.NewMutableStepNode("When", "I eat 1 bananna"))
	scenario.AddStep(nodes.NewMutableStepNode("When", "I throw 1 bananna away"))
	scenario.AddStep(nodes.NewMutableStepNode("Then", "I should still have 2 banannas"))
	scenario.AddStep(nodes.NewMutableStepNode("When", "I eat 1 bananna"))
	scenario.AddStep(nodes.NewMutableStepNode("When", "I buy 3 new banannas"))
	scenario.AddStep(nodes.NewMutableStepNode("Then", "I should have 4 banannas again"))
	gfmt.FormatScenario(scenario, os.Stdout)

	fmt.Println(">")
	// Output:
	// >
	//   Scenario: Awesome
	//     Given I have 4 banannas
	//      When I eat 1 bananna
	//       And I throw 1 bananna away
	//      Then I should still have 2 banannas
	//      When I eat 1 bananna
	//       And I buy 3 new banannas
	//      Then I should have 4 banannas again
	// >
}
<|endoftext|>


<commit_before>package gannoy

import (
	"fmt"
	"math"
	"runtime"
	"sort"
	"sync"

	"github.com/gansidui/priority_queue"
)

type GannoyIndex struct {
	meta      meta
	tree      int
	dim       int
	distance  Distance
	random    Random
	nodes     Nodes
	K         int
	numWorker int
	buildChan chan buildArgs
}

func NewGannoyIndex(metaFile string, distance Distance, random Random) (GannoyIndex, error) {

	meta, err := loadMeta(metaFile)
	if err != nil {
		return GannoyIndex{}, err
	}
	tree := meta.tree
	dim := meta.dim
	K := meta.K

	ann := meta.treePath()

	gannoy := GannoyIndex{
		meta:      meta,
		tree:      tree,
		dim:       dim,
		distance:  distance,
		random:    random,
		K:         K,
		nodes:     newNodes(ann, tree, dim, K),
		numWorker: numWorker(tree),
		buildChan: make(chan buildArgs, 1),
	}
	go gannoy.builder()
	return gannoy, nil
}

func (g GannoyIndex) Tree() {
	for i, root := range g.meta.roots() {
		n, err := g.nodes.getNode(root)
		if err != nil {
			fmt.Printf("%v\n", err)
			break
		}
		g.printTree(i, n, root, 0)
	}
}

func (g GannoyIndex) MetaFile() string {
	return g.meta.file.Name()
}

func (g *GannoyIndex) AddItem(key int, w []float64) error {
	args := buildArgs{action: ADD, key: key, w: w, result: make(chan error)}
	g.buildChan <- args
	return <-args.result
}

func (g *GannoyIndex) RemoveItem(key int) error {
	args := buildArgs{action: DELETE, key: key, result: make(chan error)}
	g.buildChan <- args
	return <-args.result
}

func (g *GannoyIndex) UpdateItem(key int, w []float64) error {
	args := buildArgs{action: UPDATE, key: key, w: w, result: make(chan error)}
	g.buildChan <- args
	return <-args.result
}

func (g *GannoyIndex) GetNnsByKey(key, n, searchK int) ([]int, error) {
	m, err := g.nodes.getNodeByKey(key)
	if err != nil || !m.isLeaf() {
		return []int{}, fmt.Errorf("Not found")
	}
	return g.getAllNns(m.v, n, searchK)
}

func (g *GannoyIndex) getAllNns(v []float64, n, searchK int) ([]int, error) {
	if searchK == -1 {
		searchK = n * g.tree
	}

	q := priority_queue.New()
	for _, root := range g.meta.roots() {
		q.Push(&Queue{priority: math.Inf(1), value: root})
	}

	nns := []int{}
	for len(nns) < searchK && q.Len() > 0 {
		top := q.Top().(*Queue)
		d := top.priority
		i := top.value

		nd, err := g.nodes.getNode(i)
		if err != nil {
			return []int{}, err
		}
		q.Pop()
		if nd.isLeaf() {
			nns = append(nns, i)
		} else if nd.nDescendants <= g.K {
			dst := nd.children
			nns = append(nns, dst...)
		} else {
			margin := g.distance.margin(nd, v)
			q.Push(&Queue{priority: math.Min(d, +margin), value: nd.children[1]})
			q.Push(&Queue{priority: math.Min(d, -margin), value: nd.children[0]})
		}
	}

	sort.Ints(nns)
	nnsDist := []sorter{}
	last := -1
	for _, j := range nns {
		if j == last {
			continue
		}
		last = j
		node, err := g.nodes.getNode(j)
		if err != nil {
			return []int{}, err
		}
		nnsDist = append(nnsDist, sorter{value: g.distance.distance(v, node.v), id: node.key})
	}

	m := len(nnsDist)
	p := m
	if n < m {
		p = n
	}

	HeapSort(nnsDist, DESC, p)

	result := make([]int, p)
	for i := 0; i < p; i++ {
		result[i] = nnsDist[m-1-i].id
	}

	return result, nil
}

func (g *GannoyIndex) addItem(key int, w []float64) error {
	if len(w) != g.dim {
		return fmt.Errorf("Dimension mismatch. expect %d, but %d.\n", g.dim, len(w))
	}
	n := g.nodes.newNode()
	n.key = key
	n.v = w
	n.parents = make([]int, g.tree)
	err := n.save()
	if err != nil {
		return err
	}
	// fmt.Printf("id %d\n", n.id)

	var wg sync.WaitGroup
	wg.Add(g.tree)
	buildChan := make(chan int, g.tree)
	worker := func(n Node) {
		for index := range buildChan {
			g.build(index, g.meta.roots()[index], n)
			wg.Done()
		}
	}

	for i := 0; i < g.numWorker; i++ {
		go worker(n)
	}

	for index, _ := range g.meta.roots() {
		buildChan <- index
	}

	wg.Wait()
	close(buildChan)
	g.nodes.maps.add(n.id, key)

	return nil
}

func (g *GannoyIndex) build(index, root int, n Node) {
	if root == -1 {
		// 最初のノード
		n.parents[index] = -1
		n.save()
		g.meta.updateRoot(index, n.id)
		return
	}
	id := g.findBranchByVector(root, n.v)
	found, _ := g.nodes.getNode(id)
	// fmt.Printf("Found %d\n", item)

	org_parent := found.parents[index]
	if found.isBucket() && len(found.children) < g.K {
		// ノードに余裕があれば追加
		// fmt.Printf("pattern bucket\n")
		n.updateParents(index, id)
		found.nDescendants++
		found.children = append(found.children, n.id)
		found.save()
	} else {
		// ノードが上限またはリーフノードであれば新しいノードを追加
		willDelete := false
		var indices []int
		if found.isLeaf() {
			// fmt.Printf("pattern leaf node\n")
			indices = []int{id, n.id}
		} else {
			// fmt.Printf("pattern full backet\n")
			indices = append(found.children, n.id)
			willDelete = true
		}

		m := g.makeTree(index, org_parent, indices)
		if root == 0 || org_parent == 0 || org_parent == -1 {
			// rootノードの入れ替え
			g.meta.updateRoot(index, m)
		} else {
			parent, _ := g.nodes.getNode(org_parent)
			parent.nDescendants++
			children := make([]int, len(parent.children))
			for i, child := range parent.children {
				if child == id {
					// 新しいノードに変更
					children[i] = m
				} else {
					// 既存のノードのまま
					children[i] = child
				}
			}
			parent.children = children
			parent.save()

		}
		if willDelete {
			found.destroy()
			g.nodes.free.push(found.id)
		}
	}
}

func (g *GannoyIndex) removeItem(key int) error {
	n, err := g.nodes.getNodeByKey(key)
	if err != nil {
		return err
	}

	var wg sync.WaitGroup
	wg.Add(g.tree)
	buildChan := make(chan int, g.tree)
	worker := func(n Node) {
		for root := range buildChan {
			g.remove(root, n)
			wg.Done()
		}
	}

	for i := 0; i < g.numWorker; i++ {
		go worker(n)
	}
	for index, _ := range g.meta.roots() {
		buildChan <- index
	}

	wg.Wait()
	close(buildChan)

	g.nodes.maps.remove(key)
	n.destroy()
	g.nodes.free.push(n.id)

	return nil
}

func (g *GannoyIndex) remove(root int, node Node) {
	if node.isRoot(root) {
		g.meta.updateRoot(root, -1)
		return
	}
	parent, _ := g.nodes.getNode(node.parents[root])
	if parent.isBucket() && len(parent.children) > 2 {
		// fmt.Printf("pattern bucket\n")
		target := -1
		for i, child := range parent.children {
			if child == node.id {
				target = i
			}
		}
		if target == -1 {
			return
		}
		children := append(parent.children[:target], parent.children[(target+1):]...)
		parent.nDescendants--
		parent.children = children
		parent.save()
	} else {
		// fmt.Printf("pattern leaf node\n")
		var other int
		for _, child := range parent.children {
			if child != node.id {
				other = child
			}
		}
		if parent.isRoot(root) {
			g.meta.updateRoot(root, other)
		} else {
			grandParent, _ := g.nodes.getNode(parent.parents[root])
			children := []int{}
			for _, child := range grandParent.children {
				if child == node.parents[root] {
					children = append(children, other)
				} else {
					children = append(children, child)
				}
			}
			grandParent.nDescendants--
			grandParent.children = children
			grandParent.save()
		}

		otherNode, _ := g.nodes.getNode(other)
		otherNode.updateParents(root, parent.parents[root])

		parent.destroy()
		g.nodes.free.push(parent.id)
	}
}

func (g GannoyIndex) findBranchByVector(id int, v []float64) int {
	node, _ := g.nodes.getNode(id)
	if node.isLeaf() || node.isBucket() {
		return id
	}
	side := g.distance.side(node, v, g.random)
	return g.findBranchByVector(node.children[side], v)
}

func (g *GannoyIndex) makeTree(root, parent int, ids []int) int {
	if len(ids) == 1 {
		n, _ := g.nodes.getNode(ids[0])
		if len(n.parents) == 0 {
			n.parents = make([]int, g.tree)
		}
		n.updateParents(root, parent)
		return ids[0]
	}

	if len(ids) <= g.K {
		m := g.nodes.newNode()
		m.parents = make([]int, g.tree)
		m.nDescendants = len(ids)
		m.parents[root] = parent
		m.children = ids
		m.save()
		for _, child := range ids {
			c, _ := g.nodes.getNode(child)
			if len(c.parents) == 0 {
				c.parents = make([]int, g.tree)
			}
			c.updateParents(root, m.id)
		}
		return m.id
	}

	children := make([]Node, len(ids))
	for i, id := range ids {
		children[i], _ = g.nodes.getNode(id)
	}

	childrenIds := [2][]int{[]int{}, []int{}}

	m := g.nodes.newNode()
	m.parents = make([]int, g.tree)
	m.nDescendants = len(ids)
	m.parents[root] = parent

	m = g.distance.createSplit(children, g.random, m)
	for _, id := range ids {
		n, _ := g.nodes.getNode(id)
		side := g.distance.side(m, n.v, g.random)
		childrenIds[side] = append(childrenIds[side], id)
	}

	for len(childrenIds[0]) == 0 || len(childrenIds[1]) == 0 {
		childrenIds[0] = []int{}
		childrenIds[1] = []int{}
		for z := 0; z < g.dim; z++ {
			m.v[z] = 0.0
		}
		for _, id := range ids {
			side := g.random.flip()
			childrenIds[side] = append(childrenIds[side], id)
		}
	}

	var flip int
	if len(childrenIds[0]) > len(childrenIds[1]) {
		flip = 1
	}

	m.save()
	for side := 0; side < 2; side++ {
		m.children[side^flip] = g.makeTree(root, m.id, childrenIds[side^flip])
	}
	m.save()

	return m.id
}

type buildArgs struct {
	action int
	key    int
	w      []float64
	result chan error
}

func (g *GannoyIndex) builder() {
	for args := range g.buildChan {
		switch args.action {
		case ADD:
			args.result <- g.addItem(args.key, args.w)
		case DELETE:
			args.result <- g.removeItem(args.key)
		case UPDATE:
			err := g.removeItem(args.key)
			if err != nil {
				args.result <- err
			} else {
				args.result <- g.addItem(args.key, args.w)
			}
		}
	}
}

func (g GannoyIndex) printTree(root int, node Node, id, tab int) {
	for i := 0; i < tab*2; i++ {
		fmt.Print(" ")
	}
	fmt.Printf("%d [%d] (%d) [nDescendants: %d, v: %v]\n", id, node.key, node.parents[root], node.nDescendants, node.v)
	if !node.isLeaf() {
		for _, child := range node.children {
			n, err := g.nodes.getNode(child)
			if err != nil {
				fmt.Printf("%v\n", err)
				break
			}
			g.printTree(root, n, child, tab+1)
		}
	}
}

func numWorker(tree int) int {
	procs := runtime.GOMAXPROCS(0) // current setting
	if tree < procs {
		return tree
	} else {
		return procs
	}
}
<commit_msg>Avoid malloc by append.<commit_after>package gannoy

import (
	"fmt"
	"math"
	"runtime"
	"sort"
	"sync"

	"github.com/gansidui/priority_queue"
)

type GannoyIndex struct {
	meta      meta
	tree      int
	dim       int
	distance  Distance
	random    Random
	nodes     Nodes
	K         int
	numWorker int
	buildChan chan buildArgs
}

func NewGannoyIndex(metaFile string, distance Distance, random Random) (GannoyIndex, error) {

	meta, err := loadMeta(metaFile)
	if err != nil {
		return GannoyIndex{}, err
	}
	tree := meta.tree
	dim := meta.dim
	K := meta.K

	ann := meta.treePath()

	gannoy := GannoyIndex{
		meta:      meta,
		tree:      tree,
		dim:       dim,
		distance:  distance,
		random:    random,
		K:         K,
		nodes:     newNodes(ann, tree, dim, K),
		numWorker: numWorker(tree),
		buildChan: make(chan buildArgs, 1),
	}
	go gannoy.builder()
	return gannoy, nil
}

func (g GannoyIndex) Tree() {
	for i, root := range g.meta.roots() {
		n, err := g.nodes.getNode(root)
		if err != nil {
			fmt.Printf("%v\n", err)
			break
		}
		g.printTree(i, n, root, 0)
	}
}

func (g GannoyIndex) MetaFile() string {
	return g.meta.file.Name()
}

func (g *GannoyIndex) AddItem(key int, w []float64) error {
	args := buildArgs{action: ADD, key: key, w: w, result: make(chan error)}
	g.buildChan <- args
	return <-args.result
}

func (g *GannoyIndex) RemoveItem(key int) error {
	args := buildArgs{action: DELETE, key: key, result: make(chan error)}
	g.buildChan <- args
	return <-args.result
}

func (g *GannoyIndex) UpdateItem(key int, w []float64) error {
	args := buildArgs{action: UPDATE, key: key, w: w, result: make(chan error)}
	g.buildChan <- args
	return <-args.result
}

func (g *GannoyIndex) GetNnsByKey(key, n, searchK int) ([]int, error) {
	m, err := g.nodes.getNodeByKey(key)
	if err != nil || !m.isLeaf() {
		return []int{}, fmt.Errorf("Not found")
	}
	return g.getAllNns(m.v, n, searchK)
}

func (g *GannoyIndex) getAllNns(v []float64, n, searchK int) ([]int, error) {
	if searchK == -1 {
		searchK = n * g.tree
	}

	q := priority_queue.New()
	for _, root := range g.meta.roots() {
		q.Push(&Queue{priority: math.Inf(1), value: root})
	}

	nns := []int{}
	for len(nns) < searchK && q.Len() > 0 {
		top := q.Top().(*Queue)
		d := top.priority
		i := top.value

		nd, err := g.nodes.getNode(i)
		if err != nil {
			return []int{}, err
		}
		q.Pop()
		if nd.isLeaf() {
			nns = append(nns, i)
		} else if nd.nDescendants <= g.K {
			dst := nd.children
			nns = append(nns, dst...)
		} else {
			margin := g.distance.margin(nd, v)
			q.Push(&Queue{priority: math.Min(d, +margin), value: nd.children[1]})
			q.Push(&Queue{priority: math.Min(d, -margin), value: nd.children[0]})
		}
	}

	sort.Ints(nns)
	nnsDist := make([]sorter, len(nns))
	last := -1
	dup := 0
	for idx, j := range nns {
		if j == last {
			dup++
			continue
		}
		last = j
		node, err := g.nodes.getNode(j)
		if err != nil {
			return []int{}, err
		}
		nnsDist[idx-dup] = sorter{value: g.distance.distance(v, node.v), id: node.key}
	}
	nnsDist = nnsDist[:len(nns)-dup]

	m := len(nnsDist)
	p := m
	if n < m {
		p = n
	}

	HeapSort(nnsDist, DESC, p)

	result := make([]int, p)
	for i := 0; i < p; i++ {
		result[i] = nnsDist[m-1-i].id
	}

	return result, nil
}

func (g *GannoyIndex) addItem(key int, w []float64) error {
	if len(w) != g.dim {
		return fmt.Errorf("Dimension mismatch. expect %d, but %d.\n", g.dim, len(w))
	}
	n := g.nodes.newNode()
	n.key = key
	n.v = w
	n.parents = make([]int, g.tree)
	err := n.save()
	if err != nil {
		return err
	}
	// fmt.Printf("id %d\n", n.id)

	var wg sync.WaitGroup
	wg.Add(g.tree)
	buildChan := make(chan int, g.tree)
	worker := func(n Node) {
		for index := range buildChan {
			g.build(index, g.meta.roots()[index], n)
			wg.Done()
		}
	}

	for i := 0; i < g.numWorker; i++ {
		go worker(n)
	}

	for index, _ := range g.meta.roots() {
		buildChan <- index
	}

	wg.Wait()
	close(buildChan)
	g.nodes.maps.add(n.id, key)

	return nil
}

func (g *GannoyIndex) build(index, root int, n Node) {
	if root == -1 {
		// 最初のノード
		n.parents[index] = -1
		n.save()
		g.meta.updateRoot(index, n.id)
		return
	}
	id := g.findBranchByVector(root, n.v)
	found, _ := g.nodes.getNode(id)
	// fmt.Printf("Found %d\n", item)

	org_parent := found.parents[index]
	if found.isBucket() && len(found.children) < g.K {
		// ノードに余裕があれば追加
		// fmt.Printf("pattern bucket\n")
		n.updateParents(index, id)
		found.nDescendants++
		found.children = append(found.children, n.id)
		found.save()
	} else {
		// ノードが上限またはリーフノードであれば新しいノードを追加
		willDelete := false
		var indices []int
		if found.isLeaf() {
			// fmt.Printf("pattern leaf node\n")
			indices = []int{id, n.id}
		} else {
			// fmt.Printf("pattern full backet\n")
			indices = append(found.children, n.id)
			willDelete = true
		}

		m := g.makeTree(index, org_parent, indices)
		if root == 0 || org_parent == 0 || org_parent == -1 {
			// rootノードの入れ替え
			g.meta.updateRoot(index, m)
		} else {
			parent, _ := g.nodes.getNode(org_parent)
			parent.nDescendants++
			children := make([]int, len(parent.children))
			for i, child := range parent.children {
				if child == id {
					// 新しいノードに変更
					children[i] = m
				} else {
					// 既存のノードのまま
					children[i] = child
				}
			}
			parent.children = children
			parent.save()

		}
		if willDelete {
			found.destroy()
			g.nodes.free.push(found.id)
		}
	}
}

func (g *GannoyIndex) removeItem(key int) error {
	n, err := g.nodes.getNodeByKey(key)
	if err != nil {
		return err
	}

	var wg sync.WaitGroup
	wg.Add(g.tree)
	buildChan := make(chan int, g.tree)
	worker := func(n Node) {
		for root := range buildChan {
			g.remove(root, n)
			wg.Done()
		}
	}

	for i := 0; i < g.numWorker; i++ {
		go worker(n)
	}
	for index, _ := range g.meta.roots() {
		buildChan <- index
	}

	wg.Wait()
	close(buildChan)

	g.nodes.maps.remove(key)
	n.destroy()
	g.nodes.free.push(n.id)

	return nil
}

func (g *GannoyIndex) remove(root int, node Node) {
	if node.isRoot(root) {
		g.meta.updateRoot(root, -1)
		return
	}
	parent, _ := g.nodes.getNode(node.parents[root])
	if parent.isBucket() && len(parent.children) > 2 {
		// fmt.Printf("pattern bucket\n")
		target := -1
		for i, child := range parent.children {
			if child == node.id {
				target = i
			}
		}
		if target == -1 {
			return
		}
		children := append(parent.children[:target], parent.children[(target+1):]...)
		parent.nDescendants--
		parent.children = children
		parent.save()
	} else {
		// fmt.Printf("pattern leaf node\n")
		var other int
		for _, child := range parent.children {
			if child != node.id {
				other = child
			}
		}
		if parent.isRoot(root) {
			g.meta.updateRoot(root, other)
		} else {
			grandParent, _ := g.nodes.getNode(parent.parents[root])
			children := []int{}
			for _, child := range grandParent.children {
				if child == node.parents[root] {
					children = append(children, other)
				} else {
					children = append(children, child)
				}
			}
			grandParent.nDescendants--
			grandParent.children = children
			grandParent.save()
		}

		otherNode, _ := g.nodes.getNode(other)
		otherNode.updateParents(root, parent.parents[root])

		parent.destroy()
		g.nodes.free.push(parent.id)
	}
}

func (g GannoyIndex) findBranchByVector(id int, v []float64) int {
	node, _ := g.nodes.getNode(id)
	if node.isLeaf() || node.isBucket() {
		return id
	}
	side := g.distance.side(node, v, g.random)
	return g.findBranchByVector(node.children[side], v)
}

func (g *GannoyIndex) makeTree(root, parent int, ids []int) int {
	if len(ids) == 1 {
		n, _ := g.nodes.getNode(ids[0])
		if len(n.parents) == 0 {
			n.parents = make([]int, g.tree)
		}
		n.updateParents(root, parent)
		return ids[0]
	}

	if len(ids) <= g.K {
		m := g.nodes.newNode()
		m.parents = make([]int, g.tree)
		m.nDescendants = len(ids)
		m.parents[root] = parent
		m.children = ids
		m.save()
		for _, child := range ids {
			c, _ := g.nodes.getNode(child)
			if len(c.parents) == 0 {
				c.parents = make([]int, g.tree)
			}
			c.updateParents(root, m.id)
		}
		return m.id
	}

	children := make([]Node, len(ids))
	for i, id := range ids {
		children[i], _ = g.nodes.getNode(id)
	}

	childrenIds := [2][]int{[]int{}, []int{}}

	m := g.nodes.newNode()
	m.parents = make([]int, g.tree)
	m.nDescendants = len(ids)
	m.parents[root] = parent

	m = g.distance.createSplit(children, g.random, m)
	for _, id := range ids {
		n, _ := g.nodes.getNode(id)
		side := g.distance.side(m, n.v, g.random)
		childrenIds[side] = append(childrenIds[side], id)
	}

	for len(childrenIds[0]) == 0 || len(childrenIds[1]) == 0 {
		childrenIds[0] = []int{}
		childrenIds[1] = []int{}
		for z := 0; z < g.dim; z++ {
			m.v[z] = 0.0
		}
		for _, id := range ids {
			side := g.random.flip()
			childrenIds[side] = append(childrenIds[side], id)
		}
	}

	var flip int
	if len(childrenIds[0]) > len(childrenIds[1]) {
		flip = 1
	}

	m.save()
	for side := 0; side < 2; side++ {
		m.children[side^flip] = g.makeTree(root, m.id, childrenIds[side^flip])
	}
	m.save()

	return m.id
}

type buildArgs struct {
	action int
	key    int
	w      []float64
	result chan error
}

func (g *GannoyIndex) builder() {
	for args := range g.buildChan {
		switch args.action {
		case ADD:
			args.result <- g.addItem(args.key, args.w)
		case DELETE:
			args.result <- g.removeItem(args.key)
		case UPDATE:
			err := g.removeItem(args.key)
			if err != nil {
				args.result <- err
			} else {
				args.result <- g.addItem(args.key, args.w)
			}
		}
	}
}

func (g GannoyIndex) printTree(root int, node Node, id, tab int) {
	for i := 0; i < tab*2; i++ {
		fmt.Print(" ")
	}
	fmt.Printf("%d [%d] (%d) [nDescendants: %d, v: %v]\n", id, node.key, node.parents[root], node.nDescendants, node.v)
	if !node.isLeaf() {
		for _, child := range node.children {
			n, err := g.nodes.getNode(child)
			if err != nil {
				fmt.Printf("%v\n", err)
				break
			}
			g.printTree(root, n, child, tab+1)
		}
	}
}

func numWorker(tree int) int {
	procs := runtime.GOMAXPROCS(0) // current setting
	if tree < procs {
		return tree
	} else {
		return procs
	}
}
<|endoftext|>


<commit_before>// garble produces pseudo random bytes based on a phrase
// and uses it to garble and ungarble files
package main

// #include <stdint.h>
// #define BSIZE 65536
//
// void xor(int64_t *a, int64_t *b) {
//     int i = BSIZE / 8;
//     while(i--) {
//         a[i] ^= b[i];
//     }
// }
import "C"
import "unsafe"

import (
	"bytes"
	cryptorand "crypto/rand"
	"crypto/sha512"
	"encoding/binary"
	"flag"
	"fmt"
	"io"
	"math/rand"
	"os"
	"runtime"
	"runtime/pprof"
	"syscall"
	"time"
)

const (
	BSIZE    = C.BSIZE
	BSIZE7   = BSIZE - BSIZEMOD
	BSIZEMOD = BSIZE % 7
	MULTI    = 4
	SOURCES  = 8
	POOL     = SOURCES * (MULTI + 1)
)

var (
	narg   = int(0)
	stdout = false
	phrase = ""
	pool   = make(chan []byte, POOL)
	loop   = make(chan []byte, POOL)
	data   = make([]chan []byte, SOURCES)
)

// randomSeed produces a int64 seed based on crypto/rand and time.
func randomSeed() int64 {
	var seed int64

	urandom := make([]byte, 8)
	cryptorand.Reader.Read(urandom)

	for key, value := range urandom {
		seed ^= (int64(value) ^ time.Now().UTC().UnixNano()) << (uint(key) * 8)
	}

	return seed
}

// randomBytes fills byte buffers with random data
func randomBytes(src rand.Source, out chan<- []byte) {
	var (
		r int64
		i = BSIZE
	)

	for buf, ok := <-pool; ok; buf, ok = <-pool {
		r = src.Int63()
		switch { // Go seems to eliminate impossible cases
		case BSIZEMOD == 6:
			buf[BSIZE-6] = byte(r >> 48)
			fallthrough
		case BSIZEMOD == 5:
			buf[BSIZE-5] = byte(r >> 32)
			fallthrough
		case BSIZEMOD == 4:
			buf[BSIZE-4] = byte(r >> 24)
			fallthrough
		case BSIZEMOD == 3:
			buf[BSIZE-3] = byte(r >> 16)
			fallthrough
		case BSIZEMOD == 2:
			buf[BSIZE-2] = byte(r >> 8)
			fallthrough
		case BSIZEMOD == 1:
			buf[BSIZE-1] = byte(r)
		}

		for i = 0; i < BSIZE7; i += 7 {
			r = src.Int63()
			buf[i] = byte(r)
			buf[i+1] = byte(r >> 8)
			buf[i+2] = byte(r >> 16)
			buf[i+3] = byte(r >> 24)
			buf[i+4] = byte(r >> 32)
			buf[i+5] = byte(r >> 40)
			buf[i+6] = byte(r >> 48)
		}

		out <- buf
	}
}

// xor a file with random data
func garble(fin *os.File, fout *os.File, in <-chan []byte, out chan<- bool) {
	var n, m int
	var err error
	data := make([]byte, BSIZE)
	var buf []byte

	for ok := true; ok; {
		// read
		n, err = fin.Read(data)
		for n != BSIZE || err != nil {
			if err != nil && err != io.EOF {
				panic(err)
			}
			if n == 0 {
				goto final
			}
			m, err = fin.Read(data[n:BSIZE])
			if m == 0 && err == io.EOF {
				// last partial block
				ok = false
				break
			}
			n += m
		}

		// xor with random data
		buf = <-in
		C.xor((*C.int64_t)(unsafe.Pointer(&data[0])), (*C.int64_t)(unsafe.Pointer(&buf[0])))
		out <- true // done with buf

		// write
		_, err = fout.Write(data[0:n])
		if err != nil {
			if err2, ok := err.(*os.PathError); ok && err2.Err == syscall.EPIPE {
				goto final
			}

			panic(err)
		}
	}

final:
	close(out)
	for {
		<-in // sleep forever
	}
	return
}

// parse command line arguments
var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")

func init() {
	flag.StringVar(&phrase, "phrase", "", "the Garble phrase, by default random")
	flag.Parse()

	narg = flag.NArg()

	if narg <= 0 {
		flag.Usage()
		os.Exit(1)
	}

	if phrase == "" {
		phrase = fmt.Sprintf("%016x", uint64(randomSeed()))
	}

	fmt.Println("Using phrase:", phrase)
}

// the main program...
func main() {
	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			panic(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	// Use available CPUs:
	if runtime.GOMAXPROCS(0) == 1 &&
		runtime.NumCPU() > 1 &&
		os.Getenv("GOMAXPROCS") == "" {
		runtime.GOMAXPROCS(runtime.NumCPU())
	}

	// Open files:
	writers := make([]chan []byte, narg)
	signals := make([]chan bool, narg)

	for i, arg := range flag.Args() {
		var fd [2]*os.File
		for i, _ := range fd {
			if !stdout && arg == "-" {
				fd[0] = os.Stdin
				fd[1] = os.Stdout
				stdout = true
				break
			}
			f, err := os.OpenFile(arg, os.O_RDWR, 0666)
			if err != nil {
				panic(err)
			}
			defer f.Close()
			fd[i] = f
		}

		writers[i] = make(chan []byte, MULTI)
		signals[i] = make(chan bool, MULTI)
		go garble(fd[0], fd[1], writers[i], signals[i])
	}

	// Allocate byte buffer pool:
	buffer := make([]byte, BSIZE*POOL)

	for i := 0; i < BSIZE*POOL; i += BSIZE {
		pool <- buffer[i : i+BSIZE]
	}

	// Initialize random sources:
	hash := sha512.New()
	sum := make([]byte, hash.Size())

	for i := 0; i < SOURCES; i++ {
		var seed, s int64
		var err error

		hash.Write([]byte(":garble:" + phrase))
		hash.Sum(sum[:0])

		buf := bytes.NewReader(sum)
		s = 0

		for err == nil {
			err = binary.Read(buf, binary.LittleEndian, &s)
			seed ^= s
		}

		src := rand.NewSource(seed)
		data[i] = make(chan []byte, MULTI)
		go randomBytes(src, data[i])
	}

	// Route data channels:
	go func(data []chan []byte, writers []chan []byte, signals []chan bool) {
		var buf []byte
		for {
			for _, r := range data {
				buf = <-r

				for i, w := range writers {
					if signals[i] != nil {
						w <- buf
					}
				}

				loop <- buf
			}
		}
	}(data, writers, signals)

	var buf []byte

	for narg > 0 {
		buf = <-loop

		for i, s := range signals {
			if s != nil {
				_, ok := <-s

				if !ok {
					signals[i] = nil
					narg--
				}
			}
		}

		pool <- buf
	}

	fmt.Println("All done!")
}
<commit_msg>range pool<commit_after>// garble produces pseudo random bytes based on a phrase
// and uses it to garble and ungarble files
package main

// #include <stdint.h>
// #define BSIZE 65536
//
// void xor(int64_t *a, int64_t *b) {
//     int i = BSIZE / 8;
//     while(i--) {
//         a[i] ^= b[i];
//     }
// }
import "C"
import "unsafe"

import (
	"bytes"
	cryptorand "crypto/rand"
	"crypto/sha512"
	"encoding/binary"
	"flag"
	"fmt"
	"io"
	"math/rand"
	"os"
	"runtime"
	"runtime/pprof"
	"syscall"
	"time"
)

const (
	BSIZE    = C.BSIZE
	BSIZE7   = BSIZE - BSIZEMOD
	BSIZEMOD = BSIZE % 7
	MULTI    = 4
	SOURCES  = 8
	POOL     = SOURCES * (MULTI + 1)
)

var (
	narg   = int(0)
	stdout = false
	phrase = ""
	pool   = make(chan []byte, POOL)
	loop   = make(chan []byte, POOL)
	data   = make([]chan []byte, SOURCES)
)

// randomSeed produces a int64 seed based on crypto/rand and time.
func randomSeed() int64 {
	var seed int64

	urandom := make([]byte, 8)
	cryptorand.Reader.Read(urandom)

	for key, value := range urandom {
		seed ^= (int64(value) ^ time.Now().UTC().UnixNano()) << (uint(key) * 8)
	}

	return seed
}

// randomBytes fills byte buffers with random data
func randomBytes(src rand.Source, out chan<- []byte) {
	var (
		r int64
		i = BSIZE
	)

	for buf := range pool {
		r = src.Int63()
		switch { // Go seems to eliminate impossible cases
		case BSIZEMOD == 6:
			buf[BSIZE-6] = byte(r >> 48)
			fallthrough
		case BSIZEMOD == 5:
			buf[BSIZE-5] = byte(r >> 32)
			fallthrough
		case BSIZEMOD == 4:
			buf[BSIZE-4] = byte(r >> 24)
			fallthrough
		case BSIZEMOD == 3:
			buf[BSIZE-3] = byte(r >> 16)
			fallthrough
		case BSIZEMOD == 2:
			buf[BSIZE-2] = byte(r >> 8)
			fallthrough
		case BSIZEMOD == 1:
			buf[BSIZE-1] = byte(r)
		}

		for i = 0; i < BSIZE7; i += 7 {
			r = src.Int63()
			buf[i] = byte(r)
			buf[i+1] = byte(r >> 8)
			buf[i+2] = byte(r >> 16)
			buf[i+3] = byte(r >> 24)
			buf[i+4] = byte(r >> 32)
			buf[i+5] = byte(r >> 40)
			buf[i+6] = byte(r >> 48)
		}

		out <- buf
	}
}

// xor a file with random data
func garble(fin *os.File, fout *os.File, in <-chan []byte, out chan<- bool) {
	var n, m int
	var err error
	data := make([]byte, BSIZE)
	var buf []byte

	for ok := true; ok; {
		// read
		n, err = fin.Read(data)
		for n != BSIZE || err != nil {
			if err != nil && err != io.EOF {
				panic(err)
			}
			if n == 0 {
				goto final
			}
			m, err = fin.Read(data[n:BSIZE])
			if m == 0 && err == io.EOF {
				// last partial block
				ok = false
				break
			}
			n += m
		}

		// xor with random data
		buf = <-in
		C.xor((*C.int64_t)(unsafe.Pointer(&data[0])), (*C.int64_t)(unsafe.Pointer(&buf[0])))
		out <- true // done with buf

		// write
		_, err = fout.Write(data[0:n])
		if err != nil {
			if err2, ok := err.(*os.PathError); ok && err2.Err == syscall.EPIPE {
				goto final
			}

			panic(err)
		}
	}

final:
	close(out)
	for {
		<-in // sleep forever
	}
	return
}

// parse command line arguments
var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")

func init() {
	flag.StringVar(&phrase, "phrase", "", "the Garble phrase, by default random")
	flag.Parse()

	narg = flag.NArg()

	if narg <= 0 {
		flag.Usage()
		os.Exit(1)
	}

	if phrase == "" {
		phrase = fmt.Sprintf("%016x", uint64(randomSeed()))
	}

	fmt.Println("Using phrase:", phrase)
}

// the main program...
func main() {
	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			panic(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	// Use available CPUs:
	if runtime.GOMAXPROCS(0) == 1 &&
		runtime.NumCPU() > 1 &&
		os.Getenv("GOMAXPROCS") == "" {
		runtime.GOMAXPROCS(runtime.NumCPU())
	}

	// Open files:
	writers := make([]chan []byte, narg)
	signals := make([]chan bool, narg)

	for i, arg := range flag.Args() {
		var fd [2]*os.File
		for i, _ := range fd {
			if !stdout && arg == "-" {
				fd[0] = os.Stdin
				fd[1] = os.Stdout
				stdout = true
				break
			}
			f, err := os.OpenFile(arg, os.O_RDWR, 0666)
			if err != nil {
				panic(err)
			}
			defer f.Close()
			fd[i] = f
		}

		writers[i] = make(chan []byte, MULTI)
		signals[i] = make(chan bool, MULTI)
		go garble(fd[0], fd[1], writers[i], signals[i])
	}

	// Allocate byte buffer pool:
	buffer := make([]byte, BSIZE*POOL)

	for i := 0; i < BSIZE*POOL; i += BSIZE {
		pool <- buffer[i : i+BSIZE]
	}

	// Initialize random sources:
	hash := sha512.New()
	sum := make([]byte, hash.Size())

	for i := 0; i < SOURCES; i++ {
		var seed, s int64
		var err error

		hash.Write([]byte(":garble:" + phrase))
		hash.Sum(sum[:0])

		buf := bytes.NewReader(sum)
		s = 0

		for err == nil {
			err = binary.Read(buf, binary.LittleEndian, &s)
			seed ^= s
		}

		src := rand.NewSource(seed)
		data[i] = make(chan []byte, MULTI)
		go randomBytes(src, data[i])
	}

	// Route data channels:
	go func(data []chan []byte, writers []chan []byte, signals []chan bool) {
		var buf []byte
		for {
			for _, r := range data {
				buf = <-r

				for i, w := range writers {
					if signals[i] != nil {
						w <- buf
					}
				}

				loop <- buf
			}
		}
	}(data, writers, signals)

	var buf []byte

	for narg > 0 {
		buf = <-loop

		for i, s := range signals {
			if s != nil {
				_, ok := <-s

				if !ok {
					signals[i] = nil
					narg--
				}
			}
		}

		pool <- buf
	}

	fmt.Println("All done!")
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/infinityworks/prometheus-rancher-exporter/measure"
	"github.com/prometheus/client_golang/prometheus"
)

// Data is used to store data from all the relevant endpoints in the API
type Data struct {
	Data []struct {
		HealthState string `json:"healthState"`
		Name        string `json:"name"`
		State       string `json:"state"`
		System      bool   `json:"system"`
		Scale       int    `json:"scale"`
		HostName    string `json:"hostname"`
		ID          string `json:"id"`
		StackID     string `json:"stackId"`
		EnvID       string `json:"environmentId"`
		BaseType    string `json:"basetype"`
		Type        string `json:"type"`
		AgentState  string `json:"agentState"`
	} `json:"data"`
}

// processMetrics - Collects the data from the API, returns data object
func (e *Exporter) processMetrics(data *Data, endpoint string, hideSys bool, ch chan<- prometheus.Metric) error {

	// Metrics - range through the data object
	for _, x := range data.Data {

		// If system services have been ignored, the loop simply skips them
		if hideSys == true && x.System == true {
			continue
		}

		// Checks the metric is of the expected type
		dataType := x.BaseType
		if dataType == "" {
			dataType = x.Type
		}
		if checkMetric(endpoint, dataType) == false {
			continue
		}

		log.Debug("Processing metrics for %s", endpoint)

		if endpoint == "hosts" {
			var s = x.HostName
			if x.Name != "" {
				s = x.Name
			}
			if err := e.setHostMetrics(s, x.State, x.AgentState); err != nil {
				log.Errorf("Error processing host metrics: %s", err)
				log.Errorf("Attempt Failed to set %s, %s, [agent] %s ", x.HostName, x.State, x.AgentState)

				continue
			}

		} else if endpoint == "stacks" {

			// Used to create a map of stackID and stackName
			// Later used as a dimension in service metrics
			stackRef = storeStackRef(x.ID, x.Name)

			if err := e.setStackMetrics(x.Name, x.State, x.HealthState, strconv.FormatBool(x.System)); err != nil {
				log.Errorf("Error processing stack metrics: %s", err)
				log.Errorf("Attempt Failed to set %s, %s, %s, %t", x.Name, x.State, x.HealthState, x.System)
				continue
			}

		} else if endpoint == "services" {

			// Retrieves the stack Name from the previous values stored.
			var stackName = retrieveStackRef(x.StackID)

			if stackName == "unknown" {
				log.Warnf("Failed to obtain stack_name for %s from the API", x.Name)
			}

			if err := e.setServiceMetrics(x.Name, stackName, x.State, x.HealthState, x.Scale); err != nil {
				log.Errorf("Error processing service metrics: %s", err)
				log.Errorf("Attempt Failed to set %s, %s, %s, %s, %d", x.Name, stackName, x.State, x.HealthState, x.Scale)
				continue
			}

			e.setServiceMetrics(x.Name, stackName, x.State, x.HealthState, x.Scale)
		}

	}

	return nil
}

// gatherData - Collects the data from thw API, invokes functions to transform that data into metrics
func (e *Exporter) gatherData(rancherURL string, accessKey string, secretKey string, endpoint string, ch chan<- prometheus.Metric) (*Data, error) {

	// Return the correct URL path
	url := setEndpoint(rancherURL, endpoint)

	// Create new data slice from Struct
	var data = new(Data)

	// Scrape EndPoint for JSON Data
	err := getJSON(url, accessKey, secretKey, &data)
	if err != nil {
		log.Error("Error getting JSON from endpoint ", endpoint)
		return nil, err
	}
	log.Debugf("JSON Fetched for: "+endpoint+": ", data)

	return data, err
}

// getJSON return json from server, return the formatted JSON
func getJSON(url string, accessKey string, secretKey string, target interface{}) error {

	start := time.Now()

	// Counter for internal exporter metrics
	measure.FunctionCountTotal.With(prometheus.Labels{"pkg": "main", "fnc": "getJSON"}).Inc()

	log.Info("Scraping: ", url)

	client := &http.Client{}
	req, err := http.NewRequest("GET", url, nil)

	if err != nil {
		log.Error("Error Collecting JSON from API: ", err)
	}

	req.SetBasicAuth(accessKey, secretKey)
	resp, err := client.Do(req)

	if err != nil {
		log.Error("Error Collecting JSON from API: ", err)
	}

	respFormatted := json.NewDecoder(resp.Body).Decode(target)

	// Timings recorded as part of internal metrics
	elapsed := float64((time.Since(start)) / time.Microsecond)
	measure.FunctionDurations.WithLabelValues("main", "getJSON").Observe(elapsed)

	// Close the response body, the underlying Transport should then close the connection.
	resp.Body.Close()

	// return formatted JSON
	return respFormatted
}

// setEndpoint - Determines the correct URL endpoint to use, gives us backwards compatibility
func setEndpoint(rancherURL string, component string) string {

	var endpoint string

	endpoint = (rancherURL + "/" + component + "/")
    endpoint = strings.Replace(endpoint, "v1", "v2-beta", 1)

	return endpoint
}

// storeStackRef stores the stackID and stack name for use as a label elsewhere
func storeStackRef(stackID string, stackName string) map[string]string {

	stackRef[stackID] = stackName

	return stackRef
}

// retrieveStackRef returns the stack name, when sending the stackID
func retrieveStackRef(stackID string) string {

	for key, value := range stackRef {
		if stackID == "" {
			return "unknown"
		} else if stackID == key {
			log.Debugf("StackRef - Key is %s, Value is %s StackID is %s", key, value, stackID)
			return value
		}
	}
	// returns unknown if no match was found
	return "unknown"
}
<commit_msg>add check on the HTTP status returned from the API<commit_after>package main

import (
	"encoding/json"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/infinityworks/prometheus-rancher-exporter/measure"
	"github.com/prometheus/client_golang/prometheus"
)

// Data is used to store data from all the relevant endpoints in the API
type Data struct {
	Data []struct {
		HealthState string `json:"healthState"`
		Name        string `json:"name"`
		State       string `json:"state"`
		System      bool   `json:"system"`
		Scale       int    `json:"scale"`
		HostName    string `json:"hostname"`
		ID          string `json:"id"`
		StackID     string `json:"stackId"`
		EnvID       string `json:"environmentId"`
		BaseType    string `json:"basetype"`
		Type        string `json:"type"`
		AgentState  string `json:"agentState"`
	} `json:"data"`
}

// processMetrics - Collects the data from the API, returns data object
func (e *Exporter) processMetrics(data *Data, endpoint string, hideSys bool, ch chan<- prometheus.Metric) error {

	// Metrics - range through the data object
	for _, x := range data.Data {

		// If system services have been ignored, the loop simply skips them
		if hideSys == true && x.System == true {
			continue
		}

		// Checks the metric is of the expected type
		dataType := x.BaseType
		if dataType == "" {
			dataType = x.Type
		}
		if checkMetric(endpoint, dataType) == false {
			continue
		}

		log.Debug("Processing metrics for %s", endpoint)

		if endpoint == "hosts" {
			var s = x.HostName
			if x.Name != "" {
				s = x.Name
			}
			if err := e.setHostMetrics(s, x.State, x.AgentState); err != nil {
				log.Errorf("Error processing host metrics: %s", err)
				log.Errorf("Attempt Failed to set %s, %s, [agent] %s ", x.HostName, x.State, x.AgentState)

				continue
			}

		} else if endpoint == "stacks" {

			// Used to create a map of stackID and stackName
			// Later used as a dimension in service metrics
			stackRef = storeStackRef(x.ID, x.Name)

			if err := e.setStackMetrics(x.Name, x.State, x.HealthState, strconv.FormatBool(x.System)); err != nil {
				log.Errorf("Error processing stack metrics: %s", err)
				log.Errorf("Attempt Failed to set %s, %s, %s, %t", x.Name, x.State, x.HealthState, x.System)
				continue
			}

		} else if endpoint == "services" {

			// Retrieves the stack Name from the previous values stored.
			var stackName = retrieveStackRef(x.StackID)

			if stackName == "unknown" {
				log.Warnf("Failed to obtain stack_name for %s from the API", x.Name)
			}

			if err := e.setServiceMetrics(x.Name, stackName, x.State, x.HealthState, x.Scale); err != nil {
				log.Errorf("Error processing service metrics: %s", err)
				log.Errorf("Attempt Failed to set %s, %s, %s, %s, %d", x.Name, stackName, x.State, x.HealthState, x.Scale)
				continue
			}

			e.setServiceMetrics(x.Name, stackName, x.State, x.HealthState, x.Scale)
		}

	}

	return nil
}

// gatherData - Collects the data from thw API, invokes functions to transform that data into metrics
func (e *Exporter) gatherData(rancherURL string, accessKey string, secretKey string, endpoint string, ch chan<- prometheus.Metric) (*Data, error) {

	// Return the correct URL path
	url := setEndpoint(rancherURL, endpoint)

	// Create new data slice from Struct
	var data = new(Data)

	// Scrape EndPoint for JSON Data
	err := getJSON(url, accessKey, secretKey, &data)
	if err != nil {
		log.Error("Error getting JSON from endpoint ", endpoint)
		return nil, err
	}
	log.Debugf("JSON Fetched for: "+endpoint+": ", data)

	return data, err
}

// getJSON return json from server, return the formatted JSON
func getJSON(url string, accessKey string, secretKey string, target interface{}) error {

	start := time.Now()

	// Counter for internal exporter metrics
	measure.FunctionCountTotal.With(prometheus.Labels{"pkg": "main", "fnc": "getJSON"}).Inc()

	log.Info("Scraping: ", url)

	client := &http.Client{}
	req, err := http.NewRequest("GET", url, nil)

	if err != nil {
		log.Error("Error Collecting JSON from API: ", err)
	}

	req.SetBasicAuth(accessKey, secretKey)
	resp, err := client.Do(req)

	if err != nil {
		log.Error("Error Collecting JSON from API: ", err)
	}

	if resp.Status != "200" {
		log.Error("Error returned from API: ",resp.Status)
	}

	respFormatted := json.NewDecoder(resp.Body).Decode(target)

	// Timings recorded as part of internal metrics
	elapsed := float64((time.Since(start)) / time.Microsecond)
	measure.FunctionDurations.WithLabelValues("main", "getJSON").Observe(elapsed)

	// Close the response body, the underlying Transport should then close the connection.
	resp.Body.Close()

	// return formatted JSON
	return respFormatted
}

// setEndpoint - Determines the correct URL endpoint to use, gives us backwards compatibility
func setEndpoint(rancherURL string, component string) string {

	var endpoint string

	endpoint = (rancherURL + "/" + component + "/")
    endpoint = strings.Replace(endpoint, "v1", "v2-beta", 1)

	return endpoint
}

// storeStackRef stores the stackID and stack name for use as a label elsewhere
func storeStackRef(stackID string, stackName string) map[string]string {

	stackRef[stackID] = stackName

	return stackRef
}

// retrieveStackRef returns the stack name, when sending the stackID
func retrieveStackRef(stackID string) string {

	for key, value := range stackRef {
		if stackID == "" {
			return "unknown"
		} else if stackID == key {
			log.Debugf("StackRef - Key is %s, Value is %s StackID is %s", key, value, stackID)
			return value
		}
	}
	// returns unknown if no match was found
	return "unknown"
}
<|endoftext|>


<commit_before>package statsd

import (
	"fmt"
	"log"
	"math/rand"
	"net"
	"strings"
	"sync"
	"time"
)

const metricTypeCount = "c"
const metricTypeGauge = "g"
const metricTypeTiming = "t"
const metricTypeSet = "s"

// The Client type
type Client struct {
	host          string
	port          int
	conn          net.Conn          // UDP connection to StatsD server
	rand          *rand.Rand        // rand generator to skip messages by sample rate
	keyBuffer     map[string]string // array of messages to send
	keyBufferLock sync.RWMutex      // mutex to lock buffer of keys
	autoflush     bool              // send metrics on every call
}

// NewClient creates new StatsD client
func NewClient(host string, port int) *Client {
	client := Client{
		host:      host,
		port:      port,
		rand:      rand.New(rand.NewSource(time.Now().Unix())),
		keyBuffer: make(map[string]string),
		autoflush: false,
	}
	return &client
}

// Open UDP connection to statsd server
func (client *Client) Open() {
	connectionString := fmt.Sprintf("%s:%d", client.host, client.port)
	conn, err := net.Dial("udp", connectionString)
	if err != nil {
		log.Println(err)
	}
	client.conn = conn
}

// Close UDP connection to statsd server
func (client *Client) Close() {
	client.conn.Close()
}

// SetAutoflush enables/disables buffered mode
// In buffered mode requires manual call of Flush()
// In autoflush mode message sends to server on every call
func (client *Client) SetAutoflush(autoflush bool) {
	client.autoflush = autoflush
}

// Timing track in milliseconds with sampling
func (client *Client) Timing(key string, time int64, sampleRate float32) {
	metricValue := fmt.Sprintf("%d|%s", time, metricTypeTiming)
	if sampleRate < 1 {
		if client.isSendAcceptedBySampleRate(sampleRate) {
			metricValue = fmt.Sprintf("%s|@%f", metricValue, sampleRate)
		} else {
			return
		}
	}

	client.addToBuffer(key, metricValue)
}

// Count tack
func (client *Client) Count(key string, delta int, sampleRate float32) {
	metricValue := fmt.Sprintf("%d|%s", delta, metricTypeCount)
	if sampleRate < 1 {
		if client.isSendAcceptedBySampleRate(sampleRate) {
			metricValue = fmt.Sprintf("%s|@%f", metricValue, sampleRate)
		} else {
			return
		}
	}

	client.addToBuffer(key, metricValue)
}

// Gauge track
func (client *Client) Gauge(key string, value int) {
	metricValue := fmt.Sprintf("%d|%s", value, metricTypeGauge)
	client.addToBuffer(key, metricValue)
}

// Set tracking
func (client *Client) Set(key string, value int) {
	metricValue := fmt.Sprintf("%d|%s", value, metricTypeSet)
	client.addToBuffer(key, metricValue)
}

// add to buffer and flush if auto flush enabled
func (client *Client) addToBuffer(key string, metricValue string) {
	// flush
	if client.autoflush {
		// send metric now
		client.send(fmt.Sprintf("%s:%s", key, metricValue))
	} else {
		// add metric to buffer for next manual flush
		client.keyBufferLock.Lock()
		client.keyBuffer[key] = metricValue
		client.keyBufferLock.Unlock()

	}
}

// Check if acceptable by sample rate
func (client *Client) isSendAcceptedBySampleRate(sampleRate float32) bool {
	if sampleRate >= 1 {
		return true
	}
	randomNumber := client.rand.Float32()
	return randomNumber <= sampleRate
}

// flush buffer to statsd daemon by UDP
func (client *Client) Flush() {
	// check if buffer has metrics
	if len(client.keyBuffer) == 0 {
		return
	}

	// prepare metric packet
	var metricPacketArray []string

	// lock
	client.keyBufferLock.Lock()

	// build packet
	for key, metricValue := range client.keyBuffer {
		metricPacketArray = append(metricPacketArray, fmt.Sprintf("%s:%s", key, metricValue))
	}
	metricPacket := strings.Join(metricPacketArray, "\n")

	// clear key buffer
	client.keyBuffer = make(map[string]string)

	// lock
	client.keyBufferLock.Unlock()

	// send packet
	client.send(metricPacket)
}

// Send StatsD packet
func (client *Client) send(metricPacket string) {
	// send metric packet
	_, err := fmt.Fprintf(client.conn, metricPacket)
	if err != nil {
		log.Println(err)
	}
}
<commit_msg>fix php doc<commit_after>package statsd

import (
	"fmt"
	"log"
	"math/rand"
	"net"
	"strings"
	"sync"
	"time"
)

const metricTypeCount = "c"
const metricTypeGauge = "g"
const metricTypeTiming = "t"
const metricTypeSet = "s"

// The Client type
type Client struct {
	host          string
	port          int
	conn          net.Conn          // UDP connection to StatsD server
	rand          *rand.Rand        // rand generator to skip messages by sample rate
	keyBuffer     map[string]string // array of messages to send
	keyBufferLock sync.RWMutex      // mutex to lock buffer of keys
	autoflush     bool              // send metrics on every call
}

// NewClient creates new StatsD client
func NewClient(host string, port int) *Client {
	client := Client{
		host:      host,
		port:      port,
		rand:      rand.New(rand.NewSource(time.Now().Unix())),
		keyBuffer: make(map[string]string),
		autoflush: false,
	}
	return &client
}

// Open UDP connection to statsd server
func (client *Client) Open() {
	connectionString := fmt.Sprintf("%s:%d", client.host, client.port)
	conn, err := net.Dial("udp", connectionString)
	if err != nil {
		log.Println(err)
	}
	client.conn = conn
}

// Close UDP connection to statsd server
func (client *Client) Close() {
	client.conn.Close()
}

// SetAutoflush enables/disables buffered mode
// In buffered mode requires manual call of Flush()
// In autoflush mode message sends to server on every call
func (client *Client) SetAutoflush(autoflush bool) {
	client.autoflush = autoflush
}

// Timing track in milliseconds with sampling
func (client *Client) Timing(key string, time int64, sampleRate float32) {
	metricValue := fmt.Sprintf("%d|%s", time, metricTypeTiming)
	if sampleRate < 1 {
		if client.isSendAcceptedBySampleRate(sampleRate) {
			metricValue = fmt.Sprintf("%s|@%f", metricValue, sampleRate)
		} else {
			return
		}
	}

	client.addToBuffer(key, metricValue)
}

// Count tack
func (client *Client) Count(key string, delta int, sampleRate float32) {
	metricValue := fmt.Sprintf("%d|%s", delta, metricTypeCount)
	if sampleRate < 1 {
		if client.isSendAcceptedBySampleRate(sampleRate) {
			metricValue = fmt.Sprintf("%s|@%f", metricValue, sampleRate)
		} else {
			return
		}
	}

	client.addToBuffer(key, metricValue)
}

// Gauge track
func (client *Client) Gauge(key string, value int) {
	metricValue := fmt.Sprintf("%d|%s", value, metricTypeGauge)
	client.addToBuffer(key, metricValue)
}

// Set tracking
func (client *Client) Set(key string, value int) {
	metricValue := fmt.Sprintf("%d|%s", value, metricTypeSet)
	client.addToBuffer(key, metricValue)
}

// add to buffer and flush if auto flush enabled
func (client *Client) addToBuffer(key string, metricValue string) {
	// flush
	if client.autoflush {
		// send metric now
		client.send(fmt.Sprintf("%s:%s", key, metricValue))
	} else {
		// add metric to buffer for next manual flush
		client.keyBufferLock.Lock()
		client.keyBuffer[key] = metricValue
		client.keyBufferLock.Unlock()

	}
}

// Check if acceptable by sample rate
func (client *Client) isSendAcceptedBySampleRate(sampleRate float32) bool {
	if sampleRate >= 1 {
		return true
	}
	randomNumber := client.rand.Float32()
	return randomNumber <= sampleRate
}

// Flush buffer to statsd daemon by UDP
func (client *Client) Flush() {
	// check if buffer has metrics
	if len(client.keyBuffer) == 0 {
		return
	}

	// prepare metric packet
	var metricPacketArray []string

	// lock
	client.keyBufferLock.Lock()

	// build packet
	for key, metricValue := range client.keyBuffer {
		metricPacketArray = append(metricPacketArray, fmt.Sprintf("%s:%s", key, metricValue))
	}
	metricPacket := strings.Join(metricPacketArray, "\n")

	// clear key buffer
	client.keyBuffer = make(map[string]string)

	// lock
	client.keyBufferLock.Unlock()

	// send packet
	client.send(metricPacket)
}

// Send StatsD packet
func (client *Client) send(metricPacket string) {
	// send metric packet
	_, err := fmt.Fprintf(client.conn, metricPacket)
	if err != nil {
		log.Println(err)
	}
}
<|endoftext|>


<commit_before>// generated by stringer -type=Status; DO NOT EDIT

package main

import "fmt"

const _Status_name = "SuccessWarningDangerInfo"

var _Status_index = [...]uint8{7, 14, 20, 24}

func (i Status) String() string {
	if i < 0 || i >= Status(len(_Status_index)) {
		return fmt.Sprintf("Status(%d)", i)
	}
	hi := _Status_index[i]
	lo := uint8(0)
	if i > 0 {
		lo = _Status_index[i-1]
	}
	return _Status_name[lo:hi]
}
<commit_msg>update by latest stringer generated code<commit_after>// generated by stringer -type=Status; DO NOT EDIT

package main

import "fmt"

const _Status_name = "SuccessWarningDangerInfo"

var _Status_index = [...]uint8{0, 7, 14, 20, 24}

func (i Status) String() string {
	if i < 0 || i >= Status(len(_Status_index)-1) {
		return fmt.Sprintf("Status(%d)", i)
	}
	return _Status_name[_Status_index[i]:_Status_index[i+1]]
}
<|endoftext|>