<commit_before>package git

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"sync"

	"context"
	"time"
)

const (
	defaultInterval = 5 * time.Minute
	defaultTimeout  = 20 * time.Second

	CheckPushTag = "flux-write-check"
)

var (
	ErrNoChanges  = errors.New("no changes made in repo")
	ErrNoConfig   = errors.New("git repo does not have valid config")
	ErrNotCloned  = errors.New("git repo has not been cloned yet")
	ErrClonedOnly = errors.New("git repo has been cloned but not yet checked for write access")
)

type NotReadyError struct {
	underlying error
}

func (err NotReadyError) Error() string {
	return "git repo not ready: " + err.underlying.Error()
}

// GitRepoStatus represents the progress made synchronising with a git
// repo. These are given below in expected order, but the status may
// go backwards if e.g., a deploy key is deleted.
type GitRepoStatus string

const (
	RepoNoConfig GitRepoStatus = "unconfigured" // configuration is empty
	RepoNew      GitRepoStatus = "new"          // no attempt made to clone it yet
	RepoCloned   GitRepoStatus = "cloned"       // has been read (cloned); no attempt made to write
	RepoReady    GitRepoStatus = "ready"        // has been written to, so ready to sync
)

type Repo struct {
	// As supplied to constructor
	origin   Remote
	branch   string
	interval time.Duration
	timeout  time.Duration
	readonly bool

	// State
	mu     sync.RWMutex
	status GitRepoStatus
	err    error
	dir    string

	notify chan struct{}
	C      chan struct{}
}

type Option interface {
	apply(*Repo)
}

type optionFunc func(*Repo)

func (f optionFunc) apply(r *Repo) {
	f(r)
}

type PollInterval time.Duration

func (p PollInterval) apply(r *Repo) {
	r.interval = time.Duration(p)
}

type Timeout time.Duration

func (t Timeout) apply(r *Repo) {
	r.timeout = time.Duration(t)
}

type Branch string

func (b Branch) apply(r *Repo) {
	r.branch = string(b)
}

type IsReadOnly bool

func (ro IsReadOnly) apply(r *Repo) {
	r.readonly = bool(ro)
}

var ReadOnly IsReadOnly = true

// NewRepo constructs a repo mirror which will sync itself.
func NewRepo(origin Remote, opts ...Option) *Repo {
	status := RepoNew
	if origin.URL == "" {
		status = RepoNoConfig
	}
	r := &Repo{
		origin:   origin,
		status:   status,
		interval: defaultInterval,
		timeout:  defaultTimeout,
		err:      ErrNotCloned,
		notify:   make(chan struct{}, 1), // `1` so that Notify doesn't block
		C:        make(chan struct{}, 1), // `1` so we don't block on completing a refresh
	}
	for _, opt := range opts {
		opt.apply(r)
	}
	return r
}

// Origin returns the Remote with which the Repo was constructed.
func (r *Repo) Origin() Remote {
	r.mu.RLock()
	defer r.mu.RUnlock()
	return r.origin
}

// Readonly returns `true` if the repo was marked as readonly, `false`
// otherwise
func (r *Repo) Readonly() bool {
	return r.readonly
}

// Dir returns the local directory into which the repo has been
// cloned, if it has been cloned.
func (r *Repo) Dir() string {
	r.mu.RLock()
	defer r.mu.RUnlock()
	return r.dir
}

// Clean removes the mirrored repo. Syncing may continue with a new
// directory, so you may need to stop that first.
func (r *Repo) Clean() {
	r.mu.Lock()
	if r.dir != "" {
		os.RemoveAll(r.dir)
	}
	r.dir = ""
	r.status = RepoNew
	r.mu.Unlock()
}

// Status reports that readiness status of this Git repo: whether it
// has been cloned and is writable, and if not, the error stopping it
// getting to the next state.
func (r *Repo) Status() (GitRepoStatus, error) {
	r.mu.RLock()
	defer r.mu.RUnlock()
	return r.status, r.err
}

func (r *Repo) setUnready(s GitRepoStatus, err error) {
	r.mu.Lock()
	r.status = s
	r.err = err
	r.mu.Unlock()
}

func (r *Repo) setReady() {
	r.mu.Lock()
	r.status = RepoReady
	r.err = nil
	r.mu.Unlock()
}

// Notify tells the repo that it should fetch from the origin as soon
// as possible. It does not block.
func (r *Repo) Notify() {
	select {
	case r.notify <- struct{}{}:
		// duly notified
	default:
		// notification already pending
	}
}

// refreshed indicates that the repo has successfully fetched from upstream.
func (r *Repo) refreshed() {
	select {
	case r.C <- struct{}{}:
	default:
	}
}

// errorIfNotReady returns the appropriate error if the repo is not
// ready, and `nil` otherwise.
func (r *Repo) errorIfNotReady() error {
	switch r.status {
	case RepoReady:
		return nil
	case RepoNoConfig:
		return ErrNoConfig
	default:
		return NotReadyError{r.err}
	}
}

// Revision returns the revision (SHA1) of the ref passed in
func (r *Repo) Revision(ctx context.Context, ref string) (string, error) {
	r.mu.RLock()
	defer r.mu.RUnlock()
	if err := r.errorIfNotReady(); err != nil {
		return "", err
	}
	return refRevision(ctx, r.dir, ref)
}

// BranchHead returns the HEAD revision (SHA1) of the configured branch
func (r *Repo) BranchHead(ctx context.Context) (string, error) {
	r.mu.RLock()
	defer r.mu.RUnlock()
	if err := r.errorIfNotReady(); err != nil {
		return "", err
	}
	return refRevision(ctx, r.dir, "heads/"+r.branch)
}

func (r *Repo) CommitsBefore(ctx context.Context, ref string, paths ...string) ([]Commit, error) {
	r.mu.RLock()
	defer r.mu.RUnlock()
	if err := r.errorIfNotReady(); err != nil {
		return nil, err
	}
	return onelinelog(ctx, r.dir, ref, paths)
}

func (r *Repo) CommitsBetween(ctx context.Context, ref1, ref2 string, paths ...string) ([]Commit, error) {
	r.mu.RLock()
	defer r.mu.RUnlock()
	if err := r.errorIfNotReady(); err != nil {
		return nil, err
	}
	return onelinelog(ctx, r.dir, ref1+".."+ref2, paths)
}

func (r *Repo) VerifyTag(ctx context.Context, tag string) (string, error) {
	r.mu.RLock()
	defer r.mu.RUnlock()
	if err := r.errorIfNotReady(); err != nil {
		return "", err
	}
	return verifyTag(ctx, r.dir, tag)
}

func (r *Repo) VerifyCommit(ctx context.Context, commit string) error {
	r.mu.RLock()
	defer r.mu.RUnlock()
	if err := r.errorIfNotReady(); err != nil {
		return err
	}
	return verifyCommit(ctx, r.dir, commit)
}

func (r *Repo) DeleteTag(ctx context.Context, tag string) error {
	r.mu.RLock()
	defer r.mu.RUnlock()
	if err := r.errorIfNotReady(); err != nil {
		return err
	}
	return deleteTag(ctx, r.dir, tag, r.origin.URL)
}

func (r *Repo) NoteRevList(ctx context.Context, notesRef string) (map[string]struct{}, error) {
	return noteRevList(ctx, r.Dir(), notesRef)
}

// GetNote gets a note for the revision specified, or nil if there is no such note.
func (r *Repo) GetNote(ctx context.Context, rev, notesRef string, note interface{}) (bool, error) {
	return getNote(ctx, r.Dir(), notesRef, rev, note)
}

// step attempts to advance the repo state machine, and returns `true`
// if it has made progress, `false` otherwise.
func (r *Repo) step(bg context.Context) bool {
	r.mu.RLock()
	url := r.origin.URL
	dir := r.dir
	status := r.status
	r.mu.RUnlock()

	switch status {

	case RepoNoConfig:
		// this is not going to change in the lifetime of this
		// process, so just exit.
		return false

	case RepoNew:
		rootdir, err := ioutil.TempDir(os.TempDir(), "flux-gitclone")
		if err != nil {
			panic(err)
		}

		ctx, cancel := context.WithTimeout(bg, r.timeout)
		dir, err = mirror(ctx, rootdir, url)
		cancel()
		if err == nil {
			r.mu.Lock()
			r.dir = dir
			ctx, cancel := context.WithTimeout(bg, r.timeout)
			err = r.fetch(ctx)
			cancel()
			r.mu.Unlock()
		}
		if err == nil {
			r.setUnready(RepoCloned, ErrClonedOnly)
			return true
		}
		dir = ""
		os.RemoveAll(rootdir)
		r.setUnready(RepoNew, err)
		return false

	case RepoCloned:
		ctx, cancel := context.WithTimeout(bg, r.timeout)
		defer cancel()

		if r.branch != "" {
			r.mu.Lock()
			// The remote may have changed between `RepoNew` and this
			// iteration of `RepoCloned`. Fetch again to pick-up any
			// changes that may have been made.
			err := r.fetch(ctx)
			r.mu.Unlock()
			if err != nil {
				r.setUnready(RepoCloned, err)
				return false
			}

			ok, err := refExists(ctx, dir, "refs/heads/"+r.branch)
			if err != nil {
				r.setUnready(RepoCloned, err)
				return false
			}
			if !ok {
				r.setUnready(RepoCloned, fmt.Errorf("configured branch '%s' does not exist", r.branch))
				return false
			}
		}

		if !r.readonly {
			err := checkPush(ctx, dir, url, r.branch)
			if err != nil {
				r.setUnready(RepoCloned, err)
				return false
			}
		}

		r.setReady()
		// Treat every transition to ready as a refresh, so
		// that any listeners can respond in the same way.
		r.refreshed()
		return true

	case RepoReady:
		return false
	}

	return false
}

// Ready tries to advance the cloning process along as far as
// possible, and returns an error if it is not able to get to a ready
// state.
func (r *Repo) Ready(ctx context.Context) error {
	for r.step(ctx) {
		// keep going!
	}
	_, err := r.Status()
	return err
}

// Start begins synchronising the repo by cloning it, then fetching
// the required tags and so on.
func (r *Repo) Start(shutdown <-chan struct{}, done *sync.WaitGroup) error {
	defer done.Done()

	for {
		ctx, cancel := context.WithTimeout(context.Background(), r.timeout)
		advanced := r.step(ctx)
		cancel()

		if advanced {
			continue
		}

		status, _ := r.Status()
		if status == RepoReady {
			if err := r.refreshLoop(shutdown); err != nil {
				r.setUnready(RepoNew, err)
				continue // with new status, skipping timer
			}
		} else if status == RepoNoConfig {
			return nil
		}

		tryAgain := time.NewTimer(10 * time.Second)
		select {
		case <-shutdown:
			if !tryAgain.Stop() {
				<-tryAgain.C
			}
			return nil
		case <-tryAgain.C:
			continue
		}
	}
	return nil
}

func (r *Repo) Refresh(ctx context.Context) error {
	// the lock here and below is difficult to avoid; possibly we
	// could clone to another repo and pull there, then swap when complete.
	r.mu.Lock()
	defer r.mu.Unlock()
	if err := r.errorIfNotReady(); err != nil {
		return err
	}
	if err := r.fetch(ctx); err != nil {
		return err
	}
	r.refreshed()
	return nil
}

func (r *Repo) refreshLoop(shutdown <-chan struct{}) error {
	gitPoll := time.NewTimer(r.interval)
	for {
		select {
		case <-shutdown:
			if !gitPoll.Stop() {
				<-gitPoll.C
			}
			return nil
		case <-gitPoll.C:
			r.Notify()
		case <-r.notify:
			if !gitPoll.Stop() {
				select {
				case <-gitPoll.C:
				default:
				}
			}
			ctx, cancel := context.WithTimeout(context.Background(), r.timeout)
			err := r.Refresh(ctx)
			cancel()
			if err != nil {
				return err
			}
			gitPoll.Reset(r.interval)
		}
	}
}

// fetch gets updated refs, and associated objects, from the upstream.
func (r *Repo) fetch(ctx context.Context) error {
	if err := fetch(ctx, r.dir, "origin"); err != nil {
		return err
	}
	return nil
}

// workingClone makes a non-bare clone, at `ref` (probably a branch),
// and returns the filesystem path to it.
func (r *Repo) workingClone(ctx context.Context, ref string) (string, error) {
	r.mu.RLock()
	defer r.mu.RUnlock()
	if err := r.errorIfNotReady(); err != nil {
		return "", err
	}
	working, err := ioutil.TempDir(os.TempDir(), "flux-working")
	if err != nil {
		return "", err
	}
	return clone(ctx, working, r.dir, ref)
}
<commit_msg>Remove redundant return code<commit_after>package git

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"sync"

	"context"
	"time"
)

const (
	defaultInterval = 5 * time.Minute
	defaultTimeout  = 20 * time.Second

	CheckPushTag = "flux-write-check"
)

var (
	ErrNoChanges  = errors.New("no changes made in repo")
	ErrNoConfig   = errors.New("git repo does not have valid config")
	ErrNotCloned  = errors.New("git repo has not been cloned yet")
	ErrClonedOnly = errors.New("git repo has been cloned but not yet checked for write access")
)

type NotReadyError struct {
	underlying error
}

func (err NotReadyError) Error() string {
	return "git repo not ready: " + err.underlying.Error()
}

// GitRepoStatus represents the progress made synchronising with a git
// repo. These are given below in expected order, but the status may
// go backwards if e.g., a deploy key is deleted.
type GitRepoStatus string

const (
	RepoNoConfig GitRepoStatus = "unconfigured" // configuration is empty
	RepoNew      GitRepoStatus = "new"          // no attempt made to clone it yet
	RepoCloned   GitRepoStatus = "cloned"       // has been read (cloned); no attempt made to write
	RepoReady    GitRepoStatus = "ready"        // has been written to, so ready to sync
)

type Repo struct {
	// As supplied to constructor
	origin   Remote
	branch   string
	interval time.Duration
	timeout  time.Duration
	readonly bool

	// State
	mu     sync.RWMutex
	status GitRepoStatus
	err    error
	dir    string

	notify chan struct{}
	C      chan struct{}
}

type Option interface {
	apply(*Repo)
}

type optionFunc func(*Repo)

func (f optionFunc) apply(r *Repo) {
	f(r)
}

type PollInterval time.Duration

func (p PollInterval) apply(r *Repo) {
	r.interval = time.Duration(p)
}

type Timeout time.Duration

func (t Timeout) apply(r *Repo) {
	r.timeout = time.Duration(t)
}

type Branch string

func (b Branch) apply(r *Repo) {
	r.branch = string(b)
}

type IsReadOnly bool

func (ro IsReadOnly) apply(r *Repo) {
	r.readonly = bool(ro)
}

var ReadOnly IsReadOnly = true

// NewRepo constructs a repo mirror which will sync itself.
func NewRepo(origin Remote, opts ...Option) *Repo {
	status := RepoNew
	if origin.URL == "" {
		status = RepoNoConfig
	}
	r := &Repo{
		origin:   origin,
		status:   status,
		interval: defaultInterval,
		timeout:  defaultTimeout,
		err:      ErrNotCloned,
		notify:   make(chan struct{}, 1), // `1` so that Notify doesn't block
		C:        make(chan struct{}, 1), // `1` so we don't block on completing a refresh
	}
	for _, opt := range opts {
		opt.apply(r)
	}
	return r
}

// Origin returns the Remote with which the Repo was constructed.
func (r *Repo) Origin() Remote {
	r.mu.RLock()
	defer r.mu.RUnlock()
	return r.origin
}

// Readonly returns `true` if the repo was marked as readonly, `false`
// otherwise
func (r *Repo) Readonly() bool {
	return r.readonly
}

// Dir returns the local directory into which the repo has been
// cloned, if it has been cloned.
func (r *Repo) Dir() string {
	r.mu.RLock()
	defer r.mu.RUnlock()
	return r.dir
}

// Clean removes the mirrored repo. Syncing may continue with a new
// directory, so you may need to stop that first.
func (r *Repo) Clean() {
	r.mu.Lock()
	if r.dir != "" {
		os.RemoveAll(r.dir)
	}
	r.dir = ""
	r.status = RepoNew
	r.mu.Unlock()
}

// Status reports that readiness status of this Git repo: whether it
// has been cloned and is writable, and if not, the error stopping it
// getting to the next state.
func (r *Repo) Status() (GitRepoStatus, error) {
	r.mu.RLock()
	defer r.mu.RUnlock()
	return r.status, r.err
}

func (r *Repo) setUnready(s GitRepoStatus, err error) {
	r.mu.Lock()
	r.status = s
	r.err = err
	r.mu.Unlock()
}

func (r *Repo) setReady() {
	r.mu.Lock()
	r.status = RepoReady
	r.err = nil
	r.mu.Unlock()
}

// Notify tells the repo that it should fetch from the origin as soon
// as possible. It does not block.
func (r *Repo) Notify() {
	select {
	case r.notify <- struct{}{}:
		// duly notified
	default:
		// notification already pending
	}
}

// refreshed indicates that the repo has successfully fetched from upstream.
func (r *Repo) refreshed() {
	select {
	case r.C <- struct{}{}:
	default:
	}
}

// errorIfNotReady returns the appropriate error if the repo is not
// ready, and `nil` otherwise.
func (r *Repo) errorIfNotReady() error {
	switch r.status {
	case RepoReady:
		return nil
	case RepoNoConfig:
		return ErrNoConfig
	default:
		return NotReadyError{r.err}
	}
}

// Revision returns the revision (SHA1) of the ref passed in
func (r *Repo) Revision(ctx context.Context, ref string) (string, error) {
	r.mu.RLock()
	defer r.mu.RUnlock()
	if err := r.errorIfNotReady(); err != nil {
		return "", err
	}
	return refRevision(ctx, r.dir, ref)
}

// BranchHead returns the HEAD revision (SHA1) of the configured branch
func (r *Repo) BranchHead(ctx context.Context) (string, error) {
	r.mu.RLock()
	defer r.mu.RUnlock()
	if err := r.errorIfNotReady(); err != nil {
		return "", err
	}
	return refRevision(ctx, r.dir, "heads/"+r.branch)
}

func (r *Repo) CommitsBefore(ctx context.Context, ref string, paths ...string) ([]Commit, error) {
	r.mu.RLock()
	defer r.mu.RUnlock()
	if err := r.errorIfNotReady(); err != nil {
		return nil, err
	}
	return onelinelog(ctx, r.dir, ref, paths)
}

func (r *Repo) CommitsBetween(ctx context.Context, ref1, ref2 string, paths ...string) ([]Commit, error) {
	r.mu.RLock()
	defer r.mu.RUnlock()
	if err := r.errorIfNotReady(); err != nil {
		return nil, err
	}
	return onelinelog(ctx, r.dir, ref1+".."+ref2, paths)
}

func (r *Repo) VerifyTag(ctx context.Context, tag string) (string, error) {
	r.mu.RLock()
	defer r.mu.RUnlock()
	if err := r.errorIfNotReady(); err != nil {
		return "", err
	}
	return verifyTag(ctx, r.dir, tag)
}

func (r *Repo) VerifyCommit(ctx context.Context, commit string) error {
	r.mu.RLock()
	defer r.mu.RUnlock()
	if err := r.errorIfNotReady(); err != nil {
		return err
	}
	return verifyCommit(ctx, r.dir, commit)
}

func (r *Repo) DeleteTag(ctx context.Context, tag string) error {
	r.mu.RLock()
	defer r.mu.RUnlock()
	if err := r.errorIfNotReady(); err != nil {
		return err
	}
	return deleteTag(ctx, r.dir, tag, r.origin.URL)
}

func (r *Repo) NoteRevList(ctx context.Context, notesRef string) (map[string]struct{}, error) {
	return noteRevList(ctx, r.Dir(), notesRef)
}

// GetNote gets a note for the revision specified, or nil if there is no such note.
func (r *Repo) GetNote(ctx context.Context, rev, notesRef string, note interface{}) (bool, error) {
	return getNote(ctx, r.Dir(), notesRef, rev, note)
}

// step attempts to advance the repo state machine, and returns `true`
// if it has made progress, `false` otherwise.
func (r *Repo) step(bg context.Context) bool {
	r.mu.RLock()
	url := r.origin.URL
	dir := r.dir
	status := r.status
	r.mu.RUnlock()

	switch status {

	case RepoNoConfig:
		// this is not going to change in the lifetime of this
		// process, so just exit.
		return false

	case RepoNew:
		rootdir, err := ioutil.TempDir(os.TempDir(), "flux-gitclone")
		if err != nil {
			panic(err)
		}

		ctx, cancel := context.WithTimeout(bg, r.timeout)
		dir, err = mirror(ctx, rootdir, url)
		cancel()
		if err == nil {
			r.mu.Lock()
			r.dir = dir
			ctx, cancel := context.WithTimeout(bg, r.timeout)
			err = r.fetch(ctx)
			cancel()
			r.mu.Unlock()
		}
		if err == nil {
			r.setUnready(RepoCloned, ErrClonedOnly)
			return true
		}
		dir = ""
		os.RemoveAll(rootdir)
		r.setUnready(RepoNew, err)
		return false

	case RepoCloned:
		ctx, cancel := context.WithTimeout(bg, r.timeout)
		defer cancel()

		if r.branch != "" {
			r.mu.Lock()
			// The remote may have changed between `RepoNew` and this
			// iteration of `RepoCloned`. Fetch again to pick-up any
			// changes that may have been made.
			err := r.fetch(ctx)
			r.mu.Unlock()
			if err != nil {
				r.setUnready(RepoCloned, err)
				return false
			}

			ok, err := refExists(ctx, dir, "refs/heads/"+r.branch)
			if err != nil {
				r.setUnready(RepoCloned, err)
				return false
			}
			if !ok {
				r.setUnready(RepoCloned, fmt.Errorf("configured branch '%s' does not exist", r.branch))
				return false
			}
		}

		if !r.readonly {
			err := checkPush(ctx, dir, url, r.branch)
			if err != nil {
				r.setUnready(RepoCloned, err)
				return false
			}
		}

		r.setReady()
		// Treat every transition to ready as a refresh, so
		// that any listeners can respond in the same way.
		r.refreshed()
		return true

	case RepoReady:
		return false
	}

	return false
}

// Ready tries to advance the cloning process along as far as
// possible, and returns an error if it is not able to get to a ready
// state.
func (r *Repo) Ready(ctx context.Context) error {
	for r.step(ctx) {
		// keep going!
	}
	_, err := r.Status()
	return err
}

// Start begins synchronising the repo by cloning it, then fetching
// the required tags and so on.
func (r *Repo) Start(shutdown <-chan struct{}, done *sync.WaitGroup) error {
	defer done.Done()

	for {
		ctx, cancel := context.WithTimeout(context.Background(), r.timeout)
		advanced := r.step(ctx)
		cancel()

		if advanced {
			continue
		}

		status, _ := r.Status()
		if status == RepoReady {
			if err := r.refreshLoop(shutdown); err != nil {
				r.setUnready(RepoNew, err)
				continue // with new status, skipping timer
			}
		} else if status == RepoNoConfig {
			return nil
		}

		tryAgain := time.NewTimer(10 * time.Second)
		select {
		case <-shutdown:
			if !tryAgain.Stop() {
				<-tryAgain.C
			}
			return nil
		case <-tryAgain.C:
			continue
		}
	}
}

func (r *Repo) Refresh(ctx context.Context) error {
	// the lock here and below is difficult to avoid; possibly we
	// could clone to another repo and pull there, then swap when complete.
	r.mu.Lock()
	defer r.mu.Unlock()
	if err := r.errorIfNotReady(); err != nil {
		return err
	}
	if err := r.fetch(ctx); err != nil {
		return err
	}
	r.refreshed()
	return nil
}

func (r *Repo) refreshLoop(shutdown <-chan struct{}) error {
	gitPoll := time.NewTimer(r.interval)
	for {
		select {
		case <-shutdown:
			if !gitPoll.Stop() {
				<-gitPoll.C
			}
			return nil
		case <-gitPoll.C:
			r.Notify()
		case <-r.notify:
			if !gitPoll.Stop() {
				select {
				case <-gitPoll.C:
				default:
				}
			}
			ctx, cancel := context.WithTimeout(context.Background(), r.timeout)
			err := r.Refresh(ctx)
			cancel()
			if err != nil {
				return err
			}
			gitPoll.Reset(r.interval)
		}
	}
}

// fetch gets updated refs, and associated objects, from the upstream.
func (r *Repo) fetch(ctx context.Context) error {
	if err := fetch(ctx, r.dir, "origin"); err != nil {
		return err
	}
	return nil
}

// workingClone makes a non-bare clone, at `ref` (probably a branch),
// and returns the filesystem path to it.
func (r *Repo) workingClone(ctx context.Context, ref string) (string, error) {
	r.mu.RLock()
	defer r.mu.RUnlock()
	if err := r.errorIfNotReady(); err != nil {
		return "", err
	}
	working, err := ioutil.TempDir(os.TempDir(), "flux-working")
	if err != nil {
		return "", err
	}
	return clone(ctx, working, r.dir, ref)
}
<|endoftext|>


<commit_before>package ip

import (
	"errors"
	"net"

	"github.com/coreos/rudder/Godeps/_workspace/src/github.com/docker/libcontainer/netlink"
)

func GetIfaceIP4Addr(iface *net.Interface) (net.IP, error) {
	addrs, err := iface.Addrs()
	if err != nil {
		return nil, err
	}

	for _, addr := range addrs {
		// Attempt to parse the address in CIDR notation
		// and assert it is IPv4
		ip, _, err := net.ParseCIDR(addr.String())
		if err == nil && ip.To4() != nil {
			return ip.To4(), nil
		}
	}

	return nil, errors.New("No IPv4 address found for given interface")
}

func GetDefaultGatewayIface() (*net.Interface, error) {
	routes, err := netlink.NetworkGetRoutes()
	if err != nil {
		return nil, err
	}

	for _, route := range routes {
		if route.Default {
			if route.Iface == nil {
				return nil, errors.New("Found default route but could not determine interface")
			}
			return route.Iface, nil
		}
	}

	return nil, errors.New("Unable to find default route")
}

func GetInterfaceByIP(ip net.IP) (*net.Interface, error) {
	ifaces, err := net.Interfaces()
	if err != nil {
		return nil, err
	}

	for _, iface := range ifaces {
		addr, err := GetIfaceIP4Addr(&iface)
		if err != nil && ip.Equal(addr) {
			return &iface, nil
		}
	}

	return nil, errors.New("No interface with given IP found")
}
<commit_msg>Fix GetInterfaceByIP()<commit_after>package ip

import (
	"errors"
	"net"

	"github.com/coreos/rudder/Godeps/_workspace/src/github.com/docker/libcontainer/netlink"
)

func GetIfaceIP4Addr(iface *net.Interface) (net.IP, error) {
	addrs, err := iface.Addrs()
	if err != nil {
		return nil, err
	}

	for _, addr := range addrs {
		// Attempt to parse the address in CIDR notation
		// and assert it is IPv4
		ip, _, err := net.ParseCIDR(addr.String())
		if err == nil && ip.To4() != nil {
			return ip.To4(), nil
		}
	}

	return nil, errors.New("No IPv4 address found for given interface")
}

func GetDefaultGatewayIface() (*net.Interface, error) {
	routes, err := netlink.NetworkGetRoutes()
	if err != nil {
		return nil, err
	}

	for _, route := range routes {
		if route.Default {
			if route.Iface == nil {
				return nil, errors.New("Found default route but could not determine interface")
			}
			return route.Iface, nil
		}
	}

	return nil, errors.New("Unable to find default route")
}

func GetInterfaceByIP(ip net.IP) (*net.Interface, error) {
	ifaces, err := net.Interfaces()
	if err != nil {
		return nil, err
	}

	for _, iface := range ifaces {
		addr, err := GetIfaceIP4Addr(&iface)
		if err == nil && ip.Equal(addr) {
			return &iface, nil
		}
	}

	return nil, errors.New("No interface with given IP found")
}
<|endoftext|>


<commit_before>package os

import (
	"fmt"
	"io"
	"os"
	"os/exec"
	"path"

	log "github.com/Sirupsen/logrus"
	"github.com/emc-advanced-dev/pkg/errors"
	"path/filepath"
)

func RunLogCommand(name string, args ...string) error {
	log.WithFields(log.Fields{"cmd": name, "args": args}).Debug("running " + name)

	out, err := exec.Command(name, args...).CombinedOutput()
	if err != nil {
		log.WithFields(log.Fields{"out": string(out)}).Error(name + " failed")

	}
	return err
}

func GetDirSize(dir string) (int64, error) {

	stat, err := os.Stat(dir)
	if err != nil {
		return 0, err
	}

	if !stat.IsDir() {
		return stat.Size(), nil
	} else {
		entries, err := listDir(dir)
		if err != nil {
			return 0, err
		}
		var sum int64 = 0
		for _, obj := range entries {
			curSize, err := GetDirSize(path.Join(dir, obj.Name()))
			if err != nil {
				return 0, err
			}
			sum += curSize
		}
		return sum, nil

	}

}

func listDir(path string) ([]os.FileInfo, error) {
	directory, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer directory.Close()
	return directory.Readdir(-1)
}

// https://www.socketloop.com/tutorials/golang-copy-directory-including-sub-directories-files

func CopyDir(source string, dest string) (err error) {
	log.Debugf("copying dir %v to %v", source, dest)

	// get properties of source dir
	sourceinfo, err := os.Stat(source)
	if err != nil {
		return err
	}

	// create dest dir

	err = os.MkdirAll(dest, sourceinfo.Mode())
	if err != nil {
		return err
	}

	directory, _ := os.Open(source)

	objects, err := directory.Readdir(-1)
	if err != nil {
		return err
	}

	for _, obj := range objects {

		sourcefilepointer := path.Join(source, obj.Name())

		destinationfilepointer := path.Join(dest, obj.Name())

		sfi, err := os.Stat(sourcefilepointer)
		if err != nil {
			return err
		}
		if sfi.IsDir() {
			// create sub-directories - recursively
			err = CopyDir(sourcefilepointer, destinationfilepointer)
			if err != nil {
				return err
			}
		} else {
			// perform copy
			err = CopyFile(sourcefilepointer, destinationfilepointer)
			if err != nil {
				return err
			}
		}
	}

	return
}

/// http://stackoverflow.com/questions/21060945/simple-way-to-copy-a-file-in-golang/21067803#21067803

// CopyFile copies a file from src to dst. If src and dst files exist, and are
// the same, then return success. Otherise, attempt to create a hard link
// between the two files. If that fail, copy the file contents from src to dst.
func CopyFile(src, dst string) error {
	log.Debugf("copying file %v to %v", src, dst)
	sfi, err := os.Stat(src)
	if err != nil {
		return err
	}
	if !sfi.Mode().IsRegular() {
		// cannot copy non-regular files (e.g., directories,
		// symlinks, devices, etc.)
		return fmt.Errorf("CopyFile: non-regular source file %s (%q)", sfi.Name(), sfi.Mode().String())
	}
	dfi, err := os.Stat(dst)
	if err != nil {
		if !os.IsNotExist(err) {
			return err
		}
	} else {
		if !(dfi.Mode().IsRegular()) {
			return fmt.Errorf("CopyFile: non-regular destination file %s (%q)", dfi.Name(), dfi.Mode().String())
		}
		if os.SameFile(sfi, dfi) {
			return nil
		}
	}
	if err = os.Link(src, dst); err == nil {
		return nil
	}
	err = copyFileContents(src, dst)
	return err
}

// copyFileContents copies the contents of the file named src to the file named
// by dst. The file will be created if it does not already exist. If the
// destination file exists, all it's contents will be replaced by the contents
// of the source file.
func copyFileContents(src, dst string) (err error) {
	in, err := os.Open(src)
	if err != nil {
		return
	}
	defer in.Close()
	out, err := os.Create(dst)
	if err != nil {
		return
	}
	defer func() {
		cerr := out.Close()
		if err == nil {
			err = cerr
		}
	}()
	if _, err = io.Copy(out, in); err != nil {
		return
	}
	err = out.Sync()
	return
}

// http://stackoverflow.com/questions/32482673/golang-how-to-get-directory-total-size
func DirSize(path string) (int64, error) {
	var size int64
	err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {
		if !info.IsDir() {
			size += info.Size()
		}
		log.Debugf("total size %v after adding file %s", (int64(size)>>20)+10, info.Name())
		return err
	})
	return size, err
}
<commit_msg>reduce logging verbosity<commit_after>package os

import (
	"fmt"
	"io"
	"os"
	"os/exec"
	"path"

	log "github.com/Sirupsen/logrus"
	"path/filepath"
)

func RunLogCommand(name string, args ...string) error {
	log.WithFields(log.Fields{"cmd": name, "args": args}).Debug("running " + name)

	out, err := exec.Command(name, args...).CombinedOutput()
	if err != nil {
		log.WithFields(log.Fields{"out": string(out)}).Error(name + " failed")

	}
	return err
}

func GetDirSize(dir string) (int64, error) {

	stat, err := os.Stat(dir)
	if err != nil {
		return 0, err
	}

	if !stat.IsDir() {
		return stat.Size(), nil
	} else {
		entries, err := listDir(dir)
		if err != nil {
			return 0, err
		}
		var sum int64 = 0
		for _, obj := range entries {
			curSize, err := GetDirSize(path.Join(dir, obj.Name()))
			if err != nil {
				return 0, err
			}
			sum += curSize
		}
		return sum, nil

	}

}

func listDir(path string) ([]os.FileInfo, error) {
	directory, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer directory.Close()
	return directory.Readdir(-1)
}

// https://www.socketloop.com/tutorials/golang-copy-directory-including-sub-directories-files

func CopyDir(source string, dest string) (err error) {
	// get properties of source dir
	sourceinfo, err := os.Stat(source)
	if err != nil {
		return err
	}

	// create dest dir

	err = os.MkdirAll(dest, sourceinfo.Mode())
	if err != nil {
		return err
	}

	directory, _ := os.Open(source)

	objects, err := directory.Readdir(-1)
	if err != nil {
		return err
	}

	for _, obj := range objects {

		sourcefilepointer := path.Join(source, obj.Name())

		destinationfilepointer := path.Join(dest, obj.Name())

		sfi, err := os.Stat(sourcefilepointer)
		if err != nil {
			return err
		}
		if sfi.IsDir() {
			// create sub-directories - recursively
			err = CopyDir(sourcefilepointer, destinationfilepointer)
			if err != nil {
				return err
			}
		} else {
			// perform copy
			err = CopyFile(sourcefilepointer, destinationfilepointer)
			if err != nil {
				return err
			}
		}
	}

	return
}

/// http://stackoverflow.com/questions/21060945/simple-way-to-copy-a-file-in-golang/21067803#21067803

// CopyFile copies a file from src to dst. If src and dst files exist, and are
// the same, then return success. Otherise, attempt to create a hard link
// between the two files. If that fail, copy the file contents from src to dst.
func CopyFile(src, dst string) error {
	sfi, err := os.Stat(src)
	if err != nil {
		return err
	}
	if !sfi.Mode().IsRegular() {
		// cannot copy non-regular files (e.g., directories,
		// symlinks, devices, etc.)
		return fmt.Errorf("CopyFile: non-regular source file %s (%q)", sfi.Name(), sfi.Mode().String())
	}
	dfi, err := os.Stat(dst)
	if err != nil {
		if !os.IsNotExist(err) {
			return err
		}
	} else {
		if !(dfi.Mode().IsRegular()) {
			return fmt.Errorf("CopyFile: non-regular destination file %s (%q)", dfi.Name(), dfi.Mode().String())
		}
		if os.SameFile(sfi, dfi) {
			return nil
		}
	}
	if err = os.Link(src, dst); err == nil {
		return nil
	}
	err = copyFileContents(src, dst)
	return err
}

// copyFileContents copies the contents of the file named src to the file named
// by dst. The file will be created if it does not already exist. If the
// destination file exists, all it's contents will be replaced by the contents
// of the source file.
func copyFileContents(src, dst string) (err error) {
	in, err := os.Open(src)
	if err != nil {
		return
	}
	defer in.Close()
	out, err := os.Create(dst)
	if err != nil {
		return
	}
	defer func() {
		cerr := out.Close()
		if err == nil {
			err = cerr
		}
	}()
	if _, err = io.Copy(out, in); err != nil {
		return
	}
	err = out.Sync()
	return
}

// http://stackoverflow.com/questions/32482673/golang-how-to-get-directory-total-size
func DirSize(path string) (int64, error) {
	var size int64
	err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {
		if !info.IsDir() {
			size += info.Size()
		}
		log.Debugf("total size %v after adding file %s", (int64(size)>>20)+10, info.Name())
		return err
	})
	return size, err
}
<|endoftext|>


<commit_before>package masterapi

import (
	"encoding/json"
	"github.com/KIT-MAMID/mamid/master"
	"github.com/KIT-MAMID/mamid/model"
	"github.com/gorilla/mux"
	"github.com/jinzhu/gorm"
	"github.com/stretchr/testify/assert"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"
	"fmt"
)

func createDBAndMasterAPI(t *testing.T) (db *gorm.DB, mainRouter *mux.Router, err error) {
	// Setup database
	db, err = model.InitializeInMemoryDB("")
	dbSlave := model.Slave{
		Hostname:             "host1",
		Port:                 1,
		MongodPortRangeBegin: 2,
		MongodPortRangeEnd:   3,
		PersistentStorage:    true,
		Mongods:              []*model.Mongod{},
		ConfiguredState:      model.SlaveStateActive,
	}
	assert.NoError(t, db.Create(&dbSlave).Error)

	dbSlave2 := model.Slave{
		Hostname:             "host2",
		Port:                 1,
		MongodPortRangeBegin: 100,
		MongodPortRangeEnd:   200,
		PersistentStorage:    false,
		Mongods:              []*model.Mongod{},
		ConfiguredState:      model.SlaveStateDisabled,
	}
	assert.NoError(t, db.Create(&dbSlave2).Error)

	// Setup masterapi
	clusterAllocator := &master.ClusterAllocator{}

	mainRouter = mux.NewRouter().StrictSlash(true)
	masterAPI := &MasterAPI{
		DB:               db,
		ClusterAllocator: clusterAllocator,
		Router:           mainRouter.PathPrefix("/api/").Subrouter(),
	}
	masterAPI.Setup()

	return
}

func TestMasterAPI_SlaveIndex(t *testing.T) {
	_, mainRouter, err := createDBAndMasterAPI(t)
	assert.NoError(t, err)

	// Test correct get
	resp := httptest.NewRecorder()

	req, err := http.NewRequest("GET", "/api/slaves", nil)
	assert.NoError(t, err)
	mainRouter.ServeHTTP(resp, req)

	var getSlaveResult []Slave
	err = json.NewDecoder(resp.Body).Decode(&getSlaveResult)
	assert.NoError(t, err)

	assert.Equal(t, 2, len(getSlaveResult))
	assert.Equal(t, "host1", getSlaveResult[0].Hostname)
	assert.EqualValues(t, 1, getSlaveResult[0].Port)
	assert.EqualValues(t, 2, getSlaveResult[0].MongodPortRangeBegin)
	assert.EqualValues(t, 3, getSlaveResult[0].MongodPortRangeEnd)
	assert.Equal(t, true, getSlaveResult[0].PersistentStorage)
	assert.Equal(t, "active", getSlaveResult[0].ConfiguredState)
}

func TestMasterAPI_SlaveById(t *testing.T) {
	_, mainRouter, err := createDBAndMasterAPI(t)
	assert.NoError(t, err)

	// Test correct get
	resp := httptest.NewRecorder()

	req, err := http.NewRequest("GET", "/api/slaves/1", nil)
	assert.NoError(t, err)
	mainRouter.ServeHTTP(resp, req)

	var getSlaveResult Slave
	err = json.NewDecoder(resp.Body).Decode(&getSlaveResult)
	assert.NoError(t, err)

	assert.Equal(t, "host1", getSlaveResult.Hostname)
	assert.EqualValues(t, 1, getSlaveResult.Port)
	assert.EqualValues(t, 2, getSlaveResult.MongodPortRangeBegin)
	assert.EqualValues(t, 3, getSlaveResult.MongodPortRangeEnd)
	assert.Equal(t, true, getSlaveResult.PersistentStorage)
	assert.Equal(t, "active", getSlaveResult.ConfiguredState)
}

func TestMasterAPI_SlavePut(t *testing.T) {
	db, mainRouter, err := createDBAndMasterAPI(t)
	assert.NoError(t, err)

	//Test correct put
	resp := httptest.NewRecorder()

	req_body := "{\"id\":0,\"hostname\":\"createdhost\",\"slave_port\":1912,\"mongod_port_range_begin\":20000,\"mongod_port_range_end\":20001,\"persistent_storage\":false,\"configured_state\":\"disabled\"}"
	req, err := http.NewRequest("PUT", "/api/slaves", strings.NewReader(req_body))
	assert.NoError(t, err)
	mainRouter.ServeHTTP(resp, req)

	if !assert.Equal(t, 200, resp.Code) {
		fmt.Println(resp.Body.String())
	}

	var createdSlave model.Slave
	db.First(&createdSlave, "hostname = ?", "createdhost")

	//Check created database entry
	assert.NotEmpty(t, createdSlave.ID)
	assert.Equal(t, "createdhost", createdSlave.Hostname)
	assert.EqualValues(t, 1912, createdSlave.Port)
	assert.EqualValues(t, 20000, createdSlave.MongodPortRangeBegin)
	assert.EqualValues(t, 20001, createdSlave.MongodPortRangeEnd)
	assert.Equal(t, false, createdSlave.PersistentStorage)
	assert.Equal(t, model.SlaveStateDisabled, createdSlave.ConfiguredState)

	//Check returned object
	var getSlaveResult Slave
	err = json.NewDecoder(resp.Body).Decode(&getSlaveResult)
	assert.NoError(t, err)

	assert.NotEmpty(t, getSlaveResult.ID)
	assert.Equal(t, "createdhost", getSlaveResult.Hostname)
}

func TestMasterAPI_SlavePut_existing_hostname(t *testing.T) {
	_, mainRouter, err := createDBAndMasterAPI(t)
	assert.NoError(t, err)

	//Test invalid put (slave with same hostname exists)
	resp := httptest.NewRecorder()

	req_body := "{\"id\":0,\"hostname\":\"host1\",\"slave_port\":1912,\"mongod_port_range_begin\":20000,\"mongod_port_range_end\":20001,\"persistent_storage\":false,\"configured_state\":\"disabled\"}"
	req, err := http.NewRequest("PUT", "/api/slaves", strings.NewReader(req_body))
	assert.NoError(t, err)
	mainRouter.ServeHTTP(resp, req)

	assert.Equal(t, 400, resp.Code)
}

func TestMasterAPI_SlavePut_invalid(t *testing.T) {
	db, mainRouter, err := createDBAndMasterAPI(t)
	assert.NoError(t, err)
	//Test invalid put (non existing field)
	resp := httptest.NewRecorder()

	req_body := "{\"id_invalid_blabla\":0,\"hostname\":\"createdhost_invalid\",\"slave_port\":1912,\"mongod_port_range_begin\":20000,\"mongod_port_range_end\":20001,\"persistent_storage\":false,\"configured_state\":\"disabled\"}"
	req, err := http.NewRequest("PUT", "/api/slaves", strings.NewReader(req_body))
	assert.NoError(t, err)
	mainRouter.ServeHTTP(resp, req)

	assert.Equal(t, 400, resp.Code)

	var invalidSlave model.Slave
	db.First(&invalidSlave, "hostname = ?", "createdhost_invalid")

	assert.Empty(t, invalidSlave.ID)
}

func TestMasterAPI_SlaveUpdate(t *testing.T) {
	db, mainRouter, err := createDBAndMasterAPI(t)
	assert.NoError(t, err)

	//Test valid update
	resp := httptest.NewRecorder()

	req_body := "{\"id\":2,\"hostname\":\"updHost\",\"slave_port\":2,\"mongod_port_range_begin\":101,\"mongod_port_range_end\":201,\"persistent_storage\":true,\"configured_state\":\"disabled\"}"
	req, err := http.NewRequest("POST", "/api/slaves/2", strings.NewReader(req_body))
	assert.NoError(t, err)
	mainRouter.ServeHTTP(resp, req)

	assert.Equal(t, 200, resp.Code)

	var updatedSlave model.Slave
	db.First(&updatedSlave, 2)

	assert.Equal(t, "updHost", updatedSlave.Hostname)
	assert.EqualValues(t, 2, updatedSlave.Port)
	assert.EqualValues(t, 101, updatedSlave.MongodPortRangeBegin)
	assert.EqualValues(t, 201, updatedSlave.MongodPortRangeEnd)
	assert.Equal(t, true, updatedSlave.PersistentStorage)
	assert.Equal(t, model.SlaveStateDisabled, updatedSlave.ConfiguredState)
}

func TestMasterAPI_SlaveUpdate_invalid(t *testing.T) {
	db, mainRouter, err := createDBAndMasterAPI(t)
	assert.NoError(t, err)

	//Test invalid update (slave is in active state)
	resp := httptest.NewRecorder()

	req_body := "{\"id\":1,\"hostname\":\"updHost\",\"slave_port\":1912,\"mongod_port_range_begin\":20000,\"mongod_port_range_end\":20001,\"persistent_storage\":false,\"configured_state\":\"active\"}"
	req, err := http.NewRequest("POST", "/api/slaves/1", strings.NewReader(req_body))
	assert.NoError(t, err)
	mainRouter.ServeHTTP(resp, req)

	assert.Equal(t, 400, resp.Code)

	var updatedSlave model.Slave
	db.First(&updatedSlave, 1)

	assert.Equal(t, "host1", updatedSlave.Hostname)
	assert.EqualValues(t, 1, updatedSlave.Port)
	assert.EqualValues(t, 2, updatedSlave.MongodPortRangeBegin)
	assert.EqualValues(t, 3, updatedSlave.MongodPortRangeEnd)
	assert.Equal(t, true, updatedSlave.PersistentStorage)
	assert.Equal(t, model.SlaveStateActive, updatedSlave.ConfiguredState)
}

func TestMasterAPI_SlaveUpdate_change_desired_state(t *testing.T) {
	db, mainRouter, err := createDBAndMasterAPI(t)
	assert.NoError(t, err)

	//Test valid state change
	resp := httptest.NewRecorder()

	req_body := "{\"id\":2,\"hostname\":\"host2\",\"slave_port\":1,\"mongod_port_range_begin\":100,\"mongod_port_range_end\":200,\"persistent_storage\":false,\"configured_state\":\"active\"}"
	req, err := http.NewRequest("POST", "/api/slaves/2", strings.NewReader(req_body))
	assert.NoError(t, err)
	mainRouter.ServeHTTP(resp, req)

	assert.Equal(t, 200, resp.Code)

	var updatedSlave model.Slave
	db.First(&updatedSlave, 2)

	assert.Equal(t, model.SlaveStateActive, updatedSlave.ConfiguredState)
}

func TestMasterAPI_SlaveUpdate_change_desired_state_invalid(t *testing.T) {
	db, mainRouter, err := createDBAndMasterAPI(t)
	assert.NoError(t, err)

	//Test invalid state change (should not be able to change state while changing another parameter)
	resp := httptest.NewRecorder()

	req_body := "{\"id\":2,\"hostname\":\"host2\",\"slave_port\":1,\"mongod_port_range_begin\":100,\"mongod_port_range_end\":150,\"persistent_storage\":false,\"configured_state\":\"active\"}"
	req, err := http.NewRequest("POST", "/api/slaves/2", strings.NewReader(req_body))
	assert.NoError(t, err)
	mainRouter.ServeHTTP(resp, req)

	assert.Equal(t, 400, resp.Code)

	var updatedSlave model.Slave
	db.First(&updatedSlave, 2)

	assert.Equal(t, model.SlaveStateDisabled, updatedSlave.ConfiguredState)
}

func TestMasterAPI_SlaveDelete(t *testing.T) {
	db, mainRouter, err := createDBAndMasterAPI(t)
	assert.NoError(t, err)

	//Test valid delete
	resp := httptest.NewRecorder()

	req, err := http.NewRequest("DELETE", "/api/slaves/2", nil)
	assert.NoError(t, err)
	mainRouter.ServeHTTP(resp, req)

	assert.Equal(t, 200, resp.Code)

	var updatedSlave model.Slave
	db.First(&updatedSlave, 2)

	assert.Empty(t, updatedSlave.ID)
}

func TestMasterAPI_SlaveDelete_invalid(t *testing.T) {
	db, mainRouter, err := createDBAndMasterAPI(t)
	assert.NoError(t, err)

	//Test invalid delete (active slave)
	resp := httptest.NewRecorder()

	req, err := http.NewRequest("DELETE", "/api/slaves/1", nil)
	assert.NoError(t, err)
	mainRouter.ServeHTTP(resp, req)

	assert.Equal(t, 400, resp.Code)

	var updatedSlave model.Slave
	db.First(&updatedSlave, 1)

	assert.NotEmpty(t, updatedSlave.ID)
}
<commit_msg>ADD/FIX: masterapi: invalid & missing fields in PUT body.<commit_after>package masterapi

import (
	"encoding/json"
	"github.com/KIT-MAMID/mamid/master"
	"github.com/KIT-MAMID/mamid/model"
	"github.com/gorilla/mux"
	"github.com/jinzhu/gorm"
	"github.com/stretchr/testify/assert"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"
	"fmt"
)

func createDBAndMasterAPI(t *testing.T) (db *gorm.DB, mainRouter *mux.Router, err error) {
	// Setup database
	db, err = model.InitializeInMemoryDB("")
	dbSlave := model.Slave{
		Hostname:             "host1",
		Port:                 1,
		MongodPortRangeBegin: 2,
		MongodPortRangeEnd:   3,
		PersistentStorage:    true,
		Mongods:              []*model.Mongod{},
		ConfiguredState:      model.SlaveStateActive,
	}
	assert.NoError(t, db.Create(&dbSlave).Error)

	dbSlave2 := model.Slave{
		Hostname:             "host2",
		Port:                 1,
		MongodPortRangeBegin: 100,
		MongodPortRangeEnd:   200,
		PersistentStorage:    false,
		Mongods:              []*model.Mongod{},
		ConfiguredState:      model.SlaveStateDisabled,
	}
	assert.NoError(t, db.Create(&dbSlave2).Error)

	// Setup masterapi
	clusterAllocator := &master.ClusterAllocator{}

	mainRouter = mux.NewRouter().StrictSlash(true)
	masterAPI := &MasterAPI{
		DB:               db,
		ClusterAllocator: clusterAllocator,
		Router:           mainRouter.PathPrefix("/api/").Subrouter(),
	}
	masterAPI.Setup()

	return
}

func TestMasterAPI_SlaveIndex(t *testing.T) {
	_, mainRouter, err := createDBAndMasterAPI(t)
	assert.NoError(t, err)

	// Test correct get
	resp := httptest.NewRecorder()

	req, err := http.NewRequest("GET", "/api/slaves", nil)
	assert.NoError(t, err)
	mainRouter.ServeHTTP(resp, req)

	var getSlaveResult []Slave
	err = json.NewDecoder(resp.Body).Decode(&getSlaveResult)
	assert.NoError(t, err)

	assert.Equal(t, 2, len(getSlaveResult))
	assert.Equal(t, "host1", getSlaveResult[0].Hostname)
	assert.EqualValues(t, 1, getSlaveResult[0].Port)
	assert.EqualValues(t, 2, getSlaveResult[0].MongodPortRangeBegin)
	assert.EqualValues(t, 3, getSlaveResult[0].MongodPortRangeEnd)
	assert.Equal(t, true, getSlaveResult[0].PersistentStorage)
	assert.Equal(t, "active", getSlaveResult[0].ConfiguredState)
}

func TestMasterAPI_SlaveById(t *testing.T) {
	_, mainRouter, err := createDBAndMasterAPI(t)
	assert.NoError(t, err)

	// Test correct get
	resp := httptest.NewRecorder()

	req, err := http.NewRequest("GET", "/api/slaves/1", nil)
	assert.NoError(t, err)
	mainRouter.ServeHTTP(resp, req)

	var getSlaveResult Slave
	err = json.NewDecoder(resp.Body).Decode(&getSlaveResult)
	assert.NoError(t, err)

	assert.Equal(t, "host1", getSlaveResult.Hostname)
	assert.EqualValues(t, 1, getSlaveResult.Port)
	assert.EqualValues(t, 2, getSlaveResult.MongodPortRangeBegin)
	assert.EqualValues(t, 3, getSlaveResult.MongodPortRangeEnd)
	assert.Equal(t, true, getSlaveResult.PersistentStorage)
	assert.Equal(t, "active", getSlaveResult.ConfiguredState)
}

func TestMasterAPI_SlavePut(t *testing.T) {
	db, mainRouter, err := createDBAndMasterAPI(t)
	assert.NoError(t, err)

	//Test correct put
	resp := httptest.NewRecorder()

	req_body := "{\"id\":0,\"hostname\":\"createdhost\",\"slave_port\":1912,\"mongod_port_range_begin\":20000,\"mongod_port_range_end\":20001,\"persistent_storage\":false,\"configured_state\":\"disabled\"}"
	req, err := http.NewRequest("PUT", "/api/slaves", strings.NewReader(req_body))
	assert.NoError(t, err)
	mainRouter.ServeHTTP(resp, req)

	if !assert.Equal(t, 200, resp.Code) {
		fmt.Println(resp.Body.String())
	}

	var createdSlave model.Slave
	db.First(&createdSlave, "hostname = ?", "createdhost")

	//Check created database entry
	assert.NotEmpty(t, createdSlave.ID)
	assert.Equal(t, "createdhost", createdSlave.Hostname)
	assert.EqualValues(t, 1912, createdSlave.Port)
	assert.EqualValues(t, 20000, createdSlave.MongodPortRangeBegin)
	assert.EqualValues(t, 20001, createdSlave.MongodPortRangeEnd)
	assert.Equal(t, false, createdSlave.PersistentStorage)
	assert.Equal(t, model.SlaveStateDisabled, createdSlave.ConfiguredState)

	//Check returned object
	var getSlaveResult Slave
	err = json.NewDecoder(resp.Body).Decode(&getSlaveResult)
	assert.NoError(t, err)

	assert.NotEmpty(t, getSlaveResult.ID)
	assert.Equal(t, "createdhost", getSlaveResult.Hostname)
}

func TestMasterAPI_SlavePut_existing_hostname(t *testing.T) {
	_, mainRouter, err := createDBAndMasterAPI(t)
	assert.NoError(t, err)

	//Test invalid put (slave with same hostname exists)
	resp := httptest.NewRecorder()

	req_body := "{\"id\":0,\"hostname\":\"host1\",\"slave_port\":1912,\"mongod_port_range_begin\":20000,\"mongod_port_range_end\":20001,\"persistent_storage\":false,\"configured_state\":\"disabled\"}"
	req, err := http.NewRequest("PUT", "/api/slaves", strings.NewReader(req_body))
	assert.NoError(t, err)
	mainRouter.ServeHTTP(resp, req)

	assert.Equal(t, 400, resp.Code)
}

func TestMasterAPI_SlavePut_additionalUnknownField(t *testing.T) {
	db, mainRouter, err := createDBAndMasterAPI(t)
	assert.NoError(t, err)
	//Test invalid put (non existing field)
	resp := httptest.NewRecorder()

	// additional unknown field:  id_invalid_blabla
	req_body := "{\"hostname\":\"createdhost\",\"slave_port\":1912,\"mongod_port_range_begin\":20000,\"mongod_port_range_end\":20002,\"persistent_storage\":false,\"configured_state\":\"disabled\"}"
	req, err := http.NewRequest("PUT", "/api/slaves", strings.NewReader(req_body))
	assert.NoError(t, err)
	mainRouter.ServeHTTP(resp, req)

	assert.Equal(t, 200, resp.Code)
	var invalidSlave model.Slave
	assert.NoError(t, db.First(&invalidSlave, "hostname = ?", "createdhost").Error)
}

func TestMasterAPI_SlavePut_missingField(t *testing.T) {
	db, mainRouter, err := createDBAndMasterAPI(t)
	assert.NoError(t, err)
	//Test invalid put (non existing field)
	resp := httptest.NewRecorder()

	// additional unknown field:  id_invalid_blabla
	// missing field: mongod_port_range_begin
	req_body := "{\"hostname\":\"createdhost_invalid\",\"slave_port\":1912,\"mongod_port_range_begin\":20000,\"persistent_storage\":false,\"configured_state\":\"disabled\"}"
	req, err := http.NewRequest("PUT", "/api/slaves", strings.NewReader(req_body))
	assert.NoError(t, err)
	mainRouter.ServeHTTP(resp, req)

	assert.Equal(t, 400, resp.Code)

	var invalidSlave model.Slave
	assert.Error(t, db.First(&invalidSlave, "hostname = ?", "createdhost_invalid").Error)
}

func TestMasterAPI_SlaveUpdate(t *testing.T) {
	db, mainRouter, err := createDBAndMasterAPI(t)
	assert.NoError(t, err)

	//Test valid update
	resp := httptest.NewRecorder()

	req_body := "{\"id\":2,\"hostname\":\"updHost\",\"slave_port\":2,\"mongod_port_range_begin\":101,\"mongod_port_range_end\":201,\"persistent_storage\":true,\"configured_state\":\"disabled\"}"
	req, err := http.NewRequest("POST", "/api/slaves/2", strings.NewReader(req_body))
	assert.NoError(t, err)
	mainRouter.ServeHTTP(resp, req)

	assert.Equal(t, 200, resp.Code)

	var updatedSlave model.Slave
	db.First(&updatedSlave, 2)

	assert.Equal(t, "updHost", updatedSlave.Hostname)
	assert.EqualValues(t, 2, updatedSlave.Port)
	assert.EqualValues(t, 101, updatedSlave.MongodPortRangeBegin)
	assert.EqualValues(t, 201, updatedSlave.MongodPortRangeEnd)
	assert.Equal(t, true, updatedSlave.PersistentStorage)
	assert.Equal(t, model.SlaveStateDisabled, updatedSlave.ConfiguredState)
}

func TestMasterAPI_SlaveUpdate_invalid(t *testing.T) {
	db, mainRouter, err := createDBAndMasterAPI(t)
	assert.NoError(t, err)

	//Test invalid update (slave is in active state)
	resp := httptest.NewRecorder()

	req_body := "{\"id\":1,\"hostname\":\"updHost\",\"slave_port\":1912,\"mongod_port_range_begin\":20000,\"mongod_port_range_end\":20001,\"persistent_storage\":false,\"configured_state\":\"active\"}"
	req, err := http.NewRequest("POST", "/api/slaves/1", strings.NewReader(req_body))
	assert.NoError(t, err)
	mainRouter.ServeHTTP(resp, req)

	assert.Equal(t, 400, resp.Code)

	var updatedSlave model.Slave
	db.First(&updatedSlave, 1)

	assert.Equal(t, "host1", updatedSlave.Hostname)
	assert.EqualValues(t, 1, updatedSlave.Port)
	assert.EqualValues(t, 2, updatedSlave.MongodPortRangeBegin)
	assert.EqualValues(t, 3, updatedSlave.MongodPortRangeEnd)
	assert.Equal(t, true, updatedSlave.PersistentStorage)
	assert.Equal(t, model.SlaveStateActive, updatedSlave.ConfiguredState)
}

func TestMasterAPI_SlaveUpdate_change_desired_state(t *testing.T) {
	db, mainRouter, err := createDBAndMasterAPI(t)
	assert.NoError(t, err)

	//Test valid state change
	resp := httptest.NewRecorder()

	req_body := "{\"id\":2,\"hostname\":\"host2\",\"slave_port\":1,\"mongod_port_range_begin\":100,\"mongod_port_range_end\":200,\"persistent_storage\":false,\"configured_state\":\"active\"}"
	req, err := http.NewRequest("POST", "/api/slaves/2", strings.NewReader(req_body))
	assert.NoError(t, err)
	mainRouter.ServeHTTP(resp, req)

	assert.Equal(t, 200, resp.Code)

	var updatedSlave model.Slave
	db.First(&updatedSlave, 2)

	assert.Equal(t, model.SlaveStateActive, updatedSlave.ConfiguredState)
}

func TestMasterAPI_SlaveUpdate_change_desired_state_invalid(t *testing.T) {
	db, mainRouter, err := createDBAndMasterAPI(t)
	assert.NoError(t, err)

	//Test invalid state change (should not be able to change state while changing another parameter)
	resp := httptest.NewRecorder()

	req_body := "{\"id\":2,\"hostname\":\"host2\",\"slave_port\":1,\"mongod_port_range_begin\":100,\"mongod_port_range_end\":150,\"persistent_storage\":false,\"configured_state\":\"active\"}"
	req, err := http.NewRequest("POST", "/api/slaves/2", strings.NewReader(req_body))
	assert.NoError(t, err)
	mainRouter.ServeHTTP(resp, req)

	assert.Equal(t, 400, resp.Code)

	var updatedSlave model.Slave
	db.First(&updatedSlave, 2)

	assert.Equal(t, model.SlaveStateDisabled, updatedSlave.ConfiguredState)
}

func TestMasterAPI_SlaveDelete(t *testing.T) {
	db, mainRouter, err := createDBAndMasterAPI(t)
	assert.NoError(t, err)

	//Test valid delete
	resp := httptest.NewRecorder()

	req, err := http.NewRequest("DELETE", "/api/slaves/2", nil)
	assert.NoError(t, err)
	mainRouter.ServeHTTP(resp, req)

	assert.Equal(t, 200, resp.Code)

	var updatedSlave model.Slave
	db.First(&updatedSlave, 2)

	assert.Empty(t, updatedSlave.ID)
}

func TestMasterAPI_SlaveDelete_invalid(t *testing.T) {
	db, mainRouter, err := createDBAndMasterAPI(t)
	assert.NoError(t, err)

	//Test invalid delete (active slave)
	resp := httptest.NewRecorder()

	req, err := http.NewRequest("DELETE", "/api/slaves/1", nil)
	assert.NoError(t, err)
	mainRouter.ServeHTTP(resp, req)

	assert.Equal(t, 400, resp.Code)

	var updatedSlave model.Slave
	db.First(&updatedSlave, 1)

	assert.NotEmpty(t, updatedSlave.ID)
}
<|endoftext|>


<commit_before>package halo

import (
	"encoding/binary"
	"os"
	"sort"

	"github.com/phil-mansfield/gotetra/render/io"
	"github.com/phil-mansfield/table"
)

// halos allows for arrays of halo properties to be sorted simultaneously.
type halos struct {
	rids []int
	xs, ys, zs, ms, rs []float64
}

func (hs *halos) Len() int { return len(hs.rs) }
func (hs *halos) Less(i, j int) bool { return hs.rs[i] < hs.rs[j] }
func (hs *halos) Swap(i, j int) {
	hs.rs[i], hs.rs[j] = hs.rs[j], hs.rs[i]
	hs.ms[i], hs.ms[j] = hs.ms[j], hs.ms[i]
	hs.xs[i], hs.xs[j] = hs.xs[j], hs.xs[i]
	hs.ys[i], hs.ys[j] = hs.ys[j], hs.ys[i]
	hs.zs[i], hs.zs[j] = hs.zs[j], hs.zs[i]
	hs.rids[i], hs.rids[j] = hs.rids[j], hs.rids[i]
}

// ReadRockstar reads halo information from the given Rockstar catalog, sorted
// from largest to smallest.
func ReadRockstar(
	file string, rType Radius, cosmo *io.CosmologyHeader,
) (rids []int, xs, ys, zs, ms, rs []float64, err error) {
	rCol := rType.RockstarColumn()
	idCol, xCol, yCol, zCol := 1, 17, 18, 19

	colIdxs := []int{ idCol, xCol, yCol, zCol, rCol }
	cols, err := table.ReadTable(file, colIdxs, nil)
	if err != nil { return nil, nil, nil, nil, nil, nil, err }

	ids := cols[0]
	xs, ys, zs = cols[1], cols[2], cols[3]
	if rType.RockstarMass() {
		ms = cols[4]
		rs = make([]float64, len(ms))
		rType.Radius(cosmo, ms, rs)
	} else {
		rs = cols[4]
		ms = make([]float64, len(rs))
		for i := range rs { rs[i] /= 1000 } // kpc -> Mpc
		rType.Mass(cosmo, rs, ms)
	}

	rids = make([]int, len(ids))
	for i := range rids { rids[i] = int(ids[i]) }

	sort.Sort(sort.Reverse(&halos{ rids, xs, ys, zs, ms, rs }))
	return rids, xs, ys, zs, ms, rs, nil
}

type Val int
const (
	Scale Val = iota
	ID
	DescScale
	DescID
	NumProg
	PID
	UPID
	DescPID
	Phantom
	SAMMVir
	MVir
	RVir
	Rs
	Vrms
	MMP
	ScaleOfLastMMP
	VMax
	X
	Y
	Z
	Vx
	Vy
	Vz
	Jx
	Jy
	Jz
	Spin
	BreadthFirstID
	DepthFirstID
	TreeRootID
	OrigHaloID
	SnapNum
	NextCoprogenitorDepthFirstID
	LastProgenitorDepthFirstID
	RsKylpin
	MVirAll
	M200b
	M200c
	M500c
	M2500c
	XOff
	Voff
	SpinBullock
	BToA
	CToA
	Ax
	Ay
	Az
	BToA500c
	CToA500c
	Ax500c
	Ay500c
	Az500c
	TU
	MAcc
	MPeak
	VAcc
	VPeak
	HalfmassScale
	AccRateInst
	AccRate100Myr
	AccRateTdyn
	valNum
	RadVir
	Rad200b
	Rad200c
	Rad500c
	Rad2500c
)

func RockstarConvert(inFile, outFile string) error {
	allIdxs := make([]int, valNum)
	for i := range allIdxs { allIdxs[i] = i }
	cols, err := table.ReadTable(inFile, allIdxs, nil)
	if err != nil { return err }

	f, err := os.Create(outFile)
	if err != nil { return err }
	defer f.Close()

	err = binary.Write(f, binary.LittleEndian, int64(len(cols[0])))
	if err != nil { return err }
	for _, col := range cols {
		err := binary.Write(f, binary.LittleEndian, col)
		if err != nil { return err }
	}

	return nil
}

func ReadBinaryRockstarVals(
	file string, cosmo *io.CosmologyHeader, valFlags ...Val,
) (ids []int, vals[][]float64, err error) {
	return readRockstarVals(file, cosmo, valFlags, binaryColGetter)
}

func ReadRockstarVals(
	file string, cosmo *io.CosmologyHeader, valFlags ...Val,
) (ids []int, vals[][]float64, err error) {
	return readRockstarVals(file, cosmo, valFlags, asciiColGetter)
}

func readRockstarVals(
	file string, cosmo *io.CosmologyHeader, valFlags []Val, getter colGetter,
) (ids []int, vals[][]float64, err error) {
	colIdxs := []int{ int(ID) }
	for _, val := range valFlags {
		switch val {
		case RadVir: colIdxs = append(colIdxs, int(MVir))
		case Rad200b: colIdxs = append(colIdxs, int(M200b))
		case Rad200c: colIdxs = append(colIdxs, int(M200c))
		case Rad500c: colIdxs = append(colIdxs, int(M500c))
		case Rad2500c: colIdxs = append(colIdxs, int(M2500c))
		default:
			colIdxs = append(colIdxs, int(val))
		}
	}

	vals, err = getter(file, colIdxs)
	if err != nil { return nil, nil, err }

	ids = make([]int, len(vals[0]))
	for i := range vals[0] {
		ids[i] = int(vals[0][i])
	}

	for i, val := range valFlags {
		switch val {
		case RadVir: RVirial.Radius(cosmo, vals[i+1], vals[i+1])
		case Rad200b: R200m.Radius(cosmo, vals[i+1], vals[i+1])
		case Rad200c: R200c.Radius(cosmo, vals[i+1], vals[i+1])
		case Rad500c: R500c.Radius(cosmo, vals[i+1], vals[i+1])
		case Rad2500c: R2500c.Radius(cosmo, vals[i+1], vals[i+1])
		}
	}

	if len(vals) == 1 {
		return ids, [][]float64{}, nil
	} else {
		return ids, vals[1:], nil
	}
}

type colGetter func(file string, colIdxs []int) ([][]float64, error)
func asciiColGetter(file string, colIdxs []int) ([][]float64, error) {
	return table.ReadTable(file, colIdxs, nil)
}
func binaryColGetter(file string, colIdxs []int) ([][]float64, error) {
	f, err := os.Open(file)
	if err != nil { return nil, err }

	n := 0
	err = binary.Read(f, binary.LittleEndian, &n)
	if err != nil { return nil, err }
	cols := make([][]float64, valNum)

	for col := range cols {
		cols[col] = make([]float64, n)
		err = binary.Read(f, binary.LittleEndian, cols[col])
		if err != nil { return nil, err }
	}
	return cols, nil
}

func init() {
	if valNum != 62 { panic("Internal gotetra setup error.") }
}
<commit_msg>Wrote ReadRockstarTopN()<commit_after>package halo

import (
	"encoding/binary"
	"os"
	"sort"

	"github.com/phil-mansfield/gotetra/render/io"
	"github.com/phil-mansfield/table"
)

// halos allows for arrays of halo properties to be sorted simultaneously.
type halos struct {
	rids []int
	xs, ys, zs, ms, rs []float64
}

func (hs *halos) Len() int { return len(hs.rs) }
func (hs *halos) Less(i, j int) bool { return hs.rs[i] < hs.rs[j] }
func (hs *halos) Swap(i, j int) {
	hs.rs[i], hs.rs[j] = hs.rs[j], hs.rs[i]
	hs.ms[i], hs.ms[j] = hs.ms[j], hs.ms[i]
	hs.xs[i], hs.xs[j] = hs.xs[j], hs.xs[i]
	hs.ys[i], hs.ys[j] = hs.ys[j], hs.ys[i]
	hs.zs[i], hs.zs[j] = hs.zs[j], hs.zs[i]
	hs.rids[i], hs.rids[j] = hs.rids[j], hs.rids[i]
}

// ReadRockstar reads halo information from the given Rockstar catalog, sorted
// from largest to smallest.
func ReadRockstar(
	file string, rType Radius, cosmo *io.CosmologyHeader,
) (rids []int, xs, ys, zs, ms, rs []float64, err error) {
	rCol := rType.RockstarColumn()
	idCol, xCol, yCol, zCol := 1, 17, 18, 19

	colIdxs := []int{ idCol, xCol, yCol, zCol, rCol }
	cols, err := table.ReadTable(file, colIdxs, nil)
	if err != nil { return nil, nil, nil, nil, nil, nil, err }

	ids := cols[0]
	xs, ys, zs = cols[1], cols[2], cols[3]
	if rType.RockstarMass() {
		ms = cols[4]
		rs = make([]float64, len(ms))
		rType.Radius(cosmo, ms, rs)
	} else {
		rs = cols[4]
		ms = make([]float64, len(rs))
		for i := range rs { rs[i] /= 1000 } // kpc -> Mpc
		rType.Mass(cosmo, rs, ms)
	}

	rids = make([]int, len(ids))
	for i := range rids { rids[i] = int(ids[i]) }

	sort.Sort(sort.Reverse(&halos{ rids, xs, ys, zs, ms, rs }))
	return rids, xs, ys, zs, ms, rs, nil
}

type Val int
const (
	Scale Val = iota
	ID
	DescScale
	DescID
	NumProg
	PID
	UPID
	DescPID
	Phantom
	SAMMVir
	MVir
	RVir
	Rs
	Vrms
	MMP
	ScaleOfLastMMP
	VMax
	X
	Y
	Z
	Vx
	Vy
	Vz
	Jx
	Jy
	Jz
	Spin
	BreadthFirstID
	DepthFirstID
	TreeRootID
	OrigHaloID
	SnapNum
	NextCoprogenitorDepthFirstID
	LastProgenitorDepthFirstID
	RsKylpin
	MVirAll
	M200b
	M200c
	M500c
	M2500c
	XOff
	Voff
	SpinBullock
	BToA
	CToA
	Ax
	Ay
	Az
	BToA500c
	CToA500c
	Ax500c
	Ay500c
	Az500c
	TU
	MAcc
	MPeak
	VAcc
	VPeak
	HalfmassScale
	AccRateInst
	AccRate100Myr
	AccRateTdyn
	valNum
	RadVir
	Rad200b
	Rad200c
	Rad500c
	Rad2500c
)

func RockstarConvert(inFile, outFile string) error {
	valIdxs := make([]int, valNum)
	for i := range valIdxs { valIdxs[i] = i }
	cols, err := table.ReadTable(inFile, valIdxs, nil)
	if err != nil { return err }

	f, err := os.Create(outFile)
	if err != nil { return err }
	defer f.Close()

	err = binary.Write(f, binary.LittleEndian, int64(len(cols[0])))
	if err != nil { return err }
	for _, col := range cols {
		err := binary.Write(f, binary.LittleEndian, col)
		if err != nil { return err }
	}

	return nil
}

type idxSet struct {
	xs []float64
	idxs []int
}

func (set idxSet) Less(i, j int) bool { return set.xs[i] < set.xs[j] }
func (set idxSet) Len() int { return len(set.xs) }
func (set idxSet) Swap(i, j int) {
	set.xs[i], set.xs[j] = set.xs[j], set.xs[i]
	set.idxs[i], set.idxs[j] = set.idxs[j], set.idxs[i]
}

func idxSort(xs []float64) []int {
	xsCopy := make([]float64, len(xs))
	copy(xsCopy, xs)
	idxs := make([]int, len(xs))
	for i := range idxs { idxs[i] = i }

	set := idxSet{}
	set.idxs = idxs
	set.xs = xsCopy
	sort.Sort(set)
	return idxs
}

func RockstarConvertTopN(inFile, outFile string, n int) error {
	valIdxs := make([]int, valNum)
	for i := range valIdxs { valIdxs[i] = i }
	cols, err := table.ReadTable(inFile, valIdxs, nil)
	if err != nil { return err }

	if n > len(cols[0]) { n = len(cols[0]) }
	idxs := idxSort(cols[M200b])

	outCols := make([][]float64, len(cols))
	for i := range cols { outCols[i] = make([]float64, len(cols[0])) }

	for j := range cols {
		for i, idx := range idxs {
			outCols[j][i] = cols[j][idx]
		}
	}

	f, err := os.Create(outFile)
	if err != nil { return err }
	defer f.Close()

	err = binary.Write(f, binary.LittleEndian, int64(len(cols[0])))
	if err != nil { return err }
	for _, col := range outCols {
		err := binary.Write(f, binary.LittleEndian, col)
		if err != nil { return err }
	}

	return nil
}

func ReadBinaryRockstarVals(
	file string, cosmo *io.CosmologyHeader, valFlags ...Val,
) (ids []int, vals[][]float64, err error) {
	return readRockstarVals(file, cosmo, valFlags, binaryColGetter)
}

func ReadRockstarVals(
	file string, cosmo *io.CosmologyHeader, valFlags ...Val,
) (ids []int, vals[][]float64, err error) {
	return readRockstarVals(file, cosmo, valFlags, asciiColGetter)
}

func readRockstarVals(
	file string, cosmo *io.CosmologyHeader, valFlags []Val, getter colGetter,
) (ids []int, vals[][]float64, err error) {
	colIdxs := []int{ int(ID) }
	for _, val := range valFlags {
		switch val {
		case RadVir: colIdxs = append(colIdxs, int(MVir))
		case Rad200b: colIdxs = append(colIdxs, int(M200b))
		case Rad200c: colIdxs = append(colIdxs, int(M200c))
		case Rad500c: colIdxs = append(colIdxs, int(M500c))
		case Rad2500c: colIdxs = append(colIdxs, int(M2500c))
		default:
			colIdxs = append(colIdxs, int(val))
		}
	}

	vals, err = getter(file, colIdxs)
	if err != nil { return nil, nil, err }

	ids = make([]int, len(vals[0]))
	for i := range vals[0] {
		ids[i] = int(vals[0][i])
	}

	for i, val := range valFlags {
		switch val {
		case RadVir: RVirial.Radius(cosmo, vals[i+1], vals[i+1])
		case Rad200b: R200m.Radius(cosmo, vals[i+1], vals[i+1])
		case Rad200c: R200c.Radius(cosmo, vals[i+1], vals[i+1])
		case Rad500c: R500c.Radius(cosmo, vals[i+1], vals[i+1])
		case Rad2500c: R2500c.Radius(cosmo, vals[i+1], vals[i+1])
		}
	}

	if len(vals) == 1 {
		return ids, [][]float64{}, nil
	} else {
		return ids, vals[1:], nil
	}
}

type colGetter func(file string, colIdxs []int) ([][]float64, error)
func asciiColGetter(file string, colIdxs []int) ([][]float64, error) {
	return table.ReadTable(file, colIdxs, nil)
}
func binaryColGetter(file string, colIdxs []int) ([][]float64, error) {
	f, err := os.Open(file)
	if err != nil { return nil, err }

	n := 0
	err = binary.Read(f, binary.LittleEndian, &n)
	if err != nil { return nil, err }
	cols := make([][]float64, valNum)

	for col := range cols {
		cols[col] = make([]float64, n)
		err = binary.Read(f, binary.LittleEndian, cols[col])
		if err != nil { return nil, err }
	}
	return cols, nil
}

func init() {
	if valNum != 62 { panic("Internal gotetra setup error.") }
}
<|endoftext|>


<commit_before>package render

import "image/color"

type Request struct {
	Width       int
	Height      int
	BBOX        [4]float64
	EPSGCode    int
	Format      string
	ScaleFactor float64
	BGColor     *color.RGBA
}
<commit_msg>render: BGColor is passed as NRGBA to mapnik<commit_after>package render

import "image/color"

type Request struct {
	Width       int
	Height      int
	BBOX        [4]float64
	EPSGCode    int
	Format      string
	ScaleFactor float64
	BGColor     *color.NRGBA
}
<|endoftext|>


<commit_before>package repl

import (
	"bytes"
	"strings"
	"testing"
)

func TestStart(t *testing.T) {
	reader = strings.NewReader("a=5\nexit")
	buf := new(bytes.Buffer)
	writer = buf

	Start()

	s := buf.String()

	if !strings.HasPrefix(s, "lbd $ ") {
		t.Error("Expected output to have prefix 'lbd $ '")
	}

	lines := strings.Split(s, "\n")
	if len(lines) != 5 {
		t.Errorf("Expected to get 5 output lines, but got %d instead.", len(lines))
	}
}
<commit_msg>Use ^D in tests as well<commit_after>package repl

import (
	"bytes"
	"strings"
	"testing"
)

func TestStart(t *testing.T) {
	reader = strings.NewReader("a=5\nb=3")
	buf := new(bytes.Buffer)
	writer = buf

	Start()

	s := buf.String()

	if !strings.HasPrefix(s, "lbd $ ") {
		t.Error("Expected output to have prefix 'lbd $ '")
	}

	lines := strings.Split(s, "\n")
	if len(lines) != 9 {
		t.Errorf("Expected to get 5 output lines, but got %d instead.", len(lines))
	}
}
<|endoftext|>


<commit_before>package jsc

import (
	"testing"

	. "github.com/smartystreets/goconvey/convey"
)

func TestPost(t *testing.T) {

	Convey("Post Tests", t, func() {

	})

}
<commit_msg>Basic POST client coverage<commit_after>package jsc

import (
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/derekdowling/go-json-spec-handler"
	"github.com/derekdowling/jsh-api"
	. "github.com/smartystreets/goconvey/convey"
)

func TestPost(t *testing.T) {

	attrs := map[string]string{
		"foo": "bar",
	}

	mock := jshapi.NewMockResource("", "test", 0, attrs)
	server := httptest.NewServer(mock)
	baseURL := server.URL

	Convey("Post Tests", t, func() {
		testObject, err := jsh.NewObject("", "test", attrs)

		_, resp, err := Post(baseURL, testObject)
		So(err, ShouldBeNil)
		So(resp.StatusCode, ShouldEqual, http.StatusCreated)
	})
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"testing"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/configservice"
	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/acctest"
	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
	"github.com/hashicorp/terraform-plugin-sdk/v2/terraform"
)

func testAccConfigRemediationConfiguration_basic(t *testing.T) {
	var rc configservice.RemediationConfiguration
	resourceName := "aws_config_remediation_configuration.foo"
	rInt := acctest.RandInt()
	expectedName := fmt.Sprintf("tf-acc-test-%d", rInt)

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckConfigRemediationConfigurationDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccConfigRemediationConfigurationConfig_basic(rInt),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckConfigRemediationConfigurationExists(resourceName, &rc),
					resource.TestCheckResourceAttr(resourceName, "config_rule_name", expectedName),
					resource.TestCheckResourceAttr(resourceName, "target_id", "SSM_DOCUMENT"),
					resource.TestCheckResourceAttr(resourceName, "target_type", "AWS-PublishSNSNotification"),
					resource.TestCheckResourceAttr(resourceName, "parameters.#", "2"),
				),
			},
			{
				ResourceName:      resourceName,
				ImportState:       true,
				ImportStateVerify: true,
			},
		},
	})
}

func testAccCheckConfigRemediationConfigurationExists(n string, obj *configservice.RemediationConfiguration) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Not Found: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("No config rule ID is set")
		}

		conn := testAccProvider.Meta().(*AWSClient).configconn
		out, err := conn.DescribeRemediationConfigurations(&configservice.DescribeRemediationConfigurationsInput{
			ConfigRuleNames: []*string{aws.String(rs.Primary.Attributes["name"])},
		})
		if err != nil {
			return fmt.Errorf("Failed to describe config rule: %s", err)
		}
		if len(out.RemediationConfigurations) < 1 {
			return fmt.Errorf("No config rule found when describing %q", rs.Primary.Attributes["name"])
		}

		rc := out.RemediationConfigurations[0]
		*obj = *rc

		return nil
	}
}

func testAccCheckConfigRemediationConfigurationDestroy(s *terraform.State) error {
	conn := testAccProvider.Meta().(*AWSClient).configconn

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "aws_config_remediation_configuration" {
			continue
		}

		resp, err := conn.DescribeRemediationConfigurations(&configservice.DescribeRemediationConfigurationsInput{
			ConfigRuleNames: []*string{aws.String(rs.Primary.Attributes["name"])},
		})

		if err == nil {
			if len(resp.RemediationConfigurations) != 0 &&
				*resp.RemediationConfigurations[0].ConfigRuleName == rs.Primary.Attributes["name"] {
				return fmt.Errorf("remediation configuration(s) still exist for rule: %s", rs.Primary.Attributes["name"])
			}
		}
	}

	return nil
}

func testAccConfigRemediationConfigurationConfig_basic(randInt int) string {
	return fmt.Sprintf(`
resource "aws_config_remediation_configuration" "test" {
	config_rule_name = aws_config_config_rule.test.name

	resource_type = ""
	target_id = "SSM_DOCUMENT"
	target_type = "AWS-PublishSNSNotification"
	target_version = "1"

	parameter {
		resource_value = "Message"
	}

	parameter {
		static_value {
			key   = "TopicArn"
		f	value = aws_sns_topic.test.arn
		}
	}

	parameter {
		static_value {
			key   = "AutomationAssumeRole"
			value = aws_iam_role.test.arn
		}
	}
}

resource "aws_sns_topic" "test" {
  name = "sns_topic_name"
}

resource "aws_config_config_rule" "test" {
  name = "tf-acc-test-%d"

  source {
    owner             = "AWS"
    source_identifier = "S3_BUCKET_VERSIONING_ENABLED"
  }

  depends_on = [aws_config_configuration_recorder.test]
}

resource "aws_config_configuration_recorder" "test" {
  name     = "tf-acc-test-%[1]d"
  role_arn = aws_iam_role.r.arn
}

resource "aws_iam_role" "test" {
  name = "tf-acc-test-awsconfig-%[1]d"

  assume_role_policy = <<EOF
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Action": "sts:AssumeRole",
      "Principal": {
        "Service": "config.amazonaws.com"
      },
      "Effect": "Allow",
      "Sid": ""
    }
  ]
}
EOF
}

resource "aws_iam_role_policy" "test" {
  name = "tf-acc-test-awsconfig-%[1]d"
  role = aws_iam_role.test.id

  policy = <<EOF
{
  "Version": "2012-10-17",
  "Statement": [
    {
        "Action": "config:Put*",
        "Effect": "Allow",
        "Resource": "*"

    }
  ]
}
EOF
}
`, randInt)
}
<commit_msg>Add test for out-of-band deletion<commit_after>package aws

import (
	"fmt"
	"testing"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/configservice"
	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/acctest"
	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
	"github.com/hashicorp/terraform-plugin-sdk/v2/terraform"
)

func testAccConfigRemediationConfiguration_basic(t *testing.T) {
	var rc configservice.RemediationConfiguration
	resourceName := "aws_config_remediation_configuration.foo"
	rInt := acctest.RandInt()
	expectedName := fmt.Sprintf("tf-acc-test-%d", rInt)

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckConfigRemediationConfigurationDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccConfigRemediationConfigurationConfig_basic(rInt),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckConfigRemediationConfigurationExists(resourceName, &rc),
					resource.TestCheckResourceAttr(resourceName, "config_rule_name", expectedName),
					resource.TestCheckResourceAttr(resourceName, "target_id", "SSM_DOCUMENT"),
					resource.TestCheckResourceAttr(resourceName, "target_type", "AWS-PublishSNSNotification"),
					resource.TestCheckResourceAttr(resourceName, "parameters.#", "2"),
				),
			},
			{
				ResourceName:      resourceName,
				ImportState:       true,
				ImportStateVerify: true,
			},
		},
	})
}

func testAccConfigRemediationConfiguration_disappears(t *testing.T) {
	var rc configservice.RemediationConfiguration
	resourceName := "aws_config_remediation_configuration.test"
	rInt := acctest.RandInt()

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckConfigRemediationConfigurationDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccConfigRemediationConfigurationConfig_basic(rInt),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckConfigRemediationConfigurationExists(resourceName, &rc),
					testAccCheckResourceDisappears(testAccProvider, resourceAwsConfigRemediationConfiguration(), resourceName),
				),
				ExpectNonEmptyPlan: true,
			},
		},
	})
}

func testAccCheckConfigRemediationConfigurationExists(n string, obj *configservice.RemediationConfiguration) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Not Found: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("No config rule ID is set")
		}

		conn := testAccProvider.Meta().(*AWSClient).configconn
		out, err := conn.DescribeRemediationConfigurations(&configservice.DescribeRemediationConfigurationsInput{
			ConfigRuleNames: []*string{aws.String(rs.Primary.Attributes["name"])},
		})
		if err != nil {
			return fmt.Errorf("Failed to describe config rule: %s", err)
		}
		if len(out.RemediationConfigurations) < 1 {
			return fmt.Errorf("No config rule found when describing %q", rs.Primary.Attributes["name"])
		}

		rc := out.RemediationConfigurations[0]
		*obj = *rc

		return nil
	}
}

func testAccCheckConfigRemediationConfigurationDestroy(s *terraform.State) error {
	conn := testAccProvider.Meta().(*AWSClient).configconn

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "aws_config_remediation_configuration" {
			continue
		}

		resp, err := conn.DescribeRemediationConfigurations(&configservice.DescribeRemediationConfigurationsInput{
			ConfigRuleNames: []*string{aws.String(rs.Primary.Attributes["name"])},
		})

		if err == nil {
			if len(resp.RemediationConfigurations) != 0 &&
				*resp.RemediationConfigurations[0].ConfigRuleName == rs.Primary.Attributes["name"] {
				return fmt.Errorf("remediation configuration(s) still exist for rule: %s", rs.Primary.Attributes["name"])
			}
		}
	}

	return nil
}

func testAccConfigRemediationConfigurationConfig_basic(randInt int) string {
	return fmt.Sprintf(`
resource "aws_config_remediation_configuration" "test" {
	config_rule_name = aws_config_config_rule.test.name

	resource_type = ""
	target_id = "SSM_DOCUMENT"
	target_type = "AWS-PublishSNSNotification"
	target_version = "1"

	parameter {
		resource_value = "Message"
	}

	parameter {
		static_value {
			key   = "TopicArn"
		f	value = aws_sns_topic.test.arn
		}
	}

	parameter {
		static_value {
			key   = "AutomationAssumeRole"
			value = aws_iam_role.test.arn
		}
	}
}

resource "aws_sns_topic" "test" {
  name = "sns_topic_name"
}

resource "aws_config_config_rule" "test" {
  name = "tf-acc-test-%d"

  source {
    owner             = "AWS"
    source_identifier = "S3_BUCKET_VERSIONING_ENABLED"
  }

  depends_on = [aws_config_configuration_recorder.test]
}

resource "aws_config_configuration_recorder" "test" {
  name     = "tf-acc-test-%[1]d"
  role_arn = aws_iam_role.r.arn
}

resource "aws_iam_role" "test" {
  name = "tf-acc-test-awsconfig-%[1]d"

  assume_role_policy = <<EOF
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Action": "sts:AssumeRole",
      "Principal": {
        "Service": "config.amazonaws.com"
      },
      "Effect": "Allow",
      "Sid": ""
    }
  ]
}
EOF
}

resource "aws_iam_role_policy" "test" {
  name = "tf-acc-test-awsconfig-%[1]d"
  role = aws_iam_role.test.id

  policy = <<EOF
{
  "Version": "2012-10-17",
  "Statement": [
    {
        "Action": "config:Put*",
        "Effect": "Allow",
        "Resource": "*"

    }
  ]
}
EOF
}
`, randInt)
}
<|endoftext|>


<commit_before>/*
http://www.apache.org/licenses/LICENSE-2.0.txt

Copyright 2015 Intel Corporation

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package strategy

import (
	"errors"
	"fmt"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/intelsdi-x/snap/core"
	"github.com/intelsdi-x/snap/pkg/chrono"
)

// GlobalCacheExpiration the default time limit for which a cache entry is valid.
// A plugin can override the GlobalCacheExpiration (default).
var GlobalCacheExpiration time.Duration

var (
	cacheLog = log.WithField("_module", "routing-cache")

	ErrCacheEntryDoesNotExist = errors.New("cache entry does not exist")
)

type cachecell struct {
	time    time.Time
	metric  core.Metric
	metrics []core.Metric
	hits    uint64
	misses  uint64
}

type cache struct {
	table map[string]*cachecell
	ttl   time.Duration
}

func NewCache(expiration time.Duration) *cache {
	return &cache{
		table: make(map[string]*cachecell),
		ttl:   expiration,
	}
}

func (c *cache) get(ns string, version int) interface{} {
	var (
		cell *cachecell
		ok   bool
	)

	key := fmt.Sprintf("%v:%v", ns, version)
	if cell, ok = c.table[key]; ok && chrono.Chrono.Now().Sub(cell.time) < c.ttl {
		cell.hits++
		cacheLog.WithFields(log.Fields{
			"namespace": key,
			"hits":      cell.hits,
			"misses":    cell.misses,
		}).Debug(fmt.Sprintf("cache hit [%s]", key))
		if cell.metric != nil {
			return cell.metric
		}
		return cell.metrics
	}
	if !ok {
		c.table[key] = &cachecell{
			time:    time.Time{},
			metrics: nil,
		}
	}
	c.table[key].misses++
	cacheLog.WithFields(log.Fields{
		"namespace": key,
		"hits":      c.table[key].hits,
		"misses":    c.table[key].misses,
	}).Debug(fmt.Sprintf("cache miss [%s]", key))
	return nil
}

func (c *cache) put(ns string, version int, m interface{}) {
	key := fmt.Sprintf("%v:%v", ns, version)
	switch metric := m.(type) {
	case core.Metric:
		if _, ok := c.table[key]; ok {
			c.table[key].time = chrono.Chrono.Now()
			c.table[key].metric = metric
		} else {
			log.Errorf("%+v", c.table)
			c.table[key] = &cachecell{
				time:   chrono.Chrono.Now(),
				metric: metric,
			}
		}
	case []core.Metric:
		if _, ok := c.table[key]; ok {
			c.table[key].time = time.Now()
			c.table[key].metrics = metric
		} else {
			c.table[key] = &cachecell{
				time:    chrono.Chrono.Now(),
				metrics: metric,
			}
		}
	default:
		cacheLog.WithFields(log.Fields{
			"namespace": key,
			"_block":    "put",
		}).Error("unsupported type")
	}
}

func (c *cache) checkCache(mts []core.Metric) (metricsToCollect []core.Metric, fromCache []core.Metric) {
	for _, mt := range mts {
		if m := c.get(core.JoinNamespace(mt.Namespace()), mt.Version()); m != nil {
			switch metric := m.(type) {
			case core.Metric:
				fromCache = append(fromCache, metric)
			case []core.Metric:
				for _, met := range metric {
					fromCache = append(fromCache, met)
				}
			default:
				cacheLog.WithFields(log.Fields{
					"_block": "checkCache",
				}).Error("unsupported type found in the cache")
			}
		} else {
			metricsToCollect = append(metricsToCollect, mt)
		}
	}
	return metricsToCollect, fromCache
}

func (c *cache) updateCache(mts []core.Metric) {
	results := []core.Metric{}
	dc := map[string][]core.Metric{}
	for _, mt := range mts {
		if mt.Labels() == nil {
			// cache the individual metric
			c.put(core.JoinNamespace(mt.Namespace()), mt.Version(), mt)
		} else {
			// collect the dynamic query results so we can cache
			ns := make([]string, len(mt.Namespace()))
			copy(ns, mt.Namespace())
			for _, label := range mt.Labels() {
				ns[label.Index] = "*"
			}
			if _, ok := dc[core.JoinNamespace(ns)]; !ok {
				dc[core.JoinNamespace(ns)] = []core.Metric{}
			}
			dc[core.JoinNamespace(ns)] = append(dc[core.JoinNamespace(ns)], mt)
			c.put(core.JoinNamespace(ns), mt.Version(), dc[core.JoinNamespace(ns)])
		}
		results = append(results, mt)
	}
}

func (c *cache) allCacheHits() uint64 {
	var hits uint64
	for _, v := range c.table {
		hits += v.hits
	}
	return hits
}

func (c *cache) allCacheMisses() uint64 {
	var misses uint64
	for _, v := range c.table {
		misses += v.misses
	}
	return misses
}

func (c *cache) cacheHits(ns string, version int) (uint64, error) {
	key := fmt.Sprintf("%v:%v", ns, version)
	if v, ok := c.table[key]; ok {
		return v.hits, nil
	}
	return 0, ErrCacheEntryDoesNotExist
}

func (c *cache) cacheMisses(ns string, version int) (uint64, error) {
	key := fmt.Sprintf("%v:%v", ns, version)
	if v, ok := c.table[key]; ok {
		return v.misses, nil
	}
	return 0, ErrCacheEntryDoesNotExist
}
<commit_msg>remove superfluous error logging in cache.go<commit_after>/*
http://www.apache.org/licenses/LICENSE-2.0.txt

Copyright 2015 Intel Corporation

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package strategy

import (
	"errors"
	"fmt"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/intelsdi-x/snap/core"
	"github.com/intelsdi-x/snap/pkg/chrono"
)

// GlobalCacheExpiration the default time limit for which a cache entry is valid.
// A plugin can override the GlobalCacheExpiration (default).
var GlobalCacheExpiration time.Duration

var (
	cacheLog = log.WithField("_module", "routing-cache")

	ErrCacheEntryDoesNotExist = errors.New("cache entry does not exist")
)

type cachecell struct {
	time    time.Time
	metric  core.Metric
	metrics []core.Metric
	hits    uint64
	misses  uint64
}

type cache struct {
	table map[string]*cachecell
	ttl   time.Duration
}

func NewCache(expiration time.Duration) *cache {
	return &cache{
		table: make(map[string]*cachecell),
		ttl:   expiration,
	}
}

func (c *cache) get(ns string, version int) interface{} {
	var (
		cell *cachecell
		ok   bool
	)

	key := fmt.Sprintf("%v:%v", ns, version)
	if cell, ok = c.table[key]; ok && chrono.Chrono.Now().Sub(cell.time) < c.ttl {
		cell.hits++
		cacheLog.WithFields(log.Fields{
			"namespace": key,
			"hits":      cell.hits,
			"misses":    cell.misses,
		}).Debug(fmt.Sprintf("cache hit [%s]", key))
		if cell.metric != nil {
			return cell.metric
		}
		return cell.metrics
	}
	if !ok {
		c.table[key] = &cachecell{
			time:    time.Time{},
			metrics: nil,
		}
	}
	c.table[key].misses++
	cacheLog.WithFields(log.Fields{
		"namespace": key,
		"hits":      c.table[key].hits,
		"misses":    c.table[key].misses,
	}).Debug(fmt.Sprintf("cache miss [%s]", key))
	return nil
}

func (c *cache) put(ns string, version int, m interface{}) {
	key := fmt.Sprintf("%v:%v", ns, version)
	switch metric := m.(type) {
	case core.Metric:
		if _, ok := c.table[key]; ok {
			c.table[key].time = chrono.Chrono.Now()
			c.table[key].metric = metric
		} else {
			c.table[key] = &cachecell{
				time:   chrono.Chrono.Now(),
				metric: metric,
			}
		}
	case []core.Metric:
		if _, ok := c.table[key]; ok {
			c.table[key].time = time.Now()
			c.table[key].metrics = metric
		} else {
			c.table[key] = &cachecell{
				time:    chrono.Chrono.Now(),
				metrics: metric,
			}
		}
	default:
		cacheLog.WithFields(log.Fields{
			"namespace": key,
			"_block":    "put",
		}).Error("unsupported type")
	}
}

func (c *cache) checkCache(mts []core.Metric) (metricsToCollect []core.Metric, fromCache []core.Metric) {
	for _, mt := range mts {
		if m := c.get(core.JoinNamespace(mt.Namespace()), mt.Version()); m != nil {
			switch metric := m.(type) {
			case core.Metric:
				fromCache = append(fromCache, metric)
			case []core.Metric:
				for _, met := range metric {
					fromCache = append(fromCache, met)
				}
			default:
				cacheLog.WithFields(log.Fields{
					"_block": "checkCache",
				}).Error("unsupported type found in the cache")
			}
		} else {
			metricsToCollect = append(metricsToCollect, mt)
		}
	}
	return metricsToCollect, fromCache
}

func (c *cache) updateCache(mts []core.Metric) {
	results := []core.Metric{}
	dc := map[string][]core.Metric{}
	for _, mt := range mts {
		if mt.Labels() == nil {
			// cache the individual metric
			c.put(core.JoinNamespace(mt.Namespace()), mt.Version(), mt)
		} else {
			// collect the dynamic query results so we can cache
			ns := make([]string, len(mt.Namespace()))
			copy(ns, mt.Namespace())
			for _, label := range mt.Labels() {
				ns[label.Index] = "*"
			}
			if _, ok := dc[core.JoinNamespace(ns)]; !ok {
				dc[core.JoinNamespace(ns)] = []core.Metric{}
			}
			dc[core.JoinNamespace(ns)] = append(dc[core.JoinNamespace(ns)], mt)
			c.put(core.JoinNamespace(ns), mt.Version(), dc[core.JoinNamespace(ns)])
		}
		results = append(results, mt)
	}
}

func (c *cache) allCacheHits() uint64 {
	var hits uint64
	for _, v := range c.table {
		hits += v.hits
	}
	return hits
}

func (c *cache) allCacheMisses() uint64 {
	var misses uint64
	for _, v := range c.table {
		misses += v.misses
	}
	return misses
}

func (c *cache) cacheHits(ns string, version int) (uint64, error) {
	key := fmt.Sprintf("%v:%v", ns, version)
	if v, ok := c.table[key]; ok {
		return v.hits, nil
	}
	return 0, ErrCacheEntryDoesNotExist
}

func (c *cache) cacheMisses(ns string, version int) (uint64, error) {
	key := fmt.Sprintf("%v:%v", ns, version)
	if v, ok := c.table[key]; ok {
		return v.misses, nil
	}
	return 0, ErrCacheEntryDoesNotExist
}
<|endoftext|>


<commit_before>package context

import (
	"fmt"
	"io"
)

// Logger defines the expected methods to be provided by logging infrastructure
type Logger interface {
	Debug(parameters ...interface{})
	Info(parameters ...interface{})
	Warn(parameters ...interface{})
	Error(parameters ...interface{})
}

// ErrorWriter adapts a context.Logger so that all output from Write() goes
// to Error(...).  This is useful for HTTP error logs.
type ErrorWriter struct {
	Logger
}

func (e *ErrorWriter) Write(data []byte) (int, error) {
	e.Error(string(data))
	return len(data), nil
}

// DefaultLogger embeds an io.Writer and sends all output to that writer.  This type
// is primarily intended for testing.
type DefaultLogger struct {
	io.Writer
}

// doWrite mimics the behavior of most logging frameworks, albeit with a simpler implementation.
func (logger DefaultLogger) doWrite(level string, parameters ...interface{}) {
	_, err := logger.Write(
		[]byte(fmt.Sprintf("[%-5.5s] ", level)),
	)

	if err != nil {
		panic(err)
	}

	if len(parameters) > 0 {
		switch head := parameters[0].(type) {
		case fmt.Stringer:
			_, err = logger.Write(
				[]byte(fmt.Sprintf(head.String(), parameters[1:]...)),
			)

		case string:
			_, err = logger.Write(
				[]byte(fmt.Sprintf(head, parameters[1:]...)),
			)
		}

		if err != nil {
			panic(err)
		}

		_, err = logger.Write([]byte("\n"))
		if err != nil {
			panic(err)
		}
	}
}

func (logger DefaultLogger) Debug(parameters ...interface{}) {
	logger.doWrite("DEBUG", parameters...)
}

func (logger DefaultLogger) Info(parameters ...interface{}) {
	logger.doWrite("INFO", parameters...)
}

func (logger DefaultLogger) Warn(parameters ...interface{}) {
	logger.doWrite("WARN", parameters...)
}

func (logger DefaultLogger) Error(parameters ...interface{}) {
	logger.doWrite("ERROR", parameters...)
}
<commit_msg>Added a standard logger for connection states<commit_after>package context

import (
	"fmt"
	"io"
	"net"
	"net/http"
)

// Logger defines the expected methods to be provided by logging infrastructure
type Logger interface {
	Debug(parameters ...interface{})
	Info(parameters ...interface{})
	Warn(parameters ...interface{})
	Error(parameters ...interface{})
}

// ErrorWriter adapts a context.Logger so that all output from Write() goes
// to Error(...).  This is useful for HTTP error logs.
type ErrorWriter struct {
	Logger
}

func (e *ErrorWriter) Write(data []byte) (int, error) {
	e.Error(string(data))
	return len(data), nil
}

// DefaultLogger embeds an io.Writer and sends all output to that writer.  This type
// is primarily intended for testing.
type DefaultLogger struct {
	io.Writer
}

// doWrite mimics the behavior of most logging frameworks, albeit with a simpler implementation.
func (logger DefaultLogger) doWrite(level string, parameters ...interface{}) {
	_, err := logger.Write(
		[]byte(fmt.Sprintf("[%-5.5s] ", level)),
	)

	if err != nil {
		panic(err)
	}

	if len(parameters) > 0 {
		switch head := parameters[0].(type) {
		case fmt.Stringer:
			_, err = logger.Write(
				[]byte(fmt.Sprintf(head.String(), parameters[1:]...)),
			)

		case string:
			_, err = logger.Write(
				[]byte(fmt.Sprintf(head, parameters[1:]...)),
			)
		}

		if err != nil {
			panic(err)
		}

		_, err = logger.Write([]byte("\n"))
		if err != nil {
			panic(err)
		}
	}
}

func (logger DefaultLogger) Debug(parameters ...interface{}) {
	logger.doWrite("DEBUG", parameters...)
}

func (logger DefaultLogger) Info(parameters ...interface{}) {
	logger.doWrite("INFO", parameters...)
}

func (logger DefaultLogger) Warn(parameters ...interface{}) {
	logger.doWrite("WARN", parameters...)
}

func (logger DefaultLogger) Error(parameters ...interface{}) {
	logger.doWrite("ERROR", parameters...)
}

// NewConnectionStateLogger produces a function appropriate for http.Server.ConnState.
// The returned function will log debug statements for each state change.
func NewConnectionStateLogger(serverName string, logger Logger) func(net.Conn, http.ConnState) {
	return func(connection net.Conn, connectionState http.ConnState) {
		logger.Debug("[%s] [%s] -> %s", serverName, connection.LocalAddr().String, connectionState)
	}
}
<|endoftext|>