<commit_before>// Copyright (c) 2014-2015 The Notify Authors. All rights reserved.
// Use of this source code is governed by the MIT license that can be
// found in the LICENSE file.

package notify

import "sync"

// nonrecursiveTree TODO(rjeczalik)
type nonrecursiveTree struct {
	rw   sync.RWMutex // protects root
	root root
	w    watcher
	c    chan EventInfo
	rec  chan EventInfo
}

// newNonrecursiveTree TODO(rjeczalik)
func newNonrecursiveTree(w watcher, c, rec chan EventInfo) *nonrecursiveTree {
	if rec == nil {
		rec = make(chan EventInfo, buffer)
	}
	t := &nonrecursiveTree{
		root: root{nd: newnode("")},
		w:    w,
		c:    c,
		rec:  rec,
	}
	go t.dispatch(c)
	go t.internal(rec)
	return t
}

// dispatch TODO(rjeczalik)
func (t *nonrecursiveTree) dispatch(c <-chan EventInfo) {
	for ei := range c {
		dbgprintf("dispatching %v on %q", ei.Event(), ei.Path())
		go func(ei EventInfo) {
			var nd node
			var isrec bool
			dir, base := split(ei.Path())
			fn := func(it node, isbase bool) error {
				isrec = isrec || it.Watch.IsRecursive()
				if isbase {
					nd = it
				} else {
					it.Watch.Dispatch(ei, recursive)
				}
				return nil
			}
			t.rw.RLock()
			// Notify recursive watchpoints found on the path.
			if err := t.root.WalkPath(dir, fn); err != nil {
				dbgprint("dispatch did not reach leaf:", err)
				t.rw.RUnlock()
				return
			}
			// Notify parent watchpoint.
			nd.Watch.Dispatch(ei, 0)
			isrec = isrec || nd.Watch.IsRecursive()
			// If leaf watchpoint exists, notify it.
			if nd, ok := nd.Child[base]; ok {
				isrec = isrec || nd.Watch.IsRecursive()
				nd.Watch.Dispatch(ei, 0)
			}
			t.rw.RUnlock()
			// If the event describes newly leaf directory created within
			if !isrec || ei.Event() != Create {
				return
			}
			if ok, err := ei.(isDirer).isDir(); !ok || err != nil {
				return
			}
			t.rec <- ei
		}(ei)
	}
}

// internal TODO(rjeczalik)
func (t *nonrecursiveTree) internal(rec <-chan EventInfo) {
	for ei := range rec {
		var nd node
		var eset = internal
		t.rw.Lock()
		t.root.WalkPath(ei.Path(), func(it node, _ bool) error {
			if e := it.Watch[t.rec]; e != 0 && e > eset {
				eset = e
			}
			nd = it
			return nil
		})
		if eset == internal {
			t.rw.Unlock()
			continue
		}
		err := nd.Add(ei.Path()).AddDir(t.recFunc(eset))
		t.rw.Unlock()
		if err != nil {
			dbgprintf("internal(%p) error: %v", rec, err)
		}
	}
}

// watchAdd TODO(rjeczalik)
func (t *nonrecursiveTree) watchAdd(nd node, c chan<- EventInfo, e Event) eventDiff {
	if e&recursive != 0 {
		diff := nd.Watch.Add(t.rec, e|Create|omit)
		nd.Watch.Add(c, e)
		return diff
	}
	return nd.Watch.Add(c, e)
}

// watchDelMin TODO(rjeczalik)
func (t *nonrecursiveTree) watchDelMin(min Event, nd node, c chan<- EventInfo, e Event) eventDiff {
	old, ok := nd.Watch[t.rec]
	if ok {
		nd.Watch[t.rec] = min
	}
	diff := nd.Watch.Del(c, e)
	if ok {
		switch old &^= diff[0] &^ diff[1]; {
		case old|internal == internal:
			delete(nd.Watch, t.rec)
			if set, ok := nd.Watch[nil]; ok && len(nd.Watch) == 1 && set == 0 {
				delete(nd.Watch, nil)
			}
		default:
			nd.Watch.Add(t.rec, old|Create)
			switch {
			case diff == none:
			case diff[1]|Create == diff[0]:
				diff = none
			default:
				diff[1] |= Create
			}
		}
	}
	return diff
}

// watchDel TODO(rjeczalik)
func (t *nonrecursiveTree) watchDel(nd node, c chan<- EventInfo, e Event) eventDiff {
	return t.watchDelMin(0, nd, c, e)
}

// Watch TODO(rjeczalik)
func (t *nonrecursiveTree) Watch(path string, c chan<- EventInfo, events ...Event) error {
	if c == nil {
		panic("notify: Watch using nil channel")
	}
	// Expanding with empty event set is a nop.
	if len(events) == 0 {
		return nil
	}
	path, isrec, err := cleanpath(path)
	if err != nil {
		return err
	}
	eset := joinevents(events)
	t.rw.Lock()
	defer t.rw.Unlock()
	nd := t.root.Add(path)
	if isrec {
		return t.watchrec(nd, c, eset|recursive)
	}
	return t.watch(nd, c, eset)
}

func (t *nonrecursiveTree) watch(nd node, c chan<- EventInfo, e Event) (err error) {
	diff := nd.Watch.Add(c, e)
	switch {
	case diff == none:
		return nil
	case diff[1] == 0:
		// TODO(rjeczalik): cleanup this panic after implementation is stable
		panic("eset is empty: " + nd.Name)
	case diff[0] == 0:
		err = t.w.Watch(nd.Name, diff[1])
	default:
		err = t.w.Rewatch(nd.Name, diff[0], diff[1])
	}
	if err != nil {
		nd.Watch.Del(c, diff.Event())
		return err
	}
	return nil
}

func (t *nonrecursiveTree) recFunc(e Event) walkFunc {
	return func(nd node) error {
		switch diff := nd.Watch.Add(t.rec, e|omit|Create); {
		case diff == none:
		case diff[1] == 0:
			// TODO(rjeczalik): cleanup this panic after implementation is stable
			panic("eset is empty: " + nd.Name)
		case diff[0] == 0:
			t.w.Watch(nd.Name, diff[1])
		default:
			t.w.Rewatch(nd.Name, diff[0], diff[1])
		}
		return nil
	}
}

func (t *nonrecursiveTree) watchrec(nd node, c chan<- EventInfo, e Event) error {
	var traverse func(walkFunc) error
	// Non-recursive tree listens on Create event for every recursive
	// watchpoint in order to automagically set a watch for every
	// created directory.
	switch diff := nd.Watch.dryAdd(t.rec, e|Create); {
	case diff == none:
		t.watchAdd(nd, c, e)
		nd.Watch.Add(t.rec, e|omit|Create)
		return nil
	case diff[1] == 0:
		// TODO(rjeczalik): cleanup this panic after implementation is stable
		panic("eset is empty: " + nd.Name)
	case diff[0] == 0:
		// TODO(rjeczalik): BFS into directories and skip subtree as soon as first
		// recursive watchpoint is encountered.
		traverse = nd.AddDir
	default:
		traverse = nd.Walk
	}
	// TODO(rjeczalik): account every path that failed to be (re)watched
	// and retry.
	if err := traverse(t.recFunc(e)); err != nil {
		return err
	}
	t.watchAdd(nd, c, e)
	return nil
}

type walkWatchpointFunc func(Event, node) error

func (t *nonrecursiveTree) walkWatchpoint(nd node, fn walkWatchpointFunc) error {
	type minode struct {
		min Event
		nd  node
	}
	mnd := minode{nd: nd}
	stack := []minode{mnd}
Traverse:
	for n := len(stack); n != 0; n = len(stack) {
		mnd, stack = stack[n-1], stack[:n-1]
		// There must be no recursive watchpoints if the node has no watchpoints
		// itself (every node in subtree rooted at recursive watchpoints must
		// have at least nil (total) and t.rec watchpoints).
		if len(mnd.nd.Watch) != 0 {
			switch err := fn(mnd.min, mnd.nd); err {
			case nil:
			case errSkip:
				continue Traverse
			default:
				return err
			}
		}
		for _, nd := range mnd.nd.Child {
			stack = append(stack, minode{mnd.nd.Watch[t.rec], nd})
		}
	}
	return nil
}

// Stop TODO(rjeczalik)
func (t *nonrecursiveTree) Stop(c chan<- EventInfo) {
	fn := func(min Event, nd node) error {
		// TODO(rjeczalik): aggregate watcher errors and retry; in worst case
		// forward to the user.
		switch diff := t.watchDelMin(min, nd, c, all); {
		case diff == none:
			return nil
		case diff[1] == 0:
			t.w.Unwatch(nd.Name)
		default:
			t.w.Rewatch(nd.Name, diff[0], diff[1])
		}
		return nil
	}
	t.rw.Lock()
	err := t.walkWatchpoint(t.root.nd, fn) // TODO(rjeczalik): store max root per c
	t.rw.Unlock()
	dbgprintf("Stop(%p) error: %v\n", c, err)
}

// Close TODO(rjeczalik)
func (t *nonrecursiveTree) Close() error {
	err := t.w.Close()
	close(t.c)
	return err
}
<commit_msg>Handle errors that could be produced in recFunc closure (fixes #76)<commit_after>// Copyright (c) 2014-2015 The Notify Authors. All rights reserved.
// Use of this source code is governed by the MIT license that can be
// found in the LICENSE file.

package notify

import "sync"

// nonrecursiveTree TODO(rjeczalik)
type nonrecursiveTree struct {
	rw   sync.RWMutex // protects root
	root root
	w    watcher
	c    chan EventInfo
	rec  chan EventInfo
}

// newNonrecursiveTree TODO(rjeczalik)
func newNonrecursiveTree(w watcher, c, rec chan EventInfo) *nonrecursiveTree {
	if rec == nil {
		rec = make(chan EventInfo, buffer)
	}
	t := &nonrecursiveTree{
		root: root{nd: newnode("")},
		w:    w,
		c:    c,
		rec:  rec,
	}
	go t.dispatch(c)
	go t.internal(rec)
	return t
}

// dispatch TODO(rjeczalik)
func (t *nonrecursiveTree) dispatch(c <-chan EventInfo) {
	for ei := range c {
		dbgprintf("dispatching %v on %q", ei.Event(), ei.Path())
		go func(ei EventInfo) {
			var nd node
			var isrec bool
			dir, base := split(ei.Path())
			fn := func(it node, isbase bool) error {
				isrec = isrec || it.Watch.IsRecursive()
				if isbase {
					nd = it
				} else {
					it.Watch.Dispatch(ei, recursive)
				}
				return nil
			}
			t.rw.RLock()
			// Notify recursive watchpoints found on the path.
			if err := t.root.WalkPath(dir, fn); err != nil {
				dbgprint("dispatch did not reach leaf:", err)
				t.rw.RUnlock()
				return
			}
			// Notify parent watchpoint.
			nd.Watch.Dispatch(ei, 0)
			isrec = isrec || nd.Watch.IsRecursive()
			// If leaf watchpoint exists, notify it.
			if nd, ok := nd.Child[base]; ok {
				isrec = isrec || nd.Watch.IsRecursive()
				nd.Watch.Dispatch(ei, 0)
			}
			t.rw.RUnlock()
			// If the event describes newly leaf directory created within
			if !isrec || ei.Event() != Create {
				return
			}
			if ok, err := ei.(isDirer).isDir(); !ok || err != nil {
				return
			}
			t.rec <- ei
		}(ei)
	}
}

// internal TODO(rjeczalik)
func (t *nonrecursiveTree) internal(rec <-chan EventInfo) {
	for ei := range rec {
		var nd node
		var eset = internal
		t.rw.Lock()
		t.root.WalkPath(ei.Path(), func(it node, _ bool) error {
			if e := it.Watch[t.rec]; e != 0 && e > eset {
				eset = e
			}
			nd = it
			return nil
		})
		if eset == internal {
			t.rw.Unlock()
			continue
		}
		err := nd.Add(ei.Path()).AddDir(t.recFunc(eset))
		t.rw.Unlock()
		if err != nil {
			dbgprintf("internal(%p) error: %v", rec, err)
		}
	}
}

// watchAdd TODO(rjeczalik)
func (t *nonrecursiveTree) watchAdd(nd node, c chan<- EventInfo, e Event) eventDiff {
	if e&recursive != 0 {
		diff := nd.Watch.Add(t.rec, e|Create|omit)
		nd.Watch.Add(c, e)
		return diff
	}
	return nd.Watch.Add(c, e)
}

// watchDelMin TODO(rjeczalik)
func (t *nonrecursiveTree) watchDelMin(min Event, nd node, c chan<- EventInfo, e Event) eventDiff {
	old, ok := nd.Watch[t.rec]
	if ok {
		nd.Watch[t.rec] = min
	}
	diff := nd.Watch.Del(c, e)
	if ok {
		switch old &^= diff[0] &^ diff[1]; {
		case old|internal == internal:
			delete(nd.Watch, t.rec)
			if set, ok := nd.Watch[nil]; ok && len(nd.Watch) == 1 && set == 0 {
				delete(nd.Watch, nil)
			}
		default:
			nd.Watch.Add(t.rec, old|Create)
			switch {
			case diff == none:
			case diff[1]|Create == diff[0]:
				diff = none
			default:
				diff[1] |= Create
			}
		}
	}
	return diff
}

// watchDel TODO(rjeczalik)
func (t *nonrecursiveTree) watchDel(nd node, c chan<- EventInfo, e Event) eventDiff {
	return t.watchDelMin(0, nd, c, e)
}

// Watch TODO(rjeczalik)
func (t *nonrecursiveTree) Watch(path string, c chan<- EventInfo, events ...Event) error {
	if c == nil {
		panic("notify: Watch using nil channel")
	}
	// Expanding with empty event set is a nop.
	if len(events) == 0 {
		return nil
	}
	path, isrec, err := cleanpath(path)
	if err != nil {
		return err
	}
	eset := joinevents(events)
	t.rw.Lock()
	defer t.rw.Unlock()
	nd := t.root.Add(path)
	if isrec {
		return t.watchrec(nd, c, eset|recursive)
	}
	return t.watch(nd, c, eset)
}

func (t *nonrecursiveTree) watch(nd node, c chan<- EventInfo, e Event) (err error) {
	diff := nd.Watch.Add(c, e)
	switch {
	case diff == none:
		return nil
	case diff[1] == 0:
		// TODO(rjeczalik): cleanup this panic after implementation is stable
		panic("eset is empty: " + nd.Name)
	case diff[0] == 0:
		err = t.w.Watch(nd.Name, diff[1])
	default:
		err = t.w.Rewatch(nd.Name, diff[0], diff[1])
	}
	if err != nil {
		nd.Watch.Del(c, diff.Event())
		return err
	}
	return nil
}

func (t *nonrecursiveTree) recFunc(e Event) walkFunc {
	return func(nd node) (err error) {
		switch diff := nd.Watch.Add(t.rec, e|omit|Create); {
		case diff == none:
		case diff[1] == 0:
			// TODO(rjeczalik): cleanup this panic after implementation is stable
			panic("eset is empty: " + nd.Name)
		case diff[0] == 0:
			err = t.w.Watch(nd.Name, diff[1])
		default:
			err = t.w.Rewatch(nd.Name, diff[0], diff[1])
		}
		return
	}
}

func (t *nonrecursiveTree) watchrec(nd node, c chan<- EventInfo, e Event) error {
	var traverse func(walkFunc) error
	// Non-recursive tree listens on Create event for every recursive
	// watchpoint in order to automagically set a watch for every
	// created directory.
	switch diff := nd.Watch.dryAdd(t.rec, e|Create); {
	case diff == none:
		t.watchAdd(nd, c, e)
		nd.Watch.Add(t.rec, e|omit|Create)
		return nil
	case diff[1] == 0:
		// TODO(rjeczalik): cleanup this panic after implementation is stable
		panic("eset is empty: " + nd.Name)
	case diff[0] == 0:
		// TODO(rjeczalik): BFS into directories and skip subtree as soon as first
		// recursive watchpoint is encountered.
		traverse = nd.AddDir
	default:
		traverse = nd.Walk
	}
	// TODO(rjeczalik): account every path that failed to be (re)watched
	// and retry.
	if err := traverse(t.recFunc(e)); err != nil {
		return err
	}
	t.watchAdd(nd, c, e)
	return nil
}

type walkWatchpointFunc func(Event, node) error

func (t *nonrecursiveTree) walkWatchpoint(nd node, fn walkWatchpointFunc) error {
	type minode struct {
		min Event
		nd  node
	}
	mnd := minode{nd: nd}
	stack := []minode{mnd}
Traverse:
	for n := len(stack); n != 0; n = len(stack) {
		mnd, stack = stack[n-1], stack[:n-1]
		// There must be no recursive watchpoints if the node has no watchpoints
		// itself (every node in subtree rooted at recursive watchpoints must
		// have at least nil (total) and t.rec watchpoints).
		if len(mnd.nd.Watch) != 0 {
			switch err := fn(mnd.min, mnd.nd); err {
			case nil:
			case errSkip:
				continue Traverse
			default:
				return err
			}
		}
		for _, nd := range mnd.nd.Child {
			stack = append(stack, minode{mnd.nd.Watch[t.rec], nd})
		}
	}
	return nil
}

// Stop TODO(rjeczalik)
func (t *nonrecursiveTree) Stop(c chan<- EventInfo) {
	fn := func(min Event, nd node) error {
		// TODO(rjeczalik): aggregate watcher errors and retry; in worst case
		// forward to the user.
		switch diff := t.watchDelMin(min, nd, c, all); {
		case diff == none:
			return nil
		case diff[1] == 0:
			t.w.Unwatch(nd.Name)
		default:
			t.w.Rewatch(nd.Name, diff[0], diff[1])
		}
		return nil
	}
	t.rw.Lock()
	err := t.walkWatchpoint(t.root.nd, fn) // TODO(rjeczalik): store max root per c
	t.rw.Unlock()
	dbgprintf("Stop(%p) error: %v\n", c, err)
}

// Close TODO(rjeczalik)
func (t *nonrecursiveTree) Close() error {
	err := t.w.Close()
	close(t.c)
	return err
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mungers

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
	"sync/atomic"
	"time"

	"github.com/golang/glog"
	githubapi "github.com/google/go-github/github"
	"k8s.io/contrib/mungegithub/github"
)

// xref k8s.io/test-infra/prow/cmd/deck/jobs.go
type prowJob struct {
	Type    string `json:"type"`
	Repo    string `json:"repo"`
	Refs    string `json:"refs"`
	State   string `json:"state"`
	Context string `json:"context"`
}

// getSuccessfulBatchJobs reads test results from Prow and returns
// all batch jobs that succeeded for the current repo.
func getSuccessfulBatchJobs(repo, url string) ([]prowJob, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)

	allJobs := []prowJob{}
	err = json.Unmarshal(body, &allJobs)
	if err != nil {
		return nil, err
	}

	jobs := []prowJob{}
	for _, job := range allJobs {
		if job.Repo == repo && job.Type == "batch" && job.State == "success" {
			jobs = append(jobs, job)
		}
	}
	return jobs, nil
}

type batchPull struct {
	Number int
	Sha    string
}

// Batch represents a specific merge state:
// a base branch and SHA, and the SHAs of each PR merged into it.
type Batch struct {
	BaseName string
	BaseSha  string
	Pulls    []batchPull
}

func (b *Batch) String() string {
	out := b.BaseName + ":" + b.BaseSha
	for _, pull := range b.Pulls {
		out += "," + strconv.Itoa(pull.Number) + ":" + pull.Sha
	}
	return out
}

// batchRefToBatch parses a string into a Batch.
// The input is a comma-separated list of colon-separated ref/sha pairs,
// like "master:abcdef0,123:f00d,456:f00f".
func batchRefToBatch(batchRef string) (Batch, error) {
	batch := Batch{}
	for i, ref := range strings.Split(batchRef, ",") {
		parts := strings.Split(ref, ":")
		if len(parts) != 2 {
			return Batch{}, errors.New("bad batchref: " + batchRef)
		}
		if i == 0 {
			batch.BaseName = parts[0]
			batch.BaseSha = parts[1]
		} else {
			num, err := strconv.ParseInt(parts[0], 10, 32)
			if err != nil {
				return Batch{}, fmt.Errorf("bad batchref: %s (%v)", batchRef, err)
			}
			batch.Pulls = append(batch.Pulls, batchPull{int(num), parts[1]})
		}
	}
	return batch, nil
}

// getCompleteBatches returns a list of Batches that passed all
// required tests.
func (sq *SubmitQueue) getCompleteBatches(jobs []prowJob) []Batch {
	// for each batch specifier, a set of successful contexts
	batchContexts := make(map[string]map[string]interface{})
	for _, job := range jobs {
		if batchContexts[job.Refs] == nil {
			batchContexts[job.Refs] = make(map[string]interface{})
		}
		batchContexts[job.Refs][job.Context] = nil
	}
	batches := []Batch{}
	for batchRef, contexts := range batchContexts {
		match := true
		// Did this succeed in all the contexts we want?
		for _, ctx := range sq.RequiredStatusContexts {
			if _, ok := contexts[ctx]; !ok {
				match = false
			}
		}
		for _, ctx := range sq.RequiredRetestContexts {
			if _, ok := contexts[ctx]; !ok {
				match = false
			}
		}
		if match {
			batch, err := batchRefToBatch(batchRef)
			if err != nil {
				continue
			}
			batches = append(batches, batch)
		}
	}
	return batches
}

// batchIntersectsQueue returns whether at least one PR in the batch is queued.
func (sq *SubmitQueue) batchIntersectsQueue(batch Batch) bool {
	sq.Lock()
	defer sq.Unlock()
	for _, pull := range batch.Pulls {
		if _, ok := sq.githubE2EQueue[pull.Number]; ok {
			return true
		}
	}
	return false
}

// matchesCommit determines if the batch can be merged given some commits.
// That is, does it contain exactly:
// 1) the batch's BaseSha
// 2) (optional) merge commits for PRs in the batch
// 3) any merged PRs in the batch are sequential from the beginning
// The return value is the number of PRs already merged, and any errors.
func (b *Batch) matchesCommits(commits []*githubapi.RepositoryCommit) (int, error) {
	if len(commits) == 0 {
		return 0, errors.New("no commits")
	}

	shaToPR := make(map[string]int)

	for _, pull := range b.Pulls {
		shaToPR[pull.Sha] = pull.Number
	}

	matchedPRs := []int{}

	// convert the list of commits into a DAG for easy following
	dag := make(map[string]*githubapi.RepositoryCommit)
	for _, commit := range commits {
		dag[*commit.SHA] = commit
	}

	ref := *commits[0].SHA
	for {
		if ref == b.BaseSha {
			break // found the base ref (condition #1)
		}
		commit, ok := dag[ref]
		if !ok {
			return 0, errors.New("ran out of commits (missing ref " + ref + ")")
		}
		if len(commit.Parents) == 2 && commit.Message != nil &&
			strings.HasPrefix(*commit.Message, "Merge") {
			// looks like a merge commit!

			// first parent is the normal branch
			ref = *commit.Parents[0].SHA
			// second parent is the PR
			pr, ok := shaToPR[*commit.Parents[1].SHA]
			if !ok {
				return 0, errors.New("Merge of something not in batch")
			}
			matchedPRs = append(matchedPRs, pr)
		} else {
			return 0, errors.New("Unknown non-merge commit " + ref)
		}
	}

	// Now, ensure that the merged PRs are ordered correctly.
	for i, pr := range matchedPRs {
		if b.Pulls[len(matchedPRs)-1-i].Number != pr {
			return 0, errors.New("Batch PRs merged out-of-order")
		}
	}
	return len(matchedPRs), nil
}

// batchIsApplicable returns whether a successful batch result can be used--
// 1) some of the batch is still unmerged and in the queue.
// 2) the recent commits are the batch head ref or merges of batch PRs.
// 3) all unmerged PRs in the batch are still in the queue.
// The return value is the number of PRs already merged, and any errors.
func (sq *SubmitQueue) batchIsApplicable(batch Batch) (int, error) {
	// batch must intersect the queue
	if !sq.batchIntersectsQueue(batch) {
		return 0, errors.New("batch has no PRs in Queue")
	}
	commits, err := sq.githubConfig.GetBranchCommits(batch.BaseName, 100)
	if err != nil {
		glog.Errorf("Error getting commits for batchIsApplicable: %v", err)
		return 0, errors.New("failed to get branch commits: " + err.Error())
	}
	return batch.matchesCommits(commits)
}

func (sq *SubmitQueue) handleGithubE2EBatchMerge() {
	repo := sq.githubConfig.Org + "/" + sq.githubConfig.Project
	for range time.Tick(1 * time.Minute) {
		jobs, err := getSuccessfulBatchJobs(repo, sq.BatchURL)
		if err != nil {
			glog.Errorf("Error reading batch jobs from Prow URL %v", sq.BatchURL)
			continue
		}
		batches := sq.getCompleteBatches(jobs)
		batchErrors := make(map[string]string)
		for _, batch := range batches {
			_, err := sq.batchIsApplicable(batch)
			if err != nil {
				batchErrors[batch.String()] = err.Error()
				continue
			}
			sq.doBatchMerge(batch)
		}
		sq.batchStatus.Error = batchErrors
	}
}

// doBatchMerge iteratively merges PRs in the batch if possible.
// If you modify this, consider modifying doGithubE2EAndMerge too.
func (sq *SubmitQueue) doBatchMerge(batch Batch) {
	sq.mergeLock.Lock()
	defer sq.mergeLock.Unlock()

	// Test again inside the merge lock, in case some other merge snuck in.
	match, err := sq.batchIsApplicable(batch)
	if err != nil {
		glog.Errorf("unexpected! batchIsApplicable failed after success %v", err)
		return
	}
	if !sq.e2eStable(true) {
		return
	}
	prs := []*github.MungeObject{}
	// Check entire batch's preconditions first.
	for _, pull := range batch.Pulls[match:] {
		obj, err := sq.githubConfig.GetObject(pull.Number)
		if err != nil {
			glog.Errorf("error getting object for pr #%d: %v", pull.Number, err)
			return
		}
		if sha, _, ok := obj.GetHeadAndBase(); !ok {
			glog.Errorf("error getting pr #%d sha", pull.Number, err)
			return
		} else if sha != pull.Sha {
			glog.Errorf("error: batch PR #%d HEAD changed: %s instead of %s",
				sha, pull.Sha)
			return
		}
		if !sq.validForMergeExt(obj, false) {
			return
		}
		prs = append(prs, obj)
	}
	// then merge each
	for _, pr := range prs {
		err := sq.mergePullRequest(pr, mergedBatch)
		if err != nil {
			return
		}
		atomic.AddInt32(&sq.batchMerges, 1)
	}
}
<commit_msg>log an explanation line when batch merging, for clarity<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mungers

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
	"sync/atomic"
	"time"

	"github.com/golang/glog"
	githubapi "github.com/google/go-github/github"
	"k8s.io/contrib/mungegithub/github"
)

// xref k8s.io/test-infra/prow/cmd/deck/jobs.go
type prowJob struct {
	Type    string `json:"type"`
	Repo    string `json:"repo"`
	Refs    string `json:"refs"`
	State   string `json:"state"`
	Context string `json:"context"`
}

// getSuccessfulBatchJobs reads test results from Prow and returns
// all batch jobs that succeeded for the current repo.
func getSuccessfulBatchJobs(repo, url string) ([]prowJob, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)

	allJobs := []prowJob{}
	err = json.Unmarshal(body, &allJobs)
	if err != nil {
		return nil, err
	}

	jobs := []prowJob{}
	for _, job := range allJobs {
		if job.Repo == repo && job.Type == "batch" && job.State == "success" {
			jobs = append(jobs, job)
		}
	}
	return jobs, nil
}

type batchPull struct {
	Number int
	Sha    string
}

// Batch represents a specific merge state:
// a base branch and SHA, and the SHAs of each PR merged into it.
type Batch struct {
	BaseName string
	BaseSha  string
	Pulls    []batchPull
}

func (b *Batch) String() string {
	out := b.BaseName + ":" + b.BaseSha
	for _, pull := range b.Pulls {
		out += "," + strconv.Itoa(pull.Number) + ":" + pull.Sha
	}
	return out
}

// batchRefToBatch parses a string into a Batch.
// The input is a comma-separated list of colon-separated ref/sha pairs,
// like "master:abcdef0,123:f00d,456:f00f".
func batchRefToBatch(batchRef string) (Batch, error) {
	batch := Batch{}
	for i, ref := range strings.Split(batchRef, ",") {
		parts := strings.Split(ref, ":")
		if len(parts) != 2 {
			return Batch{}, errors.New("bad batchref: " + batchRef)
		}
		if i == 0 {
			batch.BaseName = parts[0]
			batch.BaseSha = parts[1]
		} else {
			num, err := strconv.ParseInt(parts[0], 10, 32)
			if err != nil {
				return Batch{}, fmt.Errorf("bad batchref: %s (%v)", batchRef, err)
			}
			batch.Pulls = append(batch.Pulls, batchPull{int(num), parts[1]})
		}
	}
	return batch, nil
}

// getCompleteBatches returns a list of Batches that passed all
// required tests.
func (sq *SubmitQueue) getCompleteBatches(jobs []prowJob) []Batch {
	// for each batch specifier, a set of successful contexts
	batchContexts := make(map[string]map[string]interface{})
	for _, job := range jobs {
		if batchContexts[job.Refs] == nil {
			batchContexts[job.Refs] = make(map[string]interface{})
		}
		batchContexts[job.Refs][job.Context] = nil
	}
	batches := []Batch{}
	for batchRef, contexts := range batchContexts {
		match := true
		// Did this succeed in all the contexts we want?
		for _, ctx := range sq.RequiredStatusContexts {
			if _, ok := contexts[ctx]; !ok {
				match = false
			}
		}
		for _, ctx := range sq.RequiredRetestContexts {
			if _, ok := contexts[ctx]; !ok {
				match = false
			}
		}
		if match {
			batch, err := batchRefToBatch(batchRef)
			if err != nil {
				continue
			}
			batches = append(batches, batch)
		}
	}
	return batches
}

// batchIntersectsQueue returns whether at least one PR in the batch is queued.
func (sq *SubmitQueue) batchIntersectsQueue(batch Batch) bool {
	sq.Lock()
	defer sq.Unlock()
	for _, pull := range batch.Pulls {
		if _, ok := sq.githubE2EQueue[pull.Number]; ok {
			return true
		}
	}
	return false
}

// matchesCommit determines if the batch can be merged given some commits.
// That is, does it contain exactly:
// 1) the batch's BaseSha
// 2) (optional) merge commits for PRs in the batch
// 3) any merged PRs in the batch are sequential from the beginning
// The return value is the number of PRs already merged, and any errors.
func (b *Batch) matchesCommits(commits []*githubapi.RepositoryCommit) (int, error) {
	if len(commits) == 0 {
		return 0, errors.New("no commits")
	}

	shaToPR := make(map[string]int)

	for _, pull := range b.Pulls {
		shaToPR[pull.Sha] = pull.Number
	}

	matchedPRs := []int{}

	// convert the list of commits into a DAG for easy following
	dag := make(map[string]*githubapi.RepositoryCommit)
	for _, commit := range commits {
		dag[*commit.SHA] = commit
	}

	ref := *commits[0].SHA
	for {
		if ref == b.BaseSha {
			break // found the base ref (condition #1)
		}
		commit, ok := dag[ref]
		if !ok {
			return 0, errors.New("ran out of commits (missing ref " + ref + ")")
		}
		if len(commit.Parents) == 2 && commit.Message != nil &&
			strings.HasPrefix(*commit.Message, "Merge") {
			// looks like a merge commit!

			// first parent is the normal branch
			ref = *commit.Parents[0].SHA
			// second parent is the PR
			pr, ok := shaToPR[*commit.Parents[1].SHA]
			if !ok {
				return 0, errors.New("Merge of something not in batch")
			}
			matchedPRs = append(matchedPRs, pr)
		} else {
			return 0, errors.New("Unknown non-merge commit " + ref)
		}
	}

	// Now, ensure that the merged PRs are ordered correctly.
	for i, pr := range matchedPRs {
		if b.Pulls[len(matchedPRs)-1-i].Number != pr {
			return 0, errors.New("Batch PRs merged out-of-order")
		}
	}
	return len(matchedPRs), nil
}

// batchIsApplicable returns whether a successful batch result can be used--
// 1) some of the batch is still unmerged and in the queue.
// 2) the recent commits are the batch head ref or merges of batch PRs.
// 3) all unmerged PRs in the batch are still in the queue.
// The return value is the number of PRs already merged, and any errors.
func (sq *SubmitQueue) batchIsApplicable(batch Batch) (int, error) {
	// batch must intersect the queue
	if !sq.batchIntersectsQueue(batch) {
		return 0, errors.New("batch has no PRs in Queue")
	}
	commits, err := sq.githubConfig.GetBranchCommits(batch.BaseName, 100)
	if err != nil {
		glog.Errorf("Error getting commits for batchIsApplicable: %v", err)
		return 0, errors.New("failed to get branch commits: " + err.Error())
	}
	return batch.matchesCommits(commits)
}

func (sq *SubmitQueue) handleGithubE2EBatchMerge() {
	repo := sq.githubConfig.Org + "/" + sq.githubConfig.Project
	for range time.Tick(1 * time.Minute) {
		jobs, err := getSuccessfulBatchJobs(repo, sq.BatchURL)
		if err != nil {
			glog.Errorf("Error reading batch jobs from Prow URL %v", sq.BatchURL)
			continue
		}
		batches := sq.getCompleteBatches(jobs)
		batchErrors := make(map[string]string)
		for _, batch := range batches {
			_, err := sq.batchIsApplicable(batch)
			if err != nil {
				batchErrors[batch.String()] = err.Error()
				continue
			}
			sq.doBatchMerge(batch)
		}
		sq.batchStatus.Error = batchErrors
	}
}

// doBatchMerge iteratively merges PRs in the batch if possible.
// If you modify this, consider modifying doGithubE2EAndMerge too.
func (sq *SubmitQueue) doBatchMerge(batch Batch) {
	sq.mergeLock.Lock()
	defer sq.mergeLock.Unlock()

	// Test again inside the merge lock, in case some other merge snuck in.
	match, err := sq.batchIsApplicable(batch)
	if err != nil {
		glog.Errorf("unexpected! batchIsApplicable failed after success %v", err)
		return
	}
	if !sq.e2eStable(true) {
		return
	}

	glog.Infof("merging batch: %s", batch)
	prs := []*github.MungeObject{}
	// Check entire batch's preconditions first.
	for _, pull := range batch.Pulls[match:] {
		obj, err := sq.githubConfig.GetObject(pull.Number)
		if err != nil {
			glog.Errorf("error getting object for pr #%d: %v", pull.Number, err)
			return
		}
		if sha, _, ok := obj.GetHeadAndBase(); !ok {
			glog.Errorf("error getting pr #%d sha", pull.Number, err)
			return
		} else if sha != pull.Sha {
			glog.Errorf("error: batch PR #%d HEAD changed: %s instead of %s",
				sha, pull.Sha)
			return
		}
		if !sq.validForMergeExt(obj, false) {
			return
		}
		prs = append(prs, obj)
	}
	// then merge each
	for _, pr := range prs {
		err := sq.mergePullRequest(pr, mergedBatch)
		if err != nil {
			return
		}
		atomic.AddInt32(&sq.batchMerges, 1)
	}
}
<|endoftext|>


<commit_before><commit_msg>add: wmi/system builtin collector<commit_after><|endoftext|>


<commit_before>package brew

import (
	"bytes"
	"log"
	"path/filepath"
	"strings"
	"text/template"

	"github.com/google/go-github/github"
	"github.com/goreleaser/goreleaser/clients"
	"github.com/goreleaser/goreleaser/context"
	"github.com/goreleaser/goreleaser/sha256sum"
)

const formulae = `class {{ .Name }} < Formula
  desc "{{ .Desc }}"
  homepage "{{ .Homepage }}"
  url "https://github.com/{{ .Repo }}/releases/download/{{ .Tag }}/{{ .File }}.{{ .Format }}"
  version "{{ .Tag }}"
  sha256 "{{ .SHA256 }}"

  def install
    bin.install "{{ .BinaryName }}"
  end

  {{- if .Caveats }}

  def caveats
    "{{ .Caveats }}"
  end
  {{- end }}
end
`

type templateData struct {
	Name, Desc, Homepage, Repo, Tag, BinaryName, Caveats, File, Format, SHA256 string
}

// Pipe for brew deployment
type Pipe struct{}

// Description of the pipe
func (Pipe) Description() string {
	return "Creating homebrew formulae..."
}

// Run the pipe
func (Pipe) Run(ctx *context.Context) error {
	if ctx.Config.Brew.Repo == "" {
		return nil
	}
	client := clients.GitHub(*ctx.Token)
	path := filepath.Join(
		ctx.Config.Brew.Folder, ctx.Config.Build.BinaryName+".rb",
	)

	log.Println("Updating", path, "on", ctx.Config.Brew.Repo, "...")
	out, err := buildFormulae(ctx, client)
	if err != nil {
		return err
	}

	options := &github.RepositoryContentFileOptions{
		Committer: &github.CommitAuthor{
			Name:  github.String("goreleaserbot"),
			Email: github.String("bot@goreleaser"),
		},
		Content: out.Bytes(),
		Message: github.String(
			ctx.Config.Build.BinaryName + " version " + ctx.Git.CurrentTag,
		),
	}

	owner := ctx.BrewRepo.Owner
	repo := ctx.BrewRepo.Name
	file, _, res, err := client.Repositories.GetContents(
		owner, repo, path, &github.RepositoryContentGetOptions{},
	)
	if err != nil && res.StatusCode == 404 {
		_, _, err = client.Repositories.CreateFile(owner, repo, path, options)
		return err
	}
	options.SHA = file.SHA
	_, _, err = client.Repositories.UpdateFile(owner, repo, path, options)
	return err
}

func buildFormulae(ctx *context.Context, client *github.Client) (bytes.Buffer, error) {
	data, err := dataFor(ctx, client)
	if err != nil {
		return bytes.Buffer{}, err
	}
	return doBuildFormulae(data)
}

func doBuildFormulae(data templateData) (bytes.Buffer, error) {
	var out bytes.Buffer
	tmpl, err := template.New(data.BinaryName).Parse(formulae)
	if err != nil {
		return out, err
	}
	err = tmpl.Execute(&out, data)
	return out, err
}

func dataFor(ctx *context.Context, client *github.Client) (result templateData, err error) {
	var homepage string
	var description string
	rep, _, err := client.Repositories.Get(ctx.ReleaseRepo.Owner, ctx.ReleaseRepo.Name)
	if err != nil {
		return
	}
	file := ctx.Archives["darwinamd64"]
	sum, err := sha256sum.For("dist/" + file + "." + ctx.Config.Archive.Format)
	if err != nil {
		return
	}
	if rep.Homepage != nil && *rep.Homepage != "" {
		homepage = *rep.Homepage
	} else {
		homepage = *rep.HTMLURL
	}
	if rep.Description == nil {
		description = "TODO"
	} else {
		description = *rep.Description
	}
	return templateData{
		Name:       formulaNameFor(ctx.Config.Build.BinaryName),
		Desc:       description,
		Homepage:   homepage,
		Repo:       ctx.Config.Release.Repo,
		Tag:        ctx.Git.CurrentTag,
		BinaryName: ctx.Config.Build.BinaryName,
		Caveats:    ctx.Config.Brew.Caveats,
		File:       file,
		Format:     ctx.Config.Archive.Format,
		SHA256:     sum,
	}, err
}

func formulaNameFor(name string) string {
	name = strings.Replace(name, "-", " ", -1)
	name = strings.Replace(name, "_", " ", -1)
	return strings.Replace(strings.Title(name), " ", "", -1)
}
<commit_msg>fail brew pipe if there is no darwin_amd64 build<commit_after>package brew

import (
	"bytes"
	"errors"
	"log"
	"path/filepath"
	"strings"
	"text/template"

	"github.com/google/go-github/github"
	"github.com/goreleaser/goreleaser/clients"
	"github.com/goreleaser/goreleaser/context"
	"github.com/goreleaser/goreleaser/sha256sum"
)

// ErrNoDarwin64Build when there is no build for darwin_amd64 (goos doesn't
// contain darwin and/or goarch doesn't contain amd64)
var ErrNoDarwin64Build = errors.New("brew tap requires a darwin amd64 build")

const formulae = `class {{ .Name }} < Formula
  desc "{{ .Desc }}"
  homepage "{{ .Homepage }}"
  url "https://github.com/{{ .Repo }}/releases/download/{{ .Tag }}/{{ .File }}.{{ .Format }}"
  version "{{ .Tag }}"
  sha256 "{{ .SHA256 }}"

  def install
    bin.install "{{ .BinaryName }}"
  end

  {{- if .Caveats }}

  def caveats
    "{{ .Caveats }}"
  end
  {{- end }}
end
`

type templateData struct {
	Name, Desc, Homepage, Repo, Tag, BinaryName, Caveats, File, Format, SHA256 string
}

// Pipe for brew deployment
type Pipe struct{}

// Description of the pipe
func (Pipe) Description() string {
	return "Creating homebrew formulae..."
}

// Run the pipe
func (Pipe) Run(ctx *context.Context) error {
	if ctx.Config.Brew.Repo == "" {
		return nil
	}
	client := clients.GitHub(*ctx.Token)
	path := filepath.Join(
		ctx.Config.Brew.Folder, ctx.Config.Build.BinaryName+".rb",
	)

	log.Println("Updating", path, "on", ctx.Config.Brew.Repo, "...")
	out, err := buildFormulae(ctx, client)
	if err != nil {
		return err
	}

	options := &github.RepositoryContentFileOptions{
		Committer: &github.CommitAuthor{
			Name:  github.String("goreleaserbot"),
			Email: github.String("bot@goreleaser"),
		},
		Content: out.Bytes(),
		Message: github.String(
			ctx.Config.Build.BinaryName + " version " + ctx.Git.CurrentTag,
		),
	}

	owner := ctx.BrewRepo.Owner
	repo := ctx.BrewRepo.Name
	file, _, res, err := client.Repositories.GetContents(
		owner, repo, path, &github.RepositoryContentGetOptions{},
	)
	if err != nil && res.StatusCode == 404 {
		_, _, err = client.Repositories.CreateFile(owner, repo, path, options)
		return err
	}
	options.SHA = file.SHA
	_, _, err = client.Repositories.UpdateFile(owner, repo, path, options)
	return err
}

func buildFormulae(ctx *context.Context, client *github.Client) (bytes.Buffer, error) {
	data, err := dataFor(ctx, client)
	if err != nil {
		return bytes.Buffer{}, err
	}
	return doBuildFormulae(data)
}

func doBuildFormulae(data templateData) (bytes.Buffer, error) {
	var out bytes.Buffer
	tmpl, err := template.New(data.BinaryName).Parse(formulae)
	if err != nil {
		return out, err
	}
	err = tmpl.Execute(&out, data)
	return out, err
}

func dataFor(ctx *context.Context, client *github.Client) (result templateData, err error) {
	var homepage string
	var description string
	rep, _, err := client.Repositories.Get(ctx.ReleaseRepo.Owner, ctx.ReleaseRepo.Name)
	if err != nil {
		return
	}
	file := ctx.Archives["darwinamd64"]
	if file == "" {
		return result, ErrNoDarwin64Build
	}
	sum, err := sha256sum.For("dist/" + file + "." + ctx.Config.Archive.Format)
	if err != nil {
		return
	}
	if rep.Homepage != nil && *rep.Homepage != "" {
		homepage = *rep.Homepage
	} else {
		homepage = *rep.HTMLURL
	}
	if rep.Description == nil {
		description = "TODO"
	} else {
		description = *rep.Description
	}
	return templateData{
		Name:       formulaNameFor(ctx.Config.Build.BinaryName),
		Desc:       description,
		Homepage:   homepage,
		Repo:       ctx.Config.Release.Repo,
		Tag:        ctx.Git.CurrentTag,
		BinaryName: ctx.Config.Build.BinaryName,
		Caveats:    ctx.Config.Brew.Caveats,
		File:       file,
		Format:     ctx.Config.Archive.Format,
		SHA256:     sum,
	}, err
}

func formulaNameFor(name string) string {
	name = strings.Replace(name, "-", " ", -1)
	name = strings.Replace(name, "_", " ", -1)
	return strings.Replace(strings.Title(name), " ", "", -1)
}
<|endoftext|>


<commit_before>package apps

import (
	"encoding/json"
	"io"
	"net/url"
	"path"
	"regexp"

	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/vfs"
)

var slugReg = regexp.MustCompile(`^[A-Za-z0-9\-]+$`)

// Installer is used to install or update applications.
type Installer struct {
	fetcher Fetcher
	ctx     vfs.Context

	man  *Manifest
	src  *url.URL
	slug string

	err  error
	errc chan error
	manc chan *Manifest
}

// InstallerOptions provides the slug name of the application along with the
// source URL.
type InstallerOptions struct {
	Slug      string
	SourceURL string
}

// Fetcher interface should be implemented by the underlying transport
// used to fetch the application data.
type Fetcher interface {
	// FetchManifest should returns an io.ReadCloser to read the
	// manifest data
	FetchManifest(src *url.URL) (io.ReadCloser, error)
	// Fetch should download the application and install it in the given
	// directory.
	Fetch(src *url.URL, appDir string) error
}

// NewInstaller creates a new Installer
func NewInstaller(ctx vfs.Context, opts *InstallerOptions) (*Installer, error) {
	slug := opts.Slug
	if slug == "" || !slugReg.MatchString(slug) {
		return nil, ErrInvalidSlugName
	}

	man, err := GetBySlug(ctx, slug)
	if err != nil && !couchdb.IsNotFoundError(err) {
		return nil, err
	}

	var src *url.URL
	if opts.SourceURL != "" {
		src, err = url.Parse(opts.SourceURL)
	} else if man != nil {
		src, err = url.Parse(man.Source)
	} else {
		err = ErrNotSupportedSource
	}
	if err != nil {
		return nil, err
	}

	var fetcher Fetcher
	switch src.Scheme {
	case "git":
		fetcher = newGitFetcher(ctx)
	default:
		return nil, ErrNotSupportedSource
	}

	inst := &Installer{
		fetcher: fetcher,
		ctx:     ctx,
		src:     src,
		slug:    slug,
		man:     man,
		errc:    make(chan error),
		manc:    make(chan *Manifest, 1),
	}

	return inst, nil
}

// InstallOrUpdate will install the application linked to the installer. If the
// application is already installed, it will try to upgrade it. It will report
// its progress or error (see Poll method).
func (i *Installer) InstallOrUpdate() {
	defer i.endOfProc()

	if i.man == nil {
		i.man, i.err = i.install()
		return
	}

	state := i.man.State
	if state != Ready && state != Errored {
		i.man, i.err = nil, ErrBadState
		return
	}

	i.man, i.err = i.update()
	return
}

func (i *Installer) endOfProc() {
	man, err := i.man, i.err
	if man == nil || err == ErrBadState {
		i.errc <- err
		return
	}
	if err != nil {
		man.State = Errored
		man.Error = err.Error()
		updateManifest(i.ctx, man)
		i.errc <- err
		return
	}
	man.State = Ready
	updateManifest(i.ctx, man)
	i.manc <- i.man
}

func (i *Installer) install() (*Manifest, error) {
	man := &Manifest{}
	if err := i.ReadManifest(Installing, &man); err != nil {
		return nil, err
	}

	if err := createManifest(i.ctx, man); err != nil {
		return nil, err
	}

	i.manc <- man

	appdir := i.appDir()
	if _, err := vfs.MkdirAll(i.ctx, appdir, nil); err != nil {
		return nil, err
	}

	if err := i.fetcher.Fetch(i.src, appdir); err != nil {
		return nil, err
	}

	return man, nil
}

func (i *Installer) update() (*Manifest, error) {
	man := i.man
	version := man.Version

	if err := i.ReadManifest(Upgrading, &man); err != nil {
		return nil, err
	}

	if man.Version == version {
		return man, nil
	}

	if err := updateManifest(i.ctx, man); err != nil {
		return nil, err
	}

	i.manc <- man

	appdir := i.appDir()
	if err := i.fetcher.Fetch(i.src, appdir); err != nil {
		return nil, err
	}

	return man, nil
}

// ReadManifest will fetch the manifest and read its JSON content into the
// passed manifest pointer.
//
// The State field of the manifest will be set to the specified state.
func (i *Installer) ReadManifest(state State, man **Manifest) error {
	r, err := i.fetcher.FetchManifest(i.src)
	if err != nil {
		return err
	}
	defer r.Close()

	err = json.NewDecoder(io.LimitReader(r, ManifestMaxSize)).Decode(man)
	if err != nil {
		return ErrBadManifest
	}

	(*man).Slug = i.slug
	(*man).Source = i.src.String()
	(*man).State = state
	return nil
}

func (i *Installer) appDir() string {
	return path.Join(vfs.AppsDirName, i.slug)
}

// Poll should be used to monitor the progress of the Installer.
func (i *Installer) Poll() (man *Manifest, done bool, err error) {
	select {
	case man = <-i.manc:
		done = man.State == Ready
		return
	case err = <-i.errc:
		return
	}
}

func updateManifest(db couchdb.Database, man *Manifest) error {
	return couchdb.UpdateDoc(db, man)
}

func createManifest(db couchdb.Database, man *Manifest) error {
	return couchdb.CreateNamedDoc(db, man)
}
<commit_msg>Return the fetched manifest even if an error occur while upgrading/installing<commit_after>package apps

import (
	"encoding/json"
	"io"
	"net/url"
	"path"
	"regexp"

	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/vfs"
)

var slugReg = regexp.MustCompile(`^[A-Za-z0-9\-]+$`)

// Installer is used to install or update applications.
type Installer struct {
	fetcher Fetcher
	ctx     vfs.Context

	man  *Manifest
	src  *url.URL
	slug string

	err  error
	errc chan error
	manc chan *Manifest
}

// InstallerOptions provides the slug name of the application along with the
// source URL.
type InstallerOptions struct {
	Slug      string
	SourceURL string
}

// Fetcher interface should be implemented by the underlying transport
// used to fetch the application data.
type Fetcher interface {
	// FetchManifest should returns an io.ReadCloser to read the
	// manifest data
	FetchManifest(src *url.URL) (io.ReadCloser, error)
	// Fetch should download the application and install it in the given
	// directory.
	Fetch(src *url.URL, appDir string) error
}

// NewInstaller creates a new Installer
func NewInstaller(ctx vfs.Context, opts *InstallerOptions) (*Installer, error) {
	slug := opts.Slug
	if slug == "" || !slugReg.MatchString(slug) {
		return nil, ErrInvalidSlugName
	}

	man, err := GetBySlug(ctx, slug)
	if err != nil && !couchdb.IsNotFoundError(err) {
		return nil, err
	}

	var src *url.URL
	if opts.SourceURL != "" {
		src, err = url.Parse(opts.SourceURL)
	} else if man != nil {
		src, err = url.Parse(man.Source)
	} else {
		err = ErrNotSupportedSource
	}
	if err != nil {
		return nil, err
	}

	var fetcher Fetcher
	switch src.Scheme {
	case "git":
		fetcher = newGitFetcher(ctx)
	default:
		return nil, ErrNotSupportedSource
	}

	inst := &Installer{
		fetcher: fetcher,
		ctx:     ctx,
		src:     src,
		slug:    slug,
		man:     man,
		errc:    make(chan error),
		manc:    make(chan *Manifest, 1),
	}

	return inst, nil
}

// InstallOrUpdate will install the application linked to the installer. If the
// application is already installed, it will try to upgrade it. It will report
// its progress or error (see Poll method).
func (i *Installer) InstallOrUpdate() {
	defer i.endOfProc()

	if i.man == nil {
		i.man, i.err = i.install()
		return
	}

	state := i.man.State
	if state != Ready && state != Errored {
		i.man, i.err = nil, ErrBadState
		return
	}

	i.man, i.err = i.update()
	return
}

func (i *Installer) endOfProc() {
	man, err := i.man, i.err
	if man == nil || err == ErrBadState {
		i.errc <- err
		return
	}
	if err != nil {
		man.State = Errored
		man.Error = err.Error()
		updateManifest(i.ctx, man)
		i.errc <- err
		return
	}
	man.State = Ready
	updateManifest(i.ctx, man)
	i.manc <- i.man
}

// install will perform the installation of an application. It returns the
// freshly fetched manifest from the source along with a possible error in case
// the installation went wrong.
//
// Note that the fetched manifest is returned even if an error occured while
// upgrading.
func (i *Installer) install() (*Manifest, error) {
	man := &Manifest{}
	if err := i.ReadManifest(Installing, &man); err != nil {
		return nil, err
	}

	if err := createManifest(i.ctx, man); err != nil {
		return man, err
	}

	i.manc <- man

	appdir := i.appDir()
	if _, err := vfs.MkdirAll(i.ctx, appdir, nil); err != nil {
		return man, err
	}

	if err := i.fetcher.Fetch(i.src, appdir); err != nil {
		return man, err
	}

	return man, nil
}

// update will perform the update of an already installed application. It
// returns the freshly fetched manifest from the source along with a possible
// error in case the update went wrong.
//
// Note that the fetched manifest is returned even if an error occured while
// upgrading.
func (i *Installer) update() (*Manifest, error) {
	man := i.man
	version := man.Version

	if err := i.ReadManifest(Upgrading, &man); err != nil {
		return man, err
	}

	if man.Version == version {
		return man, nil
	}

	if err := updateManifest(i.ctx, man); err != nil {
		return man, err
	}

	i.manc <- man

	appdir := i.appDir()
	if err := i.fetcher.Fetch(i.src, appdir); err != nil {
		return man, err
	}

	return man, nil
}

// ReadManifest will fetch the manifest and read its JSON content into the
// passed manifest pointer.
//
// The State field of the manifest will be set to the specified state.
func (i *Installer) ReadManifest(state State, man **Manifest) error {
	r, err := i.fetcher.FetchManifest(i.src)
	if err != nil {
		return err
	}
	defer r.Close()

	err = json.NewDecoder(io.LimitReader(r, ManifestMaxSize)).Decode(man)
	if err != nil {
		return ErrBadManifest
	}

	(*man).Slug = i.slug
	(*man).Source = i.src.String()
	(*man).State = state
	return nil
}

func (i *Installer) appDir() string {
	return path.Join(vfs.AppsDirName, i.slug)
}

// Poll should be used to monitor the progress of the Installer.
func (i *Installer) Poll() (man *Manifest, done bool, err error) {
	select {
	case man = <-i.manc:
		done = man.State == Ready
		return
	case err = <-i.errc:
		return
	}
}

func updateManifest(db couchdb.Database, man *Manifest) error {
	return couchdb.UpdateDoc(db, man)
}

func createManifest(db couchdb.Database, man *Manifest) error {
	return couchdb.CreateNamedDoc(db, man)
}
<|endoftext|>


<commit_before>// Package tree benchmarks
// All benchmarks where down on a 13inch macbook pro with a 2.5 GHz Intel Core i5 processor and 8gb of memory
// BenchmarkTreeToArr1000-4        2000000000               0.00 ns/op
// BenchmarkTreeToArr10000-4       2000000000               0.01 ns/op
// BenchmarkTreeToArr100000-4      1000000000               0.26 ns/op
// BenchmarkTreeToArr1000000-4            1        3997007417 ns/op
// BenchmarkEdgeCount1000-4        2000000000               0.00 ns/op
// BenchmarkEdgeCount10000-4       2000000000               0.01 ns/op
// BenchmarkEdgeCount100000-4      2000000000               0.15 ns/op
// BenchmarkEdgeCount1000000-4            1        3882999625 ns/op
// BenchmarkRootShift1000-4        2000000000               0.00 ns/op
// BenchmarkRootShift10000-4       1000000000               0.02 ns/op
// BenchmarkRootShift100000-4      2000000000               0.21 ns/op
// BenchmarkRootShift1000000-4            1        7592046369 ns/op
// BenchmarkNewTree1000-4          2000000000               0.00 ns/op
// BenchmarkNewTree10000-4         2000000000               0.01 ns/op
// BenchmarkNewTree100000-4        2000000000               0.09 ns/op
// BenchmarkNewTree1000000-4              1        3040027533 ns/op
package tree

import (
	"encoding/json"
	"fmt"
	"math/rand"
	"sync"
	"time"
)

// Node is a fudemental part of what makes a tree a tree. Many Nodes creates a tree
type Node struct {
	Left  *Node
	Right *Node
	Data  int
}

// Tree basic tree structure.. Root is of type Node,
type Tree struct {
	Root      *Node
	Total     int
	NodeCount int
}

var (
	// ErrPositiveIntegers reports that only positive intergers may be added to the tree
	ErrPositiveIntegers = fmt.Errorf("only postive integers may be added")
	// ErrNodeNotFound reports that a Node wasn't found
	ErrNodeNotFound = fmt.Errorf("Node not found")
)

// NewTree ...
func NewTree() *Tree {
	return new(Tree)
}

// FindNode ...
func (t *Tree) FindNode(data int) (err error) {
	newNode := Node{
		Data: data,
	}
	if t.Root != nil {
		if t.findNode(t.Root, newNode) != nil {
			return
		}
	}
	return ErrNodeNotFound
}

func (t *Tree) findNode(search *Node, target Node) *Node {
	var returnNode *Node
	if search == nil {
		return returnNode
	}
	if search.Data == target.Data {
		return search
	}
	returnNode = t.findNode(search.Left, target)
	if returnNode == nil {
		returnNode = t.findNode(search.Right, target)
	}
	return returnNode
}

// Add appends a new node to a branch in a balanced manner
func (t *Tree) Add(data int) (err error) {
	t.Total += data
	t.NodeCount++
	if data < 0 {
		return ErrPositiveIntegers
	}
	NodeToAdd := Node{
		Data: data,
	}
	if t.Root == nil {
		t.Root = new(Node)
	}
	if t.Root.Data == 0 {
		t.Root = &NodeToAdd
		return
	}
	t.add(t.Root, NodeToAdd)
	return
}

func (t *Tree) add(oldNode *Node, newNode Node) {
	if newNode.Data < oldNode.Data {
		if oldNode.Left == nil {
			oldNode.Left = &newNode
		} else {
			t.add(oldNode.Left, newNode)
		}
	} else if newNode.Data > oldNode.Data {
		if oldNode.Right == nil {
			oldNode.Right = &newNode
		} else {
			t.add(oldNode.Right, newNode)
		}
	}
	return
}

// InOrderTraversal prints out the values in order
func (t *Tree) InOrderTraversal() {
	if t.Root != nil {
		currentNode := t.Root
		if currentNode.Left == nil && currentNode.Right == nil {
			fmt.Println(currentNode.Data)
		} else {
			t.inOrderTraversal(currentNode)
		}
	}
	return
}

func (t *Tree) inOrderTraversal(n *Node) {
	if n.Left != nil {
		t.inOrderTraversal(n.Left)
	}
	fmt.Println(n.Data)
	if n.Right != nil {
		t.inOrderTraversal(n.Right)
	}
	return
}

// Traversal prints out the values by branch side, left, right, ect...
func (t *Tree) Traversal() {
	if t.Root != nil {
		currentNode := t.Root
		if currentNode.Left == nil && currentNode.Right == nil {
			fmt.Println(currentNode.Data)
		} else {
			t.traversal(currentNode)
		}
	}
	return
}

func (t *Tree) traversal(n *Node) {
	fmt.Println(n.Data)
	if n.Left != nil {
		t.traversal(n.Left)
	}
	if n.Right != nil {
		t.traversal(n.Right)
	}
	return
}

// Sum added up all the values stored in the Nodes.. It is a redundant function because total value is kept as a Tree
// value
func (t *Tree) Sum() (total int) {
	var wg sync.WaitGroup
	c := make(chan int, 100)
	if t.Root != nil {
		currentNode := t.Root
		if currentNode.Left == nil && currentNode.Right == nil {
			return 1
		}
		wg.Add(1)
		t.sum(currentNode, c, &wg)
	}
	go func() {
		wg.Wait()
		close(c)
	}()
	for n := range c {
		total += n
	}
	return total
}

func (t *Tree) sum(n *Node, counter chan int, wg *sync.WaitGroup) {
	defer wg.Done()
	if n.Left != nil {
		wg.Add(1)
		go t.sum(n.Left, counter, wg)
	}
	counter <- n.Data
	if n.Right != nil {
		wg.Add(1)
		go t.sum(n.Right, counter, wg)
	}
	return
}

// CountEdges returns the number of edges the tree contains
func (t *Tree) CountEdges() (edges int) {
	var wg sync.WaitGroup
	c := make(chan int, 100)
	if t.Root != nil {
		currentNode := t.Root
		if currentNode.Left == nil && currentNode.Right == nil {
			return 1
		}
		wg.Add(1)
		t.countEdges(currentNode, c, &wg)
	}

	go func() {
		wg.Wait()
		close(c)
	}()
	for n := range c {
		edges += n
	}
	return edges
}

func (t *Tree) countEdges(n *Node, counter chan int, wg *sync.WaitGroup) {
	defer wg.Done()
	if n.Left != nil {
		wg.Add(1)
		go t.countEdges(n.Left, counter, wg)
	}
	counter <- 1
	if n.Right != nil {
		wg.Add(1)
		go t.countEdges(n.Right, counter, wg)
	}
	return
}

// GenerateRandomTree uses time (time.Now().Unix()) to create enthorpy for a source of random numbers to append to the Tree
func (t *Tree) GenerateRandomTree(numberOfNodesToCreate int) (err error) {
	if numberOfNodesToCreate < 0 {
		return ErrPositiveIntegers
	}
	u := time.Now()
	source := rand.NewSource(u.Unix())
	r := rand.New(source)
	arr := r.Perm(numberOfNodesToCreate)
	for _, a := range arr {
		t.Add(a)
	}
	return
}

// GetRootData returns the data stored at the root, however this does not return the root Node
func (t *Tree) GetRootData() int {
	return t.Root.Data
}

// GetTreeTotal returns the sum of the collecitve nodes on the Tree
func (t *Tree) GetTreeTotal() int {
	return t.Total
}

// TreeToArray converts to the into an int slice
func (t *Tree) TreeToArray() []int {
	var wg sync.WaitGroup
	c := make(chan int, 100)
	arr := make([]int, 0, t.NodeCount)
	if t.Root != nil {
		currentNode := t.Root
		if currentNode.Left == nil && currentNode.Right == nil {
			return []int{currentNode.Data}
		}
		wg.Add(1)
		t.traversalGetVals(currentNode, c, &wg)
	}
	go func() {
		wg.Wait()
		close(c)
	}()
	for n := range c {
		arr = append(arr, n)
	}
	return arr
}

func (t *Tree) traversalGetVals(n *Node, c chan int, wg *sync.WaitGroup) {
	defer wg.Done()
	if n.Left != nil {
		c <- n.Left.Data
		wg.Add(1)
		go t.traversalGetVals(n.Left, c, wg)
	}
	if n.Right != nil {
		c <- n.Right.Data
		wg.Add(1)
		go t.traversalGetVals(n.Right, c, wg)
	}
	return
}

// ShiftRoot rebuilds the tree with a new root
func (t *Tree) ShiftRoot(newRoot int) {
	arr := t.TreeToArray()
	n := Tree{}
	n.Add(newRoot)
	for _, i := range arr {
		n.Add(i)
	}
	*t = n
}

// PrintTree uses json.MarshalIndent() to print the Tree in an organized fashion, which can then be analysized as a JSON
// object
func (t *Tree) PrintTree() {
	b, err := json.MarshalIndent(t, "", " ")
	if err != nil {
		panic(err)
	}
	fmt.Println(string(b))
}
<commit_msg>overview<commit_after>// Package tree implements a basic balanced binary tree
package tree

import (
	"encoding/json"
	"fmt"
	"math/rand"
	"sync"
	"time"
)

// Node is a fudemental part of what makes a tree a tree. Many Nodes creates a tree
type Node struct {
	Left  *Node
	Right *Node
	Data  int
}

// Tree basic tree structure.. Root is of type Node,
type Tree struct {
	Root      *Node
	Total     int
	NodeCount int
}

var (
	/*
		All benchmarks where down on a 13inch macbook pro with a 2.5 GHz Intel Core i5 processor and 8gb of memory
		BenchmarkTreeToArr1000-4        2000000000               0.00 ns/op
		BenchmarkTreeToArr10000-4       2000000000               0.01 ns/op
		BenchmarkTreeToArr100000-4      1000000000               0.26 ns/op
		BenchmarkTreeToArr1000000-4            1        3997007417 ns/op
		BenchmarkEdgeCount1000-4        2000000000               0.00 ns/op
		BenchmarkEdgeCount10000-4       2000000000               0.01 ns/op
		BenchmarkEdgeCount100000-4      2000000000               0.15 ns/op
		BenchmarkEdgeCount1000000-4            1        3882999625 ns/op
		BenchmarkRootShift1000-4        2000000000               0.00 ns/op
		BenchmarkRootShift10000-4       1000000000               0.02 ns/op
		BenchmarkRootShift100000-4      2000000000               0.21 ns/op
		BenchmarkRootShift1000000-4            1        7592046369 ns/op
		BenchmarkNewTree1000-4          2000000000               0.00 ns/op
		BenchmarkNewTree10000-4         2000000000               0.01 ns/op
		BenchmarkNewTree100000-4        2000000000               0.09 ns/op
		BenchmarkNewTree1000000-4              1        3040027533 ns/op
	*/

	// ErrPositiveIntegers reports that only positive intergers may be added to the tree
	ErrPositiveIntegers = fmt.Errorf("only postive integers may be added")
	// ErrNodeNotFound reports that a Node wasn't found
	ErrNodeNotFound = fmt.Errorf("Node not found")
)

// NewTree ...
func NewTree() *Tree {
	return new(Tree)
}

// FindNode ...
func (t *Tree) FindNode(data int) (err error) {
	newNode := Node{
		Data: data,
	}
	if t.Root != nil {
		if t.findNode(t.Root, newNode) != nil {
			return
		}
	}
	return ErrNodeNotFound
}

func (t *Tree) findNode(search *Node, target Node) *Node {
	var returnNode *Node
	if search == nil {
		return returnNode
	}
	if search.Data == target.Data {
		return search
	}
	returnNode = t.findNode(search.Left, target)
	if returnNode == nil {
		returnNode = t.findNode(search.Right, target)
	}
	return returnNode
}

// Add appends a new node to a branch in a balanced manner
func (t *Tree) Add(data int) (err error) {
	t.Total += data
	t.NodeCount++
	if data < 0 {
		return ErrPositiveIntegers
	}
	NodeToAdd := Node{
		Data: data,
	}
	if t.Root == nil {
		t.Root = new(Node)
	}
	if t.Root.Data == 0 {
		t.Root = &NodeToAdd
		return
	}
	t.add(t.Root, NodeToAdd)
	return
}

func (t *Tree) add(oldNode *Node, newNode Node) {
	if newNode.Data < oldNode.Data {
		if oldNode.Left == nil {
			oldNode.Left = &newNode
		} else {
			t.add(oldNode.Left, newNode)
		}
	} else if newNode.Data > oldNode.Data {
		if oldNode.Right == nil {
			oldNode.Right = &newNode
		} else {
			t.add(oldNode.Right, newNode)
		}
	}
	return
}

// InOrderTraversal prints out the values in order
func (t *Tree) InOrderTraversal() {
	if t.Root != nil {
		currentNode := t.Root
		if currentNode.Left == nil && currentNode.Right == nil {
			fmt.Println(currentNode.Data)
		} else {
			t.inOrderTraversal(currentNode)
		}
	}
	return
}

func (t *Tree) inOrderTraversal(n *Node) {
	if n.Left != nil {
		t.inOrderTraversal(n.Left)
	}
	fmt.Println(n.Data)
	if n.Right != nil {
		t.inOrderTraversal(n.Right)
	}
	return
}

// Traversal prints out the values by branch side, left, right, ect...
func (t *Tree) Traversal() {
	if t.Root != nil {
		currentNode := t.Root
		if currentNode.Left == nil && currentNode.Right == nil {
			fmt.Println(currentNode.Data)
		} else {
			t.traversal(currentNode)
		}
	}
	return
}

func (t *Tree) traversal(n *Node) {
	fmt.Println(n.Data)
	if n.Left != nil {
		t.traversal(n.Left)
	}
	if n.Right != nil {
		t.traversal(n.Right)
	}
	return
}

// Sum added up all the values stored in the Nodes.. It is a redundant function because total value is kept as a Tree
// value
func (t *Tree) Sum() (total int) {
	var wg sync.WaitGroup
	c := make(chan int, 100)
	if t.Root != nil {
		currentNode := t.Root
		if currentNode.Left == nil && currentNode.Right == nil {
			return 1
		}
		wg.Add(1)
		t.sum(currentNode, c, &wg)
	}
	go func() {
		wg.Wait()
		close(c)
	}()
	for n := range c {
		total += n
	}
	return total
}

func (t *Tree) sum(n *Node, counter chan int, wg *sync.WaitGroup) {
	defer wg.Done()
	if n.Left != nil {
		wg.Add(1)
		go t.sum(n.Left, counter, wg)
	}
	counter <- n.Data
	if n.Right != nil {
		wg.Add(1)
		go t.sum(n.Right, counter, wg)
	}
	return
}

// CountEdges returns the number of edges the tree contains
func (t *Tree) CountEdges() (edges int) {
	var wg sync.WaitGroup
	c := make(chan int, 100)
	if t.Root != nil {
		currentNode := t.Root
		if currentNode.Left == nil && currentNode.Right == nil {
			return 1
		}
		wg.Add(1)
		t.countEdges(currentNode, c, &wg)
	}

	go func() {
		wg.Wait()
		close(c)
	}()
	for n := range c {
		edges += n
	}
	return edges
}

func (t *Tree) countEdges(n *Node, counter chan int, wg *sync.WaitGroup) {
	defer wg.Done()
	if n.Left != nil {
		wg.Add(1)
		go t.countEdges(n.Left, counter, wg)
	}
	counter <- 1
	if n.Right != nil {
		wg.Add(1)
		go t.countEdges(n.Right, counter, wg)
	}
	return
}

// GenerateRandomTree uses time (time.Now().Unix()) to create enthorpy for a source of random numbers to append to the Tree
func (t *Tree) GenerateRandomTree(numberOfNodesToCreate int) (err error) {
	if numberOfNodesToCreate < 0 {
		return ErrPositiveIntegers
	}
	u := time.Now()
	source := rand.NewSource(u.Unix())
	r := rand.New(source)
	arr := r.Perm(numberOfNodesToCreate)
	for _, a := range arr {
		t.Add(a)
	}
	return
}

// GetRootData returns the data stored at the root, however this does not return the root Node
func (t *Tree) GetRootData() int {
	return t.Root.Data
}

// GetTreeTotal returns the sum of the collecitve nodes on the Tree
func (t *Tree) GetTreeTotal() int {
	return t.Total
}

// TreeToArray converts to the into an int slice
func (t *Tree) TreeToArray() []int {
	var wg sync.WaitGroup
	c := make(chan int, 100)
	arr := make([]int, 0, t.NodeCount)
	if t.Root != nil {
		currentNode := t.Root
		if currentNode.Left == nil && currentNode.Right == nil {
			return []int{currentNode.Data}
		}
		wg.Add(1)
		t.traversalGetVals(currentNode, c, &wg)
	}
	go func() {
		wg.Wait()
		close(c)
	}()
	for n := range c {
		arr = append(arr, n)
	}
	return arr
}

func (t *Tree) traversalGetVals(n *Node, c chan int, wg *sync.WaitGroup) {
	defer wg.Done()
	if n.Left != nil {
		c <- n.Left.Data
		wg.Add(1)
		go t.traversalGetVals(n.Left, c, wg)
	}
	if n.Right != nil {
		c <- n.Right.Data
		wg.Add(1)
		go t.traversalGetVals(n.Right, c, wg)
	}
	return
}

// ShiftRoot rebuilds the tree with a new root
func (t *Tree) ShiftRoot(newRoot int) {
	arr := t.TreeToArray()
	n := Tree{}
	n.Add(newRoot)
	for _, i := range arr {
		n.Add(i)
	}
	*t = n
}

// PrintTree uses json.MarshalIndent() to print the Tree in an organized fashion, which can then be analysized as a JSON
// object
func (t *Tree) PrintTree() {
	b, err := json.MarshalIndent(t, "", " ")
	if err != nil {
		panic(err)
	}
	fmt.Println(string(b))
}
<|endoftext|>


<commit_before>package tree

import (
	"container/list"
	"os"
	"path/filepath"
	"strings"

	"github.com/Masterminds/glide/dependency"
	"github.com/Masterminds/glide/msg"
	gpath "github.com/Masterminds/glide/path"
	"github.com/Masterminds/glide/util"
)

// Display displays a tree view of the given project.
//
// FIXME: The output formatting could use some TLC.
func Display(b *util.BuildCtxt, basedir, myName string, level int, core bool, l *list.List) {
	deps := walkDeps(b, basedir, myName)
	for _, name := range deps {
		found := findPkg(b, name, basedir)
		if found.Loc == dependency.LocUnknown {
			m := "glide get " + found.Name
			msg.Puts("\t%s\t(%s)", found.Name, m)
			continue
		}
		if !core && found.Loc == dependency.LocGoroot || found.Loc == dependency.LocCgo {
			continue
		}
		msg.Print(strings.Repeat("|\t", level-1) + "|-- ")

		f := findInList(found.Name, l)
		if f == true {
			msg.Puts("(Recursion) %s   (%s)", found.Name, found.Path)
		} else {
			// Every branch in the tree is a copy to handle all the branches
			cl := copyList(l)
			cl.PushBack(found.Name)
			msg.Puts("%s   (%s)", found.Name, found.Path)
			Display(b, found.Path, found.Name, level+1, core, cl)
		}
	}
}

func walkDeps(b *util.BuildCtxt, base, myName string) []string {
	externalDeps := []string{}
	filepath.Walk(base, func(path string, fi os.FileInfo, err error) error {
		if !dependency.IsSrcDir(fi) {
			if fi.IsDir() {
				return filepath.SkipDir
			}
			return nil
		}

		var imps []string
		pkg, err := b.ImportDir(path, 0)
		if err != nil && strings.HasPrefix(err.Error(), "found packages ") {
			// If we got here it's because a package and multiple packages
			// declared. This is often because of an example with a package
			// or main but +build ignore as a build tag. In that case we
			// try to brute force the packages with a slower scan.
			imps, _, err = dependency.IterativeScan(path)
			if err != nil {
				msg.Err("Error walking dependencies for %s: %s", path, err)
				return err
			}
		} else if err != nil {
			if !strings.HasPrefix(err.Error(), "no buildable Go source") {
				msg.Warn("Error: %s (%s)", err, path)
				// Not sure if we should return here.
				//return err
			}
		} else {
			imps = pkg.Imports
		}

		if pkg.Goroot {
			return nil
		}

		for _, imp := range imps {
			//if strings.HasPrefix(imp, myName) {
			////Info("Skipping %s because it is a subpackage of %s", imp, myName)
			//continue
			//}
			if imp == myName {
				continue
			}
			externalDeps = append(externalDeps, imp)
		}

		return nil
	})
	return externalDeps
}

func findPkg(b *util.BuildCtxt, name, cwd string) *dependency.PkgInfo {
	var fi os.FileInfo
	var err error
	var p string

	info := &dependency.PkgInfo{
		Name: name,
	}

	if strings.HasPrefix(name, "./") || strings.HasPrefix(name, "../") {
		info.Loc = dependency.LocRelative
		return info
	}

	// Recurse backward to scan other vendor/ directories
	// If the cwd isn't an absolute path walking upwards looking for vendor/
	// folders can get into an infinate loop.
	abs, err := filepath.Abs(cwd)
	if err != nil {
		abs = cwd
	}
	if abs != "." {
		// Previously there was a check on the loop that wd := "/". The path
		// "/" is a POSIX path so this fails on Windows. Now the check is to
		// make sure the same wd isn't seen twice. When the same wd happens
		// more than once it's the beginning of looping on the same location
		// which is the top level.
		pwd := ""
		for wd := abs; wd != pwd; wd = filepath.Dir(wd) {
			pwd = wd

			// Don't look for packages outside the GOPATH
			// Note, the GOPATH may or may not end with the path separator.
			// The output of filepath.Dir does not the the path separator on the
			// end so we need to test both.
			if wd == b.GOPATH || wd+string(os.PathSeparator) == b.GOPATH {
				break
			}
			p = filepath.Join(wd, "vendor", name)
			if fi, err = os.Stat(p); err == nil && (fi.IsDir() || gpath.IsLink(fi)) {
				info.Path = p
				info.Loc = dependency.LocVendor
				info.Vendored = true
				return info
			}
		}
	}
	// Check $GOPATH
	for _, r := range strings.Split(b.GOPATH, ":") {
		p = filepath.Join(r, "src", name)
		if fi, err = os.Stat(p); err == nil && (fi.IsDir() || gpath.IsLink(fi)) {
			info.Path = p
			info.Loc = dependency.LocGopath
			return info
		}
	}

	// Check $GOROOT
	for _, r := range strings.Split(b.GOROOT, ":") {
		p = filepath.Join(r, "src", name)
		if fi, err = os.Stat(p); err == nil && (fi.IsDir() || gpath.IsLink(fi)) {
			info.Path = p
			info.Loc = dependency.LocGoroot
			return info
		}
	}

	// If this is "C", we're dealing with cgo
	if name == "C" {
		info.Loc = dependency.LocCgo
	} else if name == "appengine" || name == "appengine_internal" ||
		strings.HasPrefix(name, "appengine/") ||
		strings.HasPrefix(name, "appengine_internal/") {
		// Appengine is a special case when it comes to Go builds. It is a local
		// looking package only available within appengine. It's a special case
		// where Google products are playing with each other.
		// https://blog.golang.org/the-app-engine-sdk-and-workspaces-gopath
		info.Loc = dependency.LocAppengine
	} else if name == "context" || name == "net/http/httptrace" {
		// context and net/http/httptrace are packages being added to
		// the Go 1.7 standard library. Some packages, such as golang.org/x/net
		// are importing it with build flags in files for go1.7. Need to detect
		// this and handle it.
		info.Loc = dependency.LocGoroot
	}

	return info
}

// copyList copies an existing list to a new list.
func copyList(l *list.List) *list.List {
	n := list.New()
	for e := l.Front(); e != nil; e = e.Next() {
		n.PushBack(e.Value.(string))
	}
	return n
}

// findInList searches a list haystack for a string needle.
func findInList(n string, l *list.List) bool {
	for e := l.Front(); e != nil; e = e.Next() {
		if e.Value.(string) == n {
			return true
		}
	}

	return false
}
<commit_msg>Fixes panic while running tree command.<commit_after>package tree

import (
	"container/list"
	"os"
	"path/filepath"
	"strings"

	"github.com/Masterminds/glide/dependency"
	"github.com/Masterminds/glide/msg"
	gpath "github.com/Masterminds/glide/path"
	"github.com/Masterminds/glide/util"
)

// Display displays a tree view of the given project.
//
// FIXME: The output formatting could use some TLC.
func Display(b *util.BuildCtxt, basedir, myName string, level int, core bool, l *list.List) {
	deps := walkDeps(b, basedir, myName)
	for _, name := range deps {
		found := findPkg(b, name, basedir)
		if found.Loc == dependency.LocUnknown {
			m := "glide get " + found.Name
			msg.Puts("\t%s\t(%s)", found.Name, m)
			continue
		}
		if !core && found.Loc == dependency.LocGoroot || found.Loc == dependency.LocCgo {
			continue
		}
		msg.Print(strings.Repeat("|\t", level-1) + "|-- ")

		f := findInList(found.Name, l)
		if f == true {
			msg.Puts("(Recursion) %s   (%s)", found.Name, found.Path)
		} else {
			// Every branch in the tree is a copy to handle all the branches
			cl := copyList(l)
			cl.PushBack(found.Name)
			msg.Puts("%s   (%s)", found.Name, found.Path)
			Display(b, found.Path, found.Name, level+1, core, cl)
		}
	}
}

func walkDeps(b *util.BuildCtxt, base, myName string) []string {
	externalDeps := []string{}
	filepath.Walk(base, func(path string, fi os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if !dependency.IsSrcDir(fi) {
			if fi.IsDir() {
				return filepath.SkipDir
			}
			return nil
		}

		var imps []string
		pkg, err := b.ImportDir(path, 0)
		if err != nil && strings.HasPrefix(err.Error(), "found packages ") {
			// If we got here it's because a package and multiple packages
			// declared. This is often because of an example with a package
			// or main but +build ignore as a build tag. In that case we
			// try to brute force the packages with a slower scan.
			imps, _, err = dependency.IterativeScan(path)
			if err != nil {
				msg.Err("Error walking dependencies for %s: %s", path, err)
				return err
			}
		} else if err != nil {
			if !strings.HasPrefix(err.Error(), "no buildable Go source") {
				msg.Warn("Error: %s (%s)", err, path)
				// Not sure if we should return here.
				//return err
			}
		} else {
			imps = pkg.Imports
		}

		if pkg.Goroot {
			return nil
		}

		for _, imp := range imps {
			//if strings.HasPrefix(imp, myName) {
			////Info("Skipping %s because it is a subpackage of %s", imp, myName)
			//continue
			//}
			if imp == myName {
				continue
			}
			externalDeps = append(externalDeps, imp)
		}

		return nil
	})
	return externalDeps
}

func findPkg(b *util.BuildCtxt, name, cwd string) *dependency.PkgInfo {
	var fi os.FileInfo
	var err error
	var p string

	info := &dependency.PkgInfo{
		Name: name,
	}

	if strings.HasPrefix(name, "./") || strings.HasPrefix(name, "../") {
		info.Loc = dependency.LocRelative
		return info
	}

	// Recurse backward to scan other vendor/ directories
	// If the cwd isn't an absolute path walking upwards looking for vendor/
	// folders can get into an infinate loop.
	abs, err := filepath.Abs(cwd)
	if err != nil {
		abs = cwd
	}
	if abs != "." {
		// Previously there was a check on the loop that wd := "/". The path
		// "/" is a POSIX path so this fails on Windows. Now the check is to
		// make sure the same wd isn't seen twice. When the same wd happens
		// more than once it's the beginning of looping on the same location
		// which is the top level.
		pwd := ""
		for wd := abs; wd != pwd; wd = filepath.Dir(wd) {
			pwd = wd

			// Don't look for packages outside the GOPATH
			// Note, the GOPATH may or may not end with the path separator.
			// The output of filepath.Dir does not the the path separator on the
			// end so we need to test both.
			if wd == b.GOPATH || wd+string(os.PathSeparator) == b.GOPATH {
				break
			}
			p = filepath.Join(wd, "vendor", name)
			if fi, err = os.Stat(p); err == nil && (fi.IsDir() || gpath.IsLink(fi)) {
				info.Path = p
				info.Loc = dependency.LocVendor
				info.Vendored = true
				return info
			}
		}
	}
	// Check $GOPATH
	for _, r := range strings.Split(b.GOPATH, ":") {
		p = filepath.Join(r, "src", name)
		if fi, err = os.Stat(p); err == nil && (fi.IsDir() || gpath.IsLink(fi)) {
			info.Path = p
			info.Loc = dependency.LocGopath
			return info
		}
	}

	// Check $GOROOT
	for _, r := range strings.Split(b.GOROOT, ":") {
		p = filepath.Join(r, "src", name)
		if fi, err = os.Stat(p); err == nil && (fi.IsDir() || gpath.IsLink(fi)) {
			info.Path = p
			info.Loc = dependency.LocGoroot
			return info
		}
	}

	// If this is "C", we're dealing with cgo
	if name == "C" {
		info.Loc = dependency.LocCgo
	} else if name == "appengine" || name == "appengine_internal" ||
		strings.HasPrefix(name, "appengine/") ||
		strings.HasPrefix(name, "appengine_internal/") {
		// Appengine is a special case when it comes to Go builds. It is a local
		// looking package only available within appengine. It's a special case
		// where Google products are playing with each other.
		// https://blog.golang.org/the-app-engine-sdk-and-workspaces-gopath
		info.Loc = dependency.LocAppengine
	} else if name == "context" || name == "net/http/httptrace" {
		// context and net/http/httptrace are packages being added to
		// the Go 1.7 standard library. Some packages, such as golang.org/x/net
		// are importing it with build flags in files for go1.7. Need to detect
		// this and handle it.
		info.Loc = dependency.LocGoroot
	}

	return info
}

// copyList copies an existing list to a new list.
func copyList(l *list.List) *list.List {
	n := list.New()
	for e := l.Front(); e != nil; e = e.Next() {
		n.PushBack(e.Value.(string))
	}
	return n
}

// findInList searches a list haystack for a string needle.
func findInList(n string, l *list.List) bool {
	for e := l.Front(); e != nil; e = e.Next() {
		if e.Value.(string) == n {
			return true
		}
	}

	return false
}
<|endoftext|>


<commit_before>// Copyright 2015 Matthew Holt and The Caddy Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build gofuzz
// +build gofuzz_libfuzzer

package caddyfile

import (
	"bytes"
)

func FuzzParseCaddyfile(data []byte) (score int) {
	sb, err := Parse("Caddyfile", bytes.NewReader(data))
	if err != nil {
		// if both an error is received and some ServerBlocks,
		// then the parse was able to parse partially. Mark this
		// result as interesting to push the fuzzer further through the parser.
		if sb != nil && len(sb) > 0 {
			return 1
		}
		return 0
	}
	return 1
}
<commit_msg>v2: fuzz: update function signature of caddyfile.Parse (#3160)<commit_after>// Copyright 2015 Matthew Holt and The Caddy Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build gofuzz
// +build gofuzz_libfuzzer

package caddyfile

func FuzzParseCaddyfile(data []byte) (score int) {
	sb, err := Parse("Caddyfile", data)
	if err != nil {
		// if both an error is received and some ServerBlocks,
		// then the parse was able to parse partially. Mark this
		// result as interesting to push the fuzzer further through the parser.
		if sb != nil && len(sb) > 0 {
			return 1
		}
		return 0
	}
	return 1
}
<|endoftext|>


<commit_before>package detect

import (
	"encoding/binary"
	"net"
	"strconv"

	"github.com/mesos/mesos-go/detector"
	mesos "github.com/mesos/mesos-go/mesosproto"

	"github.com/mesosphere/mesos-dns/logging"
)

var (
	_ detector.MasterChanged = (*Masters)(nil)
	_ detector.AllMasters    = (*Masters)(nil)
)

// Masters detects changes of leader and/or master elections
// and sends these changes to a channel.
type Masters struct {
	// current masters list,
	// 1st item represents the leader,
	// the rest remaining masters
	masters []string

	// the channel leader/master changes are being sent to
	changed chan<- []string
}

// NewMasters returns a new Masters detector with the given initial masters
// and the given changed channel to which master changes will be sent to.
// Initially the leader is unknown which is represented by
// setting the first item of the sent masters slice to be empty.
func NewMasters(masters []string, changed chan<- []string) *Masters {
	return &Masters{
		masters: append([]string{""}, masters...),
		changed: changed,
	}
}

// OnMasterChanged sets the given MasterInfo as the current leader
// leaving the remaining masters unchanged and emits the current masters state.
// It implements the detector.MasterChanged interface.
func (ms *Masters) OnMasterChanged(leader *mesos.MasterInfo) {
	logging.VeryVerbose.Println("Updated leader: ", leader)

	if leader == nil {
		logging.Error.Println("No master available in Zookeeper.")
		return
	}

	ms.masters = ordered(masterHostPort(leader), ms.masters[1:])
	emit(ms.changed, ms.masters)
}

// UpdatedMasters sets the given slice of MasterInfo as the current remaining masters
// leaving the current leader unchanged and emits the current masters state.
// It implements the detector.AllMasters interface.
func (ms *Masters) UpdatedMasters(infos []*mesos.MasterInfo) {
	logging.VeryVerbose.Println("Updated masters: ", infos)

	if infos == nil {
		logging.Error.Println("No masters available in Zookeeper.")
		return
	}

	masters := make([]string, 0, len(infos))
	for _, info := range infos {
		if validMasterInfo(info) {
			masters = append(masters, masterHostPort(info))
		}
	}

	if len(masters) == 0 {
		logging.Error.Println("No valid masters available in Zookeeper.")
		return
	}

	ms.masters = ordered(ms.masters[0], masters)
	emit(ms.changed, ms.masters)
}

func emit(ch chan<- []string, s []string) {
	ch <- append(make([]string, 0, len(s)), s...)
}

// ordered returns a slice of masters with the given leader in the first position
func ordered(leader string, masters []string) []string {
	ms := append(make([]string, 0, len(masters)+1), leader)
	for _, m := range masters {
		if m != leader {
			ms = append(ms, m)
		}
	}
	return ms
}

func validMasterInfo(info *mesos.MasterInfo) bool {
	return info.GetHostname() != "" || info.GetIp() != 0
}

func masterHostPort(info *mesos.MasterInfo) string {
	// unpack IPv4
	octets := make([]byte, net.IPv4len)
	binary.LittleEndian.PutUint32(octets, info.GetIp())
	// we're using an octet slice of len IPv4len, thus no need to convert with To4()
	ipv4 := net.IP(octets)

	return net.JoinHostPort(ipv4.String(), masterPort(info))
}

func masterPort(info *mesos.MasterInfo) string {
	return strconv.FormatUint(uint64(info.GetPort()), 10)
}
<commit_msg>detect: add TODO for refactoring towards Address.ip<commit_after>package detect

import (
	"encoding/binary"
	"net"
	"strconv"

	"github.com/mesos/mesos-go/detector"
	mesos "github.com/mesos/mesos-go/mesosproto"

	"github.com/mesosphere/mesos-dns/logging"
)

var (
	_ detector.MasterChanged = (*Masters)(nil)
	_ detector.AllMasters    = (*Masters)(nil)
)

// Masters detects changes of leader and/or master elections
// and sends these changes to a channel.
type Masters struct {
	// current masters list,
	// 1st item represents the leader,
	// the rest remaining masters
	masters []string

	// the channel leader/master changes are being sent to
	changed chan<- []string
}

// NewMasters returns a new Masters detector with the given initial masters
// and the given changed channel to which master changes will be sent to.
// Initially the leader is unknown which is represented by
// setting the first item of the sent masters slice to be empty.
func NewMasters(masters []string, changed chan<- []string) *Masters {
	return &Masters{
		masters: append([]string{""}, masters...),
		changed: changed,
	}
}

// OnMasterChanged sets the given MasterInfo as the current leader
// leaving the remaining masters unchanged and emits the current masters state.
// It implements the detector.MasterChanged interface.
func (ms *Masters) OnMasterChanged(leader *mesos.MasterInfo) {
	logging.VeryVerbose.Println("Updated leader: ", leader)

	if leader == nil {
		logging.Error.Println("No master available in Zookeeper.")
		return
	}

	ms.masters = ordered(masterHostPort(leader), ms.masters[1:])
	emit(ms.changed, ms.masters)
}

// UpdatedMasters sets the given slice of MasterInfo as the current remaining masters
// leaving the current leader unchanged and emits the current masters state.
// It implements the detector.AllMasters interface.
func (ms *Masters) UpdatedMasters(infos []*mesos.MasterInfo) {
	logging.VeryVerbose.Println("Updated masters: ", infos)

	if infos == nil {
		logging.Error.Println("No masters available in Zookeeper.")
		return
	}

	masters := make([]string, 0, len(infos))
	for _, info := range infos {
		if validMasterInfo(info) {
			masters = append(masters, masterHostPort(info))
		}
	}

	if len(masters) == 0 {
		logging.Error.Println("No valid masters available in Zookeeper.")
		return
	}

	ms.masters = ordered(ms.masters[0], masters)
	emit(ms.changed, ms.masters)
}

func emit(ch chan<- []string, s []string) {
	ch <- append(make([]string, 0, len(s)), s...)
}

// ordered returns a slice of masters with the given leader in the first position
func ordered(leader string, masters []string) []string {
	ms := append(make([]string, 0, len(masters)+1), leader)
	for _, m := range masters {
		if m != leader {
			ms = append(ms, m)
		}
	}
	return ms
}

func validMasterInfo(info *mesos.MasterInfo) bool {
	return info.GetHostname() != "" || info.GetIp() != 0
}

func masterHostPort(info *mesos.MasterInfo) string {
	// unpack IPv4
	octets := make([]byte, net.IPv4len)
	// TODO(sur): refactor to use the Address.ip string field once available
	binary.LittleEndian.PutUint32(octets, info.GetIp())
	// we're using an octet slice of len IPv4len, thus no need to convert with To4()
	ipv4 := net.IP(octets)

	return net.JoinHostPort(ipv4.String(), masterPort(info))
}

func masterPort(info *mesos.MasterInfo) string {
	return strconv.FormatUint(uint64(info.GetPort()), 10)
}
<|endoftext|>


<commit_before>package handlers

import (
	"crypto/rsa"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/JustinBeckwith/go-yelp/yelp"
	"github.com/dgrijalva/jwt-go"
	"github.com/rs/cors"

	"github.com/justinas/alice"

	"github.com/gorilla/context"
	"github.com/gorilla/mux"
	"github.com/gorilla/sessions"
	"github.com/gorilla/websocket"
	"github.com/markbates/goth"
	"github.com/markbates/goth/gothic"
	"github.com/markbates/goth/providers/twitter"
)

var (
	verifyKey *rsa.PublicKey
	signKey   *rsa.PrivateKey
)

const (
	privKeyPath    = "keys/app.rsa"     // openssl genrsa -out app.rsa keysize
	pubKeyPath     = "keys/app.rsa.pub" // openssl rsa -in app.rsa -pubout > app.rsa.pub
	clientURL      = "http://localhost:8080"
	userContextKey = "userid"
)

var upgrader = websocket.Upgrader{
	ReadBufferSize:  1024,
	WriteBufferSize: 1024,
	CheckOrigin: func(r *http.Request) bool {
		return true
	},
}

// read the key files before starting http handlers
func init() {
	signBytes, err := ioutil.ReadFile(privKeyPath)
	fatal(err)

	signKey, err = jwt.ParseRSAPrivateKeyFromPEM(signBytes)
	fatal(err)

	verifyBytes, err := ioutil.ReadFile(pubKeyPath)
	fatal(err)

	verifyKey, err = jwt.ParseRSAPublicKeyFromPEM(verifyBytes)

}

func fatal(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

type httpError struct {
	status int
}

func (e httpError) Error() string {
	return http.StatusText(e.status)
}

func abortWithStatus(w http.ResponseWriter, status int) {
	abortWithError(w, httpError{status})
}

func abortWithError(w http.ResponseWriter, err error) {
	if httpError, ok := err.(httpError); ok {
		http.Error(w, httpError.Error(), httpError.status)
		return
	}
	// maybe catch other errors such as "no rows found"
	log.Println(err)
	http.Error(w, err.Error(), http.StatusInternalServerError)
}

var (
	db = initDB()
	h  = initHub()
)

// grabs user ID from context
func getUserID(r *http.Request) string {
	if userID := context.Get(r, userContextKey); userID != nil {
		return userID.(string)
	}
	return ""
}

func authenticate(w http.ResponseWriter, r *http.Request) error {
	auth := ""
	header := r.Header.Get("Authorization")
	if header == "" {
		// try the query string
		auth = r.URL.Query().Get("jwt-token")
	} else {
		parts := strings.Split(header, "Bearer")
		if len(parts) < 2 {
			return nil
		}
		auth = strings.Trim(parts[1], " ")
	}
	if auth == "" {
		return nil
	}
	t, err := jwt.Parse(auth, func(token *jwt.Token) (interface{}, error) {
		return verifyKey, nil
	})
	if err != nil {
		//check if timeout etc
		log.Println("JWTERROR", err)
		// if a timeout throw a 401, so we can prompt re-login
		return httpError{http.StatusUnauthorized}
	}
	if t.Valid {
		context.Set(r, userContextKey, t.Claims["userid"])
	} else {
		fmt.Println("NOTVALID")
	}
	return nil
}

// authentication middleware: decodes JWT token
func authenticator(next http.Handler) http.Handler {
	// token will either be in header or query string (e.g. sockets)
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if err := authenticate(w, r); err != nil {
			abortWithError(w, err)
			return
		}
		next.ServeHTTP(w, r)
	})
}

// Run runs the application.
func Run(host string) {

	twitterKey := os.Getenv("TWITTER_KEY")
	twitterSecret := os.Getenv("TWITTER_SECRET")

	goth.UseProviders(
		twitter.New(twitterKey, twitterSecret,
			"http://localhost:4000/auth/callback/?provider=twitter",
		),
	)
	gothic.Store = sessions.NewCookieStore([]byte(os.Getenv("SECRET_KEY")))

	cors := cors.New(cors.Options{
		AllowedOrigins:   []string{"*"},
		AllowedHeaders:   []string{"*"},
		AllowCredentials: true,
		Debug:            false,
	})

	authOptions := &yelp.AuthOptions{
		ConsumerKey:       os.Getenv("YELP_CONSUMER_KEY"),
		ConsumerSecret:    os.Getenv("YELP_CONSUMER_SECRET"),
		AccessToken:       os.Getenv("YELP_ACCESS_TOKEN"),
		AccessTokenSecret: os.Getenv("YELP_ACCESS_TOKEN_SECRET"),
	}

	router := mux.NewRouter()

	router.HandleFunc("/search/", func(w http.ResponseWriter, r *http.Request) {

		location := r.URL.Query().Get("location")
		if location == "" {
			abortWithStatus(w, http.StatusBadRequest)
			return
		}

		searchOptions := yelp.SearchOptions{
			LocationOptions: &yelp.LocationOptions{
				Location: location,
			},
			GeneralOptions: &yelp.GeneralOptions{
				CategoryFilter: "bars",
			},
		}

		client := yelp.New(authOptions, nil)
		result, err := client.DoSearch(searchOptions)
		if err != nil {
			abortWithError(w, err)
			return
		}

		bars := make([]Bar, len(result.Businesses))
		userID := getUserID(r)

		for i, biz := range result.Businesses {
			bars[i] = Bar{
				biz.ID,
				biz.ImageURL,
				biz.Name,
				biz.SnippetText,
				db.getTotal(biz.ID),
				db.isGoing(biz.ID, userID),
			}
		}
		w.Header().Set("Content-Type", "application/json")
		json.NewEncoder(w).Encode(bars)
	})

	router.HandleFunc("/ws/", func(w http.ResponseWriter, r *http.Request) {
		ws, err := upgrader.Upgrade(w, r, nil)
		if err != nil {
			abortWithError(w, err)
			return
		}
		cn := &conn{
			send:   make(chan *Message),
			ws:     ws,
			userID: getUserID(r),
			h:      h}
		h.register <- cn

		var wg sync.WaitGroup
		wg.Add(2)
		go cn.write(&wg)
		go cn.read(&wg)
		wg.Wait()
	})

	auth := router.PathPrefix("/auth").Subrouter()

	// redirects to provider
	auth.HandleFunc("/redirect/", gothic.BeginAuthHandler)

	// oauth provider callback
	auth.HandleFunc("/callback/", func(w http.ResponseWriter, r *http.Request) {

		creds, err := gothic.CompleteUserAuth(w, r)
		if err != nil {
			abortWithError(w, err)
			return
		}
		expires := time.Now().Add(time.Hour * 24)
		userID := fmt.Sprintf("%s:%s", creds.Provider, creds.UserID)

		token := jwt.New(jwt.SigningMethodRS256)
		token.Claims["userid"] = userID
		token.Claims["exp"] = expires.Unix()
		tokenStr, err := token.SignedString(signKey)
		if err != nil {
			abortWithError(w, err)
			return
		}
		// the client can now just read the token from the query string
		url := fmt.Sprintf("%s?jwt-token=%s", clientURL, tokenStr)
		http.Redirect(w, r, url, http.StatusTemporaryRedirect)
	})

	// kickoff the connection hub

	go h.run()

	chain := alice.New(cors.Handler, authenticator).Then(router)

	if err := http.ListenAndServe(host, chain); err != nil {
		log.Fatal(err)
	}

}
<commit_msg>Refactoring<commit_after>package handlers

import (
	"crypto/rsa"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/JustinBeckwith/go-yelp/yelp"
	"github.com/dgrijalva/jwt-go"
	"github.com/rs/cors"

	"github.com/justinas/alice"

	"github.com/gorilla/context"
	"github.com/gorilla/mux"
	"github.com/gorilla/sessions"
	"github.com/gorilla/websocket"
	"github.com/markbates/goth"
	"github.com/markbates/goth/gothic"
	"github.com/markbates/goth/providers/twitter"
)

var (
	verifyKey   *rsa.PublicKey
	signKey     *rsa.PrivateKey
	authOptions *yelp.AuthOptions
)

var (
	db = initDB()
	h  = initHub()
)

const (
	privKeyPath    = "keys/app.rsa"     // openssl genrsa -out app.rsa keysize
	pubKeyPath     = "keys/app.rsa.pub" // openssl rsa -in app.rsa -pubout > app.rsa.pub
	clientURL      = "http://localhost:8080"
	userContextKey = "userid"
)

var upgrader = websocket.Upgrader{
	ReadBufferSize:  1024,
	WriteBufferSize: 1024,
	CheckOrigin: func(r *http.Request) bool {
		return true
	},
}

// read the key files before starting http handlers
func init() {
	signBytes, err := ioutil.ReadFile(privKeyPath)
	fatal(err)

	signKey, err = jwt.ParseRSAPrivateKeyFromPEM(signBytes)
	fatal(err)

	verifyBytes, err := ioutil.ReadFile(pubKeyPath)
	fatal(err)

	verifyKey, err = jwt.ParseRSAPublicKeyFromPEM(verifyBytes)

	authOptions = &yelp.AuthOptions{
		ConsumerKey:       os.Getenv("YELP_CONSUMER_KEY"),
		ConsumerSecret:    os.Getenv("YELP_CONSUMER_SECRET"),
		AccessToken:       os.Getenv("YELP_ACCESS_TOKEN"),
		AccessTokenSecret: os.Getenv("YELP_ACCESS_TOKEN_SECRET"),
	}

	twitterKey := os.Getenv("TWITTER_KEY")
	twitterSecret := os.Getenv("TWITTER_SECRET")

	goth.UseProviders(
		twitter.New(twitterKey, twitterSecret,
			"http://localhost:4000/auth/callback/?provider=twitter",
		),
	)

	gothic.Store = sessions.NewCookieStore([]byte(os.Getenv("SECRET_KEY")))
}

func fatal(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

type httpError struct {
	status int
}

func (e httpError) Error() string {
	return http.StatusText(e.status)
}

func newHTTPError(status int) error {
	return httpError{status}
}

func renderJSON(w http.ResponseWriter, status int, payload interface{}) error {
	w.WriteHeader(status)
	w.Header().Set("Content-Type", "application/json")
	return json.NewEncoder(w).Encode(payload)
}

func handleError(w http.ResponseWriter, err error) {
	if httpError, ok := err.(httpError); ok {
		http.Error(w, httpError.Error(), httpError.status)
		return
	}
	// maybe catch other errors such as "no rows found"
	log.Println(err)
	http.Error(w, err.Error(), http.StatusInternalServerError)
}

// grabs user ID from context
func getUserID(r *http.Request) string {
	if userID := context.Get(r, userContextKey); userID != nil {
		return userID.(string)
	}
	return ""
}

func authenticate(w http.ResponseWriter, r *http.Request) error {
	auth := ""
	header := r.Header.Get("Authorization")
	if header == "" {
		// try the query string
		auth = r.URL.Query().Get("jwt-token")
	} else {
		parts := strings.Split(header, "Bearer")
		if len(parts) < 2 {
			return nil
		}
		auth = strings.Trim(parts[1], " ")
	}
	if auth == "" {
		return nil
	}
	t, err := jwt.Parse(auth, func(token *jwt.Token) (interface{}, error) {
		return verifyKey, nil
	})
	if err != nil {
		//check if timeout etc
		log.Println("JWTERROR", err)
		// if a timeout throw a 401, so we can prompt re-login
		return newHTTPError(http.StatusUnauthorized)
	}
	if t.Valid {
		context.Set(r, userContextKey, t.Claims["userid"])
	} else {
		fmt.Println("NOTVALID")
	}
	return nil
}

type handlerFunc func(http.ResponseWriter, *http.Request) error

type handler struct {
	H handlerFunc
}

func (h handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if err := authenticate(w, r); err != nil {
		handleError(w, err)
		return
	}
	if err := h.H(w, r); err != nil {
		handleError(w, err)
	}
}

func newHandler(h handlerFunc) http.Handler {
	return handler{h}
}

func socket(w http.ResponseWriter, r *http.Request) error {
	ws, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		return err
	}
	cn := &conn{
		send:   make(chan *Message),
		ws:     ws,
		userID: getUserID(r),
		h:      h}
	h.register <- cn

	var wg sync.WaitGroup
	wg.Add(2)
	go cn.write(&wg)
	go cn.read(&wg)
	wg.Wait()
	return nil

}

func authCallback(w http.ResponseWriter, r *http.Request) error {
	creds, err := gothic.CompleteUserAuth(w, r)
	if err != nil {
		return err
	}
	expires := time.Now().Add(time.Hour * 24)
	userID := fmt.Sprintf("%s:%s", creds.Provider, creds.UserID)

	token := jwt.New(jwt.SigningMethodRS256)
	token.Claims["userid"] = userID
	token.Claims["exp"] = expires.Unix()
	tokenStr, err := token.SignedString(signKey)
	if err != nil {
		return err
	}
	// the client can now just read the token from the query string
	url := fmt.Sprintf("%s?jwt-token=%s", clientURL, tokenStr)
	http.Redirect(w, r, url, http.StatusTemporaryRedirect)
	return nil
}

func searchLocation(w http.ResponseWriter, r *http.Request) error {
	location := r.URL.Query().Get("location")
	if location == "" {
		return newHTTPError(http.StatusBadRequest)
	}

	searchOptions := yelp.SearchOptions{
		LocationOptions: &yelp.LocationOptions{
			Location: location,
		},
		GeneralOptions: &yelp.GeneralOptions{
			CategoryFilter: "bars",
		},
	}

	client := yelp.New(authOptions, nil)
	result, err := client.DoSearch(searchOptions)
	if err != nil {
		return err
	}

	bars := make([]Bar, len(result.Businesses))
	userID := getUserID(r)

	for i, biz := range result.Businesses {
		bars[i] = Bar{
			biz.ID,
			biz.ImageURL,
			biz.Name,
			biz.SnippetText,
			db.getTotal(biz.ID),
			db.isGoing(biz.ID, userID),
		}
	}
	return renderJSON(w, http.StatusOK, bars)
}

// Run runs the application.
func Run(host string) {

	cors := cors.New(cors.Options{
		AllowedOrigins:   []string{"*"},
		AllowedHeaders:   []string{"*"},
		AllowCredentials: true,
		Debug:            false,
	})

	router := mux.NewRouter()

	router.Handle("/search/", newHandler(searchLocation)).Methods("GET")
	router.Handle("/ws/", newHandler(socket)).Methods("GET")

	auth := router.PathPrefix("/auth").Subrouter()

	// oauth provider callback
	auth.Handle("/callback/", newHandler(authCallback))

	// redirects to provider
	auth.HandleFunc("/redirect/", gothic.BeginAuthHandler)

	// kickoff the connection hub

	go h.run()

	chain := alice.New(cors.Handler).Then(router)

	if err := http.ListenAndServe(host, chain); err != nil {
		log.Fatal(err)
	}

}
<|endoftext|>