<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package catacomb

import (
	"fmt"
	"sync"
	"sync/atomic"

	"github.com/juju/errors"
	"launchpad.net/tomb"

	"github.com/juju/juju/worker"
)

// Catacomb is a variant of tomb.Tomb with its own internal goroutine, designed
// for coordinating the lifetimes of private workers needed by a single parent.
//
// As a client, you should only ever create zero values; these should be used
// with Invoke to manage a parent task. No Catacomb methods are meaningful
// until the catacomb has been started with a successful Invoke.
//
// See the package documentation for more detailed discussion and usage notes.
type Catacomb struct {
	tomb  tomb.Tomb
	wg    sync.WaitGroup
	adds  chan worker.Worker
	dirty int32
}

// Plan defines the strategy for an Invoke.
type Plan struct {

	// Site must point to an unused Catacomb.
	Site *Catacomb

	// Work will be run on a new goroutine, and tracked by Site.
	Work func() error

	// Init contains additional workers for which Site must be responsible.
	Init []worker.Worker
}

// Validate returns an error if the plan cannot be used. It doesn't check for
// reused catacombs: plan validity is necessary but not sufficient to determine
// that an Invoke will succeed.
func (plan Plan) Validate() error {
	if plan.Site == nil {
		return errors.NotValidf("nil Site")
	}
	if plan.Work == nil {
		return errors.NotValidf("nil Work")
	}
	for i, w := range plan.Init {
		if w == nil {
			return errors.NotValidf("nil Init item %d", i)
		}
	}
	return nil
}

// Invoke uses the plan's catacomb to run the work func. It will return an
// error if the plan is not valid, or if the catacomb has already been used.
// If Invoke returns no error, the catacomb is now controlling the work func,
// and its exported methods can be called safely.
//
// Invoke takes responsibility for all workers in plan.Init, *whether or not
// it succeeds*.
func Invoke(plan Plan) (err error) {

	defer func() {
		if err != nil {
			stopWorkers(plan.Init)
		}
	}()

	if err := plan.Validate(); err != nil {
		return errors.Trace(err)
	}
	catacomb := plan.Site
	if !atomic.CompareAndSwapInt32(&catacomb.dirty, 0, 1) {
		return errors.Errorf("catacomb %p has already been used", catacomb)
	}
	catacomb.adds = make(chan worker.Worker)

	// Add the Init workers right away, so the client can't induce data races
	// by modifying the slice post-return.
	for _, w := range plan.Init {
		catacomb.add(w)
	}

	// This goroutine listens for added workers until the catacomb is Killed.
	// We ensure the wg can't complete until we know no new workers will be
	// added.
	catacomb.wg.Add(1)
	go func() {
		defer catacomb.wg.Done()
		for {
			select {
			case <-catacomb.tomb.Dying():
				return
			case w := <-catacomb.adds:
				catacomb.add(w)
			}
		}
	}()

	// This goroutine runs the work func and stops the catacomb with its error;
	// and waits for for the listen goroutine and all added workers to complete
	// before marking the catacomb's tomb Dead.
	go func() {
		defer catacomb.tomb.Done()
		defer catacomb.wg.Wait()
		catacomb.Kill(runSafely(plan.Work))
	}()
	return nil
}

// stopWorkers stops all non-nil workers in the supplied slice, and swallows
// all errors. This is consistent, for now, because Catacomb swallows all
// errors but the first; as we come to rank or log errors, this must change
// to accommodate better practices.
func stopWorkers(workers []worker.Worker) {
	for _, w := range workers {
		if w != nil {
			worker.Stop(w)
		}
	}
}

// Add causes the supplied worker's lifetime to be bound to the catacomb's,
// relieving the client of responsibility for Kill()ing it and Wait()ing for an
// error, *whether or not this method succeeds*. If the method returns an error,
// it always indicates that the catacomb is shutting down; the value will either
// be the error from the (now-stopped) worker, or catacomb.ErrDying().
//
// If the worker completes without error, the catacomb will continue unaffected;
// otherwise the catacomb's tomb will be killed with the returned error. This
// allows clients to freely Kill() workers that have been Add()ed; any errors
// encountered will still kill the catacomb, so the workers stay under control
// until the last moment, and so can be managed pretty casually once they've
// been added.
//
// Don't try to add a worker to its own catacomb; that'll deadlock the shutdown
// procedure. I don't think there's much we can do about that.
func (catacomb *Catacomb) Add(w worker.Worker) error {
	select {
	case <-catacomb.tomb.Dying():
		if err := worker.Stop(w); err != nil {
			return errors.Trace(err)
		}
		return catacomb.ErrDying()
	case catacomb.adds <- w:
		// Note that we don't need to wait for confirmation here. This depends
		// on the catacomb.wg.Add() for the listen loop, which ensures the wg
		// won't complete until no more adds can be received.
		return nil
	}
}

// add starts two goroutines that (1) kill the catacomb's tomb with any
// error encountered by the worker; and (2) kill the worker when the
// catacomb starts dying.
func (catacomb *Catacomb) add(w worker.Worker) {
	// We must wait for _both_ goroutines to exit in
	// arbitrary order depending on the order of the worker
	// and the catacomb shutting down.
	catacomb.wg.Add(2)
	go func() {
		defer catacomb.wg.Done()
		if err := w.Wait(); err != nil {
			catacomb.Kill(err)
		}
	}()
	go func() {
		defer catacomb.wg.Done()
		<-catacomb.tomb.Dying()
		worker.Stop(w)
	}()
}

// Dying returns a channel that will be closed when Kill is called.
func (catacomb *Catacomb) Dying() <-chan struct{} {
	return catacomb.tomb.Dying()
}

// Dead returns a channel that will be closed when Invoke has completed (and
// thus when subsequent calls to Wait() are known not to block).
func (catacomb *Catacomb) Dead() <-chan struct{} {
	return catacomb.tomb.Dead()
}

// Wait blocks until Invoke completes, and returns the first non-nil and
// non-tomb.ErrDying error passed to Kill before Invoke finished.
func (catacomb *Catacomb) Wait() error {
	return catacomb.tomb.Wait()
}

// Kill kills the Catacomb's internal tomb with the supplied error, or one
// derived from it.
//  * if it's caused by this catacomb's ErrDying, it passes on tomb.ErrDying.
//  * if it's tomb.ErrDying, or caused by another catacomb's ErrDying, it passes
//    on a new error complaining about the misuse.
//  * all other errors are passed on unmodified.
// It's always safe to call Kill, but errors passed to Kill after the catacomb
// is dead will be ignored.
func (catacomb *Catacomb) Kill(err error) {
	if err == tomb.ErrDying {
		err = errors.New("bad catacomb Kill: tomb.ErrDying")
	}
	cause := errors.Cause(err)
	if match, ok := cause.(dyingError); ok {
		if catacomb != match.catacomb {
			err = errors.Errorf("bad catacomb Kill: other catacomb's ErrDying")
		} else {
			err = tomb.ErrDying
		}
	}

	// TODO(fwereade) it's pretty clear that this ought to be a Kill(nil), and
	// the catacomb should be responsible for ranking errors, just like the
	// dependency.Engine does, rather than determining priority by scheduling
	// alone.
	catacomb.tomb.Kill(err)
}

// ErrDying returns an error that can be used to Kill *this* catacomb without
// overwriting nil errors. It should only be used when the catacomb is already
// known to be dying; calling this method at any other time will return a
// different error, indicating client misuse.
func (catacomb *Catacomb) ErrDying() error {
	select {
	case <-catacomb.tomb.Dying():
		return dyingError{catacomb}
	default:
		return errors.New("bad catacomb ErrDying: still alive")
	}
}

// dyingError holds a reference to the catacomb that created it.
type dyingError struct {
	catacomb *Catacomb
}

// Error is part of the error interface.
func (err dyingError) Error() string {
	return fmt.Sprintf("catacomb %p is dying", err.catacomb)
}

// RunSafely will ensure that the function is run, and any error is returned.
// If there is a panic, then that will be returned as an error.
func runSafely(f func() error) (err error) {
	defer func() {
		if panicResult := recover(); panicResult != nil {
			err = errors.Errorf("panic resulted in: %v", panicResult)
		}
	}()
	return f()
}
<commit_msg>typo fix<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package catacomb

import (
	"fmt"
	"sync"
	"sync/atomic"

	"github.com/juju/errors"
	"launchpad.net/tomb"

	"github.com/juju/juju/worker"
)

// Catacomb is a variant of tomb.Tomb with its own internal goroutine, designed
// for coordinating the lifetimes of private workers needed by a single parent.
//
// As a client, you should only ever create zero values; these should be used
// with Invoke to manage a parent task. No Catacomb methods are meaningful
// until the catacomb has been started with a successful Invoke.
//
// See the package documentation for more detailed discussion and usage notes.
type Catacomb struct {
	tomb  tomb.Tomb
	wg    sync.WaitGroup
	adds  chan worker.Worker
	dirty int32
}

// Plan defines the strategy for an Invoke.
type Plan struct {

	// Site must point to an unused Catacomb.
	Site *Catacomb

	// Work will be run on a new goroutine, and tracked by Site.
	Work func() error

	// Init contains additional workers for which Site must be responsible.
	Init []worker.Worker
}

// Validate returns an error if the plan cannot be used. It doesn't check for
// reused catacombs: plan validity is necessary but not sufficient to determine
// that an Invoke will succeed.
func (plan Plan) Validate() error {
	if plan.Site == nil {
		return errors.NotValidf("nil Site")
	}
	if plan.Work == nil {
		return errors.NotValidf("nil Work")
	}
	for i, w := range plan.Init {
		if w == nil {
			return errors.NotValidf("nil Init item %d", i)
		}
	}
	return nil
}

// Invoke uses the plan's catacomb to run the work func. It will return an
// error if the plan is not valid, or if the catacomb has already been used.
// If Invoke returns no error, the catacomb is now controlling the work func,
// and its exported methods can be called safely.
//
// Invoke takes responsibility for all workers in plan.Init, *whether or not
// it succeeds*.
func Invoke(plan Plan) (err error) {

	defer func() {
		if err != nil {
			stopWorkers(plan.Init)
		}
	}()

	if err := plan.Validate(); err != nil {
		return errors.Trace(err)
	}
	catacomb := plan.Site
	if !atomic.CompareAndSwapInt32(&catacomb.dirty, 0, 1) {
		return errors.Errorf("catacomb %p has already been used", catacomb)
	}
	catacomb.adds = make(chan worker.Worker)

	// Add the Init workers right away, so the client can't induce data races
	// by modifying the slice post-return.
	for _, w := range plan.Init {
		catacomb.add(w)
	}

	// This goroutine listens for added workers until the catacomb is Killed.
	// We ensure the wg can't complete until we know no new workers will be
	// added.
	catacomb.wg.Add(1)
	go func() {
		defer catacomb.wg.Done()
		for {
			select {
			case <-catacomb.tomb.Dying():
				return
			case w := <-catacomb.adds:
				catacomb.add(w)
			}
		}
	}()

	// This goroutine runs the work func and stops the catacomb with its error;
	// and waits for for the listen goroutine and all added workers to complete
	// before marking the catacomb's tomb Dead.
	go func() {
		defer catacomb.tomb.Done()
		defer catacomb.wg.Wait()
		catacomb.Kill(runSafely(plan.Work))
	}()
	return nil
}

// stopWorkers stops all non-nil workers in the supplied slice, and swallows
// all errors. This is consistent, for now, because Catacomb swallows all
// errors but the first; as we come to rank or log errors, this must change
// to accommodate better practices.
func stopWorkers(workers []worker.Worker) {
	for _, w := range workers {
		if w != nil {
			worker.Stop(w)
		}
	}
}

// Add causes the supplied worker's lifetime to be bound to the catacomb's,
// relieving the client of responsibility for Kill()ing it and Wait()ing for an
// error, *whether or not this method succeeds*. If the method returns an error,
// it always indicates that the catacomb is shutting down; the value will either
// be the error from the (now-stopped) worker, or catacomb.ErrDying().
//
// If the worker completes without error, the catacomb will continue unaffected;
// otherwise the catacomb's tomb will be killed with the returned error. This
// allows clients to freely Kill() workers that have been Add()ed; any errors
// encountered will still kill the catacomb, so the workers stay under control
// until the last moment, and so can be managed pretty casually once they've
// been added.
//
// Don't try to add a worker to its own catacomb; that'll deadlock the shutdown
// procedure. I don't think there's much we can do about that.
func (catacomb *Catacomb) Add(w worker.Worker) error {
	select {
	case <-catacomb.tomb.Dying():
		if err := worker.Stop(w); err != nil {
			return errors.Trace(err)
		}
		return catacomb.ErrDying()
	case catacomb.adds <- w:
		// Note that we don't need to wait for confirmation here. This depends
		// on the catacomb.wg.Add() for the listen loop, which ensures the wg
		// won't complete until no more adds can be received.
		return nil
	}
}

// add starts two goroutines that (1) kill the catacomb's tomb with any
// error encountered by the worker; and (2) kill the worker when the
// catacomb starts dying.
func (catacomb *Catacomb) add(w worker.Worker) {
	// We must wait for _both_ goroutines to exit in
	// arbitrary order depending on the order of the worker
	// and the catacomb shutting down.
	catacomb.wg.Add(2)
	go func() {
		defer catacomb.wg.Done()
		if err := w.Wait(); err != nil {
			catacomb.Kill(err)
		}
	}()
	go func() {
		defer catacomb.wg.Done()
		<-catacomb.tomb.Dying()
		worker.Stop(w)
	}()
}

// Dying returns a channel that will be closed when Kill is called.
func (catacomb *Catacomb) Dying() <-chan struct{} {
	return catacomb.tomb.Dying()
}

// Dead returns a channel that will be closed when Invoke has completed (and
// thus when subsequent calls to Wait() are known not to block).
func (catacomb *Catacomb) Dead() <-chan struct{} {
	return catacomb.tomb.Dead()
}

// Wait blocks until Invoke completes, and returns the first non-nil and
// non-tomb.ErrDying error passed to Kill before Invoke finished.
func (catacomb *Catacomb) Wait() error {
	return catacomb.tomb.Wait()
}

// Kill kills the Catacomb's internal tomb with the supplied error, or one
// derived from it.
//  * if it's caused by this catacomb's ErrDying, it passes on tomb.ErrDying.
//  * if it's tomb.ErrDying, or caused by another catacomb's ErrDying, it passes
//    on a new error complaining about the misuse.
//  * all other errors are passed on unmodified.
// It's always safe to call Kill, but errors passed to Kill after the catacomb
// is dead will be ignored.
func (catacomb *Catacomb) Kill(err error) {
	if err == tomb.ErrDying {
		err = errors.New("bad catacomb Kill: tomb.ErrDying")
	}
	cause := errors.Cause(err)
	if match, ok := cause.(dyingError); ok {
		if catacomb != match.catacomb {
			err = errors.Errorf("bad catacomb Kill: other catacomb's ErrDying")
		} else {
			err = tomb.ErrDying
		}
	}

	// TODO(fwereade) it's pretty clear that this ought to be a Kill(nil), and
	// the catacomb should be responsible for ranking errors, just like the
	// dependency.Engine does, rather than determining priority by scheduling
	// alone.
	catacomb.tomb.Kill(err)
}

// ErrDying returns an error that can be used to Kill *this* catacomb without
// overwriting nil errors. It should only be used when the catacomb is already
// known to be dying; calling this method at any other time will return a
// different error, indicating client misuse.
func (catacomb *Catacomb) ErrDying() error {
	select {
	case <-catacomb.tomb.Dying():
		return dyingError{catacomb}
	default:
		return errors.New("bad catacomb ErrDying: still alive")
	}
}

// dyingError holds a reference to the catacomb that created it.
type dyingError struct {
	catacomb *Catacomb
}

// Error is part of the error interface.
func (err dyingError) Error() string {
	return fmt.Sprintf("catacomb %p is dying", err.catacomb)
}

// runSafely will ensure that the function is run, and any error is returned.
// If there is a panic, then that will be returned as an error.
func runSafely(f func() error) (err error) {
	defer func() {
		if panicResult := recover(); panicResult != nil {
			err = errors.Errorf("panic resulted in: %v", panicResult)
		}
	}()
	return f()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"sort"
	"strings"
	"sync"
	"syscall"
	"testing"
	"time"

	"github.com/peferron/alternate/testbin"
)

const (
	zero  time.Duration = 0
	one                 = 10 * time.Millisecond
	two                 = 2 * one
	three               = 3 * one
)

func init() {
	fmt.Println("Initializing testExit channel")
	testExit = make(chan struct{})
}

func newNilWriter() *nilWriter {
	return &nilWriter{}
}

type nilWriter struct{}

func (w *nilWriter) Write(p []byte) (n int, err error) {
	return len(p), nil
}

func newLineWriter(log bool) *lineWriter {
	return &lineWriter{
		"",
		[]string{},
		&sync.Mutex{},
		log,
	}
}

type lineWriter struct {
	buf   string
	lines []string
	mutex *sync.Mutex
	log   bool
}

func (w *lineWriter) Write(p []byte) (n int, err error) {
	w.mutex.Lock()
	defer w.mutex.Unlock()

	n = len(p)

	w.buf += string(p)
	a := strings.Split(w.buf, "\n")
	for _, s := range a {
		if s == "" {
			continue
		}
		w.lines = append(w.lines, s)
		if w.log {
			fmt.Printf("LineWriter received: %q\n", s)
		}
	}
	w.buf = a[len(a)-1]

	return
}

func (w *lineWriter) reset() {
	w.mutex.Lock()
	defer w.mutex.Unlock()

	w.buf = ""
	w.lines = []string{}
}

func (w *lineWriter) getLines() []string {
	w.mutex.Lock()
	defer w.mutex.Unlock()

	clone := make([]string, len(w.lines))
	for i, s := range w.lines {
		clone[i] = s
	}
	return clone
}

type test struct {
	t           *testing.T
	cmdStdout   *lineWriter
	cmdStderr   *lineWriter
	params      []string
	expectIndex int
}

func (test *test) reset() {
	test.cmdStdout.reset()
	test.cmdStderr.reset()
}

func (test *test) expect(d time.Duration, lines []string) {
	expectIndex := test.expectIndex
	test.expectIndex++

	time.Sleep(d)

	stdoutLines := test.cmdStdout.getLines()
	if !sameStrings(lines, stdoutLines) {
		fmt.Printf("For parameters %q expect #%d, within %v expected lines %q in stdout, was %q\n",
			test.params, expectIndex, d, lines, stdoutLines)
		test.t.Errorf("For parameters %q expect #%d, within %v expected lines %q in stdout, was %q",
			test.params, expectIndex, d, lines, stdoutLines)
	}
	stderrLines := test.cmdStderr.getLines()
	if !sameStrings(lines, stderrLines) {
		fmt.Printf("For parameters %q expect #%d, within %v expected lines %q in stderr, was %q\n",
			test.params, expectIndex, d, lines, stderrLines)
		test.t.Errorf("For parameters %q expect #%d, within %v expected lines %q in stderr, was %q",
			test.params, expectIndex, d, lines, stderrLines)
	}
}

// sameStrings makes an unordered comparison of two slices of strings, and returns true if they
// contain the same strings, or false otherwise.
func sameStrings(a []string, b []string) bool {
	if len(a) != len(b) {
		return false
	}
	if len(a) == 0 {
		return true
	}
	aa := clone(a)
	bb := clone(b)
	sort.Strings(aa)
	sort.Strings(bb)
	for i := range aa {
		if aa[i] != bb[i] {
			return false
		}
	}
	return true
}

// clone returns a deep copy of a slice of strings.
func clone(a []string) []string {
	b := make([]string, len(a))
	for i := range a {
		b[i] = a[i]
	}
	return b
}

func start(t *testing.T, params []string, overlap time.Duration) *test {
	test := &test{
		t,
		newLineWriter(false),
		newLineWriter(false),
		params,
		0,
	}
	c := testbin.Build() + " " + placeholder
	go alternate(c, placeholder, params, overlap, newNilWriter(), test.cmdStdout, test.cmdStderr)
	return test
}

func sendUsr1() {
	process().Signal(syscall.SIGUSR1)
}

func stop() {
	testExit <- struct{}{}
	time.Sleep(one)
}

func process() *os.Process {
	p, err := os.FindProcess(os.Getpid())
	if err != nil {
		panic(err)
	}
	return p
}

func TestLineWriter(t *testing.T) {
	type action struct {
		reset bool
		write string
	}
	tests := []struct {
		actions []action
		lines   []string
	}{
		{
			[]action{
				{false, "abc\nde\n\nf"},
			},
			[]string{"abc", "de", "f"},
		},
	}

	for i, test := range tests {
		writer := newLineWriter(false)
		for _, a := range test.actions {
			if a.reset {
				writer.reset()
			}
			if a.write != "" {
				writer.Write([]byte(a.write))
			}
		}

		lines := writer.getLines()

		same := true
		if len(test.lines) != len(lines) {
			same = false
		} else {
			for j := range test.lines {
				if test.lines[j] != lines[j] {
					same = false
					break
				}
			}
		}

		if !same {
			t.Errorf("For test #%d, expected lines to be %q, but was %q", i, test.lines, lines)
		}
	}
}

func TestNoOverlapNoConflict(t *testing.T) {
	paramSets := [][]string{
		{"param0", "param1"},
		{"param0", "param1", "param0"},
	}
	o := zero

	for _, v := range paramSets {
		a := testbin.SetBehavior(-one, zero, "a")
		test := start(t, v, o)
		test.expect(one, []string{
			"param0 " + a + " | start",
		})

		b := testbin.SetBehavior(-one, zero, "b")
		test.reset()
		sendUsr1()
		test.expect(one, []string{
			"param1 " + b + " | start",
			"param0 " + a + " | exit",
		})

		c := testbin.SetBehavior(-one, zero, "c")
		test.reset()
		sendUsr1()
		test.expect(one, []string{
			"param0 " + c + " | start",
			"param1 " + b + " | exit",
		})

		stop()
	}
}

func TestOverlapNoConflict(t *testing.T) {
	paramSets := [][]string{
		{"param0", "param1"},
		{"param0", "param1", "param0"},
	}
	o := two

	for _, v := range paramSets {
		a := testbin.SetBehavior(-one, zero, "a")
		test := start(t, v, o)
		test.expect(one, []string{
			"param0 " + a + " | start",
		})

		b := testbin.SetBehavior(-one, zero, "b")
		test.reset()
		sendUsr1()
		test.expect(one, []string{
			"param1 " + b + " | start",
		})
		test.reset()
		test.expect(two, []string{
			"param0 " + a + " | exit",
		})

		c := testbin.SetBehavior(-one, zero, "c")
		test.reset()
		sendUsr1()
		test.expect(one, []string{
			"param0 " + c + " | start",
		})
		test.reset()
		test.expect(two, []string{
			"param1 " + b + " | exit",
		})

		stop()
	}
}

func TestNoOverlapConflict(t *testing.T) {
	paramSets := [][]string{
		{"param0", "param1"},
		{"param0", "param1", "param0"},
	}
	o := zero

	for _, v := range paramSets {
		a := testbin.SetBehavior(-one, -one, "a")
		test := start(t, v, o)
		test.expect(one, []string{
			"param0 " + a + " | start",
		})

		b := testbin.SetBehavior(-one, zero, "b")
		test.reset()
		sendUsr1()
		test.expect(one, []string{
			"param1 " + b + " | start",
		})

		testbin.SetBehavior(-one, zero, "c")
		test.reset()
		sendUsr1()
		test.expect(one, []string{})

		stop()
	}
}

func TestOverlapConflict(t *testing.T) {
	paramSets := [][]string{
		{"param0", "param1"},
		{"param0", "param1", "param0"},
	}
	o := two

	for _, v := range paramSets {
		a := testbin.SetBehavior(-one, -one, "a")
		test := start(t, v, o)
		test.expect(one, []string{
			"param0 " + a + " | start",
		})

		b := testbin.SetBehavior(-one, zero, "b")
		test.reset()
		sendUsr1()
		test.expect(one, []string{
			"param1 " + b + " | start",
		})
		test.reset()
		test.expect(two, []string{})

		testbin.SetBehavior(-one, zero, "c")
		test.reset()
		sendUsr1()
		test.expect(three, []string{})

		stop()
	}
}

func TestPrematureCmdExit(t *testing.T) {
	paramSets := [][]string{
		{"param0", "param1"},
		{"param0", "param1", "param0"},
	}
	o := two

	for _, v := range paramSets {
		a := testbin.SetBehavior(-one, zero, "a")
		test := start(t, v, o)
		test.expect(one, []string{
			"param0 " + a + " | start",
		})

		b := testbin.SetBehavior(zero, zero, "b")
		test.reset()
		sendUsr1()
		test.expect(one, []string{
			"param1 " + b + " | start",
			"param1 " + b + " | exit",
		})
		test.reset()
		test.expect(two, []string{})

		c := testbin.SetBehavior(-one, zero, "c")
		test.reset()
		sendUsr1()
		test.expect(one, []string{
			"param1 " + c + " | start",
		})
		test.reset()
		test.expect(two, []string{
			"param0 " + a + " | exit",
		})

		stop()
	}
}

func TestCmdRunError(t *testing.T) {
	paramSets := [][]string{
		{"param0", "param1"},
		{"param0", "param1", "param0"},
	}
	o := zero

	for _, v := range paramSets {
		a := testbin.SetBehavior(-one, zero, "a")
		test := start(t, v, o)
		test.expect(one, []string{
			"param0 " + a + " | start",
		})

		b := testbin.SetBehavior(-one, zero, "b")
		test.reset()
		testbin.Clean()
		sendUsr1()
		test.expect(one, []string{})

		test.reset()
		sendUsr1()
		test.expect(one, []string{})

		testbin.Build()
		test.reset()
		sendUsr1()
		test.expect(one, []string{
			"param1 " + b + " | start",
			"param0 " + a + " | exit",
		})

		stop()
	}
}

func TestAlternateExit(t *testing.T) {
	c := testbin.Build() + " " + placeholder
	v := []string{"param0"}
	o := zero

	testbin.SetBehavior(two, zero, "a")

	exited := false
	go func() {
		alternate(c, placeholder, v, o, newNilWriter(), newNilWriter(), newNilWriter())
		exited = true
	}()

	time.Sleep(one)
	if exited {
		t.Error("Was expecting exited to be false, was true")
	}

	time.Sleep(two)
	if !exited {
		t.Error("Was expecting exited to be true, was false")
	}
}
<commit_msg>Bring back TestSameStrings wrongfully removed in commit 11dc6107b4f0c2eabd9af7900e1c1578826f12b2<commit_after>package main

import (
	"fmt"
	"os"
	"sort"
	"strings"
	"sync"
	"syscall"
	"testing"
	"time"

	"github.com/peferron/alternate/testbin"
)

const (
	zero  time.Duration = 0
	one                 = 10 * time.Millisecond
	two                 = 2 * one
	three               = 3 * one
)

func init() {
	fmt.Println("Initializing testExit channel")
	testExit = make(chan struct{})
}

func newNilWriter() *nilWriter {
	return &nilWriter{}
}

type nilWriter struct{}

func (w *nilWriter) Write(p []byte) (n int, err error) {
	return len(p), nil
}

func newLineWriter(log bool) *lineWriter {
	return &lineWriter{
		"",
		[]string{},
		&sync.Mutex{},
		log,
	}
}

type lineWriter struct {
	buf   string
	lines []string
	mutex *sync.Mutex
	log   bool
}

func (w *lineWriter) Write(p []byte) (n int, err error) {
	w.mutex.Lock()
	defer w.mutex.Unlock()

	n = len(p)

	w.buf += string(p)
	a := strings.Split(w.buf, "\n")
	for _, s := range a {
		if s == "" {
			continue
		}
		w.lines = append(w.lines, s)
		if w.log {
			fmt.Printf("LineWriter received: %q\n", s)
		}
	}
	w.buf = a[len(a)-1]

	return
}

func (w *lineWriter) reset() {
	w.mutex.Lock()
	defer w.mutex.Unlock()

	w.buf = ""
	w.lines = []string{}
}

func (w *lineWriter) getLines() []string {
	w.mutex.Lock()
	defer w.mutex.Unlock()

	clone := make([]string, len(w.lines))
	for i, s := range w.lines {
		clone[i] = s
	}
	return clone
}

type test struct {
	t           *testing.T
	cmdStdout   *lineWriter
	cmdStderr   *lineWriter
	params      []string
	expectIndex int
}

func (test *test) reset() {
	test.cmdStdout.reset()
	test.cmdStderr.reset()
}

func (test *test) expect(d time.Duration, lines []string) {
	expectIndex := test.expectIndex
	test.expectIndex++

	time.Sleep(d)

	stdoutLines := test.cmdStdout.getLines()
	if !sameStrings(lines, stdoutLines) {
		fmt.Printf("For parameters %q expect #%d, within %v expected lines %q in stdout, was %q\n",
			test.params, expectIndex, d, lines, stdoutLines)
		test.t.Errorf("For parameters %q expect #%d, within %v expected lines %q in stdout, was %q",
			test.params, expectIndex, d, lines, stdoutLines)
	}
	stderrLines := test.cmdStderr.getLines()
	if !sameStrings(lines, stderrLines) {
		fmt.Printf("For parameters %q expect #%d, within %v expected lines %q in stderr, was %q\n",
			test.params, expectIndex, d, lines, stderrLines)
		test.t.Errorf("For parameters %q expect #%d, within %v expected lines %q in stderr, was %q",
			test.params, expectIndex, d, lines, stderrLines)
	}
}

// sameStrings makes an unordered comparison of two slices of strings, and returns true if they
// contain the same strings, or false otherwise.
func sameStrings(a []string, b []string) bool {
	if len(a) != len(b) {
		return false
	}
	if len(a) == 0 {
		return true
	}
	aa := clone(a)
	bb := clone(b)
	sort.Strings(aa)
	sort.Strings(bb)
	for i := range aa {
		if aa[i] != bb[i] {
			return false
		}
	}
	return true
}

// clone returns a deep copy of a slice of strings.
func clone(a []string) []string {
	b := make([]string, len(a))
	for i := range a {
		b[i] = a[i]
	}
	return b
}

func start(t *testing.T, params []string, overlap time.Duration) *test {
	test := &test{
		t,
		newLineWriter(false),
		newLineWriter(false),
		params,
		0,
	}
	c := testbin.Build() + " " + placeholder
	go alternate(c, placeholder, params, overlap, newNilWriter(), test.cmdStdout, test.cmdStderr)
	return test
}

func sendUsr1() {
	process().Signal(syscall.SIGUSR1)
}

func stop() {
	testExit <- struct{}{}
	time.Sleep(one)
}

func process() *os.Process {
	p, err := os.FindProcess(os.Getpid())
	if err != nil {
		panic(err)
	}
	return p
}

func TestSameStrings(t *testing.T) {
	tests := []struct {
		a    []string
		b    []string
		same bool
	}{
		{[]string{"a", "b"}, []string{"a", "b"}, true},
		{[]string{"b", "a"}, []string{"a", "b"}, true},
		{[]string{"a", "b", "c"}, []string{"a", "b"}, false},
	}

	for _, test := range tests {
		same := sameStrings(test.a, test.b)
		if test.same != same {
			t.Errorf("For values %q and %q, expected same to be %t, was %t",
				test.a, test.b, test.same, same)
		}
	}
}

func TestLineWriter(t *testing.T) {
	type action struct {
		reset bool
		write string
	}
	tests := []struct {
		actions []action
		lines   []string
	}{
		{
			[]action{
				{false, "abc\nde\n\nf"},
			},
			[]string{"abc", "de", "f"},
		},
	}

	for i, test := range tests {
		writer := newLineWriter(false)
		for _, a := range test.actions {
			if a.reset {
				writer.reset()
			}
			if a.write != "" {
				writer.Write([]byte(a.write))
			}
		}

		lines := writer.getLines()

		same := true
		if len(test.lines) != len(lines) {
			same = false
		} else {
			for j := range test.lines {
				if test.lines[j] != lines[j] {
					same = false
					break
				}
			}
		}

		if !same {
			t.Errorf("For test #%d, expected lines to be %q, but was %q", i, test.lines, lines)
		}
	}
}

func TestNoOverlapNoConflict(t *testing.T) {
	paramSets := [][]string{
		{"param0", "param1"},
		{"param0", "param1", "param0"},
	}
	o := zero

	for _, v := range paramSets {
		a := testbin.SetBehavior(-one, zero, "a")
		test := start(t, v, o)
		test.expect(one, []string{
			"param0 " + a + " | start",
		})

		b := testbin.SetBehavior(-one, zero, "b")
		test.reset()
		sendUsr1()
		test.expect(one, []string{
			"param1 " + b + " | start",
			"param0 " + a + " | exit",
		})

		c := testbin.SetBehavior(-one, zero, "c")
		test.reset()
		sendUsr1()
		test.expect(one, []string{
			"param0 " + c + " | start",
			"param1 " + b + " | exit",
		})

		stop()
	}
}

func TestOverlapNoConflict(t *testing.T) {
	paramSets := [][]string{
		{"param0", "param1"},
		{"param0", "param1", "param0"},
	}
	o := two

	for _, v := range paramSets {
		a := testbin.SetBehavior(-one, zero, "a")
		test := start(t, v, o)
		test.expect(one, []string{
			"param0 " + a + " | start",
		})

		b := testbin.SetBehavior(-one, zero, "b")
		test.reset()
		sendUsr1()
		test.expect(one, []string{
			"param1 " + b + " | start",
		})
		test.reset()
		test.expect(two, []string{
			"param0 " + a + " | exit",
		})

		c := testbin.SetBehavior(-one, zero, "c")
		test.reset()
		sendUsr1()
		test.expect(one, []string{
			"param0 " + c + " | start",
		})
		test.reset()
		test.expect(two, []string{
			"param1 " + b + " | exit",
		})

		stop()
	}
}

func TestNoOverlapConflict(t *testing.T) {
	paramSets := [][]string{
		{"param0", "param1"},
		{"param0", "param1", "param0"},
	}
	o := zero

	for _, v := range paramSets {
		a := testbin.SetBehavior(-one, -one, "a")
		test := start(t, v, o)
		test.expect(one, []string{
			"param0 " + a + " | start",
		})

		b := testbin.SetBehavior(-one, zero, "b")
		test.reset()
		sendUsr1()
		test.expect(one, []string{
			"param1 " + b + " | start",
		})

		testbin.SetBehavior(-one, zero, "c")
		test.reset()
		sendUsr1()
		test.expect(one, []string{})

		stop()
	}
}

func TestOverlapConflict(t *testing.T) {
	paramSets := [][]string{
		{"param0", "param1"},
		{"param0", "param1", "param0"},
	}
	o := two

	for _, v := range paramSets {
		a := testbin.SetBehavior(-one, -one, "a")
		test := start(t, v, o)
		test.expect(one, []string{
			"param0 " + a + " | start",
		})

		b := testbin.SetBehavior(-one, zero, "b")
		test.reset()
		sendUsr1()
		test.expect(one, []string{
			"param1 " + b + " | start",
		})
		test.reset()
		test.expect(two, []string{})

		testbin.SetBehavior(-one, zero, "c")
		test.reset()
		sendUsr1()
		test.expect(three, []string{})

		stop()
	}
}

func TestPrematureCmdExit(t *testing.T) {
	paramSets := [][]string{
		{"param0", "param1"},
		{"param0", "param1", "param0"},
	}
	o := two

	for _, v := range paramSets {
		a := testbin.SetBehavior(-one, zero, "a")
		test := start(t, v, o)
		test.expect(one, []string{
			"param0 " + a + " | start",
		})

		b := testbin.SetBehavior(zero, zero, "b")
		test.reset()
		sendUsr1()
		test.expect(one, []string{
			"param1 " + b + " | start",
			"param1 " + b + " | exit",
		})
		test.reset()
		test.expect(two, []string{})

		c := testbin.SetBehavior(-one, zero, "c")
		test.reset()
		sendUsr1()
		test.expect(one, []string{
			"param1 " + c + " | start",
		})
		test.reset()
		test.expect(two, []string{
			"param0 " + a + " | exit",
		})

		stop()
	}
}

func TestCmdRunError(t *testing.T) {
	paramSets := [][]string{
		{"param0", "param1"},
		{"param0", "param1", "param0"},
	}
	o := zero

	for _, v := range paramSets {
		a := testbin.SetBehavior(-one, zero, "a")
		test := start(t, v, o)
		test.expect(one, []string{
			"param0 " + a + " | start",
		})

		b := testbin.SetBehavior(-one, zero, "b")
		test.reset()
		testbin.Clean()
		sendUsr1()
		test.expect(one, []string{})

		test.reset()
		sendUsr1()
		test.expect(one, []string{})

		testbin.Build()
		test.reset()
		sendUsr1()
		test.expect(one, []string{
			"param1 " + b + " | start",
			"param0 " + a + " | exit",
		})

		stop()
	}
}

func TestAlternateExit(t *testing.T) {
	c := testbin.Build() + " " + placeholder
	v := []string{"param0"}
	o := zero

	testbin.SetBehavior(two, zero, "a")

	exited := false
	go func() {
		alternate(c, placeholder, v, o, newNilWriter(), newNilWriter(), newNilWriter())
		exited = true
	}()

	time.Sleep(one)
	if exited {
		t.Error("Was expecting exited to be false, was true")
	}

	time.Sleep(two)
	if !exited {
		t.Error("Was expecting exited to be true, was false")
	}
}
<|endoftext|>


<commit_before>/**
 * provides functions to create wording context relations
 */

package analyse

import (
    "log"
    "net/url"
    "encoding/json"
    "sync"
    "strconv"
    "github.com/monbro/opensemanticapi-go-lang/scraper"
    "github.com/monbro/opensemanticapi-go-lang/requestStruct"
    "github.com/monbro/opensemanticapi-go-lang/database"
    "syscall"
)

type Worker struct {
    Debug bool
    Db *database.RedisMulti
    START_SEARCH_TERM string
    SNIPPET_LENGTH int
    InfiniteWorking bool
    FastMode bool
    Wg sync.WaitGroup
}

/**
 * will initially start the process
 */
func (w *Worker) Run() {

    if w.FastMode {
        // via https://stackoverflow.com/questions/17817204/how-to-set-ulimit-n-from-a-golang-program
        var rLimit syscall.Rlimit
        err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
        if err != nil {
            log.Println("Error Getting Rlimit ", err)
        }
        log.Println(rLimit)
        rLimit.Max = 999999
        rLimit.Cur = 999999
        err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit)
        if err != nil {
            log.Println("Error Setting Rlimit ", err)
        }
        err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
        if err != nil {
            log.Println("Error Getting Rlimit ", err)
        }
        log.Println("Rlimit Final", rLimit)
    }

    if w.InfiniteWorking != true {
        w.InfiniteWorking = false
    }

    // init database
    w.Db = new(database.RedisMulti)
    w.Db.InitPool("", 10)

    // initial start
    w.RunNext(w.START_SEARCH_TERM)
}

/**
 * will run the process of storing words that are related in its context
 */
func (w *Worker) RunNext(searchTerm string) {

    log.Printf("Searchterm Now: '%+v'", searchTerm)

    // declare needed variables
    var pages []string

    // initial search request to get some pages back
    pages = SearchWikipedia(searchTerm)
    searchTerm = pages[0]

    // add this page to the done collection
    w.Db.AddPageToDone(searchTerm)

    // get a slice which will exclude the first element as we processing this one soon
    pagesToQueue := pages[1:]

    // pagesToQueue = []string{"pageOne", "Geographic Names Information System"}
    w.Db.AddPagesToQueue(pagesToQueue)

    rawContent := GetWikipediaPage(pages[0])

    snippetsRaw := GetSnippetsFromText(rawContent)
    snippets := GetSnippetsFromText(rawContent)
    snippets = CleanUpSnippets(snippets)

    w.Db.Multi()

    for i := range snippets {
        if w.SNIPPET_LENGTH < len(snippets[i]) {
            log.Println("Snippet "+strconv.Itoa(i)+"/"+strconv.Itoa(len(snippets))+" with a length of "+strconv.Itoa(len(snippetsRaw[i])))

            // analyse the text block
            go w.CreateSnippetWordsRelation(snippets[i])

            // raise counter for text blocks
            w.Db.RaiseScrapedTextBlocksCounter()
        }
    }

    if !w.FastMode {
        // wait for all snippets to be finished
        w.Wg.Wait()
    }

    // flush the queued commands from the pipeline
    w.Db.Flush()

    if w.InfiniteWorking {

        // create aloop by calling it self for the next search term
        w.RunNext(w.Db.RandomPageFromQueue())
    }
}

/**
 * will search wikipedia for a search term and return existing matching pages
 */
func SearchWikipedia(searchTerm string) []string {
    // lets create a new http request object
    rb := new(scraper.RequestBit)

    rb.Url = "https://en.wikipedia.org/w/api.php?action=query&list=search&srsearch="+url.QueryEscape(searchTerm)+"&format=json"
    log.Printf("Url crawling in SearchWikipedia: %+v", rb.Url)

    // inject the struct for the json response
    rb.ResponseObjectInterface = new(requestStruct.WikiSearch)
    rb.Work() // fire the request

    w2 := *rb.ResponseObjectInterface.(*requestStruct.WikiSearch)

    var results []string

    for _, value := range w2.Query.Search {
        // results[len(results)] = value.Rev[0].RawContent
        // http://blog.golang.org/slices
        results = append(results, value.Title)
    }

    return results
}

/**
 * will get the content of a wikipedia page
 */
func GetWikipediaPage(firstPage string) string {
    // lets create a new http request object
    rb := new(scraper.RequestBit)
    rb.Url = "http://en.wikipedia.org/w/api.php?rvprop=content&format=json&prop=revisions|categories&rvprop=content&action=query&titles="+url.QueryEscape(firstPage)
    log.Printf("Url crawling in GetWikipediaPage: %+v", rb.Url)

    // inject the struct for the json response
    rb.ResponseObjectInterface = new(requestStruct.WikiPage)
    rb.Work() // fire the request

    // for type assertion we need to explicite set the type of the returned interface object again
    // @TODO it would be nice to work without an interface here at all, but on the other hand to be flexible on the struct
    w2 := *rb.ResponseObjectInterface.(*requestStruct.WikiPage)

    // as the attribute 'Pages' is a map we neet to iterate trough it and return the first result, assuming this one is the page content
    for _, value := range w2.Query.Pages {
        return value.Rev[0].RawContent
    }

    // otherwise return an empty string
    return ""
}

/**
 * will analyse a snippet by spinning relations between each word within this snippet
 *
 * @TODO should be changed to fit https://gobyexample.com/worker-pools probably?
 */
func (w *Worker) CreateSnippetWordsRelation(snippet string) {

    if !w.FastMode {
        w.Wg.Add(1)
    }

    words := GetWordsFromSnippet(snippet)
    for _, word := range words {
        // check if word has more than 2 letters and this includes checking for an empty string
        if len(word) > 2 {
            for _, relation := range words {
                // check if we not adding a relation to the word itself
                // check if the relation is more than 2 letters long and not an empty string
                if word != relation &&
                    len(relation) > 2 {
                        w.Db.AddWordRelation(word, relation)
                }
            }
        }
    }

    if !w.FastMode {
        w.Wg.Done()
    }
}

/**
 * will search wikipedia for a search term and return existing matching pages
 * will be outdated soon!
 *
 * NOT IN USE CURRENTLY
 */
func OpenSearchWikipedia(searchTerm string) []string {
    // lets create a new http request object
    rb := new(scraper.RequestBit)

    rb.Url = "http://en.wikipedia.org/w/api.php?action=opensearch&search="+url.QueryEscape(searchTerm)+"&format=json&limit=3"
    log.Printf("Url crawling in SearchWikipedia: %+v", rb.Url)
    rb.Work() // fire the request

    // as wikipedia returns a sh*t formatted json we need to assign the result in two steps
    wikiOpenSearch := new(requestStruct.WikiOpenSearch)

    // first step is to assign the result term which is the first item in the returned array
    if err := json.Unmarshal(rb.ResponseArrayRawJson[0], &wikiOpenSearch.SearchTerm); err != nil {
        log.Fatalln("expect string:", err)
    }

    // second step is to assign the second item into the 'Results' array
    if err := json.Unmarshal(rb.ResponseArrayRawJson[1], &wikiOpenSearch.Results); err != nil {
        log.Fatalln("expect []string:", err)
    }

    // log.Printf("wikiOpenSearch is: %+v", wikiOpenSearch)

    return wikiOpenSearch.Results
}
<commit_msg>changed worker to be more independend?<commit_after>/**
 * provides functions to create wording context relations
 */

package analyse

import (
    "log"
    "net/url"
    "encoding/json"
    "sync"
    "strconv"
    "github.com/monbro/opensemanticapi-go-lang/scraper"
    "github.com/monbro/opensemanticapi-go-lang/requestStruct"
    "github.com/monbro/opensemanticapi-go-lang/database"
    "syscall"
)

type Worker struct {
    Debug bool
    Db *database.RedisMulti
    START_SEARCH_TERM string
    SNIPPET_LENGTH int
    InfiniteWorking bool
    FastMode bool
    Wg sync.WaitGroup
}

/**
 * will initially start the process
 */
func (w *Worker) Run() {

    if w.FastMode {
        // via https://stackoverflow.com/questions/17817204/how-to-set-ulimit-n-from-a-golang-program
        var rLimit syscall.Rlimit
        err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
        if err != nil {
            log.Println("Error Getting Rlimit ", err)
        }
        log.Println(rLimit)
        rLimit.Max = 999999
        rLimit.Cur = 999999
        err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit)
        if err != nil {
            log.Println("Error Setting Rlimit ", err)
        }
        err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
        if err != nil {
            log.Println("Error Getting Rlimit ", err)
        }
        log.Println("Rlimit Final", rLimit)
    }

    // set flag if given
    if w.InfiniteWorking != true {
        w.InfiniteWorking = false
    }

    // init database
    w.Db = new(database.RedisMulti)
    w.Db.InitPool("", 10)

    // initial start
    w.RunNext(w.START_SEARCH_TERM)
}

/**
 * will run the process of storing words that are related in its context
 */
func (w *Worker) RunNext(searchTerm string) {

    log.Printf("Searchterm Now: '%+v'", searchTerm)

    // declare needed variables
    var pages []string

    // initial search request to get some pages back
    pages = SearchWikipedia(searchTerm)
    searchTerm = pages[0]

    // add this page to the done collection
    w.Db.AddPageToDone(searchTerm)

    // get a slice which will exclude the first element as we processing this one soon
    pagesToQueue := pages[1:]

    // pagesToQueue = []string{"pageOne", "Geographic Names Information System"}
    w.Db.AddPagesToQueue(pagesToQueue)

    rawContent := GetWikipediaPage(pages[0])

    // snippetsRaw := GetSnippetsFromText(rawContent)
    snippets := GetSnippetsFromText(rawContent)
    snippets = CleanUpSnippets(snippets)

    w.Db.Multi()

    go snippetWorker(snippets, w.Db, w.SNIPPET_LENGTH)

    // for i := range snippets {
    //     if w.SNIPPET_LENGTH < len(snippets[i]) {
    //         log.Println("Snippet "+strconv.Itoa(i)+"/"+strconv.Itoa(len(snippets))+" with a length of "+strconv.Itoa(len(snippetsRaw[i])))

    //         // analyse the text block
    //         go w.CreateSnippetWordsRelation(snippets[i])

    //         // raise counter for text blocks
    //         w.Db.RaiseScrapedTextBlocksCounter()
    //     }
    // }

    if !w.FastMode {
        // wait for all snippets to be finished
        w.Wg.Wait()
    }

    // flush the queued commands from the pipeline
    w.Db.Flush()

    if w.InfiniteWorking {

        // create aloop by calling it self for the next search term
        w.RunNext(w.Db.RandomPageFromQueue())
    }
}

/**
 * will search wikipedia for a search term and return existing matching pages
 */
func SearchWikipedia(searchTerm string) []string {
    // lets create a new http request object
    rb := new(scraper.RequestBit)

    rb.Url = "https://en.wikipedia.org/w/api.php?action=query&list=search&srsearch="+url.QueryEscape(searchTerm)+"&format=json"
    log.Printf("Url crawling in SearchWikipedia: %+v", rb.Url)

    // inject the struct for the json response
    rb.ResponseObjectInterface = new(requestStruct.WikiSearch)
    rb.Work() // fire the request

    w2 := *rb.ResponseObjectInterface.(*requestStruct.WikiSearch)

    var results []string

    for _, value := range w2.Query.Search {
        // results[len(results)] = value.Rev[0].RawContent
        // http://blog.golang.org/slices
        results = append(results, value.Title)
    }

    return results
}

/**
 * will get the content of a wikipedia page
 */
func GetWikipediaPage(firstPage string) string {
    // lets create a new http request object
    rb := new(scraper.RequestBit)
    rb.Url = "http://en.wikipedia.org/w/api.php?rvprop=content&format=json&prop=revisions|categories&rvprop=content&action=query&titles="+url.QueryEscape(firstPage)
    log.Printf("Url crawling in GetWikipediaPage: %+v", rb.Url)

    // inject the struct for the json response
    rb.ResponseObjectInterface = new(requestStruct.WikiPage)
    rb.Work() // fire the request

    // for type assertion we need to explicite set the type of the returned interface object again
    // @TODO it would be nice to work without an interface here at all, but on the other hand to be flexible on the struct
    w2 := *rb.ResponseObjectInterface.(*requestStruct.WikiPage)

    // as the attribute 'Pages' is a map we neet to iterate trough it and return the first result, assuming this one is the page content
    for _, value := range w2.Query.Pages {
        return value.Rev[0].RawContent
    }

    // otherwise return an empty string
    return ""
}

func snippetWorker(snippets []string, Db *database.RedisMulti, SNIPPET_LENGTH int) {
    for i := range snippets {
        if SNIPPET_LENGTH < len(snippets[i]) {
            log.Println("Snippet "+strconv.Itoa(i)+"/"+strconv.Itoa(len(snippets))+" with a length of "+strconv.Itoa(len(snippets[i])))

            // analyse the text block
            words := GetWordsFromSnippet(snippets[i])
            for _, word := range words {
                // check if word has more than 2 letters and this includes checking for an empty string
                if len(word) > 2 {
                    for _, relation := range words {
                        // check if we not adding a relation to the word itself
                        // check if the relation is more than 2 letters long and not an empty string
                        if word != relation &&
                            len(relation) > 2 {
                                Db.AddWordRelation(word, relation)
                        }
                    }
                }
            }

            // raise counter for text blocks
            Db.RaiseScrapedTextBlocksCounter()
        }
    }
}

/**
 * will analyse a snippet by spinning relations between each word within this snippet
 *
 * @TODO should be changed to fit https://gobyexample.com/worker-pools probably?
 */
func (w *Worker) CreateSnippetWordsRelation(snippet string) {

    if !w.FastMode {
        w.Wg.Add(1)
    }

    words := GetWordsFromSnippet(snippet)
    for _, word := range words {
        // check if word has more than 2 letters and this includes checking for an empty string
        if len(word) > 2 {
            for _, relation := range words {
                // check if we not adding a relation to the word itself
                // check if the relation is more than 2 letters long and not an empty string
                if word != relation &&
                    len(relation) > 2 {
                        w.Db.AddWordRelation(word, relation)
                }
            }
        }
    }

    if !w.FastMode {
        w.Wg.Done()
    }
}

/**
 * will search wikipedia for a search term and return existing matching pages
 * will be outdated soon!
 *
 * NOT IN USE CURRENTLY
 */
func OpenSearchWikipedia(searchTerm string) []string {
    // lets create a new http request object
    rb := new(scraper.RequestBit)

    rb.Url = "http://en.wikipedia.org/w/api.php?action=opensearch&search="+url.QueryEscape(searchTerm)+"&format=json&limit=3"
    log.Printf("Url crawling in SearchWikipedia: %+v", rb.Url)
    rb.Work() // fire the request

    // as wikipedia returns a sh*t formatted json we need to assign the result in two steps
    wikiOpenSearch := new(requestStruct.WikiOpenSearch)

    // first step is to assign the result term which is the first item in the returned array
    if err := json.Unmarshal(rb.ResponseArrayRawJson[0], &wikiOpenSearch.SearchTerm); err != nil {
        log.Fatalln("expect string:", err)
    }

    // second step is to assign the second item into the 'Results' array
    if err := json.Unmarshal(rb.ResponseArrayRawJson[1], &wikiOpenSearch.Results); err != nil {
        log.Fatalln("expect []string:", err)
    }

    // log.Printf("wikiOpenSearch is: %+v", wikiOpenSearch)

    return wikiOpenSearch.Results
}
<|endoftext|>


<commit_before>package fbmodelsend

/*
ShareContents represents a Rich Media content to be shared
*/
type ShareContents struct {
	Attachment struct {
		Type    string `json:"type"`
		Payload struct {
			TemplateType string             `json:"template_type"`
			Elements     *[]TemplateElement `json:"elements"`
		} `json:"payload"`
	} `json:"attachment"`
}
<commit_msg>It's a slice of pointers of Template Element<commit_after>package fbmodelsend

/*
ShareContents represents a Rich Media content to be shared
*/
type ShareContents struct {
	Attachment struct {
		Type    string `json:"type"`
		Payload struct {
			TemplateType string             `json:"template_type"`
			Elements     []*TemplateElement `json:"elements"`
		} `json:"payload"`
	} `json:"attachment"`
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/rand"
	"encoding/base64"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	//"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"sync"
	"time"

	oidc "github.com/coreos/go-oidc"

	"gopkg.in/yaml.v2"

	"golang.org/x/net/context"
	"golang.org/x/oauth2"
)

var (
	clientID       = os.Getenv("GOOGLE_OAUTH2_CLIENT_ID")
	clientSecret   = os.Getenv("GOOGLE_OAUTH2_CLIENT_SECRET")
	configFilename = flag.String("config", "config.yml", "The filename of the configuration")
	debug          = flag.Bool("debug", true, "Enable debug messages to console")
)

type baseConfig struct {
	ClientID     string
	ClientSecret string
	ProviderURL  string
}

type AppConfigFile struct {
	Base baseConfig
}

func loadVerifyConfigFile(configFilename string) (AppConfigFile, error) {
	var config AppConfigFile
	if _, err := os.Stat(configFilename); os.IsNotExist(err) {
		err = errors.New("mising config file failure")
		return config, err
	}
	source, err := ioutil.ReadFile(configFilename)
	if err != nil {
		//panic(err)
		err = errors.New("cannot read config file")
		return config, err
	}
	err = yaml.Unmarshal(source, &config)
	if err != nil {
		err = errors.New("Cannot parse config file")
		return config, err
	}
	return config, nil
}

type pendingConfig struct {
	ExpiresAt       time.Time
	Config          *oauth2.Config
	Provider        *oidc.Provider
	originalRequest *http.Request
	state           string
	ctx             *context.Context
}

type simpleUserInfo struct {
	ExpiresAt time.Time
	Sub       string
}

type MiddlewareState struct {
	commonConfig baseConfig
	mutex        sync.Mutex
	oidcConfig   map[string]pendingConfig //map of cookievalue back to state
	authCookie   map[string]simpleUserInfo
	ctx          *context.Context
}

var OidcAuthState MiddlewareState

const redirectPath = "/auth/google/callback"
const redirCookieName = "oidc_redir_cookie"
const authCookieName = "oidc_auth_cookie"
const randomStringEntropyBytes = 32
const maxAgeSecondsRedirCookie = 300

func writeFailureResponse(w http.ResponseWriter, code int, message string) {
	w.WriteHeader(code)
	publicErrorText := fmt.Sprintf("%d %s %s\n", code, http.StatusText(code), message)
	w.Write([]byte(publicErrorText))
}

func (state *MiddlewareState) getUserInfo(r *http.Request) (*simpleUserInfo, error) {
	cookie, err := r.Cookie(authCookieName)
	if err != nil {
		if err == http.ErrNoCookie {
			if *debug {
				log.Printf("no auth cookie present")
			}
			return nil, nil
		}
		return nil, err
	}
	index := cookie.Value
	state.mutex.Lock()
	info, ok := state.authCookie[index]
	state.mutex.Unlock()
	if !ok {
		log.Printf("Auth cookie not found")
		return nil, nil
	}
	// TODO ADD expiration check!
	log.Printf("Auth cookie found")

	return &info, nil
}

func genRandomString() (string, error) {
	size := randomStringEntropyBytes
	rb := make([]byte, size)
	_, err := rand.Read(rb)
	if err != nil {
		return "", err
	}
	return base64.URLEncoding.EncodeToString(rb), nil

}

func (state *MiddlewareState) createRedirectionToProvider(w http.ResponseWriter, r *http.Request) {
	cookieVal, err := genRandomString()
	if err != nil {
		writeFailureResponse(w, http.StatusInternalServerError, "error internal")
		log.Println(err)
		return
	}

	// we have to create new context and set redirector...
	expiration := time.Now().Add(maxAgeSecondsRedirCookie * time.Second)
	//create localstate!

	provider, err := oidc.NewProvider(*state.ctx, state.commonConfig.ProviderURL)
	if err != nil {
		log.Fatal(err)
	}
	scheme := "http"
	if r.TLS != nil {
		scheme = "https"
	}
	//scheme := r.URL.Scheme
	redirectUrl := scheme + "://" + r.Host + redirectPath

	//log.Printf("prov:%+v", provider)
	log.Printf("URL (full) ='%+v'", r.URL)
	log.Printf("URL (scheme) ='%+v'\n", r.URL.Scheme)
	log.Printf("request='%+v'\n", r)
	log.Printf("redirurl='%s'", redirectUrl)

	config := oauth2.Config{
		ClientID:     state.commonConfig.ClientID,
		ClientSecret: state.commonConfig.ClientSecret,
		Endpoint:     provider.Endpoint(),
		//RedirectURL:  "http://127.0.0.1:5556/auth/google/callback",
		RedirectURL: redirectUrl,
		//Scopes:       []string{oidc.ScopeOpenID, "profile", "email"},
		Scopes: []string{oidc.ScopeOpenID, "profile"},
	}

	log.Printf("config : %+v", config)

	stateString, err := genRandomString()
	if err != nil {
		writeFailureResponse(w, http.StatusInternalServerError, "error internal")
		log.Println(err)
		return
	}

	//stateString := "foobar" // Don't do this in production.

	//http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
	//http.Redirect(w, r, config.AuthCodeURL(state), http.StatusFound)
	//})

	cookie := http.Cookie{Name: redirCookieName, Value: cookieVal, Expires: expiration}
	http.SetCookie(w, &cookie)

	pending := pendingConfig{ExpiresAt: expiration,
		Config:          &config,
		Provider:        provider,
		originalRequest: r, state: stateString, ctx: state.ctx}
	state.mutex.Lock()
	state.oidcConfig[cookieVal] = pending
	state.mutex.Unlock()

	http.Redirect(w, r, config.AuthCodeURL(stateString), http.StatusFound)
}

func (state *MiddlewareState) handleRedirectPath(w http.ResponseWriter, r *http.Request, h http.Handler) {
	redirCookie, err := r.Cookie(redirCookieName)
	if err != nil {
		if err == http.ErrNoCookie {
			writeFailureResponse(w, http.StatusBadRequest, "Missing setup cookie!")
			log.Println(err)
			return
		}
		writeFailureResponse(w, http.StatusInternalServerError, "error internal")
		log.Println(err)
		return
	}
	index := redirCookie.Value
	state.mutex.Lock()
	pending, ok := state.oidcConfig[index]
	state.mutex.Unlock()
	if !ok {
		// clear cookie here!!!!
		writeFailureResponse(w, http.StatusBadRequest, "Invalid setup cookie!")
		log.Println(err)
		return
	}

	if r.URL.Query().Get("state") != pending.state {
		http.Error(w, "state did not match", http.StatusBadRequest)
		return
	}
	log.Printf("req : %+v", r)
	oauth2Token, err := pending.Config.Exchange(*pending.ctx, r.URL.Query().Get("code"))
	if err != nil {
		log.Printf("ctx: %+v", *pending.ctx)
		http.Error(w, "Failed to exchange token: "+err.Error(), http.StatusInternalServerError)
		return
	}

	userInfo, err := pending.Provider.UserInfo(*pending.ctx, oauth2.StaticTokenSource(oauth2Token))
	if err != nil {
		http.Error(w, "Failed to get userinfo: "+err.Error(), http.StatusInternalServerError)
		return
	}

	resp := struct {
		OAuth2Token *oauth2.Token
		UserInfo    *oidc.UserInfo
	}{oauth2Token, userInfo}
	data, err := json.MarshalIndent(resp, "", "    ")
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	//w.Write(data)
	log.Printf("%+s", data)

	//set new auth cookie!
	cookieVal, err := genRandomString()
	if err != nil {
		writeFailureResponse(w, http.StatusInternalServerError, "error internal")
		log.Println(err)
		return
	}
	savedUserInfo := simpleUserInfo{Sub: "foo"}
	state.mutex.Lock()
	state.authCookie[cookieVal] = savedUserInfo
	state.mutex.Unlock()

	// we have to create new context and set redirector...
	//set the cookie and then redirect
	expiration := time.Now().Add(3600 * time.Second)

	authCookie := http.Cookie{Name: authCookieName, Value: cookieVal, Expires: expiration, Path: "/", HttpOnly: true}

	//use handler with original request.
	http.SetCookie(w, &authCookie)

	http.Redirect(w, r, pending.originalRequest.URL.String(), http.StatusFound)
}

func simpleAuth(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if *debug {
			log.Printf("Top os simple Auth target=%s\n", r.URL.String())
		}
		userInfo, err := OidcAuthState.getUserInfo(r)
		if err != nil {
			writeFailureResponse(w, http.StatusInternalServerError, "error internal")
			log.Println(err)
			return
		}
		if userInfo != nil {
			//found actual user... call original handler verbatim
			h.ServeHTTP(w, r)
			log.Println("After")
			return
		}
		if *debug {
			log.Printf("Dont have valid user... do openidc dance")
		}
		// We dont have a valid user...
		if r.URL.Path != redirectPath {
			if *debug {
				log.Printf("Doing redirection now")
			}
			OidcAuthState.createRedirectionToProvider(w, r)
			return
		}
		OidcAuthState.handleRedirectPath(w, r, h)

	})
}

func handler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Hi there, I love %s!", r.URL.Path[1:])
}

func main() {
	flag.Parse()

	appConfig, err := loadVerifyConfigFile(*configFilename)
	if err != nil {
		panic(err)
	}
	log.Printf("appConfig: %+v\n", appConfig)
	clientID = appConfig.Base.ClientID
	clientSecret = appConfig.Base.ClientSecret

	////
	ctx := context.Background()

	OidcAuthState.ctx = &ctx
	OidcAuthState.commonConfig = appConfig.Base
	OidcAuthState.oidcConfig = make(map[string]pendingConfig)
	OidcAuthState.authCookie = make(map[string]simpleUserInfo)

	//http.HandleFunc("/", handler)
	finalHandler := http.HandlerFunc(handler)
	//http.HandleFunc("/", simpleAuth(handler))
	http.Handle("/", simpleAuth(finalHandler))
	log.Printf("listening on http://%s/", "127.0.0.1:5556")
	log.Fatal(http.ListenAndServe("127.0.0.1:5556", nil))
}
<commit_msg>cleanup<commit_after>package main

import (
	"crypto/rand"
	"encoding/base64"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	//"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"sync"
	"time"

	oidc "github.com/coreos/go-oidc"

	"gopkg.in/yaml.v2"

	"golang.org/x/net/context"
	"golang.org/x/oauth2"
)

var (
	clientID       = os.Getenv("GOOGLE_OAUTH2_CLIENT_ID")
	clientSecret   = os.Getenv("GOOGLE_OAUTH2_CLIENT_SECRET")
	configFilename = flag.String("config", "config.yml", "The filename of the configuration")
	debug          = flag.Bool("debug", true, "Enable debug messages to console")
)

type baseConfig struct {
	ClientID     string
	ClientSecret string
	ProviderURL  string
}

type AppConfigFile struct {
	Base baseConfig
}

func loadVerifyConfigFile(configFilename string) (AppConfigFile, error) {
	var config AppConfigFile
	if _, err := os.Stat(configFilename); os.IsNotExist(err) {
		err = errors.New("mising config file failure")
		return config, err
	}
	source, err := ioutil.ReadFile(configFilename)
	if err != nil {
		//panic(err)
		err = errors.New("cannot read config file")
		return config, err
	}
	err = yaml.Unmarshal(source, &config)
	if err != nil {
		err = errors.New("Cannot parse config file")
		return config, err
	}
	return config, nil
}

type pendingConfig struct {
	ExpiresAt       time.Time
	Config          *oauth2.Config
	Provider        *oidc.Provider
	originalRequest *http.Request
	state           string
	ctx             *context.Context
}

type simpleUserInfo struct {
	ExpiresAt time.Time
	Sub       string
}

type MiddlewareState struct {
	commonConfig baseConfig
	mutex        sync.Mutex
	oidcConfig   map[string]pendingConfig //map of cookievalue back to state
	authCookie   map[string]simpleUserInfo
	ctx          *context.Context
}

var OidcAuthState MiddlewareState

const redirectPath = "/auth/google/callback"
const redirCookieName = "oidc_redir_cookie"
const authCookieName = "oidc_auth_cookie"
const randomStringEntropyBytes = 32
const maxAgeSecondsRedirCookie = 300

func writeFailureResponse(w http.ResponseWriter, code int, message string) {
	w.WriteHeader(code)
	publicErrorText := fmt.Sprintf("%d %s %s\n", code, http.StatusText(code), message)
	w.Write([]byte(publicErrorText))
}

func (state *MiddlewareState) getUserInfo(r *http.Request) (*simpleUserInfo, error) {
	cookie, err := r.Cookie(authCookieName)
	if err != nil {
		if err == http.ErrNoCookie {
			if *debug {
				log.Printf("no auth cookie present")
			}
			return nil, nil
		}
		return nil, err
	}
	index := cookie.Value
	state.mutex.Lock()
	info, ok := state.authCookie[index]
	state.mutex.Unlock()
	if !ok {
		log.Printf("Auth cookie not found")
		return nil, nil
	}
	// TODO ADD expiration check!
	log.Printf("Auth cookie found")

	return &info, nil
}

func genRandomString() (string, error) {
	size := randomStringEntropyBytes
	rb := make([]byte, size)
	_, err := rand.Read(rb)
	if err != nil {
		return "", err
	}
	return base64.URLEncoding.EncodeToString(rb), nil

}

func (state *MiddlewareState) createRedirectionToProvider(w http.ResponseWriter, r *http.Request) {
	cookieVal, err := genRandomString()
	if err != nil {
		writeFailureResponse(w, http.StatusInternalServerError, "error internal")
		log.Println(err)
		return
	}

	// we have to create new context and set redirector...
	expiration := time.Now().Add(maxAgeSecondsRedirCookie * time.Second)
	//create localstate!

	provider, err := oidc.NewProvider(*state.ctx, state.commonConfig.ProviderURL)
	if err != nil {
		log.Fatal(err)
	}

	scheme := "http"
	if r.TLS != nil {
		scheme = "https"
	}
	redirectUrl := scheme + "://" + r.Host + redirectPath

	//log.Printf("prov:%+v", provider)

	config := oauth2.Config{
		ClientID:     state.commonConfig.ClientID,
		ClientSecret: state.commonConfig.ClientSecret,
		Endpoint:     provider.Endpoint(),
		RedirectURL:  redirectUrl,
		Scopes:       []string{oidc.ScopeOpenID, "profile"},
	}

	log.Printf("config : %+v", config)

	stateString, err := genRandomString()
	if err != nil {
		writeFailureResponse(w, http.StatusInternalServerError, "error internal")
		log.Println(err)
		return
	}

	cookie := http.Cookie{Name: redirCookieName, Value: cookieVal, Expires: expiration}
	http.SetCookie(w, &cookie)

	pending := pendingConfig{ExpiresAt: expiration,
		Config:          &config,
		Provider:        provider,
		originalRequest: r, state: stateString, ctx: state.ctx}
	state.mutex.Lock()
	state.oidcConfig[cookieVal] = pending
	state.mutex.Unlock()

	http.Redirect(w, r, config.AuthCodeURL(stateString), http.StatusFound)
}

func (state *MiddlewareState) handleRedirectPath(w http.ResponseWriter, r *http.Request, h http.Handler) {
	redirCookie, err := r.Cookie(redirCookieName)
	if err != nil {
		if err == http.ErrNoCookie {
			writeFailureResponse(w, http.StatusBadRequest, "Missing setup cookie!")
			log.Println(err)
			return
		}
		writeFailureResponse(w, http.StatusInternalServerError, "error internal")
		log.Println(err)
		return
	}
	index := redirCookie.Value
	state.mutex.Lock()
	pending, ok := state.oidcConfig[index]
	state.mutex.Unlock()
	if !ok {
		// clear cookie here!!!!
		writeFailureResponse(w, http.StatusBadRequest, "Invalid setup cookie!")
		log.Println(err)
		return
	}

	if r.URL.Query().Get("state") != pending.state {
		http.Error(w, "state did not match", http.StatusBadRequest)
		return
	}
	log.Printf("req : %+v", r)
	oauth2Token, err := pending.Config.Exchange(*pending.ctx, r.URL.Query().Get("code"))
	if err != nil {
		log.Printf("ctx: %+v", *pending.ctx)
		http.Error(w, "Failed to exchange token: "+err.Error(), http.StatusInternalServerError)
		return
	}

	userInfo, err := pending.Provider.UserInfo(*pending.ctx, oauth2.StaticTokenSource(oauth2Token))
	if err != nil {
		http.Error(w, "Failed to get userinfo: "+err.Error(), http.StatusInternalServerError)
		return
	}

	resp := struct {
		OAuth2Token *oauth2.Token
		UserInfo    *oidc.UserInfo
	}{oauth2Token, userInfo}
	data, err := json.MarshalIndent(resp, "", "    ")
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	//w.Write(data)
	log.Printf("%+s", data)

	//set new auth cookie!
	cookieVal, err := genRandomString()
	if err != nil {
		writeFailureResponse(w, http.StatusInternalServerError, "error internal")
		log.Println(err)
		return
	}
	savedUserInfo := simpleUserInfo{Sub: "foo"}
	state.mutex.Lock()
	state.authCookie[cookieVal] = savedUserInfo
	state.mutex.Unlock()

	// we have to create new context and set redirector...
	//set the cookie and then redirect
	expiration := time.Now().Add(3600 * time.Second)

	authCookie := http.Cookie{Name: authCookieName, Value: cookieVal, Expires: expiration, Path: "/", HttpOnly: true}

	//use handler with original request.
	http.SetCookie(w, &authCookie)

	http.Redirect(w, r, pending.originalRequest.URL.String(), http.StatusFound)
}

func simpleAuth(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if *debug {
			log.Printf("Top os simple Auth target=%s\n", r.URL.String())
		}
		userInfo, err := OidcAuthState.getUserInfo(r)
		if err != nil {
			writeFailureResponse(w, http.StatusInternalServerError, "error internal")
			log.Println(err)
			return
		}
		if userInfo != nil {
			//found actual user... call original handler verbatim
			h.ServeHTTP(w, r)
			log.Println("After")
			return
		}
		if *debug {
			log.Printf("Dont have valid user... do openidc dance")
		}
		// We dont have a valid user...
		if r.URL.Path != redirectPath {
			if *debug {
				log.Printf("Doing redirection now")
			}
			OidcAuthState.createRedirectionToProvider(w, r)
			return
		}
		OidcAuthState.handleRedirectPath(w, r, h)

	})
}

func handler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Hi there, I love %s!", r.URL.Path[1:])
}

func main() {
	flag.Parse()

	appConfig, err := loadVerifyConfigFile(*configFilename)
	if err != nil {
		panic(err)
	}
	log.Printf("appConfig: %+v\n", appConfig)
	clientID = appConfig.Base.ClientID
	clientSecret = appConfig.Base.ClientSecret

	////
	ctx := context.Background()

	OidcAuthState.ctx = &ctx
	OidcAuthState.commonConfig = appConfig.Base
	OidcAuthState.oidcConfig = make(map[string]pendingConfig)
	OidcAuthState.authCookie = make(map[string]simpleUserInfo)

	//http.HandleFunc("/", handler)
	finalHandler := http.HandlerFunc(handler)
	http.Handle("/", simpleAuth(finalHandler))
	log.Printf("listening on http://%s/", "127.0.0.1:5556")
	log.Fatal(http.ListenAndServe("127.0.0.1:5556", nil))
}
<|endoftext|>


<commit_before>package pretty

import (
	"fmt"
	"html/template"
	"io"
	"os"

	"github.com/docker/go-units"
	"github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/server/pkg/pretty"
)

// PrintRepoHeader prints a repo header.
func PrintRepoHeader(w io.Writer) {
	fmt.Fprint(w, "NAME\tCREATED\tSIZE\t\n")
}

// PrintRepoInfo pretty-prints repo info.
func PrintRepoInfo(w io.Writer, repoInfo *pfs.RepoInfo) {
	fmt.Fprintf(w, "%s\t", repoInfo.Repo.Name)
	fmt.Fprintf(
		w,
		"%s\t",
		pretty.Ago(repoInfo.Created),
	)
	fmt.Fprintf(w, "%s\t\n", units.BytesSize(float64(repoInfo.SizeBytes)))
}

// PrintDetailedRepoInfo pretty-prints detailed repo info.
func PrintDetailedRepoInfo(repoInfo *pfs.RepoInfo) error {
	template, err := template.New("RepoInfo").Funcs(funcMap).Parse(
		`Name: {{.Repo.Name}}
Created: {{prettyAgo .Created}}
Size: {{prettySize .SizeBytes}}{{if .Provenance}}
Provenance: {{range .Provenance}} {{.Name}} {{end}} {{end}}
`)
	if err != nil {
		return err
	}
	err = template.Execute(os.Stdout, repoInfo)
	if err != nil {
		return err
	}
	return nil
}

// PrintCommitInfoHeader prints a commit info header.
func PrintCommitInfoHeader(w io.Writer) {
	fmt.Fprint(w, "BRANCH\tID\tPARENT\tSTARTED\tFINISHED\tSIZE\t\n")
}

// PrintCommitInfo pretty-prints commit info.
func PrintCommitInfo(w io.Writer, commitInfo *pfs.CommitInfo) {
	fmt.Fprintf(w, "%s\t", commitInfo.Branch)
	fmt.Fprintf(w, "%s\t", commitInfo.Commit.ID)
	if commitInfo.ParentCommit != nil {
		fmt.Fprintf(w, "%s\t", commitInfo.ParentCommit.ID)
	} else {
		fmt.Fprint(w, "<none>\t")
	}
	fmt.Fprintf(
		w,
		"%s\t",
		pretty.Ago(commitInfo.Started),
	)
	finished := "\t"
	if commitInfo.Finished != nil {
		finished = fmt.Sprintf("%s\t", pretty.Ago(commitInfo.Finished))
	}
	fmt.Fprintf(w, finished)
	fmt.Fprintf(w, "%s\t\n", units.BytesSize(float64(commitInfo.SizeBytes)))
}

// PrintDetailedCommitInfo pretty-prints detailed commit info.
func PrintDetailedCommitInfo(commitInfo *pfs.CommitInfo) error {
	template, err := template.New("CommitInfo").Funcs(funcMap).Parse(
		`Commit: {{.Commit.Repo.Name}}/{{.Commit.ID}}{{if .ParentCommit}}
Parent: {{.ParentCommit.ID}} {{end}} {{if .Branch}}
Branch: {{.Branch}} {{end}}
Started: {{prettyAgo .Started}}{{if .Finished}}
Finished: {{prettyAgo .Finished}} {{end}}
Size: {{prettySize .SizeBytes}}{{if .Provenance}}
Provenance: {{range .Provenance}} {{.Repo.Name}}/{{.ID}} {{end}} {{end}}{{if .Cancelled}}
CANCELLED {{end}}
`)
	if err != nil {
		return err
	}
	err = template.Execute(os.Stdout, commitInfo)
	if err != nil {
		return err
	}
	return nil
}

// PrintFileInfoHeader prints a file info header.
func PrintFileInfoHeader(w io.Writer) {
	fmt.Fprint(w, "NAME\tTYPE\tMODIFIED\tLAST_COMMIT_MODIFIED\tSIZE\t\n")
}

// PrintFileInfo pretty-prints file info.
func PrintFileInfo(w io.Writer, fileInfo *pfs.FileInfo) {
	fmt.Fprintf(w, "%s\t", fileInfo.File.Path)
	if fileInfo.FileType == pfs.FileType_FILE_TYPE_REGULAR {
		fmt.Fprint(w, "file\t")
	} else {
		fmt.Fprint(w, "dir\t")
	}
	fmt.Fprintf(
		w,
		"%s\t",
		pretty.Ago(fileInfo.Modified),
	)
	fmt.Fprint(w, "-\t")
	fmt.Fprintf(w, "%s\t\n", units.BytesSize(float64(fileInfo.SizeBytes)))
}

// PrintDetailedFileInfo pretty-prints detailed file info.
func PrintDetailedFileInfo(fileInfo *pfs.FileInfo) error {
	template, err := template.New("FileInfo").Funcs(funcMap).Parse(
		`Path: {{.File.Commit.Repo.Name}}/{{.File.Commit.ID}}/{{.File.Path}}
Type: {{fileType .FileType}}
Modifed: {{prettyAgo .Modified}}
Size: {{prettySize .SizeBytes}}
Commit Modified: {{.CommitModified.Repo.Name}}/{{.CommitModified.ID}}{{if .Children}}
Children: {{range .Children}} {{.Path}} {{end}} {{end}}
`)
	if err != nil {
		return err
	}
	if err := template.Execute(os.Stdout, fileInfo); err != nil {
		return err
	}
	return nil
}

// PrintBlockInfoHeader prints a block info header.
func PrintBlockInfoHeader(w io.Writer) {
	fmt.Fprintf(w, "HASH\tCREATED\tSIZE\t\n")
}

// PrintBlockInfo pretty-prints block info.
func PrintBlockInfo(w io.Writer, blockInfo *pfs.BlockInfo) {
	fmt.Fprintf(w, "%s\t", blockInfo.Block.Hash)
	fmt.Fprintf(
		w,
		"%s\t",
		pretty.Ago(blockInfo.Created),
	)
	fmt.Fprintf(w, "%s\t\n", units.BytesSize(float64(blockInfo.SizeBytes)))
}

type uint64Slice []uint64

func (s uint64Slice) Len() int           { return len(s) }
func (s uint64Slice) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
func (s uint64Slice) Less(i, j int) bool { return s[i] < s[j] }

func fileType(fileType pfs.FileType) string {
	if fileType == pfs.FileType_FILE_TYPE_REGULAR {
		return "file"
	}
	return "dir"
}

var funcMap = template.FuncMap{
	"prettyAgo":  pretty.Ago,
	"prettySize": pretty.Size,
	"fileType":   fileType,
}
<commit_msg>Print repo names when listing commits.<commit_after>package pretty

import (
	"fmt"
	"html/template"
	"io"
	"os"

	"github.com/docker/go-units"
	"github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/server/pkg/pretty"
)

// PrintRepoHeader prints a repo header.
func PrintRepoHeader(w io.Writer) {
	fmt.Fprint(w, "NAME\tCREATED\tSIZE\t\n")
}

// PrintRepoInfo pretty-prints repo info.
func PrintRepoInfo(w io.Writer, repoInfo *pfs.RepoInfo) {
	fmt.Fprintf(w, "%s\t", repoInfo.Repo.Name)
	fmt.Fprintf(
		w,
		"%s\t",
		pretty.Ago(repoInfo.Created),
	)
	fmt.Fprintf(w, "%s\t\n", units.BytesSize(float64(repoInfo.SizeBytes)))
}

// PrintDetailedRepoInfo pretty-prints detailed repo info.
func PrintDetailedRepoInfo(repoInfo *pfs.RepoInfo) error {
	template, err := template.New("RepoInfo").Funcs(funcMap).Parse(
		`Name: {{.Repo.Name}}
Created: {{prettyAgo .Created}}
Size: {{prettySize .SizeBytes}}{{if .Provenance}}
Provenance: {{range .Provenance}} {{.Name}} {{end}} {{end}}
`)
	if err != nil {
		return err
	}
	err = template.Execute(os.Stdout, repoInfo)
	if err != nil {
		return err
	}
	return nil
}

// PrintCommitInfoHeader prints a commit info header.
func PrintCommitInfoHeader(w io.Writer) {
	fmt.Fprint(w, "BRANCH\tREPO/ID\tPARENT\tSTARTED\tFINISHED\tSIZE\t\n")
}

// PrintCommitInfo pretty-prints commit info.
func PrintCommitInfo(w io.Writer, commitInfo *pfs.CommitInfo) {
	fmt.Fprintf(w, "%s\t", commitInfo.Branch)
	fmt.Fprintf(w, "%s/%s\t", commitInfo.Commit.Repo.Name, commitInfo.Commit.ID)
	if commitInfo.ParentCommit != nil {
		fmt.Fprintf(w, "%s\t", commitInfo.ParentCommit.ID)
	} else {
		fmt.Fprint(w, "<none>\t")
	}
	fmt.Fprintf(
		w,
		"%s\t",
		pretty.Ago(commitInfo.Started),
	)
	finished := "\t"
	if commitInfo.Finished != nil {
		finished = fmt.Sprintf("%s\t", pretty.Ago(commitInfo.Finished))
	}
	fmt.Fprintf(w, finished)
	fmt.Fprintf(w, "%s\t\n", units.BytesSize(float64(commitInfo.SizeBytes)))
}

// PrintDetailedCommitInfo pretty-prints detailed commit info.
func PrintDetailedCommitInfo(commitInfo *pfs.CommitInfo) error {
	template, err := template.New("CommitInfo").Funcs(funcMap).Parse(
		`Commit: {{.Commit.Repo.Name}}/{{.Commit.ID}}{{if .ParentCommit}}
Parent: {{.ParentCommit.ID}} {{end}} {{if .Branch}}
Branch: {{.Branch}} {{end}}
Started: {{prettyAgo .Started}}{{if .Finished}}
Finished: {{prettyAgo .Finished}} {{end}}
Size: {{prettySize .SizeBytes}}{{if .Provenance}}
Provenance: {{range .Provenance}} {{.Repo.Name}}/{{.ID}} {{end}} {{end}}{{if .Cancelled}}
CANCELLED {{end}}
`)
	if err != nil {
		return err
	}
	err = template.Execute(os.Stdout, commitInfo)
	if err != nil {
		return err
	}
	return nil
}

// PrintFileInfoHeader prints a file info header.
func PrintFileInfoHeader(w io.Writer) {
	fmt.Fprint(w, "NAME\tTYPE\tMODIFIED\tLAST_COMMIT_MODIFIED\tSIZE\t\n")
}

// PrintFileInfo pretty-prints file info.
func PrintFileInfo(w io.Writer, fileInfo *pfs.FileInfo) {
	fmt.Fprintf(w, "%s\t", fileInfo.File.Path)
	if fileInfo.FileType == pfs.FileType_FILE_TYPE_REGULAR {
		fmt.Fprint(w, "file\t")
	} else {
		fmt.Fprint(w, "dir\t")
	}
	fmt.Fprintf(
		w,
		"%s\t",
		pretty.Ago(fileInfo.Modified),
	)
	fmt.Fprint(w, "-\t")
	fmt.Fprintf(w, "%s\t\n", units.BytesSize(float64(fileInfo.SizeBytes)))
}

// PrintDetailedFileInfo pretty-prints detailed file info.
func PrintDetailedFileInfo(fileInfo *pfs.FileInfo) error {
	template, err := template.New("FileInfo").Funcs(funcMap).Parse(
		`Path: {{.File.Commit.Repo.Name}}/{{.File.Commit.ID}}/{{.File.Path}}
Type: {{fileType .FileType}}
Modifed: {{prettyAgo .Modified}}
Size: {{prettySize .SizeBytes}}
Commit Modified: {{.CommitModified.Repo.Name}}/{{.CommitModified.ID}}{{if .Children}}
Children: {{range .Children}} {{.Path}} {{end}} {{end}}
`)
	if err != nil {
		return err
	}
	if err := template.Execute(os.Stdout, fileInfo); err != nil {
		return err
	}
	return nil
}

// PrintBlockInfoHeader prints a block info header.
func PrintBlockInfoHeader(w io.Writer) {
	fmt.Fprintf(w, "HASH\tCREATED\tSIZE\t\n")
}

// PrintBlockInfo pretty-prints block info.
func PrintBlockInfo(w io.Writer, blockInfo *pfs.BlockInfo) {
	fmt.Fprintf(w, "%s\t", blockInfo.Block.Hash)
	fmt.Fprintf(
		w,
		"%s\t",
		pretty.Ago(blockInfo.Created),
	)
	fmt.Fprintf(w, "%s\t\n", units.BytesSize(float64(blockInfo.SizeBytes)))
}

type uint64Slice []uint64

func (s uint64Slice) Len() int           { return len(s) }
func (s uint64Slice) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
func (s uint64Slice) Less(i, j int) bool { return s[i] < s[j] }

func fileType(fileType pfs.FileType) string {
	if fileType == pfs.FileType_FILE_TYPE_REGULAR {
		return "file"
	}
	return "dir"
}

var funcMap = template.FuncMap{
	"prettyAgo":  pretty.Ago,
	"prettySize": pretty.Size,
	"fileType":   fileType,
}
<|endoftext|>


<commit_before>package pretty

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"strings"
	"text/tabwriter"
	"text/template"

	"github.com/fatih/color"
	ppsclient "github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/server/pkg/pretty"
)

func PrintJobHeader(w io.Writer) {
	fmt.Fprint(w, "ID\tOUTPUT\tSTATE\t\n")
}

func PrintJobInfo(w io.Writer, jobInfo *ppsclient.JobInfo) {
	fmt.Fprintf(w, "%s\t", jobInfo.Job.ID)
	if jobInfo.OutputCommit != nil {
		fmt.Fprintf(w, "%s/%s\t", jobInfo.OutputCommit.Repo.Name, jobInfo.OutputCommit.ID)
	} else {
		fmt.Fprintf(w, "-\t")
	}
	fmt.Fprintf(w, "%s\t\n", jobState(jobInfo.State))
}

func PrintPipelineHeader(w io.Writer) {
	fmt.Fprint(w, "NAME\tINPUT\tOUTPUT\tSTATE\t\n")
}

func PrintPipelineInfo(w io.Writer, pipelineInfo *ppsclient.PipelineInfo) {
	fmt.Fprintf(w, "%s\t", pipelineInfo.Pipeline.Name)
	if len(pipelineInfo.Inputs) == 0 {
		fmt.Fprintf(w, "\t")
	} else {
		for i, input := range pipelineInfo.Inputs {
			fmt.Fprintf(w, "%s", input.Repo.Name)
			if i == len(pipelineInfo.Inputs)-1 {
				fmt.Fprintf(w, "\t")
			} else {
				fmt.Fprintf(w, ", ")
			}
		}
	}
	if pipelineInfo.OutputRepo != nil {
		fmt.Fprintf(w, "%s\t", pipelineInfo.OutputRepo.Name)
	} else {
		fmt.Fprintf(w, "\t")
	}
	fmt.Fprintf(w, "%s\t\n", pipelineState(pipelineInfo.State))
}

func PrintJobInputHeader(w io.Writer) {
	fmt.Fprint(w, "NAME\tCOMMIT\tPARTITION\tINCREMENTAL\t\n")
}

func PrintJobInput(w io.Writer, jobInput *ppsclient.JobInput) {
	fmt.Fprintf(w, "%s\t", jobInput.Commit.Repo.Name)
	fmt.Fprintf(w, "%s\t", jobInput.Commit.ID)
	fmt.Fprintf(w, "%s\t", jobInput.Method.Partition)
	fmt.Fprintf(w, "%t\t\n", jobInput.Method.Incremental)
}

func PrintPipelineInputHeader(w io.Writer) {
	fmt.Fprint(w, "NAME\tPARTITION\tINCREMENTAL\t\n")
}

func PrintPipelineInput(w io.Writer, pipelineInput *ppsclient.PipelineInput) {
	fmt.Fprintf(w, "%s\t", pipelineInput.Repo.Name)
	fmt.Fprintf(w, "%s\t", pipelineInput.Method.Partition)
	fmt.Fprintf(w, "%t\t\n", pipelineInput.Method.Incremental)
}

func PrintJobCountsHeader(w io.Writer) {
	fmt.Fprintf(w, strings.ToUpper(jobState(ppsclient.JobState_JOB_PULLING))+"\t")
	fmt.Fprintf(w, strings.ToUpper(jobState(ppsclient.JobState_JOB_RUNNING))+"\t")
	fmt.Fprintf(w, strings.ToUpper(jobState(ppsclient.JobState_JOB_FAILURE))+"\t")
	fmt.Fprintf(w, strings.ToUpper(jobState(ppsclient.JobState_JOB_SUCCESS))+"\t\n")
}

func PrintDetailedJobInfo(jobInfo *ppsclient.JobInfo) {
	template, err := template.New("JobInfo").Funcs(funcMap).Parse(
		`ID: {{.Job.ID}} {{if .ParentJob}}
Parent: {{.ParentJob.ID}} {{end}}
Created: {{prettyDuration .CreatedAt}}
State: {{jobState .State}}
Parallelism: {{.Parallelism}}
Inputs:
{{jobInputs .}}Transform:
{{prettyTransform .Transform}}`)
	if err != nil {
		fmt.Println(err.Error())
		return
	}
	err = template.Execute(os.Stdout, jobInfo)
	if err != nil {
		fmt.Println(err.Error())
		return
	}
}

func PrintDetailedPipelineInfo(pipelineInfo *ppsclient.PipelineInfo) {
	template, err := template.New("PipelineInfo").Funcs(funcMap).Parse(
		`Name: {{.Pipeline.Name}}
Created: {{prettyDuration .CreatedAt}}
State: {{pipelineState .State}}
Parallelism: {{.Parallelism}}
Inputs:
{{pipelineInputs .}}Transform:
{{prettyTransform .Transform}}
{{if .RecentError}} Recent Error: {{.RecentError}} {{end}}
Job Counts:
{{jobCounts .JobCounts}}
`)
	if err != nil {
		fmt.Println(err.Error())
		return
	}
	err = template.Execute(os.Stdout, pipelineInfo)
	if err != nil {
		fmt.Println(err.Error())
		return
	}
}

func jobState(jobState ppsclient.JobState) string {
	switch jobState {
	case ppsclient.JobState_JOB_PULLING:
		return color.New(color.FgYellow).SprintFunc()("pulling")
	case ppsclient.JobState_JOB_RUNNING:
		return color.New(color.FgYellow).SprintFunc()("running")
	case ppsclient.JobState_JOB_FAILURE:
		return color.New(color.FgRed).SprintFunc()("failure")
	case ppsclient.JobState_JOB_SUCCESS:
		return color.New(color.FgGreen).SprintFunc()("success")
	}
	return "-"
}

func pipelineState(pipelineState ppsclient.PipelineState) string {
	switch pipelineState {
	case ppsclient.PipelineState_PIPELINE_STARTING:
		return color.New(color.FgYellow).SprintFunc()("starting")
	case ppsclient.PipelineState_PIPELINE_RUNNING:
		return color.New(color.FgGreen).SprintFunc()("running")
	case ppsclient.PipelineState_PIPELINE_RESTARTING:
		return color.New(color.FgYellow).SprintFunc()("restarting")
	case ppsclient.PipelineState_PIPELINE_FAILED:
		return color.New(color.FgRed).SprintFunc()("failed")
	}
	return "-"
}

func jobInputs(jobInfo *ppsclient.JobInfo) string {
	var buffer bytes.Buffer
	writer := tabwriter.NewWriter(&buffer, 20, 1, 3, ' ', 0)
	PrintJobInputHeader(writer)
	for _, input := range jobInfo.Inputs {
		PrintJobInput(writer, input)
	}
	// can't error because buffer can't error on Write
	writer.Flush()
	return buffer.String()
}

func pipelineInputs(pipelineInfo *ppsclient.PipelineInfo) string {
	var buffer bytes.Buffer
	writer := tabwriter.NewWriter(&buffer, 20, 1, 3, ' ', 0)
	PrintPipelineInputHeader(writer)
	for _, input := range pipelineInfo.Inputs {
		PrintPipelineInput(writer, input)
	}
	// can't error because buffer can't error on Write
	writer.Flush()
	return buffer.String()
}

func jobCounts(counts map[int32]int32) string {
	var buffer bytes.Buffer
	for i := int32(ppsclient.JobState_JOB_PULLING); i <= int32(ppsclient.JobState_JOB_SUCCESS); i++ {
		fmt.Fprintf(&buffer, "%s: %d\t", jobState(ppsclient.JobState(i)), counts[i])
	}
	return buffer.String()
}

func prettyTransform(transform *ppsclient.Transform) (string, error) {
	result, err := json.MarshalIndent(transform, "", "  ")
	if err != nil {
		return "", err
	}
	return pretty.UnescapeHTML(string(result)), nil
}

var funcMap = template.FuncMap{
	"pipelineState":   pipelineState,
	"jobState":        jobState,
	"pipelineInputs":  pipelineInputs,
	"jobInputs":       jobInputs,
	"prettyDuration":  pretty.PrettyDuration,
	"jobCounts":       jobCounts,
	"prettyTransform": prettyTransform,
}
<commit_msg>Adds a newline to inspect-job.<commit_after>package pretty

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"strings"
	"text/tabwriter"
	"text/template"

	"github.com/fatih/color"
	ppsclient "github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/server/pkg/pretty"
)

func PrintJobHeader(w io.Writer) {
	fmt.Fprint(w, "ID\tOUTPUT\tSTATE\t\n")
}

func PrintJobInfo(w io.Writer, jobInfo *ppsclient.JobInfo) {
	fmt.Fprintf(w, "%s\t", jobInfo.Job.ID)
	if jobInfo.OutputCommit != nil {
		fmt.Fprintf(w, "%s/%s\t", jobInfo.OutputCommit.Repo.Name, jobInfo.OutputCommit.ID)
	} else {
		fmt.Fprintf(w, "-\t")
	}
	fmt.Fprintf(w, "%s\t\n", jobState(jobInfo.State))
}

func PrintPipelineHeader(w io.Writer) {
	fmt.Fprint(w, "NAME\tINPUT\tOUTPUT\tSTATE\t\n")
}

func PrintPipelineInfo(w io.Writer, pipelineInfo *ppsclient.PipelineInfo) {
	fmt.Fprintf(w, "%s\t", pipelineInfo.Pipeline.Name)
	if len(pipelineInfo.Inputs) == 0 {
		fmt.Fprintf(w, "\t")
	} else {
		for i, input := range pipelineInfo.Inputs {
			fmt.Fprintf(w, "%s", input.Repo.Name)
			if i == len(pipelineInfo.Inputs)-1 {
				fmt.Fprintf(w, "\t")
			} else {
				fmt.Fprintf(w, ", ")
			}
		}
	}
	if pipelineInfo.OutputRepo != nil {
		fmt.Fprintf(w, "%s\t", pipelineInfo.OutputRepo.Name)
	} else {
		fmt.Fprintf(w, "\t")
	}
	fmt.Fprintf(w, "%s\t\n", pipelineState(pipelineInfo.State))
}

func PrintJobInputHeader(w io.Writer) {
	fmt.Fprint(w, "NAME\tCOMMIT\tPARTITION\tINCREMENTAL\t\n")
}

func PrintJobInput(w io.Writer, jobInput *ppsclient.JobInput) {
	fmt.Fprintf(w, "%s\t", jobInput.Commit.Repo.Name)
	fmt.Fprintf(w, "%s\t", jobInput.Commit.ID)
	fmt.Fprintf(w, "%s\t", jobInput.Method.Partition)
	fmt.Fprintf(w, "%t\t\n", jobInput.Method.Incremental)
}

func PrintPipelineInputHeader(w io.Writer) {
	fmt.Fprint(w, "NAME\tPARTITION\tINCREMENTAL\t\n")
}

func PrintPipelineInput(w io.Writer, pipelineInput *ppsclient.PipelineInput) {
	fmt.Fprintf(w, "%s\t", pipelineInput.Repo.Name)
	fmt.Fprintf(w, "%s\t", pipelineInput.Method.Partition)
	fmt.Fprintf(w, "%t\t\n", pipelineInput.Method.Incremental)
}

func PrintJobCountsHeader(w io.Writer) {
	fmt.Fprintf(w, strings.ToUpper(jobState(ppsclient.JobState_JOB_PULLING))+"\t")
	fmt.Fprintf(w, strings.ToUpper(jobState(ppsclient.JobState_JOB_RUNNING))+"\t")
	fmt.Fprintf(w, strings.ToUpper(jobState(ppsclient.JobState_JOB_FAILURE))+"\t")
	fmt.Fprintf(w, strings.ToUpper(jobState(ppsclient.JobState_JOB_SUCCESS))+"\t\n")
}

func PrintDetailedJobInfo(jobInfo *ppsclient.JobInfo) {
	template, err := template.New("JobInfo").Funcs(funcMap).Parse(
		`ID: {{.Job.ID}} {{if .ParentJob}}
Parent: {{.ParentJob.ID}} {{end}}
Created: {{prettyDuration .CreatedAt}}
State: {{jobState .State}}
Parallelism: {{.Parallelism}}
Inputs:
{{jobInputs .}}Transform:
{{prettyTransform .Transform}}
`)
	if err != nil {
		fmt.Println(err.Error())
		return
	}
	err = template.Execute(os.Stdout, jobInfo)
	if err != nil {
		fmt.Println(err.Error())
		return
	}
}

func PrintDetailedPipelineInfo(pipelineInfo *ppsclient.PipelineInfo) {
	template, err := template.New("PipelineInfo").Funcs(funcMap).Parse(
		`Name: {{.Pipeline.Name}}
Created: {{prettyDuration .CreatedAt}}
State: {{pipelineState .State}}
Parallelism: {{.Parallelism}}
Inputs:
{{pipelineInputs .}}Transform:
{{prettyTransform .Transform}}
{{if .RecentError}} Recent Error: {{.RecentError}} {{end}}
Job Counts:
{{jobCounts .JobCounts}}
`)
	if err != nil {
		fmt.Println(err.Error())
		return
	}
	err = template.Execute(os.Stdout, pipelineInfo)
	if err != nil {
		fmt.Println(err.Error())
		return
	}
}

func jobState(jobState ppsclient.JobState) string {
	switch jobState {
	case ppsclient.JobState_JOB_PULLING:
		return color.New(color.FgYellow).SprintFunc()("pulling")
	case ppsclient.JobState_JOB_RUNNING:
		return color.New(color.FgYellow).SprintFunc()("running")
	case ppsclient.JobState_JOB_FAILURE:
		return color.New(color.FgRed).SprintFunc()("failure")
	case ppsclient.JobState_JOB_SUCCESS:
		return color.New(color.FgGreen).SprintFunc()("success")
	}
	return "-"
}

func pipelineState(pipelineState ppsclient.PipelineState) string {
	switch pipelineState {
	case ppsclient.PipelineState_PIPELINE_STARTING:
		return color.New(color.FgYellow).SprintFunc()("starting")
	case ppsclient.PipelineState_PIPELINE_RUNNING:
		return color.New(color.FgGreen).SprintFunc()("running")
	case ppsclient.PipelineState_PIPELINE_RESTARTING:
		return color.New(color.FgYellow).SprintFunc()("restarting")
	case ppsclient.PipelineState_PIPELINE_FAILED:
		return color.New(color.FgRed).SprintFunc()("failed")
	}
	return "-"
}

func jobInputs(jobInfo *ppsclient.JobInfo) string {
	var buffer bytes.Buffer
	writer := tabwriter.NewWriter(&buffer, 20, 1, 3, ' ', 0)
	PrintJobInputHeader(writer)
	for _, input := range jobInfo.Inputs {
		PrintJobInput(writer, input)
	}
	// can't error because buffer can't error on Write
	writer.Flush()
	return buffer.String()
}

func pipelineInputs(pipelineInfo *ppsclient.PipelineInfo) string {
	var buffer bytes.Buffer
	writer := tabwriter.NewWriter(&buffer, 20, 1, 3, ' ', 0)
	PrintPipelineInputHeader(writer)
	for _, input := range pipelineInfo.Inputs {
		PrintPipelineInput(writer, input)
	}
	// can't error because buffer can't error on Write
	writer.Flush()
	return buffer.String()
}

func jobCounts(counts map[int32]int32) string {
	var buffer bytes.Buffer
	for i := int32(ppsclient.JobState_JOB_PULLING); i <= int32(ppsclient.JobState_JOB_SUCCESS); i++ {
		fmt.Fprintf(&buffer, "%s: %d\t", jobState(ppsclient.JobState(i)), counts[i])
	}
	return buffer.String()
}

func prettyTransform(transform *ppsclient.Transform) (string, error) {
	result, err := json.MarshalIndent(transform, "", "  ")
	if err != nil {
		return "", err
	}
	return pretty.UnescapeHTML(string(result)), nil
}

var funcMap = template.FuncMap{
	"pipelineState":   pipelineState,
	"jobState":        jobState,
	"pipelineInputs":  pipelineInputs,
	"jobInputs":       jobInputs,
	"prettyDuration":  pretty.PrettyDuration,
	"jobCounts":       jobCounts,
	"prettyTransform": prettyTransform,
}
<|endoftext|>


<commit_before><commit_msg>ignore nr of returned values from labelvalues<commit_after><|endoftext|>


<commit_before>// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package execabs

import (
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"testing"
)

func TestFixCmd(t *testing.T) {
	cmd := &exec.Cmd{Path: "hello"}
	fixCmd("hello", cmd)
	if cmd.Path != "" {
		t.Errorf("fixCmd didn't clear cmd.Path")
	}
	expectedErr := fmt.Sprintf("hello resolves to executable in current directory (.%chello)", filepath.Separator)
	if err := cmd.Run(); err == nil {
		t.Fatal("Command.Run didn't fail")
	} else if err.Error() != expectedErr {
		t.Fatalf("Command.Run returned unexpected error: want %q, got %q", expectedErr, err.Error())
	}
}

func TestCommand(t *testing.T) {
	for _, cmd := range []func(string) *Cmd{
		func(s string) *Cmd { return Command(s) },
		func(s string) *Cmd { return CommandContext(context.Background(), s) },
	} {
		tmpDir, err := ioutil.TempDir("", "execabs-test")
		if err != nil {
			t.Fatalf("ioutil.TempDir failed: %s", err)
		}
		defer os.RemoveAll(tmpDir)
		executable := "execabs-test"
		if runtime.GOOS == "windows" {
			executable += ".exe"
		}
		if err = ioutil.WriteFile(filepath.Join(tmpDir, executable), []byte{1, 2, 3}, 0111); err != nil {
			t.Fatalf("ioutil.WriteFile failed: %s", err)
		}
		cwd, err := os.Getwd()
		if err != nil {
			t.Fatalf("os.Getwd failed: %s", err)
		}
		defer os.Chdir(cwd)
		if err = os.Chdir(tmpDir); err != nil {
			t.Fatalf("os.Chdir failed: %s", err)
		}
		if runtime.GOOS != "windows" {
			// add "." to PATH so that exec.LookPath looks in the current directory on
			// non-windows platforms as well
			origPath := os.Getenv("PATH")
			defer os.Setenv("PATH", origPath)
			os.Setenv("PATH", fmt.Sprintf(".:%s", origPath))
		}
		expectedErr := fmt.Sprintf("execabs-test resolves to executable in current directory (.%c%s)", filepath.Separator, executable)
		if err = cmd("execabs-test").Run(); err == nil {
			t.Fatalf("Command.Run didn't fail when exec.LookPath returned a relative path")
		} else if err.Error() != expectedErr {
			t.Errorf("Command.Run returned unexpected error: want %q, got %q", expectedErr, err.Error())
		}
	}
}

func TestLookPath(t *testing.T) {
	tmpDir, err := ioutil.TempDir("", "execabs-test")
	if err != nil {
		t.Fatalf("ioutil.TempDir failed: %s", err)
	}
	defer os.RemoveAll(tmpDir)
	executable := "execabs-test"
	if runtime.GOOS == "windows" {
		executable += ".exe"
	}
	if err = ioutil.WriteFile(filepath.Join(tmpDir, executable), []byte{1, 2, 3}, 0111); err != nil {
		t.Fatalf("ioutil.WriteFile failed: %s", err)
	}
	cwd, err := os.Getwd()
	if err != nil {
		t.Fatalf("os.Getwd failed: %s", err)
	}
	defer os.Chdir(cwd)
	if err = os.Chdir(tmpDir); err != nil {
		t.Fatalf("os.Chdir failed: %s", err)
	}
	if runtime.GOOS != "windows" {
		// add "." to PATH so that exec.LookPath looks in the current directory on
		// non-windows platforms as well
		origPath := os.Getenv("PATH")
		defer os.Setenv("PATH", origPath)
		os.Setenv("PATH", fmt.Sprintf(".:%s", origPath))
	}
	expectedErr := fmt.Sprintf("execabs-test resolves to executable in current directory (.%c%s)", filepath.Separator, executable)
	if _, err := LookPath("execabs-test"); err == nil {
		t.Fatalf("LookPath didn't fail when finding a non-relative path")
	} else if err.Error() != expectedErr {
		t.Errorf("LookPath returned unexpected error: want %q, got %q", expectedErr, err.Error())
	}
}
<commit_msg>execabs: disable tests on ios and js/wasm<commit_after>// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package execabs

import (
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"testing"
)

// hasExec reports whether the current system can start new processes
// using os.StartProcess or (more commonly) exec.Command.
// Copied from internal/testenv.HasExec
func hasExec() bool {
	switch runtime.GOOS {
	case "js", "ios":
		return false
	}
	return true
}

// mustHaveExec checks that the current system can start new processes
// using os.StartProcess or (more commonly) exec.Command.
// If not, mustHaveExec calls t.Skip with an explanation.
// Copied from internal/testenv.MustHaveExec
func mustHaveExec(t testing.TB) {
	if !hasExec() {
		t.Skipf("skipping test: cannot exec subprocess on %s/%s", runtime.GOOS, runtime.GOARCH)
	}
}

func TestFixCmd(t *testing.T) {
	cmd := &exec.Cmd{Path: "hello"}
	fixCmd("hello", cmd)
	if cmd.Path != "" {
		t.Errorf("fixCmd didn't clear cmd.Path")
	}
	expectedErr := fmt.Sprintf("hello resolves to executable in current directory (.%chello)", filepath.Separator)
	if err := cmd.Run(); err == nil {
		t.Fatal("Command.Run didn't fail")
	} else if err.Error() != expectedErr {
		t.Fatalf("Command.Run returned unexpected error: want %q, got %q", expectedErr, err.Error())
	}
}

func TestCommand(t *testing.T) {
	mustHaveExec(t)

	for _, cmd := range []func(string) *Cmd{
		func(s string) *Cmd { return Command(s) },
		func(s string) *Cmd { return CommandContext(context.Background(), s) },
	} {
		tmpDir, err := ioutil.TempDir("", "execabs-test")
		if err != nil {
			t.Fatalf("ioutil.TempDir failed: %s", err)
		}
		defer os.RemoveAll(tmpDir)
		executable := "execabs-test"
		if runtime.GOOS == "windows" {
			executable += ".exe"
		}
		if err = ioutil.WriteFile(filepath.Join(tmpDir, executable), []byte{1, 2, 3}, 0111); err != nil {
			t.Fatalf("ioutil.WriteFile failed: %s", err)
		}
		cwd, err := os.Getwd()
		if err != nil {
			t.Fatalf("os.Getwd failed: %s", err)
		}
		defer os.Chdir(cwd)
		if err = os.Chdir(tmpDir); err != nil {
			t.Fatalf("os.Chdir failed: %s", err)
		}
		if runtime.GOOS != "windows" {
			// add "." to PATH so that exec.LookPath looks in the current directory on
			// non-windows platforms as well
			origPath := os.Getenv("PATH")
			defer os.Setenv("PATH", origPath)
			os.Setenv("PATH", fmt.Sprintf(".:%s", origPath))
		}
		expectedErr := fmt.Sprintf("execabs-test resolves to executable in current directory (.%c%s)", filepath.Separator, executable)
		if err = cmd("execabs-test").Run(); err == nil {
			t.Fatalf("Command.Run didn't fail when exec.LookPath returned a relative path")
		} else if err.Error() != expectedErr {
			t.Errorf("Command.Run returned unexpected error: want %q, got %q", expectedErr, err.Error())
		}
	}
}

func TestLookPath(t *testing.T) {
	mustHaveExec(t)

	tmpDir, err := ioutil.TempDir("", "execabs-test")
	if err != nil {
		t.Fatalf("ioutil.TempDir failed: %s", err)
	}
	defer os.RemoveAll(tmpDir)
	executable := "execabs-test"
	if runtime.GOOS == "windows" {
		executable += ".exe"
	}
	if err = ioutil.WriteFile(filepath.Join(tmpDir, executable), []byte{1, 2, 3}, 0111); err != nil {
		t.Fatalf("ioutil.WriteFile failed: %s", err)
	}
	cwd, err := os.Getwd()
	if err != nil {
		t.Fatalf("os.Getwd failed: %s", err)
	}
	defer os.Chdir(cwd)
	if err = os.Chdir(tmpDir); err != nil {
		t.Fatalf("os.Chdir failed: %s", err)
	}
	if runtime.GOOS != "windows" {
		// add "." to PATH so that exec.LookPath looks in the current directory on
		// non-windows platforms as well
		origPath := os.Getenv("PATH")
		defer os.Setenv("PATH", origPath)
		os.Setenv("PATH", fmt.Sprintf(".:%s", origPath))
	}
	expectedErr := fmt.Sprintf("execabs-test resolves to executable in current directory (.%c%s)", filepath.Separator, executable)
	if _, err := LookPath("execabs-test"); err == nil {
		t.Fatalf("LookPath didn't fail when finding a non-relative path")
	} else if err.Error() != expectedErr {
		t.Errorf("LookPath returned unexpected error: want %q, got %q", expectedErr, err.Error())
	}
}
<|endoftext|>