<commit_before>package util

import (
	"bufio"
	"bytes"
	"encoding/base64"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"os"
	"os/exec"
	"strings"
	"syscall"
	"time"

	"github.com/golang/glog"
	"github.com/pkg/errors"

	"k8s.io/apimachinery/pkg/api/resource"
)

// CountingReader is a reader that keeps track of how much has been read
type CountingReader struct {
	Reader  io.ReadCloser
	Current int64
}

// RandAlphaNum provides an implementation to generate a random alpha numeric string of the specified length
func RandAlphaNum(n int) string {
	rand.Seed(time.Now().UnixNano())
	var letter = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")
	b := make([]rune, n)
	for i := range b {
		b[i] = letter[rand.Intn(len(letter))]
	}
	return string(b)
}

// GetNamespace returns the namespace the pod is executing in
func GetNamespace() string {
	return getNamespace("/var/run/secrets/kubernetes.io/serviceaccount/namespace")
}

func getNamespace(path string) string {
	if data, err := ioutil.ReadFile(path); err == nil {
		if ns := strings.TrimSpace(string(data)); len(ns) > 0 {
			return ns
		}
	}
	return "cdi"
}

// ParseEnvVar provides a wrapper to attempt to fetch the specified env var
func ParseEnvVar(envVarName string, decode bool) (string, error) {
	value := os.Getenv(envVarName)
	if decode {
		v, err := base64.StdEncoding.DecodeString(value)
		if err != nil {
			return "", errors.Errorf("error decoding environment variable %q", envVarName)
		}
		value = fmt.Sprintf("%s", v)
	}
	return value, nil
}

// Read reads bytes from the stream and updates the prometheus clone_progress metric according to the progress.
func (r *CountingReader) Read(p []byte) (n int, err error) {
	n, err = r.Reader.Read(p)
	r.Current += int64(n)
	return n, err
}

// Close closes the stream
func (r *CountingReader) Close() error {
	return r.Reader.Close()
}

// GetAvailableSpace gets the amount of available space at the path specified.
func GetAvailableSpace(path string) int64 {
	var stat syscall.Statfs_t
	syscall.Statfs(path, &stat)
	return int64(stat.Bavail) * int64(stat.Bsize)
}

// MinQuantity calculates the minimum of two quantities.
func MinQuantity(availableSpace, imageSize *resource.Quantity) resource.Quantity {
	if imageSize.Cmp(*availableSpace) == 1 {
		return *availableSpace
	}
	return *imageSize
}

// UnArchiveTar unarchives a tar file and streams its files
// using the specified io.Reader to the specified destination.
func UnArchiveTar(reader io.Reader, destDir string, arg ...string) error {
	glog.V(1).Infof("begin untar...\n")

	var tarOptions string
	var args = arg
	if len(arg) > 0 {
		tarOptions = arg[0]
		args = arg[1:]
	}
	options := fmt.Sprintf("-%s%s", tarOptions, "xvC")
	untar := exec.Command("/usr/bin/tar", options, destDir, strings.Join(args, ""))
	untar.Stdin = reader
	var errBuf bytes.Buffer
	untar.Stderr = &errBuf
	err := untar.Start()
	if err != nil {
		return err
	}
	err = untar.Wait()
	if err != nil {
		glog.V(3).Infof("%s\n", string(errBuf.Bytes()))
		glog.Errorf("%s\n", err.Error())
		return err
	}
	return nil
}

// ExtractSingleDirTar unarchives a tar file and streams its files
// using the specified io.Reader to the specified destination.
func ExtractSingleDirTar(reader io.Reader, destDir string, extractDir string) error {
	glog.V(1).Infof("begin untar directory ...\n")
	args := "-zxvC"
	untar := exec.Command("/usr/bin/tar", args, destDir, extractDir)
	untar.Stdin = reader
	var errBuf bytes.Buffer
	untar.Stderr = &errBuf
	err := untar.Start()
	if err != nil {
		return err
	}
	err = untar.Wait()
	if err != nil {
		glog.V(3).Infof("%s\n", string(errBuf.Bytes()))
		return err
	}

	return nil
}

// UnArchiveSingleDirFromLocalTar extracts a specified extractDir from local tar file to the specified destination.
func UnArchiveSingleDirFromLocalTar(filePath, destDir string, extractDir string) error {
	file, err := os.Open(filePath)
	if err != nil {
		return errors.Wrap(err, "could not open tar file")
	}
	fileReader := bufio.NewReader(file)
	return ExtractSingleDirTar(fileReader, destDir, extractDir)
}

// UnArchiveLocalTar unarchives a local tar file to the specified destination.
func UnArchiveLocalTar(filePath, destDir string, arg ...string) error {
	file, err := os.Open(filePath)
	if err != nil {
		return errors.Wrap(err, "could not open tar file")
	}
	fileReader := bufio.NewReader(file)
	return UnArchiveTar(fileReader, destDir, arg...)
}
<commit_msg>Remove unused code<commit_after>package util

import (
	"bufio"
	"bytes"
	"encoding/base64"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"os"
	"os/exec"
	"strings"
	"syscall"
	"time"

	"github.com/golang/glog"
	"github.com/pkg/errors"

	"k8s.io/apimachinery/pkg/api/resource"
)

// CountingReader is a reader that keeps track of how much has been read
type CountingReader struct {
	Reader  io.ReadCloser
	Current int64
}

// RandAlphaNum provides an implementation to generate a random alpha numeric string of the specified length
func RandAlphaNum(n int) string {
	rand.Seed(time.Now().UnixNano())
	var letter = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")
	b := make([]rune, n)
	for i := range b {
		b[i] = letter[rand.Intn(len(letter))]
	}
	return string(b)
}

// GetNamespace returns the namespace the pod is executing in
func GetNamespace() string {
	return getNamespace("/var/run/secrets/kubernetes.io/serviceaccount/namespace")
}

func getNamespace(path string) string {
	if data, err := ioutil.ReadFile(path); err == nil {
		if ns := strings.TrimSpace(string(data)); len(ns) > 0 {
			return ns
		}
	}
	return "cdi"
}

// ParseEnvVar provides a wrapper to attempt to fetch the specified env var
func ParseEnvVar(envVarName string, decode bool) (string, error) {
	value := os.Getenv(envVarName)
	if decode {
		v, err := base64.StdEncoding.DecodeString(value)
		if err != nil {
			return "", errors.Errorf("error decoding environment variable %q", envVarName)
		}
		value = fmt.Sprintf("%s", v)
	}
	return value, nil
}

// Read reads bytes from the stream and updates the prometheus clone_progress metric according to the progress.
func (r *CountingReader) Read(p []byte) (n int, err error) {
	n, err = r.Reader.Read(p)
	r.Current += int64(n)
	return n, err
}

// Close closes the stream
func (r *CountingReader) Close() error {
	return r.Reader.Close()
}

// GetAvailableSpace gets the amount of available space at the path specified.
func GetAvailableSpace(path string) int64 {
	var stat syscall.Statfs_t
	syscall.Statfs(path, &stat)
	return int64(stat.Bavail) * int64(stat.Bsize)
}

// MinQuantity calculates the minimum of two quantities.
func MinQuantity(availableSpace, imageSize *resource.Quantity) resource.Quantity {
	if imageSize.Cmp(*availableSpace) == 1 {
		return *availableSpace
	}
	return *imageSize
}

// UnArchiveTar unarchives a tar file and streams its files
// using the specified io.Reader to the specified destination.
func UnArchiveTar(reader io.Reader, destDir string, arg ...string) error {
	glog.V(1).Infof("begin untar...\n")

	var tarOptions string
	var args = arg
	if len(arg) > 0 {
		tarOptions = arg[0]
		args = arg[1:]
	}
	options := fmt.Sprintf("-%s%s", tarOptions, "xvC")
	untar := exec.Command("/usr/bin/tar", options, destDir, strings.Join(args, ""))
	untar.Stdin = reader
	var errBuf bytes.Buffer
	untar.Stderr = &errBuf
	err := untar.Start()
	if err != nil {
		return err
	}
	err = untar.Wait()
	if err != nil {
		glog.V(3).Infof("%s\n", string(errBuf.Bytes()))
		glog.Errorf("%s\n", err.Error())
		return err
	}
	return nil
}

// UnArchiveLocalTar unarchives a local tar file to the specified destination.
func UnArchiveLocalTar(filePath, destDir string, arg ...string) error {
	file, err := os.Open(filePath)
	if err != nil {
		return errors.Wrap(err, "could not open tar file")
	}
	fileReader := bufio.NewReader(file)
	return UnArchiveTar(fileReader, destDir, arg...)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"go/build"
	"io/ioutil"
	"os"
)

var buildCtx = build.Context{
	Compiler:   "gc",
	CgoEnabled: false,
}

var EGCC, EGLD, EGAR string

func getEnv() {
	EGCC = os.Getenv("EGCC")
	if EGCC == "" {
		die("EGCC environment variable not set")
	}
	EGLD = os.Getenv("EGLD")
	if EGLD == "" {
		die("EGLD environment variable not set")
	}
	EGAR = os.Getenv("EGAR")
	if EGAR == "" {
		die("EGAR environment variable not set")
	}
	buildCtx.GOARCH = os.Getenv("EGARCH")
	if buildCtx.GOARCH == "" {
		die("EGARCH environment variable not set")
	}
	buildCtx.GOOS = os.Getenv("EGOS")
	if buildCtx.GOOS == "" {
		die("EGOS environment variable not set")
	}
	buildCtx.GOROOT = os.Getenv("EGROOT")
	if buildCtx.GOROOT == "" {
		die("EGROOT environment variable not set")
	}
	buildCtx.GOPATH = os.Getenv("EGPATH")
	if buildCtx.GOPATH == "" {
		die("EGPATH environment variable not set")
	}
	if egtarget := os.Getenv("EGTARGET"); egtarget != "" {
		buildCtx.BuildTags = []string{egtarget}
		buildCtx.InstallSuffix = egtarget
	}
}

var (
	tmpDir    string
	verbosity int
	optLevel  string
	disableBC bool
)

func usage() {
	fmt.Println("Usage:\n  egc [flags] PKGPATH")
	flag.PrintDefaults()
}

func main() {
	flag.IntVar(&verbosity, "v", 0, "Verbosity level [0...2]")
	flag.StringVar(&optLevel, "O", "s", "GCC optimization level")
	flag.BoolVar(&disableBC, "B", false, "Disable bounds checking")
	flag.Usage = usage
	flag.Parse()
	args := flag.Args()
	if len(args) > 1 {
		usage()
		os.Exit(1)
	}

	getEnv()

	path := "."
	if len(args) == 1 {
		path = args[0]
	}

	var err error

	tmpDir, err = ioutil.TempDir("", "eg-build")
	if err != nil {
		logErr(err)
		os.Exit(1)
	}
	defer os.RemoveAll(tmpDir)

	if err = egc(path); err != nil {
		logErr(err)
		os.Exit(1)
	}
}
<commit_msg>egc: EGARCH, EGOS checking.<commit_after>package main

import (
	"flag"
	"fmt"
	"go/build"
	"io/ioutil"
	"os"
)

var buildCtx = build.Context{
	Compiler:   "gc",
	CgoEnabled: false,
}

var EGCC, EGLD, EGAR string

func getEnv() {
	EGCC = os.Getenv("EGCC")
	if EGCC == "" {
		die("EGCC environment variable not set")
	}
	EGLD = os.Getenv("EGLD")
	if EGLD == "" {
		die("EGLD environment variable not set")
	}
	EGAR = os.Getenv("EGAR")
	if EGAR == "" {
		die("EGAR environment variable not set")
	}
	buildCtx.GOARCH = os.Getenv("EGARCH")
	if buildCtx.GOARCH == "" {
		die("EGARCH environment variable not set")
	}
	if _, ok := archMap[buildCtx.GOARCH]; !ok {
		die("Unknown EGARCH: " + buildCtx.GOARCH)
	}
	buildCtx.GOOS = os.Getenv("EGOS")
	if buildCtx.GOOS == "" {
		die("EGOS environment variable not set")
	}
	if _, ok := archMap[buildCtx.GOOS]; !ok {
		die("Unknown EGOS: " + buildCtx.GOOS)
	}
	buildCtx.GOROOT = os.Getenv("EGROOT")
	if buildCtx.GOROOT == "" {
		die("EGROOT environment variable not set")
	}
	buildCtx.GOPATH = os.Getenv("EGPATH")
	if buildCtx.GOPATH == "" {
		die("EGPATH environment variable not set")
	}
	if egtarget := os.Getenv("EGTARGET"); egtarget != "" {
		buildCtx.BuildTags = []string{egtarget}
		buildCtx.InstallSuffix = egtarget
	}
}

var (
	tmpDir    string
	verbosity int
	optLevel  string
	disableBC bool
)

func usage() {
	fmt.Println("Usage:\n  egc [flags] PKGPATH")
	flag.PrintDefaults()
}

func main() {
	flag.IntVar(&verbosity, "v", 0, "Verbosity level [0...2]")
	flag.StringVar(&optLevel, "O", "s", "GCC optimization level")
	flag.BoolVar(&disableBC, "B", false, "Disable bounds checking")
	flag.Usage = usage
	flag.Parse()
	args := flag.Args()
	if len(args) > 1 {
		usage()
		os.Exit(1)
	}

	getEnv()

	path := "."
	if len(args) == 1 {
		path = args[0]
	}

	var err error

	tmpDir, err = ioutil.TempDir("", "eg-build")
	if err != nil {
		logErr(err)
		os.Exit(1)
	}
	defer os.RemoveAll(tmpDir)

	if err = egc(path); err != nil {
		logErr(err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package pretty

import (
	"fmt"
	"strings"
	"time"

	"github.com/docker/go-units"
	"github.com/fatih/color"
	"github.com/gogo/protobuf/types"
)

// UnescapeHTML returns s with < and > unescaped.
func UnescapeHTML(s string) string {
	s = strings.Replace(s, "\\u003c", "<", -1)
	s = strings.Replace(s, "\\u003e", ">", -1)
	return s
}

// Since pretty-prints the amount of time that has passed since timestamp as a
// human-readable string.
func Since(timestamp *types.Timestamp) string {
	t, _ := types.TimestampFromProto(timestamp)
	if t.Equal(time.Time{}) {
		return ""
	}
	return units.HumanDuration(time.Since(t))
}

// Ago pretty-prints the amount of time that has passed since timestamp as a
// human-readable string, and adds "ago" to the end.
func Ago(timestamp *types.Timestamp) string {
	return fmt.Sprintf("%s ago", Since(timestamp))
}

// TimeDifference pretty-prints the duration of time between from
// and to as a human-reabable string.
func TimeDifference(from *types.Timestamp, to *types.Timestamp) string {
	tFrom, _ := types.TimestampFromProto(from)
	tTo, _ := types.TimestampFromProto(to)
	return units.HumanDuration(tTo.Sub(tFrom))
}

// Duration pretty prints a duration in a human readable way.
func Duration(d *types.Duration) string {
	duration, _ := types.DurationFromProto(d)
	return units.HumanDuration(duration)
}

// Size pretty-prints size amount of bytes as a human readable string.
func Size(size uint64) string {
	return units.BytesSize(float64(size))
}

// ProgressBar pretty prints a progress bar with given width and green, yellow
// and red segments.  green, yellow and red need not add to width, they will be
// normalized. If red is nonzero there will always be at least one red segment,
// even if red is less than 1/width of the total bar.
func ProgressBar(width, green, yellow, red int) string {
	total := green + yellow + red
	var sb strings.Builder
	for i := 0; i < width; i++ {
		switch {
		case i == width-1 && red != 0:
			// if there is nonzero red then the final segment is always red,
			// this ensures that we don't present something as totally
			// successful when it wasn't
			sb.WriteString(color.RedString("▇"))
		case i*total < green*width:
			sb.WriteString(color.GreenString("▇"))
		case i*total < (green+yellow)*width:
			sb.WriteString(color.YellowString("▇"))
		case i*total < (green+yellow+red)*width:
			sb.WriteString(color.RedString("▇"))
		default:
			sb.WriteString(" ")
		}
	}
	return sb.String()
}
<commit_msg>Fix printing bug I introduced.<commit_after>package pretty

import (
	"fmt"
	"strings"
	"time"

	"github.com/docker/go-units"
	"github.com/fatih/color"
	"github.com/gogo/protobuf/types"
)

// UnescapeHTML returns s with < and > unescaped.
func UnescapeHTML(s string) string {
	s = strings.Replace(s, "\\u003c", "<", -1)
	s = strings.Replace(s, "\\u003e", ">", -1)
	return s
}

// Since pretty-prints the amount of time that has passed since timestamp as a
// human-readable string.
func Since(timestamp *types.Timestamp) string {
	t, _ := types.TimestampFromProto(timestamp)
	if t.Equal(time.Time{}) {
		return ""
	}
	return units.HumanDuration(time.Since(t))
}

// Ago pretty-prints the amount of time that has passed since timestamp as a
// human-readable string, and adds "ago" to the end.
func Ago(timestamp *types.Timestamp) string {
	since := Since(timestamp)
	if since == "" {
		return since
	}
	return fmt.Sprintf("%s ago", since)
}

// TimeDifference pretty-prints the duration of time between from
// and to as a human-reabable string.
func TimeDifference(from *types.Timestamp, to *types.Timestamp) string {
	tFrom, _ := types.TimestampFromProto(from)
	tTo, _ := types.TimestampFromProto(to)
	return units.HumanDuration(tTo.Sub(tFrom))
}

// Duration pretty prints a duration in a human readable way.
func Duration(d *types.Duration) string {
	duration, _ := types.DurationFromProto(d)
	return units.HumanDuration(duration)
}

// Size pretty-prints size amount of bytes as a human readable string.
func Size(size uint64) string {
	return units.BytesSize(float64(size))
}

// ProgressBar pretty prints a progress bar with given width and green, yellow
// and red segments.  green, yellow and red need not add to width, they will be
// normalized. If red is nonzero there will always be at least one red segment,
// even if red is less than 1/width of the total bar.
func ProgressBar(width, green, yellow, red int) string {
	total := green + yellow + red
	var sb strings.Builder
	for i := 0; i < width; i++ {
		switch {
		case i == width-1 && red != 0:
			// if there is nonzero red then the final segment is always red,
			// this ensures that we don't present something as totally
			// successful when it wasn't
			sb.WriteString(color.RedString("▇"))
		case i*total < green*width:
			sb.WriteString(color.GreenString("▇"))
		case i*total < (green+yellow)*width:
			sb.WriteString(color.YellowString("▇"))
		case i*total < (green+yellow+red)*width:
			sb.WriteString(color.RedString("▇"))
		default:
			sb.WriteString(" ")
		}
	}
	return sb.String()
}
<|endoftext|>


<commit_before>package main

import (
	"code.google.com/p/plotinum/plot"
	"code.google.com/p/plotinum/plotter"
	"flag"
	"fmt"
	"github.com/purzelrakete/bandit"
	"image/color"
	"log"
	"strconv"
	"strings"
)

// simulations maps model parameter to corresponding simulation results
type simulations map[float64]bandit.Simulation

// summary summarizes a Simulation and returns corresponding plot points.
type summary func(s bandit.Simulation) []float64

// xys turns a slice of float64 values into a plotter.XYs
func xys(data []float64) plotter.XYs {
	points := make(plotter.XYs, len(data))
	for i, datum := range data {
		points[i].X = float64(i)
		points[i].Y = datum
	}

	return points
}

// draw is a generic plotter of simulation summaries.
func draw(title, xLabel, yLabel, filename string, sims simulations, summary summary) {
	p, err := plot.New()
	if err != nil {
		log.Fatalf(err.Error())
	}

	p.Title.Text = title
	p.X.Label.Text = xLabel
	p.Y.Label.Text = yLabel

	for ε, sim := range sims {
		l, err := plotter.NewLine(xys(summary(sim)))
		if err != nil {
			log.Fatalf(err.Error())
		}

		p.Add(l)
		p.Legend.Add(fmt.Sprintf("%.2f", ε), l)
		l.LineStyle.Color = color.Gray{uint8(255 * 1.9 * ε)}
	}

	if err != nil {
		log.Fatalf(err.Error())
	}

	if err := p.Save(8, 8, filename); err != nil {
		log.Fatalf(err.Error())
	}
}

// parseArms converts command line 0.1,0.2 into a slice of floats. Returns
// the the best arm (1 indexed). In the case of equally good best arms there
// will be multiple indices in the returned slice.
func parseArms(sμ string) ([]float64, []int, error) {
	var μs []float64
	var imax []int
	max := 0.0
	for i, s := range strings.Split(sμ, ",") {
		μ, err := strconv.ParseFloat(s, 64)
		if err != nil {
			return []float64{}, []int{}, fmt.Errorf("NaN: %s", err.Error())
		}

		if μ < 0 || μ > 1 {
			return []float64{}, []int{}, fmt.Errorf("μ not in [0,1]: %.5f", μ)
		}

		// there may be multiple equally good (best) arms
		if μ > max {
			max = μ
			imax = []int{i + 1}
		} else if μ == max {
			imax = append(imax, i+1)
		}

		μs = append(μs, μ)
	}

	return μs, imax, nil
}

var (
	mcSims           = flag.Int("mcSims", 5000, "monte carlo simulations to run")
	mcHorizon        = flag.Int("mcHorizon", 300, "trials per simulation")
	mcMus            = flag.String("mcMus", "0.1,0.3,0.2,0.8", "bernoulli arm μ parameters")
	mcPerformancePng = flag.String("mcPerformancePng", "bandit_performance.png", "performance plot")
	mcAccuracyPng    = flag.String("mcAccuracyPng", "bandit_accuracy.png", "accuracy plot")
	mcCumulativePng  = flag.String("mcCumulativePng", "bandit_cumulative.png", "cumulative plot")
)

func init() {
	flag.Parse()
}

func main() {
	μs, bestArms, err := parseArms(*mcMus)
	if err != nil {
		log.Fatalf(err.Error())
	}

	var arms []bandit.Arm
	for _, μ := range μs {
		arms = append(arms, bandit.Bernoulli(μ))
	}

	sims := make(simulations)
	for _, ε := range []float64{0.1, 0.2, 0.3, 0.4, 0.5} {
		s, err := bandit.MonteCarlo(*mcSims, *mcHorizon, arms, func() (bandit.Bandit, error) {
			return bandit.EpsilonGreedyNew(len(μs), ε)
		})

		if err != nil {
			log.Fatalf(err.Error())
		}

		sims[ε] = s
	}

	title, x, y := "Greedy Accuracy", "Time", "P(selecting best arm)"
	draw(title, x, y, *mcAccuracyPng, sims, func(s bandit.Simulation) []float64 {
		return bandit.Accuracy(s, bestArms)
	})

	title, x, y = "Greedy Performance", "Time", "Reward"
	draw(title, x, y, *mcPerformancePng, sims, func(s bandit.Simulation) []float64 {
		return bandit.Performance(s)
	})

	title, x, y = "Greedy Cumulative Performance", "Time", "Cumulative Reward"
	draw(title, x, y, *mcCumulativePng, sims, func(s bandit.Simulation) []float64 {
		return bandit.Cumulative(s)
	})
}
<commit_msg>Less complected plotter<commit_after>package main

import (
	"code.google.com/p/plotinum/plot"
	"code.google.com/p/plotinum/plotter"
	"flag"
	"fmt"
	"github.com/purzelrakete/bandit"
	"image/color"
	"log"
	"strconv"
	"strings"
)

// simulations maps model parameter to corresponding simulation results
type simulations map[float64]bandit.Simulation

// summary summarizes a Simulation and returns corresponding plot points.
type summary func(s bandit.Simulation) []float64

// xys turns a slice of float64 values into a plotter.XYs
func xys(data []float64) plotter.XYs {
	points := make(plotter.XYs, len(data))
	for i, datum := range data {
		points[i].X = float64(i)
		points[i].Y = datum
	}

	return points
}

// plotLine represents labelled plot lines
type plotLines map[string][]float64

// draw is a generic plotter of labelled lines.
func draw(lines plotLines, title, xLabel, yLabel string) error {
	p, err := plot.New()
	if err != nil {
		return fmt.Errorf(err.Error())
	}

	p.Title.Text = title
	p.X.Label.Text = xLabel
	p.Y.Label.Text = yLabel

	i := 0
	for legend, data := range lines {
		i = i + 1
		l, err := plotter.NewLine(xys(data))
		if err != nil {
			return fmt.Errorf(err.Error())
		}

		p.Add(l)
		p.Legend.Add(legend, l)
		l.LineStyle.Color = color.Gray{uint8(48 * float64(i))}
	}

	if err != nil {
		return fmt.Errorf(err.Error())
	}

	filename := fmt.Sprintf("bandit_%s.png", strings.ToLower(title))
	if err := p.Save(8, 8, filename); err != nil {
		return fmt.Errorf(err.Error())
	}

	return nil
}

// parseArms converts command line 0.1,0.2 into a slice of floats. Returns
// the the best arm (1 indexed). In the case of equally good best arms there
// will be multiple indices in the returned slice.
func parseArms(sμ string) ([]float64, []int, error) {
	var μs []float64
	var imax []int
	max := 0.0
	for i, s := range strings.Split(sμ, ",") {
		μ, err := strconv.ParseFloat(s, 64)
		if err != nil {
			return []float64{}, []int{}, fmt.Errorf("NaN: %s", err.Error())
		}

		if μ < 0 || μ > 1 {
			return []float64{}, []int{}, fmt.Errorf("μ not in [0,1]: %.5f", μ)
		}

		// there may be multiple equally good (best) arms
		if μ > max {
			max = μ
			imax = []int{i + 1}
		} else if μ == max {
			imax = append(imax, i+1)
		}

		μs = append(μs, μ)
	}

	return μs, imax, nil
}

var (
	mcSims    = flag.Int("mcSims", 5000, "monte carlo simulations to run")
	mcHorizon = flag.Int("mcHorizon", 300, "trials per simulation")
	mcMus     = flag.String("mcMus", "0.1,0.3,0.2,0.8", "bernoulli arm μ parameters")
)

func init() {
	flag.Parse()
}

func main() {
	μs, bestArms, err := parseArms(*mcMus)
	if err != nil {
		log.Fatalf(err.Error())
	}

	var arms []bandit.Arm
	for _, μ := range μs {
		arms = append(arms, bandit.Bernoulli(μ))
	}

	lines := make(plotLines)
	for _, ε := range []float64{0.1, 0.2, 0.3, 0.4, 0.5} {
		s, err := bandit.MonteCarlo(*mcSims, *mcHorizon, arms, func() (bandit.Bandit, error) {
			return bandit.EpsilonGreedyNew(len(μs), ε)
		})

		if err != nil {
			log.Fatalf(err.Error())
		}

		lines[fmt.Sprintf("EpsilonGreedy(%.2f)", ε)] = bandit.Accuracy(s, bestArms)
	}

	draw(lines, "Accuracy", "Time", "P(selecting best arm)")

	lines = make(plotLines)
	for _, ε := range []float64{0.1, 0.2, 0.3, 0.4, 0.5} {
		s, err := bandit.MonteCarlo(*mcSims, *mcHorizon, arms, func() (bandit.Bandit, error) {
			return bandit.EpsilonGreedyNew(len(μs), ε)
		})

		if err != nil {
			log.Fatalf(err.Error())
		}

		lines[fmt.Sprintf("EpsilonGreedy(%.2f)", ε)] = bandit.Performance(s)
	}

	draw(lines, "Performance", "Time", "P(selecting best arm)")

	lines = make(plotLines)
	for _, ε := range []float64{0.1, 0.2, 0.3, 0.4, 0.5} {
		s, err := bandit.MonteCarlo(*mcSims, *mcHorizon, arms, func() (bandit.Bandit, error) {
			return bandit.EpsilonGreedyNew(len(μs), ε)
		})

		if err != nil {
			log.Fatalf(err.Error())
		}

		lines[fmt.Sprintf("EpsilonGreedy(%.2f)", ε)] = bandit.Cumulative(s)
	}

	draw(lines, "Cumulative", "Time", "P(selecting best arm)")
}
<|endoftext|>


<commit_before>package printer

import (
	"fmt"
	"math"
	"strings"

	"github.com/fatih/color"
	"github.com/goccy/go-yaml/ast"
	"github.com/goccy/go-yaml/token"
)

// Property additional property set for each the token
type Property struct {
	Prefix string
	Suffix string
}

// PrintFunc returns property instance
type PrintFunc func() *Property

// Printer create text from token collection or ast
type Printer struct {
	LineNumber       bool
	LineNumberFormat func(num int) string
	MapKey           PrintFunc
	Anchor           PrintFunc
	Alias            PrintFunc
	Bool             PrintFunc
	String           PrintFunc
	Number           PrintFunc
}

func defaultLineNumberFormat(num int) string {
	return fmt.Sprintf("%2d | ", num)
}

func (p *Printer) lineTexts(tk *token.Token, prop *Property) []string {
	texts := []string{}
	for idx, src := range strings.Split(tk.Origin, "\n") {
		header := ""
		if p.LineNumber {
			header = p.LineNumberFormat(tk.Position.Line + idx)
		}
		lineText := prop.Prefix + src + prop.Suffix
		texts = append(texts, fmt.Sprintf("%s%s", header, lineText))
	}
	return texts
}

func (p *Printer) property(tk *token.Token) *Property {
	prop := &Property{}
	switch tk.PreviousType() {
	case token.AnchorType:
		if p.Anchor != nil {
			return p.Anchor()
		}
		return prop
	case token.AliasType:
		if p.Alias != nil {
			return p.Alias()
		}
		return prop
	}
	switch tk.NextType() {
	case token.MappingValueType:
		if p.MapKey != nil {
			return p.MapKey()
		}
		return prop
	}
	switch tk.Type {
	case token.BoolType:
		if p.Bool != nil {
			return p.Bool()
		}
		return prop
	case token.AnchorType:
		if p.Anchor != nil {
			return p.Anchor()
		}
		return prop
	case token.AliasType:
		if p.Anchor != nil {
			return p.Alias()
		}
		return prop
	case token.StringType, token.SingleQuoteType, token.DoubleQuoteType:
		if p.String != nil {
			return p.String()
		}
		return prop
	case token.IntegerType, token.FloatType:
		if p.Number != nil {
			return p.Number()
		}
		return prop
	default:
	}
	return prop
}

// PrintTokens create text from token collection
func (p *Printer) PrintTokens(tokens token.Tokens) string {
	if len(tokens) == 0 {
		return ""
	}
	if p.LineNumber {
		if p.LineNumberFormat == nil {
			p.LineNumberFormat = defaultLineNumberFormat
		}
	}
	texts := []string{}
	lineNumber := tokens[0].Position.Line
	for _, tk := range tokens {
		lines := strings.Split(tk.Origin, "\n")
		prop := p.property(tk)
		header := ""
		if p.LineNumber {
			header = p.LineNumberFormat(lineNumber)
		}
		if len(lines) == 1 {
			line := prop.Prefix + lines[0] + prop.Suffix
			if len(texts) == 0 {
				texts = append(texts, header+line)
				lineNumber++
			} else {
				text := texts[len(texts)-1]
				texts[len(texts)-1] = text + line
			}
		} else {
			for idx, src := range lines {
				if p.LineNumber {
					header = p.LineNumberFormat(lineNumber)
				}
				line := prop.Prefix + src + prop.Suffix
				if idx == 0 {
					if len(texts) == 0 {
						texts = append(texts, header+line)
						lineNumber++
					} else {
						text := texts[len(texts)-1]
						texts[len(texts)-1] = text + line
					}
				} else {
					texts = append(texts, fmt.Sprintf("%s%s", header, line))
					lineNumber++
				}
			}
		}
	}
	return strings.Join(texts, "\n")
}

// PrintNode create text from ast.Node
func (p *Printer) PrintNode(node ast.Node) []byte {
	return []byte(fmt.Sprintf("%+v\n", node))
}

const escape = "\x1b"

func format(attr color.Attribute) string {
	return fmt.Sprintf("%s[%dm", escape, attr)
}

func (p *Printer) setDefaultColorSet() {
	p.Bool = func() *Property {
		return &Property{
			Prefix: format(color.FgHiMagenta),
			Suffix: format(color.Reset),
		}
	}
	p.Number = func() *Property {
		return &Property{
			Prefix: format(color.FgHiMagenta),
			Suffix: format(color.Reset),
		}
	}
	p.MapKey = func() *Property {
		return &Property{
			Prefix: format(color.FgHiCyan),
			Suffix: format(color.Reset),
		}
	}
	p.Anchor = func() *Property {
		return &Property{
			Prefix: format(color.FgHiYellow),
			Suffix: format(color.Reset),
		}
	}
	p.Alias = func() *Property {
		return &Property{
			Prefix: format(color.FgHiYellow),
			Suffix: format(color.Reset),
		}
	}
	p.String = func() *Property {
		return &Property{
			Prefix: format(color.FgHiGreen),
			Suffix: format(color.Reset),
		}
	}
}

func (p *Printer) PrintErrorMessage(msg string, isColored bool) string {
	if isColored {
		return fmt.Sprintf("%s%s%s",
			format(color.FgHiRed),
			msg,
			format(color.Reset),
		)
	}
	return msg
}

func (p *Printer) PrintErrorToken(tk *token.Token, isColored bool) string {
	errToken := tk
	pos := tk.Position
	curLine := pos.Line
	curExtLine := curLine + len(strings.Split(tk.Origin, "\n")) - 1
	minLine := int(math.Max(float64(curLine-3), 1))
	maxLine := curExtLine + 3
	for {
		if tk.Position.Line < minLine {
			break
		}
		if tk.Prev == nil {
			break
		}
		tk = tk.Prev
	}
	tokens := token.Tokens{}
	lastTk := tk
	for tk.Position.Line <= curExtLine {
		tokens.Add(tk)
		lastTk = tk
		tk = tk.Next
		if tk == nil {
			break
		}
	}
	org := lastTk.Origin
	trimmed := strings.TrimRight(strings.TrimRight(lastTk.Origin, " "), "\n")
	lastTk.Origin = trimmed
	if tk != nil {
		tk.Origin = org[len(org)-len(trimmed):] + tk.Origin
	}
	p.LineNumber = true
	p.LineNumberFormat = func(num int) string {
		if isColored {
			fn := color.New(color.Bold, color.FgHiWhite).SprintFunc()
			if curLine == num {
				return fn(fmt.Sprintf("> %2d | ", num))
			}
			return fn(fmt.Sprintf("  %2d | ", num))
		}
		if curLine == num {
			return fmt.Sprintf("> %2d | ", num)
		}
		return fmt.Sprintf("  %2d | ", num)
	}
	if isColored {
		p.setDefaultColorSet()
	}
	beforeSource := p.PrintTokens(tokens)
	prefixSpaceNum := len(fmt.Sprintf("  %2d | ", 1))
	annotateLine := strings.Repeat(" ", prefixSpaceNum+errToken.Position.Column-2) + "^"
	tokens = token.Tokens{}
	for tk != nil {
		if tk.Position.Line > maxLine {
			break
		}
		tokens.Add(tk)
		tk = tk.Next
	}
	afterSource := p.PrintTokens(tokens)
	return fmt.Sprintf("%s\n%s\n%s", beforeSource, annotateLine, afterSource)
}
<commit_msg>Fix printing of syntax error<commit_after>package printer

import (
	"fmt"
	"math"
	"strings"

	"github.com/fatih/color"
	"github.com/goccy/go-yaml/ast"
	"github.com/goccy/go-yaml/token"
)

// Property additional property set for each the token
type Property struct {
	Prefix string
	Suffix string
}

// PrintFunc returns property instance
type PrintFunc func() *Property

// Printer create text from token collection or ast
type Printer struct {
	LineNumber       bool
	LineNumberFormat func(num int) string
	MapKey           PrintFunc
	Anchor           PrintFunc
	Alias            PrintFunc
	Bool             PrintFunc
	String           PrintFunc
	Number           PrintFunc
}

func defaultLineNumberFormat(num int) string {
	return fmt.Sprintf("%2d | ", num)
}

func (p *Printer) lineTexts(tk *token.Token, prop *Property) []string {
	texts := []string{}
	for idx, src := range strings.Split(tk.Origin, "\n") {
		header := ""
		if p.LineNumber {
			header = p.LineNumberFormat(tk.Position.Line + idx)
		}
		lineText := prop.Prefix + src + prop.Suffix
		texts = append(texts, fmt.Sprintf("%s%s", header, lineText))
	}
	return texts
}

func (p *Printer) property(tk *token.Token) *Property {
	prop := &Property{}
	switch tk.PreviousType() {
	case token.AnchorType:
		if p.Anchor != nil {
			return p.Anchor()
		}
		return prop
	case token.AliasType:
		if p.Alias != nil {
			return p.Alias()
		}
		return prop
	}
	switch tk.NextType() {
	case token.MappingValueType:
		if p.MapKey != nil {
			return p.MapKey()
		}
		return prop
	}
	switch tk.Type {
	case token.BoolType:
		if p.Bool != nil {
			return p.Bool()
		}
		return prop
	case token.AnchorType:
		if p.Anchor != nil {
			return p.Anchor()
		}
		return prop
	case token.AliasType:
		if p.Anchor != nil {
			return p.Alias()
		}
		return prop
	case token.StringType, token.SingleQuoteType, token.DoubleQuoteType:
		if p.String != nil {
			return p.String()
		}
		return prop
	case token.IntegerType, token.FloatType:
		if p.Number != nil {
			return p.Number()
		}
		return prop
	default:
	}
	return prop
}

// PrintTokens create text from token collection
func (p *Printer) PrintTokens(tokens token.Tokens) string {
	if len(tokens) == 0 {
		return ""
	}
	if p.LineNumber {
		if p.LineNumberFormat == nil {
			p.LineNumberFormat = defaultLineNumberFormat
		}
	}
	texts := []string{}
	lineNumber := tokens[0].Position.Line
	for _, tk := range tokens {
		lines := strings.Split(tk.Origin, "\n")
		prop := p.property(tk)
		header := ""
		if p.LineNumber {
			header = p.LineNumberFormat(lineNumber)
		}
		if len(lines) == 1 {
			line := prop.Prefix + lines[0] + prop.Suffix
			if len(texts) == 0 {
				texts = append(texts, header+line)
				lineNumber++
			} else {
				text := texts[len(texts)-1]
				texts[len(texts)-1] = text + line
			}
		} else {
			for idx, src := range lines {
				if p.LineNumber {
					header = p.LineNumberFormat(lineNumber)
				}
				line := prop.Prefix + src + prop.Suffix
				if idx == 0 {
					if len(texts) == 0 {
						texts = append(texts, header+line)
						lineNumber++
					} else {
						text := texts[len(texts)-1]
						texts[len(texts)-1] = text + line
					}
				} else {
					texts = append(texts, fmt.Sprintf("%s%s", header, line))
					lineNumber++
				}
			}
		}
	}
	return strings.Join(texts, "\n")
}

// PrintNode create text from ast.Node
func (p *Printer) PrintNode(node ast.Node) []byte {
	return []byte(fmt.Sprintf("%+v\n", node))
}

const escape = "\x1b"

func format(attr color.Attribute) string {
	return fmt.Sprintf("%s[%dm", escape, attr)
}

func (p *Printer) setDefaultColorSet() {
	p.Bool = func() *Property {
		return &Property{
			Prefix: format(color.FgHiMagenta),
			Suffix: format(color.Reset),
		}
	}
	p.Number = func() *Property {
		return &Property{
			Prefix: format(color.FgHiMagenta),
			Suffix: format(color.Reset),
		}
	}
	p.MapKey = func() *Property {
		return &Property{
			Prefix: format(color.FgHiCyan),
			Suffix: format(color.Reset),
		}
	}
	p.Anchor = func() *Property {
		return &Property{
			Prefix: format(color.FgHiYellow),
			Suffix: format(color.Reset),
		}
	}
	p.Alias = func() *Property {
		return &Property{
			Prefix: format(color.FgHiYellow),
			Suffix: format(color.Reset),
		}
	}
	p.String = func() *Property {
		return &Property{
			Prefix: format(color.FgHiGreen),
			Suffix: format(color.Reset),
		}
	}
}

func (p *Printer) PrintErrorMessage(msg string, isColored bool) string {
	if isColored {
		return fmt.Sprintf("%s%s%s",
			format(color.FgHiRed),
			msg,
			format(color.Reset),
		)
	}
	return msg
}

func (p *Printer) PrintErrorToken(tk *token.Token, isColored bool) string {
	errToken := tk
	pos := tk.Position
	curLine := pos.Line
	curExtLine := curLine + len(strings.Split(strings.TrimLeft(tk.Origin, "\n"), "\n")) - 1
	minLine := int(math.Max(float64(curLine-3), 1))
	maxLine := curExtLine + 3
	for {
		if tk.Position.Line < minLine {
			break
		}
		if tk.Prev == nil {
			break
		}
		tk = tk.Prev
	}
	tokens := token.Tokens{}
	lastTk := tk
	for tk.Position.Line <= curExtLine {
		tokens.Add(tk)
		lastTk = tk
		tk = tk.Next
		if tk == nil {
			break
		}
	}
	org := lastTk.Origin
	trimmed := strings.TrimRight(strings.TrimRight(lastTk.Origin, " "), "\n")
	lastTk.Origin = trimmed
	if tk != nil {
		tk.Origin = org[len(trimmed)+1:] + tk.Origin
	}
	p.LineNumber = true
	p.LineNumberFormat = func(num int) string {
		if isColored {
			fn := color.New(color.Bold, color.FgHiWhite).SprintFunc()
			if curLine == num {
				return fn(fmt.Sprintf("> %2d | ", num))
			}
			return fn(fmt.Sprintf("  %2d | ", num))
		}
		if curLine == num {
			return fmt.Sprintf("> %2d | ", num)
		}
		return fmt.Sprintf("  %2d | ", num)
	}
	if isColored {
		p.setDefaultColorSet()
	}
	beforeSource := p.PrintTokens(tokens)
	prefixSpaceNum := len(fmt.Sprintf("  %2d | ", 1))
	annotateLine := strings.Repeat(" ", prefixSpaceNum+errToken.Position.Column-2) + "^"
	tokens = token.Tokens{}
	for tk != nil {
		if tk.Position.Line > maxLine {
			break
		}
		tokens.Add(tk)
		tk = tk.Next
	}
	afterSource := p.PrintTokens(tokens)
	return fmt.Sprintf("%s\n%s\n%s", beforeSource, annotateLine, afterSource)
}
<|endoftext|>


<commit_before>package crawler

import (
	"net"
	"net/url"
	// "path"
	"context"
	"fmt"
	"github.com/ipfs-search/ipfs-search/indexer"
	"github.com/ipfs/go-ipfs-api"
	"log"
	"strings"
	"syscall"
	"time"
)

// Indexable consists of args with a Crawler
type Indexable struct {
	*Crawler
	*Args
}

// String returns '<hash>' (<name>)
func (i *Indexable) String() string {
	if i.Name != "" {
		return fmt.Sprintf("'%s' (%s)", i.Hash, i.Name)
	}
	return fmt.Sprintf("'%s' (Unnamed)", i.Hash)
}

// handleShellError handles IPFS shell errors; returns try again bool and original error
func (i *Indexable) handleShellError(ctx context.Context, err error) (bool, error) {
	if _, ok := err.(*shell.Error); ok && (strings.Contains(err.Error(), "proto") ||
		strings.Contains(err.Error(), "unrecognized type") ||
		strings.Contains(err.Error(), "not a valid merkledag node")) {

		// Attempt to index invalid to prevent re-indexing
		i.indexInvalid(ctx, err)

		// Don't try again, return error
		return false, err
	}

	// Different error, attempt handling as URL error
	return i.handleURLError(err)
}

// handleURLError handles HTTP errors graceously, returns try again bool and original error
func (i *Indexable) handleURLError(err error) (bool, error) {
	if uerr, ok := err.(*url.Error); ok {
		if uerr.Timeout() {
			// Fail on timeouts
			return false, err
		}

		if uerr.Temporary() {
			// Retry on other temp errors
			log.Printf("Temporary URL error: %v", uerr)
			return true, nil
		}

		// Somehow, the errors below are not temp errors !?
		switch t := uerr.Err.(type) {
		case *net.OpError:
			if t.Op == "dial" {
				log.Printf("Unknown host %v", t)
				return true, nil

			} else if t.Op == "read" {
				log.Printf("Connection refused %v", t)
				return true, nil
			}

		case syscall.Errno:
			if t == syscall.ECONNREFUSED {
				log.Printf("Connection refused %v", t)
				return true, nil
			}
		}
	}

	return false, err
}

// hashURL returns the IPFS URL for a particular hash
func (i *Indexable) hashURL() string {
	return fmt.Sprintf("/ipfs/%s", i.Hash)
}

// getFileList return list of files and/or type of item (directory/file)
func (i *Indexable) getFileList(ctx context.Context) (list *shell.UnixLsObject, err error) {
	url := i.hashURL()

	tryAgain := true
	for tryAgain {
		list, err = i.Shell.FileList(url)

		tryAgain, err = i.handleShellError(ctx, err)

		if tryAgain {
			log.Printf("Retrying in %s", i.Config.RetryWait)
			time.Sleep(i.Config.RetryWait)
		}
	}

	return
}

// indexInvalid indexes invalid files to prevent indexing again
func (i *Indexable) indexInvalid(ctx context.Context, err error) {
	// Attempt to index panic to prevent re-indexing
	m := metadata{
		"error": err.Error(),
	}

	i.Indexer.IndexItem(ctx, "invalid", i.Hash, m)
}

// queueList queues any items in a given list/directory
func (i *Indexable) queueList(ctx context.Context, list *shell.UnixLsObject) (err error) {
	for _, link := range list.Links {
		dirArgs := &Args{
			Hash:       link.Hash,
			Name:       link.Name,
			Size:       link.Size,
			ParentHash: i.Hash,
		}

		switch link.Type {
		case "File":
			// Add file to crawl queue
			err = i.FileQueue.Publish(dirArgs)
		case "Directory":
			// Add directory to crawl queue
			err = i.HashQueue.Publish(dirArgs)
		default:
			log.Printf("Type '%s' skipped for %s", link.Type, i)
			i.indexInvalid(ctx, fmt.Errorf("Unknown type: %s", link.Type))
		}
	}

	return
}

// processList processes and indexes a file listing
func (i *Indexable) processList(ctx context.Context, list *shell.UnixLsObject, references []indexer.Reference) (err error) {
	switch list.Type {
	case "File":
		// Add to file crawl queue
		fileArgs := &Args{
			Hash:       i.Hash,
			Name:       i.Name,
			Size:       list.Size,
			ParentHash: i.ParentHash,
		}

		err = i.FileQueue.Publish(fileArgs)
	case "Directory":
		// Queue indexing of linked items
		err = i.queueList(ctx, list)
		if err != nil {
			return err
		}

		// Index name and size for directory and directory items
		m := metadata{
			"links":      list.Links,
			"size":       list.Size,
			"references": references,
			"first-seen": nowISO(),
		}

		err = i.Indexer.IndexItem(ctx, "directory", i.Hash, m)
	default:
		log.Printf("Type '%s' skipped for %s", list.Type, i)
	}

	return
}

// processList processes and indexes a single file
func (i *Indexable) processFile(ctx context.Context, references []indexer.Reference) error {
	m := make(metadata)

	err := i.getMetadata(&m)
	if err != nil {
		return err
	}

	// Add previously found references now
	m["size"] = i.Size
	m["references"] = references
	m["first-seen"] = nowISO()

	return i.Indexer.IndexItem(ctx, "file", i.Hash, m)
}

// preCrawl checks for and returns existing item and conditionally updates it
func (i *Indexable) preCrawl(ctx context.Context) (*existingItem, error) {
	e, err := i.getExistingItem(ctx)
	if err != nil {
		return nil, err
	}

	return e, e.update(ctx)
}

// CrawlHash crawls a particular hash (file or directory)
func (i *Indexable) CrawlHash(ctx context.Context) error {
	existing, err := i.preCrawl(ctx)

	if err != nil || !existing.shouldCrawl() {
		log.Printf("Skipping hash %s", i)
		return err
	}

	log.Printf("Crawling hash %s", i)

	list, err := i.getFileList(ctx)
	if err != nil {
		return err
	}

	err = i.processList(ctx, list, existing.references)
	if err != nil {
		return err
	}

	log.Printf("Finished hash %s", i)

	return nil
}

// CrawlFile crawls a single object, known to be a file
func (i *Indexable) CrawlFile(ctx context.Context) error {
	existing, err := i.preCrawl(ctx)

	if err != nil || !existing.shouldCrawl() {
		log.Printf("Skipping file %s", i)
		return err
	}

	log.Printf("Crawling file %s", i)

	i.processFile(ctx, existing.references)
	if err != nil {
		return err
	}

	log.Printf("Finished file %s", i)

	return nil
}
<commit_msg>Add last-seen on first indexing, closes #83.<commit_after>package crawler

import (
	"net"
	"net/url"
	// "path"
	"context"
	"fmt"
	"github.com/ipfs-search/ipfs-search/indexer"
	"github.com/ipfs/go-ipfs-api"
	"log"
	"strings"
	"syscall"
	"time"
)

// Indexable consists of args with a Crawler
type Indexable struct {
	*Crawler
	*Args
}

// String returns '<hash>' (<name>)
func (i *Indexable) String() string {
	if i.Name != "" {
		return fmt.Sprintf("'%s' (%s)", i.Hash, i.Name)
	}
	return fmt.Sprintf("'%s' (Unnamed)", i.Hash)
}

// handleShellError handles IPFS shell errors; returns try again bool and original error
func (i *Indexable) handleShellError(ctx context.Context, err error) (bool, error) {
	if _, ok := err.(*shell.Error); ok && (strings.Contains(err.Error(), "proto") ||
		strings.Contains(err.Error(), "unrecognized type") ||
		strings.Contains(err.Error(), "not a valid merkledag node")) {

		// Attempt to index invalid to prevent re-indexing
		i.indexInvalid(ctx, err)

		// Don't try again, return error
		return false, err
	}

	// Different error, attempt handling as URL error
	return i.handleURLError(err)
}

// handleURLError handles HTTP errors graceously, returns try again bool and original error
func (i *Indexable) handleURLError(err error) (bool, error) {
	if uerr, ok := err.(*url.Error); ok {
		if uerr.Timeout() {
			// Fail on timeouts
			return false, err
		}

		if uerr.Temporary() {
			// Retry on other temp errors
			log.Printf("Temporary URL error: %v", uerr)
			return true, nil
		}

		// Somehow, the errors below are not temp errors !?
		switch t := uerr.Err.(type) {
		case *net.OpError:
			if t.Op == "dial" {
				log.Printf("Unknown host %v", t)
				return true, nil

			} else if t.Op == "read" {
				log.Printf("Connection refused %v", t)
				return true, nil
			}

		case syscall.Errno:
			if t == syscall.ECONNREFUSED {
				log.Printf("Connection refused %v", t)
				return true, nil
			}
		}
	}

	return false, err
}

// hashURL returns the IPFS URL for a particular hash
func (i *Indexable) hashURL() string {
	return fmt.Sprintf("/ipfs/%s", i.Hash)
}

// getFileList return list of files and/or type of item (directory/file)
func (i *Indexable) getFileList(ctx context.Context) (list *shell.UnixLsObject, err error) {
	url := i.hashURL()

	tryAgain := true
	for tryAgain {
		list, err = i.Shell.FileList(url)

		tryAgain, err = i.handleShellError(ctx, err)

		if tryAgain {
			log.Printf("Retrying in %s", i.Config.RetryWait)
			time.Sleep(i.Config.RetryWait)
		}
	}

	return
}

// indexInvalid indexes invalid files to prevent indexing again
func (i *Indexable) indexInvalid(ctx context.Context, err error) {
	// Attempt to index panic to prevent re-indexing
	m := metadata{
		"error": err.Error(),
	}

	i.Indexer.IndexItem(ctx, "invalid", i.Hash, m)
}

// queueList queues any items in a given list/directory
func (i *Indexable) queueList(ctx context.Context, list *shell.UnixLsObject) (err error) {
	for _, link := range list.Links {
		dirArgs := &Args{
			Hash:       link.Hash,
			Name:       link.Name,
			Size:       link.Size,
			ParentHash: i.Hash,
		}

		switch link.Type {
		case "File":
			// Add file to crawl queue
			err = i.FileQueue.Publish(dirArgs)
		case "Directory":
			// Add directory to crawl queue
			err = i.HashQueue.Publish(dirArgs)
		default:
			log.Printf("Type '%s' skipped for %s", link.Type, i)
			i.indexInvalid(ctx, fmt.Errorf("Unknown type: %s", link.Type))
		}
	}

	return
}

// processList processes and indexes a file listing
func (i *Indexable) processList(ctx context.Context, list *shell.UnixLsObject, references []indexer.Reference) (err error) {
	now := nowISO()

	switch list.Type {
	case "File":
		// Add to file crawl queue
		fileArgs := &Args{
			Hash:       i.Hash,
			Name:       i.Name,
			Size:       list.Size,
			ParentHash: i.ParentHash,
		}

		err = i.FileQueue.Publish(fileArgs)
	case "Directory":
		// Queue indexing of linked items
		err = i.queueList(ctx, list)
		if err != nil {
			return err
		}

		// Index name and size for directory and directory items
		m := metadata{
			"links":      list.Links,
			"size":       list.Size,
			"references": references,
			"first-seen": now,
			"last-seen":  now,
		}

		err = i.Indexer.IndexItem(ctx, "directory", i.Hash, m)
	default:
		log.Printf("Type '%s' skipped for %s", list.Type, i)
	}

	return
}

// processList processes and indexes a single file
func (i *Indexable) processFile(ctx context.Context, references []indexer.Reference) error {
	now := nowISO()

	m := make(metadata)

	err := i.getMetadata(&m)
	if err != nil {
		return err
	}

	// Add previously found references now
	m["size"] = i.Size
	m["references"] = references
	m["first-seen"] = now
	m["last-seen"] = now

	return i.Indexer.IndexItem(ctx, "file", i.Hash, m)
}

// preCrawl checks for and returns existing item and conditionally updates it
func (i *Indexable) preCrawl(ctx context.Context) (*existingItem, error) {
	e, err := i.getExistingItem(ctx)
	if err != nil {
		return nil, err
	}

	return e, e.update(ctx)
}

// CrawlHash crawls a particular hash (file or directory)
func (i *Indexable) CrawlHash(ctx context.Context) error {
	existing, err := i.preCrawl(ctx)

	if err != nil || !existing.shouldCrawl() {
		log.Printf("Skipping hash %s", i)
		return err
	}

	log.Printf("Crawling hash %s", i)

	list, err := i.getFileList(ctx)
	if err != nil {
		return err
	}

	err = i.processList(ctx, list, existing.references)
	if err != nil {
		return err
	}

	log.Printf("Finished hash %s", i)

	return nil
}

// CrawlFile crawls a single object, known to be a file
func (i *Indexable) CrawlFile(ctx context.Context) error {
	existing, err := i.preCrawl(ctx)

	if err != nil || !existing.shouldCrawl() {
		log.Printf("Skipping file %s", i)
		return err
	}

	log.Printf("Crawling file %s", i)

	i.processFile(ctx, existing.references)
	if err != nil {
		return err
	}

	log.Printf("Finished file %s", i)

	return nil
}
<|endoftext|>


<commit_before>package program

// Version can be requested through the command line with:
//
//     c2go -v
//
// See https://github.com/elliotchance/c2go/wiki/Release-Process
const Version = "v0.24.6 Cadmium 2018-06-20"
<commit_msg>Bump version: v0.24.7 Caesium 2018-06-21<commit_after>package program

// Version can be requested through the command line with:
//
//     c2go -v
//
// See https://github.com/elliotchance/c2go/wiki/Release-Process
const Version = "v0.24.7 Caesium 2018-06-21"
<|endoftext|>


<commit_before>package helpers

import (
	"socialapi/models"
	"socialapi/request"
	"socialapi/workers/helper"

	"github.com/koding/bongo"
)

func FetchAll(channelId int64, query *request.Query, accountId int64) {
	ConvertMessagesToMessageContainers(
		FetchMessagesByIds(
			FetchMessageIdsByChannelId(channelId, query),
		),
	)
}

func FetchMessageIdsByChannelId(channelId int64, q *request.Query) ([]int64, error) {
	query := &bongo.Query{
		Selector: map[string]interface{}{
			"channel_id": channelId,
		},
		Pluck:      "message_id",
		Pagination: *bongo.NewPagination(q.Limit, q.Skip),
		Sort: map[string]string{
			"added_at": "DESC",
		},
	}

	var messageIds []int64
	if err := models.NewChannelMessageList().Some(&messageIds, query); err != nil {
		return nil, err
	}

	if messageIds == nil {
		return make([]int64, 0), nil
	}

	return messageIds, nil
}

func FetchMessagesByIds(messageIds []int64, err error) ([]models.ChannelMessage, error) {
	if err != nil {
		return make([]models.ChannelMessage, 0), err
	}

	if len(messageIds) == 0 {
		return make([]models.ChannelMessage, 0), nil
	}

	channelMessages, err := models.NewChannelMessage().FetchByIds(messageIds)
	if err != nil {
		return nil, err
	}

	return channelMessages, nil
}

func ConvertMessagesToMessageContainers(messages []models.ChannelMessage, accountId int64) ([]*models.ChannelMessageContainer, error) {
	if messages == nil {
		return make([]*models.ChannelMessageContainer, len(messages)), nil
	}

	containers := make([]*models.ChannelMessageContainer, len(messages))
	if len(messages) == 0 {
		return containers, nil
	}

	decorateContainers(containers, messages, accountId)

	return containers, nil
}

func decorateContainers(containers []*models.ChannelMessageContainer, messages []models.ChannelMessage, accountId int64) {
	log := helper.MustGetLogger()
	var err error
	for i, message := range messages {
		d := models.NewChannelMessage()
		*d = message

		containers[i], err = d.BuildEmptyMessageContainer()
		if err != nil {
			log.Error("Could not create message container for message %d: %s", containers[i].Message.Id, err)
			continue
		}

		it := models.NewInteraction()
		it.MessageId = containers[i].Message.Id

		query := request.NewQuery()
		query.Type = "like"
		query.Limit = 3

		query.AccountId = accountId

		interactionContainer, err := it.FetchInteractionContainer(query)
		if err != nil {
			log.Error("Could not fetch interactions for message %d: %s", containers[i].Message.Id, err)
			continue
		}

		containers[i].Interactions["like"] = interactionContainer
	}
}
<commit_msg>Social: Remove FetchAll message fetcher<commit_after>package helpers

import (
	"socialapi/models"
	"socialapi/request"
	"socialapi/workers/helper"

	"github.com/koding/bongo"
)

func FetchMessageIdsByChannelId(channelId int64, q *request.Query) ([]int64, error) {
	query := &bongo.Query{
		Selector: map[string]interface{}{
			"channel_id": channelId,
		},
		Pluck:      "message_id",
		Pagination: *bongo.NewPagination(q.Limit, q.Skip),
		Sort: map[string]string{
			"added_at": "DESC",
		},
	}

	var messageIds []int64
	if err := models.NewChannelMessageList().Some(&messageIds, query); err != nil {
		return nil, err
	}

	if messageIds == nil {
		return make([]int64, 0), nil
	}

	return messageIds, nil
}

func FetchMessagesByIds(messageIds []int64, err error) ([]models.ChannelMessage, error) {
	if err != nil {
		return make([]models.ChannelMessage, 0), err
	}

	if len(messageIds) == 0 {
		return make([]models.ChannelMessage, 0), nil
	}

	channelMessages, err := models.NewChannelMessage().FetchByIds(messageIds)
	if err != nil {
		return nil, err
	}

	return channelMessages, nil
}

func ConvertMessagesToMessageContainers(messages []models.ChannelMessage, accountId int64) ([]*models.ChannelMessageContainer, error) {
	if messages == nil {
		return make([]*models.ChannelMessageContainer, len(messages)), nil
	}

	containers := make([]*models.ChannelMessageContainer, len(messages))
	if len(messages) == 0 {
		return containers, nil
	}

	decorateContainers(containers, messages, accountId)

	return containers, nil
}

func decorateContainers(containers []*models.ChannelMessageContainer, messages []models.ChannelMessage, accountId int64) {
	log := helper.MustGetLogger()
	var err error
	for i, message := range messages {
		d := models.NewChannelMessage()
		*d = message

		containers[i], err = d.BuildEmptyMessageContainer()
		if err != nil {
			log.Error("Could not create message container for message %d: %s", containers[i].Message.Id, err)
			continue
		}

		it := models.NewInteraction()
		it.MessageId = containers[i].Message.Id

		query := request.NewQuery()
		query.Type = "like"
		query.Limit = 3

		query.AccountId = accountId

		interactionContainer, err := it.FetchInteractionContainer(query)
		if err != nil {
			log.Error("Could not fetch interactions for message %d: %s", containers[i].Message.Id, err)
			continue
		}

		containers[i].Interactions["like"] = interactionContainer
	}
}
<|endoftext|>


<commit_before>package pretty

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"strings"
	"text/tabwriter"
	"text/template"

	"github.com/fatih/color"
	ppsclient "github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/server/pkg/pretty"
)

// PrintJobHeader prints a job header.
func PrintJobHeader(w io.Writer) {
	// because STATE is a colorful field it has to be at the end of the line,
	// otherwise the terminal escape characters will trip up the tabwriter
	fmt.Fprint(w, "ID\tOUTPUT COMMIT\tSTARTED\tDURATION\tRESTART\tPROGRESS\tSTATE\t\n")
}

// PrintJobInfo pretty-prints job info.
func PrintJobInfo(w io.Writer, jobInfo *ppsclient.JobInfo) {
	fmt.Fprintf(w, "%s\t", jobInfo.Job.ID)
	if jobInfo.OutputCommit != nil {
		fmt.Fprintf(w, "%s/%s\t", jobInfo.OutputCommit.Repo.Name, jobInfo.OutputCommit.ID)
	} else if jobInfo.Pipeline != nil {
		fmt.Fprintf(w, "%s/-\t", jobInfo.Pipeline.Name)
	} else {
		fmt.Fprintf(w, "-\t")
	}
	fmt.Fprintf(w, "%s\t", pretty.Ago(jobInfo.Started))
	if jobInfo.Finished != nil {
		fmt.Fprintf(w, "%s\t", pretty.Duration(jobInfo.Started, jobInfo.Finished))
	} else {
		fmt.Fprintf(w, "-\t")
	}
	fmt.Fprintf(w, "%d\t", jobInfo.Restart)
	fmt.Fprintf(w, "%d / %d\t", jobInfo.DataProcessed, jobInfo.DataTotal)
	fmt.Fprintf(w, "%s\t\n", jobState(jobInfo.State))
}

// PrintPipelineHeader prints a pipeline header.
func PrintPipelineHeader(w io.Writer) {
	// because STATE is a colorful field it has to be at the end of the line,
	// otherwise the terminal escape characters will trip up the tabwriter
	fmt.Fprint(w, "NAME\tINPUT\tOUTPUT\tCREATED\tSTATE\t\n")
}

// PrintPipelineInfo pretty-prints pipeline info.
func PrintPipelineInfo(w io.Writer, pipelineInfo *ppsclient.PipelineInfo) {
	fmt.Fprintf(w, "%s\t", pipelineInfo.Pipeline.Name)
	fmt.Fprintf(w, "%s\t", shorthandInput(pipelineInfo.Input))
	fmt.Fprintf(w, "%s/%s\t", pipelineInfo.Pipeline.Name, pipelineInfo.OutputBranch)
	fmt.Fprintf(w, "%s\t", pretty.Ago(pipelineInfo.CreatedAt))
	fmt.Fprintf(w, "%s\t\n", pipelineState(pipelineInfo.State))
}

// PrintJobInputHeader pretty prints a job input header.
func PrintJobInputHeader(w io.Writer) {
	fmt.Fprint(w, "NAME\tREPO\tCOMMIT\tGLOB\tLAZY\t\n")
}

// PrintJobInput pretty-prints a job input.
func PrintJobInput(w io.Writer, jobInput *ppsclient.JobInput) {
	fmt.Fprintf(w, "%s\t", jobInput.Name)
	fmt.Fprintf(w, "%s\t", jobInput.Commit.Repo.Name)
	fmt.Fprintf(w, "%s\t", jobInput.Commit.ID)
	fmt.Fprintf(w, "%s\t", jobInput.Glob)
	fmt.Fprintf(w, "%t\t\n", jobInput.Lazy)
}

// PrintWorkerStatusHeader pretty prints a worker status header.
func PrintWorkerStatusHeader(w io.Writer) {
	fmt.Fprint(w, "WORKER\tJOB\tDATUM\tSTARTED\t\n")
}

// PrintWorkerStatus pretty prints a worker status.
func PrintWorkerStatus(w io.Writer, workerStatus *ppsclient.WorkerStatus) {
	fmt.Fprintf(w, "%s\t", workerStatus.WorkerID)
	fmt.Fprintf(w, "%s\t", workerStatus.JobID)
	for _, datum := range workerStatus.Data {
		fmt.Fprintf(w, datum.Path)
	}
	fmt.Fprintf(w, "\t")
	fmt.Fprintf(w, "%s\t\n", pretty.Ago(workerStatus.Started))
}

// PrintPipelineInputHeader prints a pipeline input header.
func PrintPipelineInputHeader(w io.Writer) {
	fmt.Fprint(w, "NAME\tREPO\tBRANCH\tGLOB\tLAZY\t\n")
}

// PrintPipelineInput pretty-prints a pipeline input.
func PrintPipelineInput(w io.Writer, pipelineInput *ppsclient.PipelineInput) {
	fmt.Fprintf(w, "%s\t", pipelineInput.Name)
	fmt.Fprintf(w, "%s\t", pipelineInput.Repo.Name)
	fmt.Fprintf(w, "%s\t", pipelineInput.Branch)
	fmt.Fprintf(w, "%s\t", pipelineInput.Glob)
	fmt.Fprintf(w, "%t\t\n", pipelineInput.Lazy)
}

// PrintJobCountsHeader prints a job counts header.
func PrintJobCountsHeader(w io.Writer) {
	fmt.Fprintf(w, strings.ToUpper(jobState(ppsclient.JobState_JOB_STARTING))+"\t")
	fmt.Fprintf(w, strings.ToUpper(jobState(ppsclient.JobState_JOB_RUNNING))+"\t")
	fmt.Fprintf(w, strings.ToUpper(jobState(ppsclient.JobState_JOB_FAILURE))+"\t")
	fmt.Fprintf(w, strings.ToUpper(jobState(ppsclient.JobState_JOB_SUCCESS))+"\t\n")
}

// PrintDetailedJobInfo pretty-prints detailed job info.
func PrintDetailedJobInfo(jobInfo *ppsclient.JobInfo) error {
	template, err := template.New("JobInfo").Funcs(funcMap).Parse(
		`ID: {{.Job.ID}} {{if .Pipeline}}
Pipeline: {{.Pipeline.Name}} {{end}} {{if .ParentJob}}
Parent: {{.ParentJob.ID}} {{end}}
Started: {{prettyAgo .Started}} {{if .Finished}}
Duration: {{prettyDuration .Started .Finished}} {{end}}
State: {{jobState .State}}
Progress: {{.DataProcessed}} / {{.DataTotal}}
Worker Status:
{{workerStatus .}}Restarts: {{.Restart}}
ParallelismSpec: {{.ParallelismSpec}}
{{ if .ResourceSpec }}ResourceSpec:
	CPU: {{ .ResourceSpec.Cpu }}
	Memory: {{ .ResourceSpec.Memory }} {{end}}
{{ if .Service }}Service:
	{{ if .Service.InternalPort }}InternalPort: {{ .Service.InternalPort }} {{end}}
	{{ if .Service.ExternalPort }}ExternalPort: {{ .Service.ExternalPort }} {{end}} {{end}}
Input:
{{jobInput .}}Transform:
{{prettyTransform .Transform}} {{if .OutputCommit}}
Output Commit: {{.OutputCommit.ID}} {{end}} {{ if .Egress }}
Egress: {{.Egress.URL}} {{end}}
`)
	if err != nil {
		return err
	}
	err = template.Execute(os.Stdout, jobInfo)
	if err != nil {
		return err
	}
	return nil
}

// PrintDetailedPipelineInfo pretty-prints detailed pipeline info.
func PrintDetailedPipelineInfo(pipelineInfo *ppsclient.PipelineInfo) error {
	template, err := template.New("PipelineInfo").Funcs(funcMap).Parse(
		`Name: {{.Pipeline.Name}}
Created: {{prettyAgo .CreatedAt}}
State: {{pipelineState .State}}
Parallelism Spec: {{.ParallelismSpec}}
{{ if .ResourceSpec }}ResourceSpec:
	CPU: {{ .ResourceSpec.Cpu }}
	Memory: {{ .ResourceSpec.Memory }} {{end}}
Input:
{{pipelineInput .}}
Output Branch: {{.OutputBranch}}
Transform:
{{prettyTransform .Transform}}
{{ if .Egress }}Egress: {{.Egress.URL}} {{end}}
{{if .RecentError}} Recent Error: {{.RecentError}} {{end}}
Job Counts:
{{jobCounts .JobCounts}}
`)
	if err != nil {
		return err
	}
	err = template.Execute(os.Stdout, pipelineInfo)
	if err != nil {
		return err
	}
	return nil
}

func jobState(jobState ppsclient.JobState) string {
	switch jobState {
	case ppsclient.JobState_JOB_STARTING:
		return color.New(color.FgYellow).SprintFunc()("starting")
	case ppsclient.JobState_JOB_RUNNING:
		return color.New(color.FgYellow).SprintFunc()("running")
	case ppsclient.JobState_JOB_FAILURE:
		return color.New(color.FgRed).SprintFunc()("failure")
	case ppsclient.JobState_JOB_SUCCESS:
		return color.New(color.FgGreen).SprintFunc()("success")
	case ppsclient.JobState_JOB_STOPPED:
		return color.New(color.FgYellow).SprintFunc()("stopped")
	}
	return "-"
}

func pipelineState(pipelineState ppsclient.PipelineState) string {
	switch pipelineState {
	case ppsclient.PipelineState_PIPELINE_STARTING:
		return color.New(color.FgYellow).SprintFunc()("starting")
	case ppsclient.PipelineState_PIPELINE_RUNNING:
		return color.New(color.FgGreen).SprintFunc()("running")
	case ppsclient.PipelineState_PIPELINE_RESTARTING:
		return color.New(color.FgYellow).SprintFunc()("restarting")
	case ppsclient.PipelineState_PIPELINE_FAILURE:
		return color.New(color.FgRed).SprintFunc()("failure")
	case ppsclient.PipelineState_PIPELINE_STOPPED:
		return color.New(color.FgYellow).SprintFunc()("stopped")
	}
	return "-"
}

func jobInput(jobInfo *ppsclient.JobInfo) string {
	if jobInfo.Input == nil {
		return ""
	}
	input, err := json.MarshalIndent(jobInfo.Input, "", "  ")
	if err != nil {
		fmt.Errorf("error marshalling input: %+v", err)
	}
	return string(input)
}

func workerStatus(jobInfo *ppsclient.JobInfo) string {
	var buffer bytes.Buffer
	writer := tabwriter.NewWriter(&buffer, 20, 1, 3, ' ', 0)
	PrintWorkerStatusHeader(writer)
	for _, workerStatus := range jobInfo.WorkerStatus {
		PrintWorkerStatus(writer, workerStatus)
	}
	// can't error because buffer can't error on Write
	writer.Flush()
	return buffer.String()
}

func pipelineInput(pipelineInfo *ppsclient.PipelineInfo) string {
	if pipelineInfo.Input == nil {
		return ""
	}
	input, err := json.MarshalIndent(pipelineInfo.Input, "", "  ")
	if err != nil {
		fmt.Errorf("error marshalling input: %+v", err)
	}
	return string(input)
}

func jobCounts(counts map[int32]int32) string {
	var buffer bytes.Buffer
	for i := int32(ppsclient.JobState_JOB_STARTING); i <= int32(ppsclient.JobState_JOB_SUCCESS); i++ {
		fmt.Fprintf(&buffer, "%s: %d\t", jobState(ppsclient.JobState(i)), counts[i])
	}
	return buffer.String()
}

func prettyTransform(transform *ppsclient.Transform) (string, error) {
	result, err := json.MarshalIndent(transform, "", "  ")
	if err != nil {
		return "", err
	}
	return pretty.UnescapeHTML(string(result)), nil
}

func shorthandInput(input *ppsclient.Input) string {
	switch {
	case input.Atom != nil:
		return fmt.Sprintf("%s:%s", input.Atom.Commit.Repo.Name, input.Atom.Glob)
	case input.Cross != nil:
		var subInput []string
		for _, input := range input.Cross.Input {
			subInput = append(subInput, shorthandInput(input))
		}
		return "(" + strings.Join(subInput, " ⨯ ") + ")"
	case input.Union != nil:
		var subInput []string
		for _, input := range input.Union.Input {
			subInput = append(subInput, shorthandInput(input))
		}
		return "(" + strings.Join(subInput, " ∪ ") + ")"
	}
	return ""
}

var funcMap = template.FuncMap{
	"pipelineState":   pipelineState,
	"jobState":        jobState,
	"workerStatus":    workerStatus,
	"pipelineInput":   pipelineInput,
	"jobInput":        jobInput,
	"prettyAgo":       pretty.Ago,
	"prettyDuration":  pretty.Duration,
	"jobCounts":       jobCounts,
	"prettyTransform": prettyTransform,
}
<commit_msg>Make InspectJob print better output<commit_after>package pretty

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"strings"
	"text/tabwriter"
	"text/template"

	"github.com/fatih/color"
	ppsclient "github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/server/pkg/pretty"
)

// PrintJobHeader prints a job header.
func PrintJobHeader(w io.Writer) {
	// because STATE is a colorful field it has to be at the end of the line,
	// otherwise the terminal escape characters will trip up the tabwriter
	fmt.Fprint(w, "ID\tOUTPUT COMMIT\tSTARTED\tDURATION\tRESTART\tPROGRESS\tSTATE\t\n")
}

// PrintJobInfo pretty-prints job info.
func PrintJobInfo(w io.Writer, jobInfo *ppsclient.JobInfo) {
	fmt.Fprintf(w, "%s\t", jobInfo.Job.ID)
	if jobInfo.OutputCommit != nil {
		fmt.Fprintf(w, "%s/%s\t", jobInfo.OutputCommit.Repo.Name, jobInfo.OutputCommit.ID)
	} else if jobInfo.Pipeline != nil {
		fmt.Fprintf(w, "%s/-\t", jobInfo.Pipeline.Name)
	} else {
		fmt.Fprintf(w, "-\t")
	}
	fmt.Fprintf(w, "%s\t", pretty.Ago(jobInfo.Started))
	if jobInfo.Finished != nil {
		fmt.Fprintf(w, "%s\t", pretty.Duration(jobInfo.Started, jobInfo.Finished))
	} else {
		fmt.Fprintf(w, "-\t")
	}
	fmt.Fprintf(w, "%d\t", jobInfo.Restart)
	fmt.Fprintf(w, "%d / %d\t", jobInfo.DataProcessed, jobInfo.DataTotal)
	fmt.Fprintf(w, "%s\t\n", jobState(jobInfo.State))
}

// PrintPipelineHeader prints a pipeline header.
func PrintPipelineHeader(w io.Writer) {
	// because STATE is a colorful field it has to be at the end of the line,
	// otherwise the terminal escape characters will trip up the tabwriter
	fmt.Fprint(w, "NAME\tINPUT\tOUTPUT\tCREATED\tSTATE\t\n")
}

// PrintPipelineInfo pretty-prints pipeline info.
func PrintPipelineInfo(w io.Writer, pipelineInfo *ppsclient.PipelineInfo) {
	fmt.Fprintf(w, "%s\t", pipelineInfo.Pipeline.Name)
	fmt.Fprintf(w, "%s\t", shorthandInput(pipelineInfo.Input))
	fmt.Fprintf(w, "%s/%s\t", pipelineInfo.Pipeline.Name, pipelineInfo.OutputBranch)
	fmt.Fprintf(w, "%s\t", pretty.Ago(pipelineInfo.CreatedAt))
	fmt.Fprintf(w, "%s\t\n", pipelineState(pipelineInfo.State))
}

// PrintJobInputHeader pretty prints a job input header.
func PrintJobInputHeader(w io.Writer) {
	fmt.Fprint(w, "NAME\tREPO\tCOMMIT\tGLOB\tLAZY\t\n")
}

// PrintJobInput pretty-prints a job input.
func PrintJobInput(w io.Writer, jobInput *ppsclient.JobInput) {
	fmt.Fprintf(w, "%s\t", jobInput.Name)
	fmt.Fprintf(w, "%s\t", jobInput.Commit.Repo.Name)
	fmt.Fprintf(w, "%s\t", jobInput.Commit.ID)
	fmt.Fprintf(w, "%s\t", jobInput.Glob)
	fmt.Fprintf(w, "%t\t\n", jobInput.Lazy)
}

// PrintWorkerStatusHeader pretty prints a worker status header.
func PrintWorkerStatusHeader(w io.Writer) {
	fmt.Fprint(w, "WORKER\tJOB\tDATUM\tSTARTED\t\n")
}

// PrintWorkerStatus pretty prints a worker status.
func PrintWorkerStatus(w io.Writer, workerStatus *ppsclient.WorkerStatus) {
	fmt.Fprintf(w, "%s\t", workerStatus.WorkerID)
	fmt.Fprintf(w, "%s\t", workerStatus.JobID)
	for _, datum := range workerStatus.Data {
		fmt.Fprintf(w, datum.Path)
	}
	fmt.Fprintf(w, "\t")
	fmt.Fprintf(w, "%s\t\n", pretty.Ago(workerStatus.Started))
}

// PrintPipelineInputHeader prints a pipeline input header.
func PrintPipelineInputHeader(w io.Writer) {
	fmt.Fprint(w, "NAME\tREPO\tBRANCH\tGLOB\tLAZY\t\n")
}

// PrintPipelineInput pretty-prints a pipeline input.
func PrintPipelineInput(w io.Writer, pipelineInput *ppsclient.PipelineInput) {
	fmt.Fprintf(w, "%s\t", pipelineInput.Name)
	fmt.Fprintf(w, "%s\t", pipelineInput.Repo.Name)
	fmt.Fprintf(w, "%s\t", pipelineInput.Branch)
	fmt.Fprintf(w, "%s\t", pipelineInput.Glob)
	fmt.Fprintf(w, "%t\t\n", pipelineInput.Lazy)
}

// PrintJobCountsHeader prints a job counts header.
func PrintJobCountsHeader(w io.Writer) {
	fmt.Fprintf(w, strings.ToUpper(jobState(ppsclient.JobState_JOB_STARTING))+"\t")
	fmt.Fprintf(w, strings.ToUpper(jobState(ppsclient.JobState_JOB_RUNNING))+"\t")
	fmt.Fprintf(w, strings.ToUpper(jobState(ppsclient.JobState_JOB_FAILURE))+"\t")
	fmt.Fprintf(w, strings.ToUpper(jobState(ppsclient.JobState_JOB_SUCCESS))+"\t\n")
}

// PrintDetailedJobInfo pretty-prints detailed job info.
func PrintDetailedJobInfo(jobInfo *ppsclient.JobInfo) error {
	template, err := template.New("JobInfo").Funcs(funcMap).Parse(
		`ID: {{.Job.ID}} {{if .Pipeline}}
Pipeline: {{.Pipeline.Name}} {{end}} {{if .ParentJob}}
Parent: {{.ParentJob.ID}} {{end}}
Started: {{prettyAgo .Started}} {{if .Finished}}
Duration: {{prettyDuration .Started .Finished}} {{end}}
State: {{jobState .State}}
Progress: {{.DataProcessed}} / {{.DataTotal}}
Worker Status:
{{workerStatus .}}Restarts: {{.Restart}}
ParallelismSpec: {{.ParallelismSpec}}
{{ if .ResourceSpec }}ResourceSpec:
	CPU: {{ .ResourceSpec.Cpu }}
	Memory: {{ .ResourceSpec.Memory }} {{end}}
{{ if .Service }}Service:
	{{ if .Service.InternalPort }}InternalPort: {{ .Service.InternalPort }} {{end}}
	{{ if .Service.ExternalPort }}ExternalPort: {{ .Service.ExternalPort }} {{end}} {{end}}Input:
{{jobInput .}}
Transform:
{{prettyTransform .Transform}} {{if .OutputCommit}}
Output Commit: {{.OutputCommit.ID}} {{end}} {{ if .Egress }}
Egress: {{.Egress.URL}} {{end}}
`)
	if err != nil {
		return err
	}
	err = template.Execute(os.Stdout, jobInfo)
	if err != nil {
		return err
	}
	return nil
}

// PrintDetailedPipelineInfo pretty-prints detailed pipeline info.
func PrintDetailedPipelineInfo(pipelineInfo *ppsclient.PipelineInfo) error {
	template, err := template.New("PipelineInfo").Funcs(funcMap).Parse(
		`Name: {{.Pipeline.Name}}
Created: {{prettyAgo .CreatedAt}}
State: {{pipelineState .State}}
Parallelism Spec: {{.ParallelismSpec}}
{{ if .ResourceSpec }}ResourceSpec:
	CPU: {{ .ResourceSpec.Cpu }}
	Memory: {{ .ResourceSpec.Memory }} {{end}}
Input:
{{pipelineInput .}}
Output Branch: {{.OutputBranch}}
Transform:
{{prettyTransform .Transform}}
{{ if .Egress }}Egress: {{.Egress.URL}} {{end}}
{{if .RecentError}} Recent Error: {{.RecentError}} {{end}}
Job Counts:
{{jobCounts .JobCounts}}
`)
	if err != nil {
		return err
	}
	err = template.Execute(os.Stdout, pipelineInfo)
	if err != nil {
		return err
	}
	return nil
}

func jobState(jobState ppsclient.JobState) string {
	switch jobState {
	case ppsclient.JobState_JOB_STARTING:
		return color.New(color.FgYellow).SprintFunc()("starting")
	case ppsclient.JobState_JOB_RUNNING:
		return color.New(color.FgYellow).SprintFunc()("running")
	case ppsclient.JobState_JOB_FAILURE:
		return color.New(color.FgRed).SprintFunc()("failure")
	case ppsclient.JobState_JOB_SUCCESS:
		return color.New(color.FgGreen).SprintFunc()("success")
	case ppsclient.JobState_JOB_STOPPED:
		return color.New(color.FgYellow).SprintFunc()("stopped")
	}
	return "-"
}

func pipelineState(pipelineState ppsclient.PipelineState) string {
	switch pipelineState {
	case ppsclient.PipelineState_PIPELINE_STARTING:
		return color.New(color.FgYellow).SprintFunc()("starting")
	case ppsclient.PipelineState_PIPELINE_RUNNING:
		return color.New(color.FgGreen).SprintFunc()("running")
	case ppsclient.PipelineState_PIPELINE_RESTARTING:
		return color.New(color.FgYellow).SprintFunc()("restarting")
	case ppsclient.PipelineState_PIPELINE_FAILURE:
		return color.New(color.FgRed).SprintFunc()("failure")
	case ppsclient.PipelineState_PIPELINE_STOPPED:
		return color.New(color.FgYellow).SprintFunc()("stopped")
	}
	return "-"
}

func jobInput(jobInfo *ppsclient.JobInfo) string {
	if jobInfo.Input == nil {
		return ""
	}
	input, err := json.MarshalIndent(jobInfo.Input, "", "  ")
	if err != nil {
		fmt.Errorf("error marshalling input: %+v", err)
	}
	return string(input) + "\n"
}

func workerStatus(jobInfo *ppsclient.JobInfo) string {
	var buffer bytes.Buffer
	writer := tabwriter.NewWriter(&buffer, 20, 1, 3, ' ', 0)
	PrintWorkerStatusHeader(writer)
	for _, workerStatus := range jobInfo.WorkerStatus {
		PrintWorkerStatus(writer, workerStatus)
	}
	// can't error because buffer can't error on Write
	writer.Flush()
	return buffer.String()
}

func pipelineInput(pipelineInfo *ppsclient.PipelineInfo) string {
	if pipelineInfo.Input == nil {
		return ""
	}
	input, err := json.MarshalIndent(pipelineInfo.Input, "", "  ")
	if err != nil {
		fmt.Errorf("error marshalling input: %+v", err)
	}
	return string(input) + "\n"
}

func jobCounts(counts map[int32]int32) string {
	var buffer bytes.Buffer
	for i := int32(ppsclient.JobState_JOB_STARTING); i <= int32(ppsclient.JobState_JOB_SUCCESS); i++ {
		fmt.Fprintf(&buffer, "%s: %d\t", jobState(ppsclient.JobState(i)), counts[i])
	}
	return buffer.String()
}

func prettyTransform(transform *ppsclient.Transform) (string, error) {
	result, err := json.MarshalIndent(transform, "", "  ")
	if err != nil {
		return "", err
	}
	return pretty.UnescapeHTML(string(result)), nil
}

func shorthandInput(input *ppsclient.Input) string {
	switch {
	case input.Atom != nil:
		return fmt.Sprintf("%s:%s", input.Atom.Commit.Repo.Name, input.Atom.Glob)
	case input.Cross != nil:
		var subInput []string
		for _, input := range input.Cross.Input {
			subInput = append(subInput, shorthandInput(input))
		}
		return "(" + strings.Join(subInput, " ⨯ ") + ")"
	case input.Union != nil:
		var subInput []string
		for _, input := range input.Union.Input {
			subInput = append(subInput, shorthandInput(input))
		}
		return "(" + strings.Join(subInput, " ∪ ") + ")"
	}
	return ""
}

var funcMap = template.FuncMap{
	"pipelineState":   pipelineState,
	"jobState":        jobState,
	"workerStatus":    workerStatus,
	"pipelineInput":   pipelineInput,
	"jobInput":        jobInput,
	"prettyAgo":       pretty.Ago,
	"prettyDuration":  pretty.Duration,
	"jobCounts":       jobCounts,
	"prettyTransform": prettyTransform,
}
<|endoftext|>


<commit_before>package pretty

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"strings"
	"text/tabwriter"
	"text/template"

	"github.com/fatih/color"
	ppsclient "github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/server/pkg/pretty"
)

// PrintJobHeader prints a job header.
func PrintJobHeader(w io.Writer) {
	fmt.Fprint(w, "ID\tOUTPUT COMMIT\tSTARTED\tDURATION\tSTATE\t\n")
}

// PrintJobInfo pretty-prints job info.
func PrintJobInfo(w io.Writer, jobInfo *ppsclient.JobInfo) {
	fmt.Fprintf(w, "%s\t", jobInfo.Job.ID)
	if jobInfo.OutputCommit != nil {
		fmt.Fprintf(w, "%s/%s\t", jobInfo.OutputCommit.Repo.Name, jobInfo.OutputCommit.ID)
	} else {
		fmt.Fprintf(w, "-\t")
	}
	fmt.Fprintf(w, "%s\t", pretty.Ago(jobInfo.Started))
	if jobInfo.Finished != nil {
		fmt.Fprintf(w, "%s\t", pretty.Duration(jobInfo.Started, jobInfo.Finished))
	} else {
		fmt.Fprintf(w, "-\t")
	}
	fmt.Fprintf(w, "%s\t\n", jobState(jobInfo.State))
}

// PrintPipelineHeader prints a pipeline header.
func PrintPipelineHeader(w io.Writer) {
	fmt.Fprint(w, "NAME\tINPUT\tOUTPUT\tSTATE\t\n")
}

// PrintPipelineInfo pretty-prints pipeline info.
func PrintPipelineInfo(w io.Writer, pipelineInfo *ppsclient.PipelineInfo) {
	fmt.Fprintf(w, "%s\t", pipelineInfo.Pipeline.Name)
	if len(pipelineInfo.Inputs) == 0 {
		fmt.Fprintf(w, "\t")
	} else {
		var inputNames []string
		for _, input := range pipelineInfo.Inputs {
			inputNames = append(inputNames, input.Name)
		}
		fmt.Fprintf(w, "%s\t", strings.Join(inputNames, ", "))
	}
	fmt.Fprintf(w, "%s/%s\t", pipelineInfo.Pipeline.Name, pipelineInfo.OutputBranch)
	fmt.Fprintf(w, "%s\t\n", pipelineState(pipelineInfo.State))
}

// PrintJobInputHeader pretty prints a job input header.
func PrintJobInputHeader(w io.Writer) {
	fmt.Fprint(w, "NAME\tREPO\tCOMMIT\tGLOB\tLAZY\t\n")
}

// PrintJobInput pretty-prints a job input.
func PrintJobInput(w io.Writer, jobInput *ppsclient.JobInput) {
	fmt.Fprintf(w, "%s\t", jobInput.Name)
	fmt.Fprintf(w, "%s\t", jobInput.Commit.Repo.Name)
	fmt.Fprintf(w, "%s\t", jobInput.Commit.ID)
	fmt.Fprintf(w, "%s\t", jobInput.Glob)
	fmt.Fprintf(w, "%t\t\n", jobInput.Lazy)
}

// PrintPipelineInputHeader prints a pipeline input header.
func PrintPipelineInputHeader(w io.Writer) {
	fmt.Fprint(w, "NAME\tREPO\tBRANCH\tGLOB\tLAZY\t\n")
}

// PrintPipelineInput pretty-prints a pipeline input.
func PrintPipelineInput(w io.Writer, pipelineInput *ppsclient.PipelineInput) {
	fmt.Fprintf(w, "%s\t", pipelineInput.Name)
	fmt.Fprintf(w, "%s\t", pipelineInput.Repo.Name)
	fmt.Fprintf(w, "%s\t", pipelineInput.Branch)
	fmt.Fprintf(w, "%s\t", pipelineInput.Glob)
	fmt.Fprintf(w, "%t\t\n", pipelineInput.Lazy)
}

// PrintJobCountsHeader prints a job counts header.
func PrintJobCountsHeader(w io.Writer) {
	fmt.Fprintf(w, strings.ToUpper(jobState(ppsclient.JobState_JOB_STARTING))+"\t")
	fmt.Fprintf(w, strings.ToUpper(jobState(ppsclient.JobState_JOB_RUNNING))+"\t")
	fmt.Fprintf(w, strings.ToUpper(jobState(ppsclient.JobState_JOB_FAILURE))+"\t")
	fmt.Fprintf(w, strings.ToUpper(jobState(ppsclient.JobState_JOB_SUCCESS))+"\t\n")
}

// PrintDetailedJobInfo pretty-prints detailed job info.
func PrintDetailedJobInfo(jobInfo *ppsclient.JobInfo) error {
	template, err := template.New("JobInfo").Funcs(funcMap).Parse(
		`ID: {{.Job.ID}} {{if .Pipeline}}
Pipeline: {{.Pipeline.Name}} {{end}} {{if .ParentJob}}
Parent: {{.ParentJob.ID}} {{end}}
Started: {{prettyAgo .Started}} {{if .Finished}}
Duration: {{prettyDuration .Started .Finished}} {{end}}
State: {{jobState .State}}
ParallelismSpec: {{.ParallelismSpec}}
{{ if .Service }}Service:
	{{ if .Service.InternalPort }}InternalPort: {{ .Service.InternalPort }} {{end}}
	{{ if .Service.ExternalPort }}ExternalPort: {{ .Service.ExternalPort }} {{end}} {{end}}
Inputs:
{{jobInputs .}}Transform:
{{prettyTransform .Transform}} {{if .OutputCommit}}
Output Commit: {{.OutputCommit.ID}} {{end}} {{ if .Egress }}
Egress: {{.Egress.URL}} {{end}}
{{ if .Error }}
Error:
{{.Error}}
{{end}}
`)
	if err != nil {
		return err
	}
	err = template.Execute(os.Stdout, jobInfo)
	if err != nil {
		return err
	}
	return nil
}

// PrintDetailedPipelineInfo pretty-prints detailed pipeline info.
func PrintDetailedPipelineInfo(pipelineInfo *ppsclient.PipelineInfo) error {
	template, err := template.New("PipelineInfo").Funcs(funcMap).Parse(
		`Name: {{.Pipeline.Name}}
Created: {{prettyAgo .CreatedAt}}
State: {{pipelineState .State}}
Parallelism Spec: {{.ParallelismSpec}}
Inputs:
{{pipelineInputs .}}
Output Branch: {{.OutputBranch}}
Transform:
{{prettyTransform .Transform}}
{{ if .Egress }}Egress: {{.Egress.URL}} {{end}}
{{if .RecentError}} Recent Error: {{.RecentError}} {{end}}
Job Counts:
{{jobCounts .JobCounts}}
`)
	if err != nil {
		return err
	}
	err = template.Execute(os.Stdout, pipelineInfo)
	if err != nil {
		return err
	}
	return nil
}

func jobState(jobState ppsclient.JobState) string {
	switch jobState {
	case ppsclient.JobState_JOB_STARTING:
		return color.New(color.FgYellow).SprintFunc()("pulling")
	case ppsclient.JobState_JOB_RUNNING:
		return color.New(color.FgYellow).SprintFunc()("running")
	case ppsclient.JobState_JOB_FAILURE:
		return color.New(color.FgRed).SprintFunc()("failure")
	case ppsclient.JobState_JOB_SUCCESS:
		return color.New(color.FgGreen).SprintFunc()("success")
	}
	return "-"
}

func pipelineState(pipelineState ppsclient.PipelineState) string {
	switch pipelineState {
	case ppsclient.PipelineState_PIPELINE_STARTING:
		return color.New(color.FgYellow).SprintFunc()("starting")
	case ppsclient.PipelineState_PIPELINE_RUNNING:
		return color.New(color.FgGreen).SprintFunc()("running")
	case ppsclient.PipelineState_PIPELINE_RESTARTING:
		return color.New(color.FgYellow).SprintFunc()("restarting")
	case ppsclient.PipelineState_PIPELINE_FAILURE:
		return color.New(color.FgRed).SprintFunc()("failure")
	case ppsclient.PipelineState_PIPELINE_STOPPED:
		return color.New(color.FgYellow).SprintFunc()("stopped")
	}
	return "-"
}

func jobInputs(jobInfo *ppsclient.JobInfo) string {
	var buffer bytes.Buffer
	writer := tabwriter.NewWriter(&buffer, 20, 1, 3, ' ', 0)
	PrintJobInputHeader(writer)
	for _, input := range jobInfo.Inputs {
		PrintJobInput(writer, input)
	}
	// can't error because buffer can't error on Write
	writer.Flush()
	return buffer.String()
}

func pipelineInputs(pipelineInfo *ppsclient.PipelineInfo) string {
	var buffer bytes.Buffer
	writer := tabwriter.NewWriter(&buffer, 20, 1, 3, ' ', 0)
	PrintPipelineInputHeader(writer)
	for _, input := range pipelineInfo.Inputs {
		PrintPipelineInput(writer, input)
	}
	// can't error because buffer can't error on Write
	writer.Flush()
	return buffer.String()
}

func jobCounts(counts map[int32]int32) string {
	var buffer bytes.Buffer
	for i := int32(ppsclient.JobState_JOB_STARTING); i <= int32(ppsclient.JobState_JOB_SUCCESS); i++ {
		fmt.Fprintf(&buffer, "%s: %d\t", jobState(ppsclient.JobState(i)), counts[i])
	}
	return buffer.String()
}

func prettyTransform(transform *ppsclient.Transform) (string, error) {
	result, err := json.MarshalIndent(transform, "", "  ")
	if err != nil {
		return "", err
	}
	return pretty.UnescapeHTML(string(result)), nil
}

var funcMap = template.FuncMap{
	"pipelineState":   pipelineState,
	"jobState":        jobState,
	"pipelineInputs":  pipelineInputs,
	"jobInputs":       jobInputs,
	"prettyAgo":       pretty.Ago,
	"prettyDuration":  pretty.Duration,
	"jobCounts":       jobCounts,
	"prettyTransform": prettyTransform,
}
<commit_msg>Fix pretty print<commit_after>package pretty

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"strings"
	"text/tabwriter"
	"text/template"

	"github.com/fatih/color"
	ppsclient "github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/server/pkg/pretty"
)

// PrintJobHeader prints a job header.
func PrintJobHeader(w io.Writer) {
	fmt.Fprint(w, "ID\tOUTPUT COMMIT\tSTARTED\tDURATION\tSTATE\t\n")
}

// PrintJobInfo pretty-prints job info.
func PrintJobInfo(w io.Writer, jobInfo *ppsclient.JobInfo) {
	fmt.Fprintf(w, "%s\t", jobInfo.Job.ID)
	if jobInfo.OutputCommit != nil {
		fmt.Fprintf(w, "%s/%s\t", jobInfo.OutputCommit.Repo.Name, jobInfo.OutputCommit.ID)
	} else {
		fmt.Fprintf(w, "-\t")
	}
	fmt.Fprintf(w, "%s\t", pretty.Ago(jobInfo.Started))
	if jobInfo.Finished != nil {
		fmt.Fprintf(w, "%s\t", pretty.Duration(jobInfo.Started, jobInfo.Finished))
	} else {
		fmt.Fprintf(w, "-\t")
	}
	fmt.Fprintf(w, "%s\t\n", jobState(jobInfo.State))
}

// PrintPipelineHeader prints a pipeline header.
func PrintPipelineHeader(w io.Writer) {
	fmt.Fprint(w, "NAME\tINPUT\tOUTPUT\tSTATE\t\n")
}

// PrintPipelineInfo pretty-prints pipeline info.
func PrintPipelineInfo(w io.Writer, pipelineInfo *ppsclient.PipelineInfo) {
	fmt.Fprintf(w, "%s\t", pipelineInfo.Pipeline.Name)
	if len(pipelineInfo.Inputs) == 0 {
		fmt.Fprintf(w, "\t")
	} else {
		var inputNames []string
		for _, input := range pipelineInfo.Inputs {
			inputNames = append(inputNames, input.Name)
		}
		fmt.Fprintf(w, "%s\t", strings.Join(inputNames, ", "))
	}
	fmt.Fprintf(w, "%s/%s\t", pipelineInfo.Pipeline.Name, pipelineInfo.OutputBranch)
	fmt.Fprintf(w, "%s\t\n", pipelineState(pipelineInfo.State))
}

// PrintJobInputHeader pretty prints a job input header.
func PrintJobInputHeader(w io.Writer) {
	fmt.Fprint(w, "NAME\tREPO\tCOMMIT\tGLOB\tLAZY\t\n")
}

// PrintJobInput pretty-prints a job input.
func PrintJobInput(w io.Writer, jobInput *ppsclient.JobInput) {
	fmt.Fprintf(w, "%s\t", jobInput.Name)
	fmt.Fprintf(w, "%s\t", jobInput.Commit.Repo.Name)
	fmt.Fprintf(w, "%s\t", jobInput.Commit.ID)
	fmt.Fprintf(w, "%s\t", jobInput.Glob)
	fmt.Fprintf(w, "%t\t\n", jobInput.Lazy)
}

// PrintPipelineInputHeader prints a pipeline input header.
func PrintPipelineInputHeader(w io.Writer) {
	fmt.Fprint(w, "NAME\tREPO\tBRANCH\tGLOB\tLAZY\t\n")
}

// PrintPipelineInput pretty-prints a pipeline input.
func PrintPipelineInput(w io.Writer, pipelineInput *ppsclient.PipelineInput) {
	fmt.Fprintf(w, "%s\t", pipelineInput.Name)
	fmt.Fprintf(w, "%s\t", pipelineInput.Repo.Name)
	fmt.Fprintf(w, "%s\t", pipelineInput.Branch)
	fmt.Fprintf(w, "%s\t", pipelineInput.Glob)
	fmt.Fprintf(w, "%t\t\n", pipelineInput.Lazy)
}

// PrintJobCountsHeader prints a job counts header.
func PrintJobCountsHeader(w io.Writer) {
	fmt.Fprintf(w, strings.ToUpper(jobState(ppsclient.JobState_JOB_STARTING))+"\t")
	fmt.Fprintf(w, strings.ToUpper(jobState(ppsclient.JobState_JOB_RUNNING))+"\t")
	fmt.Fprintf(w, strings.ToUpper(jobState(ppsclient.JobState_JOB_FAILURE))+"\t")
	fmt.Fprintf(w, strings.ToUpper(jobState(ppsclient.JobState_JOB_SUCCESS))+"\t\n")
}

// PrintDetailedJobInfo pretty-prints detailed job info.
func PrintDetailedJobInfo(jobInfo *ppsclient.JobInfo) error {
	template, err := template.New("JobInfo").Funcs(funcMap).Parse(
		`ID: {{.Job.ID}} {{if .Pipeline}}
Pipeline: {{.Pipeline.Name}} {{end}} {{if .ParentJob}}
Parent: {{.ParentJob.ID}} {{end}}
Started: {{prettyAgo .Started}} {{if .Finished}}
Duration: {{prettyDuration .Started .Finished}} {{end}}
State: {{jobState .State}}
ParallelismSpec: {{.ParallelismSpec}}
{{ if .Service }}Service:
	{{ if .Service.InternalPort }}InternalPort: {{ .Service.InternalPort }} {{end}}
	{{ if .Service.ExternalPort }}ExternalPort: {{ .Service.ExternalPort }} {{end}} {{end}}
Inputs:
{{jobInputs .}}Transform:
{{prettyTransform .Transform}} {{if .OutputCommit}}
Output Commit: {{.OutputCommit.ID}} {{end}} {{ if .Egress }}
Egress: {{.Egress.URL}} {{end}}
`)
	if err != nil {
		return err
	}
	err = template.Execute(os.Stdout, jobInfo)
	if err != nil {
		return err
	}
	return nil
}

// PrintDetailedPipelineInfo pretty-prints detailed pipeline info.
func PrintDetailedPipelineInfo(pipelineInfo *ppsclient.PipelineInfo) error {
	template, err := template.New("PipelineInfo").Funcs(funcMap).Parse(
		`Name: {{.Pipeline.Name}}
Created: {{prettyAgo .CreatedAt}}
State: {{pipelineState .State}}
Parallelism Spec: {{.ParallelismSpec}}
Inputs:
{{pipelineInputs .}}
Output Branch: {{.OutputBranch}}
Transform:
{{prettyTransform .Transform}}
{{ if .Egress }}Egress: {{.Egress.URL}} {{end}}
{{if .RecentError}} Recent Error: {{.RecentError}} {{end}}
Job Counts:
{{jobCounts .JobCounts}}
`)
	if err != nil {
		return err
	}
	err = template.Execute(os.Stdout, pipelineInfo)
	if err != nil {
		return err
	}
	return nil
}

func jobState(jobState ppsclient.JobState) string {
	switch jobState {
	case ppsclient.JobState_JOB_STARTING:
		return color.New(color.FgYellow).SprintFunc()("pulling")
	case ppsclient.JobState_JOB_RUNNING:
		return color.New(color.FgYellow).SprintFunc()("running")
	case ppsclient.JobState_JOB_FAILURE:
		return color.New(color.FgRed).SprintFunc()("failure")
	case ppsclient.JobState_JOB_SUCCESS:
		return color.New(color.FgGreen).SprintFunc()("success")
	}
	return "-"
}

func pipelineState(pipelineState ppsclient.PipelineState) string {
	switch pipelineState {
	case ppsclient.PipelineState_PIPELINE_STARTING:
		return color.New(color.FgYellow).SprintFunc()("starting")
	case ppsclient.PipelineState_PIPELINE_RUNNING:
		return color.New(color.FgGreen).SprintFunc()("running")
	case ppsclient.PipelineState_PIPELINE_RESTARTING:
		return color.New(color.FgYellow).SprintFunc()("restarting")
	case ppsclient.PipelineState_PIPELINE_FAILURE:
		return color.New(color.FgRed).SprintFunc()("failure")
	case ppsclient.PipelineState_PIPELINE_STOPPED:
		return color.New(color.FgYellow).SprintFunc()("stopped")
	}
	return "-"
}

func jobInputs(jobInfo *ppsclient.JobInfo) string {
	var buffer bytes.Buffer
	writer := tabwriter.NewWriter(&buffer, 20, 1, 3, ' ', 0)
	PrintJobInputHeader(writer)
	for _, input := range jobInfo.Inputs {
		PrintJobInput(writer, input)
	}
	// can't error because buffer can't error on Write
	writer.Flush()
	return buffer.String()
}

func pipelineInputs(pipelineInfo *ppsclient.PipelineInfo) string {
	var buffer bytes.Buffer
	writer := tabwriter.NewWriter(&buffer, 20, 1, 3, ' ', 0)
	PrintPipelineInputHeader(writer)
	for _, input := range pipelineInfo.Inputs {
		PrintPipelineInput(writer, input)
	}
	// can't error because buffer can't error on Write
	writer.Flush()
	return buffer.String()
}

func jobCounts(counts map[int32]int32) string {
	var buffer bytes.Buffer
	for i := int32(ppsclient.JobState_JOB_STARTING); i <= int32(ppsclient.JobState_JOB_SUCCESS); i++ {
		fmt.Fprintf(&buffer, "%s: %d\t", jobState(ppsclient.JobState(i)), counts[i])
	}
	return buffer.String()
}

func prettyTransform(transform *ppsclient.Transform) (string, error) {
	result, err := json.MarshalIndent(transform, "", "  ")
	if err != nil {
		return "", err
	}
	return pretty.UnescapeHTML(string(result)), nil
}

var funcMap = template.FuncMap{
	"pipelineState":   pipelineState,
	"jobState":        jobState,
	"pipelineInputs":  pipelineInputs,
	"jobInputs":       jobInputs,
	"prettyAgo":       pretty.Ago,
	"prettyDuration":  pretty.Duration,
	"jobCounts":       jobCounts,
	"prettyTransform": prettyTransform,
}
<|endoftext|>


<commit_before>package httpcanvas

import (
	"fmt"
	"math/rand"
	"net/http"
	"strings"
	"sync"
)

type CanvasHandler func(*Context)

type Canvas struct {
	handler CanvasHandler
	Width   float64
	Height  float64
	id      *int
	started *bool
	lock    *sync.RWMutex
	command chan string
}

func newCanvas(handler CanvasHandler) *Canvas {
	id := rand.Int()/2
	started := false
	return &Canvas{handler, 640, 480, &id, &started, &sync.RWMutex{}, make(chan string, 1000)}
}

func (c *Canvas) writeContainer(w http.ResponseWriter, r *http.Request) {
	// sync
	c.lock.RLock()
	id := *c.id
	c.lock.RUnlock()
	container := `<!DOCTYPE HTML>
<!-- http://www.html5canvastutorials.com/tutorials/html5-canvas-lines/ -->
<html>
  <head>
    <style>
      body {
        margin: 0px;
        padding: 0px;
      }
      .displayBox {
        border: 1px dashed rgb(170, 170, 170)
      }
    </style>
  </head>
  <body>
    <canvas id="myCanvas" class="displayBox"
      width="` + fmt.Sprintf("%d", int(c.Width)) + `"
      height="` + fmt.Sprintf("%d", int(c.Height)) + `"></canvas>
    <script>
      xmlHttp = new XMLHttpRequest();
      currentData = []
      function getNextCommands() {
        if (currentData.length == 0) {
          try {
            xmlHttp.open("GET", "/command?id=` + fmt.Sprintf("%d", id) + `", false);
            xmlHttp.send(null);
            currentData = xmlHttp.responseText.split("~");
          } catch (e) {
            currentData = ["END"]
          }
        }
      }

      function parseBool(b) {
        return b == "true"
      }

      var canvas = document.getElementById('myCanvas');
      var context = canvas.getContext('2d');
      var intervalId = 0

      function executeNextCommands() {
        getNextCommands()
        while (currentData.length > 0) {
          command = currentData.shift().split("|")
          if (command[0] == "END") {
            clearInterval(intervalId)
          } else if (command[0] == "beginPath") {
            context.beginPath();
          } else if (command[0] == "moveTo") {
            context.moveTo(parseFloat(command[1]), parseFloat(command[2]));
          } else if (command[0] == "lineTo") {
            context.lineTo(parseFloat(command[1]), parseFloat(command[2]));
          } else if (command[0] == "stroke") {
            context.stroke();
          } else if (command[0] == "arc") {
            context.arc(parseFloat(command[1]),
              parseFloat(command[2]),
              parseFloat(command[3]),
              parseFloat(command[4]),
              parseFloat(command[5]),
              parseBool(command[6]));
          } else if (command[0] == "fillStyle") {
              context.fillStyle = command[1]
          } else if (command[0] == "fill") {
              context.fill()
          } else if (command[0] == "lineWidth") {
              context.lineWidth = parseFloat(command[1])
          } else if (command[0] == "strokeStyle") {
              context.strokeStyle = command[1]
          } else if (command[0] == "fillRect") {
              context.fillRect(parseFloat(command[1]),
                parseFloat(command[2]),
                parseFloat(command[3]),
                parseFloat(command[4]))
          } else if (command[0] == "strokeRect") {
              context.strokeRect(parseFloat(command[1]),
                parseFloat(command[2]),
                parseFloat(command[3]),
                parseFloat(command[4]))
          } else if (command[0] == "clearRect") {
              context.clearRect(parseFloat(command[1]),
                parseFloat(command[2]),
                parseFloat(command[3]),
                parseFloat(command[4]))
          }
        }
      }

      intervalId = setInterval("executeNextCommands()", 10)
    </script>
  </body>
</html>`
	fmt.Fprintf(w, container)
}

func (c Canvas) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	command, _, args := stringPartition(r.RequestURI, "?")

	if command == "/" && r.Method == "GET" {
		// sync
		c.lock.Lock()
		(*c.id)++
		started := *c.started
		c.lock.Unlock()
		c.writeContainer(w, r)
		if !started {
			c.lock.Lock()
			(*c.started) = true
			go func() {
				c.handler(&Context{c.command, c.Width, c.Height})
				c.command <- "END"
			}()
			c.lock.Unlock()
		}
		return
	}

	// sync
	c.lock.RLock()
	id := *c.id
	c.lock.RUnlock()
	idExpected := fmt.Sprintf("id=%d", id)

	if command == "/command" && r.Method == "GET" {
		if args == idExpected {
			commandGroup := ""
			command := " "
			for len(command) > 0 {
				select {
				case command = <-c.command:
					if len(commandGroup) > 0 {
						commandGroup += "~"
					}
					commandGroup += command
				default:
					// if we have at least one command, then send it off
					if len(commandGroup) > 0 {
						command = ""
					}
				}
			}
			fmt.Fprintf(w, commandGroup)
			return
		} else {
			fmt.Fprintf(w, "END")
			return
		}
	}

	http.NotFound(w, r)
}

func ListenAndServe(addr string, handler CanvasHandler) (err error) {
	return http.ListenAndServe(addr, newCanvas(handler))
}

func stringPartition(s, sep string) (string, string, string) {
	sepPos := strings.Index(s, sep)
	if sepPos == -1 { // no seperator found
		return s, "", ""
	}
	split := strings.SplitN(s, sep, 2)
	return split[0], sep, split[1]
}
<commit_msg>use pointer receiver for main Canvas object<commit_after>package httpcanvas

import (
	"fmt"
	"math/rand"
	"net/http"
	"html/template"
	"strings"
)

type CanvasHandler func(*Context)

type Canvas struct {
	handler CanvasHandler
	Width   float64
	Height  float64
	Unique  string
	started bool
	command chan string
}

func newCanvas(handler CanvasHandler) *Canvas {
	return &Canvas{handler, 640, 480, "", false, make(chan string, 1000)}
}

func (c *Canvas) updateUnique() {
	c.Unique = fmt.Sprintf("%f", rand.Float64())
}

func (c *Canvas) renderHtml(w http.ResponseWriter, r *http.Request) error {
	container := `<!DOCTYPE HTML>
<!-- http://www.html5canvastutorials.com/tutorials/html5-canvas-lines/ -->
<html>
  <head>
    <style>
      body {
        margin: 0px;
        padding: 0px;
      }
      .displayBox {
        border: 1px dashed rgb(170, 170, 170)
      }
    </style>
  </head>
  <body>
    <canvas id="myCanvas" class="displayBox" width="{{.Width}}" height="{{.Height}}"></canvas>
    <script>
      xmlHttp = new XMLHttpRequest();
      currentData = []
      function getNextCommands() {
        if (currentData.length == 0) {
          try {
            xmlHttp.open("GET", "/command?id={{.Unique}}", false);
            xmlHttp.send(null);
            currentData = xmlHttp.responseText.split("~");
          } catch (e) {
            currentData = ["END"]
          }
        }
      }

      function parseBool(b) {
        return b == "true"
      }

      var canvas = document.getElementById('myCanvas');
      var context = canvas.getContext('2d');
      var intervalId = 0

      function executeNextCommands() {
        getNextCommands()
        while (currentData.length > 0) {
          command = currentData.shift().split("|")
          if (command[0] == "END") {
            clearInterval(intervalId)
          } else if (command[0] == "beginPath") {
            context.beginPath();
          } else if (command[0] == "moveTo") {
            context.moveTo(parseFloat(command[1]), parseFloat(command[2]));
          } else if (command[0] == "lineTo") {
            context.lineTo(parseFloat(command[1]), parseFloat(command[2]));
          } else if (command[0] == "stroke") {
            context.stroke();
          } else if (command[0] == "arc") {
            context.arc(parseFloat(command[1]),
              parseFloat(command[2]),
              parseFloat(command[3]),
              parseFloat(command[4]),
              parseFloat(command[5]),
              parseBool(command[6]));
          } else if (command[0] == "fillStyle") {
              context.fillStyle = command[1]
          } else if (command[0] == "fill") {
              context.fill()
          } else if (command[0] == "lineWidth") {
              context.lineWidth = parseFloat(command[1])
          } else if (command[0] == "strokeStyle") {
              context.strokeStyle = command[1]
          } else if (command[0] == "fillRect") {
              context.fillRect(parseFloat(command[1]),
                parseFloat(command[2]),
                parseFloat(command[3]),
                parseFloat(command[4]))
          } else if (command[0] == "strokeRect") {
              context.strokeRect(parseFloat(command[1]),
                parseFloat(command[2]),
                parseFloat(command[3]),
                parseFloat(command[4]))
          } else if (command[0] == "clearRect") {
              context.clearRect(parseFloat(command[1]),
                parseFloat(command[2]),
                parseFloat(command[3]),
                parseFloat(command[4]))
          }
        }
      }

      intervalId = setInterval("executeNextCommands()", 10)
    </script>
  </body>
</html>`
template, err := template.New("basic").Parse(container)
    if err != nil {
        return err
    }
    err = template.Execute(w, c)
    return err
}

func (c *Canvas) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	command, _, args := stringPartition(r.RequestURI, "?")

	if command == "/" && r.Method == "GET" {
		c.updateUnique()
		err := c.renderHtml(w, r)
		if err != nil {
			return
		}
		if !c.started {
			c.started = true
			go func() {
				c.handler(&Context{c.command, c.Width, c.Height})
				c.command <- "END"
			}()
		}
		return
	}

	uniqueExpected := fmt.Sprintf("id=%s", c.Unique)

	if command == "/command" && r.Method == "GET" {
		if args == uniqueExpected {
			commandGroup := ""
			command := " "
			for len(command) > 0 {
				select {
				case command = <-c.command:
					if len(commandGroup) > 0 {
						commandGroup += "~"
					}
					commandGroup += command
				default:
					// if we have at least one command, then send it off
					if len(commandGroup) > 0 {
						command = ""
					}
				}
			}
			fmt.Fprintf(w, commandGroup)
			return
		} else {
			fmt.Fprintf(w, "END")
			return
		}
	}

	http.NotFound(w, r)
}

func ListenAndServe(addr string, handler CanvasHandler) (err error) {
	return http.ListenAndServe(addr, newCanvas(handler))
}

func stringPartition(s, sep string) (string, string, string) {
	sepPos := strings.Index(s, sep)
	if sepPos == -1 { // no seperator found
		return s, "", ""
	}
	split := strings.SplitN(s, sep, 2)
	return split[0], sep, split[1]
}
<|endoftext|>