<commit_before>package controllers

import (
	"database/sql"
	"net/http"
	"os"

	"github.com/kayac/alphawing/app/models"

	"github.com/revel/revel"
)

type JsonResponse struct {
	Status  int      `json:"status"`
	Message []string `json:"message"`
}

type JsonResponseUploadBundle struct {
	*JsonResponse
	Content *models.BundleJsonResponse `json:"content"`
}

type ApiController struct {
	AlphaWingController
}

func (c ApiController) NewJsonResponse(stat int, mes []string) *JsonResponse {
	return &JsonResponse{
		Status:  stat,
		Message: mes,
	}
}

func (c ApiController) NewJsonResponseUploadBundle(stat int, mes []string, content *models.BundleJsonResponse) *JsonResponseUploadBundle {
	return &JsonResponseUploadBundle{
		c.NewJsonResponse(stat, mes),
		content,
	}
}

func (c ApiController) NewJsonResponseDeleteBundle(stat int, mes []string) *JsonResponse {
	return c.NewJsonResponse(stat, mes)
}

func (c ApiController) GetDocument() revel.Result {
	return c.Render()
}

func (c ApiController) PostUploadBundle(token string, description string, file *os.File) revel.Result {
	app, err := models.GetAppByApiToken(c.Txn, token)
	if err != nil {
		c.Response.Status = http.StatusUnauthorized
		return c.RenderJson(c.NewJsonResponseUploadBundle(c.Response.Status, []string{"Token is invalid."}, nil))
	}

	c.Validation.Required(file != nil).Message("File is required.")
	if c.Validation.HasErrors() {
		var errors []string
		for _, err := range c.Validation.Errors {
			errors = append(errors, err.String())
		}
		c.Response.Status = http.StatusBadRequest
		return c.RenderJson(c.NewJsonResponseUploadBundle(c.Response.Status, errors, nil))
	}

	bundle := &models.Bundle{
		Description: description,
		File:        file,
	}

	if err := app.CreateBundle(c.Txn, c.GoogleService, Conf.AaptPath, bundle); err != nil {
		if aperr, ok := err.(*models.ApkParseError); ok {
			c.Response.Status = http.StatusInternalServerError
			return c.RenderJson(c.NewJsonResponseUploadBundle(c.Response.Status, []string{aperr.Error()}, nil))
		}
		c.Response.Status = http.StatusInternalServerError
		return c.RenderJson(c.NewJsonResponseUploadBundle(c.Response.Status, []string{err.Error()}, nil))
	}

	content, err := bundle.JsonResponse(&c)
	if err != nil {
		c.Response.Status = http.StatusInternalServerError
		return c.RenderJson(c.NewJsonResponseUploadBundle(c.Response.Status, []string{err.Error()}, nil))
	}

	c.Response.Status = http.StatusOK
	return c.RenderJson(c.NewJsonResponseUploadBundle(c.Response.Status, []string{"Bundle is created!"}, content))
}

func (c ApiController) PostDeleteBundle(token string, file_id string) revel.Result {
	_, err := models.GetAppByApiToken(c.Txn, token)
	if err != nil {
		c.Response.Status = http.StatusUnauthorized
		return c.RenderJson(c.NewJsonResponseDeleteBundle(c.Response.Status, []string{"Token is invalid."}))
	}

	c.Validation.Required(file_id).Message("file_id is required.")
	if c.Validation.HasErrors() {
		var errors []string
		for _, err := range c.Validation.Errors {
			errors = append(errors, err.String())
		}
		c.Response.Status = http.StatusBadRequest
		return c.RenderJson(c.NewJsonResponseDeleteBundle(c.Response.Status, errors))
	}

	bundle, err := models.GetBundleByFileId(c.Txn, file_id)
	if err != nil {
		if err == sql.ErrNoRows {
			c.Response.Status = http.StatusInternalServerError
			return c.RenderJson(c.NewJsonResponseDeleteBundle(c.Response.Status, []string{"Bundle not found."}))
		}
		c.Response.Status = http.StatusInternalServerError
		return c.RenderJson(c.NewJsonResponseDeleteBundle(c.Response.Status, []string{err.Error()}))
	}

	err = bundle.Delete(c.Txn, c.GoogleService)
	if err != nil {
		c.Response.Status = http.StatusInternalServerError
		return c.RenderJson(c.NewJsonResponseDeleteBundle(c.Response.Status, []string{err.Error()}))
	}

	c.Response.Status = http.StatusOK
	return c.RenderJson(c.NewJsonResponseDeleteBundle(c.Response.Status, []string{"Bundle is deleted!"}))
}
<commit_msg>レビュー対応<commit_after>package controllers

import (
	"database/sql"
	"net/http"
	"os"

	"github.com/kayac/alphawing/app/models"

	"github.com/revel/revel"
)

type JsonResponse struct {
	Status  int      `json:"status"`
	Message []string `json:"message"`
}

type JsonResponseUploadBundle struct {
	*JsonResponse
	Content *models.BundleJsonResponse `json:"content"`
}

type ApiController struct {
	AlphaWingController
}

func (c ApiController) NewJsonResponse(stat int, mes []string) *JsonResponse {
	return &JsonResponse{
		Status:  stat,
		Message: mes,
	}
}

func (c ApiController) NewJsonResponseUploadBundle(stat int, mes []string, content *models.BundleJsonResponse) *JsonResponseUploadBundle {
	return &JsonResponseUploadBundle{
		c.NewJsonResponse(stat, mes),
		content,
	}
}

func (c ApiController) NewJsonResponseDeleteBundle(stat int, mes []string) *JsonResponse {
	return c.NewJsonResponse(stat, mes)
}

func (c ApiController) GetDocument() revel.Result {
	return c.Render()
}

func (c ApiController) PostUploadBundle(token string, description string, file *os.File) revel.Result {
	app, err := models.GetAppByApiToken(c.Txn, token)
	if err != nil {
		c.Response.Status = http.StatusUnauthorized
		return c.RenderJson(c.NewJsonResponseUploadBundle(c.Response.Status, []string{"Token is invalid."}, nil))
	}

	c.Validation.Required(file != nil).Message("File is required.")
	if c.Validation.HasErrors() {
		var errors []string
		for _, err := range c.Validation.Errors {
			errors = append(errors, err.String())
		}
		c.Response.Status = http.StatusBadRequest
		return c.RenderJson(c.NewJsonResponseUploadBundle(c.Response.Status, errors, nil))
	}

	bundle := &models.Bundle{
		Description: description,
		File:        file,
	}

	if err := app.CreateBundle(c.Txn, c.GoogleService, Conf.AaptPath, bundle); err != nil {
		if aperr, ok := err.(*models.ApkParseError); ok {
			c.Response.Status = http.StatusInternalServerError
			return c.RenderJson(c.NewJsonResponseUploadBundle(c.Response.Status, []string{aperr.Error()}, nil))
		}
		c.Response.Status = http.StatusInternalServerError
		return c.RenderJson(c.NewJsonResponseUploadBundle(c.Response.Status, []string{err.Error()}, nil))
	}

	content, err := bundle.JsonResponse(&c)
	if err != nil {
		c.Response.Status = http.StatusInternalServerError
		return c.RenderJson(c.NewJsonResponseUploadBundle(c.Response.Status, []string{err.Error()}, nil))
	}

	c.Response.Status = http.StatusOK
	return c.RenderJson(c.NewJsonResponseUploadBundle(c.Response.Status, []string{"Bundle is created!"}, content))
}

func (c ApiController) PostDeleteBundle(token string, file_id string) revel.Result {
	_, err := models.GetAppByApiToken(c.Txn, token)
	if err != nil {
		c.Response.Status = http.StatusUnauthorized
		return c.RenderJson(c.NewJsonResponseDeleteBundle(c.Response.Status, []string{"Token is invalid."}))
	}

	c.Validation.Required(file_id).Message("file_id is required.")
	if c.Validation.HasErrors() {
		var errors []string
		for _, err := range c.Validation.Errors {
			errors = append(errors, err.String())
		}
		c.Response.Status = http.StatusBadRequest
		return c.RenderJson(c.NewJsonResponseDeleteBundle(c.Response.Status, errors))
	}

	bundle, err := models.GetBundleByFileId(c.Txn, file_id)
	if err != nil {
		if err == sql.ErrNoRows {
			c.Response.Status = http.StatusNotFound
			return c.RenderJson(c.NewJsonResponseDeleteBundle(c.Response.Status, []string{"Bundle not found."}))
		}
		c.Response.Status = http.StatusInternalServerError
		return c.RenderJson(c.NewJsonResponseDeleteBundle(c.Response.Status, []string{err.Error()}))
	}

	err = bundle.Delete(c.Txn, c.GoogleService)
	if err != nil {
		c.Response.Status = http.StatusInternalServerError
		return c.RenderJson(c.NewJsonResponseDeleteBundle(c.Response.Status, []string{err.Error()}))
	}

	c.Response.Status = http.StatusOK
	return c.RenderJson(c.NewJsonResponseDeleteBundle(c.Response.Status, []string{"Bundle is deleted!"}))
}
<|endoftext|>


<commit_before>package conio

import (
	"context"
	"fmt"
	"io"
	"regexp"
	"strings"

	"github.com/mattn/go-runewidth"
	getch "github.com/zetamatta/go-getch"
)

var ansiCutter = regexp.MustCompile("\x1B[^a-zA-Z]*[A-Za-z]")

func BoxPrint(ctx context.Context, nodes []string, out io.Writer) bool {
	value, _, _ := boxPrint(ctx, nodes, out)
	return value
}

func boxPrint(ctx context.Context, nodes []string, out io.Writer) (bool, int, int) {
	width := int(GetScreenBufferInfo().Size.X)
	if width <= 0 || width > 999 {
		width = 80
	}
	maxLen := 1
	for _, finfo := range nodes {
		length := runewidth.StringWidth(ansiCutter.ReplaceAllString(finfo, ""))
		if length > maxLen {
			maxLen = length
		}
	}
	nodePerLine := (width - 1) / (maxLen + 1)
	if nodePerLine <= 0 {
		nodePerLine = 1
	}
	nlines := (len(nodes) + nodePerLine - 1) / nodePerLine

	lines := make([][]byte, nlines)
	row := 0
	for _, finfo := range nodes {
		lines[row] = append(lines[row], finfo...)
		w := runewidth.StringWidth(ansiCutter.ReplaceAllString(finfo, ""))
		for i, iEnd := 0, maxLen+1-w; i < iEnd; i++ {
			lines[row] = append(lines[row], ' ')
		}
		row++
		if row >= nlines {
			row = 0
		}
	}
	for _, line := range lines {
		fmt.Fprintln(out, strings.TrimSpace(string(line)))
		if ctx != nil {
			select {
			case <-ctx.Done():
				return false, nodePerLine, nlines
			default:
			}
		}
	}
	return true, nodePerLine, nlines
}

const (
	CURSOR_OFF = "\x1B[?25l"
	CURSOR_ON  = "\x1B[?25h"
	BOLD_ON    = "\x1B[0;47;30m"
	BOLD_OFF   = "\x1B[0m"

	K_LEFT  = 0x25
	K_RIGHT = 0x27
	K_UP    = 0x26
	K_DOWN  = 0x28
)

func BoxChoice(nodes []string, out io.Writer) string {
	cursor := 0
	nodes_draw := make([]string, len(nodes))
	for i := 0; i < len(nodes); i++ {
		nodes_draw[i] = nodes[i]
	}
	io.WriteString(out, CURSOR_OFF)
	defer io.WriteString(out, CURSOR_ON)
	for {
		nodes_draw[cursor] = BOLD_ON + nodes[cursor] + BOLD_OFF
		status, _, h := boxPrint(nil, nodes_draw, out)
		if !status {
			return ""
		}
		nodes_draw[cursor] = nodes[cursor]
		e := getch.All()
		if k := e.Key; k != nil {
			switch k.Rune {
			case 'h', ('b' & 0x1F):
				if cursor-h >= 0 {
					cursor -= h
				}
			case 'l', ('f' & 0x1F):
				if cursor+h < len(nodes) {
					cursor += h
				}
			case 'j', ('n' & 0x1F), ' ':
				if cursor+1 < len(nodes) {
					cursor++
				}
			case 'k', ('p' & 0x1F), '\b':
				if cursor > 0 {
					cursor--
				}
			case '\r', '\n':
				return nodes[cursor]
			case '\x1B', ('g' & 0x1F):
				return ""
			}

			switch k.Scan {
			case K_LEFT:
				if cursor-h >= 0 {
					cursor -= h
				}
			case K_RIGHT:
				if cursor+h < len(nodes) {
					cursor += h
				}
			case K_DOWN:
				if cursor+1 < len(nodes) {
					cursor++
				}
			case K_UP:
				if cursor > 0 {
					cursor--
				}
			}
		}
		fmt.Fprintf(out, "\x1B[%dA", h)
	}
}
<commit_msg>box: support roll-up/down<commit_after>package conio

import (
	"context"
	"fmt"
	"io"
	"regexp"
	"strings"

	"github.com/mattn/go-runewidth"
	getch "github.com/zetamatta/go-getch"
)

var ansiCutter = regexp.MustCompile("\x1B[^a-zA-Z]*[A-Za-z]")

func BoxPrint(ctx context.Context, nodes []string, out io.Writer) bool {
	value, _, _ := boxPrint(ctx, nodes, 0, false, out)
	return value
}

func boxPrint(ctx context.Context, nodes []string, offset int, paging bool, out io.Writer) (bool, int, int) {
	csbi := GetScreenBufferInfo()
	width := int(csbi.Size.X)
	if width <= 0 || width > 999 {
		width = 80
	}
	maxLen := 1
	for _, finfo := range nodes {
		length := runewidth.StringWidth(ansiCutter.ReplaceAllString(finfo, ""))
		if length > maxLen {
			maxLen = length
		}
	}
	nodePerLine := (width - 1) / (maxLen + 1)
	if nodePerLine <= 0 {
		nodePerLine = 1
	}
	nlines := (len(nodes) + nodePerLine - 1) / nodePerLine

	lines := make([][]byte, nlines)
	row := 0
	for _, finfo := range nodes {
		lines[row] = append(lines[row], finfo...)
		w := runewidth.StringWidth(ansiCutter.ReplaceAllString(finfo, ""))
		for i, iEnd := 0, maxLen+1-w; i < iEnd; i++ {
			lines[row] = append(lines[row], ' ')
		}
		row++
		if row >= nlines {
			row = 0
		}
	}
	i_end := len(lines)
	if paging {
		_, height := csbi.ViewSize()
		height--
		if i_end >= offset+height {
			i_end = offset + height
		}
	}

	for i := offset; i < i_end; i++ {
		fmt.Fprintln(out, string(lines[i]))
		if ctx != nil {
			select {
			case <-ctx.Done():
				return false, nodePerLine, nlines
			default:
			}
		}
	}
	return true, nodePerLine, nlines
}

const (
	CURSOR_OFF = "\x1B[?25l"
	CURSOR_ON  = "\x1B[?25h"
	BOLD_ON    = "\x1B[0;47;30m"
	BOLD_OFF   = "\x1B[0m"

	K_LEFT  = 0x25
	K_RIGHT = 0x27
	K_UP    = 0x26
	K_DOWN  = 0x28
)

func truncate(s string, w int) string {
	return runewidth.Truncate(strings.TrimSpace(s), w, "")
}

func BoxChoice(nodes []string, out io.Writer) string {
	cursor := 0
	nodes_draw := make([]string, len(nodes))
	width, height := GetScreenBufferInfo().ViewSize()
	width--
	height--
	for i := 0; i < len(nodes); i++ {
		nodes_draw[i] = truncate(nodes[i], width-1)
	}
	io.WriteString(out, CURSOR_OFF)
	defer io.WriteString(out, CURSOR_ON)

	offset := 0
	for {
		nodes_draw[cursor] = BOLD_ON +
			truncate(nodes[cursor], width-1) + BOLD_OFF
		status, _, h := boxPrint(nil, nodes_draw, offset, true, out)
		if !status {
			return ""
		}
		nodes_draw[cursor] = truncate(nodes[cursor], width-1)
		last := cursor
		for last == cursor {
			e := getch.All()
			if k := e.Key; k != nil {
				switch k.Rune {
				case 'h', ('b' & 0x1F):
					if cursor-h >= 0 {
						cursor -= h
					}
				case 'l', ('f' & 0x1F):
					if cursor+h < len(nodes) {
						cursor += h
					}
				case 'j', ('n' & 0x1F), ' ':
					if cursor+1 < len(nodes) {
						cursor++
					}
				case 'k', ('p' & 0x1F), '\b':
					if cursor > 0 {
						cursor--
					}
				case '\r', '\n':
					return nodes[cursor]
				case '\x1B', ('g' & 0x1F):
					return ""
				}

				switch k.Scan {
				case K_LEFT:
					if cursor-h >= 0 {
						cursor -= h
					}
				case K_RIGHT:
					if cursor+h < len(nodes) {
						cursor += h
					}
				case K_DOWN:
					if cursor+1 < len(nodes) {
						cursor++
					}
				case K_UP:
					if cursor > 0 {
						cursor--
					}
				}

				// x := cursor / h
				y := cursor % h
				if y < offset {
					offset--
				} else if y >= offset+height {
					offset++
				}
			}
		}
		if h < height {
			fmt.Fprintf(out, "\x1B[%dA", h)
		} else {
			fmt.Fprintf(out, "\x1B[%dA", height)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/fsouza/go-dockerclient"
	"github.com/pwaller/barrier"
)

type Container struct {
	Name      string
	ImageName string
	Args, Env []string
	Volumes   []string
	StatusURI string

	client    *docker.Client
	container *docker.Container

	Failed, Superceded, Obtained, Ready, Closing barrier.Barrier

	wg *sync.WaitGroup

	Errors  <-chan error
	errorsW chan<- error
}

type SourceType int

const (
	// Run build with current directory with context
	BuildCwd            SourceType = iota
	BuildTarballContent            // Build with specified io.Reader as context
	BuildTarballURL                // Build with specified remote URL as context
	DockerPull                     // Run a docker pull to obtain the image
	GithubRepository               // build a github repository by making a local mirror
)

type ContainerSource struct {
	Type                SourceType
	buildTarballContent io.Reader
	buildDirectory      string
	buildTarballURL     string
	dockerImageName     string
	githubURL           string
	githubRef           string
}

// Construct a *Container. When the `wg` WaitGroup is zero, there is nothing
// outstanding (such as firewall rules which need garbage collecting).
func NewContainer(client *docker.Client, name string, wg *sync.WaitGroup) *Container {

	errors := make(chan error)

	c := &Container{
		Name:    name,
		client:  client,
		wg:      wg,
		Errors:  errors,
		errorsW: errors,
	}

	// If the container fails we should assume it should be torn down.
	c.Failed.Forward(&c.Closing)

	return c
}

func makeVolumeSet(in []string) map[string]struct{} {
	volumes := map[string]struct{}{}
	for _, v := range in {
		if strings.Contains(v, ":") {
			continue
		}
		volumes[v] = struct{}{}
	}
	return volumes
}

func makeBinds(in []string) []string {
	binds := []string{}
	for _, v := range in {
		if !strings.Contains(v, ":") {
			continue
		}
		binds = append(binds, v)
	}
	return binds
}

// `docker create` the container.
func (c *Container) Create(imageName string) error {
	opts := docker.CreateContainerOptions{
		Config: &docker.Config{
			Hostname:     c.Name,
			AttachStdout: true,
			AttachStderr: true,
			Env:          c.Env,
			Cmd:          c.Args,
			Image:        imageName,
			Volumes:      makeVolumeSet(c.Volumes),
			Labels: map[string]string{
				"orchestrator":  "hanoverd",
				"hanoverd-name": c.Name,
			},
		},
	}

	var err error
	c.container, err = c.client.CreateContainer(opts)
	return err
}

// CopyOutput copies the output of the container to `w` and blocks until
// completion
func (c *Container) CopyOutput() error {

	// TODO(pwaller): at some point move this on to 'c' for configurability?
	w := os.Stderr
	// Blocks until stream closed
	return c.client.AttachToContainer(docker.AttachToContainerOptions{
		Container:    c.container.ID,
		OutputStream: w,
		ErrorStream:  w,
		Logs:         true,
		Stdout:       true,
		Stderr:       true,
		Stream:       true,
	})
}

// :todo(drj): May want to return errors for truly broken containers (timeout).
// Poll for the program inside the container being ready to accept connections
// Returns `true` for success and `false` for failure.
func (c *Container) AwaitListening() bool {

	const (
		DefaultTimeout = 5 * time.Minute
		PollFrequency  = 10 // times per second (via integer division of ns)
	)

	startDeadline := time.Now().Add(DefaultTimeout)

	for _, port := range c.container.NetworkSettings.PortMappingAPI() {
		url := fmt.Sprint("http://", port.IP, ":", port.PublicPort, c.StatusURI)
		for {
			response, err := http.Get(url)
			if response != nil && response.Body != nil {
				response.Body.Close()
			}
			if err == nil {
				switch response.StatusCode {
				case http.StatusOK:
					return true
				case http.StatusNotFound:
				default:
					log.Printf("Got non-200 status code: %v, giving up",
						response.StatusCode)
					c.Failed.Fall()
					return false
				}
			}

			if time.Now().After(startDeadline) {
				log.Printf("Took longer than %v to start, giving up", DefaultTimeout)
				return false
			}

			time.Sleep(time.Second / PollFrequency)

			select {
			case <-c.Closing.Barrier():
				// If the container has closed, cease waiting
				return false
			default:
			}
		}
	}

	return false
}

// Given an internal port, return the port mapped by docker, if there is one.
func (c *Container) MappedPort(internal int) (int, bool) {
	for _, m := range c.container.NetworkSettings.PortMappingAPI() {
		if int(m.PrivatePort) == internal {
			return int(m.PublicPort), true
		}
	}
	return -1, false
}

// Start the container (and notify it if c.Closing falls)
func (c *Container) Start() error {
	hc := &docker.HostConfig{
		PublishAllPorts: true,
		Binds:           makeBinds(c.Volumes),
	}
	err := c.client.StartContainer(c.container.ID, hc)
	if err != nil {
		return err
	}

	// Load container.NetworkSettings
	c.container, err = c.client.InspectContainer(c.container.ID)
	if err != nil {
		return err
	}

	// Listen on the Closing barrier and send a kill to the container if it
	// falls.
	c.wg.Add(1)
	go func() {
		defer c.wg.Done()

		<-c.Closing.Barrier()
		// If the container is signaled to close, send a kill signal
		err := c.client.KillContainer(docker.KillContainerOptions{
			ID: c.container.ID,
		})
		if err == nil {
			return
		}
		switch err := err.(type) {
		case *docker.NoSuchContainer:
			// The container already went away, who cares.
			return
		default:
			log.Println("Killing container failed:", c.container.ID, err)
		}
	}()
	return nil
}

// Wait until container exits
func (c *Container) Wait() (int, error) {
	return c.client.WaitContainer(c.container.ID)
}

// Internal function for raising an error.
func (c *Container) err(err error) {
	c.errorsW <- err
	c.Closing.Fall()
}

// Manage the whole lifecycle of the container in response to a request to
// start it.
func (c *Container) Run(imageSource ImageSource, payload []byte) (int, error) {

	defer c.Closing.Fall()
	defer close(c.errorsW)

	go func() {
		for err := range c.Errors {
			log.Println("BUG: Async container error:", err)
			// TODO(pwaller): If this case is hit we might not want to
			// tear the container down really.
			c.Failed.Fall()
		}
	}()

	imageName, err := imageSource.Obtain(c.client, payload)
	c.Obtained.Fall()
	if err != nil {
		c.Failed.Fall()
		return -2, err
	}

	err = c.Create(imageName)
	if err != nil {
		c.Failed.Fall()
		return -1, err
	}
	defer c.Delete()

	err = c.Start()
	if err != nil {
		c.Failed.Fall()
		return -1, err
	}

	// Must come after container start has succeeded, otherwise we end up
	// perpetually attached if it fails to succeed, which blocks program exit.
	// Program exit must be blocked ordinarily until this completes so that
	// if we are quitting we see all of the messages sent by the container
	// until it quit.
	c.wg.Add(1)
	go func() {
		defer c.wg.Done()

		err := c.CopyOutput()
		if err != nil {
			c.err(err)
		}
	}()

	go func() {
		if !c.AwaitListening() {
			c.Failed.Fall()
			return
		}
		c.Ready.Fall()
	}()

	return c.Wait()
}

func (c *Container) Delete() {
	err := c.client.RemoveContainer(docker.RemoveContainerOptions{
		ID:            c.container.ID,
		RemoveVolumes: true,
		Force:         true,
	})
	if err != nil {
		log.Println("Warn: failed to delete container:", err)
	}
}
<commit_msg>Add warning if no ports are exposed<commit_after>package main

import (
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/fsouza/go-dockerclient"
	"github.com/pwaller/barrier"
)

type Container struct {
	Name      string
	ImageName string
	Args, Env []string
	Volumes   []string
	StatusURI string

	client    *docker.Client
	container *docker.Container

	Failed, Superceded, Obtained, Ready, Closing barrier.Barrier

	wg *sync.WaitGroup

	Errors  <-chan error
	errorsW chan<- error
}

type SourceType int

const (
	// Run build with current directory with context
	BuildCwd            SourceType = iota
	BuildTarballContent            // Build with specified io.Reader as context
	BuildTarballURL                // Build with specified remote URL as context
	DockerPull                     // Run a docker pull to obtain the image
	GithubRepository               // build a github repository by making a local mirror
)

type ContainerSource struct {
	Type                SourceType
	buildTarballContent io.Reader
	buildDirectory      string
	buildTarballURL     string
	dockerImageName     string
	githubURL           string
	githubRef           string
}

// Construct a *Container. When the `wg` WaitGroup is zero, there is nothing
// outstanding (such as firewall rules which need garbage collecting).
func NewContainer(client *docker.Client, name string, wg *sync.WaitGroup) *Container {

	errors := make(chan error)

	c := &Container{
		Name:    name,
		client:  client,
		wg:      wg,
		Errors:  errors,
		errorsW: errors,
	}

	// If the container fails we should assume it should be torn down.
	c.Failed.Forward(&c.Closing)

	return c
}

func makeVolumeSet(in []string) map[string]struct{} {
	volumes := map[string]struct{}{}
	for _, v := range in {
		if strings.Contains(v, ":") {
			continue
		}
		volumes[v] = struct{}{}
	}
	return volumes
}

func makeBinds(in []string) []string {
	binds := []string{}
	for _, v := range in {
		if !strings.Contains(v, ":") {
			continue
		}
		binds = append(binds, v)
	}
	return binds
}

// `docker create` the container.
func (c *Container) Create(imageName string) error {
	opts := docker.CreateContainerOptions{
		Config: &docker.Config{
			Hostname:     c.Name,
			AttachStdout: true,
			AttachStderr: true,
			Env:          c.Env,
			Cmd:          c.Args,
			Image:        imageName,
			Volumes:      makeVolumeSet(c.Volumes),
			Labels: map[string]string{
				"orchestrator":  "hanoverd",
				"hanoverd-name": c.Name,
			},
		},
	}

	var err error
	c.container, err = c.client.CreateContainer(opts)
	return err
}

// CopyOutput copies the output of the container to `w` and blocks until
// completion
func (c *Container) CopyOutput() error {

	// TODO(pwaller): at some point move this on to 'c' for configurability?
	w := os.Stderr
	// Blocks until stream closed
	return c.client.AttachToContainer(docker.AttachToContainerOptions{
		Container:    c.container.ID,
		OutputStream: w,
		ErrorStream:  w,
		Logs:         true,
		Stdout:       true,
		Stderr:       true,
		Stream:       true,
	})
}

// :todo(drj): May want to return errors for truly broken containers (timeout).
// Poll for the program inside the container being ready to accept connections
// Returns `true` for success and `false` for failure.
func (c *Container) AwaitListening() bool {

	if len(c.container.NetworkSettings.PortMappingAPI()) == 0 {
		log.Printf("Error! No ports are exposed.")
		return false
	}

	const (
		DefaultTimeout = 5 * time.Minute
		PollFrequency  = 10 // times per second (via integer division of ns)
	)

	startDeadline := time.Now().Add(DefaultTimeout)

	for _, port := range c.container.NetworkSettings.PortMappingAPI() {
		url := fmt.Sprint("http://", port.IP, ":", port.PublicPort, c.StatusURI)
		for {
			response, err := http.Get(url)
			if response != nil && response.Body != nil {
				response.Body.Close()
			}
			if err == nil {
				switch response.StatusCode {
				case http.StatusOK:
					return true
				case http.StatusNotFound:
				default:
					log.Printf("Got non-200 status code: %v, giving up",
						response.StatusCode)
					c.Failed.Fall()
					return false
				}
			}

			if time.Now().After(startDeadline) {
				log.Printf("Took longer than %v to start, giving up", DefaultTimeout)
				return false
			}

			time.Sleep(time.Second / PollFrequency)

			select {
			case <-c.Closing.Barrier():
				// If the container has closed, cease waiting
				return false
			default:
			}
		}
	}

	return false
}

// Given an internal port, return the port mapped by docker, if there is one.
func (c *Container) MappedPort(internal int) (int, bool) {
	for _, m := range c.container.NetworkSettings.PortMappingAPI() {
		if int(m.PrivatePort) == internal {
			return int(m.PublicPort), true
		}
	}
	return -1, false
}

// Start the container (and notify it if c.Closing falls)
func (c *Container) Start() error {
	hc := &docker.HostConfig{
		PublishAllPorts: true,
		Binds:           makeBinds(c.Volumes),
	}
	err := c.client.StartContainer(c.container.ID, hc)
	if err != nil {
		return err
	}

	// Load container.NetworkSettings
	c.container, err = c.client.InspectContainer(c.container.ID)
	if err != nil {
		return err
	}

	// Listen on the Closing barrier and send a kill to the container if it
	// falls.
	c.wg.Add(1)
	go func() {
		defer c.wg.Done()

		<-c.Closing.Barrier()
		// If the container is signaled to close, send a kill signal
		err := c.client.KillContainer(docker.KillContainerOptions{
			ID: c.container.ID,
		})
		if err == nil {
			return
		}
		switch err := err.(type) {
		case *docker.NoSuchContainer:
			// The container already went away, who cares.
			return
		default:
			log.Println("Killing container failed:", c.container.ID, err)
		}
	}()
	return nil
}

// Wait until container exits
func (c *Container) Wait() (int, error) {
	return c.client.WaitContainer(c.container.ID)
}

// Internal function for raising an error.
func (c *Container) err(err error) {
	c.errorsW <- err
	c.Closing.Fall()
}

// Manage the whole lifecycle of the container in response to a request to
// start it.
func (c *Container) Run(imageSource ImageSource, payload []byte) (int, error) {

	defer c.Closing.Fall()
	defer close(c.errorsW)

	go func() {
		for err := range c.Errors {
			log.Println("BUG: Async container error:", err)
			// TODO(pwaller): If this case is hit we might not want to
			// tear the container down really.
			c.Failed.Fall()
		}
	}()

	imageName, err := imageSource.Obtain(c.client, payload)
	c.Obtained.Fall()
	if err != nil {
		c.Failed.Fall()
		return -2, err
	}

	err = c.Create(imageName)
	if err != nil {
		c.Failed.Fall()
		return -1, err
	}
	defer c.Delete()

	err = c.Start()
	if err != nil {
		c.Failed.Fall()
		return -1, err
	}

	// Must come after container start has succeeded, otherwise we end up
	// perpetually attached if it fails to succeed, which blocks program exit.
	// Program exit must be blocked ordinarily until this completes so that
	// if we are quitting we see all of the messages sent by the container
	// until it quit.
	c.wg.Add(1)
	go func() {
		defer c.wg.Done()

		err := c.CopyOutput()
		if err != nil {
			c.err(err)
		}
	}()

	go func() {
		if !c.AwaitListening() {
			c.Failed.Fall()
			return
		}
		c.Ready.Fall()
	}()

	return c.Wait()
}

func (c *Container) Delete() {
	err := c.client.RemoveContainer(docker.RemoveContainerOptions{
		ID:            c.container.ID,
		RemoveVolumes: true,
		Force:         true,
	})
	if err != nil {
		log.Println("Warn: failed to delete container:", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"sync"
	"time"

	"github.com/fsouza/go-dockerclient"
	"github.com/pwaller/barrier"
)

type Container struct {
	Name      string
	Args, Env []string

	client    *docker.Client
	container *docker.Container

	Failed, Superceded, Ready, Closing barrier.Barrier

	wg *sync.WaitGroup

	Errors  <-chan error
	errorsW chan<- error
}

type SourceType int

const (
	// Run build with current directory with context
	BuildCwd            SourceType = iota
	BuildTarballContent            // Build with specified io.Reader as context
	BuildTarballURL                // Build with specified remote URL as context
	DockerPull                     // Run a docker pull to obtain the image
)

type ContainerSource struct {
	Type                SourceType
	buildTarballContent io.Reader
	buildTarballURL     string
	dockerImageName     string
}

// Construct a *Container. When the `wg` WaitGroup is zero, there is nothing
// outstanding (such as firewall rules which need garbage collecting).
func NewContainer(client *docker.Client, name string, wg *sync.WaitGroup) *Container {

	errors := make(chan error)

	c := &Container{
		Name:    name,
		client:  client,
		wg:      wg,
		Errors:  errors,
		errorsW: errors,
	}

	// If the container fails we should assume it should be torn down.
	c.Failed.Forward(&c.Closing)

	return c
}

// Generate a docker image. This can be done through various mechanisms in
// response to an UpdateEvent (see SourceType constant declarations).
func (c *Container) Build(config UpdateEvent) error {
	if config.BuildComplete != nil {
		defer close(config.BuildComplete)
	}

	var err error
	bo := docker.BuildImageOptions{}
	bo.Name = c.Name
	bo.OutputStream = config.OutputStream
	if bo.OutputStream == nil {
		bo.OutputStream = os.Stderr
	}

	switch config.Source.Type {
	case BuildCwd:
		bo.ContextDir, err = os.Getwd()
		if err != nil {
			return err
		}
	case BuildTarballContent:
		bo.InputStream = config.Source.buildTarballContent
	default:
		return fmt.Errorf("Unimplemented ContainerSource: %v", config.Source.Type)
	}

	return c.client.BuildImage(bo)
}

// Pull an image from a docker repository.
func (c *Container) Pull(config UpdateEvent) error {
	if config.BuildComplete != nil {
		defer close(config.BuildComplete)
	}

	pio := docker.PullImageOptions{}
	pio.Repository = config.Source.dockerImageName
	pio.Registry = ""
	pio.Tag = "latest"
	pio.OutputStream = config.OutputStream
	if pio.OutputStream == nil {
		pio.OutputStream = os.Stderr
	}
	pio.RawJSONStream = false

	return c.client.PullImage(pio, docker.AuthConfiguration{})
}

// `docker create` the container.
func (c *Container) Create(source ContainerSource) error {
	opts := docker.CreateContainerOptions{
		Name: c.Name,
		Config: &docker.Config{
			Hostname:     c.Name,
			AttachStdout: true,
			AttachStderr: true,
			Env:          c.Env,
			Cmd:          c.Args,
		},
	}

	switch source.Type {
	case DockerPull:
		opts.Config.Image = source.dockerImageName
	case BuildCwd, BuildTarballContent:
		opts.Config.Image = c.Name
	}

	var err error
	c.container, err = c.client.CreateContainer(opts)

	return err
}

// CopyOutput copies the output of the container to `w` and blocks until
// completion
func (c *Container) CopyOutput() error {

	// TODO(pwaller): at some point move this on to 'c' for configurability?
	w := os.Stderr
	// Blocks until stream closed
	return c.client.AttachToContainer(docker.AttachToContainerOptions{
		Container:    c.container.ID,
		OutputStream: w,
		ErrorStream:  w,
		Logs:         true,
		Stdout:       true,
		Stderr:       true,
		Stream:       true,
	})
}

// :todo(drj): May want to return errors for truly broken containers (timeout).
// Poll for the program inside the container being ready to accept connections
// Returns `true` for success and `false` for failure.
func (c *Container) AwaitListening() bool {

	for _, port := range c.container.NetworkSettings.PortMappingAPI() {
		url := fmt.Sprint("http://", port.IP, ":", port.PublicPort, "/")
		for {
			response, err := http.Get(url)
			if response != nil && response.Body != nil {
				response.Body.Close()
			}
			if err == nil && response.StatusCode == http.StatusOK {
				return true
			}
			time.Sleep(50 * time.Millisecond)

			select {
			case <-c.Closing.Barrier():
				// If the container has closed, cease waiting
				return false
			default:
			}
		}
	}

	return false
}

// Start the container (and notify it if c.Closing falls)
func (c *Container) Start() error {
	hc := &docker.HostConfig{
		PublishAllPorts: true,
	}
	err := c.client.StartContainer(c.container.ID, hc)
	if err != nil {
		return err
	}

	// Load container.NetworkSettings
	c.container, err = c.client.InspectContainer(c.container.ID)
	if err != nil {
		return err
	}

	// Listen on the Closing barrier and send a kill to the container if it
	// falls.
	c.wg.Add(1)
	go func() {
		defer c.wg.Done()
		<-c.Closing.Barrier()
		// If the container is signaled to close, send a kill signal
		err := c.client.KillContainer(docker.KillContainerOptions{
			ID: c.container.ID,
		})
		if err == nil {
			return
		}
		switch err := err.(type) {
		case *docker.NoSuchContainer:
			// The container already went away, who cares.
			return
		default:
			log.Println("Killing container failed:", c.container.ID, err)
		}
	}()
	return nil
}

// Wait until container exits
func (c *Container) Wait() (int, error) {
	return c.client.WaitContainer(c.container.ID)
}

// Internal function for raising an error.
func (c *Container) err(err error) {
	c.errorsW <- err
	c.Closing.Fall()
}

// Manage the whole lifecycle of the container in response to a request to
// start it.
func (c *Container) Run(event UpdateEvent) (int, error) {

	defer c.Closing.Fall()
	defer close(c.errorsW)

	switch event.Source.Type {
	case DockerPull:
		err := c.Pull(event)
		if err != nil {
			return -2, err
		}
	case BuildTarballContent, BuildCwd:
		err := c.Build(event)
		if err != nil {
			return -2, err
		}
	}

	err := c.Create(event.Source)
	if err != nil {
		return -1, err
	}
	defer c.Delete()

	c.wg.Add(1)
	go func() {
		defer c.wg.Done()
		err := c.CopyOutput()
		if err != nil {
			c.err(err)
		}
	}()

	err = c.Start()
	if err != nil {
		return -1, err
	}

	go func() {
		if !c.AwaitListening() {
			c.Failed.Fall()
			return
		}
		c.Ready.Fall()
	}()

	return c.Wait()
}

func (c *Container) Delete() {
	err := c.client.RemoveContainer(docker.RemoveContainerOptions{
		ID:            c.container.ID,
		RemoveVolumes: true,
		Force:         true,
	})
	if err != nil {
		log.Println("Warn: failed to delete container:", err)
	}
}
<commit_msg>Use the RawJSONStream for the output of Pull<commit_after>package main

import (
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"sync"
	"time"

	"github.com/fsouza/go-dockerclient"
	"github.com/pwaller/barrier"
)

type Container struct {
	Name      string
	Args, Env []string

	client    *docker.Client
	container *docker.Container

	Failed, Superceded, Ready, Closing barrier.Barrier

	wg *sync.WaitGroup

	Errors  <-chan error
	errorsW chan<- error
}

type SourceType int

const (
	// Run build with current directory with context
	BuildCwd            SourceType = iota
	BuildTarballContent            // Build with specified io.Reader as context
	BuildTarballURL                // Build with specified remote URL as context
	DockerPull                     // Run a docker pull to obtain the image
)

type ContainerSource struct {
	Type                SourceType
	buildTarballContent io.Reader
	buildTarballURL     string
	dockerImageName     string
}

// Construct a *Container. When the `wg` WaitGroup is zero, there is nothing
// outstanding (such as firewall rules which need garbage collecting).
func NewContainer(client *docker.Client, name string, wg *sync.WaitGroup) *Container {

	errors := make(chan error)

	c := &Container{
		Name:    name,
		client:  client,
		wg:      wg,
		Errors:  errors,
		errorsW: errors,
	}

	// If the container fails we should assume it should be torn down.
	c.Failed.Forward(&c.Closing)

	return c
}

// Generate a docker image. This can be done through various mechanisms in
// response to an UpdateEvent (see SourceType constant declarations).
func (c *Container) Build(config UpdateEvent) error {
	if config.BuildComplete != nil {
		defer close(config.BuildComplete)
	}

	var err error
	bo := docker.BuildImageOptions{}
	bo.Name = c.Name
	bo.OutputStream = config.OutputStream
	if bo.OutputStream == nil {
		bo.OutputStream = os.Stderr
	}

	switch config.Source.Type {
	case BuildCwd:
		bo.ContextDir, err = os.Getwd()
		if err != nil {
			return err
		}
	case BuildTarballContent:
		bo.InputStream = config.Source.buildTarballContent
	default:
		return fmt.Errorf("Unimplemented ContainerSource: %v", config.Source.Type)
	}

	return c.client.BuildImage(bo)
}

// Pull an image from a docker repository.
func (c *Container) Pull(config UpdateEvent) error {
	if config.BuildComplete != nil {
		defer close(config.BuildComplete)
	}

	pio := docker.PullImageOptions{}
	pio.Repository = config.Source.dockerImageName
	log.Println("Pulling", pio.Repository)
	pio.Registry = ""
	pio.Tag = "latest"
	pio.OutputStream = config.OutputStream
	if pio.OutputStream == nil {
		pio.OutputStream = os.Stderr
	}
	pio.RawJSONStream = true

	return c.client.PullImage(pio, docker.AuthConfiguration{})
}

// `docker create` the container.
func (c *Container) Create(source ContainerSource) error {
	opts := docker.CreateContainerOptions{
		Name: c.Name,
		Config: &docker.Config{
			Hostname:     c.Name,
			AttachStdout: true,
			AttachStderr: true,
			Env:          c.Env,
			Cmd:          c.Args,
		},
	}

	switch source.Type {
	case DockerPull:
		opts.Config.Image = source.dockerImageName
	case BuildCwd, BuildTarballContent:
		opts.Config.Image = c.Name
	}

	var err error
	c.container, err = c.client.CreateContainer(opts)

	return err
}

// CopyOutput copies the output of the container to `w` and blocks until
// completion
func (c *Container) CopyOutput() error {

	// TODO(pwaller): at some point move this on to 'c' for configurability?
	w := os.Stderr
	// Blocks until stream closed
	return c.client.AttachToContainer(docker.AttachToContainerOptions{
		Container:    c.container.ID,
		OutputStream: w,
		ErrorStream:  w,
		Logs:         true,
		Stdout:       true,
		Stderr:       true,
		Stream:       true,
	})
}

// :todo(drj): May want to return errors for truly broken containers (timeout).
// Poll for the program inside the container being ready to accept connections
// Returns `true` for success and `false` for failure.
func (c *Container) AwaitListening() bool {

	for _, port := range c.container.NetworkSettings.PortMappingAPI() {
		url := fmt.Sprint("http://", port.IP, ":", port.PublicPort, "/")
		for {
			response, err := http.Get(url)
			if response != nil && response.Body != nil {
				response.Body.Close()
			}
			if err == nil && response.StatusCode == http.StatusOK {
				return true
			}
			time.Sleep(50 * time.Millisecond)

			select {
			case <-c.Closing.Barrier():
				// If the container has closed, cease waiting
				return false
			default:
			}
		}
	}

	return false
}

// Start the container (and notify it if c.Closing falls)
func (c *Container) Start() error {
	hc := &docker.HostConfig{
		PublishAllPorts: true,
	}
	err := c.client.StartContainer(c.container.ID, hc)
	if err != nil {
		return err
	}

	// Load container.NetworkSettings
	c.container, err = c.client.InspectContainer(c.container.ID)
	if err != nil {
		return err
	}

	// Listen on the Closing barrier and send a kill to the container if it
	// falls.
	c.wg.Add(1)
	go func() {
		defer c.wg.Done()
		<-c.Closing.Barrier()
		// If the container is signaled to close, send a kill signal
		err := c.client.KillContainer(docker.KillContainerOptions{
			ID: c.container.ID,
		})
		if err == nil {
			return
		}
		switch err := err.(type) {
		case *docker.NoSuchContainer:
			// The container already went away, who cares.
			return
		default:
			log.Println("Killing container failed:", c.container.ID, err)
		}
	}()
	return nil
}

// Wait until container exits
func (c *Container) Wait() (int, error) {
	return c.client.WaitContainer(c.container.ID)
}

// Internal function for raising an error.
func (c *Container) err(err error) {
	c.errorsW <- err
	c.Closing.Fall()
}

// Manage the whole lifecycle of the container in response to a request to
// start it.
func (c *Container) Run(event UpdateEvent) (int, error) {

	defer c.Closing.Fall()
	defer close(c.errorsW)

	switch event.Source.Type {
	case DockerPull:
		err := c.Pull(event)
		if err != nil {
			return -2, err
		}
	case BuildTarballContent, BuildCwd:
		err := c.Build(event)
		if err != nil {
			return -2, err
		}
	}

	err := c.Create(event.Source)
	if err != nil {
		return -1, err
	}
	defer c.Delete()

	c.wg.Add(1)
	go func() {
		defer c.wg.Done()
		err := c.CopyOutput()
		if err != nil {
			c.err(err)
		}
	}()

	err = c.Start()
	if err != nil {
		return -1, err
	}

	go func() {
		if !c.AwaitListening() {
			c.Failed.Fall()
			return
		}
		c.Ready.Fall()
	}()

	return c.Wait()
}

func (c *Container) Delete() {
	err := c.client.RemoveContainer(docker.RemoveContainerOptions{
		ID:            c.container.ID,
		RemoveVolumes: true,
		Force:         true,
	})
	if err != nil {
		log.Println("Warn: failed to delete container:", err)
	}
}
<|endoftext|>


<commit_before>package models

import (
	"fmt"
	"github.com/roadrunners/go-url-shortener/app/db"
	"github.com/roadrunners/go-url-shortener/app/models/key"
	"github.com/roadrunners/go-url-shortener/app/redis"
	"github.com/robfig/revel"
	"os"
	"os/signal"
)

const totalBufferedPulls = 1000

var (
	pulls chan *ShortUrl
)

type ShortUrl struct {
	Id   int64  `db:"id" json:"-"`
	Slug string `db:"-" json:"slug"`
	URL  string `db:"url" json:"url"`
}

func (s ShortUrl) String() string {
	return fmt.Sprintf("(%s, %s)", s.Slug, s.URL)
}

func (s *ShortUrl) pull() {
	k := fmt.Sprintf("shorturl:%d:url", s.Id)
	revel.INFO.Printf("Populating cache %v: %v", k, s.URL)
	err := redis.Client.Set(k, []byte(s.URL))
	if err != nil {
		revel.ERROR.Fatal("Could not push short url to redis")
	}
}

func ShortUrlById(id int64) (*ShortUrl, error) {
	v, err := db.DbMap.Get(ShortUrl{}, id)
	if err != nil || v == nil {
		return nil, err
	}
	s := v.(*ShortUrl)
	s.Slug = key.GenKey(s.Id)
	return s, nil
}

func ShortUrlBySlug(slug string) (*ShortUrl, error) {
	id := key.GenId(slug)
	return ShortUrlById(id)
}

func CachedShortUrlBySlug(slug string) (*ShortUrl, error) {
	id := key.GenId(slug)
	k := fmt.Sprintf("shorturl:%d:url", id)
	data, err := redis.Client.Get(k)
	if err == nil {
		s := ShortUrl{Id: id, Slug: slug, URL: string(data)}
		return &s, nil
	}
	revel.WARN.Printf("Missed cache for slug %v (id %v, key %v)", slug, id, k)
	s, err := ShortUrlById(id)
	if s != nil && err == nil {
		pulls <- s
	}
	return s, err
}

func ShortUrlCreate(url string) (*ShortUrl, error) {
	s := &ShortUrl{URL: url}
	if err := db.DbMap.Insert(s); err != nil {
		return nil, err
	}
	s.Slug = key.GenKey(s.Id)
	pulls <- s
	return s, nil
}

func pullMonitor() chan *ShortUrl {
	pulls := make(chan *ShortUrl, totalBufferedPulls)
	quit := make(chan os.Signal, 1)
	signal.Notify(quit, os.Interrupt, os.Kill)
	go func() {
		for {
			select {
			case s := <-pulls:
				s.pull()
			case <-quit:
				revel.WARN.Print("Stopping pull monitor")
				close(pulls)
				return
			}
		}
	}()
	return pulls
}

func shortUrlInit() {
	pulls = pullMonitor()
}

func init() {
	revel.OnAppStart(shortUrlInit)
}
<commit_msg>cleanup<commit_after>package models

import (
	"fmt"
	"github.com/roadrunners/go-url-shortener/app/db"
	"github.com/roadrunners/go-url-shortener/app/models/key"
	"github.com/roadrunners/go-url-shortener/app/redis"
	"github.com/robfig/revel"
	"os"
	"os/signal"
)

const redisBuffer = 10000

var (
	sendToRedis chan *ShortUrl
)

type ShortUrl struct {
	Id   int64  `db:"id" json:"-"`
	Slug string `db:"-" json:"slug"`
	URL  string `db:"url" json:"url"`
}

func (s ShortUrl) String() string {
	return fmt.Sprintf("(%s, %s)", s.Slug, s.URL)
}

func (s *ShortUrl) pull() {
	k := fmt.Sprintf("shorturl:%d:url", s.Id)
	revel.INFO.Printf("Populating cache %v: %v", k, s.URL)
	err := redis.Client.Set(k, []byte(s.URL))
	if err != nil {
		revel.ERROR.Fatal("Could not push short url to redis")
	}
}

func ShortUrlById(id int64) (*ShortUrl, error) {
	v, err := db.DbMap.Get(ShortUrl{}, id)
	if err != nil || v == nil {
		return nil, err
	}
	s := v.(*ShortUrl)
	s.Slug = key.GenKey(s.Id)
	return s, nil
}

func ShortUrlBySlug(slug string) (*ShortUrl, error) {
	id := key.GenId(slug)
	return ShortUrlById(id)
}

func CachedShortUrlBySlug(slug string) (*ShortUrl, error) {
	id := key.GenId(slug)
	k := fmt.Sprintf("shorturl:%d:url", id)
	data, err := redis.Client.Get(k)
	if err == nil {
		s := ShortUrl{Id: id, Slug: slug, URL: string(data)}
		return &s, nil
	}
	revel.WARN.Printf("Missed cache for slug %v (id %v, key %v)", slug, id, k)
	s, err := ShortUrlById(id)
	if s != nil && err == nil {
		sendToRedis <- s
	}
	return s, err
}

func ShortUrlCreate(url string) (*ShortUrl, error) {
	s := &ShortUrl{URL: url}
	if err := db.DbMap.Insert(s); err != nil {
		return nil, err
	}
	s.Slug = key.GenKey(s.Id)
	sendToRedis <- s
	return s, nil
}

func redisMonitor() chan *ShortUrl {
	sendToRedis := make(chan *ShortUrl, redisBuffer)
	quit := make(chan os.Signal, 1)
	signal.Notify(quit, os.Interrupt, os.Kill)
	go func() {
		for {
			select {
			case s := <-sendToRedis:
				s.pull()
			case <-quit:
				revel.WARN.Print("Stopping redis monitor")
				close(sendToRedis)
				return
			}
		}
	}()
	return sendToRedis
}

func shortUrlInit() {
	sendToRedis = redisMonitor()
}

func init() {
	revel.OnAppStart(shortUrlInit)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/pem"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"

	"crypto/rsa"
	"crypto/x509"

	"code.google.com/p/go.crypto/ssh"

	"code.google.com/p/gopass"
	"github.com/codegangsta/cli"
)

func decryptPEM(pemblock *pem.Block) ([]byte, error) {
	fmt.Fprintf(os.Stderr, "Enter passphrase for ~/.ssh/id_rsa: ")
	passwd, _ := gopass.GetPass("")
	fmt.Fprintf(os.Stderr, "\n")
	decryptedBytes, err := x509.DecryptPEMBlock(pemblock, []byte(passwd))
	panic_the_err(err)
	pemBytes := pem.Block{
		Type:  "RSA PRIVATE KEY",
		Bytes: decryptedBytes,
	}
	decryptedPEM := pem.EncodeToMemory(&pemBytes)
	return decryptedPEM, nil
}

func main() {
	app := cli.NewApp()
	app.Name = "Credulous"
	app.Usage = "Use it!"
	app.Version = "0.1.2"

	app.Commands = []cli.Command{
		{
			Name:  "save",
			Usage: "Save AWS credentials for a file.",
			Flags: []cli.Flag{
				cli.StringFlag{"key, k", "", "SSH public key"},
			},
			Action: func(c *cli.Context) {
				var pubkeyFile string
				if c.String("key") == "" {
					pubkeyFile = filepath.Join(os.Getenv("HOME"), "/.ssh/id_rsa.pub")
				} else {
					pubkeyFile = c.String("key")
				}

				AWSAccessKeyId := os.Getenv("AWS_ACCESS_KEY_ID")
				AWSSecretAccessKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
				if AWSAccessKeyId == "" || AWSSecretAccessKey == "" {
					fmt.Println("Can't save, no credentials in the environment")
					os.Exit(1)
				}
				username, _ := getAWSUsername(AWSAccessKeyId, AWSSecretAccessKey)
				alias, _ := getAWSAccountAlias(AWSAccessKeyId, AWSSecretAccessKey)
				fmt.Printf("saving credentials for %s@%s\n", username, alias)
				pubkeyString, err := ioutil.ReadFile(pubkeyFile)
				panic_the_err(err)
				pubkey, _, _, _, err := ssh.ParseAuthorizedKey([]byte(pubkeyString))
				SaveCredentials(username, alias, AWSAccessKeyId, AWSSecretAccessKey, pubkey)
			},
		},
		{
			Name:  "source",
			Usage: "Source AWS credentials from a file.",
			Flags: []cli.Flag{
				cli.StringFlag{"account, a", "", "AWS Account alias or id"},
				cli.StringFlag{"key, k", "", "SSH private key"},
				cli.StringFlag{"username, u", "", "IAM User"},
			},
			Action: func(c *cli.Context) {
				var privkeyFile string
				if c.String("key") == "" {
					privkeyFile = filepath.Join(os.Getenv("HOME"), "/.ssh/id_rsa")
				} else {
					privkeyFile = c.String("key")
				}
				tmp, err := ioutil.ReadFile(privkeyFile)
				panic_the_err(err)
				pemblock, _ := pem.Decode([]byte(tmp))
				if x509.IsEncryptedPEMBlock(pemblock) {
					tmp, err = decryptPEM(pemblock)
					panic_the_err(err)
				} else {
					log.Print("WARNING: Your private SSH key has no passphrase!")
				}
				key, err := ssh.ParseRawPrivateKey(tmp)
				panic_the_err(err)
				privateKey := key.(*rsa.PrivateKey)
				cred := RetrieveCredentials(c.String("account"), c.String("username"), privateKey)
				cred.Display(os.Stdout)
			},
		},
		{
			Name:  "display",
			Usage: "Display loaded AWS credentials",
			Action: func(c *cli.Context) {
				AWSAccessKeyId := os.Getenv("AWS_ACCESS_KEY_ID")
				AWSSecretAccessKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
				fmt.Printf("AWS_ACCESS_KEY_ID: %s\n", AWSAccessKeyId)
				fmt.Printf("AWS_SECRET_ACCESS_KEY: %s\n", AWSSecretAccessKey)
			},
		},
	}

	app.Run(os.Args)
}
<commit_msg>Have decryptPEM handle different files, and make bits more idiomatic<commit_after>package main

import (
	"encoding/pem"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"

	"crypto/rsa"
	"crypto/x509"

	"code.google.com/p/go.crypto/ssh"

	"code.google.com/p/gopass"
	"github.com/codegangsta/cli"
)

func decryptPEM(pemblock *pem.Block, filename string) ([]byte, error) {
	var err error
	if _, err = fmt.Fprintf(os.Stderr, "Enter passphrase for %s: ", filename); err != nil {
		return []byte(""), err
	}

	// we already emit the prompt to stderr; GetPass only emits to stdout
	var passwd string
	if passwd, err = gopass.GetPass(""); err != nil {
		return []byte(""), err
	}
	// Since the trailing CR isn't echoed, we need to fill it in
	fmt.Fprint(os.Stderr, "\n")

	var decryptedBytes []byte
	if decryptedBytes, err = x509.DecryptPEMBlock(pemblock, []byte(passwd)); err != nil {
		return []byte(""), err
	}

	pemBytes := pem.Block{
		Type:  "RSA PRIVATE KEY",
		Bytes: decryptedBytes,
	}
	decryptedPEM := pem.EncodeToMemory(&pemBytes)
	return decryptedPEM, nil
}

func main() {
	app := cli.NewApp()
	app.Name = "Credulous"
	app.Usage = "Use it!"
	app.Version = "0.1.2"

	app.Commands = []cli.Command{
		{
			Name:  "save",
			Usage: "Save AWS credentials for a file.",
			Flags: []cli.Flag{
				cli.StringFlag{"key, k", "", "SSH public key"},
			},
			Action: func(c *cli.Context) {
				var pubkeyFile string
				if c.String("key") == "" {
					pubkeyFile = filepath.Join(os.Getenv("HOME"), "/.ssh/id_rsa.pub")
				} else {
					pubkeyFile = c.String("key")
				}

				AWSAccessKeyId := os.Getenv("AWS_ACCESS_KEY_ID")
				AWSSecretAccessKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
				if AWSAccessKeyId == "" || AWSSecretAccessKey == "" {
					fmt.Println("Can't save, no credentials in the environment")
					os.Exit(1)
				}
				username, _ := getAWSUsername(AWSAccessKeyId, AWSSecretAccessKey)
				alias, _ := getAWSAccountAlias(AWSAccessKeyId, AWSSecretAccessKey)
				fmt.Printf("saving credentials for %s@%s\n", username, alias)
				pubkeyString, err := ioutil.ReadFile(pubkeyFile)
				panic_the_err(err)
				pubkey, _, _, _, err := ssh.ParseAuthorizedKey([]byte(pubkeyString))
				SaveCredentials(username, alias, AWSAccessKeyId, AWSSecretAccessKey, pubkey)
			},
		},
		{
			Name:  "source",
			Usage: "Source AWS credentials from a file.",
			Flags: []cli.Flag{
				cli.StringFlag{"account, a", "", "AWS Account alias or id"},
				cli.StringFlag{"key, k", "", "SSH private key"},
				cli.StringFlag{"username, u", "", "IAM User"},
			},
			Action: func(c *cli.Context) {
				var privkeyFile string
				var tmp []byte
				var err error

				if c.String("key") == "" {
					privkeyFile = filepath.Join(os.Getenv("HOME"), "/.ssh/id_rsa")
				} else {
					privkeyFile = c.String("key")
				}

				if tmp, err = ioutil.ReadFile(privkeyFile); err != nil {
					panic_the_err(err)
				}

				pemblock, _ := pem.Decode([]byte(tmp))
				if x509.IsEncryptedPEMBlock(pemblock) {
					if tmp, err = decryptPEM(pemblock, privkeyFile); err != nil {
						panic_the_err(err)
					}
				} else {
					log.Print("WARNING: Your private SSH key has no passphrase!")
				}

				key, err := ssh.ParseRawPrivateKey(tmp)
				if err != nil {
					panic_the_err(err)
				}
				privateKey := key.(*rsa.PrivateKey)
				cred := RetrieveCredentials(c.String("account"), c.String("username"), privateKey)
				cred.Display(os.Stdout)
			},
		},
		{
			Name:  "display",
			Usage: "Display loaded AWS credentials",
			Action: func(c *cli.Context) {
				AWSAccessKeyId := os.Getenv("AWS_ACCESS_KEY_ID")
				AWSSecretAccessKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
				fmt.Printf("AWS_ACCESS_KEY_ID: %s\n", AWSAccessKeyId)
				fmt.Printf("AWS_SECRET_ACCESS_KEY: %s\n", AWSSecretAccessKey)
			},
		},
	}

	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package app

import (
	"io/ioutil"
	"log"
	"net"
	"time"

	"golang.org/x/crypto/ssh"
)

const MAX_RETRIES_SERVER = 60 * 60
const MAX_RETRIES_CLIENT = 60 * 60

type GetServerReq struct {
	reply chan *ssh.Client
}

type ConnectionDone struct {
	client *ssh.Client
	err    error
}

// Conn wraps a net.Conn, and sets a deadline for every read
// and write operation.
type Conn struct {
	net.Conn
	ReadTimeout  time.Duration
	WriteTimeout time.Duration
}

func (c *Conn) Read(b []byte) (int, error) {
	err := c.Conn.SetReadDeadline(time.Now().Add(c.ReadTimeout))
	if err != nil {
		return 0, err
	}
	return c.Conn.Read(b)
}

func (c *Conn) Write(b []byte) (int, error) {
	err := c.Conn.SetWriteDeadline(time.Now().Add(c.WriteTimeout))
	if err != nil {
		return 0, err
	}
	return c.Conn.Write(b)
}

func directConnect(network, addr string, timeout time.Duration) (net.Conn, error) {
	conn, err := net.DialTimeout(network, addr, timeout)
	if err != nil {
		return nil, err
	}

	return &Conn{conn, timeout, timeout}, nil
}

func dialSSH(info *SSHTunnel, config *ssh.ClientConfig, proxyCommand string) (*ssh.Client, error) {
	var conn net.Conn
	var err error

	if proxyCommand == "" {
		conn, err = directConnect(`tcp`, info.Address, 5*time.Second)
	} else {
		conn, err = connectProxy(proxyCommand, info.Address)
	}
	if err != nil {
		return nil, err
	}
	c, chans, reqs, err := ssh.NewClientConn(conn, info.Address, config)
	if err != nil {
		return nil, err
	}
	return ssh.NewClient(c, chans, reqs), nil
}

func acceptAllHostKeys(hostname string, remote net.Addr, key ssh.PublicKey) error {
	return nil
}

func connectSSH(info PathInfo, resp chan<- *ssh.Client, progress chan<- ProgressCmd) {
	var err error
	log.Printf("SSH-connecting to %s\n", info.SSHTunnel.Address)

	progress <- ProgressCmd{"connection_start", nil}
	sshKey := []byte(info.SSHTunnel.SSHKeyContents)
	if info.SSHTunnel.SSHKeyFileName != "" {
		sshKey, err = ioutil.ReadFile(info.SSHTunnel.SSHKeyFileName)
		if err != nil {
			progress <- ProgressCmd{"connection_failed", "Failed to read SSH key"}
			resp <- nil
			return
		}
	}

	key, err := ssh.ParsePrivateKey(sshKey)
	if err != nil {
		progress <- ProgressCmd{"connection_failed", "Failed to parse SSH key"}
		resp <- nil
		return
	}

	config := &ssh.ClientConfig{
		User: info.SSHTunnel.Username,
		Auth: []ssh.AuthMethod{
			ssh.PublicKeys(key),
		},
		HostKeyCallback: acceptAllHostKeys,
		Timeout:         10 * time.Second,
	}

	currentRetriesServer := 0
	var sshClientConn *ssh.Client

	for {
		progress <- ProgressCmd{"connection_try", nil}
		if sshClientConn, err = dialSSH(info.SSHTunnel, config, proxyCommand); err == nil {
			break
		}

		currentRetriesServer++
		log.Printf("SSH Connection failed %s: %s\n", info.SSHTunnel.Address, err.Error())

		if currentRetriesServer < (MAX_RETRIES_SERVER / 1) {
			log.Println(`Retry...`)
			progress <- ProgressCmd{"connection_retry", nil}
			time.Sleep(1 * time.Second)
		} else {
			progress <- ProgressCmd{"connection_failed", "Connection retry limit reached"}
			resp <- nil
			return
		}
	}
	progress <- ProgressCmd{"connection_established", nil}

	runBootstrap(sshClientConn, info, progress)

	if info.SSHTunnel.Run != nil {
		session, _ := sshClientConn.NewSession()

		modes := ssh.TerminalModes{
			ssh.ECHO: 0,
		}

		if err := session.RequestPty("xterm", 80, 40, modes); err != nil {
			log.Fatalf("request for pseudo terminal failed: %s", err)
		}

		session.Start(info.SSHTunnel.Run.Command)
		time.Sleep(500 * time.Millisecond)
	}
	log.Printf("SSH-connection OK. Waiting for %s to be ready...\n", info.Backend.Address)

	progress <- ProgressCmd{"waiting_backend", nil}
	currentRetriesClient := 0
	for {
		if conn, err := sshClientConn.Dial("tcp", info.Backend.Address); err == nil {
			conn.Close()
			break
		}
		currentRetriesClient++

		if currentRetriesClient < (MAX_RETRIES_CLIENT / 5) {
			log.Println(`Retry...`)
			progress <- ProgressCmd{"waiting_backend_retry", nil}
			time.Sleep(5 * time.Second)
		} else {
			progress <- ProgressCmd{"waiting_backend_timeout", "Connection retry limit reached"}
			resp <- nil
			return
		}
	}

	progress <- ProgressCmd{"connection_success", nil}
	resp <- sshClientConn
}
<commit_msg>Added more logging for SSH client connections<commit_after>package app

import (
	"io/ioutil"
	"log"
	"net"
	"time"

	"golang.org/x/crypto/ssh"
)

const MAX_RETRIES_SERVER = 60 * 60
const MAX_RETRIES_CLIENT = 60 * 60

type GetServerReq struct {
	reply chan *ssh.Client
}

type ConnectionDone struct {
	client *ssh.Client
	err    error
}

// Conn wraps a net.Conn, and sets a deadline for every read
// and write operation.
type Conn struct {
	net.Conn
	ReadTimeout  time.Duration
	WriteTimeout time.Duration
}

func (c *Conn) Read(b []byte) (int, error) {
	err := c.Conn.SetReadDeadline(time.Now().Add(c.ReadTimeout))
	if err != nil {
		return 0, err
	}
	return c.Conn.Read(b)
}

func (c *Conn) Write(b []byte) (int, error) {
	err := c.Conn.SetWriteDeadline(time.Now().Add(c.WriteTimeout))
	if err != nil {
		return 0, err
	}
	return c.Conn.Write(b)
}

func directConnect(network, addr string, timeout time.Duration) (net.Conn, error) {
	conn, err := net.DialTimeout(network, addr, timeout)
	if err != nil {
		return nil, err
	}

	return &Conn{conn, timeout, timeout}, nil
}

func dialSSH(info *SSHTunnel, config *ssh.ClientConfig, proxyCommand string) (*ssh.Client, error) {
	var conn net.Conn
	var err error

	if proxyCommand == "" {
		conn, err = directConnect(`tcp`, info.Address, 5*time.Second)
	} else {
		conn, err = connectProxy(proxyCommand, info.Address)
	}
	if err != nil {
		return nil, err
	}
	c, chans, reqs, err := ssh.NewClientConn(conn, info.Address, config)
	if err != nil {
		return nil, err
	}
	return ssh.NewClient(c, chans, reqs), nil
}

func acceptAllHostKeys(hostname string, remote net.Addr, key ssh.PublicKey) error {
	return nil
}

func connectSSH(info PathInfo, resp chan<- *ssh.Client, progress chan<- ProgressCmd) {
	var err error
	log.Printf("SSH-connecting to %s\n", info.SSHTunnel.Address)

	progress <- ProgressCmd{"connection_start", nil}
	sshKey := []byte(info.SSHTunnel.SSHKeyContents)
	if info.SSHTunnel.SSHKeyFileName != "" {
		sshKey, err = ioutil.ReadFile(info.SSHTunnel.SSHKeyFileName)
		if err != nil {
			progress <- ProgressCmd{"connection_failed", "Failed to read SSH key"}
			resp <- nil
			return
		}
	}

	key, err := ssh.ParsePrivateKey(sshKey)
	if err != nil {
		progress <- ProgressCmd{"connection_failed", "Failed to parse SSH key"}
		resp <- nil
		return
	}

	config := &ssh.ClientConfig{
		User: info.SSHTunnel.Username,
		Auth: []ssh.AuthMethod{
			ssh.PublicKeys(key),
		},
		HostKeyCallback: acceptAllHostKeys,
		Timeout:         10 * time.Second,
	}

	currentRetriesServer := 0
	var sshClientConn *ssh.Client

	for {
		progress <- ProgressCmd{"connection_try", nil}
		if sshClientConn, err = dialSSH(info.SSHTunnel, config, proxyCommand); err == nil {
			break
		}

		currentRetriesServer++
		log.Printf("SSH Connection failed %s: %s\n", info.SSHTunnel.Address, err.Error())

		if currentRetriesServer < (MAX_RETRIES_SERVER / 1) {
			log.Println(`Retry...`)
			progress <- ProgressCmd{"connection_retry", nil}
			time.Sleep(1 * time.Second)
		} else {
			log.Println(`SSH connection limit reached. Aborting`)
			progress <- ProgressCmd{"connection_failed", "Connection retry limit reached"}
			resp <- nil
			return
		}
	}
	progress <- ProgressCmd{"connection_established", nil}

	runBootstrap(sshClientConn, info, progress)

	if info.SSHTunnel.Run != nil {
		session, _ := sshClientConn.NewSession()

		modes := ssh.TerminalModes{
			ssh.ECHO: 0,
		}

		if err := session.RequestPty("xterm", 80, 40, modes); err != nil {
			log.Fatalf("request for pseudo terminal failed: %s", err)
		}

		session.Start(info.SSHTunnel.Run.Command)
		time.Sleep(500 * time.Millisecond)
	}
	log.Printf("SSH-connection OK. Waiting for %s to be ready...\n", info.Backend.Address)

	progress <- ProgressCmd{"waiting_backend", nil}
	currentRetriesClient := 0
	for {
		log.Printf("Trying to connect to %s...\n", info.Backend.Address)
		if conn, err := sshClientConn.Dial("tcp", info.Backend.Address); err == nil {
			log.Printf("Connected to %s successfully!\n", info.Backend.Address)
			conn.Close()
			break
		}
		currentRetriesClient++

		if currentRetriesClient < (MAX_RETRIES_CLIENT / 5) {
			log.Printf("Failed to connect to %s - retrying...\n", info.Backend.Address)
			progress <- ProgressCmd{"waiting_backend_retry", nil}
			time.Sleep(5 * time.Second)
		} else {
			log.Printf("Connection limit to %s reached. Aborting.\n", info.Backend.Address)
			progress <- ProgressCmd{"waiting_backend_timeout", "Connection retry limit reached"}
			resp <- nil
			return
		}
	}

	progress <- ProgressCmd{"connection_success", nil}
	resp <- sshClientConn
}
<|endoftext|>


<commit_before>package main

import (
    "bufio"
    "encoding/csv"
    "flag"
    "fmt"
    "io"
    "os"
    "strconv"
    "strings"
)

const nilCommentRune = "TOTALLYNOTACOMMENTCHAR"

// TODO: Support values containing newlines
// TODO: Add support for specifying fields by field header name instead of just number

var inSep = flag.String("in-sep", ",", "Single character field separator used by your input")
var outSep = flag.String("out-sep", ",", "Single-character field separator to use when printing multiple columns in your output. Only valid if outputting something meant to be passed to cut/awk, and not a properly-formatted, quoted CSV file.")
var commentRune = flag.String("comment-char", nilCommentRune, "Single-character field separator to use when printing multiple columns in your output. Only valid if outputting something meant to be passed to cut/awk, and not a properly-formatted, quoted CSV file.")

var filename = flag.String("filename", "", "File to read from. If not specified, program reads from stdin.")

var fieldNumsRaw = flag.String("field-nums", "", "Comma-separated list of field indexes (starting at 0) to print to the command line")
var noRFC = flag.Bool("no-rfc", false, "Program defaults to printing RFC 4180-compliant, quoted, well-formatted CSV. If this flag is supplied, output is returned as a string naively joined by --out-sep. --no-rfc is assumed to imply you want to pass the output to naive tools like cut or awk, and in that case, it is recommended that you select an --out-sep that is unlikely to be in youc content, such as a pipe or a backtick.")

func main() {
    flag.Parse()

    var fieldNums []int

    // Identify the numbers you want to print out
    *fieldNumsRaw = strings.Trim(*fieldNumsRaw, ",")
    if *fieldNumsRaw != "" {
        for _, numStr := range strings.Split(*fieldNumsRaw, ",") {
            numStr := strings.TrimSpace(numStr)
            numInt, err := strconv.Atoi(numStr)
            if err != nil {
                panic(err)
            }
            fieldNums = append(fieldNums, numInt)
        }
    }

    csvWriter := csv.NewWriter(os.Stdout)
    csvWriter.Comma = getSeparator(*outSep)

    var reader *bufio.Reader
    if *filename == "" {
        reader = bufio.NewReader(os.Stdin)
    } else {
        f, err := os.Open(*filename)
        if err != nil {
            panic(err)
        }

        reader = bufio.NewReader(f)
    }
    for {
        line, err := reader.ReadString('\n')
        if err != nil {
            if err == io.EOF {
                break
            }
            panic(err)
        }

        if strings.TrimSpace(line) == "" {
            fmt.Println(line)
            continue
        }

        fields, err := processLine(line)
        if err != nil {
            if err == io.EOF {
                continue  // Since it's only one line, and not the whole file, it's a bogus EOF that happens when e.g. your line starts with a comment. File EOF is handled above.
            } else {
                panic(err)
            }
        }

        var toPrint []string
        if *fieldNumsRaw == "" {  // Print all fields
            for i, _ := range fields {
                toPrint = append(toPrint, fields[i])
            }
        } else {
            for _, num := range fieldNums {
                if num > len(fields) - 1 {
                    toPrint = append(toPrint, "")  // Append _something_, so printing columns out of order preserves the column index:value mapping in all columns. I.e., --field-nums=1,2,0 on a 2-column line will print "a,,b" so you always know your knew third column has certain content
                } else {
                    toPrint = append(toPrint, fields[num])
                }
            }
        }

        if *noRFC == false {
            csvWriter.Write(toPrint)
        } else {
            fmt.Println(strings.Join(toPrint, *outSep))
        }
    }
    if *noRFC == false {
        csvWriter.Flush()
    }
}

func processLine(line string) ([]string, error) {
    strReader := strings.NewReader(line)
    csvReader := csv.NewReader(strReader)
    csvReader.LazyQuotes = true
    if *commentRune != nilCommentRune {
        csvReader.Comment = ([]rune(*commentRune))[0]
    }

    sepString := *inSep

    csvReader.Comma = getSeparator(sepString)

    fields, err := csvReader.Read()
    if err != nil {
        if err == io.EOF {
            return nil, io.EOF
        } else {
            fmt.Println("Error in the following line:")
            fmt.Println(line)
            panic(err)
        }
    }

    return fields, nil
}

func getSeparator(sepString string) (sepRune rune) {
    sepString = `'` + sepString + `'`
    sepRunes, err := strconv.Unquote(sepString)
    if err != nil {
        if err.Error() == "invalid syntax" {  // Single quote was used as separator. No idea why someone would want this, but it doesn't hurt to support it
            sepString = `"` + sepString + `"`
            sepRunes, err = strconv.Unquote(sepString)
            if err != nil {
                panic(err)
            }

        } else {
            panic(err)
        }
    }
    sepRune = ([]rune(sepRunes))[0]

    return sepRune
}
<commit_msg>Now supports records with newlines inside values. Does not support values with newlines that also have quotes in the value.<commit_after>package main

import (
    "bufio"
    "encoding/csv"
    "errors"
    "flag"
    "fmt"
    "io"
    "os"
    "strconv"
    "strings"
)

const nilCommentRune = "TOTALLYNOTACOMMENTCHAR"

// TODO: Do I need to have processLine() at all? Can I just use csvReader.Read on the buffer?
// TODO: Add support for specifying fields by field header name instead of just number
// TODO: check ReadRuneFromString instead of existing technique

var inSep = flag.String("in-sep", ",", "Single character field separator used by your input")
var outSep = flag.String("out-sep", ",", "Single-character field separator to use when printing multiple columns in your output. Only valid if outputting something meant to be passed to cut/awk, and not a properly-formatted, quoted CSV file.")
var commentRune = flag.String("comment-char", nilCommentRune, "Single-character field separator to use when printing multiple columns in your output. Only valid if outputting something meant to be passed to cut/awk, and not a properly-formatted, quoted CSV file.")

var filename = flag.String("filename", "", "File to read from. If not specified, program reads from stdin.")

var fieldNumsRaw = flag.String("field-nums", "", "Comma-separated list of field indexes (starting at 0) to print to the command line")
var noRFC = flag.Bool("no-rfc", false, "Program defaults to printing RFC 4180-compliant, quoted, well-formatted CSV. If this flag is supplied, output is returned as a string naively joined by --out-sep. --no-rfc is assumed to imply you want to pass the output to naive tools like cut or awk, and in that case, it is recommended that you select an --out-sep that is unlikely to be in youc content, such as a pipe or a backtick.")

func main() {
    flag.Parse()

    var fieldNums []int

    // Identify the numbers you want to print out
    *fieldNumsRaw = strings.Trim(*fieldNumsRaw, ",")
    if *fieldNumsRaw != "" {
        for _, numStr := range strings.Split(*fieldNumsRaw, ",") {
            numStr := strings.TrimSpace(numStr)
            numInt, err := strconv.Atoi(numStr)
            if err != nil {
                panic(err)
            }
            fieldNums = append(fieldNums, numInt)
        }
    }

    csvWriter := csv.NewWriter(os.Stdout)
    csvWriter.Comma = getSeparator(*outSep)

    var reader *bufio.Reader
    if *filename == "" {
        reader = bufio.NewReader(os.Stdin)
    } else {
        f, err := os.Open(*filename)
        if err != nil {
            panic(err)
        }

        reader = bufio.NewReader(f)
    }

    for {
        line, err := reader.ReadString('\n')
        if err != nil {
            if err == io.EOF {
                break
            }
            panic(err)
        }

        if strings.TrimSpace(line) == "" {
            fmt.Println(line)
            continue
        }

        fields, err := processLine(line, false)
        if err != nil {
            if err == io.EOF {
                continue  // Since it's only one line, and not the whole file, it's a bogus EOF that happens when e.g. your line starts with a comment. File EOF is handled above.
            } else if err.Error() == "Incomplete value" {  // Handles values containing newlines
                moreLine, err := reader.ReadString('\n')
                if err != nil {
                    panic(err)
                }
                line += moreLine
                //moreLine, err = reader.ReadString('\n')
                //if err != nil {
                //    panic(err)
                //}
                //line += moreLine
                fields, err = processLine(line, false)
            } else {
                panic(err)
            }
        }

        var toPrint []string
        if *fieldNumsRaw == "" {  // Print all fields
            for i, _ := range fields {
                toPrint = append(toPrint, fields[i])
            }
        } else {
            for _, num := range fieldNums {
                if num > len(fields) - 1 {
                    toPrint = append(toPrint, "")  // Append _something_, so printing columns out of order preserves the column index:value mapping in all columns. I.e., --field-nums=1,2,0 on a 2-column line will print "a,,b" so you always know your knew third column has certain content
                } else {
                    toPrint = append(toPrint, fields[num])
                }
            }
        }

        if *noRFC == false {
            csvWriter.Write(toPrint)
        } else {
            fmt.Println(strings.Join(toPrint, *outSep))
        }
    }
    if *noRFC == false {
        csvWriter.Flush()
    }
}

func processLine(line string, lazyQuotes bool) ([]string, error) {
    strReader := strings.NewReader(line)
    csvReader := csv.NewReader(strReader)
    csvReader.LazyQuotes = lazyQuotes
    csvReader.TrailingComma = true
    if *commentRune != nilCommentRune {
        csvReader.Comment = ([]rune(*commentRune))[0]
    }

    sepString := *inSep
    csvReader.Comma = getSeparator(sepString)

    fields, err := csvReader.Read()
    if err != nil {
        if err == io.EOF {
            return nil, io.EOF
        } else if strings.Contains(err.Error(), "in non-quoted-field") {  // Field isn't quoted, but contains quotes
            return processLine(line, true)
        } else if strings.Contains(err.Error(), `extraneous " in field`) {  // Field is quoted and ends with a newline that's part of the value
            return nil, errors.New("Incomplete value")
        } else {
            fmt.Println("Error in the following line:")
            fmt.Println(line)
            panic(err)
        }
    }

    return fields, nil
}

func getSeparator(sepString string) (sepRune rune) {
    sepString = `'` + sepString + `'`
    sepRunes, err := strconv.Unquote(sepString)
    if err != nil {
        if err.Error() == "invalid syntax" {  // Single quote was used as separator. No idea why someone would want this, but it doesn't hurt to support it
            sepString = `"` + sepString + `"`
            sepRunes, err = strconv.Unquote(sepString)
            if err != nil {
                panic(err)
            }

        } else {
            panic(err)
        }
    }
    sepRune = ([]rune(sepRunes))[0]

    return sepRune
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"
)

var (
	Namespace   = "dashboard"
	ConsulAddr  = "127.0.0.1:8500"
	Version     string
	ExtAssetDir string
	Nodes       []Node
	mutex       sync.RWMutex
)

type KVPair struct {
	Key         string
	CreateIndex int64
	ModifyIndex int64
	LockIndex   int64
	Flags       int64
	Value       []byte
}

type Status int64

const (
	Success Status = iota
	Warning
	Danger
	Info
)

func (s Status) MarshalText() ([]byte, error) {
	if s <= Danger {
		return []byte(strings.ToLower(s.String())), nil
	} else {
		return []byte(strconv.FormatInt(int64(s), 10)), nil
	}
}

type Item struct {
	Category  string `json:"category"`
	Node      string `json:"node"`
	Address   string `json:"address"`
	Timestamp string `json:"timestamp"`
	Status    Status `json:"status"`
	Key       string `json:"key"`
	Data      string `json:"data"`
}

func (kv *KVPair) NewItem() Item {
	item := Item{
		Data:      string(kv.Value),
		Timestamp: time.Unix(kv.Flags/1000, 0).Format("2006-01-02 15:04:05 -0700"),
	}
	item.Status = Status(kv.Flags % 1000)

	// kv.Key : {namespace}/{category}/{node}/{key}
	path := strings.Split(kv.Key, "/")
	item.Category = path[1]
	if len(path) >= 3 {
		item.Node = path[2]
	}
	if len(path) >= 4 {
		item.Key = path[3]
	}
	return item
}

type Node struct {
	Node    string
	Address string
}

func main() {
	var (
		port        int
		showVersion bool
		trigger     string
	)
	flag.StringVar(&Namespace, "namespace", Namespace, "Consul kv top level key name. (/v1/kv/{namespace}/...)")
	flag.IntVar(&port, "port", 3000, "http listen port")
	flag.StringVar(&ExtAssetDir, "asset", "", "Serve files located in /assets from local directory. If not specified, use built-in asset.")
	flag.BoolVar(&showVersion, "v", false, "show vesion")
	flag.BoolVar(&showVersion, "version", false, "show vesion")
	flag.StringVar(&trigger, "trigger", "", "trigger command")
	flag.Parse()

	if showVersion {
		fmt.Println("consul-kv-dashboard: version:", Version)
		return
	}

	mux := http.NewServeMux()
	mux.HandleFunc("/", makeGzipHandler(indexPage))
	mux.HandleFunc("/api/", makeGzipHandler(kvApiProxy))

	if ExtAssetDir != "" {
		mux.Handle("/assets/",
			http.StripPrefix("/assets/", http.FileServer(http.Dir(ExtAssetDir))))
	} else {
		mux.Handle("/assets/",
			http.FileServer(NewAssetFileSystem("/assets/")))
	}
	http.Handle("/", mux)

	log.Println("listen port:", port)
	log.Println("asset directory:", ExtAssetDir)
	log.Println("namespace:", Namespace)
	if trigger != "" {
		log.Println("trigger:", trigger)
		go watchForTrigger(trigger)
	}
	go updateNodeList()

	log.Fatal(http.ListenAndServe(":"+strconv.Itoa(port), nil))
}

func indexPage(w http.ResponseWriter, r *http.Request) {
	var (
		data []byte
		err  error
	)
	if ExtAssetDir == "" {
		data, err = Asset("index.html")
	} else {
		var f *os.File
		f, err = os.Open(ExtAssetDir + "/index.html")
		data, err = ioutil.ReadAll(f)
	}
	if err != nil {
		log.Println(err)
	}
	w.Header().Set("Content-Type", "text/html; charset=utf-8")
	fmt.Fprint(w, string(data))
}

func kvApiProxy(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	path := strings.TrimPrefix(r.URL.Path, "/api/")
	resp, _, err := callConsulAPI(
		"/v1/kv/" + Namespace + "/" + path + "?" + r.URL.RawQuery,
	)
	if err != nil {
		http.Error(w, fmt.Sprintf("%s", err), http.StatusInternalServerError)
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode == http.StatusNotFound {
		w.Header().Set("Content-Type", "application/json")
		http.Error(w, "[]", resp.StatusCode)
		return
	}
	if resp.StatusCode != http.StatusOK {
		http.Error(w, "", resp.StatusCode)
		io.Copy(w, resp.Body)
		return
	}
	// copy response header to client
	for name, value := range resp.Header {
		if strings.HasPrefix(name, "X-") || name == "Content-Type" {
			for _, v := range value {
				w.Header().Set(name, v)
			}
		}
	}

	// keys or values
	dec := json.NewDecoder(resp.Body)
	enc := json.NewEncoder(w)
	if _, t := r.Form["keys"]; t {
		var keys []string
		uniqKeyMap := make(map[string]bool)
		dec.Decode(&keys)
		for _, key := range keys {
			path := strings.Split(key, "/")
			if len(path) >= 2 {
				uniqKeyMap[path[1]] = true
			}
		}
		uniqKeys := make([]string, 0, len(uniqKeyMap))
		for key, _ := range uniqKeyMap {
			uniqKeys = append(uniqKeys, key)
		}
		sort.Strings(uniqKeys)
		enc.Encode(uniqKeys)
	} else {
		var kvps []*KVPair
		dec.Decode(&kvps)
		items := make([]Item, 0, len(kvps))
		for _, kv := range kvps {
			item := kv.NewItem()
			if itemInNodes(&item) {
				items = append(items, item)
			}
		}
		enc.Encode(items)
	}
}

func watchForTrigger(command string) {
	var index int64
	lastStatus := make(map[string]Status)
	for {
		resp, newIndex, err := callConsulAPI(
			"/v1/kv/" + Namespace + "/?recurse&wait=55s&index=" + strconv.FormatInt(index, 10),
		)
		if err != nil {
			log.Println("[error]", err)
			time.Sleep(10 * time.Second)
			continue
		}
		index = newIndex
		defer resp.Body.Close()
		var kvps []*KVPair
		dec := json.NewDecoder(resp.Body)
		dec.Decode(&kvps)

		currentItem := make(map[string]Item)
		for _, kv := range kvps {
			item := kv.NewItem()
			if !itemInNodes(&item) {
				continue
			}
			if _, exist := currentItem[item.Category]; !exist {
				currentItem[item.Category] = item
			} else if currentItem[item.Category].Status < item.Status {
				currentItem[item.Category] = item
			}
		}
		for category, item := range currentItem {
			if _, exist := lastStatus[category]; !exist {
				// at first initialze
				lastStatus[category] = item.Status
				log.Printf("[info] %s: status %s", category, item.Status)
			} else if lastStatus[category] != item.Status {
				// status changed. invoking trigger.
				log.Printf("[info] %s: status %s -> %s", category, lastStatus[category], item.Status)
				lastStatus[category] = item.Status
				b, _ := json.Marshal(item)
				err := invokePipe(command, bytes.NewReader(b))
				if err != nil {
					log.Println("[error]", err)
				}
			}
		}
		time.Sleep(1 * time.Second)
	}
}

func invokePipe(command string, src io.Reader) error {
	log.Println("[info] Invoking command:", command)
	cmd := exec.Command("sh", "-c", command)
	stdin, err := cmd.StdinPipe()
	if err != nil {
		return err
	}
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return err
	}
	stderr, err := cmd.StderrPipe()
	if err != nil {
		return err
	}

	err = cmd.Start()
	if err != nil {
		return err
	}
	cmdCh := make(chan error)
	// src => stdin
	go func() {
		_, err := io.Copy(stdin, src)
		if err != nil {
			cmdCh <- err
		}
		stdin.Close()
	}()
	// wait for command exit
	go func() {
		cmdCh <- cmd.Wait()
	}()
	go io.Copy(os.Stdout, stdout)
	go io.Copy(os.Stderr, stderr)

	cmdErr := <-cmdCh
	return cmdErr
}

func updateNodeList() {
	var index int64
	for {
		resp, newIndex, err := callConsulAPI(
			"/v1/catalog/nodes?index=" + strconv.FormatInt(index, 10) + "&wait=55s",
		)
		if err != nil {
			log.Println("[error]", err)
			time.Sleep(10 * time.Second)
			continue
		}
		index = newIndex
		defer resp.Body.Close()
		dec := json.NewDecoder(resp.Body)
		mutex.Lock()
		dec.Decode(&Nodes)
		log.Println("[info]", Nodes)
		mutex.Unlock()
		time.Sleep(1 * time.Second)
	}
}

func itemInNodes(item *Item) bool {
	mutex.RLock()
	defer mutex.RUnlock()
	for _, node := range Nodes {
		if item.Node == node.Node {
			item.Address = node.Address
			return true
		}
	}
	return false
}

func callConsulAPI(path string) (*http.Response, int64, error) {
	var index int64
	_url := "http://" + ConsulAddr + path
	log.Println("[info] get", _url)
	resp, err := http.Get(_url)
	if err != nil {
		log.Println("[error]", err)
		return nil, index, err
	}
	_indexes := resp.Header["X-Consul-Index"]
	if len(_indexes) > 0 {
		index, _ = strconv.ParseInt(_indexes[0], 10, 64)
	}
	return resp, index, nil
}
<commit_msg>fix info status to string<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"
)

var (
	Namespace   = "dashboard"
	ConsulAddr  = "127.0.0.1:8500"
	Version     string
	ExtAssetDir string
	Nodes       []Node
	mutex       sync.RWMutex
)

type KVPair struct {
	Key         string
	CreateIndex int64
	ModifyIndex int64
	LockIndex   int64
	Flags       int64
	Value       []byte
}

type Status int64

const (
	Success Status = iota
	Warning
	Danger
	Info
)

func (s Status) MarshalText() ([]byte, error) {
	if s <= Info {
		return []byte(strings.ToLower(s.String())), nil
	} else {
		return []byte(strconv.FormatInt(int64(s), 10)), nil
	}
}

type Item struct {
	Category  string `json:"category"`
	Node      string `json:"node"`
	Address   string `json:"address"`
	Timestamp string `json:"timestamp"`
	Status    Status `json:"status"`
	Key       string `json:"key"`
	Data      string `json:"data"`
}

func (kv *KVPair) NewItem() Item {
	item := Item{
		Data:      string(kv.Value),
		Timestamp: time.Unix(kv.Flags/1000, 0).Format("2006-01-02 15:04:05 -0700"),
	}
	item.Status = Status(kv.Flags % 1000)

	// kv.Key : {namespace}/{category}/{node}/{key}
	path := strings.Split(kv.Key, "/")
	item.Category = path[1]
	if len(path) >= 3 {
		item.Node = path[2]
	}
	if len(path) >= 4 {
		item.Key = path[3]
	}
	return item
}

type Node struct {
	Node    string
	Address string
}

func main() {
	var (
		port        int
		showVersion bool
		trigger     string
	)
	flag.StringVar(&Namespace, "namespace", Namespace, "Consul kv top level key name. (/v1/kv/{namespace}/...)")
	flag.IntVar(&port, "port", 3000, "http listen port")
	flag.StringVar(&ExtAssetDir, "asset", "", "Serve files located in /assets from local directory. If not specified, use built-in asset.")
	flag.BoolVar(&showVersion, "v", false, "show vesion")
	flag.BoolVar(&showVersion, "version", false, "show vesion")
	flag.StringVar(&trigger, "trigger", "", "trigger command")
	flag.Parse()

	if showVersion {
		fmt.Println("consul-kv-dashboard: version:", Version)
		return
	}

	mux := http.NewServeMux()
	mux.HandleFunc("/", makeGzipHandler(indexPage))
	mux.HandleFunc("/api/", makeGzipHandler(kvApiProxy))

	if ExtAssetDir != "" {
		mux.Handle("/assets/",
			http.StripPrefix("/assets/", http.FileServer(http.Dir(ExtAssetDir))))
	} else {
		mux.Handle("/assets/",
			http.FileServer(NewAssetFileSystem("/assets/")))
	}
	http.Handle("/", mux)

	log.Println("listen port:", port)
	log.Println("asset directory:", ExtAssetDir)
	log.Println("namespace:", Namespace)
	if trigger != "" {
		log.Println("trigger:", trigger)
		go watchForTrigger(trigger)
	}
	go updateNodeList()

	log.Fatal(http.ListenAndServe(":"+strconv.Itoa(port), nil))
}

func indexPage(w http.ResponseWriter, r *http.Request) {
	var (
		data []byte
		err  error
	)
	if ExtAssetDir == "" {
		data, err = Asset("index.html")
	} else {
		var f *os.File
		f, err = os.Open(ExtAssetDir + "/index.html")
		data, err = ioutil.ReadAll(f)
	}
	if err != nil {
		log.Println(err)
	}
	w.Header().Set("Content-Type", "text/html; charset=utf-8")
	fmt.Fprint(w, string(data))
}

func kvApiProxy(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	path := strings.TrimPrefix(r.URL.Path, "/api/")
	resp, _, err := callConsulAPI(
		"/v1/kv/" + Namespace + "/" + path + "?" + r.URL.RawQuery,
	)
	if err != nil {
		http.Error(w, fmt.Sprintf("%s", err), http.StatusInternalServerError)
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode == http.StatusNotFound {
		w.Header().Set("Content-Type", "application/json")
		http.Error(w, "[]", resp.StatusCode)
		return
	}
	if resp.StatusCode != http.StatusOK {
		http.Error(w, "", resp.StatusCode)
		io.Copy(w, resp.Body)
		return
	}
	// copy response header to client
	for name, value := range resp.Header {
		if strings.HasPrefix(name, "X-") || name == "Content-Type" {
			for _, v := range value {
				w.Header().Set(name, v)
			}
		}
	}

	// keys or values
	dec := json.NewDecoder(resp.Body)
	enc := json.NewEncoder(w)
	if _, t := r.Form["keys"]; t {
		var keys []string
		uniqKeyMap := make(map[string]bool)
		dec.Decode(&keys)
		for _, key := range keys {
			path := strings.Split(key, "/")
			if len(path) >= 2 {
				uniqKeyMap[path[1]] = true
			}
		}
		uniqKeys := make([]string, 0, len(uniqKeyMap))
		for key, _ := range uniqKeyMap {
			uniqKeys = append(uniqKeys, key)
		}
		sort.Strings(uniqKeys)
		enc.Encode(uniqKeys)
	} else {
		var kvps []*KVPair
		dec.Decode(&kvps)
		items := make([]Item, 0, len(kvps))
		for _, kv := range kvps {
			item := kv.NewItem()
			if itemInNodes(&item) {
				items = append(items, item)
			}
		}
		enc.Encode(items)
	}
}

func watchForTrigger(command string) {
	var index int64
	lastStatus := make(map[string]Status)
	for {
		resp, newIndex, err := callConsulAPI(
			"/v1/kv/" + Namespace + "/?recurse&wait=55s&index=" + strconv.FormatInt(index, 10),
		)
		if err != nil {
			log.Println("[error]", err)
			time.Sleep(10 * time.Second)
			continue
		}
		index = newIndex
		defer resp.Body.Close()
		var kvps []*KVPair
		dec := json.NewDecoder(resp.Body)
		dec.Decode(&kvps)

		currentItem := make(map[string]Item)
		for _, kv := range kvps {
			item := kv.NewItem()
			if !itemInNodes(&item) {
				continue
			}
			if _, exist := currentItem[item.Category]; !exist {
				currentItem[item.Category] = item
			} else if currentItem[item.Category].Status < item.Status {
				currentItem[item.Category] = item
			}
		}
		for category, item := range currentItem {
			if _, exist := lastStatus[category]; !exist {
				// at first initialze
				lastStatus[category] = item.Status
				log.Printf("[info] %s: status %s", category, item.Status)
			} else if lastStatus[category] != item.Status {
				// status changed. invoking trigger.
				log.Printf("[info] %s: status %s -> %s", category, lastStatus[category], item.Status)
				lastStatus[category] = item.Status
				b, _ := json.Marshal(item)
				err := invokePipe(command, bytes.NewReader(b))
				if err != nil {
					log.Println("[error]", err)
				}
			}
		}
		time.Sleep(1 * time.Second)
	}
}

func invokePipe(command string, src io.Reader) error {
	log.Println("[info] Invoking command:", command)
	cmd := exec.Command("sh", "-c", command)
	stdin, err := cmd.StdinPipe()
	if err != nil {
		return err
	}
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return err
	}
	stderr, err := cmd.StderrPipe()
	if err != nil {
		return err
	}

	err = cmd.Start()
	if err != nil {
		return err
	}
	cmdCh := make(chan error)
	// src => stdin
	go func() {
		_, err := io.Copy(stdin, src)
		if err != nil {
			cmdCh <- err
		}
		stdin.Close()
	}()
	// wait for command exit
	go func() {
		cmdCh <- cmd.Wait()
	}()
	go io.Copy(os.Stdout, stdout)
	go io.Copy(os.Stderr, stderr)

	cmdErr := <-cmdCh
	return cmdErr
}

func updateNodeList() {
	var index int64
	for {
		resp, newIndex, err := callConsulAPI(
			"/v1/catalog/nodes?index=" + strconv.FormatInt(index, 10) + "&wait=55s",
		)
		if err != nil {
			log.Println("[error]", err)
			time.Sleep(10 * time.Second)
			continue
		}
		index = newIndex
		defer resp.Body.Close()
		dec := json.NewDecoder(resp.Body)
		mutex.Lock()
		dec.Decode(&Nodes)
		log.Println("[info]", Nodes)
		mutex.Unlock()
		time.Sleep(1 * time.Second)
	}
}

func itemInNodes(item *Item) bool {
	mutex.RLock()
	defer mutex.RUnlock()
	for _, node := range Nodes {
		if item.Node == node.Node {
			item.Address = node.Address
			return true
		}
	}
	return false
}

func callConsulAPI(path string) (*http.Response, int64, error) {
	var index int64
	_url := "http://" + ConsulAddr + path
	log.Println("[info] get", _url)
	resp, err := http.Get(_url)
	if err != nil {
		log.Println("[error]", err)
		return nil, index, err
	}
	_indexes := resp.Header["X-Consul-Index"]
	if len(_indexes) > 0 {
		index, _ = strconv.ParseInt(_indexes[0], 10, 64)
	}
	return resp, index, nil
}
<|endoftext|>


<commit_before>// Copyright © 2015 Steve Francia <spf@spf13.com>.
//
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

// Package remote integrates the remote features of Viper.
package remote

import (
	"bytes"
	crypt "github.com/ltick/crypt/config"
	"github.com/samt42/viper"
	"io"
	"os"
)

type remoteConfigProvider struct{}

func (rc remoteConfigProvider) Set(rp viper.RemoteProvider, value []byte) error {
    cm, err := getConfigManager(rp)
    if err != nil {
        return err
    }
    err = cm.Set(rp.Path(), value)
    if err != nil {
        return err
    }
    return nil
}

func (rc remoteConfigProvider) List(rp viper.RemoteProvider) (map[string]io.Reader, error) {
    cm, err := getConfigManager(rp)
    if err != nil {
        return err
    }
    kvPairs, err := cm.List(rp.Path())
    if err != nil {
        return err
    }
    list := make(map[string][]byte, 0)
    for _, kvPair := range kvPairs {
        list[kvPair.Key] = bytes.NewReader(kvPair.Value)
    }
    return list, nil
}

func (rc remoteConfigProvider) Get(rp viper.RemoteProvider) (io.Reader, error) {
	cm, err := getConfigManager(rp)
	if err != nil {
		return nil, err
	}
	b, err := cm.Get(rp.Path())
	if err != nil {
		return nil, err
	}
	return bytes.NewReader(b), nil
}

func (rc remoteConfigProvider) Watch(rp viper.RemoteProvider) (io.Reader, error) {
	cm, err := getConfigManager(rp)
	if err != nil {
		return nil, err
	}
	resp, err := cm.Get(rp.Path())
	if err != nil {
		return nil, err
	}

	return bytes.NewReader(resp), nil
}

func (rc remoteConfigProvider) WatchChannel(rp viper.RemoteProvider) (<-chan *viper.RemoteResponse, chan bool) {
	cm, err := getConfigManager(rp)
	if err != nil {
		return nil, nil
	}
	quit := make(chan bool)
	quitwc := make(chan bool)
	viperResponsCh := make(chan *viper.RemoteResponse)
	cryptoResponseCh := cm.Watch(rp.Path(), quit)
	// need this function to convert the Channel response form crypt.Response to viper.Response
	go func(cr <-chan *crypt.Response, vr chan<- *viper.RemoteResponse, quitwc <-chan bool, quit chan<- bool) {
		for {
			select {
			case <-quitwc:
				quit <- true
				return
			case resp := <-cr:
				vr <- &viper.RemoteResponse{
					Error: resp.Error,
					Value: resp.Value,
				}

			}

		}
	}(cryptoResponseCh, viperResponsCh, quitwc, quit)

	return viperResponsCh, quitwc
}

func getConfigManager(rp viper.RemoteProvider) (crypt.ConfigManager, error) {
	var cm crypt.ConfigManager
	var err error

	if rp.SecretKeyring() != "" {
		kr, err := os.Open(rp.SecretKeyring())
		defer kr.Close()
		if err != nil {
			return nil, err
		}
		switch rp.Provider() {
		case "etcd":
			cm, err = crypt.NewEtcdConfigManager([]string{rp.Endpoint()}, kr)
		case "consul":
			cm, err = crypt.NewConsulConfigManager([]string{rp.Endpoint()}, kr)
		case "memcache":
			cm, err = crypt.NewMemcacheConfigManager([]string{rp.Endpoint()}, kr)
		case "zookeeper":
			config := rp.Config()
			cm, err = crypt.NewZookeeperConfigManager([]string{rp.Endpoint()}, config["user"], config["password"], kr)
		}
	} else {
		switch rp.Provider() {
		case "etcd":
			cm, err = crypt.NewStandardEtcdConfigManager([]string{rp.Endpoint()})
		case "consul":
			cm, err = crypt.NewStandardConsulConfigManager([]string{rp.Endpoint()})
		case "memcache":
			cm, err = crypt.NewStandardMemcacheConfigManager([]string{rp.Endpoint()})
		case "zookeeper":
			config := rp.Config()
			cm, err = crypt.NewStandardZookeeperConfigManager([]string{rp.Endpoint()}, config["user"], config["password"])
		}
	}
	if err != nil {
		return nil, err
	}
	return cm, nil
}

func init() {
	viper.RemoteConfig = &remoteConfigProvider{}
}
<commit_msg>update<commit_after>// Copyright © 2015 Steve Francia <spf@spf13.com>.
//
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

// Package remote integrates the remote features of Viper.
package remote

import (
	"bytes"
	crypt "github.com/ltick/crypt/config"
	"github.com/samt42/viper"
	"io"
	"os"
)

type remoteConfigProvider struct{}

func (rc remoteConfigProvider) Set(rp viper.RemoteProvider, value []byte) error {
	cm, err := getConfigManager(rp)
	if err != nil {
		return err
	}
	err = cm.Set(rp.Path(), value)
	if err != nil {
		return err
	}
	return nil
}

func (rc remoteConfigProvider) List(rp viper.RemoteProvider) (map[string]io.Reader, error) {
	cm, err := getConfigManager(rp)
	if err != nil {
		return nil, err
	}
	kvPairs, err := cm.List(rp.Path())
	if err != nil {
		return nil, err
	}
	list := make(map[string]io.Reader, 0)
	for _, kvPair := range kvPairs {
		list[kvPair.Key] = bytes.NewReader(kvPair.Value)
	}
	return list, nil
}

func (rc remoteConfigProvider) Get(rp viper.RemoteProvider) (io.Reader, error) {
	cm, err := getConfigManager(rp)
	if err != nil {
		return nil, err
	}
	b, err := cm.Get(rp.Path())
	if err != nil {
		return nil, err
	}
	return bytes.NewReader(b), nil
}

func (rc remoteConfigProvider) Watch(rp viper.RemoteProvider) (io.Reader, error) {
	cm, err := getConfigManager(rp)
	if err != nil {
		return nil, err
	}
	resp, err := cm.Get(rp.Path())
	if err != nil {
		return nil, err
	}

	return bytes.NewReader(resp), nil
}

func (rc remoteConfigProvider) WatchChannel(rp viper.RemoteProvider) (<-chan *viper.RemoteResponse, chan bool) {
	cm, err := getConfigManager(rp)
	if err != nil {
		return nil, nil
	}
	quit := make(chan bool)
	quitwc := make(chan bool)
	viperResponsCh := make(chan *viper.RemoteResponse)
	cryptoResponseCh := cm.Watch(rp.Path(), quit)
	// need this function to convert the Channel response form crypt.Response to viper.Response
	go func(cr <-chan *crypt.Response, vr chan<- *viper.RemoteResponse, quitwc <-chan bool, quit chan<- bool) {
		for {
			select {
			case <-quitwc:
				quit <- true
				return
			case resp := <-cr:
				vr <- &viper.RemoteResponse{
					Error: resp.Error,
					Value: resp.Value,
				}

			}

		}
	}(cryptoResponseCh, viperResponsCh, quitwc, quit)

	return viperResponsCh, quitwc
}

func getConfigManager(rp viper.RemoteProvider) (crypt.ConfigManager, error) {
	var cm crypt.ConfigManager
	var err error

	if rp.SecretKeyring() != "" {
		kr, err := os.Open(rp.SecretKeyring())
		defer kr.Close()
		if err != nil {
			return nil, err
		}
		switch rp.Provider() {
		case "etcd":
			cm, err = crypt.NewEtcdConfigManager([]string{rp.Endpoint()}, kr)
		case "consul":
			cm, err = crypt.NewConsulConfigManager([]string{rp.Endpoint()}, kr)
		case "memcache":
			cm, err = crypt.NewMemcacheConfigManager([]string{rp.Endpoint()}, kr)
		case "zookeeper":
			config := rp.Config()
			cm, err = crypt.NewZookeeperConfigManager([]string{rp.Endpoint()}, config["user"], config["password"], kr)
		}
	} else {
		switch rp.Provider() {
		case "etcd":
			cm, err = crypt.NewStandardEtcdConfigManager([]string{rp.Endpoint()})
		case "consul":
			cm, err = crypt.NewStandardConsulConfigManager([]string{rp.Endpoint()})
		case "memcache":
			cm, err = crypt.NewStandardMemcacheConfigManager([]string{rp.Endpoint()})
		case "zookeeper":
			config := rp.Config()
			cm, err = crypt.NewStandardZookeeperConfigManager([]string{rp.Endpoint()}, config["user"], config["password"])
		}
	}
	if err != nil {
		return nil, err
	}
	return cm, nil
}

func init() {
	viper.RemoteConfig = &remoteConfigProvider{}
}
<|endoftext|>