<commit_before>package dochaincore

import (
	"bytes"
	"context"
	"crypto/rand"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"html/template"
	"net/http"
	"net/url"
	"os"
	"path"
	"strings"
	"sync"
	"time"
)

const indexPageHTML = `<!DOCTYPE html>

		<title>One-Click Chain Core DigitalOcean</title>
		<link rel="stylesheet" href="/static/style.css">

Install Chain Core on a DigitalOcean droplet. This installer creates a new 1gb droplet and a 100gb block storage volume on your DigitalOcean account. It installs Chain Core on the droplet using the attached volume for storage. The approximate cost on DigitalOcean is $20/month.

			Install Chain Core

`
const progressPageHTML = `
<!DOCTYPE html>

		<title>One-Click Chain Core DigitalOcean</title>
		<link rel="stylesheet" href="/static/style.css">

Initializing droplet…

Success! Chain Core has been installed on your DigitalOcean droplet. To access
				Chain Core's API and Dashboard, you'll need your client token:

URL:

Token:

				Open dashboard

When destroying the droplet, remember to also destroy its block storage volume.

`

func Handler(oauthClientID, oauthClientSecret, host string) http.Handler {
	h := &handler{
		oauthClientID:     oauthClientID,
		oauthClientSecret: oauthClientSecret,
		host:              host,
		progressTmpl:      template.Must(template.New("progresspage").Parse(progressPageHTML)),
		indexTmpl:         template.Must(template.New("index").Parse(indexPageHTML)),
		installs:          make(map[string]*install),
	}
	mux := http.NewServeMux()
	mux.HandleFunc("/status/", h.status)
	mux.HandleFunc("/grant", h.grant)
	mux.HandleFunc("/install/", h.progressPage)
	mux.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("static"))))
	mux.HandleFunc("/", h.index)
	return mux
}

type handler struct {
	oauthClientID     string
	oauthClientSecret string
	host              string
	progressTmpl      *template.Template
	indexTmpl         *template.Template

	installMu sync.Mutex
	installs  map[string]*install
}

func (h *handler) index(rw http.ResponseWriter, req *http.Request) {
	b := make([]byte, 16)
	_, err := rand.Read(b)
	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
		return
	}
	installID := hex.EncodeToString(b)

	h.installMu.Lock()
	h.installs[installID] = &install{Status: "pending auth"}
	h.installMu.Unlock()

	vals := make(url.Values)
	vals.Set("response_type", "code")
	vals.Set("client_id", h.oauthClientID)
	vals.Set("state", installID)
	vals.Set("scope", "read write")
	vals.Set("redirect_uri", h.host+"/grant")
	u := url.URL{
		Scheme:   "https",
		Host:     "cloud.digitalocean.com",
		Path:     "/v1/oauth/authorize",
		RawQuery: vals.Encode(),
	}

	tmplData := struct {
		InstallLink string
	}{
		InstallLink: u.String(),
	}
	h.indexTmpl.Execute(rw, tmplData)
}

func (h *handler) grant(rw http.ResponseWriter, req *http.Request) {
	code, state := req.FormValue("code"), req.FormValue("state")
	if code == "" || state == "" {
		http.Error(rw, "invalid oauth2 grant", http.StatusBadRequest)
		return
	}

	h.installMu.Lock()
	curr := h.installs[state]
	h.installMu.Unlock()
	if curr == nil {
		http.Error(rw, "invalid oauth2 state", http.StatusBadRequest)
		return
	}

	// Claim the code grant
	vals := make(url.Values)
	vals.Set("grant_type", "authorization_code")
	vals.Set("code", code)
	vals.Set("client_id", h.oauthClientID)
	vals.Set("client_secret", h.oauthClientSecret)
	vals.Set("redirect_uri", h.host+"/grant")
	u := url.URL{
		Scheme:   "https",
		Host:     "cloud.digitalocean.com",
		Path:     "/v1/oauth/token",
		RawQuery: vals.Encode(),
	}
	resp, err := http.Post(u.String(), "application/x-www-form-urlencoded", nil)
	if err != nil {
		http.Error(rw, "internal server error", http.StatusInternalServerError)
		return
	}
	defer resp.Body.Close()

	var decodedResponse struct {
		AccessToken  string `json:"access_token"`
		TokenType    string `json:"bearer"`
		ExpiresIn    int    `json:"expires_in"`
		RefreshToken string `json:"refresh_token"`
		Scope        string `json:"scope"`
		Info         struct {
			Name  string `json:"name"`
			Email string `json:"email"`
			UUID  string `json:"uuid"`
		} `json:"info"`
	}
	err = json.NewDecoder(resp.Body).Decode(&decodedResponse)
	if err != nil {
		http.Error(rw, "err decoding access token grant", http.StatusBadRequest)
		return
	}
	if decodedResponse.AccessToken == "" {
		http.Error(rw, "missing access token", http.StatusBadRequest)
		return
	}
	if decodedResponse.Scope != "read write" {
		http.Error(rw, "need read write OAuth scope", http.StatusBadRequest)
		return
	}

	curr.mu.Lock()
	curr.accessToken = decodedResponse.AccessToken
	curr.mu.Unlock()

	http.Redirect(rw, req, "/install/"+state, http.StatusFound)
}

func (h *handler) progressPage(rw http.ResponseWriter, req *http.Request) {
	id := path.Base(req.URL.Path)
	h.installMu.Lock()
	curr := h.installs[id]
	h.installMu.Unlock()

	if curr == nil {
		http.NotFound(rw, req)
		return
	}

	go curr.init(id)

	tmplData := struct {
		InstallID string
	}{
		InstallID: id,
	}
	err := h.progressTmpl.Execute(rw, tmplData)
	if err != nil {
		fmt.Fprintf(os.Stderr, "executing template: %s", err.Error())
	}
}

func (h *handler) status(rw http.ResponseWriter, req *http.Request) {
	id := path.Base(req.URL.Path)
	h.installMu.Lock()
	curr := h.installs[id]
	h.installMu.Unlock()

	if curr == nil {
		http.NotFound(rw, req)
		return
	}

	// Marshal to a buffer first so that a really slow request can't
	// keep curr.mu locked indefinitely.
	var buf bytes.Buffer
	curr.mu.Lock()
	_ = json.NewEncoder(&buf).Encode(curr)
	curr.mu.Unlock()

	rw.Header().Set("Content-Type", "application/json")
	rw.Write(buf.Bytes())
}

type install struct {
	mu          sync.Mutex
	Status      string `json:"status"`
	ClientToken string `json:"client_token"`
	IPAddress   string `json:"ip_address"`
	accessToken string
	c           *Core
}

func (i *install) setStatus(status string) {
	i.mu.Lock()
	defer i.mu.Unlock()
	i.Status = status
}

func (i *install) init(state string) {
	defer revoke(i.accessToken)

	// Set a 10 minute timeout for the installation. From beginning
	// to end it should only take a ~2 minutes, but make sure we
	// cleanup and revoke the access token even if it takes longer.
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
	defer cancel()

	var core *Core
	var err error

	defer func() {
		if err != nil {
			i.setStatus(err.Error())
		}
	}()

	// Start deploying and create the droplet.
	core, err = Deploy(ctx, i.accessToken, DropletName("chain-core-"+state[:6]))
	if err != nil {
		return
	}

	i.mu.Lock()
	i.IPAddress = core.IPv4Address
	i.c = core
	i.Status = "waiting for ssh"
	i.mu.Unlock()

	err = WaitForSSH(ctx, core)
	if err != nil {
		return
	}

	i.setStatus("waiting for http")
	err = WaitForHTTP(ctx, core)
	if err != nil {
		return
	}

	i.setStatus("creating client token")
	token, err := CreateClientToken(ctx, core)
	if err != nil {
		return
	}

	i.mu.Lock()
	i.Status = "done"
	i.ClientToken = token
	i.c = nil // garbage collect the SSH keys
	i.mu.Unlock()
}

func revoke(accessToken string) error {
	body := strings.NewReader(url.Values{"token": {accessToken}}.Encode())
	req, err := http.NewRequest("POST", "https://cloud.digitalocean.com/v1/oauth/revoke", body)
	if err != nil {
		return err
	}
	req.Header.Set("Authorization", "Bearer "+accessToken)
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("revoke endpoint returned %d status code", resp.StatusCode)
	}
	return nil
}
<commit_msg>handler: clean up<commit_after>package dochaincore

import (
	"bytes"
	"context"
	"crypto/rand"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"html/template"
	"net/http"
	"net/url"
	"os"
	"path"
	"strings"
	"sync"
	"time"
)

const indexPageHTML = `<!DOCTYPE html>

		<title>One-Click Chain Core DigitalOcean</title>
		<link rel="stylesheet" href="/static/style.css">

Install Chain Core on a DigitalOcean droplet. This installer creates a new 1gb droplet and a 100gb block storage volume on your DigitalOcean account. It installs Chain Core on the droplet using the attached volume for storage. The approximate cost on DigitalOcean is $20/month.

			Install Chain Core

`
const progressPageHTML = `
<!DOCTYPE html>

		<title>One-Click Chain Core DigitalOcean</title>
		<link rel="stylesheet" href="/static/style.css">

Initializing droplet…

Success! Chain Core has been installed on your DigitalOcean droplet. To access
				Chain Core's API and Dashboard, you'll need your client token:

URL:

Token:

				Open dashboard

When destroying the droplet, remember to also destroy its block storage volume.

`

func Handler(oauthClientID, oauthClientSecret, host string) http.Handler {
	h := &handler{
		oauthClientID:     oauthClientID,
		oauthClientSecret: oauthClientSecret,
		host:              host,
		progressTmpl:      template.Must(template.New("progresspage").Parse(progressPageHTML)),
		indexTmpl:         template.Must(template.New("index").Parse(indexPageHTML)),
		installs:          make(map[string]*install),
	}
	mux := http.NewServeMux()
	mux.HandleFunc("/status/", h.status)
	mux.HandleFunc("/grant", h.grant)
	mux.HandleFunc("/install/", h.progressPage)
	mux.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("static"))))
	mux.HandleFunc("/", h.index)
	return mux
}

type handler struct {
	oauthClientID     string
	oauthClientSecret string
	host              string
	progressTmpl      *template.Template
	indexTmpl         *template.Template

	installMu sync.Mutex
	installs  map[string]*install
}

func (h *handler) index(rw http.ResponseWriter, req *http.Request) {
	b := make([]byte, 16)
	_, err := rand.Read(b)
	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
		return
	}
	installID := hex.EncodeToString(b)

	h.installMu.Lock()
	h.installs[installID] = &install{Status: "pending auth"}
	h.installMu.Unlock()

	vals := make(url.Values)
	vals.Set("response_type", "code")
	vals.Set("client_id", h.oauthClientID)
	vals.Set("state", installID)
	vals.Set("scope", "read write")
	vals.Set("redirect_uri", h.host+"/grant")
	u := url.URL{
		Scheme:   "https",
		Host:     "cloud.digitalocean.com",
		Path:     "/v1/oauth/authorize",
		RawQuery: vals.Encode(),
	}

	h.indexTmpl.Execute(rw, struct {
		InstallLink string
	}{
		InstallLink: u.String(),
	})
}

func (h *handler) grant(rw http.ResponseWriter, req *http.Request) {
	code, state := req.FormValue("code"), req.FormValue("state")
	if code == "" || state == "" {
		http.Error(rw, "invalid oauth2 grant", http.StatusBadRequest)
		return
	}

	h.installMu.Lock()
	curr := h.installs[state]
	h.installMu.Unlock()
	if curr == nil {
		http.Error(rw, "invalid oauth2 state", http.StatusBadRequest)
		return
	}

	// Claim the code grant
	vals := make(url.Values)
	vals.Set("grant_type", "authorization_code")
	vals.Set("code", code)
	vals.Set("client_id", h.oauthClientID)
	vals.Set("client_secret", h.oauthClientSecret)
	vals.Set("redirect_uri", h.host+"/grant")
	u := url.URL{
		Scheme:   "https",
		Host:     "cloud.digitalocean.com",
		Path:     "/v1/oauth/token",
		RawQuery: vals.Encode(),
	}
	resp, err := http.Post(u.String(), "application/x-www-form-urlencoded", nil)
	if err != nil {
		http.Error(rw, "internal server error", http.StatusInternalServerError)
		return
	}
	defer resp.Body.Close()

	var decodedResponse struct {
		AccessToken  string `json:"access_token"`
		TokenType    string `json:"bearer"`
		ExpiresIn    int    `json:"expires_in"`
		RefreshToken string `json:"refresh_token"`
		Scope        string `json:"scope"`
		Info         struct {
			Name  string `json:"name"`
			Email string `json:"email"`
			UUID  string `json:"uuid"`
		} `json:"info"`
	}
	err = json.NewDecoder(resp.Body).Decode(&decodedResponse)
	if err != nil {
		http.Error(rw, "err decoding access token grant", http.StatusBadRequest)
		return
	}
	if decodedResponse.AccessToken == "" {
		http.Error(rw, "missing access token", http.StatusBadRequest)
		return
	}
	if decodedResponse.Scope != "read write" {
		http.Error(rw, "need read write OAuth scope", http.StatusBadRequest)
		return
	}

	curr.mu.Lock()
	curr.accessToken = decodedResponse.AccessToken
	curr.mu.Unlock()

	http.Redirect(rw, req, "/install/"+state, http.StatusFound)
}

func (h *handler) progressPage(rw http.ResponseWriter, req *http.Request) {
	id := path.Base(req.URL.Path)
	h.installMu.Lock()
	curr := h.installs[id]
	h.installMu.Unlock()

	if curr == nil {
		http.NotFound(rw, req)
		return
	}

	go curr.init(id)
	err := h.progressTmpl.Execute(rw, struct {
		InstallID string
	}{
		InstallID: id,
	})
	if err != nil {
		fmt.Fprintf(os.Stderr, "executing template: %s", err.Error())
	}
}

func (h *handler) status(rw http.ResponseWriter, req *http.Request) {
	id := path.Base(req.URL.Path)
	h.installMu.Lock()
	curr := h.installs[id]
	h.installMu.Unlock()

	if curr == nil {
		http.NotFound(rw, req)
		return
	}

	// Marshal to a buffer first so that a really slow request can't
	// keep curr.mu locked indefinitely.
	var buf bytes.Buffer
	curr.mu.Lock()
	_ = json.NewEncoder(&buf).Encode(curr)
	curr.mu.Unlock()

	rw.Header().Set("Content-Type", "application/json")
	rw.Write(buf.Bytes())
}

type install struct {
	mu          sync.Mutex
	Status      string `json:"status"`
	ClientToken string `json:"client_token"`
	IPAddress   string `json:"ip_address"`
	accessToken string
	c           *Core
}

func (i *install) setStatus(status string) {
	i.mu.Lock()
	defer i.mu.Unlock()
	i.Status = status
}

func (i *install) init(state string) {
	defer revoke(i.accessToken)

	// Set a 10 minute timeout for the installation. From beginning
	// to end it should only take a ~2 minutes, but make sure we
	// cleanup and revoke the access token even if it takes longer.
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
	defer cancel()

	var core *Core
	var err error
	defer func() {
		if err != nil {
			i.setStatus(err.Error())
		}
	}()

	// Start deploying and create the droplet.
	core, err = Deploy(ctx, i.accessToken, DropletName("chain-core-"+state[:6]))
	if err != nil {
		return
	}

	i.mu.Lock()
	i.IPAddress = core.IPv4Address
	i.c = core
	i.Status = "waiting for ssh"
	i.mu.Unlock()

	err = WaitForSSH(ctx, core)
	if err != nil {
		return
	}

	i.setStatus("waiting for http")
	err = WaitForHTTP(ctx, core)
	if err != nil {
		return
	}

	i.setStatus("creating client token")
	token, err := CreateClientToken(ctx, core)
	if err != nil {
		return
	}

	i.mu.Lock()
	i.Status = "done"
	i.ClientToken = token
	i.c = nil // garbage collect the SSH keys
	i.mu.Unlock()
}

func revoke(accessToken string) error {
	body := strings.NewReader(url.Values{"token": {accessToken}}.Encode())
	req, err := http.NewRequest("POST", "https://cloud.digitalocean.com/v1/oauth/revoke", body)
	if err != nil {
		return err
	}
	req.Header.Set("Authorization", "Bearer "+accessToken)
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("revoke endpoint returned %d status code", resp.StatusCode)
	}
	return nil
}
<|endoftext|>


<commit_before>// Package slack15 is log15 (https://github.com/inconshreveable/log15)
// handler for sending log messages to Slack.
package slack15

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"os"

	"github.com/inconshreveable/log15"
)

// ErrNoWebHook is returned if no WebHook URL is provided nor it
// could be found in environment
var ErrNoWebHook = errors.New("No Slack WebHook URL specified")

// Handler implements log15.Handler interface
type Handler struct {
	// WebHook URL (if empty taken from $SLACK_WEBHOOK)
	URL string

	// Message formatter – if nil default will be used
	Formatter log15.Format

	// Envelope The following fields allow for ovewritting default values
	// for webhook (as set in slack.com/services)
	Envelope
}

// func NewHandler()

type ctxReader struct {
	ctx []interface{}

	key   string
	value interface{}
	err   error
}

func (r *ctxReader) Pairs() int {
	return len(r.ctx) / 2
}

func (r *ctxReader) Next() bool {
	if len(r.ctx) < 2 {
		return false
	}
	var ok bool
	r.key, ok = r.ctx[0].(string)
	if !ok {
		r.err = fmt.Errorf("%+v is not a string key", r.ctx[0])
		r.key = "?"
	}
	r.value = r.ctx[1]
	r.ctx = r.ctx[2:]
	return true
}

func newCtxReader(ctx []interface{}) *ctxReader {
	return &ctxReader{ctx: ctx}
}

func (r *ctxReader) Key() string {
	return r.key
}

func (r *ctxReader) Value() interface{} {
	return r.value
}

func (r *ctxReader) Err() error {
	return r.err
}

// Log logs records by sending it to Slack
func (h *Handler) Log(r *log15.Record) error {
	msg, err := h.getMsg(r)
	// send message anyway if error occured

	url := h.URL
	if url == "" {
		url = os.Getenv("SLACK_WEBHOOK_URL")
		if url == "" {
			return ErrNoWebHook
		}
	}

	payload, err := json.Marshal(msg)
	if err != nil {
		return err
	}

	resp, err := http.Post(url, "", bytes.NewReader(payload))
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		err = fmt.Errorf("slack responsed with code %d", resp.StatusCode)
	}

	return err
}

func (h *Handler) getMsg(r *log15.Record) (*message, error) {
	var err error
	msg := &message{
		Envelope: h.Envelope,
	}

	color := "#32C8C8" // blue
	switch r.Lvl {
	case log15.LvlInfo:
		color = "good" // green
	case log15.LvlWarn:
		color = "warning" // yellow
	case log15.LvlError:
		color = "danger" // red
	case log15.LvlCrit:
		color = "#C832C8" // purple
	}

	if h.Formatter != nil {
		txt := string(h.Formatter.Format(r))
		msg.Attachments = []attachment{{
			Text:     txt,
			Fallback: txt,
			Color:    color,
		}}
	} else {
		ctx := newCtxReader(r.Ctx)
		fields := make([]field, 0, ctx.Pairs()+1)

		for ctx.Next() {
			v := fmt.Sprint(ctx.Value())
			fields = append(fields, field{
				Title: ctx.Key(),
				Value: v,
				Short: true,
			})
		}
		err = ctx.Err()

		msg.Attachments = []attachment{{
			Text:     r.Msg,
			Fallback: string(log15.LogfmtFormat().Format(r)),
			Fields:   fields,
			Color:    color,
		}}
	}

	return msg, err
}
<commit_msg>Improve docs<commit_after>// Package slack15 is log15 (https://github.com/inconshreveable/log15)
// handler for sending log messages to Slack.
package slack15

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"os"

	"github.com/inconshreveable/log15"
)

// ErrNoWebHook is returned if no WebHook URL is provided nor it
// could be found in environment
var ErrNoWebHook = errors.New("No Slack WebHook URL specified")

// Handler implements log15.Handler interface
type Handler struct {
	// WebHook URL (if empty taken from $SLACK_WEBHOOK)
	URL string

	// Message formatter – if nil default will be used
	Formatter log15.Format

	// Envelope allowing to overwrite webhook's defaults
	Envelope
}

// Log logs records by sending it to Slack
func (h *Handler) Log(r *log15.Record) error {
	msg, err := h.getMsg(r)
	// send message anyway if error occured

	url := h.URL
	if url == "" {
		url = os.Getenv("SLACK_WEBHOOK_URL")
		if url == "" {
			return ErrNoWebHook
		}
	}

	payload, err := json.Marshal(msg)
	if err != nil {
		return err
	}

	resp, err := http.Post(url, "", bytes.NewReader(payload))
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		err = fmt.Errorf("slack responsed with code %d", resp.StatusCode)
	}

	return err
}

// ctxReader extracts key-value pairs from log15.Record.Ctx
type ctxReader struct {
	ctx []interface{}

	key   string
	value interface{}
	err   error
}

func newCtxReader(ctx []interface{}) *ctxReader {
	return &ctxReader{ctx: ctx}
}

// Pairs returns number of key-value pairs left
func (r *ctxReader) Pairs() int {
	return len(r.ctx) / 2
}

// Next process next key-value pair.
// Note true can be returned even if internal error is set.
func (r *ctxReader) Next() bool {
	if len(r.ctx) < 2 {
		return false
	}
	var ok bool
	r.key, ok = r.ctx[0].(string)
	if !ok {
		r.err = fmt.Errorf("%+v is not a string key", r.ctx[0])
		r.key = "?"
	}
	r.value = r.ctx[1]
	r.ctx = r.ctx[2:]
	return true
}

func (r *ctxReader) Key() string {
	return r.key
}

func (r *ctxReader) Value() interface{} {
	return r.value
}

func (r *ctxReader) Err() error {
	return r.err
}

// getMsg returns message which should be sent to Slack
func (h *Handler) getMsg(r *log15.Record) (*message, error) {
	var err error
	msg := &message{
		Envelope: h.Envelope,
	}

	color := "#32C8C8" // blue
	switch r.Lvl {
	case log15.LvlInfo:
		color = "good" // green
	case log15.LvlWarn:
		color = "warning" // yellow
	case log15.LvlError:
		color = "danger" // red
	case log15.LvlCrit:
		color = "#C832C8" // purple
	}

	if h.Formatter != nil {
		txt := string(h.Formatter.Format(r))
		msg.Attachments = []attachment{{
			Text:     txt,
			Fallback: txt,
			Color:    color,
		}}
	} else {
		ctx := newCtxReader(r.Ctx)
		fields := make([]field, 0, ctx.Pairs()+1)

		for ctx.Next() {
			v := fmt.Sprint(ctx.Value())
			fields = append(fields, field{
				Title: ctx.Key(),
				Value: v,
				Short: true,
			})
		}
		err = ctx.Err()

		msg.Attachments = []attachment{{
			Text:     r.Msg,
			Fallback: string(log15.LogfmtFormat().Format(r)),
			Fields:   fields,
			Color:    color,
		}}
	}

	return msg, err
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"strings"
	"text/tabwriter"

	"golang.org/x/net/context"

	"github.com/pachyderm/pachyderm/src/pfs"
	"github.com/pachyderm/pachyderm/src/pps"
	"github.com/pachyderm/pachyderm/src/pps/pretty"

	"go.pedge.io/env"
	"go.pedge.io/pkg/cobra"
	"go.pedge.io/proto/stream"

	"google.golang.org/grpc"

	"github.com/spf13/cobra"
)

type appEnv struct {
	PachydermPpsd1Port string `env:"PACHYDERM_PPSD_1_PORT"`
	Address            string `env:"PPS_ADDRESS,default=0.0.0.0:651"`
}

func main() {
	env.Main(do, &appEnv{}, defaultEnv)
}

func do(appEnvObj interface{}) error {
	appEnv := appEnvObj.(*appEnv)
	address := appEnv.PachydermPpsd1Port
	if address == "" {
		address = appEnv.Address
	} else {
		address = strings.Replace(address, "tcp://", "", -1)
	}
	clientConn, err := grpc.Dial(address, grpc.WithInsecure())
	if err != nil {
		return err
	}
	jobAPIClient := pps.NewJobAPIClient(clientConn)
	pipelineAPIClient := pps.NewPipelineAPIClient(clientConn)
	rootCmd := &cobra.Command{
		Use: "pps",
		Long: `Access the PPS API.

Note that this CLI is experimental and does not even check for common errors.
The environment variable PPS_ADDRESS controls what server the CLI connects to, the default is 0.0.0.0:651.`,
	}

	var image string
	var outParentCommitID string
	createJob := &cobra.Command{
		Use:   "create-job in-repo-name in-commit-id out-repo-name command [args]",
		Short: "Create a new job. Returns the id of the created job.",
		Long: `Create a new job. With repo-name/commit-id as input and
out-repo-name as output. A commit will be created for the output.
You can find out the name of the commit with inspect-job.`,
		Run: func(cmd *cobra.Command, args []string) {
			job, err := jobAPIClient.CreateJob(
				context.Background(),
				&pps.CreateJobRequest{
					Spec: &pps.CreateJobRequest_Transform{
						Transform: &pps.Transform{
							Image: image,
							Cmd:   args[3:],
						},
					},
					Input: &pfs.Commit{
						Repo: &pfs.Repo{
							Name: args[0],
						},
						Id: args[1],
					},
					OutputParent: &pfs.Commit{
						Repo: &pfs.Repo{
							Name: args[2],
						},
						Id: outParentCommitID,
					},
				})
			if err != nil {
				errorAndExit("Error from CreateJob: %s", err.Error())
			}
			fmt.Println(job.Id)
		},
	}
	createJob.Flags().StringVarP(&image, "image", "i", "ubuntu", "The image to run the job in.")
	createJob.Flags().StringVarP(&outParentCommitID, "parent", "p", "", "The parent to use for the output commit.")

	inspectJob := &cobra.Command{
		Use:   "inspect-job job-id",
		Short: "Return info about a job.",
		Long:  "Return info about a job.",
		Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
			jobInfo, err := jobAPIClient.InspectJob(
				context.Background(),
				&pps.InspectJobRequest{
					Job: &pps.Job{
						Id: args[0],
					},
				},
			)
			if err != nil {
				errorAndExit("Error from InspectJob: %s", err.Error())
			}
			if jobInfo == nil {
				errorAndExit("Job %s not found.", args[0])
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintJobHeader(writer)
			pretty.PrintJobInfo(writer, jobInfo)
			return writer.Flush()
		}),
	}

	var pipelineName string
	listJob := &cobra.Command{
		Use:   "list-job -p pipeline-name",
		Short: "Return info about all jobs.",
		Long:  "Return info about all jobs.",
		Run: pkgcobra.RunFixedArgs(0, func(args []string) error {
			var pipeline *pps.Pipeline
			if pipelineName != "" {
				pipeline = &pps.Pipeline{
					Name: pipelineName,
				}
			}
			jobInfos, err := jobAPIClient.ListJob(
				context.Background(),
				&pps.ListJobRequest{
					Pipeline: pipeline,
				},
			)
			if err != nil {
				errorAndExit("Error from InspectJob: %s", err.Error())
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintJobHeader(writer)
			for _, jobInfo := range jobInfos.JobInfo {
				pretty.PrintJobInfo(writer, jobInfo)
			}
			return writer.Flush()
		}),
	}
	listJob.Flags().StringVarP(&pipelineName, "pipeline", "p", "", "Limit to jobs made by pipeline.")

	getJobLogs := &cobra.Command{
		Use:   "logs job-id",
		Short: "Return logs from a job.",
		Long:  "Return logs from a job.",
		Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
			logsClient, err := jobAPIClient.GetJobLogs(
				context.Background(),
				&pps.GetJobLogsRequest{
					Job: &pps.Job{
						Id: args[0],
					},
					OutputStream: pps.OutputStream_OUTPUT_STREAM_ALL,
				},
			)
			if err != nil {
				errorAndExit("Error from InspectJob: %s", err.Error())
			}
			if err := protostream.WriteFromStreamingBytesClient(logsClient, os.Stdout); err != nil {
				return err
			}
			return nil
		}),
	}

	createPipeline := &cobra.Command{
		Use:   "create-pipeline pipeline-name input-repo output-repo command [args]",
		Short: "Create a new pipeline.",
		Long:  "Create a new pipeline.",
		Run: func(cmd *cobra.Command, args []string) {
			if _, err := pipelineAPIClient.CreatePipeline(
				context.Background(),
				&pps.CreatePipelineRequest{
					Pipeline: &pps.Pipeline{
						Name: args[0],
					},
					Transform: &pps.Transform{
						Image: image,
						Cmd:   args[3:],
					},
					Input: &pfs.Repo{
						Name: args[1],
					},
					Output: &pfs.Repo{
						Name: args[2],
					},
				},
			); err != nil {
				errorAndExit("Error from CreatePipeline: %s", err.Error())
			}
		},
	}
	createPipeline.Flags().StringVarP(&image, "image", "i", "ubuntu", "The image to run the pipeline's jobs in.")

	inspectPipeline := &cobra.Command{
		Use:   "inspect-pipeline pipeline-name",
		Short: "Return info about a pipeline.",
		Long:  "Return info about a pipeline.",
		Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
			pipelineInfo, err := pipelineAPIClient.InspectPipeline(
				context.Background(),
				&pps.InspectPipelineRequest{
					Pipeline: &pps.Pipeline{
						Name: args[0],
					},
				},
			)
			if err != nil {
				errorAndExit("Error from InspectPipeline: %s", err.Error())
			}
			if pipelineInfo == nil {
				errorAndExit("Pipeline %s not found.", args[0])
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintPipelineHeader(writer)
			pretty.PrintPipelineInfo(writer, pipelineInfo)
			return writer.Flush()
		}),
	}

	listPipeline := &cobra.Command{
		Use:   "list-pipeline",
		Short: "Return info about all pipelines.",
		Long:  "Return info about all pipelines.",
		Run: pkgcobra.RunFixedArgs(0, func(args []string) error {
			pipelineInfos, err := pipelineAPIClient.ListPipeline(
				context.Background(),
				&pps.ListPipelineRequest{},
			)
			if err != nil {
				errorAndExit("Error from ListPipeline: %s", err.Error())
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintPipelineHeader(writer)
			for _, pipelineInfo := range pipelineInfos.PipelineInfo {
				pretty.PrintPipelineInfo(writer, pipelineInfo)
			}
			return writer.Flush()
		}),
	}

	deletePipeline := &cobra.Command{
		Use:   "delete-pipeline pipeline-name",
		Short: "Delete a pipeline.",
		Long:  "Delete a pipeline.",
		Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
			if _, err := pipelineAPIClient.DeletePipeline(
				context.Background(),
				&pps.DeletePipelineRequest{
					Pipeline: &pps.Pipeline{
						Name: args[0],
					},
				},
			); err != nil {
				errorAndExit("Error from DeletePipeline: %s", err.Error())
			}
			return nil
		}),
	}

	rootCmd.AddCommand(createJob)
	rootCmd.AddCommand(inspectJob)
	rootCmd.AddCommand(listJob)
	rootCmd.AddCommand(getJobLogs)
	rootCmd.AddCommand(createPipeline)
	rootCmd.AddCommand(inspectPipeline)
	rootCmd.AddCommand(listPipeline)
	rootCmd.AddCommand(deletePipeline)
	return rootCmd.Execute()
}

func errorAndExit(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, "%s\n", fmt.Sprintf(format, args...))
	os.Exit(1)
}
<commit_msg>fix env for pps<commit_after>package main

import (
	"fmt"
	"os"
	"strings"
	"text/tabwriter"

	"golang.org/x/net/context"

	"github.com/pachyderm/pachyderm/src/pfs"
	"github.com/pachyderm/pachyderm/src/pps"
	"github.com/pachyderm/pachyderm/src/pps/pretty"

	"go.pedge.io/env"
	"go.pedge.io/pkg/cobra"
	"go.pedge.io/proto/stream"

	"google.golang.org/grpc"

	"github.com/spf13/cobra"
)

type appEnv struct {
	PachydermPpsd1Port string `env:"PACHYDERM_PPSD_1_PORT"`
	Address            string `env:"PPS_ADDRESS,default=0.0.0.0:651"`
}

func main() {
	env.Main(do, &appEnv{})
}

func do(appEnvObj interface{}) error {
	appEnv := appEnvObj.(*appEnv)
	address := appEnv.PachydermPpsd1Port
	if address == "" {
		address = appEnv.Address
	} else {
		address = strings.Replace(address, "tcp://", "", -1)
	}
	clientConn, err := grpc.Dial(address, grpc.WithInsecure())
	if err != nil {
		return err
	}
	jobAPIClient := pps.NewJobAPIClient(clientConn)
	pipelineAPIClient := pps.NewPipelineAPIClient(clientConn)
	rootCmd := &cobra.Command{
		Use: "pps",
		Long: `Access the PPS API.

Note that this CLI is experimental and does not even check for common errors.
The environment variable PPS_ADDRESS controls what server the CLI connects to, the default is 0.0.0.0:651.`,
	}

	var image string
	var outParentCommitID string
	createJob := &cobra.Command{
		Use:   "create-job in-repo-name in-commit-id out-repo-name command [args]",
		Short: "Create a new job. Returns the id of the created job.",
		Long: `Create a new job. With repo-name/commit-id as input and
out-repo-name as output. A commit will be created for the output.
You can find out the name of the commit with inspect-job.`,
		Run: func(cmd *cobra.Command, args []string) {
			job, err := jobAPIClient.CreateJob(
				context.Background(),
				&pps.CreateJobRequest{
					Spec: &pps.CreateJobRequest_Transform{
						Transform: &pps.Transform{
							Image: image,
							Cmd:   args[3:],
						},
					},
					Input: &pfs.Commit{
						Repo: &pfs.Repo{
							Name: args[0],
						},
						Id: args[1],
					},
					OutputParent: &pfs.Commit{
						Repo: &pfs.Repo{
							Name: args[2],
						},
						Id: outParentCommitID,
					},
				})
			if err != nil {
				errorAndExit("Error from CreateJob: %s", err.Error())
			}
			fmt.Println(job.Id)
		},
	}
	createJob.Flags().StringVarP(&image, "image", "i", "ubuntu", "The image to run the job in.")
	createJob.Flags().StringVarP(&outParentCommitID, "parent", "p", "", "The parent to use for the output commit.")

	inspectJob := &cobra.Command{
		Use:   "inspect-job job-id",
		Short: "Return info about a job.",
		Long:  "Return info about a job.",
		Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
			jobInfo, err := jobAPIClient.InspectJob(
				context.Background(),
				&pps.InspectJobRequest{
					Job: &pps.Job{
						Id: args[0],
					},
				},
			)
			if err != nil {
				errorAndExit("Error from InspectJob: %s", err.Error())
			}
			if jobInfo == nil {
				errorAndExit("Job %s not found.", args[0])
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintJobHeader(writer)
			pretty.PrintJobInfo(writer, jobInfo)
			return writer.Flush()
		}),
	}

	var pipelineName string
	listJob := &cobra.Command{
		Use:   "list-job -p pipeline-name",
		Short: "Return info about all jobs.",
		Long:  "Return info about all jobs.",
		Run: pkgcobra.RunFixedArgs(0, func(args []string) error {
			var pipeline *pps.Pipeline
			if pipelineName != "" {
				pipeline = &pps.Pipeline{
					Name: pipelineName,
				}
			}
			jobInfos, err := jobAPIClient.ListJob(
				context.Background(),
				&pps.ListJobRequest{
					Pipeline: pipeline,
				},
			)
			if err != nil {
				errorAndExit("Error from InspectJob: %s", err.Error())
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintJobHeader(writer)
			for _, jobInfo := range jobInfos.JobInfo {
				pretty.PrintJobInfo(writer, jobInfo)
			}
			return writer.Flush()
		}),
	}
	listJob.Flags().StringVarP(&pipelineName, "pipeline", "p", "", "Limit to jobs made by pipeline.")

	getJobLogs := &cobra.Command{
		Use:   "logs job-id",
		Short: "Return logs from a job.",
		Long:  "Return logs from a job.",
		Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
			logsClient, err := jobAPIClient.GetJobLogs(
				context.Background(),
				&pps.GetJobLogsRequest{
					Job: &pps.Job{
						Id: args[0],
					},
					OutputStream: pps.OutputStream_OUTPUT_STREAM_ALL,
				},
			)
			if err != nil {
				errorAndExit("Error from InspectJob: %s", err.Error())
			}
			if err := protostream.WriteFromStreamingBytesClient(logsClient, os.Stdout); err != nil {
				return err
			}
			return nil
		}),
	}

	createPipeline := &cobra.Command{
		Use:   "create-pipeline pipeline-name input-repo output-repo command [args]",
		Short: "Create a new pipeline.",
		Long:  "Create a new pipeline.",
		Run: func(cmd *cobra.Command, args []string) {
			if _, err := pipelineAPIClient.CreatePipeline(
				context.Background(),
				&pps.CreatePipelineRequest{
					Pipeline: &pps.Pipeline{
						Name: args[0],
					},
					Transform: &pps.Transform{
						Image: image,
						Cmd:   args[3:],
					},
					Input: &pfs.Repo{
						Name: args[1],
					},
					Output: &pfs.Repo{
						Name: args[2],
					},
				},
			); err != nil {
				errorAndExit("Error from CreatePipeline: %s", err.Error())
			}
		},
	}
	createPipeline.Flags().StringVarP(&image, "image", "i", "ubuntu", "The image to run the pipeline's jobs in.")

	inspectPipeline := &cobra.Command{
		Use:   "inspect-pipeline pipeline-name",
		Short: "Return info about a pipeline.",
		Long:  "Return info about a pipeline.",
		Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
			pipelineInfo, err := pipelineAPIClient.InspectPipeline(
				context.Background(),
				&pps.InspectPipelineRequest{
					Pipeline: &pps.Pipeline{
						Name: args[0],
					},
				},
			)
			if err != nil {
				errorAndExit("Error from InspectPipeline: %s", err.Error())
			}
			if pipelineInfo == nil {
				errorAndExit("Pipeline %s not found.", args[0])
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintPipelineHeader(writer)
			pretty.PrintPipelineInfo(writer, pipelineInfo)
			return writer.Flush()
		}),
	}

	listPipeline := &cobra.Command{
		Use:   "list-pipeline",
		Short: "Return info about all pipelines.",
		Long:  "Return info about all pipelines.",
		Run: pkgcobra.RunFixedArgs(0, func(args []string) error {
			pipelineInfos, err := pipelineAPIClient.ListPipeline(
				context.Background(),
				&pps.ListPipelineRequest{},
			)
			if err != nil {
				errorAndExit("Error from ListPipeline: %s", err.Error())
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintPipelineHeader(writer)
			for _, pipelineInfo := range pipelineInfos.PipelineInfo {
				pretty.PrintPipelineInfo(writer, pipelineInfo)
			}
			return writer.Flush()
		}),
	}

	deletePipeline := &cobra.Command{
		Use:   "delete-pipeline pipeline-name",
		Short: "Delete a pipeline.",
		Long:  "Delete a pipeline.",
		Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
			if _, err := pipelineAPIClient.DeletePipeline(
				context.Background(),
				&pps.DeletePipelineRequest{
					Pipeline: &pps.Pipeline{
						Name: args[0],
					},
				},
			); err != nil {
				errorAndExit("Error from DeletePipeline: %s", err.Error())
			}
			return nil
		}),
	}

	rootCmd.AddCommand(createJob)
	rootCmd.AddCommand(inspectJob)
	rootCmd.AddCommand(listJob)
	rootCmd.AddCommand(getJobLogs)
	rootCmd.AddCommand(createPipeline)
	rootCmd.AddCommand(inspectPipeline)
	rootCmd.AddCommand(listPipeline)
	rootCmd.AddCommand(deletePipeline)
	return rootCmd.Execute()
}

func errorAndExit(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, "%s\n", fmt.Sprintf(format, args...))
	os.Exit(1)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"strings"
	"text/tabwriter"

	"golang.org/x/net/context"

	"github.com/pachyderm/pachyderm/src/pfs"
	"github.com/pachyderm/pachyderm/src/pps"
	"github.com/pachyderm/pachyderm/src/pps/pretty"

	"go.pedge.io/env"
	"go.pedge.io/pkg/cobra"
	"go.pedge.io/proto/stream"

	"google.golang.org/grpc"

	"github.com/spf13/cobra"
)

var (
	defaultEnv = map[string]string{
		"PPS_ADDRESS": "0.0.0.0:651",
	}
)

type appEnv struct {
	PachydermPpsd1Port string `env:"PACHYDERM_PPSD_1_PORT"`
	Address            string `env:"PPS_ADDRESS"`
}

func main() {
	env.Main(do, &appEnv{}, defaultEnv)
}

func do(appEnvObj interface{}) error {
	appEnv := appEnvObj.(*appEnv)
	address := appEnv.PachydermPpsd1Port
	if address == "" {
		address = appEnv.Address
	} else {
		address = strings.Replace(address, "tcp://", "", -1)
	}
	clientConn, err := grpc.Dial(address, grpc.WithInsecure())
	if err != nil {
		return err
	}
	jobAPIClient := pps.NewJobAPIClient(clientConn)
	pipelineAPIClient := pps.NewPipelineAPIClient(clientConn)
	rootCmd := &cobra.Command{
		Use: "pps",
		Long: `Access the PPS API.

Note that this CLI is experimental and does not even check for common errors.
The environment variable PPS_ADDRESS controls what server the CLI connects to, the default is 0.0.0.0:651.`,
	}

	var image string
	var outParentCommitId string
	createJob := &cobra.Command{
		Use:   "create-job in-repo-name in-commit-id out-repo-name -i image -p out-parent-commit-id command [args]",
		Short: "Create a new job. Returns the id of the created job.",
		Long: `Create a new job. With repo-name/commit-id as input and
out-repo-name as output. A commit will be created for the output.
You can find out the name of the commit with inspect-job.`,
		Run: func(cmd *cobra.Command, args []string) {
			job, err := jobAPIClient.CreateJob(
				context.Background(),
				&pps.CreateJobRequest{
					Spec: &pps.CreateJobRequest_Transform{
						Transform: &pps.Transform{
							Image: image,
							Cmd:   args[3:],
						},
					},
					Input: &pfs.Commit{
						Repo: &pfs.Repo{
							Name: args[0],
						},
						Id: args[1],
					},
					OutputParent: &pfs.Commit{
						Repo: &pfs.Repo{
							Name: args[2],
						},
						Id: outParentCommitId,
					},
				})
			if err != nil {
				errorAndExit("Error from CreateJob: %s", err.Error())
			}
			fmt.Println(job.Id)
		},
	}
	createJob.Flags().StringVarP(&image, "image", "i", "ubuntu", "The image to run the job in.")
	createJob.Flags().StringVarP(&outParentCommitId, "parent", "p", "", "The parent to use for the output commit.")

	inspectJob := &cobra.Command{
		Use:   "inspect-job job-id",
		Short: "Return info about a job.",
		Long:  "Return info about a job.",
		Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
			jobInfo, err := jobAPIClient.InspectJob(
				context.Background(),
				&pps.InspectJobRequest{
					Job: &pps.Job{
						Id: args[0],
					},
				},
			)
			if err != nil {
				errorAndExit("Error from InspectJob: %s", err.Error())
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintJobHeader(writer)
			pretty.PrintJobInfo(writer, jobInfo)
			return writer.Flush()
		}),
	}

	var pipelineName string
	listJob := &cobra.Command{
		Use:   "list-job -p pipeline-name",
		Short: "Return info about all jobs.",
		Long:  "Return info about all jobs.",
		Run: pkgcobra.RunFixedArgs(0, func(args []string) error {
			var pipeline *pps.Pipeline
			if pipelineName != "" {
				pipeline = &pps.Pipeline{
					Name: pipelineName,
				}
			}
			jobInfos, err := jobAPIClient.ListJob(
				context.Background(),
				&pps.ListJobRequest{
					Pipeline: pipeline,
				},
			)
			if err != nil {
				errorAndExit("Error from InspectJob: %s", err.Error())
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintJobHeader(writer)
			for _, jobInfo := range jobInfos.JobInfo {
				pretty.PrintJobInfo(writer, jobInfo)
			}
			return writer.Flush()
		}),
	}
	listJob.Flags().StringVarP(&pipelineName, "pipeline", "p", "", "Limit to jobs made by pipeline.")

	getJobLogs := &cobra.Command{
		Use:   "logs job-id",
		Short: "Return logs from a job.",
		Long:  "Return logs from a job.",
		Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
			logsClient, err := jobAPIClient.GetJobLogs(
				context.Background(),
				&pps.GetJobLogsRequest{
					Job: &pps.Job{
						Id: args[0],
					},
					OutputStream: pps.OutputStream_OUTPUT_STREAM_ALL,
				},
			)
			if err != nil {
				errorAndExit("Error from InspectJob: %s", err.Error())
			}
			if err := protostream.WriteFromStreamingBytesClient(logsClient, os.Stdout); err != nil {
				return err
			}
			return nil
		}),
	}

	createPipeline := &cobra.Command{
		Use:   "create-pipeline pipeline-name input-repo output-repo -i image command [args]",
		Short: "Create a new pipeline.",
		Long:  "Create a new pipeline.",
		Run: func(cmd *cobra.Command, args []string) {
			if _, err := pipelineAPIClient.CreatePipeline(
				context.Background(),
				&pps.CreatePipelineRequest{
					Pipeline: &pps.Pipeline{
						Name: args[0],
					},
					Transform: &pps.Transform{
						Image: image,
						Cmd:   args[3:],
					},
					Input: &pfs.Repo{
						Name: args[1],
					},
					Output: &pfs.Repo{
						Name: args[2],
					},
				},
			); err != nil {
				errorAndExit("Error from CreatePipeline: %s", err.Error())
			}
		},
	}
	createPipeline.Flags().StringVarP(&image, "image", "i", "ubuntu", "The image to run the pipeline's jobs in.")

	inspectPipeline := &cobra.Command{
		Use:   "inspect-pipeline pipeline-name",
		Short: "Return info about a pipeline.",
		Long:  "Return info about a pipeline.",
		Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
			pipelineInfo, err := pipelineAPIClient.InspectPipeline(
				context.Background(),
				&pps.InspectPipelineRequest{
					Pipeline: &pps.Pipeline{
						Name: args[0],
					},
				},
			)
			if err != nil {
				errorAndExit("Error from InspectPipeline: %s", err.Error())
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintPipelineHeader(writer)
			pretty.PrintPipelineInfo(writer, pipelineInfo)
			return writer.Flush()
		}),
	}

	listPipeline := &cobra.Command{
		Use:   "list-pipeline",
		Short: "Return info about all pipelines.",
		Long:  "Return info about all pipelines.",
		Run: pkgcobra.RunFixedArgs(0, func(args []string) error {
			pipelineInfos, err := pipelineAPIClient.ListPipeline(
				context.Background(),
				&pps.ListPipelineRequest{},
			)
			if err != nil {
				errorAndExit("Error from ListPipeline: %s", err.Error())
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintPipelineHeader(writer)
			for _, pipelineInfo := range pipelineInfos.PipelineInfo {
				pretty.PrintPipelineInfo(writer, pipelineInfo)
			}
			return writer.Flush()
		}),
	}

	deletePipeline := &cobra.Command{
		Use:   "delete-pipeline pipeline-name",
		Short: "Delete a pipeline.",
		Long:  "Delete a pipeline.",
		Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
			if _, err := pipelineAPIClient.DeletePipeline(
				context.Background(),
				&pps.DeletePipelineRequest{
					Pipeline: &pps.Pipeline{
						Name: args[0],
					},
				},
			); err != nil {
				errorAndExit("Error from DeletePipeline: %s", err.Error())
			}
			return nil
		}),
	}

	rootCmd.AddCommand(createJob)
	rootCmd.AddCommand(inspectJob)
	rootCmd.AddCommand(listJob)
	rootCmd.AddCommand(getJobLogs)
	rootCmd.AddCommand(createPipeline)
	rootCmd.AddCommand(inspectPipeline)
	rootCmd.AddCommand(listPipeline)
	rootCmd.AddCommand(deletePipeline)
	return rootCmd.Execute()
}

func errorAndExit(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, "%s\n", fmt.Sprintf(format, args...))
	os.Exit(1)
}
<commit_msg>Don't put flags in usage. Cobra does it for me :).<commit_after>package main

import (
	"fmt"
	"os"
	"strings"
	"text/tabwriter"

	"golang.org/x/net/context"

	"github.com/pachyderm/pachyderm/src/pfs"
	"github.com/pachyderm/pachyderm/src/pps"
	"github.com/pachyderm/pachyderm/src/pps/pretty"

	"go.pedge.io/env"
	"go.pedge.io/pkg/cobra"
	"go.pedge.io/proto/stream"

	"google.golang.org/grpc"

	"github.com/spf13/cobra"
)

var (
	defaultEnv = map[string]string{
		"PPS_ADDRESS": "0.0.0.0:651",
	}
)

type appEnv struct {
	PachydermPpsd1Port string `env:"PACHYDERM_PPSD_1_PORT"`
	Address            string `env:"PPS_ADDRESS"`
}

func main() {
	env.Main(do, &appEnv{}, defaultEnv)
}

func do(appEnvObj interface{}) error {
	appEnv := appEnvObj.(*appEnv)
	address := appEnv.PachydermPpsd1Port
	if address == "" {
		address = appEnv.Address
	} else {
		address = strings.Replace(address, "tcp://", "", -1)
	}
	clientConn, err := grpc.Dial(address, grpc.WithInsecure())
	if err != nil {
		return err
	}
	jobAPIClient := pps.NewJobAPIClient(clientConn)
	pipelineAPIClient := pps.NewPipelineAPIClient(clientConn)
	rootCmd := &cobra.Command{
		Use: "pps",
		Long: `Access the PPS API.

Note that this CLI is experimental and does not even check for common errors.
The environment variable PPS_ADDRESS controls what server the CLI connects to, the default is 0.0.0.0:651.`,
	}

	var image string
	var outParentCommitId string
	createJob := &cobra.Command{
		Use:   "create-job in-repo-name in-commit-id out-repo-name command [args]",
		Short: "Create a new job. Returns the id of the created job.",
		Long: `Create a new job. With repo-name/commit-id as input and
out-repo-name as output. A commit will be created for the output.
You can find out the name of the commit with inspect-job.`,
		Run: func(cmd *cobra.Command, args []string) {
			job, err := jobAPIClient.CreateJob(
				context.Background(),
				&pps.CreateJobRequest{
					Spec: &pps.CreateJobRequest_Transform{
						Transform: &pps.Transform{
							Image: image,
							Cmd:   args[3:],
						},
					},
					Input: &pfs.Commit{
						Repo: &pfs.Repo{
							Name: args[0],
						},
						Id: args[1],
					},
					OutputParent: &pfs.Commit{
						Repo: &pfs.Repo{
							Name: args[2],
						},
						Id: outParentCommitId,
					},
				})
			if err != nil {
				errorAndExit("Error from CreateJob: %s", err.Error())
			}
			fmt.Println(job.Id)
		},
	}
	createJob.Flags().StringVarP(&image, "image", "i", "ubuntu", "The image to run the job in.")
	createJob.Flags().StringVarP(&outParentCommitId, "parent", "p", "", "The parent to use for the output commit.")

	inspectJob := &cobra.Command{
		Use:   "inspect-job job-id",
		Short: "Return info about a job.",
		Long:  "Return info about a job.",
		Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
			jobInfo, err := jobAPIClient.InspectJob(
				context.Background(),
				&pps.InspectJobRequest{
					Job: &pps.Job{
						Id: args[0],
					},
				},
			)
			if err != nil {
				errorAndExit("Error from InspectJob: %s", err.Error())
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintJobHeader(writer)
			pretty.PrintJobInfo(writer, jobInfo)
			return writer.Flush()
		}),
	}

	var pipelineName string
	listJob := &cobra.Command{
		Use:   "list-job -p pipeline-name",
		Short: "Return info about all jobs.",
		Long:  "Return info about all jobs.",
		Run: pkgcobra.RunFixedArgs(0, func(args []string) error {
			var pipeline *pps.Pipeline
			if pipelineName != "" {
				pipeline = &pps.Pipeline{
					Name: pipelineName,
				}
			}
			jobInfos, err := jobAPIClient.ListJob(
				context.Background(),
				&pps.ListJobRequest{
					Pipeline: pipeline,
				},
			)
			if err != nil {
				errorAndExit("Error from InspectJob: %s", err.Error())
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintJobHeader(writer)
			for _, jobInfo := range jobInfos.JobInfo {
				pretty.PrintJobInfo(writer, jobInfo)
			}
			return writer.Flush()
		}),
	}
	listJob.Flags().StringVarP(&pipelineName, "pipeline", "p", "", "Limit to jobs made by pipeline.")

	getJobLogs := &cobra.Command{
		Use:   "logs job-id",
		Short: "Return logs from a job.",
		Long:  "Return logs from a job.",
		Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
			logsClient, err := jobAPIClient.GetJobLogs(
				context.Background(),
				&pps.GetJobLogsRequest{
					Job: &pps.Job{
						Id: args[0],
					},
					OutputStream: pps.OutputStream_OUTPUT_STREAM_ALL,
				},
			)
			if err != nil {
				errorAndExit("Error from InspectJob: %s", err.Error())
			}
			if err := protostream.WriteFromStreamingBytesClient(logsClient, os.Stdout); err != nil {
				return err
			}
			return nil
		}),
	}

	createPipeline := &cobra.Command{
		Use:   "create-pipeline pipeline-name input-repo output-repo command [args]",
		Short: "Create a new pipeline.",
		Long:  "Create a new pipeline.",
		Run: func(cmd *cobra.Command, args []string) {
			if _, err := pipelineAPIClient.CreatePipeline(
				context.Background(),
				&pps.CreatePipelineRequest{
					Pipeline: &pps.Pipeline{
						Name: args[0],
					},
					Transform: &pps.Transform{
						Image: image,
						Cmd:   args[3:],
					},
					Input: &pfs.Repo{
						Name: args[1],
					},
					Output: &pfs.Repo{
						Name: args[2],
					},
				},
			); err != nil {
				errorAndExit("Error from CreatePipeline: %s", err.Error())
			}
		},
	}
	createPipeline.Flags().StringVarP(&image, "image", "i", "ubuntu", "The image to run the pipeline's jobs in.")

	inspectPipeline := &cobra.Command{
		Use:   "inspect-pipeline pipeline-name",
		Short: "Return info about a pipeline.",
		Long:  "Return info about a pipeline.",
		Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
			pipelineInfo, err := pipelineAPIClient.InspectPipeline(
				context.Background(),
				&pps.InspectPipelineRequest{
					Pipeline: &pps.Pipeline{
						Name: args[0],
					},
				},
			)
			if err != nil {
				errorAndExit("Error from InspectPipeline: %s", err.Error())
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintPipelineHeader(writer)
			pretty.PrintPipelineInfo(writer, pipelineInfo)
			return writer.Flush()
		}),
	}

	listPipeline := &cobra.Command{
		Use:   "list-pipeline",
		Short: "Return info about all pipelines.",
		Long:  "Return info about all pipelines.",
		Run: pkgcobra.RunFixedArgs(0, func(args []string) error {
			pipelineInfos, err := pipelineAPIClient.ListPipeline(
				context.Background(),
				&pps.ListPipelineRequest{},
			)
			if err != nil {
				errorAndExit("Error from ListPipeline: %s", err.Error())
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintPipelineHeader(writer)
			for _, pipelineInfo := range pipelineInfos.PipelineInfo {
				pretty.PrintPipelineInfo(writer, pipelineInfo)
			}
			return writer.Flush()
		}),
	}

	deletePipeline := &cobra.Command{
		Use:   "delete-pipeline pipeline-name",
		Short: "Delete a pipeline.",
		Long:  "Delete a pipeline.",
		Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
			if _, err := pipelineAPIClient.DeletePipeline(
				context.Background(),
				&pps.DeletePipelineRequest{
					Pipeline: &pps.Pipeline{
						Name: args[0],
					},
				},
			); err != nil {
				errorAndExit("Error from DeletePipeline: %s", err.Error())
			}
			return nil
		}),
	}

	rootCmd.AddCommand(createJob)
	rootCmd.AddCommand(inspectJob)
	rootCmd.AddCommand(listJob)
	rootCmd.AddCommand(getJobLogs)
	rootCmd.AddCommand(createPipeline)
	rootCmd.AddCommand(inspectPipeline)
	rootCmd.AddCommand(listPipeline)
	rootCmd.AddCommand(deletePipeline)
	return rootCmd.Execute()
}

func errorAndExit(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, "%s\n", fmt.Sprintf(format, args...))
	os.Exit(1)
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 Niklas Wolber
// This file is licensed under the MIT license.
// See the LICENSE file for more information.

package job

import (
	"context"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"sync"
	"time"

	"golang.org/x/crypto/ssh"
)

type storeElement struct {
	ref      int
	client   *sshClient
	lastUsed time.Time
}

type sshClientStore struct {
	clients map[string]*storeElement
	m       sync.Mutex
}

var (
	errUnknownUser = errors.New("keyboard interactive: unknown user")
)

var (
	store *sshClientStore
)

// InitializeSSHClientStore initialies the global SSH connection store and
// sets the time-to-live for unused connections.
func InitializeSSHClientStore(ttl time.Duration) {
	store = &sshClientStore{
		clients: make(map[string]*storeElement),
	}

	// This go routine runs for the lifetime of the program.
	go func() {
		for {
			watchTime := time.Duration(float64(ttl.Nanoseconds()) * 0.1)
			<-time.Tick(watchTime)

			func() {
				store.m.Lock()
				defer store.m.Unlock()

				for key, elem := range store.clients {
					if diff := time.Now().Sub(elem.lastUsed); elem.ref <= 0 && diff > ttl {
						log.Println("connection to", key, "unused for", diff, "closing")
						elem.client.c.Close()
						delete(store.clients, key)
					}
				}
			}()
		}
	}()
}

func newSSHClient(ctx context.Context, addr, user, keyFile, password string, keyboardInteractive map[string]string) (*sshClient, error) {
	key := fmt.Sprintf("%s@%s", user, addr)

	store.m.Lock()
	defer store.m.Unlock()

	elem, ok := store.clients[key]

	if !ok {
		client, err := createClient(addr, user, keyFile, password, keyboardInteractive)
		if err != nil {
			return nil, err
		}

		go func(client *sshClient) {
			err := client.c.Wait()
			log.Println("connection closed, removing from store:", err)

			store.m.Lock()
			defer store.m.Unlock()
			if _, ok := store.clients[key]; ok {
				delete(store.clients, key)
			} else {
				log.Println("didn't find a client to remove")
			}
		}(client)

		elem = &storeElement{
			client: client,
		}
		store.clients[key] = elem
	} else {
		log.Println("reusing existing connection")
	}

	elem.ref++
	elem.lastUsed = time.Now()

	go func(ctx context.Context, client *sshClient) {
		<-ctx.Done()
		store.m.Lock()
		defer store.m.Unlock()

		store.clients[key].ref--
	}(ctx, elem.client)

	return elem.client, nil
}

type sshClient struct {
	c *ssh.Client
}

func createClient(addr, user, keyFile, password string, keyboardInteractive map[string]string) (*sshClient, error) {
	config := &ssh.ClientConfig{
		User: user,
		Auth: []ssh.AuthMethod{},
	}

	if keyFile != "" {
		s, _, err := readPrivateKeyFile(keyFile, nil)
		if err != nil {
			err = fmt.Errorf("Unable to read private key %s", err)
			log.Println(err)
			return nil, err
		}

		signer, err := ssh.NewSignerFromSigner(s)
		if err != nil {
			err = fmt.Errorf("Unable to turn signer into signer %s", err)
			log.Println(err)
			return nil, err
		}
		config.Auth = append(config.Auth, ssh.PublicKeys(signer))
	}

	if password != "" {
		config.Auth = append(config.Auth, ssh.Password(password))
	}

	if keyboardInteractive != nil && len(keyboardInteractive) > 0 {
		config.Auth = append(config.Auth, ssh.KeyboardInteractive(keyboardInteractiveChallenge(user, keyboardInteractive)))
	}

	log.Println("no existing connection, connecting to", addr)
	client, err := ssh.Dial("tcp", addr, config)
	if err != nil {
		return nil, err
	}

	log.Println("connected to", addr)
	return &sshClient{
		c: client,
	}, nil
}

func keyboardInteractiveChallenge(user string, keyboardInteractive map[string]string) ssh.KeyboardInteractiveChallenge {
	return func(challengeUser, instruction string, questions []string, echos []bool) ([]string, error) {
		if len(questions) == 0 {
			return nil, nil
		}

		var answers []string
		for _, question := range questions {
			if answer, ok := keyboardInteractive[question]; ok {
				answers = append(answers, answer)
			}
		}

		return answers, nil
	}
}

func (s *sshClient) executeCommand(ctx context.Context, command string, stdout, stderr io.Writer) error {
	l, ok := ctx.Value(loggerKey).(Logger)
	if !ok || l == nil {
		l = log.New(os.Stderr, "", log.LstdFlags)
	}

	select {
	case <-ctx.Done():
		l.Printf("won't execute %q because context is done", command)
		return nil
	default:
	}

	session, err := s.c.NewSession()
	if err != nil {
		l.Println("failed to create session:", err)
	}
	defer session.Close()

	if stdout != nil {
		session.Stdout = stdout
	}

	if stderr != nil {
		session.Stderr = stderr
	}

	l.Printf("executing %q", command)
	if err := session.Start(command); err != nil {
		l.Printf("failed to start: %q, %s", command, err)
	}

	done := make(chan error)
	go func() {
		done <- session.Wait()
	}()

	select {
	case <-ctx.Done():
		l.Println("closing session, context done")
		return nil
	case err, _ := <-done:
		if err != nil {
			l.Printf("executing %q failed: %s", command, err)
			return err
		}
	}

	l.Printf("%q executed successfully", command)
	return nil
}

func (s *sshClient) forwardRemote(ctx context.Context, remoteAddr, localAddr string) {
	forwardRemote(ctx, s.c, remoteAddr, localAddr)
}

func (s *sshClient) forwardTunnel(ctx context.Context, remoteAddr, localAddr string) {
	forwardLocal(ctx, s.c, remoteAddr, localAddr)
}
<commit_msg>Remove confusing log message<commit_after>// Copyright (c) 2016 Niklas Wolber
// This file is licensed under the MIT license.
// See the LICENSE file for more information.

package job

import (
	"context"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"sync"
	"time"

	"golang.org/x/crypto/ssh"
)

type storeElement struct {
	ref      int
	client   *sshClient
	lastUsed time.Time
}

type sshClientStore struct {
	clients map[string]*storeElement
	m       sync.Mutex
}

var (
	errUnknownUser = errors.New("keyboard interactive: unknown user")
)

var (
	store *sshClientStore
)

// InitializeSSHClientStore initialies the global SSH connection store and
// sets the time-to-live for unused connections.
func InitializeSSHClientStore(ttl time.Duration) {
	store = &sshClientStore{
		clients: make(map[string]*storeElement),
	}

	// This go routine runs for the lifetime of the program.
	go func() {
		for {
			watchTime := time.Duration(float64(ttl.Nanoseconds()) * 0.1)
			<-time.Tick(watchTime)

			func() {
				store.m.Lock()
				defer store.m.Unlock()

				for key, elem := range store.clients {
					if diff := time.Now().Sub(elem.lastUsed); elem.ref <= 0 && diff > ttl {
						log.Println("connection to", key, "unused for", diff, "closing")
						elem.client.c.Close()
						delete(store.clients, key)
					}
				}
			}()
		}
	}()
}

func newSSHClient(ctx context.Context, addr, user, keyFile, password string, keyboardInteractive map[string]string) (*sshClient, error) {
	key := fmt.Sprintf("%s@%s", user, addr)

	store.m.Lock()
	defer store.m.Unlock()

	elem, ok := store.clients[key]

	if !ok {
		client, err := createClient(addr, user, keyFile, password, keyboardInteractive)
		if err != nil {
			return nil, err
		}

		go func(client *sshClient) {
			err := client.c.Wait()
			log.Println("connection closed, removing from store:", err)

			store.m.Lock()
			defer store.m.Unlock()
			if _, ok := store.clients[key]; ok {
				delete(store.clients, key)
			}
		}(client)

		elem = &storeElement{
			client: client,
		}
		store.clients[key] = elem
	} else {
		log.Println("reusing existing connection")
	}

	elem.ref++
	elem.lastUsed = time.Now()

	go func(ctx context.Context, client *sshClient) {
		<-ctx.Done()
		store.m.Lock()
		defer store.m.Unlock()

		store.clients[key].ref--
	}(ctx, elem.client)

	return elem.client, nil
}

type sshClient struct {
	c *ssh.Client
}

func createClient(addr, user, keyFile, password string, keyboardInteractive map[string]string) (*sshClient, error) {
	config := &ssh.ClientConfig{
		User: user,
		Auth: []ssh.AuthMethod{},
	}

	if keyFile != "" {
		s, _, err := readPrivateKeyFile(keyFile, nil)
		if err != nil {
			err = fmt.Errorf("Unable to read private key %s", err)
			log.Println(err)
			return nil, err
		}

		signer, err := ssh.NewSignerFromSigner(s)
		if err != nil {
			err = fmt.Errorf("Unable to turn signer into signer %s", err)
			log.Println(err)
			return nil, err
		}
		config.Auth = append(config.Auth, ssh.PublicKeys(signer))
	}

	if password != "" {
		config.Auth = append(config.Auth, ssh.Password(password))
	}

	if keyboardInteractive != nil && len(keyboardInteractive) > 0 {
		config.Auth = append(config.Auth, ssh.KeyboardInteractive(keyboardInteractiveChallenge(user, keyboardInteractive)))
	}

	log.Println("no existing connection, connecting to", addr)
	client, err := ssh.Dial("tcp", addr, config)
	if err != nil {
		return nil, err
	}

	log.Println("connected to", addr)
	return &sshClient{
		c: client,
	}, nil
}

func keyboardInteractiveChallenge(user string, keyboardInteractive map[string]string) ssh.KeyboardInteractiveChallenge {
	return func(challengeUser, instruction string, questions []string, echos []bool) ([]string, error) {
		if len(questions) == 0 {
			return nil, nil
		}

		var answers []string
		for _, question := range questions {
			if answer, ok := keyboardInteractive[question]; ok {
				answers = append(answers, answer)
			}
		}

		return answers, nil
	}
}

func (s *sshClient) executeCommand(ctx context.Context, command string, stdout, stderr io.Writer) error {
	l, ok := ctx.Value(loggerKey).(Logger)
	if !ok || l == nil {
		l = log.New(os.Stderr, "", log.LstdFlags)
	}

	select {
	case <-ctx.Done():
		l.Printf("won't execute %q because context is done", command)
		return nil
	default:
	}

	session, err := s.c.NewSession()
	if err != nil {
		l.Println("failed to create session:", err)
	}
	defer session.Close()

	if stdout != nil {
		session.Stdout = stdout
	}

	if stderr != nil {
		session.Stderr = stderr
	}

	l.Printf("executing %q", command)
	if err := session.Start(command); err != nil {
		l.Printf("failed to start: %q, %s", command, err)
	}

	done := make(chan error)
	go func() {
		done <- session.Wait()
	}()

	select {
	case <-ctx.Done():
		l.Println("closing session, context done")
		return nil
	case err, _ := <-done:
		if err != nil {
			l.Printf("executing %q failed: %s", command, err)
			return err
		}
	}

	l.Printf("%q executed successfully", command)
	return nil
}

func (s *sshClient) forwardRemote(ctx context.Context, remoteAddr, localAddr string) {
	forwardRemote(ctx, s.c, remoteAddr, localAddr)
}

func (s *sshClient) forwardTunnel(ctx context.Context, remoteAddr, localAddr string) {
	forwardLocal(ctx, s.c, remoteAddr, localAddr)
}
<|endoftext|>


<commit_before>package job

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"net/http"
	"os/exec"
	"strings"
	"time"

	"github.com/mattn/go-shellwords"
	log "github.com/sirupsen/logrus"
)

type JobRunner struct {
	job  *Job
	meta Metadata

	numberOfAttempts uint
	currentRetries   uint
	currentStat      *JobStat
}

var (
	ErrJobDisabled    = errors.New("Job cannot run, as it is disabled")
	ErrCmdIsEmpty     = errors.New("Job Command is empty.")
	ErrJobTypeInvalid = errors.New("Job Type is not valid.")
)

// Run calls the appropriate run function, collects metadata around the success
// or failure of the Job's execution, and schedules the next run.
func (j *JobRunner) Run(cache JobCache) (*JobStat, Metadata, error) {
	j.job.lock.RLock()
	defer j.job.lock.RUnlock()

	j.meta.LastAttemptedRun = j.job.clk.Time().Now()

	if j.job.Disabled {
		log.Infof("Job %s tried to run, but exited early because its disabled.", j.job.Name)
		return nil, j.meta, ErrJobDisabled
	}

	log.Infof("Job %s:%s started.", j.job.Name, j.job.Id)

	j.runSetup()

	for {
		var err error
		if j.job.JobType == LocalJob {
			err = j.LocalRun()
		} else if j.job.JobType == RemoteJob {
			err = j.RemoteRun()
		} else {
			err = ErrJobTypeInvalid
		}

		if err != nil {
			// Log Error in Metadata
			// TODO - Error Reporting, email error
			log.Errorln("Error running job:", j.currentStat.JobId)
			log.Errorln(err)

			j.meta.ErrorCount++
			j.meta.LastError = j.job.clk.Time().Now()

			// Handle retrying
			if j.shouldRetry() {
				j.currentRetries--
				continue
			}

			j.collectStats(false)
			j.meta.NumberOfFinishedRuns++

			// TODO: Wrap error into something better.
			return j.currentStat, j.meta, err
		} else {
			break
		}
	}

	log.Infof("Job %s:%s finished.", j.job.Name, j.job.Id)
	j.meta.SuccessCount++
	j.meta.NumberOfFinishedRuns++
	j.meta.LastSuccess = j.job.clk.Time().Now()

	j.collectStats(true)

	// Run Dependent Jobs
	if len(j.job.DependentJobs) != 0 {
		for _, id := range j.job.DependentJobs {
			newJob, err := cache.Get(id)
			if err != nil {
				log.Errorf("Error retrieving dependent job with id of %s", id)
			} else {
				newJob.Run(cache)
			}
		}
	}

	return j.currentStat, j.meta, nil
}

// LocalRun executes the Job's local shell command
func (j *JobRunner) LocalRun() error {
	return j.runCmd()
}

// RemoteRun sends a http request, and checks if the response is valid in time,
func (j *JobRunner) RemoteRun() error {
	// Calculate a response timeout
	timeout := j.responseTimeout()

	ctx, cncl := context.Background(), func() {}
	if timeout > 0 {
		ctx, cncl = context.WithTimeout(ctx, timeout)
		defer cncl()
	}

	// Normalize the method passed by the user
	method := strings.ToUpper(j.job.RemoteProperties.Method)
	bodyBuffer := bytes.NewBufferString(j.job.RemoteProperties.Body)
	req, err := http.NewRequest(method, j.job.RemoteProperties.Url, bodyBuffer)
	if err != nil {
		return err
	}

	// Set default or user's passed headers
	j.setHeaders(req)

	// Do the request
	res, err := http.DefaultClient.Do(req.WithContext(ctx))
	if err != nil {
		return err
	}

	// Check if we got any of the status codes the user asked for
	if j.checkExpected(res.StatusCode) {
		return nil
	} else {
		return errors.New(res.Status)
	}
}

func initShParser() *shellwords.Parser {
	shParser := shellwords.NewParser()
	shParser.ParseEnv = true
	shParser.ParseBacktick = true
	return shParser
}

func (j *JobRunner) runCmd() error {
	j.numberOfAttempts++

	// Execute command
	shParser := initShParser()
	args, err := shParser.Parse(j.job.Command)
	if err != nil {
		return err
	}
	if len(args) == 0 {
		return ErrCmdIsEmpty
	}

	cmd := exec.Command(args[0], args[1:]...)
	out, err := cmd.CombinedOutput()
	if err != nil {
		return fmt.Errorf("%s: %s", err, strings.TrimSpace(string(out)))
	}
	return nil
}

func (j *JobRunner) shouldRetry() bool {
	// Check number of retries left
	if j.currentRetries == 0 {
		return false
	}

	// Check Epsilon
	if j.job.Epsilon != "" && j.job.Schedule != "" {
		if !j.job.epsilonDuration.IsZero() {
			timeSinceStart := j.job.clk.Time().Now().Sub(j.job.NextRunAt)
			timeLeftToRetry := j.job.epsilonDuration.RelativeTo(j.job.clk.Time().Now()) - timeSinceStart
			if timeLeftToRetry < 0 {
				return false
			}
		}
	}

	return true
}

func (j *JobRunner) runSetup() {
	// Setup Job Stat
	j.currentStat = NewJobStat(j.job.Id)

	// Init retries
	j.currentRetries = j.job.Retries
}

func (j *JobRunner) collectStats(success bool) {
	j.currentStat.ExecutionDuration = j.job.clk.Time().Now().Sub(j.currentStat.RanAt)
	j.currentStat.Success = success
	j.currentStat.NumberOfRetries = j.job.Retries - j.currentRetries
}

func (j *JobRunner) checkExpected(statusCode int) bool {
	// If no expected response codes passed, add 200 status code as expected
	if len(j.job.RemoteProperties.ExpectedResponseCodes) == 0 {
		j.job.RemoteProperties.ExpectedResponseCodes = append(j.job.RemoteProperties.ExpectedResponseCodes, 200)
	}
	for _, expected := range j.job.RemoteProperties.ExpectedResponseCodes {
		if expected == statusCode {
			return true
		}
	}

	return false
}

// responseTimeout sets a default timeout if none specified
func (j *JobRunner) responseTimeout() time.Duration {
	responseTimeout := j.job.RemoteProperties.Timeout
	if responseTimeout == 0 {

		// set default to 30 seconds
		responseTimeout = 30
	}
	return time.Duration(responseTimeout) * time.Second
}

// setHeaders sets default and user specific headers to the http request
func (j *JobRunner) setHeaders(req *http.Request) {
	// A valid assumption is that the user is sending something in json cause we're past 2017
	if j.job.RemoteProperties.Headers["Content-Type"] == nil {
		j.job.RemoteProperties.Headers["Content-Type"] = []string{"application/json"}
	}
	req.Header = j.job.RemoteProperties.Headers
}
<commit_msg>Fix panic related to headers.<commit_after>package job

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"net/http"
	"os/exec"
	"strings"
	"time"

	"github.com/mattn/go-shellwords"
	log "github.com/sirupsen/logrus"
)

type JobRunner struct {
	job  *Job
	meta Metadata

	numberOfAttempts uint
	currentRetries   uint
	currentStat      *JobStat
}

var (
	ErrJobDisabled    = errors.New("Job cannot run, as it is disabled")
	ErrCmdIsEmpty     = errors.New("Job Command is empty.")
	ErrJobTypeInvalid = errors.New("Job Type is not valid.")
)

// Run calls the appropriate run function, collects metadata around the success
// or failure of the Job's execution, and schedules the next run.
func (j *JobRunner) Run(cache JobCache) (*JobStat, Metadata, error) {
	j.job.lock.RLock()
	defer j.job.lock.RUnlock()

	j.meta.LastAttemptedRun = j.job.clk.Time().Now()

	if j.job.Disabled {
		log.Infof("Job %s tried to run, but exited early because its disabled.", j.job.Name)
		return nil, j.meta, ErrJobDisabled
	}

	log.Infof("Job %s:%s started.", j.job.Name, j.job.Id)

	j.runSetup()

	for {
		var err error
		if j.job.JobType == LocalJob {
			err = j.LocalRun()
		} else if j.job.JobType == RemoteJob {
			err = j.RemoteRun()
		} else {
			err = ErrJobTypeInvalid
		}

		if err != nil {
			// Log Error in Metadata
			// TODO - Error Reporting, email error
			log.Errorln("Error running job:", j.currentStat.JobId)
			log.Errorln(err)

			j.meta.ErrorCount++
			j.meta.LastError = j.job.clk.Time().Now()

			// Handle retrying
			if j.shouldRetry() {
				j.currentRetries--
				continue
			}

			j.collectStats(false)
			j.meta.NumberOfFinishedRuns++

			// TODO: Wrap error into something better.
			return j.currentStat, j.meta, err
		} else {
			break
		}
	}

	log.Infof("Job %s:%s finished.", j.job.Name, j.job.Id)
	j.meta.SuccessCount++
	j.meta.NumberOfFinishedRuns++
	j.meta.LastSuccess = j.job.clk.Time().Now()

	j.collectStats(true)

	// Run Dependent Jobs
	if len(j.job.DependentJobs) != 0 {
		for _, id := range j.job.DependentJobs {
			newJob, err := cache.Get(id)
			if err != nil {
				log.Errorf("Error retrieving dependent job with id of %s", id)
			} else {
				newJob.Run(cache)
			}
		}
	}

	return j.currentStat, j.meta, nil
}

// LocalRun executes the Job's local shell command
func (j *JobRunner) LocalRun() error {
	return j.runCmd()
}

// RemoteRun sends a http request, and checks if the response is valid in time,
func (j *JobRunner) RemoteRun() error {
	// Calculate a response timeout
	timeout := j.responseTimeout()

	ctx, cncl := context.Background(), func() {}
	if timeout > 0 {
		ctx, cncl = context.WithTimeout(ctx, timeout)
		defer cncl()
	}

	// Normalize the method passed by the user
	method := strings.ToUpper(j.job.RemoteProperties.Method)
	bodyBuffer := bytes.NewBufferString(j.job.RemoteProperties.Body)
	req, err := http.NewRequest(method, j.job.RemoteProperties.Url, bodyBuffer)
	if err != nil {
		return err
	}

	// Set default or user's passed headers
	j.setHeaders(req)

	// Do the request
	res, err := http.DefaultClient.Do(req.WithContext(ctx))
	if err != nil {
		return err
	}

	// Check if we got any of the status codes the user asked for
	if j.checkExpected(res.StatusCode) {
		return nil
	} else {
		return errors.New(res.Status)
	}
}

func initShParser() *shellwords.Parser {
	shParser := shellwords.NewParser()
	shParser.ParseEnv = true
	shParser.ParseBacktick = true
	return shParser
}

func (j *JobRunner) runCmd() error {
	j.numberOfAttempts++

	// Execute command
	shParser := initShParser()
	args, err := shParser.Parse(j.job.Command)
	if err != nil {
		return err
	}
	if len(args) == 0 {
		return ErrCmdIsEmpty
	}

	cmd := exec.Command(args[0], args[1:]...)
	out, err := cmd.CombinedOutput()
	if err != nil {
		return fmt.Errorf("%s: %s", err, strings.TrimSpace(string(out)))
	}
	return nil
}

func (j *JobRunner) shouldRetry() bool {
	// Check number of retries left
	if j.currentRetries == 0 {
		return false
	}

	// Check Epsilon
	if j.job.Epsilon != "" && j.job.Schedule != "" {
		if !j.job.epsilonDuration.IsZero() {
			timeSinceStart := j.job.clk.Time().Now().Sub(j.job.NextRunAt)
			timeLeftToRetry := j.job.epsilonDuration.RelativeTo(j.job.clk.Time().Now()) - timeSinceStart
			if timeLeftToRetry < 0 {
				return false
			}
		}
	}

	return true
}

func (j *JobRunner) runSetup() {
	// Setup Job Stat
	j.currentStat = NewJobStat(j.job.Id)

	// Init retries
	j.currentRetries = j.job.Retries
}

func (j *JobRunner) collectStats(success bool) {
	j.currentStat.ExecutionDuration = j.job.clk.Time().Now().Sub(j.currentStat.RanAt)
	j.currentStat.Success = success
	j.currentStat.NumberOfRetries = j.job.Retries - j.currentRetries
}

func (j *JobRunner) checkExpected(statusCode int) bool {
	// If no expected response codes passed, add 200 status code as expected
	if len(j.job.RemoteProperties.ExpectedResponseCodes) == 0 {
		j.job.RemoteProperties.ExpectedResponseCodes = append(j.job.RemoteProperties.ExpectedResponseCodes, 200)
	}
	for _, expected := range j.job.RemoteProperties.ExpectedResponseCodes {
		if expected == statusCode {
			return true
		}
	}

	return false
}

// responseTimeout sets a default timeout if none specified
func (j *JobRunner) responseTimeout() time.Duration {
	responseTimeout := j.job.RemoteProperties.Timeout
	if responseTimeout == 0 {

		// set default to 30 seconds
		responseTimeout = 30
	}
	return time.Duration(responseTimeout) * time.Second
}

// setHeaders sets default and user specific headers to the http request
func (j *JobRunner) setHeaders(req *http.Request) {
	if j.job.RemoteProperties.Headers == nil {
		j.job.RemoteProperties.Headers = http.Header{}
	}
	// A valid assumption is that the user is sending something in json cause we're past 2017
	if j.job.RemoteProperties.Headers["Content-Type"] == nil {
		j.job.RemoteProperties.Headers["Content-Type"] = []string{"application/json"}
	}
	req.Header = j.job.RemoteProperties.Headers
}
<|endoftext|>


<commit_before>package quark

import (
	"encoding/base64"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"

	"google.golang.org/grpc/metadata"

	"github.com/gkarlik/quark-go/broker"
	"github.com/gkarlik/quark-go/metrics"
	"github.com/gkarlik/quark-go/service/trace"
	opentracing "github.com/opentracing/opentracing-go"
	"golang.org/x/net/context"
)

// GetEnvVar gets environment variable by key. Panics is variable is not set.
func GetEnvVar(key string) string {
	v := os.Getenv(key)
	if v == "" {
		panic(fmt.Sprintf("Environment variable %q is not set!", key))
	}
	return v
}

// GetHostAddress returns host address and optionally port on which service is hosted.
// If port is 0 only address is returned.
func GetHostAddress(port int) (*url.URL, error) {
	ip, err := getLocalIPAddress()
	if err != nil {
		return nil, err
	}

	u := fmt.Sprintf("%s:%d", ip, port)
	if port == 0 {
		u = fmt.Sprintf(ip)
	}

	return url.Parse(u)
}

func getLocalIPAddress() (string, error) {
	ifaces, error := net.Interfaces()
	if error != nil {
		return "", error
	}
	for _, iface := range ifaces {
		if iface.Flags&net.FlagUp == 0 {
			continue // interface down
		}
		if iface.Flags&net.FlagLoopback != 0 {
			continue // loopback interface
		}
		addrs, error := iface.Addrs()
		if error != nil {
			return "", error
		}
		for _, addr := range addrs {
			var ip net.IP
			switch v := addr.(type) {
			case *net.IPNet:
				ip = v.IP
			case *net.IPAddr:
				ip = v.IP
			}
			if ip == nil || ip.IsLoopback() {
				continue
			}
			ip = ip.To4()
			if ip == nil {
				continue // not an ipv4 address
			}
			return ip.String(), nil
		}
	}
	return "", errors.New("Network not available")
}

// ReportServiceValue sends metric with name and value using service instance.
func ReportServiceValue(s Service, name string, value interface{}) error {
	m := metrics.Metric{
		Name: name,
		Tags: map[string]string{"service": s.Info().Name},
		Values: map[string]interface{}{
			"value": value,
		},
	}
	return s.Metrics().Report(m)
}

// CallHTTPService calls HTTP service at specified url with HTTP method and body.
func CallHTTPService(s Service, method string, url string, body io.Reader, parent trace.Span) ([]byte, error) {
	req, err := http.NewRequest(method, url, body)
	if err != nil {
		return nil, err
	}

	s.Tracer().InjectSpan(parent, opentracing.HTTPHeaders, opentracing.HTTPHeadersCarrier(req.Header))

	client := http.Client{
		Timeout: 10 * time.Second,
	}

	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf(resp.Status)
	}

	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	return data, nil
}

// RPCMetadataCarrier represents carrier for span propagation using gRPC metadata.
type RPCMetadataCarrier struct {
	MD *metadata.MD // gRPC metadata
}

// Set sets metadata value inside gRPC metadata.
func (c RPCMetadataCarrier) Set(key, val string) {
	k := strings.ToLower(key)
	if strings.HasSuffix(k, "-bin") {
		val = string(base64.StdEncoding.EncodeToString([]byte(val)))
	}

	(*c.MD)[k] = append((*c.MD)[k], val)
}

// ForeachKey iterates over gRPC metadata key and values.
func (c RPCMetadataCarrier) ForeachKey(handler func(key, val string) error) error {
	for k, vals := range *c.MD {
		for _, v := range vals {
			if err := handler(k, v); err != nil {
				return err
			}
		}
	}
	return nil
}

// StartRPCSpan starts span with name and parent span taken from RPC context.
func StartRPCSpan(s Service, name string, ctx context.Context) trace.Span {
	var span trace.Span
	var err error

	md, ok := metadata.FromContext(ctx)
	if ok {
		span, err = s.Tracer().ExtractSpan(name, opentracing.TextMap, RPCMetadataCarrier{MD: &md})
	}

	if err != nil || !ok {
		span = s.Tracer().StartSpan(name)
	}

	return span
}

// MessageContextCarrier represents carrier for span propagation using broker message context.
type MessageContextCarrier struct {
	Context *broker.MessageContext
}

// Set sets metadata value inside broker message context.
func (c MessageContextCarrier) Set(key, val string) {
	k := strings.ToLower(key)
	if strings.HasSuffix(k, "-bin") {
		val = string(base64.StdEncoding.EncodeToString([]byte(val)))
	}

	(*c.Context)[k] = val
}

// ForeachKey iterates over broker message context key.
func (c MessageContextCarrier) ForeachKey(handler func(key, val string) error) error {
	for k, v := range *c.Context {
		if err := handler(k, v); err != nil {
			return err
		}
	}
	return nil
}

// StartMessageSpan starts span with name and parent span taken from message.
func StartMessageSpan(s Service, name string, m broker.Message) trace.Span {
	span, err := s.Tracer().ExtractSpan(name, opentracing.TextMap, MessageContextCarrier{Context: &m.Context})
	if err != nil {
		span = s.Tracer().StartSpan(name)
	}

	return span
}
<commit_msg>Add interrupt signal handler.<commit_after>package quark

import (
	"encoding/base64"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/gkarlik/quark-go/broker"
	"github.com/gkarlik/quark-go/metrics"
	"github.com/gkarlik/quark-go/service/trace"
	opentracing "github.com/opentracing/opentracing-go"
	"golang.org/x/net/context"
	"google.golang.org/grpc/metadata"
)

// GetEnvVar gets environment variable by key. Panics is variable is not set.
func GetEnvVar(key string) string {
	v := os.Getenv(key)
	if v == "" {
		panic(fmt.Sprintf("Environment variable %q is not set!", key))
	}
	return v
}

// GetHostAddress returns host address and optionally port on which service is hosted.
// If port is 0 only address is returned.
func GetHostAddress(port int) (*url.URL, error) {
	ip, err := getLocalIPAddress()
	if err != nil {
		return nil, err
	}

	u := fmt.Sprintf("%s:%d", ip, port)
	if port == 0 {
		u = fmt.Sprintf(ip)
	}

	return url.Parse(u)
}

func getLocalIPAddress() (string, error) {
	ifaces, error := net.Interfaces()
	if error != nil {
		return "", error
	}
	for _, iface := range ifaces {
		if iface.Flags&net.FlagUp == 0 {
			continue // interface down
		}
		if iface.Flags&net.FlagLoopback != 0 {
			continue // loopback interface
		}
		addrs, error := iface.Addrs()
		if error != nil {
			return "", error
		}
		for _, addr := range addrs {
			var ip net.IP
			switch v := addr.(type) {
			case *net.IPNet:
				ip = v.IP
			case *net.IPAddr:
				ip = v.IP
			}
			if ip == nil || ip.IsLoopback() {
				continue
			}
			ip = ip.To4()
			if ip == nil {
				continue // not an ipv4 address
			}
			return ip.String(), nil
		}
	}
	return "", errors.New("Network not available")
}

// ReportServiceValue sends metric with name and value using service instance.
func ReportServiceValue(s Service, name string, value interface{}) error {
	m := metrics.Metric{
		Name: name,
		Tags: map[string]string{"service": s.Info().Name},
		Values: map[string]interface{}{
			"value": value,
		},
	}
	return s.Metrics().Report(m)
}

// CallHTTPService calls HTTP service at specified url with HTTP method and body.
func CallHTTPService(s Service, method string, url string, body io.Reader, parent trace.Span) ([]byte, error) {
	req, err := http.NewRequest(method, url, body)
	if err != nil {
		return nil, err
	}

	s.Tracer().InjectSpan(parent, opentracing.HTTPHeaders, opentracing.HTTPHeadersCarrier(req.Header))

	client := http.Client{
		Timeout: 10 * time.Second,
	}

	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf(resp.Status)
	}

	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	return data, nil
}

// RPCMetadataCarrier represents carrier for span propagation using gRPC metadata.
type RPCMetadataCarrier struct {
	MD *metadata.MD // gRPC metadata
}

// Set sets metadata value inside gRPC metadata.
func (c RPCMetadataCarrier) Set(key, val string) {
	k := strings.ToLower(key)
	if strings.HasSuffix(k, "-bin") {
		val = string(base64.StdEncoding.EncodeToString([]byte(val)))
	}

	(*c.MD)[k] = append((*c.MD)[k], val)
}

// ForeachKey iterates over gRPC metadata key and values.
func (c RPCMetadataCarrier) ForeachKey(handler func(key, val string) error) error {
	for k, vals := range *c.MD {
		for _, v := range vals {
			if err := handler(k, v); err != nil {
				return err
			}
		}
	}
	return nil
}

// StartRPCSpan starts span with name and parent span taken from RPC context.
func StartRPCSpan(s Service, name string, ctx context.Context) trace.Span {
	var span trace.Span
	var err error

	md, ok := metadata.FromContext(ctx)
	if ok {
		span, err = s.Tracer().ExtractSpan(name, opentracing.TextMap, RPCMetadataCarrier{MD: &md})
	}

	if err != nil || !ok {
		span = s.Tracer().StartSpan(name)
	}

	return span
}

// MessageContextCarrier represents carrier for span propagation using broker message context.
type MessageContextCarrier struct {
	Context *broker.MessageContext
}

// Set sets metadata value inside broker message context.
func (c MessageContextCarrier) Set(key, val string) {
	k := strings.ToLower(key)
	if strings.HasSuffix(k, "-bin") {
		val = string(base64.StdEncoding.EncodeToString([]byte(val)))
	}

	(*c.Context)[k] = val
}

// ForeachKey iterates over broker message context key.
func (c MessageContextCarrier) ForeachKey(handler func(key, val string) error) error {
	for k, v := range *c.Context {
		if err := handler(k, v); err != nil {
			return err
		}
	}
	return nil
}

// StartMessageSpan starts span with name and parent span taken from message.
func StartMessageSpan(s Service, name string, m broker.Message) trace.Span {
	span, err := s.Tracer().ExtractSpan(name, opentracing.TextMap, MessageContextCarrier{Context: &m.Context})
	if err != nil {
		span = s.Tracer().StartSpan(name)
	}

	return span
}

// HandleInterrupt handles interrupt signal receiced by the service.
func HandleInterrupt(s Service) <-chan bool {
	signals := make(chan os.Signal, 1)
	done := make(chan bool, 1)

	signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)

	go func() {
		<-signals
		s.Log().Info("Received interrupt signal")
		done <- true
	}()

	return done
}
<|endoftext|>


<commit_before>package meter

import (
	"context"
	"net/url"
	"strings"
	"time"

	"github.com/alxarch/go-timecodec"
)

type DateRangeParserFunc func(string, string, time.Duration) (time.Time, time.Time, error)

func DateRangeParser(dec tc.TimeDecoder) DateRangeParserFunc {
	return func(s, e string, max time.Duration) (start, end time.Time, err error) {
		now := time.Now()
		if e != "" {
			if end, err = dec.UnmarshalTime(e); err != nil {
				return
			}
		}
		if end.IsZero() || end.After(now) {
			end = now
		}
		if s != "" {
			if start, err = dec.UnmarshalTime(s); err != nil {
				return
			}
		}
		min := end.Add(-max)
		if start.IsZero() || start.After(end) || start.Before(min) {
			start = min
		}
		return
	}
}

func TimeSequence(start time.Time, end time.Time, unit time.Duration) []time.Time {
	start = start.Round(unit)
	end = end.Round(unit)
	if unit == 0 {
		return []time.Time{}
	}
	n := end.Sub(start) / unit

	results := make([]time.Time, 0, n)

	for s := start; end.Sub(s) >= 0; s = s.Add(unit) {
		results = append(results, s)
	}
	return results
}

func Join(sep string, parts ...string) string {
	return strings.Join(parts, sep)
}

func PermutationPairs(input url.Values) [][]string {
	result := [][]string{}
	for k, vv := range input {
		first := len(result) == 0
		for i, v := range vv {
			if first {
				result = append(result, []string{k, v})
			} else if i == 0 {
				for i, r := range result {
					result[i] = append(r, k, v)
				}
			} else {
				for _, r := range result {
					rr := make([]string, len(r), len(r)+2)
					copy(rr, r)
					rr = append(rr, k, v)
					result = append(result, rr)
				}
			}
		}
	}
	return result
}

func SetInterval(d time.Duration, callback func(tm time.Time)) (cancel func()) {
	done := make(chan struct{})
	cancel = func() {
		close(done)
	}
	go RunInterval(d, callback, done)
	return
}

func RunInterval(d time.Duration, callback func(tm time.Time), done <-chan struct{}) {
	tick := time.NewTicker(d)
	defer tick.Stop()
	for {
		select {
		case <-done:
			return
		case t := <-tick.C:
			callback(t)
		}
	}
}

func SetIntervalContext(parent context.Context, d time.Duration, callback func(tm time.Time)) (ctx context.Context, cancel context.CancelFunc) {
	if parent == nil {
		parent = context.Background()
	}
	ctx, cancel = context.WithCancel(parent)
	go RunInterval(d, callback, ctx.Done())
	return

}

type Interval struct {
	cancel   context.CancelFunc
	interval time.Duration
}

func NewInterval(dt time.Duration, callback func(t time.Time)) *Interval {
	return &Interval{
		cancel:   SetInterval(dt, callback),
		interval: dt,
	}
}

func (i *Interval) Close() {
	i.cancel()
}
func (i *Interval) Interval() time.Duration {
	return i.interval
}
<commit_msg>Remove irrelevant code<commit_after>package meter

import (
	"context"
	"net/url"
	"strings"
	"time"

	"github.com/alxarch/go-timecodec"
)

type DateRangeParserFunc func(string, string, time.Duration) (time.Time, time.Time, error)

func DateRangeParser(dec tc.TimeDecoder) DateRangeParserFunc {
	return func(s, e string, max time.Duration) (start, end time.Time, err error) {
		now := time.Now()
		if e != "" {
			if end, err = dec.UnmarshalTime(e); err != nil {
				return
			}
		}
		if end.IsZero() || end.After(now) {
			end = now
		}
		if s != "" {
			if start, err = dec.UnmarshalTime(s); err != nil {
				return
			}
		}
		min := end.Add(-max)
		if start.IsZero() || start.After(end) || start.Before(min) {
			start = min
		}
		return
	}
}

func TimeSequence(start time.Time, end time.Time, unit time.Duration) []time.Time {
	start = start.Round(unit)
	end = end.Round(unit)
	if unit == 0 {
		return []time.Time{}
	}
	n := end.Sub(start) / unit

	results := make([]time.Time, 0, n)

	for s := start; end.Sub(s) >= 0; s = s.Add(unit) {
		results = append(results, s)
	}
	return results
}

func Join(sep string, parts ...string) string {
	return strings.Join(parts, sep)
}

func PermutationPairs(input url.Values) [][]string {
	result := [][]string{}
	for k, vv := range input {
		first := len(result) == 0
		for i, v := range vv {
			if first {
				result = append(result, []string{k, v})
			} else if i == 0 {
				for i, r := range result {
					result[i] = append(r, k, v)
				}
			} else {
				for _, r := range result {
					rr := make([]string, len(r), len(r)+2)
					copy(rr, r)
					rr = append(rr, k, v)
					result = append(result, rr)
				}
			}
		}
	}
	return result
}

func SetInterval(d time.Duration, callback func(tm time.Time)) (cancel func()) {
	done := make(chan struct{})
	cancel = func() {
		close(done)
	}
	go RunInterval(d, callback, done)
	return
}

func RunInterval(d time.Duration, callback func(tm time.Time), done <-chan struct{}) {
	tick := time.NewTicker(d)
	defer tick.Stop()
	for {
		select {
		case <-done:
			return
		case t := <-tick.C:
			callback(t)
		}
	}
}

func SetIntervalContext(parent context.Context, d time.Duration, callback func(tm time.Time)) (ctx context.Context, cancel context.CancelFunc) {
	if parent == nil {
		parent = context.Background()
	}
	ctx, cancel = context.WithCancel(parent)
	go RunInterval(d, callback, ctx.Done())
	return

}

// type Interval struct {
// 	cancel   context.CancelFunc
// 	interval time.Duration
// }
//
// func NewInterval(dt time.Duration, callback func(t time.Time)) *Interval {
// 	return &Interval{
// 		cancel:   SetInterval(dt, callback),
// 		interval: dt,
// 	}
// }
//
// func (i *Interval) Close() {
// 	i.cancel()
// }
// func (i *Interval) Interval() time.Duration {
// 	return i.interval
// }
<|endoftext|>