<commit_before>package command

import (
	"fmt"
	"os"
	"reflect"
	"time"

	"github.com/hashicorp/vault/api"
	"github.com/hashicorp/vault/command/token"
	"github.com/mitchellh/cli"
)

// DefaultTokenHelper returns the token helper that is configured for Vault.
func DefaultTokenHelper() (token.TokenHelper, error) {
	config, err := LoadConfig("")
	if err != nil {
		return nil, err
	}

	path := config.TokenHelper
	if path == "" {
		return &token.InternalTokenHelper{}, nil
	}

	path, err = token.ExternalTokenHelperPath(path)
	if err != nil {
		return nil, err
	}
	return &token.ExternalTokenHelper{BinaryPath: path}, nil
}

func PrintRawField(ui cli.Ui, secret *api.Secret, field string) int {
	var val interface{}
	switch {
	case secret.Auth != nil:
		switch field {
		case "token":
			val = secret.Auth.ClientToken
		case "token_accessor":
			val = secret.Auth.Accessor
		case "token_duration":
			val = secret.Auth.LeaseDuration
		case "token_renewable":
			val = secret.Auth.Renewable
		case "token_policies":
			val = secret.Auth.Policies
		default:
			val = secret.Data[field]
		}

	case secret.WrapInfo != nil:
		switch field {
		case "wrapping_token":
			val = secret.WrapInfo.Token
		case "wrapping_token_ttl":
			val = secret.WrapInfo.TTL
		case "wrapping_token_creation_time":
			val = secret.WrapInfo.CreationTime.Format(time.RFC3339Nano)
		case "wrapped_accessor":
			val = secret.WrapInfo.WrappedAccessor
		default:
			val = secret.Data[field]
		}

	default:
		switch field {
		case "refresh_interval":
			val = secret.LeaseDuration
		default:
			val = secret.Data[field]
		}
	}

	if val != nil {
		// c.Ui.Output() prints a CR character which in this case is
		// not desired. Since Vault CLI currently only uses BasicUi,
		// which writes to standard output, os.Stdout is used here to
		// directly print the message. If mitchellh/cli exposes method
		// to print without CR, this check needs to be removed.
		if reflect.TypeOf(ui).String() == "*cli.BasicUi" {
			fmt.Fprintf(os.Stdout, fmt.Sprintf("%v", val))
		} else {
			ui.Output(fmt.Sprintf("%v", val))
		}
		return 0
	} else {
		ui.Error(fmt.Sprintf(
			"Field %s not present in secret", field))
		return 1
	}
}
<commit_msg>cli: fix bug with 'vault read -field=...' when the field value contains a printf formatting verb (#2109)<commit_after>package command

import (
	"fmt"
	"os"
	"reflect"
	"time"

	"github.com/hashicorp/vault/api"
	"github.com/hashicorp/vault/command/token"
	"github.com/mitchellh/cli"
)

// DefaultTokenHelper returns the token helper that is configured for Vault.
func DefaultTokenHelper() (token.TokenHelper, error) {
	config, err := LoadConfig("")
	if err != nil {
		return nil, err
	}

	path := config.TokenHelper
	if path == "" {
		return &token.InternalTokenHelper{}, nil
	}

	path, err = token.ExternalTokenHelperPath(path)
	if err != nil {
		return nil, err
	}
	return &token.ExternalTokenHelper{BinaryPath: path}, nil
}

func PrintRawField(ui cli.Ui, secret *api.Secret, field string) int {
	var val interface{}
	switch {
	case secret.Auth != nil:
		switch field {
		case "token":
			val = secret.Auth.ClientToken
		case "token_accessor":
			val = secret.Auth.Accessor
		case "token_duration":
			val = secret.Auth.LeaseDuration
		case "token_renewable":
			val = secret.Auth.Renewable
		case "token_policies":
			val = secret.Auth.Policies
		default:
			val = secret.Data[field]
		}

	case secret.WrapInfo != nil:
		switch field {
		case "wrapping_token":
			val = secret.WrapInfo.Token
		case "wrapping_token_ttl":
			val = secret.WrapInfo.TTL
		case "wrapping_token_creation_time":
			val = secret.WrapInfo.CreationTime.Format(time.RFC3339Nano)
		case "wrapped_accessor":
			val = secret.WrapInfo.WrappedAccessor
		default:
			val = secret.Data[field]
		}

	default:
		switch field {
		case "refresh_interval":
			val = secret.LeaseDuration
		default:
			val = secret.Data[field]
		}
	}

	if val != nil {
		// c.Ui.Output() prints a CR character which in this case is
		// not desired. Since Vault CLI currently only uses BasicUi,
		// which writes to standard output, os.Stdout is used here to
		// directly print the message. If mitchellh/cli exposes method
		// to print without CR, this check needs to be removed.
		if reflect.TypeOf(ui).String() == "*cli.BasicUi" {
			fmt.Fprintf(os.Stdout, "%v", val)
		} else {
			ui.Output(fmt.Sprintf("%v", val))
		}
		return 0
	} else {
		ui.Error(fmt.Sprintf(
			"Field %s not present in secret", field))
		return 1
	}
}
<|endoftext|>


<commit_before>package command

import (
	"encoding/xml"
	"log"
	"net/url"
	"strings"

	"github.com/davecgh/go-spew/spew"
	"github.com/igungor/tlbot"
)

func init() {
	register(cmdWiki)
}

const wikiURL = "https://en.wikipedia.org/w/api.php"

var cmdWiki = &Command{
	Name:      "bkz",
	ShortLine: "bakınız çok ilginçtir",
	Run:       runWiki,
}

func runWiki(b *tlbot.Bot, msg *tlbot.Message) {
	args := msg.Args()
	var txt string
	if len(args) == 0 {
		err := b.SendMessage(msg.Chat, "neye referans vereyim? mesela bana bakın: */bkz İlber Ortaylı*", tlbot.ModeMarkdown, false, nil)
		if err != nil {
			log.Printf("[wiki] Error while sending message '%v'. Err: %v\n", txt, err)
		}
		return
	}

	u, err := url.Parse(wikiURL)
	if err != nil {
		log.Printf("[wiki] Error while parsing URL '%v'. Err: %v\n", wikiURL, err)
		return
	}

	qs := strings.Join(args, " ")
	params := u.Query()
	params.Set("action", "opensearch")
	params.Set("format", "xml")
	params.Set("search", qs)
	u.RawQuery = params.Encode()

	resp, err := httpclient.Get(u.String())
	if err != nil {
		log.Printf("[wiki] Error while fetching article for query '%v'. Err: %v\n", qs, err)
		return
	}
	defer resp.Body.Close()

	var res result
	err = xml.NewDecoder(resp.Body).Decode(&res)
	if err != nil {
		log.Printf("[wiki] Error while decoding response '%v'. Err: %v\n", res, err)
		return
	}

	spew.Dump(res)
	return

	if len(res.Section.Items) == 0 {
		b.SendMessage(msg.Chat, "aradığın referansı bulamadım 🙈", tlbot.ModeMarkdown, true, nil)
		return
	}

	article := res.Section.Items[0]
	txt = article.URL
	err = b.SendMessage(msg.Chat, txt, tlbot.ModeNone, true, nil)
	if err != nil {
		log.Printf("[wiki] Error while sending message to Telegram servers. Err: '%v'\nMessage: %v", err, txt)
		return
	}
}

// wikipedia api response
type result struct {
	XMLName xml.Name `xml:"SearchSuggestion"`
	Section struct {
		Items []struct {
			Title       string `xml:"Text"`
			URL         string `xml:"Url"`
			Description string
			Image       string
		} `xml:"Item"`
	} `xml:"Section"`
}
<commit_msg>Oops. Revert debug returns.<commit_after>package command

import (
	"encoding/xml"
	"log"
	"net/url"
	"strings"

	"github.com/igungor/tlbot"
)

func init() {
	register(cmdWiki)
}

const wikiURL = "https://en.wikipedia.org/w/api.php"

var cmdWiki = &Command{
	Name:      "bkz",
	ShortLine: "bakınız çok ilginçtir",
	Run:       runWiki,
}

func runWiki(b *tlbot.Bot, msg *tlbot.Message) {
	args := msg.Args()
	var txt string
	if len(args) == 0 {
		err := b.SendMessage(msg.Chat, "neye referans vereyim? mesela bana bakın: */bkz İlber Ortaylı*", tlbot.ModeMarkdown, false, nil)
		if err != nil {
			log.Printf("[wiki] Error while sending message '%v'. Err: %v\n", txt, err)
		}
		return
	}

	u, err := url.Parse(wikiURL)
	if err != nil {
		log.Printf("[wiki] Error while parsing URL '%v'. Err: %v\n", wikiURL, err)
		return
	}

	qs := strings.Join(args, " ")
	params := u.Query()
	params.Set("action", "opensearch")
	params.Set("format", "xml")
	params.Set("search", qs)
	u.RawQuery = params.Encode()

	resp, err := httpclient.Get(u.String())
	if err != nil {
		log.Printf("[wiki] Error while fetching article for query '%v'. Err: %v\n", qs, err)
		return
	}
	defer resp.Body.Close()

	var res result
	err = xml.NewDecoder(resp.Body).Decode(&res)
	if err != nil {
		log.Printf("[wiki] Error while decoding response '%v'. Err: %v\n", res, err)
		return
	}

	if len(res.Section.Items) == 0 {
		b.SendMessage(msg.Chat, "aradığın referansı bulamadım 🙈", tlbot.ModeMarkdown, true, nil)
		return
	}

	article := res.Section.Items[0]
	txt = article.URL
	err = b.SendMessage(msg.Chat, txt, tlbot.ModeNone, true, nil)
	if err != nil {
		log.Printf("[wiki] Error while sending message to Telegram servers. Err: '%v'\nMessage: %v", err, txt)
		return
	}
}

// wikipedia api response
type result struct {
	XMLName xml.Name `xml:"SearchSuggestion"`
	Section struct {
		Items []struct {
			Title       string `xml:"Text"`
			URL         string `xml:"Url"`
			Description string
			Image       string
		} `xml:"Item"`
	} `xml:"Section"`
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"

	"github.com/boltdb/bolt"
	"github.com/codegangsta/cli"
)

func registerJobs(app cli.App) *cli.App {
	app.Commands = append(app.Commands,
		[]cli.Command{
			{
				Name:  `jobs`,
				Usage: `SUBCOMMANDS for job information`,
				Subcommands: []cli.Command{
					{
						Name:   `list`,
						Usage:  `List outstanding jobs`,
						Action: runtime(cmdJobList),
					},
					{
						Name:   `show`,
						Usage:  `Show details about a job`,
						Action: runtime(cmdJobShow),
					},
					{
						Name:  `local`,
						Usage: `SUBCOMMANDS for locally saved jobs`,
						Subcommands: []cli.Command{
							{
								Name:   `outstanding`,
								Usage:  `List outstanding locally saved Jobs`,
								Action: runtime(cmdJobLocalOutstanding),
							},
							{
								Name:   `update`,
								Usage:  `Check and update status of oustanding locally cached jobs`,
								Action: runtime(cmdJobLocalUpdate),
							},
							{
								Name:   `list`,
								Usage:  `List all locally cached jobs`,
								Action: runtime(cmdJobLocalList),
							},
							{
								Name:   `prune`,
								Usage:  `Delete completed jobs from local cache`,
								Action: runtime(cmdJobLocalPrune),
							},
						},
					},
				},
			},
		}...,
	)
	return &app
}

func cmdJobList(c *cli.Context) error {
	utl.ValidateCliArgumentCount(c, 0)
	if resp, err := adm.GetReq(`/jobs/`); err != nil {
		return err
	} else {
		fmt.Println(resp)
	}
	return nil
}

func cmdJobShow(c *cli.Context) error {
	utl.ValidateCliArgumentCount(c, 1)
	if !utl.IsUUID(c.Args().First()) {
		return fmt.Errorf("Argument is not a UUID: %s", c.Args().First())
	}
	path := fmt.Sprintf("/jobs/%s", c.Args().First())
	if resp, err := adm.GetReq(path); err != nil {
		return err
	} else {
		fmt.Println(resp)
	}
	return nil
}

func cmdJobLocalOutstanding(c *cli.Context) error {
	jobs, err := store.ActiveJobs()
	if err != nil && err != bolt.ErrBucketNotFound {
		return err
	}

	pj := []proto.Job{}
	for _, iArray := range jobs {
		pj = append(pj, proto.Job{
			Id:       iArray[1],
			TsQueued: iArray[2],
			Type:     iArray[3],
		})
	}

	if enc, err := json.Marshal(&pj); err != nil {
		return err
	} else {
		fmt.Println(string(enc))
	}
	return nil
}

func cmdJobLocalUpdate(c *cli.Context) error {
	return nil
}

func cmdJobLocalList(c *cli.Context) error {
	active, err := store.ActiveJobs()
	if err != nil && err != bolt.ErrBucketNotFound {
		return err
	}

	jobs := []proto.Job{}
	for _, iArray := range active {
		jobs = append(jobs, proto.Job{
			Id:       iArray[1],
			TsQueued: iArray[2],
			Type:     iArray[3],
		})
	}

	finished, err := store.FinishedJobs()
	if err != nil && err != bolt.ErrBucketNotFound {
		return err
	}

	jobs = append(jobs, finished...)
	if enc, err := json.Marshal(&jobs); err != nil {
		return err
	} else {
		fmt.Println(string(enc))
	}
	return nil
}

func cmdJobLocalPrune(c *cli.Context) error {
	return store.PruneFinishedJobs()
}

// vim: ts=4 sw=4 sts=4 noet fenc=utf-8 ffs=unix
<commit_msg>Update jobs command description<commit_after>package main

import (
	"encoding/json"
	"fmt"

	"github.com/boltdb/bolt"
	"github.com/codegangsta/cli"
)

func registerJobs(app cli.App) *cli.App {
	app.Commands = append(app.Commands,
		[]cli.Command{
			{
				Name:  `jobs`,
				Usage: `SUBCOMMANDS for job information`,
				Subcommands: []cli.Command{
					{
						Name:   `list`,
						Usage:  `List outstanding jobs (remote)`,
						Action: runtime(cmdJobList),
					},
					{
						Name:   `show`,
						Usage:  `Show details about a job (remote)`,
						Action: runtime(cmdJobShow),
					},
					{
						Name:  `local`,
						Usage: `SUBCOMMANDS for locally saved jobs`,
						Subcommands: []cli.Command{
							{
								Name:   `outstanding`,
								Usage:  `List outstanding locally saved Jobs`,
								Action: runtime(cmdJobLocalOutstanding),
							},
							{
								Name:   `update`,
								Usage:  `Check and update status of oustanding locally cached jobs`,
								Action: runtime(cmdJobLocalUpdate),
							},
							{
								Name:   `list`,
								Usage:  `List all locally cached jobs`,
								Action: runtime(cmdJobLocalList),
							},
							{
								Name:   `prune`,
								Usage:  `Delete completed jobs from local cache`,
								Action: runtime(cmdJobLocalPrune),
							},
						},
					},
				},
			},
		}...,
	)
	return &app
}

func cmdJobList(c *cli.Context) error {
	utl.ValidateCliArgumentCount(c, 0)
	if resp, err := adm.GetReq(`/jobs/`); err != nil {
		return err
	} else {
		fmt.Println(resp)
	}
	return nil
}

func cmdJobShow(c *cli.Context) error {
	utl.ValidateCliArgumentCount(c, 1)
	if !utl.IsUUID(c.Args().First()) {
		return fmt.Errorf("Argument is not a UUID: %s", c.Args().First())
	}
	path := fmt.Sprintf("/jobs/%s", c.Args().First())
	if resp, err := adm.GetReq(path); err != nil {
		return err
	} else {
		fmt.Println(resp)
	}
	return nil
}

func cmdJobLocalOutstanding(c *cli.Context) error {
	jobs, err := store.ActiveJobs()
	if err != nil && err != bolt.ErrBucketNotFound {
		return err
	}

	pj := []proto.Job{}
	for _, iArray := range jobs {
		pj = append(pj, proto.Job{
			Id:       iArray[1],
			TsQueued: iArray[2],
			Type:     iArray[3],
		})
	}

	if enc, err := json.Marshal(&pj); err != nil {
		return err
	} else {
		fmt.Println(string(enc))
	}
	return nil
}

func cmdJobLocalUpdate(c *cli.Context) error {
	return nil
}

func cmdJobLocalList(c *cli.Context) error {
	active, err := store.ActiveJobs()
	if err != nil && err != bolt.ErrBucketNotFound {
		return err
	}

	jobs := []proto.Job{}
	for _, iArray := range active {
		jobs = append(jobs, proto.Job{
			Id:       iArray[1],
			TsQueued: iArray[2],
			Type:     iArray[3],
		})
	}

	finished, err := store.FinishedJobs()
	if err != nil && err != bolt.ErrBucketNotFound {
		return err
	}

	jobs = append(jobs, finished...)
	if enc, err := json.Marshal(&jobs); err != nil {
		return err
	} else {
		fmt.Println(string(enc))
	}
	return nil
}

func cmdJobLocalPrune(c *cli.Context) error {
	return store.PruneFinishedJobs()
}

// vim: ts=4 sw=4 sts=4 noet fenc=utf-8 ffs=unix
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package textproto

import (
	"bufio"
	"bytes"
	"io"
	"reflect"
	"strings"
	"testing"
)

type canonicalHeaderKeyTest struct {
	in, out string
}

var canonicalHeaderKeyTests = []canonicalHeaderKeyTest{
	{"a-b-c", "A-B-C"},
	{"a-1-c", "A-1-C"},
	{"User-Agent", "User-Agent"},
	{"uSER-aGENT", "User-Agent"},
	{"user-agent", "User-Agent"},
	{"USER-AGENT", "User-Agent"},
	{"üser-agenT", "üser-Agent"}, // non-ASCII unchanged
}

func TestCanonicalMIMEHeaderKey(t *testing.T) {
	for _, tt := range canonicalHeaderKeyTests {
		if s := CanonicalMIMEHeaderKey(tt.in); s != tt.out {
			t.Errorf("CanonicalMIMEHeaderKey(%q) = %q, want %q", tt.in, s, tt.out)
		}
	}
}

func reader(s string) *Reader {
	return NewReader(bufio.NewReader(strings.NewReader(s)))
}

func TestReadLine(t *testing.T) {
	r := reader("line1\nline2\n")
	s, err := r.ReadLine()
	if s != "line1" || err != nil {
		t.Fatalf("Line 1: %s, %v", s, err)
	}
	s, err = r.ReadLine()
	if s != "line2" || err != nil {
		t.Fatalf("Line 2: %s, %v", s, err)
	}
	s, err = r.ReadLine()
	if s != "" || err != io.EOF {
		t.Fatalf("EOF: %s, %v", s, err)
	}
}

func TestReadContinuedLine(t *testing.T) {
	r := reader("line1\nline\n 2\nline3\n")
	s, err := r.ReadContinuedLine()
	if s != "line1" || err != nil {
		t.Fatalf("Line 1: %s, %v", s, err)
	}
	s, err = r.ReadContinuedLine()
	if s != "line 2" || err != nil {
		t.Fatalf("Line 2: %s, %v", s, err)
	}
	s, err = r.ReadContinuedLine()
	if s != "line3" || err != nil {
		t.Fatalf("Line 3: %s, %v", s, err)
	}
	s, err = r.ReadContinuedLine()
	if s != "" || err != io.EOF {
		t.Fatalf("EOF: %s, %v", s, err)
	}
}

func TestReadCodeLine(t *testing.T) {
	r := reader("123 hi\n234 bye\n345 no way\n")
	code, msg, err := r.ReadCodeLine(0)
	if code != 123 || msg != "hi" || err != nil {
		t.Fatalf("Line 1: %d, %s, %v", code, msg, err)
	}
	code, msg, err = r.ReadCodeLine(23)
	if code != 234 || msg != "bye" || err != nil {
		t.Fatalf("Line 2: %d, %s, %v", code, msg, err)
	}
	code, msg, err = r.ReadCodeLine(346)
	if code != 345 || msg != "no way" || err == nil {
		t.Fatalf("Line 3: %d, %s, %v", code, msg, err)
	}
	if e, ok := err.(*Error); !ok || e.Code != code || e.Msg != msg {
		t.Fatalf("Line 3: wrong error %v\n", err)
	}
	code, msg, err = r.ReadCodeLine(1)
	if code != 0 || msg != "" || err != io.EOF {
		t.Fatalf("EOF: %d, %s, %v", code, msg, err)
	}
}

func TestReadDotLines(t *testing.T) {
	r := reader("dotlines\r\n.foo\r\n..bar\n...baz\nquux\r\n\r\n.\r\nanother\n")
	s, err := r.ReadDotLines()
	want := []string{"dotlines", "foo", ".bar", "..baz", "quux", ""}
	if !reflect.DeepEqual(s, want) || err != nil {
		t.Fatalf("ReadDotLines: %v, %v", s, err)
	}

	s, err = r.ReadDotLines()
	want = []string{"another"}
	if !reflect.DeepEqual(s, want) || err != io.ErrUnexpectedEOF {
		t.Fatalf("ReadDotLines2: %v, %v", s, err)
	}
}

func TestReadDotBytes(t *testing.T) {
	r := reader("dotlines\r\n.foo\r\n..bar\n...baz\nquux\r\n\r\n.\r\nanot.her\r\n")
	b, err := r.ReadDotBytes()
	want := []byte("dotlines\nfoo\n.bar\n..baz\nquux\n\n")
	if !reflect.DeepEqual(b, want) || err != nil {
		t.Fatalf("ReadDotBytes: %q, %v", b, err)
	}

	b, err = r.ReadDotBytes()
	want = []byte("anot.her\n")
	if !reflect.DeepEqual(b, want) || err != io.ErrUnexpectedEOF {
		t.Fatalf("ReadDotBytes2: %q, %v", b, err)
	}
}

func TestReadMIMEHeader(t *testing.T) {
	r := reader("my-key: Value 1  \r\nLong-key: Even \n Longer Value\r\nmy-Key: Value 2\r\n\n")
	m, err := r.ReadMIMEHeader()
	want := MIMEHeader{
		"My-Key":   {"Value 1", "Value 2"},
		"Long-Key": {"Even Longer Value"},
	}
	if !reflect.DeepEqual(m, want) || err != nil {
		t.Fatalf("ReadMIMEHeader: %v, %v; want %v", m, err, want)
	}
}

func TestReadMIMEHeaderSingle(t *testing.T) {
	r := reader("Foo: bar\n\n")
	m, err := r.ReadMIMEHeader()
	want := MIMEHeader{"Foo": {"bar"}}
	if !reflect.DeepEqual(m, want) || err != nil {
		t.Fatalf("ReadMIMEHeader: %v, %v; want %v", m, err, want)
	}
}

func TestLargeReadMIMEHeader(t *testing.T) {
	data := make([]byte, 16*1024)
	for i := 0; i < len(data); i++ {
		data[i] = 'x'
	}
	sdata := string(data)
	r := reader("Cookie: " + sdata + "\r\n\n")
	m, err := r.ReadMIMEHeader()
	if err != nil {
		t.Fatalf("ReadMIMEHeader: %v", err)
	}
	cookie := m.Get("Cookie")
	if cookie != sdata {
		t.Fatalf("ReadMIMEHeader: %v bytes, want %v bytes", len(cookie), len(sdata))
	}
}

// Test that we read slightly-bogus MIME headers seen in the wild,
// with spaces before colons, and spaces in keys.
func TestReadMIMEHeaderNonCompliant(t *testing.T) {
	// Invalid HTTP response header as sent by an Axis security
	// camera: (this is handled by IE, Firefox, Chrome, curl, etc.)
	r := reader("Foo: bar\r\n" +
		"Content-Language: en\r\n" +
		"SID : 0\r\n" +
		"Audio Mode : None\r\n" +
		"Privilege : 127\r\n\r\n")
	m, err := r.ReadMIMEHeader()
	want := MIMEHeader{
		"Foo":              {"bar"},
		"Content-Language": {"en"},
		"Sid":              {"0"},
		"Audio-Mode":       {"None"},
		"Privilege":        {"127"},
	}
	if !reflect.DeepEqual(m, want) || err != nil {
		t.Fatalf("ReadMIMEHeader =\n%v, %v; want:\n%v", m, err, want)
	}
}

type readResponseTest struct {
	in       string
	inCode   int
	wantCode int
	wantMsg  string
}

var readResponseTests = []readResponseTest{
	{"230-Anonymous access granted, restrictions apply\n" +
		"Read the file README.txt,\n" +
		"230  please",
		23,
		230,
		"Anonymous access granted, restrictions apply\nRead the file README.txt,\n please",
	},

	{"230 Anonymous access granted, restrictions apply\n",
		23,
		230,
		"Anonymous access granted, restrictions apply",
	},

	{"400-A\n400-B\n400 C",
		4,
		400,
		"A\nB\nC",
	},

	{"400-A\r\n400-B\r\n400 C\r\n",
		4,
		400,
		"A\nB\nC",
	},
}

// See http://www.ietf.org/rfc/rfc959.txt page 36.
func TestRFC959Lines(t *testing.T) {
	for i, tt := range readResponseTests {
		r := reader(tt.in + "\nFOLLOWING DATA")
		code, msg, err := r.ReadResponse(tt.inCode)
		if err != nil {
			t.Errorf("#%d: ReadResponse: %v", i, err)
			continue
		}
		if code != tt.wantCode {
			t.Errorf("#%d: code=%d, want %d", i, code, tt.wantCode)
		}
		if msg != tt.wantMsg {
			t.Errorf("#%d: msg=%q, want %q", i, msg, tt.wantMsg)
		}
	}
}

func TestCommonHeaders(t *testing.T) {
	// need to disable the commonHeaders-based optimization
	// during this check, or we'd not be testing anything
	oldch := commonHeaders
	commonHeaders = []string{}
	defer func() { commonHeaders = oldch }()

	last := ""
	for _, h := range oldch {
		if last > h {
			t.Errorf("%v is out of order", h)
		}
		if last == h {
			t.Errorf("%v is duplicated", h)
		}
		if canon := CanonicalMIMEHeaderKey(h); h != canon {
			t.Errorf("%v is not canonical", h)
		}
		last = h
	}
}

var clientHeaders = strings.Replace(`Host: golang.org
Connection: keep-alive
Cache-Control: max-age=0
Accept: application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5
User-Agent: Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.472.63 Safari/534.3
Accept-Encoding: gzip,deflate,sdch
Accept-Language: en-US,en;q=0.8,fr-CH;q=0.6
Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.3
COOKIE: __utma=000000000.0000000000.0000000000.0000000000.0000000000.00; __utmb=000000000.0.00.0000000000; __utmc=000000000; __utmz=000000000.0000000000.00.0.utmcsr=code.google.com|utmccn=(referral)|utmcmd=referral|utmcct=/p/go/issues/detail
Non-Interned: test

`, "\n", "\r\n", -1)

var serverHeaders = strings.Replace(`Content-Type: text/html; charset=utf-8
Content-Encoding: gzip
Date: Thu, 27 Sep 2012 09:03:33 GMT
Server: Google Frontend
Cache-Control: private
Content-Length: 2298
VIA: 1.1 proxy.example.com:80 (XXX/n.n.n-nnn)
Connection: Close
Non-Interned: test

`, "\n", "\r\n", -1)

func BenchmarkReadMIMEHeader(b *testing.B) {
	var buf bytes.Buffer
	br := bufio.NewReader(&buf)
	r := NewReader(br)
	for i := 0; i < b.N; i++ {
		var want int
		var find string
		if (i & 1) == 1 {
			buf.WriteString(clientHeaders)
			want = 10
			find = "Cookie"
		} else {
			buf.WriteString(serverHeaders)
			want = 9
			find = "Via"
		}
		h, err := r.ReadMIMEHeader()
		if err != nil {
			b.Fatal(err)
		}
		if len(h) != want {
			b.Fatalf("wrong number of headers: got %d, want %d", len(h), want)
		}
		if _, ok := h[find]; !ok {
			b.Fatalf("did not find key %s", find)
		}
	}
}

func BenchmarkUncommon(b *testing.B) {
	var buf bytes.Buffer
	br := bufio.NewReader(&buf)
	r := NewReader(br)
	for i := 0; i < b.N; i++ {
		buf.WriteString("uncommon-header-for-benchmark: foo\r\n\r\n")
		h, err := r.ReadMIMEHeader()
		if err != nil {
			b.Fatal(err)
		}
		if _, ok := h["Uncommon-Header-For-Benchmark"]; !ok {
			b.Fatal("Missing result header.")
		}
	}
}
<commit_msg>net/textproto: report allocs in benchmarks<commit_after>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package textproto

import (
	"bufio"
	"bytes"
	"io"
	"reflect"
	"strings"
	"testing"
)

type canonicalHeaderKeyTest struct {
	in, out string
}

var canonicalHeaderKeyTests = []canonicalHeaderKeyTest{
	{"a-b-c", "A-B-C"},
	{"a-1-c", "A-1-C"},
	{"User-Agent", "User-Agent"},
	{"uSER-aGENT", "User-Agent"},
	{"user-agent", "User-Agent"},
	{"USER-AGENT", "User-Agent"},
	{"üser-agenT", "üser-Agent"}, // non-ASCII unchanged
}

func TestCanonicalMIMEHeaderKey(t *testing.T) {
	for _, tt := range canonicalHeaderKeyTests {
		if s := CanonicalMIMEHeaderKey(tt.in); s != tt.out {
			t.Errorf("CanonicalMIMEHeaderKey(%q) = %q, want %q", tt.in, s, tt.out)
		}
	}
}

func reader(s string) *Reader {
	return NewReader(bufio.NewReader(strings.NewReader(s)))
}

func TestReadLine(t *testing.T) {
	r := reader("line1\nline2\n")
	s, err := r.ReadLine()
	if s != "line1" || err != nil {
		t.Fatalf("Line 1: %s, %v", s, err)
	}
	s, err = r.ReadLine()
	if s != "line2" || err != nil {
		t.Fatalf("Line 2: %s, %v", s, err)
	}
	s, err = r.ReadLine()
	if s != "" || err != io.EOF {
		t.Fatalf("EOF: %s, %v", s, err)
	}
}

func TestReadContinuedLine(t *testing.T) {
	r := reader("line1\nline\n 2\nline3\n")
	s, err := r.ReadContinuedLine()
	if s != "line1" || err != nil {
		t.Fatalf("Line 1: %s, %v", s, err)
	}
	s, err = r.ReadContinuedLine()
	if s != "line 2" || err != nil {
		t.Fatalf("Line 2: %s, %v", s, err)
	}
	s, err = r.ReadContinuedLine()
	if s != "line3" || err != nil {
		t.Fatalf("Line 3: %s, %v", s, err)
	}
	s, err = r.ReadContinuedLine()
	if s != "" || err != io.EOF {
		t.Fatalf("EOF: %s, %v", s, err)
	}
}

func TestReadCodeLine(t *testing.T) {
	r := reader("123 hi\n234 bye\n345 no way\n")
	code, msg, err := r.ReadCodeLine(0)
	if code != 123 || msg != "hi" || err != nil {
		t.Fatalf("Line 1: %d, %s, %v", code, msg, err)
	}
	code, msg, err = r.ReadCodeLine(23)
	if code != 234 || msg != "bye" || err != nil {
		t.Fatalf("Line 2: %d, %s, %v", code, msg, err)
	}
	code, msg, err = r.ReadCodeLine(346)
	if code != 345 || msg != "no way" || err == nil {
		t.Fatalf("Line 3: %d, %s, %v", code, msg, err)
	}
	if e, ok := err.(*Error); !ok || e.Code != code || e.Msg != msg {
		t.Fatalf("Line 3: wrong error %v\n", err)
	}
	code, msg, err = r.ReadCodeLine(1)
	if code != 0 || msg != "" || err != io.EOF {
		t.Fatalf("EOF: %d, %s, %v", code, msg, err)
	}
}

func TestReadDotLines(t *testing.T) {
	r := reader("dotlines\r\n.foo\r\n..bar\n...baz\nquux\r\n\r\n.\r\nanother\n")
	s, err := r.ReadDotLines()
	want := []string{"dotlines", "foo", ".bar", "..baz", "quux", ""}
	if !reflect.DeepEqual(s, want) || err != nil {
		t.Fatalf("ReadDotLines: %v, %v", s, err)
	}

	s, err = r.ReadDotLines()
	want = []string{"another"}
	if !reflect.DeepEqual(s, want) || err != io.ErrUnexpectedEOF {
		t.Fatalf("ReadDotLines2: %v, %v", s, err)
	}
}

func TestReadDotBytes(t *testing.T) {
	r := reader("dotlines\r\n.foo\r\n..bar\n...baz\nquux\r\n\r\n.\r\nanot.her\r\n")
	b, err := r.ReadDotBytes()
	want := []byte("dotlines\nfoo\n.bar\n..baz\nquux\n\n")
	if !reflect.DeepEqual(b, want) || err != nil {
		t.Fatalf("ReadDotBytes: %q, %v", b, err)
	}

	b, err = r.ReadDotBytes()
	want = []byte("anot.her\n")
	if !reflect.DeepEqual(b, want) || err != io.ErrUnexpectedEOF {
		t.Fatalf("ReadDotBytes2: %q, %v", b, err)
	}
}

func TestReadMIMEHeader(t *testing.T) {
	r := reader("my-key: Value 1  \r\nLong-key: Even \n Longer Value\r\nmy-Key: Value 2\r\n\n")
	m, err := r.ReadMIMEHeader()
	want := MIMEHeader{
		"My-Key":   {"Value 1", "Value 2"},
		"Long-Key": {"Even Longer Value"},
	}
	if !reflect.DeepEqual(m, want) || err != nil {
		t.Fatalf("ReadMIMEHeader: %v, %v; want %v", m, err, want)
	}
}

func TestReadMIMEHeaderSingle(t *testing.T) {
	r := reader("Foo: bar\n\n")
	m, err := r.ReadMIMEHeader()
	want := MIMEHeader{"Foo": {"bar"}}
	if !reflect.DeepEqual(m, want) || err != nil {
		t.Fatalf("ReadMIMEHeader: %v, %v; want %v", m, err, want)
	}
}

func TestLargeReadMIMEHeader(t *testing.T) {
	data := make([]byte, 16*1024)
	for i := 0; i < len(data); i++ {
		data[i] = 'x'
	}
	sdata := string(data)
	r := reader("Cookie: " + sdata + "\r\n\n")
	m, err := r.ReadMIMEHeader()
	if err != nil {
		t.Fatalf("ReadMIMEHeader: %v", err)
	}
	cookie := m.Get("Cookie")
	if cookie != sdata {
		t.Fatalf("ReadMIMEHeader: %v bytes, want %v bytes", len(cookie), len(sdata))
	}
}

// Test that we read slightly-bogus MIME headers seen in the wild,
// with spaces before colons, and spaces in keys.
func TestReadMIMEHeaderNonCompliant(t *testing.T) {
	// Invalid HTTP response header as sent by an Axis security
	// camera: (this is handled by IE, Firefox, Chrome, curl, etc.)
	r := reader("Foo: bar\r\n" +
		"Content-Language: en\r\n" +
		"SID : 0\r\n" +
		"Audio Mode : None\r\n" +
		"Privilege : 127\r\n\r\n")
	m, err := r.ReadMIMEHeader()
	want := MIMEHeader{
		"Foo":              {"bar"},
		"Content-Language": {"en"},
		"Sid":              {"0"},
		"Audio-Mode":       {"None"},
		"Privilege":        {"127"},
	}
	if !reflect.DeepEqual(m, want) || err != nil {
		t.Fatalf("ReadMIMEHeader =\n%v, %v; want:\n%v", m, err, want)
	}
}

type readResponseTest struct {
	in       string
	inCode   int
	wantCode int
	wantMsg  string
}

var readResponseTests = []readResponseTest{
	{"230-Anonymous access granted, restrictions apply\n" +
		"Read the file README.txt,\n" +
		"230  please",
		23,
		230,
		"Anonymous access granted, restrictions apply\nRead the file README.txt,\n please",
	},

	{"230 Anonymous access granted, restrictions apply\n",
		23,
		230,
		"Anonymous access granted, restrictions apply",
	},

	{"400-A\n400-B\n400 C",
		4,
		400,
		"A\nB\nC",
	},

	{"400-A\r\n400-B\r\n400 C\r\n",
		4,
		400,
		"A\nB\nC",
	},
}

// See http://www.ietf.org/rfc/rfc959.txt page 36.
func TestRFC959Lines(t *testing.T) {
	for i, tt := range readResponseTests {
		r := reader(tt.in + "\nFOLLOWING DATA")
		code, msg, err := r.ReadResponse(tt.inCode)
		if err != nil {
			t.Errorf("#%d: ReadResponse: %v", i, err)
			continue
		}
		if code != tt.wantCode {
			t.Errorf("#%d: code=%d, want %d", i, code, tt.wantCode)
		}
		if msg != tt.wantMsg {
			t.Errorf("#%d: msg=%q, want %q", i, msg, tt.wantMsg)
		}
	}
}

func TestCommonHeaders(t *testing.T) {
	// need to disable the commonHeaders-based optimization
	// during this check, or we'd not be testing anything
	oldch := commonHeaders
	commonHeaders = []string{}
	defer func() { commonHeaders = oldch }()

	last := ""
	for _, h := range oldch {
		if last > h {
			t.Errorf("%v is out of order", h)
		}
		if last == h {
			t.Errorf("%v is duplicated", h)
		}
		if canon := CanonicalMIMEHeaderKey(h); h != canon {
			t.Errorf("%v is not canonical", h)
		}
		last = h
	}
}

var clientHeaders = strings.Replace(`Host: golang.org
Connection: keep-alive
Cache-Control: max-age=0
Accept: application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5
User-Agent: Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.472.63 Safari/534.3
Accept-Encoding: gzip,deflate,sdch
Accept-Language: en-US,en;q=0.8,fr-CH;q=0.6
Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.3
COOKIE: __utma=000000000.0000000000.0000000000.0000000000.0000000000.00; __utmb=000000000.0.00.0000000000; __utmc=000000000; __utmz=000000000.0000000000.00.0.utmcsr=code.google.com|utmccn=(referral)|utmcmd=referral|utmcct=/p/go/issues/detail
Non-Interned: test

`, "\n", "\r\n", -1)

var serverHeaders = strings.Replace(`Content-Type: text/html; charset=utf-8
Content-Encoding: gzip
Date: Thu, 27 Sep 2012 09:03:33 GMT
Server: Google Frontend
Cache-Control: private
Content-Length: 2298
VIA: 1.1 proxy.example.com:80 (XXX/n.n.n-nnn)
Connection: Close
Non-Interned: test

`, "\n", "\r\n", -1)

func BenchmarkReadMIMEHeader(b *testing.B) {
	b.ReportAllocs()
	var buf bytes.Buffer
	br := bufio.NewReader(&buf)
	r := NewReader(br)
	for i := 0; i < b.N; i++ {
		var want int
		var find string
		if (i & 1) == 1 {
			buf.WriteString(clientHeaders)
			want = 10
			find = "Cookie"
		} else {
			buf.WriteString(serverHeaders)
			want = 9
			find = "Via"
		}
		h, err := r.ReadMIMEHeader()
		if err != nil {
			b.Fatal(err)
		}
		if len(h) != want {
			b.Fatalf("wrong number of headers: got %d, want %d", len(h), want)
		}
		if _, ok := h[find]; !ok {
			b.Fatalf("did not find key %s", find)
		}
	}
}

func BenchmarkUncommon(b *testing.B) {
	b.ReportAllocs()
	var buf bytes.Buffer
	br := bufio.NewReader(&buf)
	r := NewReader(br)
	for i := 0; i < b.N; i++ {
		buf.WriteString("uncommon-header-for-benchmark: foo\r\n\r\n")
		h, err := r.ReadMIMEHeader()
		if err != nil {
			b.Fatal(err)
		}
		if _, ok := h["Uncommon-Header-For-Benchmark"]; !ok {
			b.Fatal("Missing result header.")
		}
	}
}
<|endoftext|>


<commit_before>package common

import (
	"bytes"
	"errors"
	"fmt"
	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/helpers"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"
)

type BuildState string

const (
	Pending BuildState = "pending"
	Running            = "running"
	Failed             = "failed"
	Success            = "success"
)

type Build struct {
	GetBuildResponse `yaml:",inline"`
	BuildState       BuildState     `json:"build_state"`
	BuildStarted     time.Time      `json:"build_started"`
	BuildFinished    time.Time      `json:"build_finished"`
	BuildDuration    time.Duration  `json:"build_duration"`
	BuildAbort       chan os.Signal `json:"-" yaml:"-"`
	RootDir          string         `json:"-" yaml:"-"`
	BuildDir         string         `json:"-" yaml:"-"`
	CacheDir         string         `json:"-" yaml:"-"`
	Hostname         string         `json:"-" yaml:"-"`
	Runner           *RunnerConfig  `json:"runner"`

	// Unique ID for all running builds (globally)
	GlobalID int `json:"global_id"`

	// Unique ID for all running builds on this runner
	RunnerID int `json:"runner_id"`

	// Unique ID for all running builds on this runner and this project
	ProjectRunnerID int `json:"project_runner_id"`

	buildLog     bytes.Buffer
	buildLogLock sync.RWMutex
}

func (b *Build) AssignID(otherBuilds ...*Build) {
	globals := make(map[int]bool)
	runners := make(map[int]bool)
	projectRunners := make(map[int]bool)

	for _, otherBuild := range otherBuilds {
		globals[otherBuild.GlobalID] = true

		if otherBuild.Runner.ShortDescription() != b.Runner.ShortDescription() {
			continue
		}
		runners[otherBuild.RunnerID] = true

		if otherBuild.ProjectID != b.ProjectID {
			continue
		}
		projectRunners[otherBuild.ProjectRunnerID] = true
	}

	for i := 0; ; i++ {
		if !globals[i] {
			b.GlobalID = i
			break
		}
	}

	for i := 0; ; i++ {
		if !runners[i] {
			b.RunnerID = i
			break
		}
	}

	for i := 0; ; i++ {
		if !projectRunners[i] {
			b.ProjectRunnerID = i
			break
		}
	}
}

func (b *Build) ProjectUniqueName() string {
	return fmt.Sprintf("runner-%s-project-%d-concurrent-%d",
		b.Runner.ShortDescription(), b.ProjectID, b.ProjectRunnerID)
}

func (b *Build) ProjectSlug() (string, error) {
	url, err := url.Parse(b.RepoURL)
	if err != nil {
		return "", err
	}
	if url.Host == "" {
		return "", errors.New("only URI reference supported")
	}

	slug := url.Path
	slug = strings.TrimSuffix(slug, ".git")
	slug = filepath.Clean(slug)
	if slug == "." {
		return "", errors.New("invalid path")
	}
	if strings.Contains(slug, "..") {
		return "", errors.New("it doesn't look like a valid path")
	}
	return slug, nil
}

func (b *Build) ProjectUniqueDir(sharedDir bool) string {
	dir, err := b.ProjectSlug()
	if err != nil {
		dir = fmt.Sprintf("project-%d", b.ProjectID)
	}

	// for shared dirs path is constructed like this:
	// <some-path>/runner-short-id/concurrent-id/group-name/project-name/
	// ex.<some-path>/01234567/0/group/repo/
	if sharedDir {
		dir = filepath.Join(
			fmt.Sprintf("%s", b.Runner.ShortDescription()),
			fmt.Sprintf("%d", b.ProjectRunnerID),
			dir,
		)
	}
	return dir
}

func (b *Build) FullProjectDir() string {
	return b.BuildDir
}

func (b *Build) CacheFileForRef(ref string) string {
	if b.CacheDir != "" {
		return filepath.Join(b.CacheDir, ref, b.Name+".tgz")
	}
	return ""
}

func (b *Build) CacheFile() string {
	// For tags we don't create cache
	if b.Tag {
		return ""
	}
	return b.CacheFileForRef(b.RefName)
}

func (b *Build) StartBuild(rootDir, cacheDir string, sharedDir bool) {
	b.BuildStarted = time.Now()
	b.BuildState = Pending
	b.RootDir = rootDir
	b.BuildDir = filepath.Join(rootDir, b.ProjectUniqueDir(sharedDir))
	b.CacheDir = filepath.Join(cacheDir, b.ProjectUniqueDir(false))
}

func (b *Build) FinishBuild(buildState BuildState) {
	b.BuildState = buildState
	b.BuildFinished = time.Now()
	b.BuildDuration = b.BuildFinished.Sub(b.BuildStarted)
}

func (b *Build) BuildLog() string {
	b.buildLogLock.RLock()
	defer b.buildLogLock.RUnlock()
	return b.buildLog.String()
}

func (b *Build) BuildLogLen() int {
	b.buildLogLock.RLock()
	defer b.buildLogLock.RUnlock()
	return b.buildLog.Len()
}

func (b *Build) WriteString(data string) (int, error) {
	b.buildLogLock.Lock()
	defer b.buildLogLock.Unlock()
	return b.buildLog.WriteString(data)
}

func (b *Build) WriteRune(r rune) (int, error) {
	b.buildLogLock.Lock()
	defer b.buildLogLock.Unlock()
	return b.buildLog.WriteRune(r)
}

func (b *Build) SendBuildLog() {
	var buildTrace string

	buildTrace = b.BuildLog()
	for {
		if UpdateBuild(*b.Runner, b.ID, b.BuildState, buildTrace) != UpdateFailed {
			break
		} else {
			time.Sleep(UpdateRetryInterval * time.Second)
		}
	}
}

func (b *Build) Run(globalConfig *Config) error {
	executor := NewExecutor(b.Runner.Executor)
	if executor == nil {
		b.WriteString("Executor not found: " + b.Runner.Executor)
		b.SendBuildLog()
		return errors.New("executor not found")
	}

	err := executor.Prepare(globalConfig, b.Runner, b)
	if err == nil {
		err = executor.Start()
	}
	if err == nil {
		err = executor.Wait()
	}
	executor.Finish(err)
	if executor != nil {
		executor.Cleanup()
	}
	return err
}

func (b *Build) String() string {
	return helpers.ToYAML(b)
}
<commit_msg>Calculate cache storage path using relative base path<commit_after>package common

import (
	"bytes"
	"errors"
	"fmt"
	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/helpers"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"
)

type BuildState string

const (
	Pending BuildState = "pending"
	Running            = "running"
	Failed             = "failed"
	Success            = "success"
)

type Build struct {
	GetBuildResponse `yaml:",inline"`
	BuildState       BuildState     `json:"build_state"`
	BuildStarted     time.Time      `json:"build_started"`
	BuildFinished    time.Time      `json:"build_finished"`
	BuildDuration    time.Duration  `json:"build_duration"`
	BuildAbort       chan os.Signal `json:"-" yaml:"-"`
	RootDir          string         `json:"-" yaml:"-"`
	BuildDir         string         `json:"-" yaml:"-"`
	CacheDir         string         `json:"-" yaml:"-"`
	Hostname         string         `json:"-" yaml:"-"`
	Runner           *RunnerConfig  `json:"runner"`

	// Unique ID for all running builds (globally)
	GlobalID int `json:"global_id"`

	// Unique ID for all running builds on this runner
	RunnerID int `json:"runner_id"`

	// Unique ID for all running builds on this runner and this project
	ProjectRunnerID int `json:"project_runner_id"`

	buildLog     bytes.Buffer
	buildLogLock sync.RWMutex
}

func (b *Build) AssignID(otherBuilds ...*Build) {
	globals := make(map[int]bool)
	runners := make(map[int]bool)
	projectRunners := make(map[int]bool)

	for _, otherBuild := range otherBuilds {
		globals[otherBuild.GlobalID] = true

		if otherBuild.Runner.ShortDescription() != b.Runner.ShortDescription() {
			continue
		}
		runners[otherBuild.RunnerID] = true

		if otherBuild.ProjectID != b.ProjectID {
			continue
		}
		projectRunners[otherBuild.ProjectRunnerID] = true
	}

	for i := 0; ; i++ {
		if !globals[i] {
			b.GlobalID = i
			break
		}
	}

	for i := 0; ; i++ {
		if !runners[i] {
			b.RunnerID = i
			break
		}
	}

	for i := 0; ; i++ {
		if !projectRunners[i] {
			b.ProjectRunnerID = i
			break
		}
	}
}

func (b *Build) ProjectUniqueName() string {
	return fmt.Sprintf("runner-%s-project-%d-concurrent-%d",
		b.Runner.ShortDescription(), b.ProjectID, b.ProjectRunnerID)
}

func (b *Build) ProjectSlug() (string, error) {
	url, err := url.Parse(b.RepoURL)
	if err != nil {
		return "", err
	}
	if url.Host == "" {
		return "", errors.New("only URI reference supported")
	}

	slug := url.Path
	slug = strings.TrimSuffix(slug, ".git")
	slug = filepath.Clean(slug)
	if slug == "." {
		return "", errors.New("invalid path")
	}
	if strings.Contains(slug, "..") {
		return "", errors.New("it doesn't look like a valid path")
	}
	return slug, nil
}

func (b *Build) ProjectUniqueDir(sharedDir bool) string {
	dir, err := b.ProjectSlug()
	if err != nil {
		dir = fmt.Sprintf("project-%d", b.ProjectID)
	}

	// for shared dirs path is constructed like this:
	// <some-path>/runner-short-id/concurrent-id/group-name/project-name/
	// ex.<some-path>/01234567/0/group/repo/
	if sharedDir {
		dir = filepath.Join(
			fmt.Sprintf("%s", b.Runner.ShortDescription()),
			fmt.Sprintf("%d", b.ProjectRunnerID),
			dir,
		)
	}
	return dir
}

func (b *Build) FullProjectDir() string {
	return b.BuildDir
}

func (b *Build) CacheFileForRef(ref string) string {
	if b.CacheDir != "" {
		cacheFile := filepath.Join(b.CacheDir, ref, b.Name+".tgz")
		cacheFile, err := filepath.Rel(b.BuildDir, cacheFile)
		if err != nil {
			return ""
		}
		return cacheFile
	}
	return ""
}

func (b *Build) CacheFile() string {
	// For tags we don't create cache
	if b.Tag {
		return ""
	}
	return b.CacheFileForRef(b.RefName)
}

func (b *Build) StartBuild(rootDir, cacheDir string, sharedDir bool) {
	b.BuildStarted = time.Now()
	b.BuildState = Pending
	b.RootDir = rootDir
	b.BuildDir = filepath.Join(rootDir, b.ProjectUniqueDir(sharedDir))
	b.CacheDir = filepath.Join(cacheDir, b.ProjectUniqueDir(false))
}

func (b *Build) FinishBuild(buildState BuildState) {
	b.BuildState = buildState
	b.BuildFinished = time.Now()
	b.BuildDuration = b.BuildFinished.Sub(b.BuildStarted)
}

func (b *Build) BuildLog() string {
	b.buildLogLock.RLock()
	defer b.buildLogLock.RUnlock()
	return b.buildLog.String()
}

func (b *Build) BuildLogLen() int {
	b.buildLogLock.RLock()
	defer b.buildLogLock.RUnlock()
	return b.buildLog.Len()
}

func (b *Build) WriteString(data string) (int, error) {
	b.buildLogLock.Lock()
	defer b.buildLogLock.Unlock()
	return b.buildLog.WriteString(data)
}

func (b *Build) WriteRune(r rune) (int, error) {
	b.buildLogLock.Lock()
	defer b.buildLogLock.Unlock()
	return b.buildLog.WriteRune(r)
}

func (b *Build) SendBuildLog() {
	var buildTrace string

	buildTrace = b.BuildLog()
	for {
		if UpdateBuild(*b.Runner, b.ID, b.BuildState, buildTrace) != UpdateFailed {
			break
		} else {
			time.Sleep(UpdateRetryInterval * time.Second)
		}
	}
}

func (b *Build) Run(globalConfig *Config) error {
	executor := NewExecutor(b.Runner.Executor)
	if executor == nil {
		b.WriteString("Executor not found: " + b.Runner.Executor)
		b.SendBuildLog()
		return errors.New("executor not found")
	}

	err := executor.Prepare(globalConfig, b.Runner, b)
	if err == nil {
		err = executor.Start()
	}
	if err == nil {
		err = executor.Wait()
	}
	executor.Finish(err)
	if executor != nil {
		executor.Cleanup()
	}
	return err
}

func (b *Build) String() string {
	return helpers.ToYAML(b)
}
<|endoftext|>


<commit_before>// Copyright 2015 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package parser

import (
	"testing"

	"github.com/prometheus/prometheus/util/testutil"
)

func TestExprString(t *testing.T) {
	// A list of valid expressions that are expected to be
	// returned as out when calling String(). If out is empty the output
	// is expected to equal the input.
	inputs := []struct {
		in, out string
	}{
		{
			in:  `sum by() (task:errors:rate10s{job="s"})`,
			out: `sum(task:errors:rate10s{job="s"})`,
		},
		{
			in: `sum by(code) (task:errors:rate10s{job="s"})`,
		},
		{
			in: `sum without() (task:errors:rate10s{job="s"})`,
		},
		{
			in: `sum without(instance) (task:errors:rate10s{job="s"})`,
		},
		{
			in: `topk(5, task:errors:rate10s{job="s"})`,
		},
		{
			in: `count_values("value", task:errors:rate10s{job="s"})`,
		},
		{
			in: `a - on() c`,
		},
		{
			in: `a - on(b) c`,
		},
		{
			in: `a - on(b) group_left(x) c`,
		},
		{
			in: `a - on(b) group_left(x, y) c`,
		},
		{
			in:  `a - on(b) group_left c`,
			out: `a - on(b) group_left() c`,
		},
		{
			in: `a - on(b) group_left() (c)`,
		},
		{
			in: `a - ignoring(b) c`,
		},
		{
			in:  `a - ignoring() c`,
			out: `a - c`,
		},
		{
			in: `up > bool 0`,
		},
		{
			in: `a offset 1m`,
		},
		{
			in: `a{c="d"}[5m] offset 1m`,
		},
		{
			in: `a[5m] offset 1m`,
		},
		{
			in: `a[1h:5m] offset 1m`,
		},
		{
			in: `{__name__="a"}`,
		},
	}

	for _, test := range inputs {
		expr, err := ParseExpr(test.in)
		testutil.Ok(t, err)

		exp := test.in
		if test.out != "" {
			exp = test.out
		}

		testutil.Equals(t, exp, expr.String())
	}
}
<commit_msg>Add unit test case to improve test coverage for matcher.go (#7658)<commit_after>// Copyright 2015 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package parser

import (
	"testing"

	"github.com/prometheus/prometheus/util/testutil"
)

func TestExprString(t *testing.T) {
	// A list of valid expressions that are expected to be
	// returned as out when calling String(). If out is empty the output
	// is expected to equal the input.
	inputs := []struct {
		in, out string
	}{
		{
			in:  `sum by() (task:errors:rate10s{job="s"})`,
			out: `sum(task:errors:rate10s{job="s"})`,
		},
		{
			in: `sum by(code) (task:errors:rate10s{job="s"})`,
		},
		{
			in: `sum without() (task:errors:rate10s{job="s"})`,
		},
		{
			in: `sum without(instance) (task:errors:rate10s{job="s"})`,
		},
		{
			in: `topk(5, task:errors:rate10s{job="s"})`,
		},
		{
			in: `count_values("value", task:errors:rate10s{job="s"})`,
		},
		{
			in: `a - on() c`,
		},
		{
			in: `a - on(b) c`,
		},
		{
			in: `a - on(b) group_left(x) c`,
		},
		{
			in: `a - on(b) group_left(x, y) c`,
		},
		{
			in:  `a - on(b) group_left c`,
			out: `a - on(b) group_left() c`,
		},
		{
			in: `a - on(b) group_left() (c)`,
		},
		{
			in: `a - ignoring(b) c`,
		},
		{
			in:  `a - ignoring() c`,
			out: `a - c`,
		},
		{
			in: `up > bool 0`,
		},
		{
			in: `a offset 1m`,
		},
		{
			in: `a{c="d"}[5m] offset 1m`,
		},
		{
			in: `a[5m] offset 1m`,
		},
		{
			in: `a[1h:5m] offset 1m`,
		},
		{
			in: `{__name__="a"}`,
		},
		{
			in: `a{b!="c"}[1m]`,
		},
		{
			in: `a{b=~"c"}[1m]`,
		},
		{
			in: `a{b!~"c"}[1m]`,
		},
	}

	for _, test := range inputs {
		expr, err := ParseExpr(test.in)
		testutil.Ok(t, err)

		exp := test.in
		if test.out != "" {
			exp = test.out
		}

		testutil.Equals(t, exp, expr.String())
	}
}
<|endoftext|>


<commit_before>package config

import (
	"flag"
	"fmt"
	"os"
)

type ProcessConfig interface {
	VerboseMode() bool
	SuperVerboseMode() bool
	EnableDiscord() bool
}

type configHolder struct {
	verbose      bool
	superVerbose bool
	discord      bool
}

func (c configHolder) VerboseMode() bool {
	return c.verbose || c.superVerbose
}

func (c configHolder) SuperVerboseMode() bool {
	return c.superVerbose
}

func (c configHolder) EnableDiscord() bool {
	return c.discord
}

var config *configHolder

func Configuration() ProcessConfig {
	if config == nil {
		verboseModeFlag := flag.Bool("v", false, "more logging")
		superVerboseModeFlag := flag.Bool("vv", false, "more more logging")
		flag.Parse()
		discord := len(os.Getenv("discord_token")) > 0
		logLevel := os.Getenv("log_level")
		superVerboseMode := *superVerboseModeFlag || logLevel == "debug"
		verboseMode := *verboseModeFlag || superVerboseMode || logLevel == "info"
		config = &configHolder{verboseMode, *superVerboseModeFlag, discord}

		if superVerboseMode {
			fmt.Print("!!!! SUPER Verbose Logging enabled !!!!\n\n")
		} else if verboseMode {
			fmt.Print("!!!! Verbose Logging enabled !!!!\n\n")
		}
	}

	return config
}
<commit_msg>Properly load super verbose mode from environment variables<commit_after>package config

import (
	"flag"
	"fmt"
	"os"
)

type ProcessConfig interface {
	VerboseMode() bool
	SuperVerboseMode() bool
	EnableDiscord() bool
}

type configHolder struct {
	verbose      bool
	superVerbose bool
	discord      bool
}

func (c configHolder) VerboseMode() bool {
	return c.verbose || c.superVerbose
}

func (c configHolder) SuperVerboseMode() bool {
	return c.superVerbose
}

func (c configHolder) EnableDiscord() bool {
	return c.discord
}

var config *configHolder

func Configuration() ProcessConfig {
	if config == nil {
		verboseModeFlag := flag.Bool("v", false, "more logging")
		superVerboseModeFlag := flag.Bool("vv", false, "more more logging")
		flag.Parse()
		discord := len(os.Getenv("discord_token")) > 0
		logLevel := os.Getenv("log_level")
		superVerboseMode := *superVerboseModeFlag || logLevel == "debug"
		verboseMode := *verboseModeFlag || superVerboseMode || logLevel == "info"
		config = &configHolder{verboseMode, superVerboseMode, discord}

		if superVerboseMode {
			fmt.Print("!!!! SUPER Verbose Logging enabled !!!!\n\n")
		} else if verboseMode {
			fmt.Print("!!!! Verbose Logging enabled !!!!\n\n")
		}
	}

	return config
}
<|endoftext|>


<commit_before>package pipelineserver

import (
	"errors"
	"fmt"
	"sync"
	"time"

	"go.pachyderm.com/pachyderm/src/pfs"
	"go.pachyderm.com/pachyderm/src/pps"
	"go.pachyderm.com/pachyderm/src/pps/persist"
	"go.pedge.io/google-protobuf"
	"go.pedge.io/proto/rpclog"
	"go.pedge.io/protolog"
	"golang.org/x/net/context"
)

const (
	changeEventTypeCreate changeEventType = iota
	changeEventTypeDelete
)

type changeEventType int

type apiServer struct {
	protorpclog.Logger
	pfsAPIClient     pfs.APIClient
	jobAPIClient     pps.JobAPIClient
	persistAPIClient persist.APIClient

	started                          bool
	pipelineNameToPipelineController map[string]*pipelineController
	lock                             *sync.Mutex
}

func newAPIServer(
	pfsAPIClient pfs.APIClient,
	jobAPIClient pps.JobAPIClient,
	persistAPIClient persist.APIClient,
) *apiServer {
	return &apiServer{
		protorpclog.NewLogger("pachyderm.pps.PipelineAPI"),
		pfsAPIClient,
		jobAPIClient,
		persistAPIClient,
		false,
		make(map[string]*pipelineController),
		&sync.Mutex{},
	}
}

func (a *apiServer) Start() error {
	a.lock.Lock()
	defer a.lock.Unlock()
	// TODO(pedge): volatile bool?
	if a.started {
		// TODO(pedge): abstract error to public variable
		return errors.New("pachyderm.pps.pipelineserver: already started")
	}
	a.started = true
	pipelineInfos, err := a.ListPipeline(context.Background(), &pps.ListPipelineRequest{})
	if err != nil {
		return err
	}
	for _, pipelineInfo := range pipelineInfos.PipelineInfo {
		if err := a.addPipelineController(pipelineInfo); err != nil {
			return err
		}
	}
	return nil
}

func (a *apiServer) CreatePipeline(ctx context.Context, request *pps.CreatePipelineRequest) (response *google_protobuf.Empty, err error) {
	defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now())
	if _, err := a.persistAPIClient.CreatePipelineInfo(
		ctx,
		&persist.PipelineInfo{
			PipelineName: request.Pipeline.Name,
			Transform:    request.Transform,
			Input:        request.Input,
			Output:       request.Output,
		},
	); err != nil {
		return nil, err
	}
	if err := a.registerChangeEvent(
		ctx,
		&changeEvent{
			Type:         changeEventTypeCreate,
			PipelineName: request.Pipeline.Name,
		},
	); err != nil {
		// TODO(pedge): need to roll back the db create
		return nil, err
	}
	return google_protobuf.EmptyInstance, nil
}

func (a *apiServer) InspectPipeline(ctx context.Context, request *pps.InspectPipelineRequest) (response *pps.PipelineInfo, err error) {
	defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now())
	persistPipelineInfo, err := a.persistAPIClient.GetPipelineInfo(ctx, request.Pipeline)
	if err != nil {
		return nil, err
	}
	return persistPipelineInfoToPipelineInfo(persistPipelineInfo), nil
}

func (a *apiServer) ListPipeline(ctx context.Context, request *pps.ListPipelineRequest) (response *pps.PipelineInfos, err error) {
	defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now())
	persistPipelineInfos, err := a.persistAPIClient.ListPipelineInfos(ctx, google_protobuf.EmptyInstance)
	if err != nil {
		return nil, err
	}
	pipelineInfos := make([]*pps.PipelineInfo, len(persistPipelineInfos.PipelineInfo))
	for i, persistPipelineInfo := range persistPipelineInfos.PipelineInfo {
		pipelineInfos[i] = persistPipelineInfoToPipelineInfo(persistPipelineInfo)
	}
	return &pps.PipelineInfos{
		PipelineInfo: pipelineInfos,
	}, nil
}

func (a *apiServer) DeletePipeline(ctx context.Context, request *pps.DeletePipelineRequest) (response *google_protobuf.Empty, err error) {
	if _, err := a.persistAPIClient.DeletePipelineInfo(ctx, request.Pipeline); err != nil {
		return nil, err
	}
	if err := a.registerChangeEvent(
		ctx,
		&changeEvent{
			Type:         changeEventTypeDelete,
			PipelineName: request.Pipeline.Name,
		},
	); err != nil {
		return nil, err
	}
	return google_protobuf.EmptyInstance, nil
}

func persistPipelineInfoToPipelineInfo(persistPipelineInfo *persist.PipelineInfo) *pps.PipelineInfo {
	return &pps.PipelineInfo{
		Pipeline: &pps.Pipeline{
			Name: persistPipelineInfo.PipelineName,
		},
		Transform: persistPipelineInfo.Transform,
		Input:     persistPipelineInfo.Input,
		Output:    persistPipelineInfo.Output,
	}
}

type changeEvent struct {
	Type         changeEventType
	PipelineName string
}

// TODO(pedge): this is relateively out of date, we can just do this directly in the functions, and
// with the create at least, we avoid a db read
func (a *apiServer) registerChangeEvent(ctx context.Context, request *changeEvent) error {
	a.lock.Lock()
	defer a.lock.Unlock()
	switch request.Type {
	case changeEventTypeCreate:
		if !a.pipelineRegistered(request.PipelineName) {
			if err := a.addPipelineControllerByName(ctx, request.PipelineName); err != nil {
				return err
			}
			// TODO(pedge): what to do?
		} else {
			protolog.Warnf("pachyderm.pps.pipelineserver: had a create change event for an existing pipeline: %v", request)
			if err := a.removePipelineController(request.PipelineName); err != nil {
				return err
			}
			if err := a.addPipelineControllerByName(ctx, request.PipelineName); err != nil {
				return err
			}
		}
	case changeEventTypeDelete:
		if !a.pipelineRegistered(request.PipelineName) {
			protolog.Warnf("pachyderm.pps.pipelineserver: had a delete change event for a pipeline that was not registered: %v", request)
		} else {
			if err := a.removePipelineController(request.PipelineName); err != nil {
				return err
			}
		}
	default:
		return fmt.Errorf("pachyderm.pps.pipelineserver: unknown change event type: %v", request.Type)
	}
	return nil
}

func (a *apiServer) pipelineRegistered(name string) bool {
	_, ok := a.pipelineNameToPipelineController[name]
	return ok
}

func (a *apiServer) addPipelineControllerByName(ctx context.Context, name string) error {
	pipelineInfo, err := a.InspectPipeline(
		ctx,
		&pps.InspectPipelineRequest{
			Pipeline: &pps.Pipeline{
				Name: name,
			},
		},
	)
	if err != nil {
		return err
	}
	return a.addPipelineController(pipelineInfo)
}

func (a *apiServer) addPipelineController(pipelineInfo *pps.PipelineInfo) error {
	pipelineController := newPipelineController(
		a.pfsAPIClient,
		a.jobAPIClient,
		pps.NewLocalPipelineAPIClient(a),
		pipelineInfo,
	)
	a.pipelineNameToPipelineController[pipelineInfo.Pipeline.Name] = pipelineController
	return pipelineController.Start()
}

func (a *apiServer) removePipelineController(name string) error {
	pipelineController, ok := a.pipelineNameToPipelineController[name]
	if !ok {
		return fmt.Errorf("pachyderm.pps.pipelineserver: no pipeline registered for name: %s", name)
	}
	pipelineController.Cancel()
	return nil
}
<commit_msg>remove changeEvent, changeEventType, registerChangeEvent, and associated code from pps api pipeline server impl<commit_after>package pipelineserver

import (
	"errors"
	"sync"
	"time"

	"go.pachyderm.com/pachyderm/src/pfs"
	"go.pachyderm.com/pachyderm/src/pps"
	"go.pachyderm.com/pachyderm/src/pps/persist"
	"go.pedge.io/google-protobuf"
	"go.pedge.io/proto/rpclog"
	"go.pedge.io/protolog"
	"golang.org/x/net/context"
)

type apiServer struct {
	protorpclog.Logger
	pfsAPIClient     pfs.APIClient
	jobAPIClient     pps.JobAPIClient
	persistAPIClient persist.APIClient

	started                          bool
	pipelineNameToPipelineController map[string]*pipelineController
	lock                             *sync.Mutex
}

func newAPIServer(
	pfsAPIClient pfs.APIClient,
	jobAPIClient pps.JobAPIClient,
	persistAPIClient persist.APIClient,
) *apiServer {
	return &apiServer{
		protorpclog.NewLogger("pachyderm.pps.PipelineAPI"),
		pfsAPIClient,
		jobAPIClient,
		persistAPIClient,
		false,
		make(map[string]*pipelineController),
		&sync.Mutex{},
	}
}

func (a *apiServer) Start() error {
	a.lock.Lock()
	defer a.lock.Unlock()
	// TODO(pedge): volatile bool?
	if a.started {
		// TODO(pedge): abstract error to public variable
		return errors.New("pachyderm.pps.pipelineserver: already started")
	}
	a.started = true
	pipelineInfos, err := a.ListPipeline(context.Background(), &pps.ListPipelineRequest{})
	if err != nil {
		return err
	}
	for _, pipelineInfo := range pipelineInfos.PipelineInfo {
		if err := a.addPipelineController(pipelineInfo); err != nil {
			return err
		}
	}
	return nil
}

func (a *apiServer) CreatePipeline(ctx context.Context, request *pps.CreatePipelineRequest) (response *google_protobuf.Empty, err error) {
	defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now())
	persistPipelineInfo := &persist.PipelineInfo{
		PipelineName: request.Pipeline.Name,
		Transform:    request.Transform,
		Input:        request.Input,
		Output:       request.Output,
	}
	if _, err := a.persistAPIClient.CreatePipelineInfo(ctx, persistPipelineInfo); err != nil {
		return nil, err
	}
	a.lock.Lock()
	defer a.lock.Unlock()
	if err := a.addPipelineController(persistPipelineInfoToPipelineInfo(persistPipelineInfo)); err != nil {
		// TODO(pedge): need to roll back the db create
		return nil, err
	}
	return google_protobuf.EmptyInstance, nil
}

func (a *apiServer) InspectPipeline(ctx context.Context, request *pps.InspectPipelineRequest) (response *pps.PipelineInfo, err error) {
	defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now())
	persistPipelineInfo, err := a.persistAPIClient.GetPipelineInfo(ctx, request.Pipeline)
	if err != nil {
		return nil, err
	}
	return persistPipelineInfoToPipelineInfo(persistPipelineInfo), nil
}

func (a *apiServer) ListPipeline(ctx context.Context, request *pps.ListPipelineRequest) (response *pps.PipelineInfos, err error) {
	defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now())
	persistPipelineInfos, err := a.persistAPIClient.ListPipelineInfos(ctx, google_protobuf.EmptyInstance)
	if err != nil {
		return nil, err
	}
	pipelineInfos := make([]*pps.PipelineInfo, len(persistPipelineInfos.PipelineInfo))
	for i, persistPipelineInfo := range persistPipelineInfos.PipelineInfo {
		pipelineInfos[i] = persistPipelineInfoToPipelineInfo(persistPipelineInfo)
	}
	return &pps.PipelineInfos{
		PipelineInfo: pipelineInfos,
	}, nil
}

func (a *apiServer) DeletePipeline(ctx context.Context, request *pps.DeletePipelineRequest) (response *google_protobuf.Empty, err error) {
	if _, err := a.persistAPIClient.DeletePipelineInfo(ctx, request.Pipeline); err != nil {
		return nil, err
	}
	a.lock.Lock()
	defer a.lock.Unlock()
	if err := a.removePipelineController(request.Pipeline.Name); err != nil {
		return nil, err
	}
	return google_protobuf.EmptyInstance, nil
}

func persistPipelineInfoToPipelineInfo(persistPipelineInfo *persist.PipelineInfo) *pps.PipelineInfo {
	return &pps.PipelineInfo{
		Pipeline: &pps.Pipeline{
			Name: persistPipelineInfo.PipelineName,
		},
		Transform: persistPipelineInfo.Transform,
		Input:     persistPipelineInfo.Input,
		Output:    persistPipelineInfo.Output,
	}
}

func (a *apiServer) addPipelineController(pipelineInfo *pps.PipelineInfo) error {
	if _, ok := a.pipelineNameToPipelineController[pipelineInfo.Pipeline.Name]; ok {
		protolog.Warnf("pachyderm.pps.pipelineserver: had a create change event for an existing pipeline: %v", pipelineInfo)
		if err := a.removePipelineController(pipelineInfo.Pipeline.Name); err != nil {
			return err
		}
	}
	pipelineController := newPipelineController(
		a.pfsAPIClient,
		a.jobAPIClient,
		pps.NewLocalPipelineAPIClient(a),
		pipelineInfo,
	)
	a.pipelineNameToPipelineController[pipelineInfo.Pipeline.Name] = pipelineController
	return pipelineController.Start()
}

func (a *apiServer) removePipelineController(name string) error {
	pipelineController, ok := a.pipelineNameToPipelineController[name]
	if !ok {
		protolog.Warnf("pachyderm.pps.pipelineserver: no pipeline registered for name: %s", name)
	} else {
		pipelineController.Cancel()
	}
	return nil
}
<|endoftext|>


<commit_before>// +build linux,cgo,!agent

package db_test

import (
	"testing"

	"github.com/lxc/lxd/lxd/db"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

// The GetNetworksLocalConfigs method returns only node-specific config values.
func TestGetNetworksLocalConfigs(t *testing.T) {
	cluster, cleanup := db.NewTestCluster(t)
	defer cleanup()

	_, err := cluster.CreateNetwork("lxdbr0", "", db.NetworkTypeBridge, map[string]string{
		"dns.mode":                   "none",
		"bridge.external_interfaces": "vlan0",
	})
	require.NoError(t, err)

	var config map[string]map[string]string

	err = cluster.Transaction(func(tx *db.ClusterTx) error {
		var err error
		config, err = tx.GetNetworksLocalConfig()
		return err
	})
	require.NoError(t, err)

	assert.Equal(t, config, map[string]map[string]string{
		"lxdbr0": {"bridge.external_interfaces": "vlan0"},
	})
}

func TestCreatePendingNetwork(t *testing.T) {
	tx, cleanup := db.NewTestClusterTx(t)
	defer cleanup()

	_, err := tx.CreateNode("buzz", "1.2.3.4:666")
	require.NoError(t, err)
	_, err = tx.CreateNode("rusp", "5.6.7.8:666")
	require.NoError(t, err)

	config := map[string]string{"bridge.external_interfaces": "foo"}
	err = tx.CreatePendingNetwork("buzz", "network1", db.NetworkTypeBridge, config)
	require.NoError(t, err)

	networkID, err := tx.GetNetworkID("network1")
	require.NoError(t, err)
	assert.True(t, networkID > 0)

	config = map[string]string{"bridge.external_interfaces": "bar"}
	err = tx.CreatePendingNetwork("rusp", "network1", db.NetworkTypeBridge, config)
	require.NoError(t, err)

	// The initial node (whose name is 'none' by default) is missing.
	_, err = tx.NetworkNodeConfigs(networkID)
	require.EqualError(t, err, "Network not defined on nodes: none")

	config = map[string]string{"bridge.external_interfaces": "egg"}
	err = tx.CreatePendingNetwork("none", "network1", db.NetworkTypeBridge, config)
	require.NoError(t, err)

	// Now the storage is defined on all nodes.
	configs, err := tx.NetworkNodeConfigs(networkID)
	require.NoError(t, err)
	assert.Len(t, configs, 3)
	assert.Equal(t, map[string]string{"bridge.external_interfaces": "foo"}, configs["buzz"])
	assert.Equal(t, map[string]string{"bridge.external_interfaces": "bar"}, configs["rusp"])
	assert.Equal(t, map[string]string{"bridge.external_interfaces": "egg"}, configs["none"])
}

// If an entry for the given network and node already exists, an error is
// returned.
func TestNetworksCreatePending_AlreadyDefined(t *testing.T) {
	tx, cleanup := db.NewTestClusterTx(t)
	defer cleanup()

	_, err := tx.CreateNode("buzz", "1.2.3.4:666")
	require.NoError(t, err)

	err = tx.CreatePendingNetwork("buzz", "network1", db.NetworkTypeBridge, map[string]string{})
	require.NoError(t, err)

	err = tx.CreatePendingNetwork("buzz", "network1", db.NetworkTypeBridge, map[string]string{})
	require.Equal(t, db.ErrAlreadyDefined, err)
}

// If no node with the given name is found, an error is returned.
func TestNetworksCreatePending_NonExistingNode(t *testing.T) {
	tx, cleanup := db.NewTestClusterTx(t)
	defer cleanup()

	err := tx.CreatePendingNetwork("buzz", "network1", db.NetworkTypeBridge, map[string]string{})
	require.Equal(t, db.ErrNoSuchObject, err)
}
<commit_msg>lxd/db/networks/test: Updates tests to use project.Default for new networks<commit_after>// +build linux,cgo,!agent

package db_test

import (
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"

	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/project"
)

// The GetNetworksLocalConfigs method returns only node-specific config values.
func TestGetNetworksLocalConfigs(t *testing.T) {
	cluster, cleanup := db.NewTestCluster(t)
	defer cleanup()

	_, err := cluster.CreateNetwork(project.Default, "lxdbr0", "", db.NetworkTypeBridge, map[string]string{
		"dns.mode":                   "none",
		"bridge.external_interfaces": "vlan0",
	})
	require.NoError(t, err)

	var config map[string]map[string]string

	err = cluster.Transaction(func(tx *db.ClusterTx) error {
		var err error
		config, err = tx.GetNetworksLocalConfig()
		return err
	})
	require.NoError(t, err)

	assert.Equal(t, config, map[string]map[string]string{
		"lxdbr0": {"bridge.external_interfaces": "vlan0"},
	})
}

func TestCreatePendingNetwork(t *testing.T) {
	tx, cleanup := db.NewTestClusterTx(t)
	defer cleanup()

	_, err := tx.CreateNode("buzz", "1.2.3.4:666")
	require.NoError(t, err)
	_, err = tx.CreateNode("rusp", "5.6.7.8:666")
	require.NoError(t, err)

	config := map[string]string{"bridge.external_interfaces": "foo"}
	err = tx.CreatePendingNetwork("buzz", project.Default, "network1", db.NetworkTypeBridge, config)
	require.NoError(t, err)

	networkID, err := tx.GetNetworkID("network1")
	require.NoError(t, err)
	assert.True(t, networkID > 0)

	config = map[string]string{"bridge.external_interfaces": "bar"}
	err = tx.CreatePendingNetwork("rusp", project.Default, "network1", db.NetworkTypeBridge, config)
	require.NoError(t, err)

	// The initial node (whose name is 'none' by default) is missing.
	_, err = tx.NetworkNodeConfigs(networkID)
	require.EqualError(t, err, "Network not defined on nodes: none")

	config = map[string]string{"bridge.external_interfaces": "egg"}
	err = tx.CreatePendingNetwork("none", project.Default, "network1", db.NetworkTypeBridge, config)
	require.NoError(t, err)

	// Now the storage is defined on all nodes.
	configs, err := tx.NetworkNodeConfigs(networkID)
	require.NoError(t, err)
	assert.Len(t, configs, 3)
	assert.Equal(t, map[string]string{"bridge.external_interfaces": "foo"}, configs["buzz"])
	assert.Equal(t, map[string]string{"bridge.external_interfaces": "bar"}, configs["rusp"])
	assert.Equal(t, map[string]string{"bridge.external_interfaces": "egg"}, configs["none"])
}

// If an entry for the given network and node already exists, an error is
// returned.
func TestNetworksCreatePending_AlreadyDefined(t *testing.T) {
	tx, cleanup := db.NewTestClusterTx(t)
	defer cleanup()

	_, err := tx.CreateNode("buzz", "1.2.3.4:666")
	require.NoError(t, err)

	err = tx.CreatePendingNetwork("buzz", project.Default, "network1", db.NetworkTypeBridge, map[string]string{})
	require.NoError(t, err)

	err = tx.CreatePendingNetwork("buzz", project.Default, "network1", db.NetworkTypeBridge, map[string]string{})
	require.Equal(t, db.ErrAlreadyDefined, err)
}

// If no node with the given name is found, an error is returned.
func TestNetworksCreatePending_NonExistingNode(t *testing.T) {
	tx, cleanup := db.NewTestClusterTx(t)
	defer cleanup()

	err := tx.CreatePendingNetwork("buzz", project.Default, "network1", db.NetworkTypeBridge, map[string]string{})
	require.Equal(t, db.ErrNoSuchObject, err)
}
<|endoftext|>


<commit_before>package resources

import (
	"bufio"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/pkg/errors"

	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/units"
)

var sysDevicesNode = "/sys/devices/system/node"
var sysDevicesSystemMemory = "/sys/devices/system/memory"

type meminfo struct {
	Cached         uint64
	Buffers        uint64
	Total          uint64
	Free           uint64
	Used           uint64
	HugepagesTotal uint64
	HugepagesFree  uint64
	HugepagesSize  uint64
}

func parseMeminfo(path string) (*meminfo, error) {
	memory := meminfo{}

	// Open meminfo
	f, err := os.Open(path)
	if err != nil {
		return nil, errors.Wrapf(err, "Failed to open \"%s\"", path)
	}
	defer f.Close()
	memInfo := bufio.NewScanner(f)

	// Get common memory information
	for memInfo.Scan() {
		line := strings.TrimSpace(memInfo.Text())

		// Get key/value
		fields := strings.SplitN(line, ":", 2)
		key := strings.TrimSpace(fields[0])
		keyFields := strings.Split(key, " ")
		key = keyFields[len(keyFields)-1]
		value := strings.TrimSpace(fields[1])
		value = strings.Replace(value, " kB", "KiB", 1)

		if key == "MemTotal" {
			bytes, err := units.ParseByteSizeString(value)
			if err != nil {
				return nil, errors.Wrap(err, "Failed to parse MemTotal")
			}

			memory.Total = uint64(bytes)
			continue
		}

		if key == "MemFree" {
			bytes, err := units.ParseByteSizeString(value)
			if err != nil {
				return nil, errors.Wrap(err, "Failed to parse MemFree")
			}

			memory.Free = uint64(bytes)
			continue
		}

		if key == "MemUsed" {
			bytes, err := units.ParseByteSizeString(value)
			if err != nil {
				return nil, errors.Wrap(err, "Failed to parse MemUsed")
			}

			memory.Used = uint64(bytes)
			continue
		}

		if key == "Cached" {
			bytes, err := units.ParseByteSizeString(value)
			if err != nil {
				return nil, errors.Wrap(err, "Failed to parse Cached")
			}

			memory.Cached = uint64(bytes)
			continue
		}

		if key == "Buffers" {
			bytes, err := units.ParseByteSizeString(value)
			if err != nil {
				return nil, errors.Wrap(err, "Failed to parse Buffers")
			}

			memory.Buffers = uint64(bytes)
			continue
		}

		if key == "HugePages_Total" {
			bytes, err := units.ParseByteSizeString(value)
			if err != nil {
				return nil, errors.Wrap(err, "Failed to parse HugePages_Total")
			}

			memory.HugepagesTotal = uint64(bytes)
			continue
		}

		if key == "HugePages_Free" {
			bytes, err := units.ParseByteSizeString(value)
			if err != nil {
				return nil, errors.Wrap(err, "Failed to parse HugePages_Free")
			}

			memory.HugepagesFree = uint64(bytes)
			continue
		}

		if key == "Hugepagesize" {
			bytes, err := units.ParseByteSizeString(value)
			if err != nil {
				return nil, errors.Wrap(err, "Failed to parse Hugepagesize")
			}

			memory.HugepagesSize = uint64(bytes)
			continue
		}
	}

	return &memory, nil
}

func getTotalMemory() uint64 {
	memoryBlockSizePath := filepath.Join(sysDevicesSystemMemory, "block_size_bytes")

	if !sysfsExists(memoryBlockSizePath) {
		return 0
	}

	// Get block size
	content, err := ioutil.ReadFile(memoryBlockSizePath)
	if err != nil {
		return 0
	}

	blockSize, err := strconv.ParseUint(strings.TrimSpace(string(content)), 16, 64)
	if err != nil {
		return 0
	}

	entries, err := ioutil.ReadDir(sysDevicesSystemMemory)
	if err != nil {
		return 0
	}

	// Count the number of blocks
	var count uint64
	for _, entry := range entries {
		// Only consider directories
		if !entry.IsDir() {
			continue
		}

		entryName := entry.Name()
		entryPath := filepath.Join(sysDevicesSystemMemory, entryName)

		// Ignore directories not starting with "memory"
		if !strings.HasPrefix(entryName, "memory") {
			continue
		}

		content, err := ioutil.ReadFile(filepath.Join(entryPath, "online"))
		if err != nil {
			return 0
		}

		// Only count the block if it's online
		if strings.TrimSpace(string(content)) == "1" {
			count++
		}
	}

	return blockSize * count
}

// GetMemory returns a filled api.ResourcesMemory struct ready for use by LXD
func GetMemory() (*api.ResourcesMemory, error) {
	memory := api.ResourcesMemory{}

	// Parse main meminfo
	info, err := parseMeminfo("/proc/meminfo")
	if err != nil {
		return nil, errors.Wrap(err, "Failed to parse /proc/meminfo")
	}

	// Calculate the total memory from /sys/devices/system/memory, as the previously determined
	// value reports the amount of available system memory minus the amount reserved for the kernel.
	// If successful, replace the previous value, retrieved from /proc/meminfo.
	memTotal := getTotalMemory()
	if memTotal > 0 {
		info.Total = memTotal
	}

	// Fill used values
	memory.HugepagesUsed = (info.HugepagesTotal - info.HugepagesFree) * info.HugepagesSize
	memory.HugepagesTotal = info.HugepagesTotal * info.HugepagesSize
	memory.HugepagesSize = info.HugepagesSize

	memory.Used = info.Total - info.Free - info.Cached - info.Buffers
	memory.Total = info.Total

	// Get NUMA information
	if sysfsExists(sysDevicesNode) {
		memory.Nodes = []api.ResourcesMemoryNode{}

		// List all the nodes
		entries, err := ioutil.ReadDir(sysDevicesNode)
		if err != nil {
			return nil, errors.Wrapf(err, "Failed to list \"%s\"", sysDevicesNode)
		}

		// Iterate and add to our list
		for _, entry := range entries {
			entryName := entry.Name()
			entryPath := filepath.Join(sysDevicesNode, entryName)

			if !sysfsExists(filepath.Join(entryPath, "meminfo")) {
				continue
			}

			// Get NUMA node number
			nodeName := strings.TrimPrefix(entryName, "node")
			nodeNumber, err := strconv.ParseUint(nodeName, 10, 64)
			if err != nil {
				return nil, errors.Wrap(err, "Failed to find NUMA node")
			}

			// Parse NUMA meminfo
			info, err := parseMeminfo(filepath.Join(entryPath, "meminfo"))
			if err != nil {
				return nil, errors.Wrapf(err, "Failed to parse \"%s\"", filepath.Join(entryPath, "meminfo"))
			}

			// Setup the entry
			node := api.ResourcesMemoryNode{}
			node.NUMANode = nodeNumber

			node.HugepagesUsed = (info.HugepagesTotal - info.HugepagesFree) * memory.HugepagesSize
			node.HugepagesTotal = info.HugepagesTotal * memory.HugepagesSize

			node.Used = info.Used
			node.Total = info.Total

			memory.Nodes = append(memory.Nodes, node)
		}
	}

	return &memory, nil
}
<commit_msg>lxd/resources: Fix total memory for per NUMA node<commit_after>package resources

import (
	"bufio"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/pkg/errors"

	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/units"
)

var sysDevicesNode = "/sys/devices/system/node"
var sysDevicesSystemMemory = "/sys/devices/system/memory"

type meminfo struct {
	Cached         uint64
	Buffers        uint64
	Total          uint64
	Free           uint64
	Used           uint64
	HugepagesTotal uint64
	HugepagesFree  uint64
	HugepagesSize  uint64
}

func parseMeminfo(path string) (*meminfo, error) {
	memory := meminfo{}

	// Open meminfo
	f, err := os.Open(path)
	if err != nil {
		return nil, errors.Wrapf(err, "Failed to open \"%s\"", path)
	}
	defer f.Close()
	memInfo := bufio.NewScanner(f)

	// Get common memory information
	for memInfo.Scan() {
		line := strings.TrimSpace(memInfo.Text())

		// Get key/value
		fields := strings.SplitN(line, ":", 2)
		key := strings.TrimSpace(fields[0])
		keyFields := strings.Split(key, " ")
		key = keyFields[len(keyFields)-1]
		value := strings.TrimSpace(fields[1])
		value = strings.Replace(value, " kB", "KiB", 1)

		if key == "MemTotal" {
			bytes, err := units.ParseByteSizeString(value)
			if err != nil {
				return nil, errors.Wrap(err, "Failed to parse MemTotal")
			}

			memory.Total = uint64(bytes)
			continue
		}

		if key == "MemFree" {
			bytes, err := units.ParseByteSizeString(value)
			if err != nil {
				return nil, errors.Wrap(err, "Failed to parse MemFree")
			}

			memory.Free = uint64(bytes)
			continue
		}

		if key == "MemUsed" {
			bytes, err := units.ParseByteSizeString(value)
			if err != nil {
				return nil, errors.Wrap(err, "Failed to parse MemUsed")
			}

			memory.Used = uint64(bytes)
			continue
		}

		if key == "Cached" {
			bytes, err := units.ParseByteSizeString(value)
			if err != nil {
				return nil, errors.Wrap(err, "Failed to parse Cached")
			}

			memory.Cached = uint64(bytes)
			continue
		}

		if key == "Buffers" {
			bytes, err := units.ParseByteSizeString(value)
			if err != nil {
				return nil, errors.Wrap(err, "Failed to parse Buffers")
			}

			memory.Buffers = uint64(bytes)
			continue
		}

		if key == "HugePages_Total" {
			bytes, err := units.ParseByteSizeString(value)
			if err != nil {
				return nil, errors.Wrap(err, "Failed to parse HugePages_Total")
			}

			memory.HugepagesTotal = uint64(bytes)
			continue
		}

		if key == "HugePages_Free" {
			bytes, err := units.ParseByteSizeString(value)
			if err != nil {
				return nil, errors.Wrap(err, "Failed to parse HugePages_Free")
			}

			memory.HugepagesFree = uint64(bytes)
			continue
		}

		if key == "Hugepagesize" {
			bytes, err := units.ParseByteSizeString(value)
			if err != nil {
				return nil, errors.Wrap(err, "Failed to parse Hugepagesize")
			}

			memory.HugepagesSize = uint64(bytes)
			continue
		}
	}

	return &memory, nil
}

func getMemoryBlockSizeBytes() uint64 {
	memoryBlockSizePath := filepath.Join(sysDevicesSystemMemory, "block_size_bytes")

	if !sysfsExists(memoryBlockSizePath) {
		return 0
	}

	// Get block size
	content, err := ioutil.ReadFile(memoryBlockSizePath)
	if err != nil {
		return 0
	}

	blockSize, err := strconv.ParseUint(strings.TrimSpace(string(content)), 16, 64)
	if err != nil {
		return 0
	}

	return blockSize
}

func getTotalMemory(sysDevicesBase string) uint64 {
	blockSize := getMemoryBlockSizeBytes()
	if blockSize == 0 {
		return 0
	}

	entries, err := ioutil.ReadDir(sysDevicesBase)
	if err != nil {
		return 0
	}

	// Count the number of blocks
	var count uint64
	for _, entry := range entries {
		entryName := entry.Name()
		entryPath := filepath.Join(sysDevicesBase, entryName)

		// Ignore directories not starting with "memory"
		if !strings.HasPrefix(entryName, "memory") {
			continue
		}

		// Ignore invalid entries.
		if !sysfsExists(filepath.Join(entryPath, "online")) {
			continue
		}

		content, err := ioutil.ReadFile(filepath.Join(entryPath, "online"))
		if err != nil {
			return 0
		}

		// Only count the block if it's online
		if strings.TrimSpace(string(content)) == "1" {
			count++
		}
	}

	return blockSize * count
}

// GetMemory returns a filled api.ResourcesMemory struct ready for use by LXD
func GetMemory() (*api.ResourcesMemory, error) {
	memory := api.ResourcesMemory{}

	// Parse main meminfo
	info, err := parseMeminfo("/proc/meminfo")
	if err != nil {
		return nil, errors.Wrap(err, "Failed to parse /proc/meminfo")
	}

	// Calculate the total memory from /sys/devices/system/memory, as the previously determined
	// value reports the amount of available system memory minus the amount reserved for the kernel.
	// If successful, replace the previous value, retrieved from /proc/meminfo.
	memTotal := getTotalMemory(sysDevicesSystemMemory)
	if memTotal > 0 {
		info.Total = memTotal
	}

	// Fill used values
	memory.HugepagesUsed = (info.HugepagesTotal - info.HugepagesFree) * info.HugepagesSize
	memory.HugepagesTotal = info.HugepagesTotal * info.HugepagesSize
	memory.HugepagesSize = info.HugepagesSize

	memory.Used = info.Total - info.Free - info.Cached - info.Buffers
	memory.Total = info.Total

	// Get NUMA information
	if sysfsExists(sysDevicesNode) {
		memory.Nodes = []api.ResourcesMemoryNode{}

		// List all the nodes
		entries, err := ioutil.ReadDir(sysDevicesNode)
		if err != nil {
			return nil, errors.Wrapf(err, "Failed to list \"%s\"", sysDevicesNode)
		}

		// Iterate and add to our list
		for _, entry := range entries {
			entryName := entry.Name()
			entryPath := filepath.Join(sysDevicesNode, entryName)

			if !sysfsExists(filepath.Join(entryPath, "meminfo")) {
				continue
			}

			// Get NUMA node number
			nodeName := strings.TrimPrefix(entryName, "node")
			nodeNumber, err := strconv.ParseUint(nodeName, 10, 64)
			if err != nil {
				return nil, errors.Wrap(err, "Failed to find NUMA node")
			}

			// Parse NUMA meminfo
			info, err := parseMeminfo(filepath.Join(entryPath, "meminfo"))
			if err != nil {
				return nil, errors.Wrapf(err, "Failed to parse \"%s\"", filepath.Join(entryPath, "meminfo"))
			}

			// Setup the entry
			node := api.ResourcesMemoryNode{}
			node.NUMANode = nodeNumber

			node.HugepagesUsed = (info.HugepagesTotal - info.HugepagesFree) * memory.HugepagesSize
			node.HugepagesTotal = info.HugepagesTotal * memory.HugepagesSize

			node.Used = info.Used
			node.Total = info.Total

			// Calculate the total memory from /sys/devices/system/node/memory, as the previously determined
			// value reports the amount of available system memory minus the amount reserved for the kernel.
			// If successful, replace the previous value, retrieved from /sys/devices/system/node/meminfo.
			memTotal := getTotalMemory(entryPath)
			if memTotal > 0 {
				node.Total = memTotal
			}

			memory.Nodes = append(memory.Nodes, node)
		}
	}

	return &memory, nil
}
<|endoftext|>


<commit_before>package lxdatabase_test

import (
	"github.com/layer-x/layerx-commons/lxdatabase"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/layer-x/layerx-mesos-tpi/test_helpers"
)

var _ = Describe("Lxdb", func() {
	test_helpers.StartETCD()

	BeforeEach(func() {
		Describe("initializes without error", func() {
			err := lxdatabase.Init([]string{"http://127.0.0.1:2379"})
			Expect(err).To(BeNil())
		})
	})
	Describe("lxdatabase.Set(key, val)", func() {
		It("sets keys", func() {
			err := lxdatabase.Set("foo", "bar")
			Expect(err).To(BeNil())
		})
	})
	Describe("lxdatabase.Get(key)", func() {
		It("gets values", func() {
			val, err := lxdatabase.Get("foo")
			Expect(err).To(BeNil())
			Expect(val).To(Equal("bar"))
		})
	})
	Describe("lxdatabase.Rm(key)", func() {
		It("deletes keys", func() {
			err := lxdatabase.Rm("foo")
			Expect(err).To(BeNil())
			val, err := lxdatabase.Get("foo")
			Expect(val).To(Equal(""))
			Expect(err).To(Not(BeNil()))
			Expect(err.Error()).To(ContainSubstring("Key not found"))
		})
	})
	Describe("lxdatabase.Mkdir(key)", func() {
		It("makes directories", func() {
			err := lxdatabase.Rmdir("/foo_dir", true)
			Expect(err).To(BeNil())
			err = lxdatabase.Mkdir("foo_dir")
			Expect(err).To(BeNil())
		})
	})
	Describe("lxdatabase.GetKeys(key)", func() {
		It("lists keys in directory", func() {
			keys, err := lxdatabase.GetKeys("foo_dir")
			Expect(err).To(BeNil())
			Expect(keys).To(BeEmpty())
			err = lxdatabase.Set("foo_dir/foo", "bar")
			Expect(err).To(BeNil())
			keys, err = lxdatabase.GetKeys("foo_dir")
			Expect(err).To(BeNil())
			Expect(keys).To(ContainElement("bar"))
		})
	})
	Describe("lxdatabase.GetSubdirectories(dir)", func() {
		It("lists sbudirs in directory", func() {
			dirs, err := lxdatabase.GetSubdirectories("foo_dir")
			Expect(err).To(BeNil())
			Expect(dirs).To(BeEmpty())
			err = lxdatabase.Mkdir("foo_dir/bar_dir")
			Expect(err).To(BeNil())
			dirs, err = lxdatabase.GetSubdirectories("foo_dir")
			Expect(err).To(BeNil())
			Expect(dirs).To(ContainElement("/foo_dir/bar_dir"))
		})
	})
	Describe("cleanup", func() {
		It("cleans up etcd", func() {
			test_helpers.CleanupETCD()
		})
	})
})
<commit_msg>gofmtw<commit_after>package lxdatabase_test

import (
	"github.com/layer-x/layerx-commons/lxdatabase"

	"github.com/layer-x/layerx-mesos-tpi/test_helpers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Lxdb", func() {
	test_helpers.StartETCD()

	BeforeEach(func() {
		Describe("initializes without error", func() {
			err := lxdatabase.Init([]string{"http://127.0.0.1:2379"})
			Expect(err).To(BeNil())
		})
	})
	Describe("lxdatabase.Set(key, val)", func() {
		It("sets keys", func() {
			err := lxdatabase.Set("foo", "bar")
			Expect(err).To(BeNil())
		})
	})
	Describe("lxdatabase.Get(key)", func() {
		It("gets values", func() {
			val, err := lxdatabase.Get("foo")
			Expect(err).To(BeNil())
			Expect(val).To(Equal("bar"))
		})
	})
	Describe("lxdatabase.Rm(key)", func() {
		It("deletes keys", func() {
			err := lxdatabase.Rm("foo")
			Expect(err).To(BeNil())
			val, err := lxdatabase.Get("foo")
			Expect(val).To(Equal(""))
			Expect(err).To(Not(BeNil()))
			Expect(err.Error()).To(ContainSubstring("Key not found"))
		})
	})
	Describe("lxdatabase.Mkdir(key)", func() {
		It("makes directories", func() {
			err := lxdatabase.Rmdir("/foo_dir", true)
			Expect(err).To(BeNil())
			err = lxdatabase.Mkdir("foo_dir")
			Expect(err).To(BeNil())
		})
	})
	Describe("lxdatabase.GetKeys(key)", func() {
		It("lists keys in directory", func() {
			keys, err := lxdatabase.GetKeys("foo_dir")
			Expect(err).To(BeNil())
			Expect(keys).To(BeEmpty())
			err = lxdatabase.Set("foo_dir/foo", "bar")
			Expect(err).To(BeNil())
			keys, err = lxdatabase.GetKeys("foo_dir")
			Expect(err).To(BeNil())
			Expect(keys).To(ContainElement("bar"))
		})
	})
	Describe("lxdatabase.GetSubdirectories(dir)", func() {
		It("lists sbudirs in directory", func() {
			dirs, err := lxdatabase.GetSubdirectories("foo_dir")
			Expect(err).To(BeNil())
			Expect(dirs).To(BeEmpty())
			err = lxdatabase.Mkdir("foo_dir/bar_dir")
			Expect(err).To(BeNil())
			dirs, err = lxdatabase.GetSubdirectories("foo_dir")
			Expect(err).To(BeNil())
			Expect(dirs).To(ContainElement("/foo_dir/bar_dir"))
		})
	})
	Describe("cleanup", func() {
		It("cleans up etcd", func() {
			test_helpers.CleanupETCD()
		})
	})
})
<|endoftext|>


<commit_before>package command

import (
	"fmt"
	"os"
	"reflect"
	"text/tabwriter"

	"github.com/briandowns/wasteband/config"
	"github.com/briandowns/wasteband/utils"
	"github.com/fatih/flags"
	"github.com/mitchellh/cli"
)

// Show holds in the passed in configuration
type Show struct {
	config *config.Configuration
}

// NewShow creates a new instance of Delete
func NewShow(conf *config.Configuration) cli.CommandFactory {
	return func() (cli.Command, error) {
		return &Show{
			config: conf,
		}, nil
	}
}

// Run shows a given resource.
func (s *Show) Run(args []string) int {
	if flags.Has("help", args) || len(args) < 1 {
		fmt.Print(s.Help())
		return 1
	}

	// make sure that the command is a valid one
	if !utils.InSlice(args[0], allowedResources["show"]) {
		fmt.Print("ERROR: invalid option for show\n\n")
		fmt.Print(s.Help())
		return 1
	}

	// process the subcommand and it's options
	switch args[0] {
	case "index":
		_, err := utils.ESConn(s.config.Endpoint).Health()
		if err != nil {
			fmt.Printf("ERROR: %v\n", err.Error())
			return 1
		}
	case "cluster-health":
		result, err := utils.ESConn(s.config.Endpoint).Health()
		if err != nil {
			fmt.Printf("ERROR: %v\n", err.Error())
			return 1
		}

		w := new(tabwriter.Writer)
		w.Init(os.Stdout, 0, 8, 0, '\t', 0)

		v := reflect.ValueOf(result)

		fmt.Fprint(w, "\n")

		// iterate through the values of the struct and write to the tabwriter
		for i := 0; i < v.NumField(); i++ {
			// colorize the value of Status field based on it's content
			switch v.Field(i).Interface() {
			case "green":
				fmt.Fprintf(w, "%s\t%v\n", v.Type().Field(i).Name, green(v.Field(i).Interface().(string)))
				continue
			case "yellow":
				fmt.Fprintf(w, "%s\t%v\n", v.Type().Field(i).Name, yellow(v.Field(i).Interface().(string)))
				continue
			case "red":
				fmt.Fprintf(w, "%s\t%v\n", v.Type().Field(i).Name, red(v.Field(i).Interface().(string)))
				continue
			}
			fmt.Fprintf(w, "%s\t%v\n", v.Type().Field(i).Name, v.Field(i).Interface())
		}

		fmt.Fprintf(w, "\n")
		w.Flush() // dump out of writing to the tabwriter / os.Stdout
	}
	return 1
}

// Help provides full help inforamation for the subcommand
func (s *Show) Help() string { return "" }

// Synopsis provides a brief description of the command
func (s *Show) Synopsis() string {
	return "Show an Elasticsearch resource"
}
<commit_msg>show work<commit_after>package command

import (
	"fmt"
	"os"
	"reflect"
	"text/tabwriter"

	"github.com/briandowns/wasteband/config"
	"github.com/briandowns/wasteband/utils"
	"github.com/fatih/flags"
	"github.com/mitchellh/cli"
)

// Show holds in the passed in configuration
type Show struct {
	config *config.Configuration
}

// NewShow creates a new instance of Delete
func NewShow(conf *config.Configuration) cli.CommandFactory {
	return func() (cli.Command, error) {
		return &Show{
			config: conf,
		}, nil
	}
}

// Run shows a given resource.
func (s *Show) Run(args []string) int {
	if flags.Has("help", args) || len(args) < 1 {
		fmt.Print(s.Help())
		return 1
	}

	// make sure that the command is a valid one
	if !utils.InSlice(args[0], allowedResources["show"]) {
		fmt.Print("ERROR: invalid option for show\n\n")
		fmt.Print(s.Help())
		return 1
	}

	// process the subcommand and it's options
	switch args[0] {
	case "indexes":
		// most of this functionaity will have to wait on PR 209 in the elastigo
		// project to be merged.
		result := utils.ESConn(s.config.Endpoint).GetCatIndexInfo("")

		for _, i := range result {
			fmt.Println(i.Name)
		}

		_, err := utils.ESConn(s.config.Endpoint).Health()
		if err != nil {
			fmt.Printf("ERROR: %v\n", err.Error())
			return 1
		}
	case "cluster-health":
		result, err := utils.ESConn(s.config.Endpoint).Health()
		if err != nil {
			fmt.Printf("ERROR: %v\n", err.Error())
			return 1
		}

		w := new(tabwriter.Writer)
		w.Init(os.Stdout, 0, 8, 0, '\t', 0)

		v := reflect.ValueOf(result)

		fmt.Fprint(w, "\n")

		// iterate through the values of the struct and write to the tabwriter
		for i := 0; i < v.NumField(); i++ {
			// colorize the value of Status field based on it's content
			switch v.Field(i).Interface() {
			case "green":
				fmt.Fprintf(w, "%s\t%v\n", v.Type().Field(i).Name, green(v.Field(i).Interface().(string)))
				continue
			case "yellow":
				fmt.Fprintf(w, "%s\t%v\n", v.Type().Field(i).Name, yellow(v.Field(i).Interface().(string)))
				continue
			case "red":
				fmt.Fprintf(w, "%s\t%v\n", v.Type().Field(i).Name, red(v.Field(i).Interface().(string)))
				continue
			}
			fmt.Fprintf(w, "%s\t%v\n", v.Type().Field(i).Name, v.Field(i).Interface())
		}

		fmt.Fprintf(w, "\n")
		w.Flush() // dump out of writing to the tabwriter / os.Stdout
	case "cluster-stats":
		//
	}
	return 1
}

// Help provides full help inforamation for the subcommand
func (s *Show) Help() string { return "" }

// Synopsis provides a brief description of the command
func (s *Show) Synopsis() string {
	return "Show an Elasticsearch resource"
}
<|endoftext|>


<commit_before>package command

import (
	"fmt"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/Wheeeel/todobot/task"
	tg "github.com/go-telegram-bot-api/telegram-bot-api"
	"github.com/pkg/errors"
)

func TODO(bot *tg.BotAPI, req *tg.Message) {
	msg := tg.NewMessage(req.Chat.ID, "")
	args := strings.Split(req.CommandArguments(), ",")
	if args[0] == "" {
		msg.Text = "usage: `/todo taskObj1,taskObj2,taskObj3`\ntaskObj: `<description>##<enrollCnt>`\ne.g: `/todo 吃包##2`"
		msg.ParseMode = tg.ModeMarkdown
		bot.Send(msg)
		return
	}
	textTpl := `
	*%d TODO Items Added*
	`
	cnt := 0
	for _, arg := range args {
		arg = strings.TrimLeft(arg, " ")
		tmp := strings.Split(arg, "##")
		var enrollCnt int
		taskStr := tmp[0]
		if len(tmp) == 2 {
			fmt.Sscanf(tmp[1], "%d", &enrollCnt)
		}
		if len(tmp) == 1 {
			enrollCnt = 1
		}
		tid, err := task.AddTask(task.DB, taskStr, enrollCnt, req.Chat.ID)
		if err != nil {
			err = errors.Wrap(err, "cmd todo error")
			log.Error(err)
			textTpl = textTpl + "[ERROR] Server error, not all items added\n"
			break
		}
		cnt++
		textTpl = textTpl + fmt.Sprintf("*TODO [%d]* _", tid) + taskStr + "_\n"
	}
	textTpl = fmt.Sprintf(textTpl, cnt)
	msg.ParseMode = tg.ModeMarkdown
	msg.Text = textTpl
	bot.Send(msg)
	return
}
<commit_msg>Add more hint to todo<commit_after>package command

import (
	"fmt"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/Wheeeel/todobot/task"
	tg "github.com/go-telegram-bot-api/telegram-bot-api"
	"github.com/pkg/errors"
)

func TODO(bot *tg.BotAPI, req *tg.Message) {
	msg := tg.NewMessage(req.Chat.ID, "")
	args := strings.Split(req.CommandArguments(), ",")
	if args[0] == "" {
		msg.Text = "usage: `/todo taskObj1,taskObj2,taskObj3`\ntaskObj: `<description>##<enrollCnt>`\ne.g: `/todo 吃包##2`"
		msg.ParseMode = tg.ModeMarkdown
		bot.Send(msg)
		return
	}
	textTpl := `
	*%d TODO Items Added*
	`
	cnt := 0
	for _, arg := range args {
		arg = strings.TrimLeft(arg, " ")
		tmp := strings.Split(arg, "##")
		var enrollCnt int
		taskStr := tmp[0]
		if len(tmp) == 2 {
			fmt.Sscanf(tmp[1], "%d", &enrollCnt)
		}
		if len(tmp) == 1 {
			enrollCnt = 1
		}
		tid, err := task.AddTask(task.DB, taskStr, enrollCnt, req.Chat.ID)
		if err != nil {
			err = errors.Wrap(err, "cmd todo error")
			log.Error(err)
			textTpl = textTpl + "[ERROR] Server error, not all items added\n"
			break
		}
		cnt++
		textTpl = textTpl + fmt.Sprintf("*TODO [%d]* _", tid) + taskStr + "_\n"
	}
	textTpl = fmt.Sprintf(textTpl, cnt)
	textTpl += "Use /workon to focus on one of your job!"
	msg.ParseMode = tg.ModeMarkdown
	msg.Text = textTpl
	bot.Send(msg)
	return
}
<|endoftext|>