<commit_before>package data

// this package uses panic, as data validation is very important here.
// if something goes wrong, panic is safer than getting corrupted data.

import "unicode/utf8"

func runeToBytes(r rune) []byte {
	bs := make([]byte, utf8.RuneLen(r))
	utf8.EncodeRune(bs, r)
	return bs
}

type Clip struct {
	data     []byte
	newlines []int
}

func DataClip(data []byte) Clip {
	newlines := []int{}
	for i, b := range data {
		if b == '\n' {
			newlines = append(newlines, i)
		}
	}
	return Clip{
		data:     data,
		newlines: newlines,
	}
}

func Clips(datas ...[]byte) []Clip {
	clips := make([]Clip, 0)
	for _, data := range datas {
		c := DataClip(data)
		clips = append(clips, c)
	}
	return clips
}

func (c Clip) Len() int {
	return len(c.data)
}

func cut(c Clip, o int) (a, b Clip) {
	aNewlines := make([]int, 0)
	bNewlines := make([]int, 0)
	for _, n := range c.newlines {
		if o < n {
			aNewlines = append(aNewlines, n)
		} else {
			bNewlines = append(bNewlines, n-o)
		}
	}
	a = Clip{data: c.data[:o], newlines: aNewlines}
	b = Clip{data: c.data[o:], newlines: bNewlines}
	return a, b
}

func (c Clip) Cut(o int) (a, b Clip) {
	aNewlines := make([]int, 0)
	bNewlines := make([]int, 0)
	for _, n := range c.newlines {
		if o < n {
			aNewlines = append(aNewlines, n)
		} else {
			bNewlines = append(bNewlines, n-o)
		}
	}
	a = Clip{data: c.data[:o], newlines: aNewlines}
	b = Clip{data: c.data[o:], newlines: bNewlines}
	return a, b
}

func (c Clip) PopFirst() Clip {
	if c.Len() <= 0 {
		panic("cannot pop")
	}
	r, n := utf8.DecodeRune(c.data)
	c.data = c.data[:len(c.data)-n]
	if r == '\n' {
		c.newlines = c.newlines[:len(c.newlines)-1]
	}
	return c
}

func (c Clip) PopLast() Clip {
	if c.Len() <= 0 {
		panic("cannot pop")
	}
	r, n := utf8.DecodeLastRune(c.data)
	c.data = c.data[n:]
	if r == '\n' {
		c.newlines = c.newlines[1:]
	}
	return c
}

func (c Clip) Append(r rune) Clip {
	if r == '\n' {
		c.newlines = append(c.newlines, len(c.data))
	}
	c.data = append(c.data, runeToBytes(r)...)
	return c
}

type Cursor struct {
	clips []Clip

	i int // clip index
	o int // byte offset on the clip

	appending bool
}

func NewCursor(clips []Clip) *Cursor {
	return &Cursor{clips: clips}
}

func nextOffset(data []byte, o int) int {
	remain := data[o:]
	r, n := utf8.DecodeRune(remain)
	remain = remain[n:]
	if r == '\r' {
		r, _ := utf8.DecodeRune(remain)
		if r == '\n' {
			n += 1
		}
	}
	o += n
	if o == len(data) {
		return -1
	}
	return o
}

func prevOffset(data []byte, o int) int {
	if o == 0 {
		return -1
	}
	remain := data[:o]
	r, n := utf8.DecodeLastRune(remain)
	remain = remain[:len(remain)-n]
	if r == '\n' {
		r, _ := utf8.DecodeLastRune(remain)
		if r == '\r' {
			n += 1
		}
	}
	return o - n
}

func (c *Cursor) MoveNext() {
	c.appending = false
	if c.i == len(c.clips) {
		if c.o != 0 {
			panic("c.o should 0 when c.i == len(c.clips)")
		}
		return
	}
	o := nextOffset(c.clips[c.i].data, c.o)
	if o == -1 {
		c.i++
		c.o = 0
		return
	}
	c.o = o
}

func (c *Cursor) MovePrev() {
	c.appending = false
	if c.i == 0 && c.o == 0 {
		return
	}
	if c.i == len(c.clips) {
		if c.o != 0 {
			panic("c.o should 0 when c.i == len(c.clips)")
		}
		c.i--
		c.o = len(c.clips[c.i].data)
	}
	if c.o == 0 {
		c.i--
		c.o = prevOffset(c.clips[c.i].data, len(c.clips[c.i].data))
		return
	}
	c.o = prevOffset(c.clips[c.i].data, c.o)
}

func (c *Cursor) Move(o int) {
	if o == 0 {
		return
	}
	if o > 0 {
		for i := 0; i < o; i++ {
			c.MoveNext()
		}
	} else {
		for i := 0; i > o; i-- {
			c.MovePrev()
		}
	}
}

func (c *Cursor) GotoStart() {
	c.appending = false
	c.i = 0
	c.o = 0
}

func (c *Cursor) GotoEnd() {
	c.appending = false
	c.i = len(c.clips)
	c.o = 0
}

func (c *Cursor) GotoNextLine() {
	c.appending = false
	if len(c.clips) == 0 {
		panic("length of clips should not be zero")
	}
	for {
		if c.i == len(c.clips) {
			if c.o != 0 {
				panic("c.o should 0 when c.i == len(c.clips)")
			}
			return
		}
		nls := c.clips[c.i].newlines
		for i := range nls {
			o := nls[i]
			if o <= c.o {
				continue
			}
			c.o = o
			return
		}
		c.i++
		c.o = 0
	}
}

func (c *Cursor) GotoPrevLine() {
	c.appending = false
	if len(c.clips) == 0 {
		panic("length of clips should not be zero")
	}
	if c.i == len(c.clips) {
		c.i--
		c.o = len(c.clips[c.i].data)
	}
	for {
		nls := c.clips[c.i].newlines
		for i := range nls {
			o := nls[len(nls)-1-i]
			if o >= c.o {
				continue
			}
			c.o = o
			return
		}
		if c.i == 0 {
			// no more previous clip
			c.o = 0
			return
		}
		c.i--
		c.o = len(c.clips[c.i].data)
	}
}

// Cut cuts the underlying clip it stands.
// If it is standing at edge of clip(s), it doesn't do anything.
func (c *Cursor) Cut() {
	if c.i == len(c.clips) {
		if c.o != 0 {
			panic("c.o should 0 when c.i == len(c.clips)")
		}
		return
	}
	if c.o == 0 {
		// edge
		return
	}
	clipA, clipB := c.clips[c.i].Cut(c.o)
	c.clips = append(c.clips[:c.i], append([]Clip{clipA, clipB}, c.clips[c.i+1:]...)...)
	c.i++
	c.o = 0
}

func (c *Cursor) Write(r rune) {
	if c.appending {
		if c.o != 0 {
			panic("c.o should 0 when appending")
		}
		c.clips[c.i-1] = c.clips[c.i-1].Append(r)
		return
	}
	c.appending = true
	c.Cut()
	clipInsert := DataClip(runeToBytes(r))
	c.clips = append(c.clips[:c.i], append([]Clip{clipInsert}, c.clips[c.i:]...)...)
	c.i++
	c.o = 0
}

func (c *Cursor) Delete() {
	c.appending = false
	if c.i == len(c.clips) {
		if c.o != 0 {
			panic("c.o should 0 when c.i == len(c.clips)")
		}
		return
	}
	c.Cut()
	p := nextOffset(c.clips[c.i].data, 0)
	if p == -1 {
		c.clips = append(c.clips[:c.i], c.clips[c.i+1:]...)
		return
	}
	_, c.clips[c.i] = c.clips[c.i].Cut(p)
}

func (c *Cursor) Backspace() {
	c.appending = false
	if c.i == 0 && c.o == 0 {
		return
	}
	c.Cut()
	p := prevOffset(c.clips[c.i-1].data, len(c.clips[c.i-1].data))
	if p == 0 {
		c.clips = append(c.clips[:c.i-1], c.clips[c.i:]...)
		c.i--
		return
	}
	c.clips[c.i-1], _ = c.clips[c.i-1].Cut(p)
}
<commit_msg>data: remove Clip.Pop{First,Last}<commit_after>package data

// this package uses panic, as data validation is very important here.
// if something goes wrong, panic is safer than getting corrupted data.

import "unicode/utf8"

func runeToBytes(r rune) []byte {
	bs := make([]byte, utf8.RuneLen(r))
	utf8.EncodeRune(bs, r)
	return bs
}

type Clip struct {
	data     []byte
	newlines []int
}

func DataClip(data []byte) Clip {
	newlines := []int{}
	for i, b := range data {
		if b == '\n' {
			newlines = append(newlines, i)
		}
	}
	return Clip{
		data:     data,
		newlines: newlines,
	}
}

func Clips(datas ...[]byte) []Clip {
	clips := make([]Clip, 0)
	for _, data := range datas {
		c := DataClip(data)
		clips = append(clips, c)
	}
	return clips
}

func (c Clip) Len() int {
	return len(c.data)
}

func cut(c Clip, o int) (a, b Clip) {
	aNewlines := make([]int, 0)
	bNewlines := make([]int, 0)
	for _, n := range c.newlines {
		if o < n {
			aNewlines = append(aNewlines, n)
		} else {
			bNewlines = append(bNewlines, n-o)
		}
	}
	a = Clip{data: c.data[:o], newlines: aNewlines}
	b = Clip{data: c.data[o:], newlines: bNewlines}
	return a, b
}

func (c Clip) Cut(o int) (a, b Clip) {
	aNewlines := make([]int, 0)
	bNewlines := make([]int, 0)
	for _, n := range c.newlines {
		if o < n {
			aNewlines = append(aNewlines, n)
		} else {
			bNewlines = append(bNewlines, n-o)
		}
	}
	a = Clip{data: c.data[:o], newlines: aNewlines}
	b = Clip{data: c.data[o:], newlines: bNewlines}
	return a, b
}

func (c Clip) Append(r rune) Clip {
	if r == '\n' {
		c.newlines = append(c.newlines, len(c.data))
	}
	c.data = append(c.data, runeToBytes(r)...)
	return c
}

type Cursor struct {
	clips []Clip

	i int // clip index
	o int // byte offset on the clip

	appending bool
}

func NewCursor(clips []Clip) *Cursor {
	return &Cursor{clips: clips}
}

func nextOffset(data []byte, o int) int {
	remain := data[o:]
	r, n := utf8.DecodeRune(remain)
	remain = remain[n:]
	if r == '\r' {
		r, _ := utf8.DecodeRune(remain)
		if r == '\n' {
			n += 1
		}
	}
	o += n
	if o == len(data) {
		return -1
	}
	return o
}

func prevOffset(data []byte, o int) int {
	if o == 0 {
		return -1
	}
	remain := data[:o]
	r, n := utf8.DecodeLastRune(remain)
	remain = remain[:len(remain)-n]
	if r == '\n' {
		r, _ := utf8.DecodeLastRune(remain)
		if r == '\r' {
			n += 1
		}
	}
	return o - n
}

func (c *Cursor) MoveNext() {
	c.appending = false
	if c.i == len(c.clips) {
		if c.o != 0 {
			panic("c.o should 0 when c.i == len(c.clips)")
		}
		return
	}
	o := nextOffset(c.clips[c.i].data, c.o)
	if o == -1 {
		c.i++
		c.o = 0
		return
	}
	c.o = o
}

func (c *Cursor) MovePrev() {
	c.appending = false
	if c.i == 0 && c.o == 0 {
		return
	}
	if c.i == len(c.clips) {
		if c.o != 0 {
			panic("c.o should 0 when c.i == len(c.clips)")
		}
		c.i--
		c.o = len(c.clips[c.i].data)
	}
	if c.o == 0 {
		c.i--
		c.o = prevOffset(c.clips[c.i].data, len(c.clips[c.i].data))
		return
	}
	c.o = prevOffset(c.clips[c.i].data, c.o)
}

func (c *Cursor) Move(o int) {
	if o == 0 {
		return
	}
	if o > 0 {
		for i := 0; i < o; i++ {
			c.MoveNext()
		}
	} else {
		for i := 0; i > o; i-- {
			c.MovePrev()
		}
	}
}

func (c *Cursor) GotoStart() {
	c.appending = false
	c.i = 0
	c.o = 0
}

func (c *Cursor) GotoEnd() {
	c.appending = false
	c.i = len(c.clips)
	c.o = 0
}

func (c *Cursor) GotoNextLine() {
	c.appending = false
	if len(c.clips) == 0 {
		panic("length of clips should not be zero")
	}
	for {
		if c.i == len(c.clips) {
			if c.o != 0 {
				panic("c.o should 0 when c.i == len(c.clips)")
			}
			return
		}
		nls := c.clips[c.i].newlines
		for i := range nls {
			o := nls[i]
			if o <= c.o {
				continue
			}
			c.o = o
			return
		}
		c.i++
		c.o = 0
	}
}

func (c *Cursor) GotoPrevLine() {
	c.appending = false
	if len(c.clips) == 0 {
		panic("length of clips should not be zero")
	}
	if c.i == len(c.clips) {
		c.i--
		c.o = len(c.clips[c.i].data)
	}
	for {
		nls := c.clips[c.i].newlines
		for i := range nls {
			o := nls[len(nls)-1-i]
			if o >= c.o {
				continue
			}
			c.o = o
			return
		}
		if c.i == 0 {
			// no more previous clip
			c.o = 0
			return
		}
		c.i--
		c.o = len(c.clips[c.i].data)
	}
}

// Cut cuts the underlying clip it stands.
// If it is standing at edge of clip(s), it doesn't do anything.
func (c *Cursor) Cut() {
	if c.i == len(c.clips) {
		if c.o != 0 {
			panic("c.o should 0 when c.i == len(c.clips)")
		}
		return
	}
	if c.o == 0 {
		// edge
		return
	}
	clipA, clipB := c.clips[c.i].Cut(c.o)
	c.clips = append(c.clips[:c.i], append([]Clip{clipA, clipB}, c.clips[c.i+1:]...)...)
	c.i++
	c.o = 0
}

func (c *Cursor) Write(r rune) {
	if c.appending {
		if c.o != 0 {
			panic("c.o should 0 when appending")
		}
		c.clips[c.i-1] = c.clips[c.i-1].Append(r)
		return
	}
	c.appending = true
	c.Cut()
	clipInsert := DataClip(runeToBytes(r))
	c.clips = append(c.clips[:c.i], append([]Clip{clipInsert}, c.clips[c.i:]...)...)
	c.i++
	c.o = 0
}

func (c *Cursor) Delete() {
	c.appending = false
	if c.i == len(c.clips) {
		if c.o != 0 {
			panic("c.o should 0 when c.i == len(c.clips)")
		}
		return
	}
	c.Cut()
	p := nextOffset(c.clips[c.i].data, 0)
	if p == -1 {
		c.clips = append(c.clips[:c.i], c.clips[c.i+1:]...)
		return
	}
	_, c.clips[c.i] = c.clips[c.i].Cut(p)
}

func (c *Cursor) Backspace() {
	c.appending = false
	if c.i == 0 && c.o == 0 {
		return
	}
	c.Cut()
	p := prevOffset(c.clips[c.i-1].data, len(c.clips[c.i-1].data))
	if p == 0 {
		c.clips = append(c.clips[:c.i-1], c.clips[c.i:]...)
		c.i--
		return
	}
	c.clips[c.i-1], _ = c.clips[c.i-1].Cut(p)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
)

type entry struct {
	value int
	name  string
}

var v = []entry{
	{0, "zero"},
	{1, "one"},
	{2, "two"},
	{3, "three"},
	{4, "four"},
	{5, "five"},
	{6, "six"},
}

func increase() {
	for i := 0; i <= len(v); i++ {
		v[i].value++
	}
}

func main() {
	fmt.Println(os.Args)
	increase()

}
<commit_msg>Revert main.go<commit_after>package main

import (
	"io"
	"net/http"

	"github.com/frozzare/go-config"

	_ "github.com/frozzare/go-config/plugins/yaml"
)

// Example file

func main() {
	// Use file + env as middlewares.
	config.Use(config.NewFromFile("config.yml"))
	config.Use(config.NewEnv())

	// Watch config (replaces first middleware since it's the same file path).
	config.WatchFile("config.yml")

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		v, _ := config.String("name")
		io.WriteString(w, v)
	})

	http.ListenAndServe(":8899", nil)
}
0
<|endoftext|>


<commit_before>package main

import (
	"testing"
	"time"
)

func TestClientSmokeTest(t *testing.T) {
	server, err := newLumberjackServer(&serverOptions{
		Network: "tcp",
		Address: "127.0.0.1:0", // random port

		WriteTimeout: 2 * time.Second,
		ReadTimeout:  2 * time.Second,
	})

	if err != nil {
		t.Fatal(err)
	}
	defer server.Close()

	dataCh := make(chan Data, 1)
	go server.ServeInto(dataCh)

	client := NewLumberjackClient(&LumberjackClientOptions{
		Network:           "tcp",
		Address:           server.Addr().String(),
		ConnectionTimeout: 2 * time.Second,
		WriteTimeout:      2 * time.Second,
		ReadTimeout:       2 * time.Second,
	})

	lines := []Data{
		Data{"line": "foo bar baz", "offset": "25"},
	}
	err = client.Send(lines)
	if err != nil {
		t.Error(err)
	}

	select {
	case receivedLine := <-dataCh:
		if receivedLine["line"] != lines[0]["line"] {
			t.Fatalf("Got line of %s, expected %s", receivedLine["line"], lines[0]["line"])
		}
	case <-time.After(250 * time.Millisecond):
		t.Fatal("Timeout waiting for lines to arrive")
	}
}

func TestClientReconnectSmokeTest(t *testing.T) {
	server, err := newLumberjackServer(&serverOptions{
		Network: "tcp",
		Address: "127.0.0.1:0", // random port

		WriteTimeout: 2 * time.Second,
		ReadTimeout:  2 * time.Second,
	})

	if err != nil {
		t.Fatal(err)
	}
	defer server.Close()

	// Without the server accepting connections, we should run into a connection
	// timeout
	client := NewLumberjackClient(&LumberjackClientOptions{
		Network:           "tcp",
		Address:           server.Addr().String(),
		ConnectionTimeout: 1 * time.Second,
		WriteTimeout:      1 * time.Second,
		ReadTimeout:       1 * time.Second,
	})

	lines := []Data{
		Data{"line": "foo bar baz", "offset": "25"},
	}
	err = client.Send(lines)
	if err == nil {
		t.Fatalf("Expected Send to timeout, but did not")
	}

	// Now, setup the server properly, things should go through
	dataCh := make(chan Data, 1)
	go server.ServeInto(dataCh)

	err = client.Send(lines)
	if err != nil {
		t.Error(err)
	}

	select {
	case <-dataCh:
		// success
	case <-time.After(250 * time.Millisecond):
		t.Fatal("Timeout waiting for lines to arrive")
	}
}
<commit_msg>Uses the correct field name in tests<commit_after>package main

import (
	"testing"
	"time"
)

func TestClientSmokeTest(t *testing.T) {
	server, err := newLumberjackServer(&serverOptions{
		Network: "tcp",
		Address: "127.0.0.1:0", // random port

		WriteTimeout: 2 * time.Second,
		ReadTimeout:  2 * time.Second,
	})

	if err != nil {
		t.Fatal(err)
	}
	defer server.Close()

	dataCh := make(chan Data, 1)
	go server.ServeInto(dataCh)

	client := NewLumberjackClient(&LumberjackClientOptions{
		Network:           "tcp",
		Address:           server.Addr().String(),
		ConnectionTimeout: 2 * time.Second,
		SendTimeout:       2 * time.Second,
	})

	lines := []Data{
		Data{"line": "foo bar baz", "offset": "25"},
	}
	err = client.Send(lines)
	if err != nil {
		t.Error(err)
	}

	select {
	case receivedLine := <-dataCh:
		if receivedLine["line"] != lines[0]["line"] {
			t.Fatalf("Got line of %s, expected %s", receivedLine["line"], lines[0]["line"])
		}
	case <-time.After(250 * time.Millisecond):
		t.Fatal("Timeout waiting for lines to arrive")
	}
}

func TestClientReconnectSmokeTest(t *testing.T) {
	server, err := newLumberjackServer(&serverOptions{
		Network: "tcp",
		Address: "127.0.0.1:0", // random port

		WriteTimeout: 2 * time.Second,
		ReadTimeout:  2 * time.Second,
	})

	if err != nil {
		t.Fatal(err)
	}
	defer server.Close()

	// Without the server accepting connections, we should run into a connection
	// timeout
	client := NewLumberjackClient(&LumberjackClientOptions{
		Network:           "tcp",
		Address:           server.Addr().String(),
		ConnectionTimeout: 1 * time.Second,
		SendTimeout:       1 * time.Second,
	})

	lines := []Data{
		Data{"line": "foo bar baz", "offset": "25"},
	}
	err = client.Send(lines)
	if err == nil {
		t.Fatalf("Expected Send to timeout, but did not")
	}

	// Now, setup the server properly, things should go through
	dataCh := make(chan Data, 1)
	go server.ServeInto(dataCh)

	err = client.Send(lines)
	if err != nil {
		t.Error(err)
	}

	select {
	case <-dataCh:
		// success
	case <-time.After(250 * time.Millisecond):
		t.Fatal("Timeout waiting for lines to arrive")
	}
}
<|endoftext|>


<commit_before>package db

import (
	"fmt"
	"strings"

	"github.com/lxc/lxd/lxd/db/generate/lex"
)

// Return the table name for the given database entity.
func entityTable(entity string, override string) string {
	if override != "" {
		return override
	}

	entityParts := strings.Split(lex.Snake(entity), "_")
	tableParts := make([]string, len(entityParts))
	for i, part := range entityParts {
		if strings.HasSuffix(part, "ty") || strings.HasSuffix(part, "ly") {
			tableParts[i] = part
		} else {
			tableParts[i] = lex.Plural(part)
		}
	}

	return strings.Join(tableParts, "_")
}

// Return the name of the Filter struct for the given database entity.
func entityFilter(entity string) string {
	return fmt.Sprintf("%sFilter", lex.Camel(entity))
}

// Return the name of the global variable holding the registration code for
// the given kind of statement aganst the given entity.
func stmtCodeVar(entity string, kind string, filters ...string) string {
	prefix := lex.Minuscule(lex.Camel(entity))
	name := fmt.Sprintf("%s%s", prefix, lex.Camel(kind))

	if len(filters) > 0 {
		name += "By"
		name += strings.Join(filters, "And")
	}

	return name
}

// operation returns the kind of operation being performed, without filter fields.
func operation(kind string) string {
	return strings.Split(kind, "-by-")[0]
}

// activeFilters returns the filters mentioned in the command name.
func activeFilters(kind string) []string {
	startIndex := strings.Index(kind, "-by-") + len("-by-")
	return strings.Split(kind[startIndex:], "-and-")
}

// Return an expression evaluating if a filter should be used (based on active
// criteria).
func activeCriteria(filter []string, ignoredFilter []string) string {
	expr := ""
	for i, name := range filter {
		if i > 0 {
			expr += " && "
		}

		expr += fmt.Sprintf("filter.%s != nil", name)
	}

	for _, name := range ignoredFilter {
		if len(expr) > 0 {
			expr += " && "
		}

		expr += fmt.Sprintf("filter.%s == nil", name)
	}

	return expr
}

// Return the code for a "dest" function, to be passed as parameter to
// query.SelectObjects in order to scan a single row.
func destFunc(slice string, typ string, fields []*Field) string {
	f := fmt.Sprintf(`func(i int) []any {
                      %s = append(%s, %s{})
                      return []any{
`, slice, slice, typ)

	for _, field := range fields {
		f += fmt.Sprintf("&%s[i].%s,\n", slice, field.Name)
	}

	f += "        }\n"
	f += "}"

	return f
}
<commit_msg>lxd/db/generate/db/lex: Use QueryScan rowFunc in generated code<commit_after>package db

import (
	"fmt"
	"strings"

	"github.com/lxc/lxd/lxd/db/generate/lex"
)

// Return the table name for the given database entity.
func entityTable(entity string, override string) string {
	if override != "" {
		return override
	}

	entityParts := strings.Split(lex.Snake(entity), "_")
	tableParts := make([]string, len(entityParts))
	for i, part := range entityParts {
		if strings.HasSuffix(part, "ty") || strings.HasSuffix(part, "ly") {
			tableParts[i] = part
		} else {
			tableParts[i] = lex.Plural(part)
		}
	}

	return strings.Join(tableParts, "_")
}

// Return the name of the Filter struct for the given database entity.
func entityFilter(entity string) string {
	return fmt.Sprintf("%sFilter", lex.Camel(entity))
}

// Return the name of the global variable holding the registration code for
// the given kind of statement aganst the given entity.
func stmtCodeVar(entity string, kind string, filters ...string) string {
	prefix := lex.Minuscule(lex.Camel(entity))
	name := fmt.Sprintf("%s%s", prefix, lex.Camel(kind))

	if len(filters) > 0 {
		name += "By"
		name += strings.Join(filters, "And")
	}

	return name
}

// operation returns the kind of operation being performed, without filter fields.
func operation(kind string) string {
	return strings.Split(kind, "-by-")[0]
}

// activeFilters returns the filters mentioned in the command name.
func activeFilters(kind string) []string {
	startIndex := strings.Index(kind, "-by-") + len("-by-")
	return strings.Split(kind[startIndex:], "-and-")
}

// Return an expression evaluating if a filter should be used (based on active
// criteria).
func activeCriteria(filter []string, ignoredFilter []string) string {
	expr := ""
	for i, name := range filter {
		if i > 0 {
			expr += " && "
		}

		expr += fmt.Sprintf("filter.%s != nil", name)
	}

	for _, name := range ignoredFilter {
		if len(expr) > 0 {
			expr += " && "
		}

		expr += fmt.Sprintf("filter.%s == nil", name)
	}

	return expr
}

// Return the code for a "dest" function, to be passed as parameter to
// query.SelectObjects in order to scan a single row.
func destFunc(slice string, typ string, fields []*Field) string {
	f := fmt.Sprintf(`func(i int) []any {
                      %s = append(%s, %s{})
                      return []any{
`, slice, slice, typ)

	for _, field := range fields {
		f += fmt.Sprintf("&%s[i].%s,\n", slice, field.Name)
	}

	f += "        }\n"
	f += "}"

	return f
}

// destFuncQueryScan returns code for QueryScan's rowFunc.
// TODO: Replace the above destFunc with this one once SelectObjects is updated to use the new rowFunc callback.
func destFuncQueryScan(slice string, typ string, fields []*Field) string {
	varName := lex.Minuscule(string(typ[0]))
	args := make([]string, 0, len(fields))
	for _, field := range fields {
		arg := fmt.Sprintf("&%s.%s", varName, field.Name)
		args = append(args, arg)
	}

	f := fmt.Sprintf(`func(scan func(dest ...any) error) error {
                      %s := %s{}
                      err := scan(%s)
                      if err != nil {
                        return err
                      }

                      %s = append(%s, %s)

                      return nil
                    }
`, varName, typ, strings.Join(args, ", "), slice, slice, varName)
	return f
}
<|endoftext|>


<commit_before>package storage

import (
	"io"

	"github.com/lxc/lxd/lxd/instance/instancetype"
	"github.com/lxc/lxd/lxd/migration"
	"github.com/lxc/lxd/lxd/operations"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/lxd/storage/drivers"
	"github.com/lxc/lxd/shared/api"
)

// Instance represents the storage relevant subset of a LXD instance.
type Instance interface {
	Name() string
	Project() string
	Type() instancetype.Type
	Path() string

	IsRunning() bool
	TemplateApply(trigger string) error
}

// Pool represents a LXD storage pool.
type Pool interface {
	// Internal.
	DaemonState() *state.State

	// Pool.
	ID() int64
	Name() string
	Driver() drivers.Driver

	GetResources() (*api.ResourcesStoragePool, error)
	Delete(op *operations.Operation) error

	Mount() (bool, error)
	Unmount() (bool, error)

	// Instances.
	CreateInstance(i Instance, op *operations.Operation) error
	CreateInstanceFromBackup(i Instance, sourcePath string, op *operations.Operation) error
	CreateInstanceFromCopy(i Instance, src Instance, snapshots bool, op *operations.Operation) error
	CreateInstanceFromImage(i Instance, fingerprint string, op *operations.Operation) error
	CreateInstanceFromMigration(i Instance, conn io.ReadWriteCloser, args migration.SinkArgs, op *operations.Operation) error
	RenameInstance(i Instance, newName string, op *operations.Operation) error
	DeleteInstance(i Instance, op *operations.Operation) error

	MigrateInstance(i Instance, snapshots bool, args migration.SourceArgs) (migration.StorageSourceDriver, error)
	RefreshInstance(i Instance, src Instance, snapshots bool, op *operations.Operation) error
	BackupInstance(i Instance, targetPath string, optimized bool, snapshots bool, op *operations.Operation) error

	GetInstanceUsage(i Instance) (int64, error)
	SetInstanceQuota(i Instance, quota uint64) error

	MountInstance(i Instance) (bool, error)
	UnmountInstance(i Instance) (bool, error)
	GetInstanceDisk(i Instance) (string, string, error)

	// Instance snapshots.
	CreateInstanceSnapshot(i Instance, name string, op *operations.Operation) error
	RenameInstanceSnapshot(i Instance, newName string, op *operations.Operation) error
	DeleteInstanceSnapshot(i Instance, op *operations.Operation) error
	RestoreInstanceSnapshot(i Instance, op *operations.Operation) error
	MountInstanceSnapshot(i Instance) (bool, error)
	UnmountInstanceSnapshot(i Instance) (bool, error)

	// Images.
	CreateImage(fingerprint string, op *operations.Operation) error
	DeleteImage(fingerprint string, op *operations.Operation) error

	// Custom volumes.
	CreateCustomVolume(volName, desc string, config map[string]string, op *operations.Operation) error
	CreateCustomVolumeFromCopy(volName, desc string, config map[string]string, srcPoolName, srcVolName string, srcVolOnly bool, op *operations.Operation) error
	UpdateCustomVolume(volName, newDesc string, newConfig map[string]string, op *operations.Operation) error
	RenameCustomVolume(volName string, newVolName string, op *operations.Operation) error
	DeleteCustomVolume(volName string, op *operations.Operation) error
	GetCustomVolumeUsage(volName string) (int64, error)
	MountCustomVolume(volName string, op *operations.Operation) (bool, error)
	UnmountCustomVolume(volName string, op *operations.Operation) (bool, error)

	// Custom volume snapshots.
	CreateCustomVolumeSnapshot(volName string, newSnapshotName string, op *operations.Operation) error
	RenameCustomVolumeSnapshot(volName string, newSnapshotName string, op *operations.Operation) error
	DeleteCustomVolumeSnapshot(volName string, op *operations.Operation) error
	RestoreCustomVolume(volName string, snapshotName string, op *operations.Operation) error

	// Custom volume migration.
	MigrationTypes(contentType drivers.ContentType) []migration.Type
	CreateCustomVolumeFromMigration(conn io.ReadWriteCloser, args migration.VolumeTargetArgs, op *operations.Operation) error
	MigrateCustomVolume(conn io.ReadWriteCloser, args migration.VolumeSourceArgs, op *operations.Operation) error
}
<commit_msg>lxd/storage/interfaces: Renames CreateImage to EnsureImage<commit_after>package storage

import (
	"io"

	"github.com/lxc/lxd/lxd/instance/instancetype"
	"github.com/lxc/lxd/lxd/migration"
	"github.com/lxc/lxd/lxd/operations"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/lxd/storage/drivers"
	"github.com/lxc/lxd/shared/api"
)

// Instance represents the storage relevant subset of a LXD instance.
type Instance interface {
	Name() string
	Project() string
	Type() instancetype.Type
	Path() string

	IsRunning() bool
	TemplateApply(trigger string) error
}

// Pool represents a LXD storage pool.
type Pool interface {
	// Internal.
	DaemonState() *state.State

	// Pool.
	ID() int64
	Name() string
	Driver() drivers.Driver

	GetResources() (*api.ResourcesStoragePool, error)
	Delete(op *operations.Operation) error

	Mount() (bool, error)
	Unmount() (bool, error)

	// Instances.
	CreateInstance(i Instance, op *operations.Operation) error
	CreateInstanceFromBackup(i Instance, sourcePath string, op *operations.Operation) error
	CreateInstanceFromCopy(i Instance, src Instance, snapshots bool, op *operations.Operation) error
	CreateInstanceFromImage(i Instance, fingerprint string, op *operations.Operation) error
	CreateInstanceFromMigration(i Instance, conn io.ReadWriteCloser, args migration.SinkArgs, op *operations.Operation) error
	RenameInstance(i Instance, newName string, op *operations.Operation) error
	DeleteInstance(i Instance, op *operations.Operation) error

	MigrateInstance(i Instance, snapshots bool, args migration.SourceArgs) (migration.StorageSourceDriver, error)
	RefreshInstance(i Instance, src Instance, snapshots bool, op *operations.Operation) error
	BackupInstance(i Instance, targetPath string, optimized bool, snapshots bool, op *operations.Operation) error

	GetInstanceUsage(i Instance) (int64, error)
	SetInstanceQuota(i Instance, quota uint64) error

	MountInstance(i Instance) (bool, error)
	UnmountInstance(i Instance) (bool, error)
	GetInstanceDisk(i Instance) (string, string, error)

	// Instance snapshots.
	CreateInstanceSnapshot(i Instance, name string, op *operations.Operation) error
	RenameInstanceSnapshot(i Instance, newName string, op *operations.Operation) error
	DeleteInstanceSnapshot(i Instance, op *operations.Operation) error
	RestoreInstanceSnapshot(i Instance, op *operations.Operation) error
	MountInstanceSnapshot(i Instance) (bool, error)
	UnmountInstanceSnapshot(i Instance) (bool, error)

	// Images.
	EnsureImage(fingerprint string, op *operations.Operation) error
	DeleteImage(fingerprint string, op *operations.Operation) error

	// Custom volumes.
	CreateCustomVolume(volName, desc string, config map[string]string, op *operations.Operation) error
	CreateCustomVolumeFromCopy(volName, desc string, config map[string]string, srcPoolName, srcVolName string, srcVolOnly bool, op *operations.Operation) error
	UpdateCustomVolume(volName, newDesc string, newConfig map[string]string, op *operations.Operation) error
	RenameCustomVolume(volName string, newVolName string, op *operations.Operation) error
	DeleteCustomVolume(volName string, op *operations.Operation) error
	GetCustomVolumeUsage(volName string) (int64, error)
	MountCustomVolume(volName string, op *operations.Operation) (bool, error)
	UnmountCustomVolume(volName string, op *operations.Operation) (bool, error)

	// Custom volume snapshots.
	CreateCustomVolumeSnapshot(volName string, newSnapshotName string, op *operations.Operation) error
	RenameCustomVolumeSnapshot(volName string, newSnapshotName string, op *operations.Operation) error
	DeleteCustomVolumeSnapshot(volName string, op *operations.Operation) error
	RestoreCustomVolume(volName string, snapshotName string, op *operations.Operation) error

	// Custom volume migration.
	MigrationTypes(contentType drivers.ContentType) []migration.Type
	CreateCustomVolumeFromMigration(conn io.ReadWriteCloser, args migration.VolumeTargetArgs, op *operations.Operation) error
	MigrateCustomVolume(conn io.ReadWriteCloser, args migration.VolumeSourceArgs, op *operations.Operation) error
}
<|endoftext|>


<commit_before>//go:generate stringer -type=FeatureFlag

package features

import (
	"fmt"
	"sync"
)

type FeatureFlag int

const (
	unused FeatureFlag = iota // unused is used for testing
	//   Deprecated features, these can be removed once stripped from production configs
	WriteIssuedNamesPrecert
	HeadNonceStatusOK
	RemoveWFE2AccountID
	CheckRenewalFirst
	ParallelCheckFailedValidation
	DeleteUnusedChallenges
	BlockedKeyTable

	//   Currently in-use features
	// Check CAA and respect validationmethods parameter.
	CAAValidationMethods
	// Check CAA and respect accounturi parameter.
	CAAAccountURI
	// EnforceMultiVA causes the VA to block on remote VA PerformValidation
	// requests in order to make a valid/invalid decision with the results.
	EnforceMultiVA
	// MultiVAFullResults will cause the main VA to wait for all of the remote VA
	// results, not just the threshold required to make a decision.
	MultiVAFullResults
	// MandatoryPOSTAsGET forbids legacy unauthenticated GET requests for ACME
	// resources.
	MandatoryPOSTAsGET
	// Allow creation of new registrations in ACMEv1.
	AllowV1Registration
	// V1DisableNewValidations disables validations for new domain names in the V1
	// API.
	V1DisableNewValidations
	// PrecertificateRevocation allows revocation of precertificates with the
	// ACMEv2 interface.
	PrecertificateRevocation
	// StripDefaultSchemePort enables stripping of default scheme ports from HTTP
	// request Host headers
	StripDefaultSchemePort
	// StoreIssuerInfo enables storage of information identifying the issuer of
	// a certificate in the certificateStatus table.
	StoreIssuerInfo
	// StoreKeyHashes enables storage of SPKI hashes associated with certificates.
	StoreKeyHashes
	// StoreRevokerInfo enables storage of the revoker and a bool indicating if the row
	// was checked for extant unrevoked certificates in the blockedKeys table.
	StoreRevokerInfo
	// RestrictRSAKeySizes enables restriction of acceptable RSA public key moduli to
	// the common sizes (2048, 3072, and 4096 bits).
	RestrictRSAKeySizes
	// FasterNewOrdersRateLimit enables use of a separate table for counting the
	// new orders rate limit.
	FasterNewOrdersRateLimit
)

// List of features and their default value, protected by fMu
var features = map[FeatureFlag]bool{
	unused:                        false,
	CAAValidationMethods:          false,
	CAAAccountURI:                 false,
	HeadNonceStatusOK:             false,
	EnforceMultiVA:                false,
	MultiVAFullResults:            false,
	RemoveWFE2AccountID:           false,
	CheckRenewalFirst:             false,
	MandatoryPOSTAsGET:            false,
	AllowV1Registration:           true,
	ParallelCheckFailedValidation: false,
	DeleteUnusedChallenges:        false,
	V1DisableNewValidations:       false,
	PrecertificateRevocation:      false,
	StripDefaultSchemePort:        false,
	StoreIssuerInfo:               false,
	WriteIssuedNamesPrecert:       false,
	StoreKeyHashes:                false,
	StoreRevokerInfo:              false,
	RestrictRSAKeySizes:           false,
	FasterNewOrdersRateLimit:      false,
}

var fMu = new(sync.RWMutex)

var initial = map[FeatureFlag]bool{}

var nameToFeature = make(map[string]FeatureFlag, len(features))

func init() {
	for f, v := range features {
		nameToFeature[f.String()] = f
		initial[f] = v
	}
}

// Set accepts a list of features and whether they should
// be enabled or disabled, it will return a error if passed
// a feature name that it doesn't know
func Set(featureSet map[string]bool) error {
	fMu.Lock()
	defer fMu.Unlock()
	for n, v := range featureSet {
		f, present := nameToFeature[n]
		if !present {
			return fmt.Errorf("feature '%s' doesn't exist", n)
		}
		features[f] = v
	}
	return nil
}

// Enabled returns true if the feature is enabled or false
// if it isn't, it will panic if passed a feature that it
// doesn't know.
func Enabled(n FeatureFlag) bool {
	fMu.RLock()
	defer fMu.RUnlock()
	v, present := features[n]
	if !present {
		panic(fmt.Sprintf("feature '%s' doesn't exist", n.String()))
	}
	return v
}

// Reset resets the features to their initial state
func Reset() {
	fMu.Lock()
	defer fMu.Unlock()
	for k, v := range initial {
		features[k] = v
	}
}
<commit_msg>Restore BlockedKeyTable to map of flag values. (#4893)<commit_after>//go:generate stringer -type=FeatureFlag

package features

import (
	"fmt"
	"sync"
)

type FeatureFlag int

const (
	unused FeatureFlag = iota // unused is used for testing
	//   Deprecated features, these can be removed once stripped from production configs
	WriteIssuedNamesPrecert
	HeadNonceStatusOK
	RemoveWFE2AccountID
	CheckRenewalFirst
	ParallelCheckFailedValidation
	DeleteUnusedChallenges
	BlockedKeyTable

	//   Currently in-use features
	// Check CAA and respect validationmethods parameter.
	CAAValidationMethods
	// Check CAA and respect accounturi parameter.
	CAAAccountURI
	// EnforceMultiVA causes the VA to block on remote VA PerformValidation
	// requests in order to make a valid/invalid decision with the results.
	EnforceMultiVA
	// MultiVAFullResults will cause the main VA to wait for all of the remote VA
	// results, not just the threshold required to make a decision.
	MultiVAFullResults
	// MandatoryPOSTAsGET forbids legacy unauthenticated GET requests for ACME
	// resources.
	MandatoryPOSTAsGET
	// Allow creation of new registrations in ACMEv1.
	AllowV1Registration
	// V1DisableNewValidations disables validations for new domain names in the V1
	// API.
	V1DisableNewValidations
	// PrecertificateRevocation allows revocation of precertificates with the
	// ACMEv2 interface.
	PrecertificateRevocation
	// StripDefaultSchemePort enables stripping of default scheme ports from HTTP
	// request Host headers
	StripDefaultSchemePort
	// StoreIssuerInfo enables storage of information identifying the issuer of
	// a certificate in the certificateStatus table.
	StoreIssuerInfo
	// StoreKeyHashes enables storage of SPKI hashes associated with certificates.
	StoreKeyHashes
	// StoreRevokerInfo enables storage of the revoker and a bool indicating if the row
	// was checked for extant unrevoked certificates in the blockedKeys table.
	StoreRevokerInfo
	// RestrictRSAKeySizes enables restriction of acceptable RSA public key moduli to
	// the common sizes (2048, 3072, and 4096 bits).
	RestrictRSAKeySizes
	// FasterNewOrdersRateLimit enables use of a separate table for counting the
	// new orders rate limit.
	FasterNewOrdersRateLimit
)

// List of features and their default value, protected by fMu
var features = map[FeatureFlag]bool{
	unused:                        false,
	CAAValidationMethods:          false,
	CAAAccountURI:                 false,
	HeadNonceStatusOK:             false,
	EnforceMultiVA:                false,
	MultiVAFullResults:            false,
	RemoveWFE2AccountID:           false,
	CheckRenewalFirst:             false,
	MandatoryPOSTAsGET:            false,
	AllowV1Registration:           true,
	ParallelCheckFailedValidation: false,
	DeleteUnusedChallenges:        false,
	V1DisableNewValidations:       false,
	PrecertificateRevocation:      false,
	StripDefaultSchemePort:        false,
	StoreIssuerInfo:               false,
	WriteIssuedNamesPrecert:       false,
	StoreKeyHashes:                false,
	StoreRevokerInfo:              false,
	RestrictRSAKeySizes:           false,
	FasterNewOrdersRateLimit:      false,
	BlockedKeyTable:               false,
}

var fMu = new(sync.RWMutex)

var initial = map[FeatureFlag]bool{}

var nameToFeature = make(map[string]FeatureFlag, len(features))

func init() {
	for f, v := range features {
		nameToFeature[f.String()] = f
		initial[f] = v
	}
}

// Set accepts a list of features and whether they should
// be enabled or disabled, it will return a error if passed
// a feature name that it doesn't know
func Set(featureSet map[string]bool) error {
	fMu.Lock()
	defer fMu.Unlock()
	for n, v := range featureSet {
		f, present := nameToFeature[n]
		if !present {
			return fmt.Errorf("feature '%s' doesn't exist", n)
		}
		features[f] = v
	}
	return nil
}

// Enabled returns true if the feature is enabled or false
// if it isn't, it will panic if passed a feature that it
// doesn't know.
func Enabled(n FeatureFlag) bool {
	fMu.RLock()
	defer fMu.RUnlock()
	v, present := features[n]
	if !present {
		panic(fmt.Sprintf("feature '%s' doesn't exist", n.String()))
	}
	return v
}

// Reset resets the features to their initial state
func Reset() {
	fMu.Lock()
	defer fMu.Unlock()
	for k, v := range initial {
		features[k] = v
	}
}
<|endoftext|>


<commit_before>package lib

import (
	"crypto/md5"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"runtime"
	"strings"
	"time"

	"github.com/cheggaaa/pb"
	"github.com/nightlyone/lockfile"
	"github.com/pivotal-golang/archiver/extractor"

	"github.com/subutai-io/base/agent/config"
	"github.com/subutai-io/base/agent/lib/container"
	"github.com/subutai-io/base/agent/lib/gpg"
	"github.com/subutai-io/base/agent/lib/template"
	"github.com/subutai-io/base/agent/log"
)

var (
	lock lockfile.Lockfile
)

type templ struct {
	name    string
	file    string
	version string
	branch  string
	id      string
	hash    string
}

func templId(t templ, arch string, kurjun *http.Client) string {
	url := config.Cdn.Kurjun + "/template/info?name=" + t.name + "&version=" + t.version + "-" + t.branch + "&type=text"
	if len(t.branch) == 0 {
		url = config.Cdn.Kurjun + "/template/info?name=" + t.name + "&version=" + t.version + "&type=text"
	}
	response, err := kurjun.Get(url)
	log.Debug("Retrieving id, get: " + url)
	if err == nil && response.StatusCode == 204 {
		log.Warn("Cannot get template with specified version, trying without version")
		response, err = kurjun.Get(config.Cdn.Kurjun + "/template/info?name=" + t.name + "&type=text")
	}
	if log.Check(log.WarnLevel, "Getting kurjun response", err) || response.StatusCode != 200 {
		return ""
	}

	hash, err := ioutil.ReadAll(response.Body)
	response.Body.Close()
	if log.Check(log.WarnLevel, "Reading response body", err) {
		return ""
	}

	log.Debug("Template id: " + string(hash))
	return string(hash)
}

func md5sum(filePath string) string {
	file, err := os.Open(filePath)
	if err != nil {
		return ""
	}
	defer file.Close()

	hash := md5.New()
	if _, err := io.Copy(hash, file); err != nil {
		return ""
	}
	return fmt.Sprintf("%x", hash.Sum(nil))
}

func checkLocal(t templ) bool {
	var response string
	files, _ := ioutil.ReadDir(config.Agent.LxcPrefix + "tmpdir")
	for _, f := range files {
		if t.file == f.Name() {
			if len(t.hash) == 0 {
				fmt.Print("Cannot check md5 of local archive. Trust anyway? (y/n)")
				_, err := fmt.Scanln(&response)
				log.Check(log.FatalLevel, "Reading input", err)
				if response == "y" {
					return true
				}
			}
			if t.hash == md5sum(config.Agent.LxcPrefix+"tmpdir/"+f.Name()) {
				return true
			}
		}
	}
	return false
}

func download(t templ, kurjun *http.Client) bool {
	out, err := os.Create(config.Agent.LxcPrefix + "tmpdir/" + t.file)
	log.Check(log.FatalLevel, "Creating file "+t.file, err)
	defer out.Close()
	response, err := kurjun.Get(config.Cdn.Kurjun + "/template/get?id=" + t.id)
	log.Check(log.FatalLevel, "Getting "+config.Cdn.Kurjun+"/template/get?id="+t.id, err)
	defer response.Body.Close()

	bar := pb.New(int(response.ContentLength)).SetUnits(pb.U_BYTES)
	bar.Start()
	rd := bar.NewProxyReader(response.Body)

	_, err = io.Copy(out, rd)
	log.Check(log.FatalLevel, "Writing file "+t.file, err)
	if t.hash == md5sum(config.Agent.LxcPrefix+"tmpdir/"+t.file) {
		return true
	}
	log.Error("Failed to check MD5 after download. Please check your connection and try again.")
	return false
}
func lockSubutai(file string) bool {
	lock, err := lockfile.New("/var/run/lock/subutai." + file)
	if log.Check(log.DebugLevel, "Init lock "+file, err) {
		return false
	}

	err = lock.TryLock()
	if log.Check(log.DebugLevel, "Locking file "+file, err) {
		return false
	}

	return true
}

func unlockSubutai() {
	lock.Unlock()
}

func LxcImport(name, version, token string) {
	if container.IsContainer(name) && name == "management" && len(token) > 1 {
		gpg.ExchageAndEncrypt("management", token)
		return
	}

	if container.IsContainer(name) {
		log.Info(name + " instance exist")
		return
	}

	log.Info("Importing " + name)
	for !lockSubutai(name + ".import") {
		time.Sleep(time.Second * 1)
	}
	defer unlockSubutai()

	var t templ

	t.name = name
	t.version = config.Template.Version
	t.branch = config.Template.Branch
	if len(version) != 0 {
		if strings.Contains(version, "-") {
			verstr := strings.Split(version, "-")
			if len(verstr) == 2 {
				t.version = verstr[0]
				t.branch = verstr[1]
			} else {
				log.Error("Invalid version")
			}
		} else {
			if strings.Contains(version, ".") {
				t.version = version
				t.branch = config.Template.Branch
			} else {
				t.version = config.Template.Version
				t.branch = version
			}
		}
	}

	log.Info("Version: " + t.version + ", branch: " + t.branch)

	if t.branch == "stable" {
		t.branch = ""
		t.file = t.name + "-subutai-template_" + t.version + "_" + config.Template.Arch + ".tar.gz"
	} else {
		t.file = t.name + "-subutai-template_" + t.version + "-" + t.branch + "_" + config.Template.Arch + ".tar.gz"
	}

	kurjun := config.CheckKurjun()
	t.id = templId(t, runtime.GOARCH, kurjun)
	if len(strings.Split(t.id, ".")) > 1 {
		t.hash = strings.Split(t.id, ".")[1]
	}

	if !checkLocal(t) && !download(t, kurjun) {
		log.Error(t.name + " template not found")
	}

	log.Info("Unpacking template " + t.name)
	tgz := extractor.NewTgz()
	templdir := config.Agent.LxcPrefix + "tmpdir/" + t.name
	tgz.Extract(config.Agent.LxcPrefix+"tmpdir/"+t.file, templdir)
	parent := container.GetConfigItem(templdir+"/config", "subutai.parent")
	if parent != "" && parent != t.name && !container.IsTemplate(parent) {
		log.Info("Parent template required: " + parent)
		LxcImport(parent, "", token)
	}

	log.Info("Installing template " + t.name)
	template.Install(parent, t.name)
	// TODO following lines kept for back compatibility with old templates, should be deleted when all templates will be replaced.
	os.Rename(config.Agent.LxcPrefix+t.name+"/"+t.name+"-home", config.Agent.LxcPrefix+t.name+"/home")
	os.Rename(config.Agent.LxcPrefix+t.name+"/"+t.name+"-var", config.Agent.LxcPrefix+t.name+"/var")
	os.Rename(config.Agent.LxcPrefix+t.name+"/"+t.name+"-opt", config.Agent.LxcPrefix+t.name+"/opt")
	log.Check(log.FatalLevel, "Removing temp dir "+templdir, os.RemoveAll(templdir))

	if t.name == "management" {
		template.MngInit()
		return
	}

	container.SetContainerConf(t.name, [][]string{
		{"lxc.rootfs", config.Agent.LxcPrefix + t.name + "/rootfs"},
		{"lxc.mount", config.Agent.LxcPrefix + t.name + "/fstab"},
		{"lxc.hook.pre-start", ""},
		{"lxc.include", config.Agent.AppPrefix + "share/lxc/config/ubuntu.common.conf"},
		{"lxc.include", config.Agent.AppPrefix + "share/lxc/config/ubuntu.userns.conf"},
		{"subutai.config.path", config.Agent.AppPrefix + "etc"},
		{"lxc.network.script.up", config.Agent.AppPrefix + "bin/create_ovs_interface"},
		{"lxc.mount.entry", config.Agent.LxcPrefix + t.name + "/home home none bind,rw 0 0"},
		{"lxc.mount.entry", config.Agent.LxcPrefix + t.name + "/opt opt none bind,rw 0 0"},
		{"lxc.mount.entry", config.Agent.LxcPrefix + t.name + "/var var none bind,rw 0 0"},
	})
}
<commit_msg>Removed "stable" option in import<commit_after>package lib

import (
	"crypto/md5"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"runtime"
	"strings"
	"time"

	"github.com/cheggaaa/pb"
	"github.com/nightlyone/lockfile"
	"github.com/pivotal-golang/archiver/extractor"

	"github.com/subutai-io/base/agent/config"
	"github.com/subutai-io/base/agent/lib/container"
	"github.com/subutai-io/base/agent/lib/gpg"
	"github.com/subutai-io/base/agent/lib/template"
	"github.com/subutai-io/base/agent/log"
)

var (
	lock lockfile.Lockfile
)

type templ struct {
	name    string
	file    string
	version string
	branch  string
	id      string
	hash    string
}

func templId(t templ, arch string, kurjun *http.Client) string {
	url := config.Cdn.Kurjun + "/template/info?name=" + t.name + "&version=" + t.version + "-" + t.branch + "&type=text"
	if len(t.branch) == 0 {
		url = config.Cdn.Kurjun + "/template/info?name=" + t.name + "&version=" + t.version + "&type=text"
	}
	response, err := kurjun.Get(url)
	log.Debug("Retrieving id, get: " + url)
	if err == nil && response.StatusCode == 204 {
		log.Warn("Cannot get template with specified version, trying without version")
		response, err = kurjun.Get(config.Cdn.Kurjun + "/template/info?name=" + t.name + "&type=text")
	}
	if log.Check(log.WarnLevel, "Getting kurjun response", err) || response.StatusCode != 200 {
		return ""
	}

	hash, err := ioutil.ReadAll(response.Body)
	response.Body.Close()
	if log.Check(log.WarnLevel, "Reading response body", err) {
		return ""
	}

	log.Debug("Template id: " + string(hash))
	return string(hash)
}

func md5sum(filePath string) string {
	file, err := os.Open(filePath)
	if err != nil {
		return ""
	}
	defer file.Close()

	hash := md5.New()
	if _, err := io.Copy(hash, file); err != nil {
		return ""
	}
	return fmt.Sprintf("%x", hash.Sum(nil))
}

func checkLocal(t templ) bool {
	var response string
	files, _ := ioutil.ReadDir(config.Agent.LxcPrefix + "tmpdir")
	for _, f := range files {
		if t.file == f.Name() {
			if len(t.hash) == 0 {
				fmt.Print("Cannot check md5 of local archive. Trust anyway? (y/n)")
				_, err := fmt.Scanln(&response)
				log.Check(log.FatalLevel, "Reading input", err)
				if response == "y" {
					return true
				}
			}
			if t.hash == md5sum(config.Agent.LxcPrefix+"tmpdir/"+f.Name()) {
				return true
			}
		}
	}
	return false
}

func download(t templ, kurjun *http.Client) bool {
	out, err := os.Create(config.Agent.LxcPrefix + "tmpdir/" + t.file)
	log.Check(log.FatalLevel, "Creating file "+t.file, err)
	defer out.Close()
	response, err := kurjun.Get(config.Cdn.Kurjun + "/template/get?id=" + t.id)
	log.Check(log.FatalLevel, "Getting "+config.Cdn.Kurjun+"/template/get?id="+t.id, err)
	defer response.Body.Close()

	bar := pb.New(int(response.ContentLength)).SetUnits(pb.U_BYTES)
	bar.Start()
	rd := bar.NewProxyReader(response.Body)

	_, err = io.Copy(out, rd)
	log.Check(log.FatalLevel, "Writing file "+t.file, err)
	if t.hash == md5sum(config.Agent.LxcPrefix+"tmpdir/"+t.file) {
		return true
	}
	log.Error("Failed to check MD5 after download. Please check your connection and try again.")
	return false
}
func lockSubutai(file string) bool {
	lock, err := lockfile.New("/var/run/lock/subutai." + file)
	if log.Check(log.DebugLevel, "Init lock "+file, err) {
		return false
	}

	err = lock.TryLock()
	if log.Check(log.DebugLevel, "Locking file "+file, err) {
		return false
	}

	return true
}

func unlockSubutai() {
	lock.Unlock()
}

func LxcImport(name, version, token string) {
	if container.IsContainer(name) && name == "management" && len(token) > 1 {
		gpg.ExchageAndEncrypt("management", token)
		return
	}

	if container.IsContainer(name) {
		log.Info(name + " instance exist")
		return
	}

	log.Info("Importing " + name)
	for !lockSubutai(name + ".import") {
		time.Sleep(time.Second * 1)
	}
	defer unlockSubutai()

	var t templ

	t.name = name
	t.version = config.Template.Version
	t.branch = config.Template.Branch
	if len(version) != 0 {
		if strings.Contains(version, "-") {
			verstr := strings.Split(version, "-")
			if len(verstr) == 2 {
				t.version = verstr[0]
				t.branch = verstr[1]
			} else {
				log.Error("Invalid version")
			}
		} else {
			if strings.Contains(version, ".") {
				t.version = version
				t.branch = config.Template.Branch
			} else {
				t.version = config.Template.Version
				t.branch = version
			}
		}
	}

	log.Info("Version: " + t.version + ", branch: " + t.branch)

	if t.branch == "" {
		t.file = t.name + "-subutai-template_" + t.version + "_" + config.Template.Arch + ".tar.gz"
	} else {
		t.file = t.name + "-subutai-template_" + t.version + "-" + t.branch + "_" + config.Template.Arch + ".tar.gz"
	}

	kurjun := config.CheckKurjun()
	t.id = templId(t, runtime.GOARCH, kurjun)
	if len(strings.Split(t.id, ".")) > 1 {
		t.hash = strings.Split(t.id, ".")[1]
	}

	if !checkLocal(t) && !download(t, kurjun) {
		log.Error(t.name + " template not found")
	}

	log.Info("Unpacking template " + t.name)
	tgz := extractor.NewTgz()
	templdir := config.Agent.LxcPrefix + "tmpdir/" + t.name
	tgz.Extract(config.Agent.LxcPrefix+"tmpdir/"+t.file, templdir)
	parent := container.GetConfigItem(templdir+"/config", "subutai.parent")
	if parent != "" && parent != t.name && !container.IsTemplate(parent) {
		log.Info("Parent template required: " + parent)
		LxcImport(parent, "", token)
	}

	log.Info("Installing template " + t.name)
	template.Install(parent, t.name)
	// TODO following lines kept for back compatibility with old templates, should be deleted when all templates will be replaced.
	os.Rename(config.Agent.LxcPrefix+t.name+"/"+t.name+"-home", config.Agent.LxcPrefix+t.name+"/home")
	os.Rename(config.Agent.LxcPrefix+t.name+"/"+t.name+"-var", config.Agent.LxcPrefix+t.name+"/var")
	os.Rename(config.Agent.LxcPrefix+t.name+"/"+t.name+"-opt", config.Agent.LxcPrefix+t.name+"/opt")
	log.Check(log.FatalLevel, "Removing temp dir "+templdir, os.RemoveAll(templdir))

	if t.name == "management" {
		template.MngInit()
		return
	}

	container.SetContainerConf(t.name, [][]string{
		{"lxc.rootfs", config.Agent.LxcPrefix + t.name + "/rootfs"},
		{"lxc.mount", config.Agent.LxcPrefix + t.name + "/fstab"},
		{"lxc.hook.pre-start", ""},
		{"lxc.include", config.Agent.AppPrefix + "share/lxc/config/ubuntu.common.conf"},
		{"lxc.include", config.Agent.AppPrefix + "share/lxc/config/ubuntu.userns.conf"},
		{"subutai.config.path", config.Agent.AppPrefix + "etc"},
		{"lxc.network.script.up", config.Agent.AppPrefix + "bin/create_ovs_interface"},
		{"lxc.mount.entry", config.Agent.LxcPrefix + t.name + "/home home none bind,rw 0 0"},
		{"lxc.mount.entry", config.Agent.LxcPrefix + t.name + "/opt opt none bind,rw 0 0"},
		{"lxc.mount.entry", config.Agent.LxcPrefix + t.name + "/var var none bind,rw 0 0"},
	})
}
<|endoftext|>


<commit_before>package lib

import (
	"crypto/tls"
	"encoding/json"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"

	"github.com/cheggaaa/pb"

	"github.com/subutai-io/base/agent/config"
	"github.com/subutai-io/base/agent/lib/container"
	"github.com/subutai-io/base/agent/log"
)

type snap struct {
	Md5Sum  string   `json:"md5Sum"`
	Name    string   `json:"name"`
	Owner   []string `json:"owner"`
	Version string   `json:"version"`
}

func ifUpdateable(installed int) string {
	var update []snap

	tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
	client := &http.Client{Transport: tr}
	if !config.Cdn.Allowinsecure {
		client = &http.Client{}
	}
	resp, err := client.Get("https://" + config.Cdn.Url + ":" + config.Cdn.Sslport + "/kurjun/rest/file/info?name=subutai_")
	log.Check(log.FatalLevel, "GET: https://"+config.Cdn.Url+":"+config.Cdn.Sslport+"/kurjun/rest/file/info?name=subutai_", err)
	defer resp.Body.Close()
	js, err := ioutil.ReadAll(resp.Body)
	log.Check(log.FatalLevel, "Reading response", err)
	log.Check(log.FatalLevel, "Parsing file list", json.Unmarshal(js, &update))

	hash := ""
	for _, v := range update {
		available, err := strconv.Atoi(v.Version)
		log.Check(log.ErrorLevel, "Matching update, "+v.Version, err)
		if strings.HasSuffix(v.Name, ".snap") && installed < available && ourCI(v.Owner) && strings.Contains(v.Name, config.Template.Arch) {
			if len(config.Template.Branch) != 0 && strings.Contains(v.Name, config.Template.Branch) {
				log.Debug("Found newer snap: " + v.Name + ", " + v.Version)
				installed = available
				hash = v.Md5Sum
			} else if len(config.Template.Branch) == 0 {
				log.Debug("Found newer snap: " + v.Name + ", " + v.Version)
				installed = available
				hash = v.Md5Sum
			}
		}
	}

	log.Debug("Latest version: " + strconv.Itoa(installed))
	return hash
}

func ourCI(owners []string) bool {
	for _, v := range owners {
		if v == "jenkins" {
			return true
		}
	}
	return false
}

func getInstalled() string {
	f, err := ioutil.ReadFile(config.Agent.AppPrefix + "/meta/package.yaml")
	if !log.Check(log.DebugLevel, "Reading file package.yaml", err) {
		lines := strings.Split(string(f), "\n")
		for _, v := range lines {
			if strings.HasPrefix(v, "version: ") {
				if version := strings.Split(strings.TrimPrefix(v, "version: "), "-"); len(version) > 1 {
					log.Debug("Installed: " + version[1])
					return version[1]
				}
			}
		}
	}
	return "0"
}

func upgradeRh(packet string) {
	log.Info("Updating Resource host")
	file, err := os.Create("/tmp/" + packet)
	log.Check(log.FatalLevel, "Creating update file", err)
	defer file.Close()
	tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
	client := &http.Client{Transport: tr}
	if !config.Cdn.Allowinsecure {
		client = &http.Client{}
	}
	resp, err := client.Get("https://" + config.Cdn.Url + ":" + config.Cdn.Sslport + "/kurjun/rest/file/get?id=" + packet)
	log.Check(log.FatalLevel, "GET: https://"+config.Cdn.Url+":"+config.Cdn.Sslport+"/kurjun/rest/file/get?id="+packet, err)
	defer resp.Body.Close()
	log.Info("Downloading snap package")
	bar := pb.New(int(resp.ContentLength)).SetUnits(pb.U_BYTES)
	bar.Start()
	rd := bar.NewProxyReader(resp.Body)

	_, err = io.Copy(file, rd)
	log.Check(log.FatalLevel, "Writing response to file", err)

	log.Check(log.FatalLevel, "Installing update /tmp/"+packet,
		exec.Command("snappy", "install", "--allow-unauthenticated", "/tmp/"+packet).Run())
	log.Check(log.FatalLevel, "Removing update file /tmp/"+packet, os.Remove("/tmp/"+packet))

}

func Update(name string, check bool) {
	if !lockSubutai(name + ".update") {
		log.Error("Another update process is already running")
	}
	defer unlockSubutai()
	switch name {
	case "rh":

		installed, err := strconv.Atoi(getInstalled())
		log.Check(log.FatalLevel, "Converting installed package timestamp to int", err)

		newsnap := ifUpdateable(installed)
		if len(newsnap) == 0 {
			log.Info("No update is available")
			os.Exit(1)
		} else if check {
			log.Info("Update is available")
			os.Exit(0)
		}
		upgradeRh(newsnap)

	default:
		if !container.IsContainer(name) {
			log.Error("no such instance \"" + name + "\"")
		}
		_, err := container.AttachExec(name, []string{"apt-get", "-qq", "update", "-y", "--force-yes", "-o", "Acquire::http::Timeout=5"})
		log.Check(log.FatalLevel, "Updating apt index", err)
		output, err := container.AttachExec(name, []string{"apt-get", "-qq", "upgrade", "-y", "--force-yes", "-o", "Acquire::http::Timeout=5", "-s"})
		log.Check(log.FatalLevel, "Checking for available update", err)
		if len(output) == 0 {
			log.Info("No update is available")
			os.Exit(1)
		} else if check {
			log.Info("Update is available")
			os.Exit(0)
		}
		_, err = container.AttachExec(name, []string{"apt-get", "-qq", "upgrade", "-y", "--force-yes", "-o", "Acquire::http::Timeout=5"})
		log.Check(log.FatalLevel, "Updating container", err)
	}
}
<commit_msg>Update matching fix, fixes subutai-io/subos#507<commit_after>package lib

import (
	"crypto/tls"
	"encoding/json"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"

	"github.com/cheggaaa/pb"

	"github.com/subutai-io/base/agent/config"
	"github.com/subutai-io/base/agent/lib/container"
	"github.com/subutai-io/base/agent/log"
)

type snap struct {
	Md5Sum  string   `json:"md5Sum"`
	Name    string   `json:"name"`
	Owner   []string `json:"owner"`
	Version string   `json:"version"`
}

func ifUpdateable(installed int) string {
	var update []snap

	tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
	client := &http.Client{Transport: tr}
	if !config.Cdn.Allowinsecure {
		client = &http.Client{}
	}
	resp, err := client.Get("https://" + config.Cdn.Url + ":" + config.Cdn.Sslport + "/kurjun/rest/file/info?name=subutai_")
	log.Check(log.FatalLevel, "GET: https://"+config.Cdn.Url+":"+config.Cdn.Sslport+"/kurjun/rest/file/info?name=subutai_", err)
	defer resp.Body.Close()
	js, err := ioutil.ReadAll(resp.Body)
	log.Check(log.FatalLevel, "Reading response", err)
	log.Check(log.FatalLevel, "Parsing file list", json.Unmarshal(js, &update))

	hash := ""
	for _, v := range update {
		available, err := strconv.Atoi(v.Version)
		log.Check(log.ErrorLevel, "Matching update "+v.Version, err)
		if len(config.Template.Branch) != 0 && strings.HasSuffix(v.Name, config.Template.Arch+"-"+config.Template.Branch+".snap") && installed < available && ourCI(v.Owner) {
			log.Debug("Found newer snap: " + v.Name + ", " + v.Version)
			installed = available
			hash = v.Md5Sum
		} else if len(config.Template.Branch) == 0 && strings.HasSuffix(v.Name, config.Template.Arch+".snap") && installed < available && ourCI(v.Owner) {
			log.Debug("Found newer snap: " + v.Name + ", " + v.Version)
			installed = available
			hash = v.Md5Sum
		}
	}

	log.Debug("Latest version: " + strconv.Itoa(installed))
	return hash
}

func ourCI(owners []string) bool {
	for _, v := range owners {
		if v == "jenkins" {
			return true
		}
	}
	return false
}

func getInstalled() string {
	f, err := ioutil.ReadFile(config.Agent.AppPrefix + "/meta/package.yaml")
	if !log.Check(log.DebugLevel, "Reading file package.yaml", err) {
		lines := strings.Split(string(f), "\n")
		for _, v := range lines {
			if strings.HasPrefix(v, "version: ") {
				if version := strings.Split(strings.TrimPrefix(v, "version: "), "-"); len(version) > 1 {
					log.Debug("Installed: " + version[1])
					return version[1]
				}
			}
		}
	}
	return "0"
}

func upgradeRh(packet string) {
	log.Info("Updating Resource host")
	file, err := os.Create("/tmp/" + packet)
	log.Check(log.FatalLevel, "Creating update file", err)
	defer file.Close()
	tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
	client := &http.Client{Transport: tr}
	if !config.Cdn.Allowinsecure {
		client = &http.Client{}
	}
	resp, err := client.Get("https://" + config.Cdn.Url + ":" + config.Cdn.Sslport + "/kurjun/rest/file/get?id=" + packet)
	log.Check(log.FatalLevel, "GET: https://"+config.Cdn.Url+":"+config.Cdn.Sslport+"/kurjun/rest/file/get?id="+packet, err)
	defer resp.Body.Close()
	log.Info("Downloading snap package")
	bar := pb.New(int(resp.ContentLength)).SetUnits(pb.U_BYTES)
	bar.Start()
	rd := bar.NewProxyReader(resp.Body)

	_, err = io.Copy(file, rd)
	log.Check(log.FatalLevel, "Writing response to file", err)

	log.Check(log.FatalLevel, "Installing update /tmp/"+packet,
		exec.Command("snappy", "install", "--allow-unauthenticated", "/tmp/"+packet).Run())
	log.Check(log.FatalLevel, "Removing update file /tmp/"+packet, os.Remove("/tmp/"+packet))

}

func Update(name string, check bool) {
	if !lockSubutai(name + ".update") {
		log.Error("Another update process is already running")
	}
	defer unlockSubutai()
	switch name {
	case "rh":

		installed, err := strconv.Atoi(getInstalled())
		log.Check(log.FatalLevel, "Converting installed package timestamp to int", err)

		newsnap := ifUpdateable(installed)
		if len(newsnap) == 0 {
			log.Info("No update is available")
			os.Exit(1)
		} else if check {
			log.Info("Update is available")
			os.Exit(0)
		}
		upgradeRh(newsnap)

	default:
		if !container.IsContainer(name) {
			log.Error("no such instance \"" + name + "\"")
		}
		_, err := container.AttachExec(name, []string{"apt-get", "-qq", "update", "-y", "--force-yes", "-o", "Acquire::http::Timeout=5"})
		log.Check(log.FatalLevel, "Updating apt index", err)
		output, err := container.AttachExec(name, []string{"apt-get", "-qq", "upgrade", "-y", "--force-yes", "-o", "Acquire::http::Timeout=5", "-s"})
		log.Check(log.FatalLevel, "Checking for available update", err)
		if len(output) == 0 {
			log.Info("No update is available")
			os.Exit(1)
		} else if check {
			log.Info("Update is available")
			os.Exit(0)
		}
		_, err = container.AttachExec(name, []string{"apt-get", "-qq", "upgrade", "-y", "--force-yes", "-o", "Acquire::http::Timeout=5"})
		log.Check(log.FatalLevel, "Updating container", err)
	}
}
<|endoftext|>


<commit_before>package device

type Device struct {
	Permision       Permisions
	Owner           string
	Status          string
	Name            string
	Codename        string
	Vendor          string
	Manufacturer    string
	Type            string
	Platform        string
	Cpu             string
	Gpu             string
	Ram             string
	Weight          string
	Dimensions      string
	ScreenDimension string
	Resolution      string
	ScreenDensity   string
	InternalStorage string
	SdCard          string
	Bluetooth       string
	WiFi            string
	MainCamera      string
	SecondaryCamera string
	Power           string
	Peripherals     []string
}

type Permissions struct {
	Organization Permision
	Team         Permision
}

type Permision struct {
	Run     bool
	Results bool
	Info    bool
}
<commit_msg>Changed Peripherals to string type and fixed some typos<commit_after>package device

type Device struct {
	Permission      Permissions
	Owner           string
	Status          string
	Name            string
	Codename        string
	Vendor          string
	Manufacturer    string
	Type            string
	Platform        string
	Cpu             string
	Gpu             string
	Ram             string
	Weight          string
	Dimensions      string
	ScreenDimension string
	Resolution      string
	ScreenDensity   string
	InternalStorage string
	SdCard          string
	Bluetooth       string
	WiFi            string
	MainCamera      string
	SecondaryCamera string
	Power           string
	Peripherals     string
}

type Permissions struct {
	Organization Permission
	Team         Permission
}

type Permission struct {
	Run     bool
	Results bool
	Info    bool
}
<|endoftext|>


<commit_before>package lib

import (
	"encoding/json"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"

	"github.com/cheggaaa/pb"

	"github.com/subutai-io/base/agent/config"
	"github.com/subutai-io/base/agent/lib/container"
	"github.com/subutai-io/base/agent/log"
)

type snap struct {
	Version string `json:"version"`
}

func getAvailable(name string) string {
	var update snap
	client := &http.Client{}
	resp, err := client.Get("https://" + config.Cdn.Url + ":" + config.Cdn.Sslport + "/kurjun/rest/file/info?name=" + name)
	log.Check(log.FatalLevel, "GET: https://"+config.Cdn.Url+":"+config.Cdn.Sslport+"/kurjun/rest/file/info?name="+name, err)
	defer resp.Body.Close()
	js, err := ioutil.ReadAll(resp.Body)
	log.Check(log.FatalLevel, "Reading response", err)
	log.Check(log.FatalLevel, "Parsing file list", json.Unmarshal(js, &update))
	log.Debug("Available: " + update.Version)
	return update.Version
}

func getInstalled() string {
	f, err := ioutil.ReadFile(config.Agent.AppPrefix + "/meta/package.yaml")
	if !log.Check(log.DebugLevel, "Reading file package.yaml", err) {
		lines := strings.Split(string(f), "\n")
		for _, v := range lines {
			if strings.HasPrefix(v, "version: ") {
				if version := strings.Split(strings.TrimPrefix(v, "version: "), "-"); len(version) > 1 {
					log.Debug("Installed: " + version[1])
					return version[1]
				}
			}
		}
	}
	return "0"
}

func upgradeRh(packet string) {
	log.Info("Updating Resource host")
	file, err := os.Create("/tmp/" + packet)
	log.Check(log.FatalLevel, "Creating update file", err)
	defer file.Close()
	client := &http.Client{}
	resp, err := client.Get("https://" + config.Cdn.Url + ":" + config.Cdn.Sslport + "/kurjun/rest/file/get?name=" + packet)
	log.Check(log.FatalLevel, "GET: https://"+config.Cdn.Url+":"+config.Cdn.Sslport+"/kurjun/rest/file/get?name="+packet, err)
	defer resp.Body.Close()
	log.Info("Downloading snap package")
	bar := pb.New(int(resp.ContentLength)).SetUnits(pb.U_BYTES)
	bar.Start()
	rd := bar.NewProxyReader(resp.Body)

	_, err = io.Copy(file, rd)
	log.Check(log.FatalLevel, "Writing response to file", err)

	log.Check(log.FatalLevel, "Installing update /tmp/"+packet,
		exec.Command("snappy", "install", "--allow-unauthenticated", "/tmp/"+packet).Run())
	log.Check(log.FatalLevel, "Removing update file /tmp/"+packet, os.Remove("/tmp/"+packet))

}

func Update(name string, check bool) {
	switch name {
	case "rh":
		if !lockSubutai("rh.update") {
			log.Error("Another update process is already running")
		}
		defer unlockSubutai()

		packet := "subutai_" + config.Template.Version + "_" + config.Template.Arch + ".snap"
		if len(config.Template.Branch) != 0 {
			packet = "subutai_" + config.Template.Version + "_" + config.Template.Arch + "-" + config.Template.Branch + ".snap"
		}

		installed, err := strconv.Atoi(getInstalled())
		log.Check(log.FatalLevel, "Converting installed package timestamp to int", err)
		available, err := strconv.Atoi(getAvailable(packet))
		log.Check(log.FatalLevel, "Converting available package timestamp to int", err)

		if installed >= available {
			log.Info("No update is available")
			os.Exit(1)
		} else if check {
			log.Info("Update is available")
			os.Exit(0)
		}

		upgradeRh(packet)

	default:
		if !container.IsContainer(name) {
			log.Error("no such instance \"" + name + "\"")
		}
		_, err := container.AttachExec(name, []string{"apt-get", "-qq", "update", "-y", "--force-yes", "-o", "Acquire::http::Timeout=5"})
		log.Check(log.FatalLevel, "Updating apt index", err)
		output, err := container.AttachExec(name, []string{"apt-get", "-qq", "upgrade", "-y", "--force-yes", "-o", "Acquire::http::Timeout=5", "-s"})
		log.Check(log.FatalLevel, "Checking for available update", err)
		if len(output) == 0 {
			log.Info("No update is available")
			os.Exit(1)
		} else if check {
			log.Info("Update is available")
			os.Exit(0)
		}
		_, err = container.AttachExec(name, []string{"apt-get", "-qq", "upgrade", "-y", "--force-yes", "-o", "Acquire::http::Timeout=5"})
		log.Check(log.FatalLevel, "Updating container", err)
	}
}
<commit_msg>Allowinsecure flag applied for update operation<commit_after>package lib

import (
	"crypto/tls"
	"encoding/json"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"

	"github.com/cheggaaa/pb"

	"github.com/subutai-io/base/agent/config"
	"github.com/subutai-io/base/agent/lib/container"
	"github.com/subutai-io/base/agent/log"
)

type snap struct {
	Version string `json:"version"`
}

func getAvailable(name string) string {
	var update snap
	tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
	client := &http.Client{Transport: tr}
	if !config.Cdn.Allowinsecure {
		client = &http.Client{}
	}
	resp, err := client.Get("https://" + config.Cdn.Url + ":" + config.Cdn.Sslport + "/kurjun/rest/file/info?name=" + name)
	log.Check(log.FatalLevel, "GET: https://"+config.Cdn.Url+":"+config.Cdn.Sslport+"/kurjun/rest/file/info?name="+name, err)
	defer resp.Body.Close()
	js, err := ioutil.ReadAll(resp.Body)
	log.Check(log.FatalLevel, "Reading response", err)
	log.Check(log.FatalLevel, "Parsing file list", json.Unmarshal(js, &update))
	log.Debug("Available: " + update.Version)
	return update.Version
}

func getInstalled() string {
	f, err := ioutil.ReadFile(config.Agent.AppPrefix + "/meta/package.yaml")
	if !log.Check(log.DebugLevel, "Reading file package.yaml", err) {
		lines := strings.Split(string(f), "\n")
		for _, v := range lines {
			if strings.HasPrefix(v, "version: ") {
				if version := strings.Split(strings.TrimPrefix(v, "version: "), "-"); len(version) > 1 {
					log.Debug("Installed: " + version[1])
					return version[1]
				}
			}
		}
	}
	return "0"
}

func upgradeRh(packet string) {
	log.Info("Updating Resource host")
	file, err := os.Create("/tmp/" + packet)
	log.Check(log.FatalLevel, "Creating update file", err)
	defer file.Close()
	tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
	client := &http.Client{Transport: tr}
	if !config.Cdn.Allowinsecure {
		client = &http.Client{}
	}
	resp, err := client.Get("https://" + config.Cdn.Url + ":" + config.Cdn.Sslport + "/kurjun/rest/file/get?name=" + packet)
	log.Check(log.FatalLevel, "GET: https://"+config.Cdn.Url+":"+config.Cdn.Sslport+"/kurjun/rest/file/get?name="+packet, err)
	defer resp.Body.Close()
	log.Info("Downloading snap package")
	bar := pb.New(int(resp.ContentLength)).SetUnits(pb.U_BYTES)
	bar.Start()
	rd := bar.NewProxyReader(resp.Body)

	_, err = io.Copy(file, rd)
	log.Check(log.FatalLevel, "Writing response to file", err)

	log.Check(log.FatalLevel, "Installing update /tmp/"+packet,
		exec.Command("snappy", "install", "--allow-unauthenticated", "/tmp/"+packet).Run())
	log.Check(log.FatalLevel, "Removing update file /tmp/"+packet, os.Remove("/tmp/"+packet))

}

func Update(name string, check bool) {
	switch name {
	case "rh":
		if !lockSubutai("rh.update") {
			log.Error("Another update process is already running")
		}
		defer unlockSubutai()

		packet := "subutai_" + config.Template.Version + "_" + config.Template.Arch + ".snap"
		if len(config.Template.Branch) != 0 {
			packet = "subutai_" + config.Template.Version + "_" + config.Template.Arch + "-" + config.Template.Branch + ".snap"
		}

		installed, err := strconv.Atoi(getInstalled())
		log.Check(log.FatalLevel, "Converting installed package timestamp to int", err)
		available, err := strconv.Atoi(getAvailable(packet))
		log.Check(log.FatalLevel, "Converting available package timestamp to int", err)

		if installed >= available {
			log.Info("No update is available")
			os.Exit(1)
		} else if check {
			log.Info("Update is available")
			os.Exit(0)
		}

		upgradeRh(packet)

	default:
		if !container.IsContainer(name) {
			log.Error("no such instance \"" + name + "\"")
		}
		_, err := container.AttachExec(name, []string{"apt-get", "-qq", "update", "-y", "--force-yes", "-o", "Acquire::http::Timeout=5"})
		log.Check(log.FatalLevel, "Updating apt index", err)
		output, err := container.AttachExec(name, []string{"apt-get", "-qq", "upgrade", "-y", "--force-yes", "-o", "Acquire::http::Timeout=5", "-s"})
		log.Check(log.FatalLevel, "Checking for available update", err)
		if len(output) == 0 {
			log.Info("No update is available")
			os.Exit(1)
		} else if check {
			log.Info("Update is available")
			os.Exit(0)
		}
		_, err = container.AttachExec(name, []string{"apt-get", "-qq", "upgrade", "-y", "--force-yes", "-o", "Acquire::http::Timeout=5"})
		log.Check(log.FatalLevel, "Updating container", err)
	}
}
<|endoftext|>


<commit_before>package agent

import (
	"fmt"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/buildkite/agent/api"
	"github.com/buildkite/agent/logger"
	"github.com/buildkite/agent/process"
	"github.com/buildkite/agent/retry"
)

type JobRunner struct {
	// The job being run
	Job *api.Job

	// The APIClient that will be used when updating the job
	APIClient *api.Client

	// The endpoint that should be used when communicating with the API
	Endpoint string

	// The registred agent API record running this job
	Agent *api.Agent

	// The configuration of the agent from the CLI
	AgentConfiguration *AgentConfiguration

	// The interal process of the job
	process *process.Process

	// The internal header time streamer
	headerTimesStreamer *HeaderTimesStreamer

	// The internal log streamer
	logStreamer *LogStreamer

	// If the job is being cancelled
	cancelled bool

	// Used to wait on various routines that we spin up
	routineWaitGroup sync.WaitGroup
}

// Initializes the job runner
func (r JobRunner) Create() (runner *JobRunner, err error) {
	runner = &r

	// Our own APIClient using the endpoint and the agents access token
	runner.APIClient = APIClient{Endpoint: r.Endpoint, Token: r.Agent.AccessToken}.Create()

	// // Create our header times struct
	runner.headerTimesStreamer = &HeaderTimesStreamer{UploadCallback: r.onUploadHeaderTime}

	// The log streamer that will take the output chunks, and send them to
	// the Buildkite Agent API
	runner.logStreamer = LogStreamer{MaxChunkSizeBytes: r.Job.ChunksMaxSizeBytes, Callback: r.onUploadChunk}.New()

	// The process that will run the bootstrap script
	runner.process = &process.Process{
		Script:        r.AgentConfiguration.BootstrapScript,
		Env:           r.createEnvironment(),
		PTY:           r.AgentConfiguration.RunInPty,
		StartCallback: r.onProcessStartCallback,
		LineCallback:  runner.headerTimesStreamer.Scan,
	}

	return
}

// Runs the job
func (r *JobRunner) Run() error {
	logger.Info("Starting job %s", r.Job.ID)

	// Start the build in the Buildkite Agent API. This is the first thing
	// we do so if it fails, we don't have to worry about cleaning things
	// up like started log streamer workers, etc.
	if err := r.startJob(time.Now()); err != nil {
		return err
	}

	// Start the header time streamer
	if err := r.headerTimesStreamer.Start(); err != nil {
		return err
	}

	// Start the log streamer
	if err := r.logStreamer.Start(); err != nil {
		return err
	}

	// Start the process. This will block until it finishes.
	if err := r.process.Start(); err != nil {
		// Send the error as output
		r.logStreamer.Process(fmt.Sprintf("%s", err))
	} else {
		// Add the final output to the streamer
		r.logStreamer.Process(r.process.Output())
	}

	// Store the finished at time
	finishedAt := time.Now()

	// Stop the header time streamer. This will block until all the chunks
	// have been uploaded
	r.headerTimesStreamer.Stop()

	// Stop the log streamer. This will block until all the chunks have
	// been uploaded
	r.logStreamer.Stop()

	// Warn about failed chunks
	if r.logStreamer.ChunksFailedCount > 0 {
		logger.Warn("%d chunks failed to upload for this job", r.logStreamer.ChunksFailedCount)
	}

	// Finish the build in the Buildkite Agent API
	r.finishJob(finishedAt, r.process.ExitStatus, int(r.logStreamer.ChunksFailedCount))

	// Wait for the routines that we spun up to finish
	logger.Debug("[JobRunner] Waiting for all other routines to finish")
	r.routineWaitGroup.Wait()

	logger.Info("Finished job %s", r.Job.ID)

	return nil
}

func (r *JobRunner) Kill() error {
	if !r.cancelled {
		logger.Info("Canceling job %s", r.Job.ID)
		r.cancelled = true

		if r.process != nil {
			r.process.Kill()
		} else {
			logger.Error("No process to kill")
		}
	}

	return nil
}

// Creates the environment variables that will be used in the process
func (r *JobRunner) createEnvironment() []string {
	// Create a clone of our jobs environment. We'll then set the
	// environment variables provided by the agent, which will override any
	// sent by Buildkite. The variables below should always take
	// precedence.
	env := make(map[string]string)
	for key, value := range r.Job.Env {
		env[key] = value
	}

	// Add agent environment variables
	env["BUILDKITE_AGENT_ENDPOINT"] = r.Endpoint
	env["BUILDKITE_AGENT_ACCESS_TOKEN"] = r.Agent.AccessToken
	env["BUILDKITE_AGENT_DEBUG"] = fmt.Sprintf("%t", logger.GetLevel() == logger.DEBUG)
	env["BUILDKITE_AGENT_PID"] = fmt.Sprintf("%d", os.Getpid())

	// We know the BUILDKITE_BIN_PATH dir, because it's the path to the
	// currently running file (there is only 1 binary)
	dir, _ := filepath.Abs(filepath.Dir(os.Args[0]))
	env["BUILDKITE_BIN_PATH"] = dir

	// Add misc options
	env["BUILDKITE_BUILD_PATH"] = r.AgentConfiguration.BuildPath
	env["BUILDKITE_HOOKS_PATH"] = r.AgentConfiguration.HooksPath
	env["BUILDKITE_PLUGINS_PATH"] = r.AgentConfiguration.PluginsPath
	env["BUILDKITE_SSH_FINGERPRINT_VERIFICATION"] = fmt.Sprintf("%t", r.AgentConfiguration.SSHFingerprintVerification)
	env["BUILDKITE_COMMAND_EVAL"] = fmt.Sprintf("%t", r.AgentConfiguration.CommandEval)
	env["BUILDKITE_GIT_CLONE_FLAGS"] = r.AgentConfiguration.GitCloneFlags
	env["BUILDKITE_GIT_CLEAN_FLAGS"] = r.AgentConfiguration.GitCleanFlags

	// Convert the env map into a slice (which is what the script gear
	// needs)
	envSlice := []string{}
	for key, value := range env {
		envSlice = append(envSlice, fmt.Sprintf("%s=%s", key, value))
	}

	return envSlice
}

// Starts the job in the Buildkite Agent API. We'll retry on connection-related
// issues, but if a connection succeeds and we get an error response back from
// Buildkite, we won't bother retrying. For example, a "no such host" will
// retry, but a 422 from Buildkite won't.
func (r *JobRunner) startJob(startedAt time.Time) error {
	r.Job.StartedAt = startedAt.UTC().Format(time.RFC3339Nano)

	return retry.Do(func(s *retry.Stats) error {
		_, err := r.APIClient.Jobs.Start(r.Job)

		if err != nil {
			if api.IsRetryableError(err) {
				logger.Warn("%s (%s)", err, s)
			} else {
				logger.Warn("Buildkite rejected the call to start the job (%s)", err)
				s.Break()
			}
		}

		return err
	}, &retry.Config{Maximum: 30, Interval: 5 * time.Second})
}

// Finishes the job in the Buildkite Agent API. This call will keep on retrying
// forever until it finally gets a successfull response from the API.
func (r *JobRunner) finishJob(finishedAt time.Time, exitStatus string, failedChunkCount int) error {
	r.Job.FinishedAt = finishedAt.UTC().Format(time.RFC3339Nano)
	r.Job.ExitStatus = exitStatus
	r.Job.ChunksFailedCount = failedChunkCount

	return retry.Do(func(s *retry.Stats) error {
		response, err := r.APIClient.Jobs.Finish(r.Job)
		if err != nil {
			// If the API returns with a 422, that means that we
			// succesfully tried to finish the job, but Buildkite
			// rejected the finish for some reason. This can
			// sometimes mean that Buildkite has cancelled the job
			// before we get a chance to send the final API call
			// (maybe this agent took too long to kill the
			// process). In that case, we don't want to keep trying
			// to finish the job forever so we'll just bail out and
			// go find some more work to do.
			if response != nil && response.StatusCode == 422 {
				logger.Warn("Buildkite rejected the call to finish the job (%s)", err)
				s.Break()
			} else {
				logger.Warn("%s (%s)", err, s)
			}
		}

		return err
	}, &retry.Config{Forever: true, Interval: 1 * time.Second})
}

func (r *JobRunner) onProcessStartCallback() {
	// Since we're spinning up 2 routines here, we might as well add them
	// to the routine wait group here.
	r.routineWaitGroup.Add(2)

	// Start a routine that will grab the output every few seconds and send
	// it back to Buildkite
	go func() {
		for r.process.IsRunning() {
			// Send the output of the process to the log streamer
			// for processing
			r.logStreamer.Process(r.process.Output())

			// Check the output in another second
			time.Sleep(1 * time.Second)
		}

		// Mark this routine as done in the wait group
		r.routineWaitGroup.Done()

		logger.Debug("[JobRunner] Routine that processes the log has finished")
	}()

	// Start a routine that will constantly ping Buildkite to see if the
	// job has been canceled
	go func() {
		for r.process.IsRunning() {
			// Re-get the job and check it's status to see if it's been
			// cancelled
			jobState, _, err := r.APIClient.Jobs.GetState(r.Job.ID)
			if err != nil {
				// We don't really care if it fails, we'll just
				// try again soon anyway
				logger.Warn("Problem with getting job state %s (%s)", r.Job.ID, err)
			} else if jobState.State == "canceling" || jobState.State == "canceled" {
				r.Kill()
			}

			// Check for cancellations
			time.Sleep(time.Duration(r.Agent.JobStatusInterval))
		}

		// Mark this routine as done in the wait group
		r.routineWaitGroup.Done()

		logger.Debug("[JobRunner] Routine that refreshes the job has finished")
	}()
}

func (r *JobRunner) onUploadHeaderTime(cursor int, total int, times map[string]string) {
	retry.Do(func(s *retry.Stats) error {
		_, err := r.APIClient.HeaderTimes.Save(r.Job.ID, &api.HeaderTimes{Times: times})
		if err != nil {
			logger.Warn("%s (%s)", err, s)
		}

		return err
	}, &retry.Config{Maximum: 10, Interval: 5 * time.Second})
}

// Call when a chunk is ready for upload. It retry the chunk upload with an
// interval before giving up.
func (r *JobRunner) onUploadChunk(chunk *LogStreamerChunk) error {
	return retry.Do(func(s *retry.Stats) error {
		_, err := r.APIClient.Chunks.Upload(r.Job.ID, &api.Chunk{
			Data:     chunk.Data,
			Sequence: chunk.Order,
			Offset:   chunk.Offset,
			Size:     chunk.Size,
		})
		if err != nil {
			logger.Warn("%s (%s)", err, s)
		}

		return err
	}, &retry.Config{Maximum: 10, Interval: 5 * time.Second})
}
<commit_msg>Seconds, not milliseconds<commit_after>package agent

import (
	"fmt"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/buildkite/agent/api"
	"github.com/buildkite/agent/logger"
	"github.com/buildkite/agent/process"
	"github.com/buildkite/agent/retry"
)

type JobRunner struct {
	// The job being run
	Job *api.Job

	// The APIClient that will be used when updating the job
	APIClient *api.Client

	// The endpoint that should be used when communicating with the API
	Endpoint string

	// The registred agent API record running this job
	Agent *api.Agent

	// The configuration of the agent from the CLI
	AgentConfiguration *AgentConfiguration

	// The interal process of the job
	process *process.Process

	// The internal header time streamer
	headerTimesStreamer *HeaderTimesStreamer

	// The internal log streamer
	logStreamer *LogStreamer

	// If the job is being cancelled
	cancelled bool

	// Used to wait on various routines that we spin up
	routineWaitGroup sync.WaitGroup
}

// Initializes the job runner
func (r JobRunner) Create() (runner *JobRunner, err error) {
	runner = &r

	// Our own APIClient using the endpoint and the agents access token
	runner.APIClient = APIClient{Endpoint: r.Endpoint, Token: r.Agent.AccessToken}.Create()

	// // Create our header times struct
	runner.headerTimesStreamer = &HeaderTimesStreamer{UploadCallback: r.onUploadHeaderTime}

	// The log streamer that will take the output chunks, and send them to
	// the Buildkite Agent API
	runner.logStreamer = LogStreamer{MaxChunkSizeBytes: r.Job.ChunksMaxSizeBytes, Callback: r.onUploadChunk}.New()

	// The process that will run the bootstrap script
	runner.process = &process.Process{
		Script:        r.AgentConfiguration.BootstrapScript,
		Env:           r.createEnvironment(),
		PTY:           r.AgentConfiguration.RunInPty,
		StartCallback: r.onProcessStartCallback,
		LineCallback:  runner.headerTimesStreamer.Scan,
	}

	return
}

// Runs the job
func (r *JobRunner) Run() error {
	logger.Info("Starting job %s", r.Job.ID)

	// Start the build in the Buildkite Agent API. This is the first thing
	// we do so if it fails, we don't have to worry about cleaning things
	// up like started log streamer workers, etc.
	if err := r.startJob(time.Now()); err != nil {
		return err
	}

	// Start the header time streamer
	if err := r.headerTimesStreamer.Start(); err != nil {
		return err
	}

	// Start the log streamer
	if err := r.logStreamer.Start(); err != nil {
		return err
	}

	// Start the process. This will block until it finishes.
	if err := r.process.Start(); err != nil {
		// Send the error as output
		r.logStreamer.Process(fmt.Sprintf("%s", err))
	} else {
		// Add the final output to the streamer
		r.logStreamer.Process(r.process.Output())
	}

	// Store the finished at time
	finishedAt := time.Now()

	// Stop the header time streamer. This will block until all the chunks
	// have been uploaded
	r.headerTimesStreamer.Stop()

	// Stop the log streamer. This will block until all the chunks have
	// been uploaded
	r.logStreamer.Stop()

	// Warn about failed chunks
	if r.logStreamer.ChunksFailedCount > 0 {
		logger.Warn("%d chunks failed to upload for this job", r.logStreamer.ChunksFailedCount)
	}

	// Finish the build in the Buildkite Agent API
	r.finishJob(finishedAt, r.process.ExitStatus, int(r.logStreamer.ChunksFailedCount))

	// Wait for the routines that we spun up to finish
	logger.Debug("[JobRunner] Waiting for all other routines to finish")
	r.routineWaitGroup.Wait()

	logger.Info("Finished job %s", r.Job.ID)

	return nil
}

func (r *JobRunner) Kill() error {
	if !r.cancelled {
		logger.Info("Canceling job %s", r.Job.ID)
		r.cancelled = true

		if r.process != nil {
			r.process.Kill()
		} else {
			logger.Error("No process to kill")
		}
	}

	return nil
}

// Creates the environment variables that will be used in the process
func (r *JobRunner) createEnvironment() []string {
	// Create a clone of our jobs environment. We'll then set the
	// environment variables provided by the agent, which will override any
	// sent by Buildkite. The variables below should always take
	// precedence.
	env := make(map[string]string)
	for key, value := range r.Job.Env {
		env[key] = value
	}

	// Add agent environment variables
	env["BUILDKITE_AGENT_ENDPOINT"] = r.Endpoint
	env["BUILDKITE_AGENT_ACCESS_TOKEN"] = r.Agent.AccessToken
	env["BUILDKITE_AGENT_DEBUG"] = fmt.Sprintf("%t", logger.GetLevel() == logger.DEBUG)
	env["BUILDKITE_AGENT_PID"] = fmt.Sprintf("%d", os.Getpid())

	// We know the BUILDKITE_BIN_PATH dir, because it's the path to the
	// currently running file (there is only 1 binary)
	dir, _ := filepath.Abs(filepath.Dir(os.Args[0]))
	env["BUILDKITE_BIN_PATH"] = dir

	// Add misc options
	env["BUILDKITE_BUILD_PATH"] = r.AgentConfiguration.BuildPath
	env["BUILDKITE_HOOKS_PATH"] = r.AgentConfiguration.HooksPath
	env["BUILDKITE_PLUGINS_PATH"] = r.AgentConfiguration.PluginsPath
	env["BUILDKITE_SSH_FINGERPRINT_VERIFICATION"] = fmt.Sprintf("%t", r.AgentConfiguration.SSHFingerprintVerification)
	env["BUILDKITE_COMMAND_EVAL"] = fmt.Sprintf("%t", r.AgentConfiguration.CommandEval)
	env["BUILDKITE_GIT_CLONE_FLAGS"] = r.AgentConfiguration.GitCloneFlags
	env["BUILDKITE_GIT_CLEAN_FLAGS"] = r.AgentConfiguration.GitCleanFlags

	// Convert the env map into a slice (which is what the script gear
	// needs)
	envSlice := []string{}
	for key, value := range env {
		envSlice = append(envSlice, fmt.Sprintf("%s=%s", key, value))
	}

	return envSlice
}

// Starts the job in the Buildkite Agent API. We'll retry on connection-related
// issues, but if a connection succeeds and we get an error response back from
// Buildkite, we won't bother retrying. For example, a "no such host" will
// retry, but a 422 from Buildkite won't.
func (r *JobRunner) startJob(startedAt time.Time) error {
	r.Job.StartedAt = startedAt.UTC().Format(time.RFC3339Nano)

	return retry.Do(func(s *retry.Stats) error {
		_, err := r.APIClient.Jobs.Start(r.Job)

		if err != nil {
			if api.IsRetryableError(err) {
				logger.Warn("%s (%s)", err, s)
			} else {
				logger.Warn("Buildkite rejected the call to start the job (%s)", err)
				s.Break()
			}
		}

		return err
	}, &retry.Config{Maximum: 30, Interval: 5 * time.Second})
}

// Finishes the job in the Buildkite Agent API. This call will keep on retrying
// forever until it finally gets a successfull response from the API.
func (r *JobRunner) finishJob(finishedAt time.Time, exitStatus string, failedChunkCount int) error {
	r.Job.FinishedAt = finishedAt.UTC().Format(time.RFC3339Nano)
	r.Job.ExitStatus = exitStatus
	r.Job.ChunksFailedCount = failedChunkCount

	return retry.Do(func(s *retry.Stats) error {
		response, err := r.APIClient.Jobs.Finish(r.Job)
		if err != nil {
			// If the API returns with a 422, that means that we
			// succesfully tried to finish the job, but Buildkite
			// rejected the finish for some reason. This can
			// sometimes mean that Buildkite has cancelled the job
			// before we get a chance to send the final API call
			// (maybe this agent took too long to kill the
			// process). In that case, we don't want to keep trying
			// to finish the job forever so we'll just bail out and
			// go find some more work to do.
			if response != nil && response.StatusCode == 422 {
				logger.Warn("Buildkite rejected the call to finish the job (%s)", err)
				s.Break()
			} else {
				logger.Warn("%s (%s)", err, s)
			}
		}

		return err
	}, &retry.Config{Forever: true, Interval: 1 * time.Second})
}

func (r *JobRunner) onProcessStartCallback() {
	// Since we're spinning up 2 routines here, we might as well add them
	// to the routine wait group here.
	r.routineWaitGroup.Add(2)

	// Start a routine that will grab the output every few seconds and send
	// it back to Buildkite
	go func() {
		for r.process.IsRunning() {
			// Send the output of the process to the log streamer
			// for processing
			r.logStreamer.Process(r.process.Output())

			// Check the output in another second
			time.Sleep(1 * time.Second)
		}

		// Mark this routine as done in the wait group
		r.routineWaitGroup.Done()

		logger.Debug("[JobRunner] Routine that processes the log has finished")
	}()

	// Start a routine that will constantly ping Buildkite to see if the
	// job has been canceled
	go func() {
		for r.process.IsRunning() {
			// Re-get the job and check it's status to see if it's been
			// cancelled
			jobState, _, err := r.APIClient.Jobs.GetState(r.Job.ID)
			if err != nil {
				// We don't really care if it fails, we'll just
				// try again soon anyway
				logger.Warn("Problem with getting job state %s (%s)", r.Job.ID, err)
			} else if jobState.State == "canceling" || jobState.State == "canceled" {
				r.Kill()
			}

			// Check for cancellations
			time.Sleep(time.Duration(r.Agent.JobStatusInterval) * time.Second)
		}

		// Mark this routine as done in the wait group
		r.routineWaitGroup.Done()

		logger.Debug("[JobRunner] Routine that refreshes the job has finished")
	}()
}

func (r *JobRunner) onUploadHeaderTime(cursor int, total int, times map[string]string) {
	retry.Do(func(s *retry.Stats) error {
		_, err := r.APIClient.HeaderTimes.Save(r.Job.ID, &api.HeaderTimes{Times: times})
		if err != nil {
			logger.Warn("%s (%s)", err, s)
		}

		return err
	}, &retry.Config{Maximum: 10, Interval: 5 * time.Second})
}

// Call when a chunk is ready for upload. It retry the chunk upload with an
// interval before giving up.
func (r *JobRunner) onUploadChunk(chunk *LogStreamerChunk) error {
	return retry.Do(func(s *retry.Stats) error {
		_, err := r.APIClient.Chunks.Upload(r.Job.ID, &api.Chunk{
			Data:     chunk.Data,
			Sequence: chunk.Order,
			Offset:   chunk.Offset,
			Size:     chunk.Size,
		})
		if err != nil {
			logger.Warn("%s (%s)", err, s)
		}

		return err
	}, &retry.Config{Maximum: 10, Interval: 5 * time.Second})
}
<|endoftext|>


<commit_before>package dialer

import (
	"log"
	"net"

	"github.com/linkosmosis/godns"
)

// Dialer -
type Dialer struct {
	*net.Dialer

	AddrsPool *godns.Pool
}

// New - initalize dial.Dialer wrapper
func New() *Dialer {
	return &Dialer{
		Dialer:    &net.Dialer{},
		AddrsPool: godns.New(),
	}
}

// Dial - lightweight version of dialer.Dial, this has cached
// dns hostport and shorter TCP connection setup
func (d *Dialer) Dial(network, address string) (net.Conn, error) {
	tcpAddr, err := d.AddrsPool.Get(address)
	if err != nil {
		log.Println("Failed to resolve:", err)
		return d.Dialer.Dial(network, address)
	}
	c, err := net.DialTCP(network, nil, tcpAddr)
	if err != nil {
		log.Println("Failed to setup DialTCP:", err)
		return d.Dialer.Dial(network, address)
	}
	if d.KeepAlive != 0 {
		c.SetKeepAlive(true)
		c.SetKeepAlivePeriod(d.KeepAlive)
		c.SetLinger(0)
		c.SetNoDelay(true)
	}
	return c, err
}
<commit_msg>Dialer log more friendly messages<commit_after>package dialer

import (
	"log"
	"net"

	"github.com/linkosmosis/godns"
)

// Dialer -
type Dialer struct {
	*net.Dialer

	AddrsPool *godns.Pool
}

// New - initalize dial.Dialer wrapper
func New() *Dialer {
	return &Dialer{
		Dialer:    &net.Dialer{},
		AddrsPool: godns.New(),
	}
}

// Dial - lightweight version of dialer.Dial, this has cached
// dns hostport and shorter TCP connection setup
func (d *Dialer) Dial(network, address string) (net.Conn, error) {
	tcpAddr, err := d.AddrsPool.Get(address)
	if err != nil {
		log.Printf("Failed to resolve: %s, fallback dialer.Dial", address)
		return d.Dialer.Dial(network, address)
	}
	c, err := net.DialTCP(network, nil, tcpAddr)
	if err != nil {
		log.Printf("Failed to setup DialTCP: %s, fallback dialer.Dial", err)
		return d.Dialer.Dial(network, address)
	}
	if d.KeepAlive != 0 {
		c.SetKeepAlive(true)
		c.SetKeepAlivePeriod(d.KeepAlive)
		c.SetLinger(0)
		c.SetNoDelay(true)
	}
	return c, err
}
<|endoftext|>