<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"os"
	"sort"
	"strings"
	"time"

	"github.com/tidwall/gjson"
)

var videosMD = "content/page/videos.md"
var videosNoYoutube = "scripts/videosNoYoutube.txt"
var videosLanding = "content/post/vigotech-charlas.md"
var projectsMD = "content/page/proxectos.md"
var projectsLanding = "content/post/vigotech-proxectos.md"
var channelJSON = "scripts/channels.json"
var projectsJSON = "scripts/projects.json"

type channelType []struct {
	ChannelName string `json:"channelName"`
	ChannelID   string `json:"channelID"`
}

var channels = channelType{}

type video struct {
	title, videoID, channel, publishedAt string
}

type videosType []video

func (slice videosType) Len() int {
	return len(slice)
}

func (slice videosType) Less(i, j int) bool {
	return slice[i].publishedAt < slice[j].publishedAt
}

func (slice videosType) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}

var videos = videosType{}

type projectsType []struct {
	User string `json:"user"`
	Repo string `json:"repo"`
}

var projects = projectsType{}

func main() {

	token := os.Getenv("YOUTUBE_TOKEN")
	now := time.Now().Format(time.RFC3339)

	// Read JSON channels
	c, err6 := os.Open(channelJSON)
	if err6 != nil {
		log.Fatal("Not able to open ", channelJSON)
	}

	jsonParser := json.NewDecoder(c)
	if err6 = jsonParser.Decode(&channels); err6 != nil {
		log.Fatal("Parsing channels JSONfile", err6.Error())
	}

	// Read JSON channels
	c1, err7 := os.Open(projectsJSON)
	if err7 != nil {
		log.Fatal("Not able to open ", projectsJSON)
	}

	jsonParser = json.NewDecoder(c1)
	if err7 = jsonParser.Decode(&projects); err7 != nil {
		log.Fatal("Parsing projects JSON file", err7.Error())
	}

	// Prepare Videos Markdown page
	_ = os.Truncate(videosMD, 0)
	var file, err = os.OpenFile(videosMD, os.O_RDWR, 0644)
	if err != nil {
		log.Fatal("Not able to open ", videosMD)
	}
	defer file.Close()

	s := `+++
date = "$now"
draft = false
title = "Charlas"
type = "page"
weight = 1
+++

----
Charlas gravadas polos distintos grupos:

`

	file.WriteString(strings.Replace(s, "$now", now, 1))

	for _, c := range channels {
		// Processing YouTube Requests
		url := fmt.Sprintf("https://www.googleapis.com/youtube/v3/search?"+
			"key=%s&channelId=%s&part=snippet,id&order=date&maxResults=50",
			token, c.ChannelID)

		req, err := http.NewRequest("GET", url, nil)
		if err != nil {
			log.Fatal("NewRequest: ", err)
			return
		}

		client := &http.Client{}

		resp, err := client.Do(req)
		if err != nil {
			log.Fatal("Do: ", err)
			return
		}

		defer resp.Body.Close()

		if resp.StatusCode != 200 {
			log.Fatal("YouTube response is: ", resp.StatusCode)
		}

		bodyBytes, _ := ioutil.ReadAll(resp.Body)
		record := string(bodyBytes)

		totalResults := gjson.Get(record, "pageInfo.totalResults")

		if totalResults.Int() > 49 {
			log.Fatal("Returned 50 videos, we could loose records")
		}
		s1 := fmt.Sprintf(
			"\n\n### [%s](https://www.youtube.com/channel/%s)\n\n",
			c.ChannelName,
			c.ChannelID)
		file.WriteString(s1)

		r := gjson.Get(record, "items")
		for _, item := range r.Array() {

			kind := gjson.Get(item.String(), "id.kind")

			if kind.String() != "youtube#video" {
				continue
			}

			title := gjson.Get(item.String(), "snippet.title").String()
			videoID := gjson.Get(item.String(), "id.videoId").String()
			publishedAt := gjson.Get(
				item.String(),
				"snippet.publishedAt").String()

			s2 := fmt.Sprintf("- [%s](https://www.youtube.com/watch?v=%s)\n",
				title, videoID)
			file.WriteString(s2)

			videos = append(videos, video{
				title:       title,
				videoID:     videoID,
				channel:     c.ChannelName,
				publishedAt: publishedAt,
			})
		}
	}

	// Add videos which aren't hosted in YouTube
	dat, err := ioutil.ReadFile(videosNoYoutube)
	if err != nil {
		log.Fatal("Not able to open ", videosNoYoutube)
	}
	file.WriteString(string(dat))

	// Update the videos landing page
	_ = os.Truncate(videosLanding, 0)
	var fileLanding, err3 = os.OpenFile(videosLanding, os.O_RDWR, 0644)
	if err3 != nil {
		log.Fatal("Not able to open ", videosLanding)
	}
	defer fileLanding.Close()

	s = `+++
date = "$now"
draft = false
title = "Charlas"
weight = 102
type = "post"
class="post last talks"

+++

A maioría de charlas está gravadas e dispoñibles para o seu visionado.
Están son as catro últimas:

`
	fileLanding.WriteString(strings.Replace(s, "$now", now, 1))

	sort.Sort(sort.Reverse(videos))

	for i, v := range videos {

		s = fmt.Sprintf("* [%s](https://www.youtube.com/watch?v=%s) (%s)\n",
			v.title,
			v.videoID,
			v.channel)

		//fileLanding.WriteString(s)

		s = fmt.Sprintf("
"+
			"

\n",
			v.videoID)

		fileLanding.WriteString(s)

		if i == 3 {
			break
		}
	}
	s = "\n\n

\n\n"
	fileLanding.WriteString(s)

	s = `
[Preme aquí para ver tódalas charlas](./page/videos/)
`

	fileLanding.WriteString(s)

	// Prepare Proxectos file output
	_ = os.Truncate(projectsMD, 0)
	var fileProjects, err4 = os.OpenFile(projectsMD, os.O_RDWR, 0644)
	if err4 != nil {
		log.Fatal("Not able to open ", projectsMD)
	}
	defer fileProjects.Close()

	s = `+++
date = "$now"
draft = false
title = "Proxectos"
type = "page"
weight = 1
+++

----
Proxectos de código aberto creados por xente da comunidade:

`

	fileProjects.WriteString(strings.Replace(s, "$now", now, 1))

	fileProjects.WriteString("
\n\n")

	for i, v := range projects {

		if i%2 == 0 {
			fileProjects.WriteString("
\n")
		}

		s = fmt.Sprintf("\n
"+
			"

\n",
			v.User,
			v.Repo)

		fileProjects.WriteString(s)

		if i%2 == 1 || len(projects)-1 == i {
			fileProjects.WriteString("
\n")
		}
	}
	fileProjects.WriteString("
\n\n")

	fileProjects.WriteString("")

	// Update the projects landing page
	_ = os.Truncate(projectsLanding, 0)
	var fileProjectsLanding, err5 = os.OpenFile(
		projectsLanding, os.O_RDWR, 0644)
	if err5 != nil {
		log.Fatal("Not able to open ", projectsLanding)
	}
	defer fileProjectsLanding.Close()

	s = `+++
date = "$now"
draft = false
title = "Proxectos"
weight = 103
type = "post"
class="post last projects"

+++

Proxectos de código aberto creados por xente da comunidade:
`
	fileProjectsLanding.WriteString(strings.Replace(s, "$now", now, 1))

	rand.Seed(time.Now().Unix())
	random := rand.Int()

	fileProjectsLanding.WriteString("
\n\n")

	for i := 0; i < 6; i++ {

		if i%2 == 0 {
			fileProjectsLanding.WriteString("
\n")
		}

		n := (random + i) % len(projects)
		s = fmt.Sprintf("\n
"+
			"

\n",
			projects[n].User,
			projects[n].Repo)

		fileProjectsLanding.WriteString(s)

		if i%2 == 1 {
			fileProjectsLanding.WriteString("
\n")
		}
	}

	fileProjectsLanding.WriteString("
\n\n")

	s = `

[Preme aquí para ver tódolos proxectos](./page/proxectos/)

`
	fileProjectsLanding.WriteString(s)

}
<commit_msg>Temporary disable projects<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"os"
	"sort"
	"strings"
	"time"

	"github.com/tidwall/gjson"
)

var videosMD = "content/page/videos.md"
var videosNoYoutube = "scripts/videosNoYoutube.txt"
var videosLanding = "content/post/vigotech-charlas.md"
var projectsMD = "content/page/proxectos.md"
var projectsLanding = "content/post/vigotech-proxectos.md"
var channelJSON = "scripts/channels.json"
var projectsJSON = "scripts/projects.json"

type channelType []struct {
	ChannelName string `json:"channelName"`
	ChannelID   string `json:"channelID"`
}

var channels = channelType{}

type video struct {
	title, videoID, channel, publishedAt string
}

type videosType []video

func (slice videosType) Len() int {
	return len(slice)
}

func (slice videosType) Less(i, j int) bool {
	return slice[i].publishedAt < slice[j].publishedAt
}

func (slice videosType) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}

var videos = videosType{}

type projectsType []struct {
	User string `json:"user"`
	Repo string `json:"repo"`
}

var projects = projectsType{}

func main() {

	token := os.Getenv("YOUTUBE_TOKEN")
	now := time.Now().Format(time.RFC3339)

	// Read JSON channels
	c, err6 := os.Open(channelJSON)
	if err6 != nil {
		log.Fatal("Not able to open ", channelJSON)
	}

	jsonParser := json.NewDecoder(c)
	if err6 = jsonParser.Decode(&channels); err6 != nil {
		log.Fatal("Parsing channels JSONfile", err6.Error())
	}

	// Read JSON channels
	c1, err7 := os.Open(projectsJSON)
	if err7 != nil {
		log.Fatal("Not able to open ", projectsJSON)
	}

	jsonParser = json.NewDecoder(c1)
	if err7 = jsonParser.Decode(&projects); err7 != nil {
		log.Fatal("Parsing projects JSON file", err7.Error())
	}

	// Prepare Videos Markdown page
	_ = os.Truncate(videosMD, 0)
	var file, err = os.OpenFile(videosMD, os.O_RDWR, 0644)
	if err != nil {
		log.Fatal("Not able to open ", videosMD)
	}
	defer file.Close()

	s := `+++
date = "$now"
draft = false
title = "Charlas"
type = "page"
weight = 1
+++

----
Charlas gravadas polos distintos grupos:

`

	file.WriteString(strings.Replace(s, "$now", now, 1))

	for _, c := range channels {
		// Processing YouTube Requests
		url := fmt.Sprintf("https://www.googleapis.com/youtube/v3/search?"+
			"key=%s&channelId=%s&part=snippet,id&order=date&maxResults=50",
			token, c.ChannelID)

		req, err := http.NewRequest("GET", url, nil)
		if err != nil {
			log.Fatal("NewRequest: ", err)
			return
		}

		client := &http.Client{}

		resp, err := client.Do(req)
		if err != nil {
			log.Fatal("Do: ", err)
			return
		}

		defer resp.Body.Close()

		if resp.StatusCode != 200 {
			log.Fatal("YouTube response is: ", resp.StatusCode)
		}

		bodyBytes, _ := ioutil.ReadAll(resp.Body)
		record := string(bodyBytes)

		totalResults := gjson.Get(record, "pageInfo.totalResults")

		if totalResults.Int() > 49 {
			log.Fatal("Returned 50 videos, we could loose records")
		}
		s1 := fmt.Sprintf(
			"\n\n### [%s](https://www.youtube.com/channel/%s)\n\n",
			c.ChannelName,
			c.ChannelID)
		file.WriteString(s1)

		r := gjson.Get(record, "items")
		for _, item := range r.Array() {

			kind := gjson.Get(item.String(), "id.kind")

			if kind.String() != "youtube#video" {
				continue
			}

			title := gjson.Get(item.String(), "snippet.title").String()
			videoID := gjson.Get(item.String(), "id.videoId").String()
			publishedAt := gjson.Get(
				item.String(),
				"snippet.publishedAt").String()

			s2 := fmt.Sprintf("- [%s](https://www.youtube.com/watch?v=%s)\n",
				title, videoID)
			file.WriteString(s2)

			videos = append(videos, video{
				title:       title,
				videoID:     videoID,
				channel:     c.ChannelName,
				publishedAt: publishedAt,
			})
		}
	}

	// Add videos which aren't hosted in YouTube
	dat, err := ioutil.ReadFile(videosNoYoutube)
	if err != nil {
		log.Fatal("Not able to open ", videosNoYoutube)
	}
	file.WriteString(string(dat))

	// Update the videos landing page
	_ = os.Truncate(videosLanding, 0)
	var fileLanding, err3 = os.OpenFile(videosLanding, os.O_RDWR, 0644)
	if err3 != nil {
		log.Fatal("Not able to open ", videosLanding)
	}
	defer fileLanding.Close()

	s = `+++
date = "$now"
draft = false
title = "Charlas"
weight = 102
type = "post"
class="post last talks"

+++

A maioría de charlas está gravadas e dispoñibles para o seu visionado.
Están son as catro últimas:

`
	fileLanding.WriteString(strings.Replace(s, "$now", now, 1))

	sort.Sort(sort.Reverse(videos))

	for i, v := range videos {

		s = fmt.Sprintf("* [%s](https://www.youtube.com/watch?v=%s) (%s)\n",
			v.title,
			v.videoID,
			v.channel)

		//fileLanding.WriteString(s)

		s = fmt.Sprintf("
"+
			"

\n",
			v.videoID)

		fileLanding.WriteString(s)

		if i == 3 {
			break
		}
	}
	s = "\n\n

\n\n"
	fileLanding.WriteString(s)

	s = `
[Preme aquí para ver tódalas charlas](./page/videos/)
`

	fileLanding.WriteString(s)

	// Prepare Proxectos file output
	_ = os.Truncate(projectsMD, 0)
	var fileProjects, err4 = os.OpenFile(projectsMD, os.O_RDWR, 0644)
	if err4 != nil {
		log.Fatal("Not able to open ", projectsMD)
	}
	defer fileProjects.Close()

	s = `+++
date = "$now"
draft = false
title = "Proxectos"
type = "page"
weight = 1
+++

----
Proxectos de código aberto creados por xente da comunidade:

`

	fileProjects.WriteString(strings.Replace(s, "$now", now, 1))

	fileProjects.WriteString("
\n\n")

	for i, v := range projects {

		if i%2 == 0 {
			fileProjects.WriteString("
\n")
		}

		s = fmt.Sprintf("\n
"+
			"

\n",
			v.User,
			v.Repo)

		fileProjects.WriteString(s)

		if i%2 == 1 || len(projects)-1 == i {
			fileProjects.WriteString("
\n")
		}
	}
	fileProjects.WriteString("
\n\n")

	fileProjects.WriteString("")

	// Update the projects landing page
	_ = os.Truncate(projectsLanding, 0)
	var fileProjectsLanding, err5 = os.OpenFile(
		projectsLanding, os.O_RDWR, 0644)
	if err5 != nil {
		log.Fatal("Not able to open ", projectsLanding)
	}
	defer fileProjectsLanding.Close()

	s = `+++
date = "$now"
draft = true
title = "Proxectos"
weight = 103
type = "post"
class="post last projects"

+++

Proxectos de código aberto creados por xente da comunidade:
`
	fileProjectsLanding.WriteString(strings.Replace(s, "$now", now, 1))

	rand.Seed(time.Now().Unix())
	random := rand.Int()

	fileProjectsLanding.WriteString("
\n\n")

	for i := 0; i < 6; i++ {

		if i%2 == 0 {
			fileProjectsLanding.WriteString("
\n")
		}

		n := (random + i) % len(projects)
		s = fmt.Sprintf("\n
"+
			"

\n",
			projects[n].User,
			projects[n].Repo)

		fileProjectsLanding.WriteString(s)

		if i%2 == 1 {
			fileProjectsLanding.WriteString("
\n")
		}
	}

	fileProjectsLanding.WriteString("
\n\n")

	s = `

[Preme aquí para ver tódolos proxectos](./page/proxectos/)

`
	fileProjectsLanding.WriteString(s)

}
<|endoftext|>


<commit_before>package sexpconv

import (
	"exn"
	"go/ast"
	"go/token"
	"go/types"
	"magic_pkg/emacs/rt"
	"sexp"
	"sys_info/function"
	"xtypes"
)

func (conv *converter) Expr(node ast.Expr) sexp.Form {
	switch node := node.(type) {
	case *ast.ParenExpr:
		return conv.Expr(node.X)
	case *ast.Ident:
		return conv.Ident(node)
	case *ast.BasicLit:
		return conv.Constant(node)
	case *ast.BinaryExpr:
		return conv.BinaryExpr(node)
	case *ast.CallExpr:
		return conv.CallExpr(node)
	case *ast.SelectorExpr:
		return conv.SelectorExpr(node)
	case *ast.TypeAssertExpr:
		return conv.TypeAssertExpr(node)
	case *ast.IndexExpr:
		return conv.IndexExpr(node)
	case *ast.UnaryExpr:
		return conv.UnaryExpr(node)
	case *ast.CompositeLit:
		return conv.CompositeLit(node)
	case *ast.SliceExpr:
		return conv.SliceExpr(node)

	default:
		panic(errUnexpectedExpr(conv, node))
	}
}

func (conv *converter) Ident(node *ast.Ident) sexp.Form {
	if cv := conv.Constant(node); cv != nil {
		return cv
	}

	obj := conv.info.Uses[node]
	typ := obj.Type()

	if typ, ok := typ.(*types.Basic); ok {
		// Coerce untyped nil to correct value depending on
		// the context type.
		if typ.Kind() == types.UntypedNil {
			switch conv.ctxType.(type) {
			case *types.Map:
				return nilMap
			case *types.Slice:
				return nilSlice
			case *types.Signature:
				return nilFunc
			case *types.Interface:
				return nilInterface
			}
		}
	}

	if xtypes.IsGlobal(obj) {
		return sexp.Var{Name: conv.env.InternVar(nil, node.Name), Typ: typ}
	}
	return sexp.Var{Name: node.Name, Typ: typ}
}

func (conv *converter) BinaryExpr(node *ast.BinaryExpr) sexp.Form {
	if cv := conv.Constant(node); cv != nil {
		return cv
	}

	typ := conv.basicTypeOf(node.X)
	x, y := conv.Expr(node.X), conv.Expr(node.Y)

	if typ.Info()&types.IsNumeric != 0 {
		switch node.Op {
		case token.ADD:
			return sexp.NewAdd(x, y)
		case token.SUB:
			return sexp.NewSub(x, y)
		case token.MUL:
			return sexp.NewMul(x, y)
		case token.QUO:
			return sexp.NewQuo(x, y)
		case token.EQL:
			return sexp.NewNumEq(x, y)
		case token.NEQ:
			return sexp.NewNumNeq(x, y)
		case token.LSS:
			return sexp.NewNumLt(x, y)
		case token.GTR:
			return sexp.NewNumGt(x, y)
		case token.LEQ:
			return sexp.NewNumLte(x, y)
		case token.GEQ:
			return sexp.NewNumGte(x, y)
		case token.AND:
			return sexp.NewBitAnd(x, y)
		case token.OR:
			return sexp.NewBitOr(x, y)
		case token.SHL:
			return sexp.NewShl(x, y)
		case token.SHR:
			return sexp.NewShr(x, y)

		default:
			panic(errUnexpectedExpr(conv, node))
		}
	}

	if typ.Kind() == types.String {
		switch node.Op {
		case token.ADD:
			return sexp.NewConcat(x, y)
		case token.EQL:
			return sexp.NewStrEq(x, y)
		case token.LSS:
			return sexp.NewStrLt(x, y)
		case token.GTR:
			return sexp.NewStrGt(x, y)
		case token.LEQ:
			panic(errUnexpectedExpr(conv, node))
			// return sexp.NewStrLte(x, y)
		case token.GEQ:
			panic(errUnexpectedExpr(conv, node))
			// return sexp.NewStrGte(x, y)

		default:
			panic(errUnexpectedExpr(conv, node))
		}
	}

	if node.Op == token.LAND {
		return &sexp.And{X: x, Y: y}
	}
	if node.Op == token.LOR {
		return &sexp.Or{X: x, Y: y}
	}

	panic(errUnexpectedExpr(conv, node))
}

func (conv *converter) structIndex(typ *types.Struct, node *ast.SelectorExpr) *sexp.StructIndex {
	return &sexp.StructIndex{
		Struct: conv.Expr(node.X),
		Index:  xtypes.LookupField(node.Sel.Name, typ),
		Typ:    typ,
	}
}

func (conv *converter) SelectorExpr(node *ast.SelectorExpr) sexp.Form {
	if cv := conv.Constant(node); cv != nil {
		return cv
	}

	typ := conv.typeOf(node.X)
	if typ, ok := typ.Underlying().(*types.Struct); ok {
		return conv.structIndex(typ, node)
	}
	if typ, ok := typ.(*types.Pointer); ok {
		// Single level of indirection for structs is the same
		// as not having indirection at all.
		if derefTyp, ok := typ.Elem().Underlying().(*types.Struct); ok {
			return conv.structIndex(derefTyp, node)
		}
	}

	id, ok := node.X.(*ast.Ident)
	if !ok {
		panic(errUnexpectedExpr(conv, node))
	}
	switch obj := conv.info.Uses[id].(type) {
	case *types.PkgName:
		return sexp.Symbol{
			Val: conv.env.InternVar(obj.Imported(), node.Sel.Name),
		}
	default:
		panic(errUnexpectedExpr(conv, node))
	}
}

func (conv *converter) UnaryExpr(node *ast.UnaryExpr) sexp.Form {
	if cv := conv.Constant(node); cv != nil {
		return cv
	}

	x := conv.Expr(node.X)

	switch node.Op {
	case token.NOT:
		return sexp.NewNot(x)
	case token.SUB:
		return sexp.NewNeg(x)
	case token.ADD:
		return x
	case token.AND:
		return conv.takeAddr(x)
	}

	panic(errUnexpectedExpr(conv, node))
}

func (conv *converter) takeAddr(form sexp.Form) sexp.Form {
	if xtypes.IsStruct(form.Type()) {
		return form // #REFS: 25
	}

	panic(exn.NoImpl("take address operation"))
}

func (conv *converter) TypeAssertExpr(node *ast.TypeAssertExpr) sexp.Form {
	expr := conv.Expr(node.X)
	assertTyp := conv.typeOf(node.Type)
	return &sexp.TypeAssert{Expr: expr, Typ: assertTyp}
}

func (conv *converter) IndexExpr(node *ast.IndexExpr) sexp.Form {
	switch typ := conv.typeOf(node.X).(type) {
	case *types.Map:
		return &sexp.LispCall{
			Fn: function.Gethash,
			Args: conv.valueCopyList([]sexp.Form{
				conv.Expr(node.Index),
				conv.Expr(node.X),
				ZeroValue(typ.Elem()),
			}),
		}

	case *types.Array:
		return &sexp.ArrayIndex{
			Array: conv.Expr(node.X),
			Index: conv.Expr(node.Index),
		}

	case *types.Slice:
		return conv.call(rt.FnSliceGet, node.X, node.Index)

	// #TODO: strings
	default:
		panic(errUnexpectedExpr(conv, node))
	}
}

func (conv *converter) SliceExpr(node *ast.SliceExpr) sexp.Form {
	var low, high sexp.Form
	if node.Low != nil {
		low = conv.Expr(node.Low)
	}
	if node.High != nil {
		high = conv.Expr(node.High)
	}
	x := conv.Expr(node.X)
	switch conv.typeOf(node.X).Underlying().(type) {
	case *types.Array:
		return sexp.NewArraySlice(x, low, high)
	case *types.Basic:
		return sexp.NewSubstr(x, low, high)
	default:
		return sexp.NewSubslice(x, low, high)
	}
}

func (conv *converter) CompositeLit(node *ast.CompositeLit) sexp.Form {
	switch typ := conv.typeOf(node).(type) {
	case *types.Array:
		return conv.arrayLit(node, typ)
	case *types.Slice:
		return conv.sliceLit(node, typ)
	case *types.Named:
		return conv.structLit(node, typ.Underlying().(*types.Struct))

	default:
		panic(errUnexpectedExpr(conv, node))
	}
}

func (conv *converter) sliceLit(node *ast.CompositeLit, typ *types.Slice) sexp.Form {
	if len(node.Elts) == 0 {
		return ZeroValue(typ)
	}

	return &sexp.SliceLit{Vals: conv.exprList(node.Elts), Typ: typ}
}

func (conv *converter) arrayLit(node *ast.CompositeLit, typ *types.Array) sexp.Form {
	if len(node.Elts) == 0 {
		return ZeroValue(typ)
	}

	if len(node.Elts) != int(typ.Len()) {
		vals := make(map[int]sexp.Form, len(node.Elts))
		for i, elt := range node.Elts {
			vals[i] = conv.Expr(elt)
		}
		ctor := &sexp.LispCall{
			Fn: function.MakeVector,
			Args: []sexp.Form{
				sexp.Int(typ.Len()),
				ZeroValue(typ.Elem()),
			},
		}
		return &sexp.SparseArrayLit{
			Vals: vals,
			Ctor: ctor,
			Typ:  typ,
		}
	}

	// Each element has explicit initializer.
	return &sexp.ArrayLit{Vals: conv.exprList(node.Elts), Typ: typ}
}

func (conv *converter) structLit(node *ast.CompositeLit, typ *types.Struct) sexp.Form {
	vals := make([]sexp.Form, typ.NumFields())
	for _, elt := range node.Elts {
		kv := elt.(*ast.KeyValueExpr)
		key := kv.Key.(*ast.Ident)
		vals[xtypes.LookupField(key.Name, typ)] = conv.Expr(kv.Value)
	}
	for i, val := range vals {
		if val == nil {
			vals[i] = ZeroValue(typ.Field(i).Type())
		}
	}
	return &sexp.StructLit{Vals: vals, Typ: typ}
}
<commit_msg>struct composite lit initializers are copied<commit_after>package sexpconv

import (
	"exn"
	"go/ast"
	"go/token"
	"go/types"
	"magic_pkg/emacs/rt"
	"sexp"
	"sys_info/function"
	"xtypes"
)

func (conv *converter) Expr(node ast.Expr) sexp.Form {
	switch node := node.(type) {
	case *ast.ParenExpr:
		return conv.Expr(node.X)
	case *ast.Ident:
		return conv.Ident(node)
	case *ast.BasicLit:
		return conv.Constant(node)
	case *ast.BinaryExpr:
		return conv.BinaryExpr(node)
	case *ast.CallExpr:
		return conv.CallExpr(node)
	case *ast.SelectorExpr:
		return conv.SelectorExpr(node)
	case *ast.TypeAssertExpr:
		return conv.TypeAssertExpr(node)
	case *ast.IndexExpr:
		return conv.IndexExpr(node)
	case *ast.UnaryExpr:
		return conv.UnaryExpr(node)
	case *ast.CompositeLit:
		return conv.CompositeLit(node)
	case *ast.SliceExpr:
		return conv.SliceExpr(node)

	default:
		panic(errUnexpectedExpr(conv, node))
	}
}

func (conv *converter) Ident(node *ast.Ident) sexp.Form {
	if cv := conv.Constant(node); cv != nil {
		return cv
	}

	obj := conv.info.Uses[node]
	typ := obj.Type()

	if typ, ok := typ.(*types.Basic); ok {
		// Coerce untyped nil to correct value depending on
		// the context type.
		if typ.Kind() == types.UntypedNil {
			switch conv.ctxType.(type) {
			case *types.Map:
				return nilMap
			case *types.Slice:
				return nilSlice
			case *types.Signature:
				return nilFunc
			case *types.Interface:
				return nilInterface
			}
		}
	}

	if xtypes.IsGlobal(obj) {
		return sexp.Var{Name: conv.env.InternVar(nil, node.Name), Typ: typ}
	}
	return sexp.Var{Name: node.Name, Typ: typ}
}

func (conv *converter) BinaryExpr(node *ast.BinaryExpr) sexp.Form {
	if cv := conv.Constant(node); cv != nil {
		return cv
	}

	typ := conv.basicTypeOf(node.X)
	x, y := conv.Expr(node.X), conv.Expr(node.Y)

	if typ.Info()&types.IsNumeric != 0 {
		switch node.Op {
		case token.ADD:
			return sexp.NewAdd(x, y)
		case token.SUB:
			return sexp.NewSub(x, y)
		case token.MUL:
			return sexp.NewMul(x, y)
		case token.QUO:
			return sexp.NewQuo(x, y)
		case token.EQL:
			return sexp.NewNumEq(x, y)
		case token.NEQ:
			return sexp.NewNumNeq(x, y)
		case token.LSS:
			return sexp.NewNumLt(x, y)
		case token.GTR:
			return sexp.NewNumGt(x, y)
		case token.LEQ:
			return sexp.NewNumLte(x, y)
		case token.GEQ:
			return sexp.NewNumGte(x, y)
		case token.AND:
			return sexp.NewBitAnd(x, y)
		case token.OR:
			return sexp.NewBitOr(x, y)
		case token.SHL:
			return sexp.NewShl(x, y)
		case token.SHR:
			return sexp.NewShr(x, y)

		default:
			panic(errUnexpectedExpr(conv, node))
		}
	}

	if typ.Kind() == types.String {
		switch node.Op {
		case token.ADD:
			return sexp.NewConcat(x, y)
		case token.EQL:
			return sexp.NewStrEq(x, y)
		case token.LSS:
			return sexp.NewStrLt(x, y)
		case token.GTR:
			return sexp.NewStrGt(x, y)
		case token.LEQ:
			panic(errUnexpectedExpr(conv, node))
			// return sexp.NewStrLte(x, y)
		case token.GEQ:
			panic(errUnexpectedExpr(conv, node))
			// return sexp.NewStrGte(x, y)

		default:
			panic(errUnexpectedExpr(conv, node))
		}
	}

	if node.Op == token.LAND {
		return &sexp.And{X: x, Y: y}
	}
	if node.Op == token.LOR {
		return &sexp.Or{X: x, Y: y}
	}

	panic(errUnexpectedExpr(conv, node))
}

func (conv *converter) structIndex(typ *types.Struct, node *ast.SelectorExpr) *sexp.StructIndex {
	return &sexp.StructIndex{
		Struct: conv.Expr(node.X),
		Index:  xtypes.LookupField(node.Sel.Name, typ),
		Typ:    typ,
	}
}

func (conv *converter) SelectorExpr(node *ast.SelectorExpr) sexp.Form {
	if cv := conv.Constant(node); cv != nil {
		return cv
	}

	typ := conv.typeOf(node.X)
	if typ, ok := typ.Underlying().(*types.Struct); ok {
		return conv.structIndex(typ, node)
	}
	if typ, ok := typ.(*types.Pointer); ok {
		// Single level of indirection for structs is the same
		// as not having indirection at all.
		if derefTyp, ok := typ.Elem().Underlying().(*types.Struct); ok {
			return conv.structIndex(derefTyp, node)
		}
	}

	id, ok := node.X.(*ast.Ident)
	if !ok {
		panic(errUnexpectedExpr(conv, node))
	}
	switch obj := conv.info.Uses[id].(type) {
	case *types.PkgName:
		return sexp.Symbol{
			Val: conv.env.InternVar(obj.Imported(), node.Sel.Name),
		}
	default:
		panic(errUnexpectedExpr(conv, node))
	}
}

func (conv *converter) UnaryExpr(node *ast.UnaryExpr) sexp.Form {
	if cv := conv.Constant(node); cv != nil {
		return cv
	}

	x := conv.Expr(node.X)

	switch node.Op {
	case token.NOT:
		return sexp.NewNot(x)
	case token.SUB:
		return sexp.NewNeg(x)
	case token.ADD:
		return x
	case token.AND:
		return conv.takeAddr(x)
	}

	panic(errUnexpectedExpr(conv, node))
}

func (conv *converter) takeAddr(form sexp.Form) sexp.Form {
	if xtypes.IsStruct(form.Type()) {
		return form // #REFS: 25
	}

	panic(exn.NoImpl("take address operation"))
}

func (conv *converter) TypeAssertExpr(node *ast.TypeAssertExpr) sexp.Form {
	expr := conv.Expr(node.X)
	assertTyp := conv.typeOf(node.Type)
	return &sexp.TypeAssert{Expr: expr, Typ: assertTyp}
}

func (conv *converter) IndexExpr(node *ast.IndexExpr) sexp.Form {
	switch typ := conv.typeOf(node.X).(type) {
	case *types.Map:
		return &sexp.LispCall{
			Fn: function.Gethash,
			Args: conv.valueCopyList([]sexp.Form{
				conv.Expr(node.Index),
				conv.Expr(node.X),
				ZeroValue(typ.Elem()),
			}),
		}

	case *types.Array:
		return &sexp.ArrayIndex{
			Array: conv.Expr(node.X),
			Index: conv.Expr(node.Index),
		}

	case *types.Slice:
		return conv.call(rt.FnSliceGet, node.X, node.Index)

	// #TODO: strings
	default:
		panic(errUnexpectedExpr(conv, node))
	}
}

func (conv *converter) SliceExpr(node *ast.SliceExpr) sexp.Form {
	var low, high sexp.Form
	if node.Low != nil {
		low = conv.Expr(node.Low)
	}
	if node.High != nil {
		high = conv.Expr(node.High)
	}
	x := conv.Expr(node.X)
	switch conv.typeOf(node.X).Underlying().(type) {
	case *types.Array:
		return sexp.NewArraySlice(x, low, high)
	case *types.Basic:
		return sexp.NewSubstr(x, low, high)
	default:
		return sexp.NewSubslice(x, low, high)
	}
}

func (conv *converter) CompositeLit(node *ast.CompositeLit) sexp.Form {
	switch typ := conv.typeOf(node).(type) {
	case *types.Array:
		return conv.arrayLit(node, typ)
	case *types.Slice:
		return conv.sliceLit(node, typ)
	case *types.Named:
		return conv.structLit(node, typ.Underlying().(*types.Struct))

	default:
		panic(errUnexpectedExpr(conv, node))
	}
}

func (conv *converter) sliceLit(node *ast.CompositeLit, typ *types.Slice) sexp.Form {
	if len(node.Elts) == 0 {
		return ZeroValue(typ)
	}

	return &sexp.SliceLit{Vals: conv.exprList(node.Elts), Typ: typ}
}

func (conv *converter) arrayLit(node *ast.CompositeLit, typ *types.Array) sexp.Form {
	if len(node.Elts) == 0 {
		return ZeroValue(typ)
	}

	if len(node.Elts) != int(typ.Len()) {
		vals := make(map[int]sexp.Form, len(node.Elts))
		for i, elt := range node.Elts {
			vals[i] = conv.Expr(elt)
		}
		ctor := &sexp.LispCall{
			Fn: function.MakeVector,
			Args: []sexp.Form{
				sexp.Int(typ.Len()),
				ZeroValue(typ.Elem()),
			},
		}
		return &sexp.SparseArrayLit{
			Vals: vals,
			Ctor: ctor,
			Typ:  typ,
		}
	}

	// Each element has explicit initializer.
	return &sexp.ArrayLit{Vals: conv.exprList(node.Elts), Typ: typ}
}

func (conv *converter) structLit(node *ast.CompositeLit, typ *types.Struct) sexp.Form {
	vals := make([]sexp.Form, typ.NumFields())
	for _, elt := range node.Elts {
		kv := elt.(*ast.KeyValueExpr)
		key := kv.Key.(*ast.Ident)
		vals[xtypes.LookupField(key.Name, typ)] = conv.valueCopy(conv.Expr(kv.Value))
	}
	for i, val := range vals {
		if val == nil {
			vals[i] = ZeroValue(typ.Field(i).Type())
		}
	}
	return &sexp.StructLit{Vals: vals, Typ: typ}
}
<|endoftext|>


<commit_before>package ghw

import (
    "testing"
)

func TestTotalPhysicalBytes(t *testing.T) {
    tpb := memTotalPhysicalBytes()

    if tpb < 1 {
        t.Errorf("Expected >0 total physical memory, got %d", tpb)
    }
}

func TestTotalUsableBytes(t *testing.T) {
    tpb := memTotalUsableBytes()

    if tpb < 1 {
        t.Errorf("Expected >0 total usable memory, got %d", tpb)
    }
}
<commit_msg>linux: add unit test for supported page sizes<commit_after>package ghw

import (
    "testing"
)

func TestTotalPhysicalBytes(t *testing.T) {
    tpb := memTotalPhysicalBytes()

    if tpb < 1 {
        t.Errorf("Expected >0 total physical memory, got %d", tpb)
    }
}

func TestTotalUsableBytes(t *testing.T) {
    tpb := memTotalUsableBytes()

    if tpb < 1 {
        t.Errorf("Expected >0 total usable memory, got %d", tpb)
    }
}

func TestSupportedPageSizes(t *testing.T) {
    sps := memSupportedPageSizes()

    if sps == nil {
        t.Errorf("Expected non-nil supported page sizes, but got nil")
    }
    if len(sps) == 0 {
        t.Errorf("Expected >0 supported page sizes, but got 0.")
    }
}
<|endoftext|>


<commit_before>package minhash

import (
	"math"
	"math/big"
	"math/rand"
)

var bitMask = uint32(0x1)
var hashFuncNum = 128

func minKey(l map[string]uint32) (string, uint32) {
	var result string
	m := uint32(math.MaxUint32)
	for k := range l {
		if m > l[k] {
			m = l[k]
			result = k
		}
	}
	return result, m
}

func minHash(data []string, seed uint32) uint32 {
	vector := make(map[string]uint32)
	for k := range data {
		vector[data[k]] = Murmurhash3_32(data[k], seed)
	}
	_, value := minKey(vector)
	return value
}

func signatureInt32(data []string) uint32 {
	rand.Seed(1)
	sig := uint32(0)
	for i := 0; i < hashFuncNum; i++ {
		sig += (minHash(data, rand.Uint32()) & bitMask) << uint32(i)
	}
	return sig
}

func Signature(data []string) *big.Int {
	rand.Seed(1)
	sigBig := big.NewInt(0)
	for i := 0; i < hashFuncNum; i++ {
		sigBig.SetBit(sigBig, i, uint(minHash(data, rand.Uint32())&bitMask))
	}
	return sigBig
}

func popCount(bits uint32) uint32 {
	bits = (bits & 0x55555555) + (bits >> 1 & 0x55555555)
	bits = (bits & 0x33333333) + (bits >> 2 & 0x33333333)
	bits = (bits & 0x0f0f0f0f) + (bits >> 4 & 0x0f0f0f0f)
	bits = (bits & 0x00ff00ff) + (bits >> 8 & 0x00ff00ff)
	return (bits & 0x0000ffff) + (bits >> 16 & 0x0000ffff)
}

func popCountBig(bits *big.Int) int {
	result := 0
	for _, v := range bits.Bytes() {
		result += int(popCount(uint32(v)))
	}
	return result
}

func MinhashFromSignature(sig1, sig2 *big.Int) float32 {
	commonBig := big.NewInt(0)
	commonBig.Xor(sig1, sig2)
	return 2.0 * (float32((hashFuncNum-popCountBig(commonBig)))/float32(hashFuncNum) - 0.5)
}

func Minhash(v1, v2 []string) float32 {
	return MinhashFromSignature(Signature(v1), Signature(v2))
}
<commit_msg>remove unused func<commit_after>package minhash

import (
	"math"
	"math/big"
	"math/rand"
)

var bitMask = uint32(0x1)
var hashFuncNum = 128

func minKey(l map[string]uint32) (string, uint32) {
	var result string
	m := uint32(math.MaxUint32)
	for k := range l {
		if m > l[k] {
			m = l[k]
			result = k
		}
	}
	return result, m
}

func minHash(data []string, seed uint32) uint32 {
	vector := make(map[string]uint32)
	for k := range data {
		vector[data[k]] = Murmurhash3_32(data[k], seed)
	}
	_, value := minKey(vector)
	return value
}

func Signature(data []string) *big.Int {
	rand.Seed(1)
	sigBig := big.NewInt(0)
	for i := 0; i < hashFuncNum; i++ {
		sigBig.SetBit(sigBig, i, uint(minHash(data, rand.Uint32())&bitMask))
	}
	return sigBig
}

func popCount(bits uint32) uint32 {
	bits = (bits & 0x55555555) + (bits >> 1 & 0x55555555)
	bits = (bits & 0x33333333) + (bits >> 2 & 0x33333333)
	bits = (bits & 0x0f0f0f0f) + (bits >> 4 & 0x0f0f0f0f)
	bits = (bits & 0x00ff00ff) + (bits >> 8 & 0x00ff00ff)
	return (bits & 0x0000ffff) + (bits >> 16 & 0x0000ffff)
}

func popCountBig(bits *big.Int) int {
	result := 0
	for _, v := range bits.Bytes() {
		result += int(popCount(uint32(v)))
	}
	return result
}

func MinhashFromSignature(sig1, sig2 *big.Int) float32 {
	commonBig := big.NewInt(0)
	commonBig.Xor(sig1, sig2)
	return 2.0 * (float32((hashFuncNum-popCountBig(commonBig)))/float32(hashFuncNum) - 0.5)
}

func Minhash(v1, v2 []string) float32 {
	return MinhashFromSignature(Signature(v1), Signature(v2))
}
<|endoftext|>


<commit_before>package lasr

import (
	"bytes"
	"context"
	"sync"
	"testing"

	"github.com/boltdb/bolt"
)

func TestSendReceive(t *testing.T) {
	q, cleanup := newQ(t)
	defer cleanup()

	msg := []byte("foobar")
	id, err := q.Send(msg)
	if err != nil {
		t.Fatal(err)
	}
	idb, err := id.MarshalBinary()
	if err != nil {
		t.Fatal(err)
	}
	message, err := q.Receive(context.Background())
	if err != nil {
		t.Fatal(err)
	}
	if !bytes.Equal(message.ID, idb) {
		t.Error("messages do not match")
	}
	if !bytes.Equal(message.Body, msg) {
		t.Error("messages do not match")
	}
	if err := message.Ack(); err != nil {
		t.Fatal(err)
	}
}

func TestSendReceiveNackNoRetry(t *testing.T) {
	q, cleanup := newQ(t)
	defer cleanup()

	msg := []byte("foobar")
	if _, err := q.Send(msg); err != nil {
		t.Fatal(err)
	}
	message, err := q.Receive(context.Background())
	if err != nil {
		t.Fatal(err)
	}
	if err := message.Nack(false); err != nil {
		t.Fatal(err)
	}
	if err := message.Ack(); err != ErrAckNack {
		t.Error("expected ErrAckNack")
	}
}

func TestSendReceiveNackWithRetry(t *testing.T) {
	q, cleanup := newQ(t)
	defer cleanup()

	msg := []byte("foobar")
	id, err := q.Send(msg)
	if err != nil {
		t.Fatal(err)
	}
	idb, err := id.MarshalBinary()
	if err != nil {
		t.Fatal(err)
	}
	message, err := q.Receive(context.Background())
	if err != nil {
		t.Fatal(err)
	}
	if err := message.Nack(true); err != nil {
		t.Fatal(err)
	}
	message, err = q.Receive(context.Background())
	if err != nil {
		t.Fatal(err)
	}
	if !bytes.Equal(message.ID, idb) {
		t.Errorf("bad id: %v", message.ID)
	}
	if !bytes.Equal(message.Body, msg) {
		t.Errorf("bad body: %q", string(message.Body))
	}
	if err := message.Ack(); err != nil {
		t.Fatal(err)
	}
}

func TestReceiveContextCancelled(t *testing.T) {
	q, cleanup := newQ(t)
	defer cleanup()

	ctx, cancel := context.WithCancel(context.Background())

	done := make(chan struct{})
	go func() {
		defer close(done)
		q.Receive(ctx)
	}()
	cancel()
	<-done // blocks forever if the Receive never gets cancelled
}

func TestUnacked(t *testing.T) {
	q, cleanup := newQ(t)
	defer cleanup()

	msg := []byte("foo")
	id, err := q.Send(msg)
	if err != nil {
		t.Fatal(err)
	}
	idb, err := id.MarshalBinary()
	if err != nil {
		t.Fatal(err)
	}

	if _, err := q.Receive(context.Background()); err != nil {
		t.Fatal(err)
	}

	// deadlock unless we clear the wg here
	q.inFlight = sync.WaitGroup{}

	// Make sure the unacked message is in the unacked queue
	err = q.db.Update(func(tx *bolt.Tx) error {
		bucket, err := q.bucket(tx, q.keys.unacked)
		if err != nil {
			return err
		}
		if bucket == nil {
			t.Error("nil bucket")
			return nil
		}
		stats := bucket.Stats()
		if got, want := stats.KeyN, 1; got != want {
			t.Errorf("got %d unacked messages, want %d", got, want)
		}
		return nil
	})
	if err != nil {
		t.Fatal(err)
	}

	// Assume at this point the program crashed and we loaded the db again.
	q, err = NewQ(q.db, "testing")
	if err != nil {
		t.Fatal(err)
	}

	// Make sure all the messages are moved out of the unacked bucket
	err = q.db.Update(func(tx *bolt.Tx) error {
		bucket, err := q.bucket(tx, q.keys.unacked)
		if err != nil {
			return err
		}
		k, _ := bucket.Cursor().First()
		if k != nil {
			t.Errorf("expected empty bucket, found key %x", k)
		}
		return nil
	})

	if err != nil {
		t.Fatal(err)
	}

	m, err := q.Receive(context.Background())
	if err != nil {
		t.Fatal(err)
	}

	defer func() {
		if err := m.Ack(); err != nil {
			t.Fatal(err)
		}
	}()

	if got, want := string(m.Body), "foo"; got != want {
		t.Errorf("bad body: got %q, want %q", got, want)
	}

	if got, want := m.ID, idb; !bytes.Equal(got, want) {
		t.Errorf("bad id: got %v, want %v", got, want)
	}
}

func BenchmarkSend_64(b *testing.B) {
	benchSend(b, 64)
}

func benchSend(b *testing.B, msgSize int) {
	q, cleanup := newQ(b)
	defer cleanup()
	msg := make([]byte, msgSize)
	for i := 0; i < len(msg); i++ {
		msg[i] = byte(i % 256)
	}
	b.ResetTimer()

	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			if _, err := q.Send(msg); err != nil {
				b.Fatal(err)
			}
		}
	})
}

func benchRoundtrip(b *testing.B, msgSize int) {
	q, cleanup := newQ(b)
	defer cleanup()
	msg := make([]byte, msgSize)
	for i := 0; i < len(msg); i++ {
		msg[i] = byte(i % 256)
	}
	ctx := context.Background()
	b.ResetTimer()
	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			if _, err := q.Send(msg); err != nil {
				b.Fatal(err)
			}
			msg, err := q.Receive(ctx)
			if err != nil {
				b.Fatal(err)
			}
			if err := msg.Ack(); err != nil {
				b.Fatal(err)
			}
		}
	})
}

func BenchmarkRoundtrip_64(b *testing.B) {
	benchRoundtrip(b, 64)
}
<commit_msg>Greater variety of benchmark message sizes.<commit_after>package lasr

import (
	"bytes"
	"context"
	"sync"
	"testing"

	"github.com/boltdb/bolt"
)

func TestSendReceive(t *testing.T) {
	q, cleanup := newQ(t)
	defer cleanup()

	msg := []byte("foobar")
	id, err := q.Send(msg)
	if err != nil {
		t.Fatal(err)
	}
	idb, err := id.MarshalBinary()
	if err != nil {
		t.Fatal(err)
	}
	message, err := q.Receive(context.Background())
	if err != nil {
		t.Fatal(err)
	}
	if !bytes.Equal(message.ID, idb) {
		t.Error("messages do not match")
	}
	if !bytes.Equal(message.Body, msg) {
		t.Error("messages do not match")
	}
	if err := message.Ack(); err != nil {
		t.Fatal(err)
	}
}

func TestSendReceiveNackNoRetry(t *testing.T) {
	q, cleanup := newQ(t)
	defer cleanup()

	msg := []byte("foobar")
	if _, err := q.Send(msg); err != nil {
		t.Fatal(err)
	}
	message, err := q.Receive(context.Background())
	if err != nil {
		t.Fatal(err)
	}
	if err := message.Nack(false); err != nil {
		t.Fatal(err)
	}
	if err := message.Ack(); err != ErrAckNack {
		t.Error("expected ErrAckNack")
	}
}

func TestSendReceiveNackWithRetry(t *testing.T) {
	q, cleanup := newQ(t)
	defer cleanup()

	msg := []byte("foobar")
	id, err := q.Send(msg)
	if err != nil {
		t.Fatal(err)
	}
	idb, err := id.MarshalBinary()
	if err != nil {
		t.Fatal(err)
	}
	message, err := q.Receive(context.Background())
	if err != nil {
		t.Fatal(err)
	}
	if err := message.Nack(true); err != nil {
		t.Fatal(err)
	}
	message, err = q.Receive(context.Background())
	if err != nil {
		t.Fatal(err)
	}
	if !bytes.Equal(message.ID, idb) {
		t.Errorf("bad id: %v", message.ID)
	}
	if !bytes.Equal(message.Body, msg) {
		t.Errorf("bad body: %q", string(message.Body))
	}
	if err := message.Ack(); err != nil {
		t.Fatal(err)
	}
}

func TestReceiveContextCancelled(t *testing.T) {
	q, cleanup := newQ(t)
	defer cleanup()

	ctx, cancel := context.WithCancel(context.Background())

	done := make(chan struct{})
	go func() {
		defer close(done)
		q.Receive(ctx)
	}()
	cancel()
	<-done // blocks forever if the Receive never gets cancelled
}

func TestUnacked(t *testing.T) {
	q, cleanup := newQ(t)
	defer cleanup()

	msg := []byte("foo")
	id, err := q.Send(msg)
	if err != nil {
		t.Fatal(err)
	}
	idb, err := id.MarshalBinary()
	if err != nil {
		t.Fatal(err)
	}

	if _, err := q.Receive(context.Background()); err != nil {
		t.Fatal(err)
	}

	// deadlock unless we clear the wg here
	q.inFlight = sync.WaitGroup{}

	// Make sure the unacked message is in the unacked queue
	err = q.db.Update(func(tx *bolt.Tx) error {
		bucket, err := q.bucket(tx, q.keys.unacked)
		if err != nil {
			return err
		}
		if bucket == nil {
			t.Error("nil bucket")
			return nil
		}
		stats := bucket.Stats()
		if got, want := stats.KeyN, 1; got != want {
			t.Errorf("got %d unacked messages, want %d", got, want)
		}
		return nil
	})
	if err != nil {
		t.Fatal(err)
	}

	// Assume at this point the program crashed and we loaded the db again.
	q, err = NewQ(q.db, "testing")
	if err != nil {
		t.Fatal(err)
	}

	// Make sure all the messages are moved out of the unacked bucket
	err = q.db.Update(func(tx *bolt.Tx) error {
		bucket, err := q.bucket(tx, q.keys.unacked)
		if err != nil {
			return err
		}
		k, _ := bucket.Cursor().First()
		if k != nil {
			t.Errorf("expected empty bucket, found key %x", k)
		}
		return nil
	})

	if err != nil {
		t.Fatal(err)
	}

	m, err := q.Receive(context.Background())
	if err != nil {
		t.Fatal(err)
	}

	defer func() {
		if err := m.Ack(); err != nil {
			t.Fatal(err)
		}
	}()

	if got, want := string(m.Body), "foo"; got != want {
		t.Errorf("bad body: got %q, want %q", got, want)
	}

	if got, want := m.ID, idb; !bytes.Equal(got, want) {
		t.Errorf("bad id: got %v, want %v", got, want)
	}
}

func BenchmarkSend_4K(b *testing.B) {
	benchSend(b, 4096)
}

func BenchmarkSend_1M(b *testing.B) {
	benchSend(b, 1<<20)
}

func BenchmarkSend_16M(b *testing.B) {
	benchSend(b, 1<<24)
}

func benchSend(b *testing.B, msgSize int) {
	q, cleanup := newQ(b)
	defer cleanup()
	msg := make([]byte, msgSize)
	for i := 0; i < len(msg); i++ {
		msg[i] = byte(i % 256)
	}
	b.ResetTimer()

	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			if _, err := q.Send(msg); err != nil {
				b.Fatal(err)
			}
		}
	})
}

func benchRoundtrip(b *testing.B, msgSize int) {
	q, cleanup := newQ(b)
	defer cleanup()
	msg := make([]byte, msgSize)
	for i := 0; i < len(msg); i++ {
		msg[i] = byte(i % 256)
	}
	ctx := context.Background()
	b.ResetTimer()
	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			if _, err := q.Send(msg); err != nil {
				b.Fatal(err)
			}
			msg, err := q.Receive(ctx)
			if err != nil {
				b.Fatal(err)
			}
			if err := msg.Ack(); err != nil {
				b.Fatal(err)
			}
		}
	})
}

func BenchmarkRoundtrip_4K(b *testing.B) {
	benchRoundtrip(b, 4096)
}

func BenchmarkRoundTrip_1M(b *testing.B) {
	benchRoundtrip(b, 1<<20)
}

func BenchmarkRoundTrip_16M(b *testing.B) {
	benchRoundtrip(b, 1<<24)
}
<|endoftext|>


<commit_before>package mesos

import (
	"regexp"
	"strings"

	"github.com/CiscoCloud/mesos-consul/registry"

	"github.com/mesosphere/mesos-dns/records/state"
)

type CheckVar struct {
	Host string
	Port string
}

var globalCV *CheckVar

// Task Methods

// GetCheck()
//   Build a Check structure from the Task labels
//
func GetCheck(t *state.Task, cv *CheckVar) *registry.Check {
	c := registry.DefaultCheck()

	for _, l := range t.DiscoveryInfo.Labels.Labels {
		k := strings.ToLower(l.Key)

		switch k {
		case "check_http":
			c.HTTP = interpolate(cv, l.Value)
		case "check_script":
			c.Script = interpolate(cv, l.Value)
		case "check_ttl":
			c.TTL = interpolate(cv, l.Value)
		case "check_interval":
			c.Interval = l.Value
		}
	}

	return c
}

// Replace {variables} with values
//
func interpolate(cv *CheckVar, s string) string {
	r := regexp.MustCompile("{[^}]*}")

	globalCV = cv
	rval := r.ReplaceAllStringFunc(s, varReplace)

	return string(rval)
}

// Replacement function
//
func varReplace(s string) string {
	switch s {
	case "{port}":
		return globalCV.Port
	case "{host}":
		return globalCV.Host
	default:
		return s
	}

	return s
}
<commit_msg>Remove unnecessary return `go vet`<commit_after>package mesos

import (
	"regexp"
	"strings"

	"github.com/CiscoCloud/mesos-consul/registry"

	"github.com/mesosphere/mesos-dns/records/state"
)

type CheckVar struct {
	Host string
	Port string
}

var globalCV *CheckVar

// Task Methods

// GetCheck()
//   Build a Check structure from the Task labels
//
func GetCheck(t *state.Task, cv *CheckVar) *registry.Check {
	c := registry.DefaultCheck()

	for _, l := range t.DiscoveryInfo.Labels.Labels {
		k := strings.ToLower(l.Key)

		switch k {
		case "check_http":
			c.HTTP = interpolate(cv, l.Value)
		case "check_script":
			c.Script = interpolate(cv, l.Value)
		case "check_ttl":
			c.TTL = interpolate(cv, l.Value)
		case "check_interval":
			c.Interval = l.Value
		}
	}

	return c
}

// Replace {variables} with values
//
func interpolate(cv *CheckVar, s string) string {
	r := regexp.MustCompile("{[^}]*}")

	globalCV = cv
	rval := r.ReplaceAllStringFunc(s, varReplace)

	return string(rval)
}

// Replacement function
//
func varReplace(s string) string {
	switch s {
	case "{port}":
		return globalCV.Port
	case "{host}":
		return globalCV.Host
	default:
		return s
	}
}
<|endoftext|>


<commit_before>package cpu

import (
	"bufio"
	"os"
	"regexp"
	"strconv"
)

type Cpu struct{}

const name = "cpu"

func (self *Cpu) Name() string {
	return name
}

func (self *Cpu) Collect() (result interface{}, err error) {
	result, err = getCpuInfo()
	return
}

var cpuMap = map[string]string{
	"vendor_id":   "vendor_id",
	"model name":  "model_name",
	"cpu cores":   "cpu_cores",
	"cpu MHz":     "mhz",
	"cache size":  "cache_size",
	// "processor":   "processor",
	// "cpu family":  "family",
	// "model":       "model",
	// "stepping":    "stepping",
	// "physical id": "physical_id",
	// "flags":       "flags",
}

// fixme tri This collector saves only the last processor's info

func getCpuInfo() (cpuInfo map[string]string, err error) {
	file, err := os.Open("/proc/cpuinfo")

	if err != nil {
		return
	}

	var lines []string
	scanner := bufio.NewScanner(file)

	for scanner.Scan() {
		lines = append(lines, scanner.Text())
	}

	if scanner.Err() != nil {
		err = scanner.Err()
		return
	}

	cpuInfo = make(map[string]string)
	count := 0

	for _, line := range lines {
		pair := regexp.MustCompile("\t: ").Split(line, 2)

		key, ok := cpuMap[pair[0]]
		if ok {
			cpuInfo[key] = pair[1]
			if pair[0] == "processor" {
				count += 1
			}
		}
	}

	cpuInfo["total"] = strconv.Itoa(count)
	return
}
<commit_msg>Update cpu MHz<commit_after>package cpu

import (
	"bufio"
	"os"
	"regexp"
	"strconv"
)

type Cpu struct{}

const name = "cpu"

func (self *Cpu) Name() string {
	return name
}

func (self *Cpu) Collect() (result interface{}, err error) {
	result, err = getCpuInfo()
	return
}

var cpuMap = map[string]string{
	"vendor_id":   "vendor_id",
	"model name":  "model_name",
	"cpu cores":   "cpu_cores",
	"cpu MHz\t":     "mhz",
	"cache size":  "cache_size",
	// "processor":   "processor",
	// "cpu family":  "family",
	// "model":       "model",
	// "stepping":    "stepping",
	// "physical id": "physical_id",
	// "flags":       "flags",
}

// fixme tri This collector saves only the last processor's info

func getCpuInfo() (cpuInfo map[string]string, err error) {
	file, err := os.Open("/proc/cpuinfo")

	if err != nil {
		return
	}

	var lines []string
	scanner := bufio.NewScanner(file)

	for scanner.Scan() {
		lines = append(lines, scanner.Text())
	}

	if scanner.Err() != nil {
		err = scanner.Err()
		return
	}

	cpuInfo = make(map[string]string)
	count := 0

	for _, line := range lines {
		pair := regexp.MustCompile("\t: ").Split(line, 2)

		key, ok := cpuMap[pair[0]]
		if pair[0] == "processor" {
			count += 1
		}
		if ok {
			cpuInfo[key] = pair[1]
		}
	}

	cpuInfo["total"] = strconv.Itoa(count)
	return
}
<|endoftext|>


<commit_before>package minion

import (
	"os"
	"os/exec"
	"os/signal"
	"log"
	"bytes"
	"time"
	"strings"
	"strconv"
	"path/filepath"
	"encoding/json"

	"code.google.com/p/go-uuid/uuid"
	"github.com/coreos/etcd/Godeps/_workspace/src/golang.org/x/net/context"
	etcdclient "github.com/coreos/etcd/client"
)

// Key spaces in etcd
const EtcdRootKeySpace = "/gru"
var EtcdMinionSpace = filepath.Join(EtcdRootKeySpace, "minion")

// Etcd Minion
type EtcdMinion struct {
	// Name of this minion
	Name string

	// Minion root node in etcd
	MinionRootDir string

	// Minion queue node in etcd
	QueueDir string

	// Log directory to keep previously executed tasks
	LogDir string

	// Root node for classifiers in etcd
	ClassifierDir string

	// Minion unique identifier
	UUID uuid.UUID

	// KeysAPI client to etcd
	KAPI etcdclient.KeysAPI
}

// Etcd minion task
type EtcdTask struct {
	// Command to be executed by the minion
	Command string

	// Command arguments
	Args []string

	// Time when the command was sent for processing
	TimeReceived int64

	// Time when the command was processed
	TimeProcessed int64

	// Task unique identifier
	TaskID uuid.UUID

	// Result of task after processing
	Result string

	// Task error, if any
	Error string
}

// Unmarshals task from etcd and removes it from the queue
func UnmarshalEtcdTask(node *etcdclient.Node) (*EtcdTask, error) {
	task := new(EtcdTask)
	err := json.Unmarshal([]byte(node.Value), &task)

	if err != nil {
		log.Printf("Invalid task: key: %s\n", node.Key)
		log.Printf("Invalid task: value: %s\n", node.Value)
		log.Printf("Invalid task: error: %s\n", err)
	}

	return task, err
}

func NewEtcdTask(command string, args ...string) MinionTask {
	t := &EtcdTask{
		Command: command,
		Args: args,
		TimeReceived: time.Now().Unix(),
		TaskID: uuid.NewRandom(),
	}

	return t
}

// Gets the task unique identifier
func (t *EtcdTask) GetTaskID() uuid.UUID {
	return t.TaskID
}

// Gets the task command to be executed
func (t *EtcdTask) GetCommand() (string, error) {
	return t.Command, nil
}

// Gets the task arguments
func (t *EtcdTask) GetArgs() ([]string, error) {
	return t.Args, nil
}

// Returns the time a task has been received for processing
func (t *EtcdTask) GetTimeReceived() (int64, error) {
	return t.TimeReceived, nil
}

// Returns the titme when a task has been processed
func (t *EtcdTask) GetTimeProcessed() (int64, error) {
	return t.TimeProcessed, nil
}

// Returns the result of the task
func (t *EtcdTask) GetResult() (string, error) {
	return t.Result, nil
}

// Returns the task error, if any
func (t *EtcdTask) GetError() string {
	return t.Error
}

// Processes a task
func (t *EtcdTask) Process() error {
	var buf bytes.Buffer
	taskID := t.GetTaskID()
	command, _ := t.GetCommand()
	args, _ := t.GetArgs()
	cmd := exec.Command(command, args...)
	cmd.Stdout = &buf
	cmd.Stderr = &buf

	log.Printf("Processing task %s\n", taskID)

	cmdError := cmd.Run()
	t.TimeProcessed = time.Now().Unix()
	t.Result = buf.String()

	if cmdError != nil {
		log.Printf("Failed to process task %s\n", taskID)
		t.Error = cmdError.Error()
	} else {
		log.Printf("Finished processing task %s\n", taskID)
	}

	return cmdError
}

// Creates a new etcd minion
func NewEtcdMinion(name string, cfg etcdclient.Config) Minion {
	c, err := etcdclient.New(cfg)
	if err != nil {
		log.Fatal(err)
	}

	kapi := etcdclient.NewKeysAPI(c)
	minionUUID := GenerateUUID(name)
	minionRootDir := filepath.Join(EtcdMinionSpace, minionUUID.String())
	queueDir := filepath.Join(minionRootDir, "queue")
	classifierDir := filepath.Join(minionRootDir, "classifier")
	logDir := filepath.Join(minionRootDir, "log")

	log.Printf("Created minion with uuid %s\n", minionUUID)

	m := &EtcdMinion{
		Name: name,
		MinionRootDir: minionRootDir,
		QueueDir: queueDir,
		ClassifierDir: classifierDir,
		LogDir: logDir,
		UUID: minionUUID,
		KAPI: kapi,
	}

	return m
}

// Get the minion UUID
func (m *EtcdMinion) GetUUID() uuid.UUID {
	return m.UUID
}

// Get the human-readable name of the minion
func (m *EtcdMinion) GetName() (string, error) {
	return m.Name, nil
}

// Set the human-readable name of the minion
func (m *EtcdMinion) SetName(name string) error {
	key := filepath.Join(m.MinionRootDir, "name")
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
	}

	_, err := m.KAPI.Set(context.Background(), key, m.Name, opts)

	return err
}

// Set the time the minion was last seen in seconds since the Epoch
func (m *EtcdMinion) SetLastseen(s int64) error {
	key := filepath.Join(m.MinionRootDir, "lastseen")
	lastseen := strconv.FormatInt(s, 10)
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
	}

	_, err := m.KAPI.Set(context.Background(), key, lastseen, opts)

	return err
}

// Get classifier for a minion
func (m *EtcdMinion) GetClassifier(key string) (MinionClassifier, error) {
	klassifier := new(SimpleClassifier)
	klassifierNode := filepath.Join(m.ClassifierDir, key, "info")

	// Get classifier from etcd and deserialize
	resp, err := m.KAPI.Get(context.Background(), klassifierNode, nil)

	if err != nil {
		return klassifier, err
	}

	err = json.Unmarshal([]byte(resp.Node.Value), &klassifier)

	return klassifier, err
}

// Classify a minion  a given key and value
func (m *EtcdMinion) SetClassifier(c MinionClassifier) error {
	// Classifiers in etcd expire after an hour
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
		TTL: time.Hour,
	}

	// Get classifier values
	key, err := c.GetKey()
	description, err := c.GetDescription()
	value, err := c.GetValue(m)

	if err != nil {
		return err
	}

	// Create a simple classifier and serialize to JSON
	klassifier := NewSimpleClassifier(key, value, description)
	data, err := json.Marshal(klassifier)

	if err != nil {
		log.Printf("Failed to serialize classifier: %s\n", key)
		return err
	}

	// Set classifier in etcd
	klassifierNode := filepath.Join(m.ClassifierDir, key)
	_, err = m.KAPI.Set(context.Background(), klassifierNode, string(data), opts)

	return err
}

// Runs periodic jobs such as refreshing classifiers and updating lastseen
func (m *EtcdMinion) Refresh(ticker *time.Ticker) error {
	for {
		// Update classifiers
		for _, classifier := range ClassifierRegistry {
			m.SetClassifier(classifier)
		}

		// Update lastseen time
		now := time.Now().Unix()
		err := m.SetLastseen(now)
		if err != nil {
			log.Printf("Failed to update lastseen time: %s\n", err)
		}

		<- ticker.C
	}

	return nil
}

// Monitors etcd for new tasks for processing
func (m *EtcdMinion) TaskListener(c chan<- MinionTask) error {
	log.Printf("Starting task listener")

	watcherOpts := &etcdclient.WatcherOptions{
		Recursive: true,
	}
	watcher := m.KAPI.Watcher(m.QueueDir, watcherOpts)

	for {
		resp, err := watcher.Next(context.Background())
		if err != nil {
			log.Printf("Failed to read task: %s\n", err)
			continue
		}

		// Ignore "delete" events when removing a task from the queue
		action := strings.ToLower(resp.Action)
		if strings.EqualFold(action, "delete") {
			continue
		}

		task, err := UnmarshalEtcdTask(resp.Node)
		m.KAPI.Delete(context.Background(), resp.Node.Key, nil)

		if err != nil {
			continue
		}

		log.Printf("Received task %s\n", task.GetTaskID())

		c <- task
	}

	return nil
}

// Processes new tasks
func (m *EtcdMinion) TaskRunner(c <-chan MinionTask) error {
	for {
		task := <-c
		task.Process()
		m.SaveTaskResult(task)
	}

	return nil
}

// Saves a task in the minion's log of previously executed tasks
func (m *EtcdMinion) SaveTaskResult(t MinionTask) error {
	taskID := t.GetTaskID()
	taskNode := filepath.Join(m.LogDir, taskID.String())

	data, err := json.Marshal(t)
	if err != nil {
		log.Printf("Failed to save task %s: %s\n", taskID, err)
		return err
	}
	_, err = m.KAPI.Create(context.Background(), taskNode, string(data))

	return err
}

// Checks for any tasks in backlog
func (m *EtcdMinion) CheckForBacklog(c chan<- MinionTask) error {
	opts := &etcdclient.GetOptions{
		Recursive: true,
		Sort: true,
	}

	// Get backlog tasks if any
	resp, err := m.KAPI.Get(context.Background(), m.QueueDir, opts)
	if err != nil {
		return nil
	}

	backlog := resp.Node.Nodes

	if len(backlog) == 0 {
		// No backlog tasks found
		return nil
	}

	log.Printf("Found %d tasks in backlog", len(backlog))
	for _, node := range backlog {
		task, err := UnmarshalEtcdTask(node)
		m.KAPI.Delete(context.Background(), node.Key, nil)

		if err != nil {
			continue
		}

		c <- task
	}

	return nil
}

// Main entry point of the minion
func (m *EtcdMinion) Serve() error {
	// Channel on which we send the quit signal
	quit := make(chan os.Signal, 1)
	signal.Notify(quit, os.Interrupt)

	// Run any periodic tasks every hour
	ticker := time.NewTicker(time.Minute * 15)
	go m.Refresh(ticker)

	// Check for backlog tasks and start task listener and runner
	tasks := make(chan MinionTask)
	go m.TaskListener(tasks)
	go m.CheckForBacklog(tasks)
	go m.TaskRunner(tasks)

	// Block until a stop signal is received
	s := <-quit
	log.Printf("Received %s signal, shutting down", s)
	close(quit)
	close(tasks)

	return nil
}

<commit_msg>Comment updates<commit_after>package minion

import (
	"os"
	"os/exec"
	"os/signal"
	"log"
	"bytes"
	"time"
	"strings"
	"strconv"
	"path/filepath"
	"encoding/json"

	"code.google.com/p/go-uuid/uuid"
	"github.com/coreos/etcd/Godeps/_workspace/src/golang.org/x/net/context"
	etcdclient "github.com/coreos/etcd/client"
)

// Key spaces in etcd
const EtcdRootKeySpace = "/gru"
var EtcdMinionSpace = filepath.Join(EtcdRootKeySpace, "minion")

// Etcd Minion
type EtcdMinion struct {
	// Name of this minion
	Name string

	// Minion root node in etcd
	MinionRootDir string

	// Minion queue node in etcd
	QueueDir string

	// Log directory to keep previously executed tasks
	LogDir string

	// Root node for classifiers in etcd
	ClassifierDir string

	// Minion unique identifier
	UUID uuid.UUID

	// KeysAPI client to etcd
	KAPI etcdclient.KeysAPI
}

// Etcd minion task
type EtcdTask struct {
	// Command to be executed by the minion
	Command string

	// Command arguments
	Args []string

	// Time when the command was sent for processing
	TimeReceived int64

	// Time when the command was processed
	TimeProcessed int64

	// Task unique identifier
	TaskID uuid.UUID

	// Result of task after processing
	Result string

	// Task error, if any
	Error string
}

// Unmarshals task from etcd and removes it from the queue
func UnmarshalEtcdTask(node *etcdclient.Node) (*EtcdTask, error) {
	task := new(EtcdTask)
	err := json.Unmarshal([]byte(node.Value), &task)

	if err != nil {
		log.Printf("Invalid task: key: %s\n", node.Key)
		log.Printf("Invalid task: value: %s\n", node.Value)
		log.Printf("Invalid task: error: %s\n", err)
	}

	return task, err
}

func NewEtcdTask(command string, args ...string) MinionTask {
	t := &EtcdTask{
		Command: command,
		Args: args,
		TimeReceived: time.Now().Unix(),
		TaskID: uuid.NewRandom(),
	}

	return t
}

// Gets the task unique identifier
func (t *EtcdTask) GetTaskID() uuid.UUID {
	return t.TaskID
}

// Gets the task command to be executed
func (t *EtcdTask) GetCommand() (string, error) {
	return t.Command, nil
}

// Gets the task arguments
func (t *EtcdTask) GetArgs() ([]string, error) {
	return t.Args, nil
}

// Returns the time a task has been received for processing
func (t *EtcdTask) GetTimeReceived() (int64, error) {
	return t.TimeReceived, nil
}

// Returns the time when a task has been processed
func (t *EtcdTask) GetTimeProcessed() (int64, error) {
	return t.TimeProcessed, nil
}

// Returns the result of the task
func (t *EtcdTask) GetResult() (string, error) {
	return t.Result, nil
}

// Returns the task error, if any
func (t *EtcdTask) GetError() string {
	return t.Error
}

// Processes a task
func (t *EtcdTask) Process() error {
	var buf bytes.Buffer
	taskID := t.GetTaskID()
	command, _ := t.GetCommand()
	args, _ := t.GetArgs()
	cmd := exec.Command(command, args...)
	cmd.Stdout = &buf
	cmd.Stderr = &buf

	log.Printf("Processing task %s\n", taskID)

	cmdError := cmd.Run()
	t.TimeProcessed = time.Now().Unix()
	t.Result = buf.String()

	if cmdError != nil {
		log.Printf("Failed to process task %s\n", taskID)
		t.Error = cmdError.Error()
	} else {
		log.Printf("Finished processing task %s\n", taskID)
	}

	return cmdError
}

// Creates a new etcd minion
func NewEtcdMinion(name string, cfg etcdclient.Config) Minion {
	c, err := etcdclient.New(cfg)
	if err != nil {
		log.Fatal(err)
	}

	kapi := etcdclient.NewKeysAPI(c)
	minionUUID := GenerateUUID(name)
	minionRootDir := filepath.Join(EtcdMinionSpace, minionUUID.String())
	queueDir := filepath.Join(minionRootDir, "queue")
	classifierDir := filepath.Join(minionRootDir, "classifier")
	logDir := filepath.Join(minionRootDir, "log")

	log.Printf("Created minion with uuid %s\n", minionUUID)

	m := &EtcdMinion{
		Name: name,
		MinionRootDir: minionRootDir,
		QueueDir: queueDir,
		ClassifierDir: classifierDir,
		LogDir: logDir,
		UUID: minionUUID,
		KAPI: kapi,
	}

	return m
}

// Get the minion UUID
func (m *EtcdMinion) GetUUID() uuid.UUID {
	return m.UUID
}

// Get the human-readable name of the minion
func (m *EtcdMinion) GetName() (string, error) {
	return m.Name, nil
}

// Set the human-readable name of the minion
func (m *EtcdMinion) SetName(name string) error {
	key := filepath.Join(m.MinionRootDir, "name")
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
	}

	_, err := m.KAPI.Set(context.Background(), key, m.Name, opts)

	return err
}

// Set the time the minion was last seen in seconds since the Epoch
func (m *EtcdMinion) SetLastseen(s int64) error {
	key := filepath.Join(m.MinionRootDir, "lastseen")
	lastseen := strconv.FormatInt(s, 10)
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
	}

	_, err := m.KAPI.Set(context.Background(), key, lastseen, opts)

	return err
}

// Get classifier for a minion
func (m *EtcdMinion) GetClassifier(key string) (MinionClassifier, error) {
	klassifier := new(SimpleClassifier)
	klassifierNode := filepath.Join(m.ClassifierDir, key, "info")

	// Get classifier from etcd and deserialize
	resp, err := m.KAPI.Get(context.Background(), klassifierNode, nil)

	if err != nil {
		return klassifier, err
	}

	err = json.Unmarshal([]byte(resp.Node.Value), &klassifier)

	return klassifier, err
}

// Classify a minion with a given key and value
func (m *EtcdMinion) SetClassifier(c MinionClassifier) error {
	// Classifiers in etcd expire after an hour
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
		TTL: time.Hour,
	}

	// Get classifier values
	key, err := c.GetKey()
	description, err := c.GetDescription()
	value, err := c.GetValue(m)

	if err != nil {
		return err
	}

	// Create a simple classifier and serialize to JSON
	klassifier := NewSimpleClassifier(key, value, description)
	data, err := json.Marshal(klassifier)

	if err != nil {
		log.Printf("Failed to serialize classifier: %s\n", key)
		return err
	}

	// Set classifier in etcd
	klassifierNode := filepath.Join(m.ClassifierDir, key)
	_, err = m.KAPI.Set(context.Background(), klassifierNode, string(data), opts)

	return err
}

// Runs periodic jobs such as refreshing classifiers and updating lastseen
func (m *EtcdMinion) Refresh(ticker *time.Ticker) error {
	for {
		// Update classifiers
		for _, classifier := range ClassifierRegistry {
			m.SetClassifier(classifier)
		}

		// Update lastseen time
		now := time.Now().Unix()
		err := m.SetLastseen(now)
		if err != nil {
			log.Printf("Failed to update lastseen time: %s\n", err)
		}

		<- ticker.C
	}

	return nil
}

// Monitors etcd for new tasks for processing
func (m *EtcdMinion) TaskListener(c chan<- MinionTask) error {
	log.Printf("Starting task listener")

	watcherOpts := &etcdclient.WatcherOptions{
		Recursive: true,
	}
	watcher := m.KAPI.Watcher(m.QueueDir, watcherOpts)

	for {
		resp, err := watcher.Next(context.Background())
		if err != nil {
			log.Printf("Failed to read task: %s\n", err)
			continue
		}

		// Ignore "delete" events when removing a task from the queue
		action := strings.ToLower(resp.Action)
		if strings.EqualFold(action, "delete") {
			continue
		}

		task, err := UnmarshalEtcdTask(resp.Node)
		m.KAPI.Delete(context.Background(), resp.Node.Key, nil)

		if err != nil {
			continue
		}

		log.Printf("Received task %s\n", task.GetTaskID())

		c <- task
	}

	return nil
}

// Processes new tasks
func (m *EtcdMinion) TaskRunner(c <-chan MinionTask) error {
	for {
		task := <-c
		task.Process()
		m.SaveTaskResult(task)
	}

	return nil
}

// Saves a task in the minion's log of previously executed tasks
func (m *EtcdMinion) SaveTaskResult(t MinionTask) error {
	taskID := t.GetTaskID()
	taskNode := filepath.Join(m.LogDir, taskID.String())

	data, err := json.Marshal(t)
	if err != nil {
		log.Printf("Failed to save task %s: %s\n", taskID, err)
		return err
	}
	_, err = m.KAPI.Create(context.Background(), taskNode, string(data))

	return err
}

// Checks for any tasks in backlog
func (m *EtcdMinion) CheckForBacklog(c chan<- MinionTask) error {
	opts := &etcdclient.GetOptions{
		Recursive: true,
		Sort: true,
	}

	// Get backlog tasks if any
	resp, err := m.KAPI.Get(context.Background(), m.QueueDir, opts)
	if err != nil {
		return nil
	}

	backlog := resp.Node.Nodes

	if len(backlog) == 0 {
		// No backlog tasks found
		return nil
	}

	log.Printf("Found %d tasks in backlog", len(backlog))
	for _, node := range backlog {
		task, err := UnmarshalEtcdTask(node)
		m.KAPI.Delete(context.Background(), node.Key, nil)

		if err != nil {
			continue
		}

		c <- task
	}

	return nil
}

// Main entry point of the minion
func (m *EtcdMinion) Serve() error {
	// Channel on which we send the quit signal
	quit := make(chan os.Signal, 1)
	signal.Notify(quit, os.Interrupt)

	// Run any periodic tasks every hour
	ticker := time.NewTicker(time.Minute * 15)
	go m.Refresh(ticker)

	// Check for backlog tasks and start task listener and runner
	tasks := make(chan MinionTask)
	go m.TaskListener(tasks)
	go m.CheckForBacklog(tasks)
	go m.TaskRunner(tasks)

	// Block until a stop signal is received
	s := <-quit
	log.Printf("Received %s signal, shutting down", s)
	close(quit)
	close(tasks)

	return nil
}

<|endoftext|>


<commit_before>package minion

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"math/rand"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/dnaeon/backoff"
	"github.com/dnaeon/gru/classifier"
	"github.com/dnaeon/gru/task"
	"github.com/dnaeon/gru/utils"
	"github.com/libgit2/git2go"

	etcdclient "github.com/coreos/etcd/client"
	"github.com/pborman/uuid"
	"golang.org/x/net/context"
)

// EtcdMinionSpace is the keyspace in etcd used by minions
const EtcdMinionSpace = "/gru/minion"

// Etcd Minion
type etcdMinion struct {
	// Name of the minion
	name string

	// Minion root directory in etcd
	rootDir string

	// Minion queue directory in etcd
	queueDir string

	// Log directory of previously executed tasks
	logDir string

	// Classifier directory in etcd
	classifierDir string

	// Minion unique identifier
	id uuid.UUID

	// KeysAPI client to etcd
	kapi etcdclient.KeysAPI

	// Channel over which tasks are sent for processing
	taskQueue chan *task.Task

	// Git repo from which to sync modules and data
	gitRepo string

	// Site directory contains the modules and data synced from Git
	siteDir string

	// Channel used to signal for shutdown time
	done chan struct{}
}

// EtcdMinionConfig contains configuration settings for
// minions which use the etcd backend implementation
type EtcdMinionConfig struct {
	// Name of the minion
	Name string

	// GitRepo provides the url from which to sync modules and data
	GitRepo string

	// EtcdConfig provides etcd-related configuration settings
	EtcdConfig etcdclient.Config
}

// NewEtcdMinion creates a new minion with etcd backend
func NewEtcdMinion(config *EtcdMinionConfig) (Minion, error) {
	c, err := etcdclient.New(config.EtcdConfig)
	if err != nil {
		return nil, err
	}

	cwd, err := os.Getwd()
	if err != nil {
		return nil, err
	}

	kapi := etcdclient.NewKeysAPI(c)
	id := utils.GenerateUUID(config.Name)
	rootDir := filepath.Join(EtcdMinionSpace, id.String())
	queueDir := filepath.Join(rootDir, "queue")
	classifierDir := filepath.Join(rootDir, "classifier")
	logDir := filepath.Join(rootDir, "log")
	siteDir := filepath.Join(cwd, "site")
	taskQueue := make(chan *task.Task)
	done := make(chan struct{})

	m := &etcdMinion{
		name:          config.Name,
		rootDir:       rootDir,
		queueDir:      queueDir,
		classifierDir: classifierDir,
		logDir:        logDir,
		id:            id,
		kapi:          kapi,
		taskQueue:     taskQueue,
		gitRepo:       config.GitRepo,
		siteDir:       siteDir,
		done:          done,
	}

	return m, nil
}

// Checks for any pending tasks and sends them
// for processing if there are any
func (m *etcdMinion) checkQueue() error {
	opts := &etcdclient.GetOptions{
		Recursive: true,
		Sort:      true,
	}

	// Get backlog tasks if any
	// If the directory key in etcd is missing that is okay, since
	// it means there are no pending tasks for processing
	resp, err := m.kapi.Get(context.Background(), m.queueDir, opts)
	if err != nil {
		if eerr, ok := err.(etcdclient.Error); !ok || eerr.Code == etcdclient.ErrorCodeKeyNotFound {
			return err
		}
	}

	backlog := resp.Node.Nodes
	if len(backlog) == 0 {
		// No backlog tasks found
		return nil
	}

	log.Printf("Found %d pending tasks in queue", len(backlog))
	for _, node := range backlog {
		t, err := EtcdUnmarshalTask(node)
		m.kapi.Delete(context.Background(), node.Key, nil)

		if err != nil {
			continue
		}

		m.taskQueue <- t
	}

	return nil
}

// Runs periodic jobs such as refreshing classifiers and
// updating the lastseen time every five minutes
func (m *etcdMinion) periodicRunner() {
	schedule := time.Minute * 5
	ticker := time.NewTicker(schedule)
	log.Printf("Periodic scheduler set to run every %s\n", schedule)

	for {
		select {
		case <-m.done:
			break
		case now := <-ticker.C:
			// Run periodic jobs
			m.classify()
			m.checkQueue()
			m.SetLastseen(now.Unix())
		}
	}
}

// Processes new tasks
func (m *etcdMinion) processTask(t *task.Task) error {
	// Update state of task to indicate that we are now processing it
	t.State = task.TaskStateProcessing
	m.SaveTaskResult(t)

	log.Printf("Processing task %s\n", t.TaskID)

	var buf bytes.Buffer
	fmt.Fprintf(&buf, "Loaded %d resources from catalog\n", t.Catalog.Len())
	err := t.Catalog.Run(&buf)
	t.TimeProcessed = time.Now().Unix()
	t.Result = buf.String()

	if err != nil {
		log.Printf("Failed to process task %s: %s\n", t.TaskID, err)
		t.State = task.TaskStateFailed
	} else {
		log.Printf("Finished processing task %s\n", t.TaskID)
		t.State = task.TaskStateSuccess
	}

	m.SaveTaskResult(t)

	return err
}

// Classifies the minion
func (m *etcdMinion) classify() error {
	for key := range classifier.Registry {
		klassifier, err := classifier.Get(key)

		if err != nil {
			log.Printf("Failed to get classifier %s: %s\n", key, err)
			continue
		}

		m.SetClassifier(klassifier)
	}

	return nil
}

// EtcdUnmarshalTask unmarshals tasks from an etcd node
func EtcdUnmarshalTask(node *etcdclient.Node) (*task.Task, error) {
	task := new(task.Task)
	err := json.Unmarshal([]byte(node.Value), &task)

	return task, err
}

// ID returns the minion unique identifier
func (m *etcdMinion) ID() uuid.UUID {
	return m.id
}

// SetName sets the human-readable name of the minion in etcd
func (m *etcdMinion) SetName(name string) error {
	nameKey := filepath.Join(m.rootDir, "name")
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
	}

	_, err := m.kapi.Set(context.Background(), nameKey, name, opts)
	if err != nil {
		log.Printf("Failed to set name of minion: %s\n", err)
	}

	return err
}

// SetLastseen sets the time the minion was last seen in
// seconds since the Epoch
func (m *etcdMinion) SetLastseen(s int64) error {
	lastseenKey := filepath.Join(m.rootDir, "lastseen")
	lastseenValue := strconv.FormatInt(s, 10)
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
	}

	_, err := m.kapi.Set(context.Background(), lastseenKey, lastseenValue, opts)
	if err != nil {
		log.Printf("Failed to set lastseen time: %s\n", err)
	}

	return err
}

// SetClassifier sets a classifier for the minion in etcd
func (m *etcdMinion) SetClassifier(c *classifier.Classifier) error {
	// Classifiers in etcd expire after an hour
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
		TTL:       time.Hour,
	}

	// Serialize classifier to JSON and save it in etcd
	data, err := json.Marshal(c)
	if err != nil {
		log.Printf("Failed to serialize classifier %s: %s\n", c.Key, err)
		return err
	}

	// Classifier key in etcd
	klassifierKey := filepath.Join(m.classifierDir, c.Key)
	_, err = m.kapi.Set(context.Background(), klassifierKey, string(data), opts)

	if err != nil {
		log.Printf("Failed to set classifier %s: %s\n", c.Key, err)
		return err
	}

	return nil
}

// TaskListener monitors etcd for new tasks
func (m *etcdMinion) TaskListener(c chan<- *task.Task) error {
	log.Printf("Task listener is watching %s\n", m.queueDir)

	rand.Seed(time.Now().UTC().UnixNano())
	b := backoff.Backoff{
		Min:    1 * time.Second,
		Max:    10 * time.Minute,
		Factor: 2.0,
		Jitter: true,
	}

	watcherOpts := &etcdclient.WatcherOptions{
		Recursive: true,
	}
	watcher := m.kapi.Watcher(m.queueDir, watcherOpts)

	for {
		resp, err := watcher.Next(context.Background())
		if err != nil {
			// Use a backoff and retry later again
			duration := b.Duration()
			log.Printf("%s, retrying in %s\n", err, duration)
			time.Sleep(duration)
			continue
		}

		// Reset the backoff counter on successful receive
		b.Reset()

		// Ignore "delete" events when removing a task from the queue
		action := strings.ToLower(resp.Action)
		if strings.EqualFold(action, "delete") {
			continue
		}

		// Unmarshal and remove task from the queue
		t, err := EtcdUnmarshalTask(resp.Node)
		m.kapi.Delete(context.Background(), resp.Node.Key, nil)

		if err != nil {
			log.Printf("Received invalid task %s: %s\n", resp.Node.Key, err)
			continue
		}

		log.Printf("Received task %s\n", t.TaskID)
		c <- t
	}

	return nil
}

// TaskRunner processes new tasks
func (m *etcdMinion) TaskRunner(c <-chan *task.Task) error {
	log.Println("Starting task runner")

	for {
		select {
		case <-m.done:
			break
		case t := <-c:
			t.State = task.TaskStateQueued
			t.TimeReceived = time.Now().Unix()
			m.SaveTaskResult(t)

			if t.IsConcurrent {
				go m.processTask(t)
			} else {
				m.processTask(t)
			}
		}
	}

	return nil
}

// SaveTaskResult stores the result of a task in etcd
func (m *etcdMinion) SaveTaskResult(t *task.Task) error {
	taskKey := filepath.Join(m.logDir, t.TaskID.String())

	data, err := json.Marshal(t)
	if err != nil {
		log.Printf("Failed to serialize task %s: %s\n", t.TaskID, err)
		return err
	}

	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
	}

	_, err = m.kapi.Set(context.Background(), taskKey, string(data), opts)
	if err != nil {
		log.Printf("Failed to save task %s: %s\n", t.TaskID, err)
	}

	return err
}

// Sync syncs module and data files from the upstream Git repository
func (m *etcdMinion) Sync() error {
	fi, err := os.Stat(m.siteDir)
	if err != nil {
		return err
	}

	// Site directory does not exist, clone the Git repository from upstream
	if os.IsNotExist(err) {
		log.Printf("site repo is missing, performing initial sync from %s\n", m.gitRepo)
		opts := &git.CloneOptions{}
		if _, err := git.Clone(m.gitRepo, m.siteDir, opts); err != nil {
			return err
		}
	}

	// File exists, ensure that it is a valid Git repository
	if !fi.IsDir() {
		return fmt.Errorf("%s exists, but is not a directory", m.siteDir)
	}

	// Open the repository and fetch from the default remote
	repo, err := git.OpenRepository(m.siteDir)
	if err != nil {
		return err
	}

	origin, err := repo.Remotes.Lookup("origin")
	if err != nil {
		return err
	}

	err = origin.Fetch([]string{}, &git.FetchOptions{}, "")

	return err
}

// Seve starts the minion
func (m *etcdMinion) Serve() error {
	err := m.SetName(m.name)
	if err != nil {
		return err
	}

	now := time.Now().Unix()
	err = m.SetLastseen(now)
	if err != nil {
		return err
	}

	// Start minion services
	go m.classify()
	go m.checkQueue()
	go m.periodicRunner()
	go m.TaskRunner(m.taskQueue)
	go m.TaskListener(m.taskQueue)

	log.Printf("Minion %s is ready to serve", m.ID())

	return nil
}

// Stop shutdowns the minions and its services
func (m *etcdMinion) Stop() error {
	log.Println("Minion is shutting down")

	close(m.taskQueue)
	close(m.done)

	return nil
}
<commit_msg>Doc comment updates<commit_after>package minion

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"math/rand"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/dnaeon/backoff"
	"github.com/dnaeon/gru/classifier"
	"github.com/dnaeon/gru/task"
	"github.com/dnaeon/gru/utils"
	"github.com/libgit2/git2go"

	etcdclient "github.com/coreos/etcd/client"
	"github.com/pborman/uuid"
	"golang.org/x/net/context"
)

// EtcdMinionSpace is the keyspace in etcd used by minions
const EtcdMinionSpace = "/gru/minion"

// Etcd Minion
type etcdMinion struct {
	// Name of the minion
	name string

	// Minion root directory in etcd
	rootDir string

	// Minion queue directory in etcd
	queueDir string

	// Log directory of previously executed tasks
	logDir string

	// Classifier directory in etcd
	classifierDir string

	// Minion unique identifier
	id uuid.UUID

	// KeysAPI client to etcd
	kapi etcdclient.KeysAPI

	// Channel over which tasks are sent for processing
	taskQueue chan *task.Task

	// Git repo from which to sync modules and data
	gitRepo string

	// Site directory contains the modules and data synced from Git
	siteDir string

	// Channel used to signal for shutdown time
	done chan struct{}
}

// EtcdMinionConfig contains configuration settings for
// minions which use the etcd backend implementation
type EtcdMinionConfig struct {
	// Name of the minion
	Name string

	// GitRepo provides the url from which to sync modules and data
	GitRepo string

	// EtcdConfig provides etcd-related configuration settings
	EtcdConfig etcdclient.Config
}

// NewEtcdMinion creates a new minion with etcd backend
func NewEtcdMinion(config *EtcdMinionConfig) (Minion, error) {
	c, err := etcdclient.New(config.EtcdConfig)
	if err != nil {
		return nil, err
	}

	cwd, err := os.Getwd()
	if err != nil {
		return nil, err
	}

	kapi := etcdclient.NewKeysAPI(c)
	id := utils.GenerateUUID(config.Name)
	rootDir := filepath.Join(EtcdMinionSpace, id.String())
	queueDir := filepath.Join(rootDir, "queue")
	classifierDir := filepath.Join(rootDir, "classifier")
	logDir := filepath.Join(rootDir, "log")
	siteDir := filepath.Join(cwd, "site")
	taskQueue := make(chan *task.Task)
	done := make(chan struct{})

	m := &etcdMinion{
		name:          config.Name,
		rootDir:       rootDir,
		queueDir:      queueDir,
		classifierDir: classifierDir,
		logDir:        logDir,
		id:            id,
		kapi:          kapi,
		taskQueue:     taskQueue,
		gitRepo:       config.GitRepo,
		siteDir:       siteDir,
		done:          done,
	}

	return m, nil
}

// Checks for any pending tasks and sends them
// for processing if there are any
func (m *etcdMinion) checkQueue() error {
	opts := &etcdclient.GetOptions{
		Recursive: true,
		Sort:      true,
	}

	// Get backlog tasks if any
	// If the directory key in etcd is missing that is okay, since
	// it means there are no pending tasks for processing
	resp, err := m.kapi.Get(context.Background(), m.queueDir, opts)
	if err != nil {
		if eerr, ok := err.(etcdclient.Error); !ok || eerr.Code == etcdclient.ErrorCodeKeyNotFound {
			return err
		}
	}

	backlog := resp.Node.Nodes
	if len(backlog) == 0 {
		// No backlog tasks found
		return nil
	}

	log.Printf("Found %d pending tasks in queue", len(backlog))
	for _, node := range backlog {
		t, err := EtcdUnmarshalTask(node)
		m.kapi.Delete(context.Background(), node.Key, nil)

		if err != nil {
			continue
		}

		m.taskQueue <- t
	}

	return nil
}

// Runs periodic jobs such as refreshing classifiers and
// updating the lastseen time every five minutes
func (m *etcdMinion) periodicRunner() {
	schedule := time.Minute * 5
	ticker := time.NewTicker(schedule)
	log.Printf("Periodic scheduler set to run every %s\n", schedule)

	for {
		select {
		case <-m.done:
			break
		case now := <-ticker.C:
			// Run periodic jobs
			m.classify()
			m.checkQueue()
			m.SetLastseen(now.Unix())
		}
	}
}

// Processes new tasks
func (m *etcdMinion) processTask(t *task.Task) error {
	// Update state of task to indicate that we are now processing it
	t.State = task.TaskStateProcessing
	m.SaveTaskResult(t)

	log.Printf("Processing task %s\n", t.TaskID)

	var buf bytes.Buffer
	fmt.Fprintf(&buf, "Loaded %d resources from catalog\n", t.Catalog.Len())
	err := t.Catalog.Run(&buf)
	t.TimeProcessed = time.Now().Unix()
	t.Result = buf.String()

	if err != nil {
		log.Printf("Failed to process task %s: %s\n", t.TaskID, err)
		t.State = task.TaskStateFailed
	} else {
		log.Printf("Finished processing task %s\n", t.TaskID)
		t.State = task.TaskStateSuccess
	}

	m.SaveTaskResult(t)

	return err
}

// Classifies the minion
func (m *etcdMinion) classify() error {
	for key := range classifier.Registry {
		klassifier, err := classifier.Get(key)

		if err != nil {
			log.Printf("Failed to get classifier %s: %s\n", key, err)
			continue
		}

		m.SetClassifier(klassifier)
	}

	return nil
}

// EtcdUnmarshalTask unmarshals tasks from an etcd node
func EtcdUnmarshalTask(node *etcdclient.Node) (*task.Task, error) {
	task := new(task.Task)
	err := json.Unmarshal([]byte(node.Value), &task)

	return task, err
}

// ID returns the minion unique identifier
func (m *etcdMinion) ID() uuid.UUID {
	return m.id
}

// SetName sets the human-readable name of the minion in etcd
func (m *etcdMinion) SetName(name string) error {
	nameKey := filepath.Join(m.rootDir, "name")
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
	}

	_, err := m.kapi.Set(context.Background(), nameKey, name, opts)
	if err != nil {
		log.Printf("Failed to set name of minion: %s\n", err)
	}

	return err
}

// SetLastseen sets the time the minion was last seen in
// seconds since the Epoch
func (m *etcdMinion) SetLastseen(s int64) error {
	lastseenKey := filepath.Join(m.rootDir, "lastseen")
	lastseenValue := strconv.FormatInt(s, 10)
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
	}

	_, err := m.kapi.Set(context.Background(), lastseenKey, lastseenValue, opts)
	if err != nil {
		log.Printf("Failed to set lastseen time: %s\n", err)
	}

	return err
}

// SetClassifier sets a classifier for the minion in etcd
func (m *etcdMinion) SetClassifier(c *classifier.Classifier) error {
	// Classifiers in etcd expire after an hour
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
		TTL:       time.Hour,
	}

	// Serialize classifier to JSON and save it in etcd
	data, err := json.Marshal(c)
	if err != nil {
		log.Printf("Failed to serialize classifier %s: %s\n", c.Key, err)
		return err
	}

	// Classifier key in etcd
	klassifierKey := filepath.Join(m.classifierDir, c.Key)
	_, err = m.kapi.Set(context.Background(), klassifierKey, string(data), opts)

	if err != nil {
		log.Printf("Failed to set classifier %s: %s\n", c.Key, err)
		return err
	}

	return nil
}

// TaskListener monitors etcd for new tasks
func (m *etcdMinion) TaskListener(c chan<- *task.Task) error {
	log.Printf("Task listener is watching %s\n", m.queueDir)

	rand.Seed(time.Now().UTC().UnixNano())
	b := backoff.Backoff{
		Min:    1 * time.Second,
		Max:    10 * time.Minute,
		Factor: 2.0,
		Jitter: true,
	}

	watcherOpts := &etcdclient.WatcherOptions{
		Recursive: true,
	}
	watcher := m.kapi.Watcher(m.queueDir, watcherOpts)

	for {
		resp, err := watcher.Next(context.Background())
		if err != nil {
			// Use a backoff and retry later again
			duration := b.Duration()
			log.Printf("%s, retrying in %s\n", err, duration)
			time.Sleep(duration)
			continue
		}

		// Reset the backoff counter on successful receive
		b.Reset()

		// Ignore "delete" events when removing a task from the queue
		action := strings.ToLower(resp.Action)
		if strings.EqualFold(action, "delete") {
			continue
		}

		// Unmarshal and remove task from the queue
		t, err := EtcdUnmarshalTask(resp.Node)
		m.kapi.Delete(context.Background(), resp.Node.Key, nil)

		if err != nil {
			log.Printf("Received invalid task %s: %s\n", resp.Node.Key, err)
			continue
		}

		log.Printf("Received task %s\n", t.TaskID)
		c <- t
	}

	return nil
}

// TaskRunner processes new tasks
func (m *etcdMinion) TaskRunner(c <-chan *task.Task) error {
	log.Println("Starting task runner")

	for {
		select {
		case <-m.done:
			break
		case t := <-c:
			t.State = task.TaskStateQueued
			t.TimeReceived = time.Now().Unix()
			m.SaveTaskResult(t)

			if t.IsConcurrent {
				go m.processTask(t)
			} else {
				m.processTask(t)
			}
		}
	}

	return nil
}

// SaveTaskResult stores the result of a task in etcd
func (m *etcdMinion) SaveTaskResult(t *task.Task) error {
	taskKey := filepath.Join(m.logDir, t.TaskID.String())

	data, err := json.Marshal(t)
	if err != nil {
		log.Printf("Failed to serialize task %s: %s\n", t.TaskID, err)
		return err
	}

	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
	}

	_, err = m.kapi.Set(context.Background(), taskKey, string(data), opts)
	if err != nil {
		log.Printf("Failed to save task %s: %s\n", t.TaskID, err)
	}

	return err
}

// Sync syncs module and data files from the upstream Git repository
func (m *etcdMinion) Sync() error {
	fi, err := os.Stat(m.siteDir)
	if err != nil {
		return err
	}

	// Site directory does not exist, clone the Git repository from upstream
	if os.IsNotExist(err) {
		log.Printf("site repo is missing, performing initial sync from %s\n", m.gitRepo)
		opts := &git.CloneOptions{}
		if _, err := git.Clone(m.gitRepo, m.siteDir, opts); err != nil {
			return err
		}
	}

	// File exists, ensure that it is a valid Git repository
	if !fi.IsDir() {
		return fmt.Errorf("%s exists, but is not a directory", m.siteDir)
	}

	// Open the repository and fetch from the default remote
	repo, err := git.OpenRepository(m.siteDir)
	if err != nil {
		return err
	}

	origin, err := repo.Remotes.Lookup("origin")
	if err != nil {
		return err
	}

	err = origin.Fetch([]string{}, &git.FetchOptions{}, "")

	return err
}

// Serve starts the minion
func (m *etcdMinion) Serve() error {
	err := m.SetName(m.name)
	if err != nil {
		return err
	}

	now := time.Now().Unix()
	err = m.SetLastseen(now)
	if err != nil {
		return err
	}

	// Start minion services
	go m.classify()
	go m.checkQueue()
	go m.periodicRunner()
	go m.TaskRunner(m.taskQueue)
	go m.TaskListener(m.taskQueue)

	log.Printf("Minion %s is ready to serve", m.ID())

	return nil
}

// Stop shutdowns the minions and its services
func (m *etcdMinion) Stop() error {
	log.Println("Minion is shutting down")

	close(m.taskQueue)
	close(m.done)

	return nil
}
<|endoftext|>