<commit_before>package scaffold

import (
	"bytes"
	"strings"
	"testing"
)

func TestSplitFilename(t *testing.T) {

	tests := []struct {
		input string
		bare  string
		ext   string
	}{
		{"test.txt", "test", ".txt"},
		{"test.a.b", "test.a", ".b"},
	}

	for _, test := range tests {
		bare, ext := splitFilename(test.input)

		if bare != test.bare || ext != test.ext {
			t.Errorf("splitFilename(%#v) = %#v, %#v; want %#v, %#v", test.input, bare, ext, test.bare, test.ext)
		}
	}

}

func TestIsLowercase(t *testing.T) {

	tests := []struct {
		input    string
		expected bool
	}{
		{"A", false},
		{"B", false},
		{"C", false},
		{"D", false},
		{"X", false},
		{"Y", false},
		{"Z", false},
		{"a", true},
		{"b", true},
		{"c", true},
		{"d", true},
		{"x", true},
		{"y", true},
		{"z", true},
	}

	for _, test := range tests {

		if got, want := isLowercase(test.input), test.expected; got != want {
			t.Errorf("isLowercase(%#v) = %v; want %v", test.input, got, want)
		}
	}

}

func TestFileName(t *testing.T) {

	/*
		([^-a-zA-Z_0-9])
	*/

	tests := []struct {
		input, expected string
	}{
		{" \\A/ want's 853 : until . # * +-0?@µ ", "A-wants-853-until-0"},
	}

	for _, test := range tests {

		if got, want := FileName(test.input), test.expected; got != want {
			t.Errorf("FileName(%#v) = %#v; want %#v", test.input, got, want)
		}
	}

}

func TestCamelCase1(t *testing.T) {

	tests := []struct {
		input, expected string
	}{
		{"field_name", "FieldName"},
		{"FieldName", "FieldName"},
		{"Field_name", "FieldName"},
		{"field_Name", "FieldName"},
		{"fieldname", "Fieldname"},
	}

	for _, test := range tests {

		if got, want := CamelCase1(test.input), test.expected; got != want {
			t.Errorf("CamelCase1(%v) = %v; want %v", test.input, got, want)
		}
	}

}

func TestCamelCase2(t *testing.T) {

	tests := []struct {
		input, expected string
	}{
		{"field_name", "fieldName"},
		{"FieldName", "FieldName"},
		{"Field_name", "FieldName"},
		{"field_Name", "fieldName"},
		{"fieldname", "fieldname"},
	}

	for _, test := range tests {

		if got, want := CamelCase2(test.input), test.expected; got != want {
			t.Errorf("CamelCase2(%#v) = %#v; want %#v", test.input, got, want)
		}
	}

}

func TestReplace(t *testing.T) {

	tests := []struct {
		input, src, dest, expected string
	}{
		{"field_name", "_", "*", "field*name"},
		{"field_name_x", "_", "*", "field*name*x"},
		{"_field_name_", "_", "*", "*field*name*"},
	}

	for _, test := range tests {

		if got, want := Replace(test.input, test.src, test.dest), test.expected; got != want {
			t.Errorf("Replace(%#v, %#v, %#v) = %#v; want %#v", test.input, test.src, test.dest, got, want)
		}
	}

}

var validHead = `{
	"Models": [
		{
			"Name": "",
			"Fields": [
				{"Name": "", "Type": ""}
			]
		}
	]
}`

var validBody = `{{range .Models}}
>>>models/
>>>{{toLower .Name}}/
>>>model.go
package {{replace .Name "_" "."}}

type {{camelCase1 .Name}} struct {
{{range .Fields}}
	{{camelCase1 .Name}} {{.Type}}
{{end}}
}

<<<model.go
<<<{{toLower .Name}}/
<<<models/
{{end}}`

var validJSON = `
{
	"Models": [
		{
			"Name": "person",
			"Fields": [
				{
					"Name": "first_name",
					"Type": "string"
				},
				{
					"Name": "last_name",
					"Type": "string"
				}
			]
		},
		{
			"Name": "address",
			"Fields": [
				{
					"Name": "street_no",
					"Type": "string"
				},
				{
					"Name": "city",
					"Type": "string"
				}
			]
		}
	]
}
`

var validTemplate = validHead + "\n\n" + validBody

func TestSplitTemplate(t *testing.T) {
	h, b := SplitTemplate(validTemplate)

	if h != validHead {
		t.Errorf("wrong head: %#v, expecting %#v", h, validHead)
	}

	if b != validBody {
		t.Errorf("wrong body: %#v, expecting %#v", b, validBody)
	}
}

func TestRun(t *testing.T) {

	tests := []struct {
		dir, body, json, expected string
	}{
		{
			"start",
			">>>file.txt\n<<<file.txt",
			`{}`,
			"start/file.txt\n",
		},
		{
			"start",
			">>>file1.txt\n<<<file1.txt\n>>>file2.txt\n<<<file2.txt",
			`{}`,
			"start/file1.txt\nstart/file2.txt\n",
		},
		{
			"start",
			">>>a/\n>>>b/\n>>>file1.txt\n<<<file1.txt\n>>>file2.txt\n<<<file2.txt\n<<<b/\n<<<a/\n",
			`{}`,
			"start/a/b/file1.txt\nstart/a/b/file2.txt\n",
		},
		{
			"a/dir",
			"{{range .Files}}>>>{{.Name}}.txt\n<<<{{.Name}}.txt\n{{end}}",
			`{"Files": [{"Name": "file1"},{"Name": "file2"}]}`,
			"a/dir/file1.txt\na/dir/file2.txt\n",
		},
		{
			"start/dir",
			validBody,
			validJSON,
			"start/dir/models/person/model.go\nstart/dir/models/address/model.go\n",
		},
	}

	for _, test := range tests {
		var log bytes.Buffer
		err := Run(test.dir, test.body, strings.NewReader(test.json), &log, true)
		if err != nil {
			t.Errorf("Run(%#v, %#v, %#v,...) returned error: %v", test.dir, test.body, test.json, err)
			continue
		}
		if got, want := log.String(), test.expected; got != want {
			t.Errorf("Run(%#v, %#v, %#v,...) = %#v; want %#v", test.dir, test.body, test.json, got, want)
		}
	}
}

func TestRunErrors(t *testing.T) {

	tests := []struct {
		dir, body, json string
	}{
		{
			"start",
			">>>file.txt\n<<<file.txt",
			`{`,
		},
		{
			"start",
			">>file.txt\n<<<file.txt",
			`{}`,
		},
		{
			"start",
			">>>file1.txt\n<<<file2.txt",
			`{}`,
		},
		{
			"start",
			">>>a/\n>>>b\n>>>file1.txt\n<<<file1.txt\n<<<a/\n<<<b\n",
			`{}`,
		},
		{
			"start",
			">>>file1.txt\nho\n>>>file2.txt\nhu<<<file2.txt\n<<<file1.txt\n",
			`{}`,
		},
		{
			"start",
			"{{range .x}}",
			`{}`,
		},
	}

	for _, test := range tests {
		var log bytes.Buffer
		err := Run(test.dir, test.body, strings.NewReader(test.json), &log, true)
		if err == nil {
			t.Errorf("Run(%#v, %#v, %#v,...) returned no error", test.dir, test.body, test.json)
		}
	}
}
<commit_msg>try to fix paths in tests on windows<commit_after>package scaffold

import (
	"bytes"
	"strings"
	"testing"
)

func TestSplitFilename(t *testing.T) {

	tests := []struct {
		input string
		bare  string
		ext   string
	}{
		{"test.txt", "test", ".txt"},
		{"test.a.b", "test.a", ".b"},
	}

	for _, test := range tests {
		bare, ext := splitFilename(test.input)

		if bare != test.bare || ext != test.ext {
			t.Errorf("splitFilename(%#v) = %#v, %#v; want %#v, %#v", test.input, bare, ext, test.bare, test.ext)
		}
	}

}

func TestIsLowercase(t *testing.T) {

	tests := []struct {
		input    string
		expected bool
	}{
		{"A", false},
		{"B", false},
		{"C", false},
		{"D", false},
		{"X", false},
		{"Y", false},
		{"Z", false},
		{"a", true},
		{"b", true},
		{"c", true},
		{"d", true},
		{"x", true},
		{"y", true},
		{"z", true},
	}

	for _, test := range tests {

		if got, want := isLowercase(test.input), test.expected; got != want {
			t.Errorf("isLowercase(%#v) = %v; want %v", test.input, got, want)
		}
	}

}

func TestFileName(t *testing.T) {

	/*
		([^-a-zA-Z_0-9])
	*/

	tests := []struct {
		input, expected string
	}{
		{" \\A/ want's 853 : until . # * +-0?@µ ", "A-wants-853-until-0"},
	}

	for _, test := range tests {

		if got, want := FileName(test.input), test.expected; got != want {
			t.Errorf("FileName(%#v) = %#v; want %#v", test.input, got, want)
		}
	}

}

func TestCamelCase1(t *testing.T) {

	tests := []struct {
		input, expected string
	}{
		{"field_name", "FieldName"},
		{"FieldName", "FieldName"},
		{"Field_name", "FieldName"},
		{"field_Name", "FieldName"},
		{"fieldname", "Fieldname"},
	}

	for _, test := range tests {

		if got, want := CamelCase1(test.input), test.expected; got != want {
			t.Errorf("CamelCase1(%v) = %v; want %v", test.input, got, want)
		}
	}

}

func TestCamelCase2(t *testing.T) {

	tests := []struct {
		input, expected string
	}{
		{"field_name", "fieldName"},
		{"FieldName", "FieldName"},
		{"Field_name", "FieldName"},
		{"field_Name", "fieldName"},
		{"fieldname", "fieldname"},
	}

	for _, test := range tests {

		if got, want := CamelCase2(test.input), test.expected; got != want {
			t.Errorf("CamelCase2(%#v) = %#v; want %#v", test.input, got, want)
		}
	}

}

func TestReplace(t *testing.T) {

	tests := []struct {
		input, src, dest, expected string
	}{
		{"field_name", "_", "*", "field*name"},
		{"field_name_x", "_", "*", "field*name*x"},
		{"_field_name_", "_", "*", "*field*name*"},
	}

	for _, test := range tests {

		if got, want := Replace(test.input, test.src, test.dest), test.expected; got != want {
			t.Errorf("Replace(%#v, %#v, %#v) = %#v; want %#v", test.input, test.src, test.dest, got, want)
		}
	}

}

var validHead = `{
	"Models": [
		{
			"Name": "",
			"Fields": [
				{"Name": "", "Type": ""}
			]
		}
	]
}`

var validBody = `{{range .Models}}
>>>models/
>>>{{toLower .Name}}/
>>>model.go
package {{replace .Name "_" "."}}

type {{camelCase1 .Name}} struct {
{{range .Fields}}
	{{camelCase1 .Name}} {{.Type}}
{{end}}
}

<<<model.go
<<<{{toLower .Name}}/
<<<models/
{{end}}`

var validJSON = `
{
	"Models": [
		{
			"Name": "person",
			"Fields": [
				{
					"Name": "first_name",
					"Type": "string"
				},
				{
					"Name": "last_name",
					"Type": "string"
				}
			]
		},
		{
			"Name": "address",
			"Fields": [
				{
					"Name": "street_no",
					"Type": "string"
				},
				{
					"Name": "city",
					"Type": "string"
				}
			]
		}
	]
}
`

var validTemplate = validHead + "\n\n" + validBody

func TestSplitTemplate(t *testing.T) {
	h, b := SplitTemplate(validTemplate)

	if h != validHead {
		t.Errorf("wrong head: %#v, expecting %#v", h, validHead)
	}

	if b != validBody {
		t.Errorf("wrong body: %#v, expecting %#v", b, validBody)
	}
}

func TestRun(t *testing.T) {

	tests := []struct {
		dir, body, json, expected string
	}{
		{
			"start",
			">>>file.txt\n<<<file.txt",
			`{}`,
			"start/file.txt\n",
		},
		{
			"start",
			">>>file1.txt\n<<<file1.txt\n>>>file2.txt\n<<<file2.txt",
			`{}`,
			"start/file1.txt\nstart/file2.txt\n",
		},
		{
			"start",
			">>>a/\n>>>b/\n>>>file1.txt\n<<<file1.txt\n>>>file2.txt\n<<<file2.txt\n<<<b/\n<<<a/\n",
			`{}`,
			"start/a/b/file1.txt\nstart/a/b/file2.txt\n",
		},
		{
			"a/dir",
			"{{range .Files}}>>>{{.Name}}.txt\n<<<{{.Name}}.txt\n{{end}}",
			`{"Files": [{"Name": "file1"},{"Name": "file2"}]}`,
			"a/dir/file1.txt\na/dir/file2.txt\n",
		},
		{
			"start/dir",
			validBody,
			validJSON,
			"start/dir/models/person/model.go\nstart/dir/models/address/model.go\n",
		},
	}

	for _, test := range tests {
		var log bytes.Buffer
		err := Run(test.dir, test.body, strings.NewReader(test.json), &log, true)
		if err != nil {
			t.Errorf("Run(%#v, %#v, %#v,...) returned error: %v", test.dir, test.body, test.json, err)
			continue
		}
		if got, want := log.String(), strings.Replace(test.expected, "\\\\", "/", -1); got != want {
			t.Errorf("Run(%#v, %#v, %#v,...) = %#v; want %#v", test.dir, test.body, test.json, got, want)
		}
	}
}

func TestRunErrors(t *testing.T) {

	tests := []struct {
		dir, body, json string
	}{
		{
			"start",
			">>>file.txt\n<<<file.txt",
			`{`,
		},
		{
			"start",
			">>file.txt\n<<<file.txt",
			`{}`,
		},
		{
			"start",
			">>>file1.txt\n<<<file2.txt",
			`{}`,
		},
		{
			"start",
			">>>a/\n>>>b\n>>>file1.txt\n<<<file1.txt\n<<<a/\n<<<b\n",
			`{}`,
		},
		{
			"start",
			">>>file1.txt\nho\n>>>file2.txt\nhu<<<file2.txt\n<<<file1.txt\n",
			`{}`,
		},
		{
			"start",
			"{{range .x}}",
			`{}`,
		},
	}

	for _, test := range tests {
		var log bytes.Buffer
		err := Run(test.dir, test.body, strings.NewReader(test.json), &log, true)
		if err == nil {
			t.Errorf("Run(%#v, %#v, %#v,...) returned no error", test.dir, test.body, test.json)
		}
	}
}
<|endoftext|>


<commit_before>package schema

type Base struct {
	Origin string `json:"root"`
	//ignore the tmpl attribute
}

type Avatar struct {
	X           int    `json:"x"`
	Y           int    `json:"y"`
	Size        int    `json:"size"`
	SourceFile  string `json:"source"`
	CroppedFile string `json:"crop"`
}

type User struct {
	Id         int    `json:"id"`
	ScreenName string `json:"name"`
	Avatar     Avatar `json:"icon"`
}

type UserBase struct {
	Id   int  `json:"id"`
	Base Base `json:"base"`
	User User `json:"user"`
}

type Group struct {
	Id          int        `json:"id"`
	Name        string     `json:"name"`
	Founder     UserBase   `json:"user"`
	Team        []UserBase `json:"admins"`
	Description string     `json:"intro"`
	Avatar      Avatar     `json:"icon"`
	Clubs       []string   `json:"clubs"`
}

type GroupBase struct {
	Id    int   `json:"id"`
	Base  Base  `json:"base"`
	Group Group `json:"group"`
	//just ignore the bbs attributes, will generate our own hot threads later
}

type Attachment struct {
	FileName string `json:"source"`
	Height   int    `json:"height"`
	Width    int    `json:"width"`
	Type     string `json:"type"`
}

type Message struct {
	Id int `json:"id"`
	//go is typed language while javascript is not.
	//we make it interface{} in case we fall into pits like this:
	//{"msgid": "3"}
	//It may hurt like hell.
	ThreadId    interface{}  `json:"msgid"`
	Title       string       `json:"title"`
	Content     string       `json:"content"`
	Attachments []Attachment `json:"attachment2"`
	Author      UserBase     `json:"user"`
	Group       GroupBase    `json:"object"`
	CreatedAt   string       `json:"created"`
}

type MessageBase struct {
	Id      int     `json:"id"`
	Base    Base    `json:"base"`
	Message Message `json:"message"`
}
<commit_msg>fix typo<commit_after>package schema

type Base struct {
	Origin string `json:"root"`
	//ignore the tmpl attribute
}

type Avatar struct {
	X           int    `json:"x"`
	Y           int    `json:"y"`
	Size        int    `json:"size"`
	SourceFile  string `json:"source"`
	CroppedFile string `json:"crop"`
}

type User struct {
	Id         int    `json:"id"`
	ScreenName string `json:"name"`
	Avatar     Avatar `json:"icon"`
}

type UserBase struct {
	Id   int  `json:"id"`
	Base Base `json:"base"`
	User User `json:"user"`
}

type Group struct {
	Id          int        `json:"id"`
	Name        string     `json:"name"`
	Founder     UserBase   `json:"user"`
	Team        []UserBase `json:"admins"`
	Description string     `json:"intro"`
	Avatar      Avatar     `json:"icon"`
	Clubs       []string   `json:"clubs"`
}

type GroupBase struct {
	Id    int   `json:"id"`
	Base  Base  `json:"base"`
	Group Group `json:"group"`
	//just ignore the bbs attributes, will generate our own hot threads later
}

type Attachment struct {
	FileName string `json:"source"`
	Height   int    `json:"height"`
	Width    int    `json:"width"`
	Type     string `json:"type"`
}

type Message struct {
	Id int `json:"id"`
	//go is typed language while javascript is not.
	//we make it interface{} in case we fall into pitfalls like this:
	//{"msgid": "3"}
	//It may hurt like hell.
	ThreadId    interface{}  `json:"msgid"`
	Title       string       `json:"title"`
	Content     string       `json:"content"`
	Attachments []Attachment `json:"attachment2"`
	Author      UserBase     `json:"user"`
	Group       GroupBase    `json:"object"`
	CreatedAt   string       `json:"created"`
}

type MessageBase struct {
	Id      int     `json:"id"`
	Base    Base    `json:"base"`
	Message Message `json:"message"`
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"
	"strconv"
	"time"

	contracts "github.com/estafette/estafette-ci-contracts"
	"github.com/opentracing/opentracing-go"
	"github.com/opentracing/opentracing-go/ext"

	"github.com/rs/zerolog/log"
	"github.com/sethgrid/pester"
)

// EndOfLifeHelper has methods to shutdown the runner after a fatal or successful run
type EndOfLifeHelper interface {
	handleFatal(context.Context, contracts.BuildLog, error, string)
	sendBuildFinishedEvent(ctx context.Context, buildStatus string) error
	sendBuildCleanEvent(ctx context.Context, buildStatus string) error
	sendBuildJobLogEvent(ctx context.Context, buildLog contracts.BuildLog) error
}

type endOfLifeHelperImpl struct {
	runAsJob bool
	config   contracts.BuilderConfig
}

// NewEndOfLifeHelper returns a new EndOfLifeHelper
func NewEndOfLifeHelper(runAsJob bool, config contracts.BuilderConfig) EndOfLifeHelper {
	return &endOfLifeHelperImpl{
		runAsJob: runAsJob,
		config:   config,
	}
}

func (elh *endOfLifeHelperImpl) handleFatal(ctx context.Context, buildLog contracts.BuildLog, err error, message string) {

	// add error messages as step to show in logs
	fatalStep := contracts.BuildLogStep{
		Step:     "init",
		LogLines: []contracts.BuildLogLine{},
		ExitCode: -1,
		Status:   "FAILED",
	}
	lineNumber := 1

	if err != nil {
		fatalStep.LogLines = append(fatalStep.LogLines, contracts.BuildLogLine{
			LineNumber: lineNumber,
			Timestamp:  time.Now().UTC(),
			StreamType: "stderr",
			Text:       err.Error(),
		})
		lineNumber++
	}
	if message != "" {
		fatalStep.LogLines = append(fatalStep.LogLines, contracts.BuildLogLine{
			LineNumber: lineNumber,
			Timestamp:  time.Now().UTC(),
			StreamType: "stderr",
			Text:       message,
		})
		lineNumber++
	}

	buildLog.Steps = append(buildLog.Steps, fatalStep)

	_ = elh.sendBuildFinishedEvent(ctx, "failed")
	_ = elh.sendBuildJobLogEvent(ctx, buildLog)
	_ = elh.sendBuildCleanEvent(ctx, "failed")

	if elh.runAsJob {
		log.Error().Err(err).Msg(message)
		os.Exit(0)
	} else {
		log.Fatal().Err(err).Msg(message)
	}
}

func (elh *endOfLifeHelperImpl) sendBuildJobLogEvent(ctx context.Context, buildLog contracts.BuildLog) (err error) {

	err = elh.sendBuildJobLogEventCore(ctx, buildLog)

	if err == nil {
		return
	}

	// strip log lines from successful steps to reduce size of the logs and still keep the useful information
	slimBuildLog := buildLog
	slimBuildLog.Steps = []contracts.BuildLogStep{}
	for _, s := range buildLog.Steps {
		slimBuildLogStep := s
		if s.Status == "SUCCEEDED" {
			if len(s.LogLines) > 0 {
				slimBuildLogStep.LogLines = []contracts.BuildLogLine{
					contracts.BuildLogLine{
						LineNumber: s.LogLines[0].LineNumber,
						Timestamp:  s.LogLines[0].Timestamp,
						StreamType: "stdout",
						Text:       "Truncated logs for reducing total log size; to prevent this use less verbose logging",
					},
				}
			}
		}

		slimBuildLog.Steps = append(slimBuildLog.Steps, slimBuildLogStep)
	}

	return elh.sendBuildJobLogEventCore(ctx, slimBuildLog)
}

func (elh *endOfLifeHelperImpl) sendBuildJobLogEventCore(ctx context.Context, buildLog contracts.BuildLog) (err error) {

	span, ctx := opentracing.StartSpanFromContext(ctx, "SendLog")
	defer span.Finish()

	ciServerBuilderPostLogsURL := elh.config.CIServer.PostLogsURL
	ciAPIKey := elh.config.CIServer.APIKey
	jobName := *elh.config.JobName

	if ciServerBuilderPostLogsURL != "" && ciAPIKey != "" && jobName != "" {

		// convert BuildJobLogs to json
		var requestBody io.Reader

		var data []byte
		if *elh.config.Action == "release" {
			// copy buildLog to releaseLog and marshal that
			releaseLog := contracts.ReleaseLog{
				ID:         buildLog.ID,
				RepoSource: buildLog.RepoSource,
				RepoOwner:  buildLog.RepoOwner,
				RepoName:   buildLog.RepoName,
				ReleaseID:  strconv.Itoa(elh.config.ReleaseParams.ReleaseID),
				Steps:      buildLog.Steps,
				InsertedAt: buildLog.InsertedAt,
			}
			data, err = json.Marshal(releaseLog)
			if err != nil {
				log.Error().Err(err).Msgf("Failed marshalling ReleaseLog for job %v", jobName)
				return
			}
		} else {
			data, err = json.Marshal(buildLog)
			if err != nil {
				log.Error().Err(err).Msgf("Failed marshalling BuildLog for job %v", jobName)
				return
			}
		}

		requestBody = bytes.NewReader(data)

		// create client, in order to add headers
		client := pester.New()
		client.MaxRetries = 5
		client.Backoff = pester.ExponentialJitterBackoff
		client.KeepLog = true
		client.Timeout = time.Second * 120
		request, err := http.NewRequest("POST", ciServerBuilderPostLogsURL, requestBody)
		if err != nil {
			log.Error().Err(err).Msgf("Failed creating http client for job %v", jobName)
			return err
		}

		// add tracing context
		ext.SpanKindRPCClient.Set(span)
		ext.HTTPUrl.Set(span, request.URL.String())
		ext.HTTPMethod.Set(span, request.Method)
		span.Tracer().Inject(
			span.Context(),
			opentracing.HTTPHeaders,
			opentracing.HTTPHeadersCarrier(request.Header),
		)

		// add headers
		request.Header.Add("Authorization", fmt.Sprintf("Bearer %v", ciAPIKey))
		request.Header.Add("Content-Type", "application/json")

		// perform actual request
		response, err := client.Do(request)
		if err != nil {
			log.Error().Err(err).Str("logs", client.LogString()).Msgf("Failed shipping logs to %v for job %v", ciServerBuilderPostLogsURL, jobName)
			return err
		}

		defer response.Body.Close()

		log.Debug().Str("logs", client.LogString()).Msgf("Successfully shipped logs to %v for job %v", ciServerBuilderPostLogsURL, jobName)
	}

	return nil
}

func (elh *endOfLifeHelperImpl) sendBuildFinishedEvent(ctx context.Context, buildStatus string) error {
	return elh.sendBuilderEvent(ctx, buildStatus, fmt.Sprintf("builder:%v", buildStatus))
}

func (elh *endOfLifeHelperImpl) sendBuildCleanEvent(ctx context.Context, buildStatus string) error {
	return elh.sendBuilderEvent(ctx, buildStatus, "builder:clean")
}

func (elh *endOfLifeHelperImpl) sendBuilderEvent(ctx context.Context, buildStatus, event string) (err error) {

	span, ctx := opentracing.StartSpanFromContext(ctx, "SendBuildStatus")
	defer span.Finish()
	span.SetTag("build-status", buildStatus)
	span.SetTag("event-type", event)

	ciServerBuilderEventsURL := elh.config.CIServer.BuilderEventsURL
	ciAPIKey := elh.config.CIServer.APIKey
	jobName := *elh.config.JobName

	if ciServerBuilderEventsURL != "" && ciAPIKey != "" && jobName != "" {
		// convert EstafetteCiBuilderEvent to json
		var requestBody io.Reader

		buildID := ""
		if elh.config.BuildParams != nil {
			buildID = strconv.Itoa(elh.config.BuildParams.BuildID)
		}

		releaseID := ""
		if elh.config.ReleaseParams != nil {
			releaseID = strconv.Itoa(elh.config.ReleaseParams.ReleaseID)
		}

		ciBuilderEvent := EstafetteCiBuilderEvent{
			JobName:      jobName,
			RepoSource:   elh.config.Git.RepoSource,
			RepoOwner:    elh.config.Git.RepoOwner,
			RepoName:     elh.config.Git.RepoName,
			RepoBranch:   elh.config.Git.RepoBranch,
			RepoRevision: elh.config.Git.RepoRevision,
			ReleaseID:    releaseID,
			BuildID:      buildID,
			BuildStatus:  buildStatus,
		}
		data, err := json.Marshal(ciBuilderEvent)
		if err != nil {
			log.Error().Err(err).Msgf("Failed marshalling EstafetteCiBuilderEvent for job %v", jobName)
			return err
		}
		requestBody = bytes.NewReader(data)

		// create client, in order to add headers
		client := pester.New()
		client.MaxRetries = 5
		client.Backoff = pester.ExponentialJitterBackoff
		client.KeepLog = true
		client.Timeout = time.Second * 10
		request, err := http.NewRequest("POST", ciServerBuilderEventsURL, requestBody)
		if err != nil {
			log.Error().Err(err).Msgf("Failed creating http client for job %v", jobName)
			return err
		}

		// add tracing context
		ext.SpanKindRPCClient.Set(span)
		ext.HTTPUrl.Set(span, request.URL.String())
		ext.HTTPMethod.Set(span, request.Method)
		span.Tracer().Inject(
			span.Context(),
			opentracing.HTTPHeaders,
			opentracing.HTTPHeadersCarrier(request.Header),
		)

		// add headers
		request.Header.Add("X-Estafette-Event", event)
		request.Header.Add("X-Estafette-Event-Job-Name", jobName)
		request.Header.Add("Authorization", fmt.Sprintf("Bearer %v", ciAPIKey))

		// perform actual request
		response, err := client.Do(request)
		if err != nil {
			log.Error().Err(err).Str("pesterLogs", client.LogString()).Msgf("Failed performing http request to %v for job %v", ciServerBuilderEventsURL, jobName)
			return err
		}

		defer response.Body.Close()

		log.Debug().Str("pesterLogs", client.LogString()).Str("url", ciServerBuilderEventsURL).Msgf("Succesfully sent %v event to api", event)
	}

	return nil
}
<commit_msg>set http status code tag after requests<commit_after>package main

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"
	"strconv"
	"time"

	contracts "github.com/estafette/estafette-ci-contracts"
	"github.com/opentracing/opentracing-go"
	"github.com/opentracing/opentracing-go/ext"

	"github.com/rs/zerolog/log"
	"github.com/sethgrid/pester"
)

// EndOfLifeHelper has methods to shutdown the runner after a fatal or successful run
type EndOfLifeHelper interface {
	handleFatal(context.Context, contracts.BuildLog, error, string)
	sendBuildFinishedEvent(ctx context.Context, buildStatus string) error
	sendBuildCleanEvent(ctx context.Context, buildStatus string) error
	sendBuildJobLogEvent(ctx context.Context, buildLog contracts.BuildLog) error
}

type endOfLifeHelperImpl struct {
	runAsJob bool
	config   contracts.BuilderConfig
}

// NewEndOfLifeHelper returns a new EndOfLifeHelper
func NewEndOfLifeHelper(runAsJob bool, config contracts.BuilderConfig) EndOfLifeHelper {
	return &endOfLifeHelperImpl{
		runAsJob: runAsJob,
		config:   config,
	}
}

func (elh *endOfLifeHelperImpl) handleFatal(ctx context.Context, buildLog contracts.BuildLog, err error, message string) {

	// add error messages as step to show in logs
	fatalStep := contracts.BuildLogStep{
		Step:     "init",
		LogLines: []contracts.BuildLogLine{},
		ExitCode: -1,
		Status:   "FAILED",
	}
	lineNumber := 1

	if err != nil {
		fatalStep.LogLines = append(fatalStep.LogLines, contracts.BuildLogLine{
			LineNumber: lineNumber,
			Timestamp:  time.Now().UTC(),
			StreamType: "stderr",
			Text:       err.Error(),
		})
		lineNumber++
	}
	if message != "" {
		fatalStep.LogLines = append(fatalStep.LogLines, contracts.BuildLogLine{
			LineNumber: lineNumber,
			Timestamp:  time.Now().UTC(),
			StreamType: "stderr",
			Text:       message,
		})
		lineNumber++
	}

	buildLog.Steps = append(buildLog.Steps, fatalStep)

	_ = elh.sendBuildFinishedEvent(ctx, "failed")
	_ = elh.sendBuildJobLogEvent(ctx, buildLog)
	_ = elh.sendBuildCleanEvent(ctx, "failed")

	if elh.runAsJob {
		log.Error().Err(err).Msg(message)
		os.Exit(0)
	} else {
		log.Fatal().Err(err).Msg(message)
	}
}

func (elh *endOfLifeHelperImpl) sendBuildJobLogEvent(ctx context.Context, buildLog contracts.BuildLog) (err error) {

	err = elh.sendBuildJobLogEventCore(ctx, buildLog)

	if err == nil {
		return
	}

	// strip log lines from successful steps to reduce size of the logs and still keep the useful information
	slimBuildLog := buildLog
	slimBuildLog.Steps = []contracts.BuildLogStep{}
	for _, s := range buildLog.Steps {
		slimBuildLogStep := s
		if s.Status == "SUCCEEDED" {
			if len(s.LogLines) > 0 {
				slimBuildLogStep.LogLines = []contracts.BuildLogLine{
					contracts.BuildLogLine{
						LineNumber: s.LogLines[0].LineNumber,
						Timestamp:  s.LogLines[0].Timestamp,
						StreamType: "stdout",
						Text:       "Truncated logs for reducing total log size; to prevent this use less verbose logging",
					},
				}
			}
		}

		slimBuildLog.Steps = append(slimBuildLog.Steps, slimBuildLogStep)
	}

	return elh.sendBuildJobLogEventCore(ctx, slimBuildLog)
}

func (elh *endOfLifeHelperImpl) sendBuildJobLogEventCore(ctx context.Context, buildLog contracts.BuildLog) (err error) {

	span, ctx := opentracing.StartSpanFromContext(ctx, "SendLog")
	defer span.Finish()

	ciServerBuilderPostLogsURL := elh.config.CIServer.PostLogsURL
	ciAPIKey := elh.config.CIServer.APIKey
	jobName := *elh.config.JobName

	if ciServerBuilderPostLogsURL != "" && ciAPIKey != "" && jobName != "" {

		// convert BuildJobLogs to json
		var requestBody io.Reader

		var data []byte
		if *elh.config.Action == "release" {
			// copy buildLog to releaseLog and marshal that
			releaseLog := contracts.ReleaseLog{
				ID:         buildLog.ID,
				RepoSource: buildLog.RepoSource,
				RepoOwner:  buildLog.RepoOwner,
				RepoName:   buildLog.RepoName,
				ReleaseID:  strconv.Itoa(elh.config.ReleaseParams.ReleaseID),
				Steps:      buildLog.Steps,
				InsertedAt: buildLog.InsertedAt,
			}
			data, err = json.Marshal(releaseLog)
			if err != nil {
				log.Error().Err(err).Msgf("Failed marshalling ReleaseLog for job %v", jobName)
				return
			}
		} else {
			data, err = json.Marshal(buildLog)
			if err != nil {
				log.Error().Err(err).Msgf("Failed marshalling BuildLog for job %v", jobName)
				return
			}
		}

		requestBody = bytes.NewReader(data)

		// create client, in order to add headers
		client := pester.New()
		client.MaxRetries = 5
		client.Backoff = pester.ExponentialJitterBackoff
		client.KeepLog = true
		client.Timeout = time.Second * 120
		request, err := http.NewRequest("POST", ciServerBuilderPostLogsURL, requestBody)
		if err != nil {
			log.Error().Err(err).Msgf("Failed creating http client for job %v", jobName)
			return err
		}

		// add tracing context
		ext.SpanKindRPCClient.Set(span)
		ext.HTTPUrl.Set(span, request.URL.String())
		ext.HTTPMethod.Set(span, request.Method)
		span.Tracer().Inject(
			span.Context(),
			opentracing.HTTPHeaders,
			opentracing.HTTPHeadersCarrier(request.Header),
		)

		// add headers
		request.Header.Add("Authorization", fmt.Sprintf("Bearer %v", ciAPIKey))
		request.Header.Add("Content-Type", "application/json")

		// perform actual request
		response, err := client.Do(request)
		if err != nil {
			log.Error().Err(err).Str("logs", client.LogString()).Msgf("Failed shipping logs to %v for job %v", ciServerBuilderPostLogsURL, jobName)
			return err
		}

		ext.HTTPStatusCode.Set(span, uint16(response.StatusCode))

		defer response.Body.Close()

		log.Debug().Str("logs", client.LogString()).Msgf("Successfully shipped logs to %v for job %v", ciServerBuilderPostLogsURL, jobName)
	}

	return nil
}

func (elh *endOfLifeHelperImpl) sendBuildFinishedEvent(ctx context.Context, buildStatus string) error {
	return elh.sendBuilderEvent(ctx, buildStatus, fmt.Sprintf("builder:%v", buildStatus))
}

func (elh *endOfLifeHelperImpl) sendBuildCleanEvent(ctx context.Context, buildStatus string) error {
	return elh.sendBuilderEvent(ctx, buildStatus, "builder:clean")
}

func (elh *endOfLifeHelperImpl) sendBuilderEvent(ctx context.Context, buildStatus, event string) (err error) {

	span, ctx := opentracing.StartSpanFromContext(ctx, "SendBuildStatus")
	defer span.Finish()
	span.SetTag("build-status", buildStatus)
	span.SetTag("event-type", event)

	ciServerBuilderEventsURL := elh.config.CIServer.BuilderEventsURL
	ciAPIKey := elh.config.CIServer.APIKey
	jobName := *elh.config.JobName

	if ciServerBuilderEventsURL != "" && ciAPIKey != "" && jobName != "" {
		// convert EstafetteCiBuilderEvent to json
		var requestBody io.Reader

		buildID := ""
		if elh.config.BuildParams != nil {
			buildID = strconv.Itoa(elh.config.BuildParams.BuildID)
		}

		releaseID := ""
		if elh.config.ReleaseParams != nil {
			releaseID = strconv.Itoa(elh.config.ReleaseParams.ReleaseID)
		}

		ciBuilderEvent := EstafetteCiBuilderEvent{
			JobName:      jobName,
			RepoSource:   elh.config.Git.RepoSource,
			RepoOwner:    elh.config.Git.RepoOwner,
			RepoName:     elh.config.Git.RepoName,
			RepoBranch:   elh.config.Git.RepoBranch,
			RepoRevision: elh.config.Git.RepoRevision,
			ReleaseID:    releaseID,
			BuildID:      buildID,
			BuildStatus:  buildStatus,
		}
		data, err := json.Marshal(ciBuilderEvent)
		if err != nil {
			log.Error().Err(err).Msgf("Failed marshalling EstafetteCiBuilderEvent for job %v", jobName)
			return err
		}
		requestBody = bytes.NewReader(data)

		// create client, in order to add headers
		client := pester.New()
		client.MaxRetries = 5
		client.Backoff = pester.ExponentialJitterBackoff
		client.KeepLog = true
		client.Timeout = time.Second * 10
		request, err := http.NewRequest("POST", ciServerBuilderEventsURL, requestBody)
		if err != nil {
			log.Error().Err(err).Msgf("Failed creating http client for job %v", jobName)
			return err
		}

		// add tracing context
		ext.SpanKindRPCClient.Set(span)
		ext.HTTPUrl.Set(span, request.URL.String())
		ext.HTTPMethod.Set(span, request.Method)
		span.Tracer().Inject(
			span.Context(),
			opentracing.HTTPHeaders,
			opentracing.HTTPHeadersCarrier(request.Header),
		)

		// add headers
		request.Header.Add("X-Estafette-Event", event)
		request.Header.Add("X-Estafette-Event-Job-Name", jobName)
		request.Header.Add("Authorization", fmt.Sprintf("Bearer %v", ciAPIKey))

		// perform actual request
		response, err := client.Do(request)
		if err != nil {
			log.Error().Err(err).Str("pesterLogs", client.LogString()).Msgf("Failed performing http request to %v for job %v", ciServerBuilderEventsURL, jobName)
			return err
		}

		ext.HTTPStatusCode.Set(span, uint16(response.StatusCode))

		defer response.Body.Close()

		log.Debug().Str("pesterLogs", client.LogString()).Str("url", ciServerBuilderEventsURL).Msgf("Succesfully sent %v event to api", event)
	}

	return nil
}
<|endoftext|>


<commit_before>// pcsclite binding
//
// http://pcsclite.alioth.debian.org/pcsclite.html
// http://pcsclite.alioth.debian.org/api/group__API.html
//
package scard

// #cgo pkg-config: libpcsclite
// #include <stdlib.h>
// #include <winscard.h>
import "C"

import (
	"bytes"
	"unsafe"
)

// Version returns the libpcsclite version string
func Version() string {
	return C.PCSCLITE_VERSION_NUMBER
}

type Context struct {
	ctx C.SCARDCONTEXT
}

type Card struct {
	handle         C.SCARDHANDLE
	activeProtocol Protocol
}

type Protocol uint32

const (
	PROTOCOL_UNDEFINED Protocol = C.SCARD_PROTOCOL_UNDEFINED
	PROTOCOL_T0        Protocol = C.SCARD_PROTOCOL_T0
	PROTOCOL_T1        Protocol = C.SCARD_PROTOCOL_T1
	PROTOCOL_RAW       Protocol = C.SCARD_PROTOCOL_RAW
	PROTOCOL_ANY       Protocol = C.SCARD_PROTOCOL_ANY
)

type ShareMode uint32

const (
	SHARE_EXCLUSIVE ShareMode = C.SCARD_SHARE_EXCLUSIVE
	SHARE_SHARED    ShareMode = C.SCARD_SHARE_SHARED
	SHARE_DIRECT    ShareMode = C.SCARD_SHARE_DIRECT
)

type Disposition uint32

const (
	LEAVE_CARD   Disposition = C.SCARD_LEAVE_CARD
	RESET_CARD   Disposition = C.SCARD_RESET_CARD
	UNPOWER_CARD Disposition = C.SCARD_UNPOWER_CARD
	EJECT_CARD   Disposition = C.SCARD_EJECT_CARD
)

type CardStatus struct {
	Reader         string
	State          uint32
	ActiveProtocol Protocol
	ATR            []byte
}

type ReaderState struct {
	Reader       string
	UserData     interface{}
	CurrentState StateFlag
	EventState   StateFlag
	// TODO: ATR
}

type StateFlag uint32

const (
	STATE_UNAWARE     StateFlag = C.SCARD_STATE_UNAWARE
	STATE_IGNORE      StateFlag = C.SCARD_STATE_IGNORE
	STATE_CHANGED     StateFlag = C.SCARD_STATE_CHANGED
	STATE_UNKNOWN     StateFlag = C.SCARD_STATE_UNKNOWN
	STATE_UNAVAILABLE StateFlag = C.SCARD_STATE_UNAVAILABLE
	STATE_EMPTY       StateFlag = C.SCARD_STATE_EMPTY
	STATE_PRESENT     StateFlag = C.SCARD_STATE_PRESENT
	STATE_ATRMATCH    StateFlag = C.SCARD_STATE_ATRMATCH
	STATE_EXCLUSIVE   StateFlag = C.SCARD_STATE_EXCLUSIVE
	STATE_INUSE       StateFlag = C.SCARD_STATE_INUSE
	STATE_MUTE        StateFlag = C.SCARD_STATE_MUTE
	STATE_UNPOWERED   StateFlag = C.SCARD_STATE_UNPOWERED
)

type Timeout uint32

const (
	INFINITE Timeout = C.INFINITE
)

// wraps SCardEstablishContext
func EstablishContext() (*Context, error) {
	var ctx Context

	r := C.SCardEstablishContext(C.SCARD_SCOPE_SYSTEM, nil, nil, &ctx.ctx)
	if r != C.SCARD_S_SUCCESS {
		return nil, newError(r)
	}

	return &ctx, nil
}

// wraps SCardIsValidContext
func (ctx *Context) IsValid() (bool, error) {
	r := C.SCardIsValidContext(ctx.ctx)
	switch r {
	case C.SCARD_S_SUCCESS:
		return true, nil
	case C.SCARD_E_INVALID_HANDLE:
		return false, nil
	default:
		return false, newError(r)
	}
	panic("unreachable")
}

// wraps SCardCancel
func (ctx *Context) Cancel() error {
	r := C.SCardCancel(ctx.ctx)
	if r != C.SCARD_S_SUCCESS {
		return newError(r)
	}
	return nil
}

// wraps SCardReleaseContext
func (ctx *Context) Release() error {
	r := C.SCardReleaseContext(ctx.ctx)
	if r != C.SCARD_S_SUCCESS {
		return newError(r)
	}
	return nil
}

// wraps SCardListReaders
func (ctx *Context) ListReaders() ([]string, error) {
	var needed C.DWORD

	r := C.SCardListReaders(ctx.ctx, nil, nil, &needed)
	if r != C.SCARD_S_SUCCESS {
		return nil, newError(r)
	}

	data := make([]byte, needed)
	cdata := (*C.char)(unsafe.Pointer(&data[0]))

	r = C.SCardListReaders(ctx.ctx, nil, cdata, &needed)
	if r != C.SCARD_S_SUCCESS {
		return nil, newError(r)
	}

	var readers []string
	for _, b := range bytes.Split(data, []byte{0}) {
		if len(b) > 0 {
			readers = append(readers, string(b))
		}
	}

	return readers, nil
}

// wraps SCardListReaderGroups
func (ctx *Context) ListReaderGroups() ([]string, error) {
	var needed C.DWORD

	r := C.SCardListReaderGroups(ctx.ctx, nil, &needed)
	if r != C.SCARD_S_SUCCESS {
		return nil, newError(r)
	}

	data := make([]byte, needed)
	cdata := (*C.char)(unsafe.Pointer(&data[0]))

	r = C.SCardListReaderGroups(ctx.ctx, cdata, &needed)
	if r != C.SCARD_S_SUCCESS {
		return nil, newError(r)
	}

	var groups []string
	for _, b := range bytes.Split(data, []byte{0}) {
		if len(b) > 0 {
			groups = append(groups, string(b))
		}
	}

	return groups, nil
}

// wraps SCardGetStatusChange
func (ctx *Context) GetStatusChange(readerStates []ReaderState, timeout Timeout) error {

	crs := make([]C.SCARD_READERSTATE, len(readerStates))

	for i := range readerStates {
		crs[i].szReader = C.CString(readerStates[i].Reader)
		defer C.free(unsafe.Pointer(crs[i].szReader))
		crs[i].dwCurrentState = C.DWORD(readerStates[i].CurrentState)
	}

	r := C.SCardGetStatusChange(ctx.ctx, C.DWORD(timeout),
		(C.LPSCARD_READERSTATE)(unsafe.Pointer(&crs[0])),
		C.DWORD(len(crs)))

	if r != C.SCARD_S_SUCCESS {
		return newError(r)
	}

	for i := range readerStates {
		readerStates[i].EventState = StateFlag(crs[i].dwEventState)
	}

	return nil
}

// wraps SCardConnect
func (ctx *Context) Connect(reader string, mode ShareMode, proto Protocol) (*Card, error) {
	var card Card
	var activeProtocol C.DWORD

	creader := C.CString(reader)
	defer C.free(unsafe.Pointer(creader))

	r := C.SCardConnect(ctx.ctx, creader, C.DWORD(mode), C.DWORD(proto), &card.handle, &activeProtocol)
	if r != C.SCARD_S_SUCCESS {
		return nil, newError(r)
	}

	card.activeProtocol = Protocol(activeProtocol)
	return &card, nil
}

// wraps SCardDisconnect
func (card *Card) Disconnect(d Disposition) error {
	r := C.SCardDisconnect(card.handle, C.DWORD(d))
	if r != C.SCARD_S_SUCCESS {
		return newError(r)
	}
	return nil
}

// wraps SCardReconnect
func (card *Card) Reconnect(mode ShareMode, protocol Protocol, init Disposition) error {
	var activeProtocol C.DWORD

	r := C.SCardReconnect(card.handle, C.DWORD(mode), C.DWORD(protocol), C.DWORD(init), &activeProtocol)
	if r != C.SCARD_S_SUCCESS {
		return newError(r)
	}

	card.activeProtocol = Protocol(activeProtocol)

	return nil
}

// wraps SCardBeginTransaction
func (card *Card) BeginTransaction() error {
	r := C.SCardBeginTransaction(card.handle)
	if r != C.SCARD_S_SUCCESS {
		return newError(r)
	}
	return nil
}

// wraps SCardEndTransaction
func (card *Card) EndTransaction(d Disposition) error {
	r := C.SCardEndTransaction(card.handle, C.DWORD(d))
	if r != C.SCARD_S_SUCCESS {
		return newError(r)
	}
	return nil
}

// wraps SCardStatus
func (card *Card) Status() (*CardStatus, error) {
	var reader [C.MAX_READERNAME + 1]byte
	var readerLen = C.DWORD(len(reader))
	var state, proto C.DWORD
	var atr [C.MAX_ATR_SIZE]byte
	var atrLen = C.DWORD(len(atr))

	r := C.SCardStatus(card.handle, (C.LPSTR)(unsafe.Pointer(&reader[0])), &readerLen, &state, &proto, (*C.BYTE)(&atr[0]), &atrLen)
	if r != C.SCARD_S_SUCCESS {
		return nil, newError(r)
	}

	status := &CardStatus{
		Reader:         string(reader[0:readerLen]),
		State:          uint32(state),
		ActiveProtocol: Protocol(proto),
		ATR:            atr[0:atrLen],
	}

	return status, nil
}

// wraps SCardTransmit
func (card *Card) Transmit(cmd []byte) ([]byte, error) {
	var sendpci *C.SCARD_IO_REQUEST
	var recvpci C.SCARD_IO_REQUEST

	switch card.activeProtocol {
	case PROTOCOL_T0:
		sendpci = &C.g_rgSCardT0Pci
	case PROTOCOL_T1:
		sendpci = &C.g_rgSCardT1Pci
	case PROTOCOL_RAW:
		sendpci = &C.g_rgSCardRawPci
	default:
		panic("unknown protocol")
	}

	var recv [C.MAX_BUFFER_SIZE_EXTENDED]byte
	var recvlen C.DWORD = C.DWORD(len(recv))

	r := C.SCardTransmit(card.handle, sendpci, (*C.BYTE)(&cmd[0]), C.DWORD(len(cmd)), &recvpci, (*C.BYTE)(&recv[0]), &recvlen)
	if r != C.SCARD_S_SUCCESS {
		return nil, newError(r)
	}

	rsp := make([]byte, recvlen)
	copy(rsp, recv[0:recvlen])

	return rsp, nil
}

// wraps SCardControl
func (card *Card) Control(ctrl uint32, cmd []byte) ([]byte, error) {
	var recv [C.MAX_BUFFER_SIZE_EXTENDED]byte
	var recvlen C.DWORD
	var r C.LONG

	if len(cmd) == 0 {
		r = C.SCardControl(card.handle, C.DWORD(ctrl),
			(C.LPCVOID)(nil), 0,
			(C.LPVOID)(unsafe.Pointer(&recv[0])), C.DWORD(len(recv)), &recvlen)
	} else {
		r = C.SCardControl(card.handle, C.DWORD(ctrl),
			(C.LPCVOID)(unsafe.Pointer(&cmd[0])), C.DWORD(len(cmd)),
			(C.LPVOID)(unsafe.Pointer(&recv[0])), C.DWORD(len(recv)), &recvlen)
	}
	if r != C.SCARD_S_SUCCESS {
		return nil, newError(r)
	}

	rsp := make([]byte, recvlen)
	copy(rsp, recv[0:recvlen])

	return rsp, nil
}

// wraps SCardGetAttrib
func (card *Card) GetAttrib(id uint32) ([]byte, error) {
	var needed C.DWORD

	r := C.SCardGetAttrib(card.handle, C.DWORD(id), nil, &needed)
	if r != C.SCARD_S_SUCCESS {
		return nil, newError(r)
	}

	var attrib = make([]byte, needed)

	r = C.SCardGetAttrib(card.handle, C.DWORD(id), (*C.BYTE)(&attrib[0]), &needed)
	if r != C.SCARD_S_SUCCESS {
		return nil, newError(r)
	}

	return attrib[0:needed], nil
}

// wraps SCardSetAttrib
func (card *Card) SetAttrib(id uint32, data []byte) error {
	r := C.SCardSetAttrib(card.handle, C.DWORD(id), (*C.BYTE)(&data[0]), C.DWORD(len(data)))
	if r != C.SCARD_S_SUCCESS {
		return newError(r)
	}
	return nil
}

// SCardFreeMemory is not needed. We (hopefuly) never return buffers allocated by libpcsclite
<commit_msg>work around tip cgo strangeness<commit_after>// pcsclite binding
//
// http://pcsclite.alioth.debian.org/pcsclite.html
// http://pcsclite.alioth.debian.org/api/group__API.html
//
package scard

// #cgo pkg-config: libpcsclite
// #include <stdlib.h>
// #include <winscard.h>
import "C"

import (
	"bytes"
	"unsafe"
)

// Version returns the libpcsclite version string
func Version() string {
	return C.PCSCLITE_VERSION_NUMBER
}

type Context struct {
	ctx C.SCARDCONTEXT
}

type Card struct {
	handle         C.SCARDHANDLE
	activeProtocol Protocol
}

type Protocol uint32

const (
	PROTOCOL_UNDEFINED Protocol = C.SCARD_PROTOCOL_UNDEFINED
	PROTOCOL_T0        Protocol = C.SCARD_PROTOCOL_T0
	PROTOCOL_T1        Protocol = C.SCARD_PROTOCOL_T1
	PROTOCOL_RAW       Protocol = C.SCARD_PROTOCOL_RAW
	PROTOCOL_ANY       Protocol = C.SCARD_PROTOCOL_ANY
)

type ShareMode uint32

const (
	SHARE_EXCLUSIVE ShareMode = C.SCARD_SHARE_EXCLUSIVE
	SHARE_SHARED    ShareMode = C.SCARD_SHARE_SHARED
	SHARE_DIRECT    ShareMode = C.SCARD_SHARE_DIRECT
)

type Disposition uint32

const (
	LEAVE_CARD   Disposition = C.SCARD_LEAVE_CARD
	RESET_CARD   Disposition = C.SCARD_RESET_CARD
	UNPOWER_CARD Disposition = C.SCARD_UNPOWER_CARD
	EJECT_CARD   Disposition = C.SCARD_EJECT_CARD
)

type CardStatus struct {
	Reader         string
	State          uint32
	ActiveProtocol Protocol
	ATR            []byte
}

type ReaderState struct {
	Reader       string
	UserData     interface{}
	CurrentState StateFlag
	EventState   StateFlag
	// TODO: ATR
}

type StateFlag uint32

const (
	STATE_UNAWARE     StateFlag = C.SCARD_STATE_UNAWARE
	STATE_IGNORE      StateFlag = C.SCARD_STATE_IGNORE
	STATE_CHANGED     StateFlag = C.SCARD_STATE_CHANGED
	STATE_UNKNOWN     StateFlag = C.SCARD_STATE_UNKNOWN
	STATE_UNAVAILABLE StateFlag = C.SCARD_STATE_UNAVAILABLE
	STATE_EMPTY       StateFlag = C.SCARD_STATE_EMPTY
	STATE_PRESENT     StateFlag = C.SCARD_STATE_PRESENT
	STATE_ATRMATCH    StateFlag = C.SCARD_STATE_ATRMATCH
	STATE_EXCLUSIVE   StateFlag = C.SCARD_STATE_EXCLUSIVE
	STATE_INUSE       StateFlag = C.SCARD_STATE_INUSE
	STATE_MUTE        StateFlag = C.SCARD_STATE_MUTE
	STATE_UNPOWERED   StateFlag = C.SCARD_STATE_UNPOWERED
)

type Timeout uint32

const (
	INFINITE Timeout = C.INFINITE
)

// wraps SCardEstablishContext
func EstablishContext() (*Context, error) {
	var ctx Context

	r := C.SCardEstablishContext(C.SCARD_SCOPE_SYSTEM, nil, nil, &ctx.ctx)
	if r != C.SCARD_S_SUCCESS {
		return nil, newError(r)
	}

	return &ctx, nil
}

// wraps SCardIsValidContext
func (ctx *Context) IsValid() (bool, error) {
	r := C.SCardIsValidContext(ctx.ctx)
	switch r {
	case C.SCARD_S_SUCCESS:
		return true, nil
	case C.SCARD_E_INVALID_HANDLE:
		return false, nil
	default:
		return false, newError(r)
	}
	panic("unreachable")
}

// wraps SCardCancel
func (ctx *Context) Cancel() error {
	r := C.SCardCancel(ctx.ctx)
	if r != C.SCARD_S_SUCCESS {
		return newError(r)
	}
	return nil
}

// wraps SCardReleaseContext
func (ctx *Context) Release() error {
	r := C.SCardReleaseContext(ctx.ctx)
	if r != C.SCARD_S_SUCCESS {
		return newError(r)
	}
	return nil
}

// wraps SCardListReaders
func (ctx *Context) ListReaders() ([]string, error) {
	var needed C.DWORD

	r := C.SCardListReaders(ctx.ctx, nil, nil, &needed)
	if r != C.SCARD_S_SUCCESS {
		return nil, newError(r)
	}

	data := make([]byte, needed)
	cdata := (*C.char)(unsafe.Pointer(&data[0]))

	r = C.SCardListReaders(ctx.ctx, nil, cdata, &needed)
	if r != C.SCARD_S_SUCCESS {
		return nil, newError(r)
	}

	var readers []string
	for _, b := range bytes.Split(data, []byte{0}) {
		if len(b) > 0 {
			readers = append(readers, string(b))
		}
	}

	return readers, nil
}

// wraps SCardListReaderGroups
func (ctx *Context) ListReaderGroups() ([]string, error) {
	var needed C.DWORD

	r := C.SCardListReaderGroups(ctx.ctx, nil, &needed)
	if r != C.SCARD_S_SUCCESS {
		return nil, newError(r)
	}

	data := make([]byte, needed)
	cdata := (*C.char)(unsafe.Pointer(&data[0]))

	r = C.SCardListReaderGroups(ctx.ctx, cdata, &needed)
	if r != C.SCARD_S_SUCCESS {
		return nil, newError(r)
	}

	var groups []string
	for _, b := range bytes.Split(data, []byte{0}) {
		if len(b) > 0 {
			groups = append(groups, string(b))
		}
	}

	return groups, nil
}

// wraps SCardGetStatusChange
func (ctx *Context) GetStatusChange(readerStates []ReaderState, timeout Timeout) error {

	crs := make([]C.SCARD_READERSTATE, len(readerStates))

	for i := range readerStates {
		crs[i].szReader = C.CString(readerStates[i].Reader)
		defer C.free(unsafe.Pointer(crs[i].szReader))
		crs[i].dwCurrentState = C.DWORD(readerStates[i].CurrentState)
	}

	r := C.SCardGetStatusChange(ctx.ctx, C.DWORD(timeout),
		(C.LPSCARD_READERSTATE)(unsafe.Pointer(&crs[0])),
		C.DWORD(len(crs)))

	if r != C.SCARD_S_SUCCESS {
		return newError(r)
	}

	for i := range readerStates {
		readerStates[i].EventState = StateFlag(crs[i].dwEventState)
	}

	return nil
}

// wraps SCardConnect
func (ctx *Context) Connect(reader string, mode ShareMode, proto Protocol) (*Card, error) {
	var card Card
	var activeProtocol C.DWORD

	creader := C.CString(reader)
	defer C.free(unsafe.Pointer(creader))

	r := C.SCardConnect(ctx.ctx, creader, C.DWORD(mode), C.DWORD(proto), &card.handle, &activeProtocol)
	if r != C.SCARD_S_SUCCESS {
		return nil, newError(r)
	}

	card.activeProtocol = Protocol(activeProtocol)
	return &card, nil
}

// wraps SCardDisconnect
func (card *Card) Disconnect(d Disposition) error {
	r := C.SCardDisconnect(card.handle, C.DWORD(d))
	if r != C.SCARD_S_SUCCESS {
		return newError(r)
	}
	return nil
}

// wraps SCardReconnect
func (card *Card) Reconnect(mode ShareMode, protocol Protocol, init Disposition) error {
	var activeProtocol C.DWORD

	r := C.SCardReconnect(card.handle, C.DWORD(mode), C.DWORD(protocol), C.DWORD(init), &activeProtocol)
	if r != C.SCARD_S_SUCCESS {
		return newError(r)
	}

	card.activeProtocol = Protocol(activeProtocol)

	return nil
}

// wraps SCardBeginTransaction
func (card *Card) BeginTransaction() error {
	r := C.SCardBeginTransaction(card.handle)
	if r != C.SCARD_S_SUCCESS {
		return newError(r)
	}
	return nil
}

// wraps SCardEndTransaction
func (card *Card) EndTransaction(d Disposition) error {
	r := C.SCardEndTransaction(card.handle, C.DWORD(d))
	if r != C.SCARD_S_SUCCESS {
		return newError(r)
	}
	return nil
}

// wraps SCardStatus
func (card *Card) Status() (*CardStatus, error) {
	var reader [C.MAX_READERNAME + 1]byte
	var readerLen = C.DWORD(len(reader))
	var state, proto C.DWORD
	var atr [C.MAX_ATR_SIZE]byte
	var atrLen = C.DWORD(len(atr))

	r := C.SCardStatus(card.handle, (C.LPSTR)(unsafe.Pointer(&reader[0])), &readerLen, &state, &proto, (*C.BYTE)(&atr[0]), &atrLen)
	if r != C.SCARD_S_SUCCESS {
		return nil, newError(r)
	}

	status := &CardStatus{
		Reader:         string(reader[0:readerLen]),
		State:          uint32(state),
		ActiveProtocol: Protocol(proto),
		ATR:            atr[0:atrLen],
	}

	return status, nil
}

// wraps SCardTransmit
func (card *Card) Transmit(cmd []byte) ([]byte, error) {
	var sendpci C.SCARD_IO_REQUEST
	var recvpci C.SCARD_IO_REQUEST

	switch card.activeProtocol {
	case PROTOCOL_T0:
		sendpci.dwProtocol = C.SCARD_PROTOCOL_T0
	case PROTOCOL_T1:
		sendpci.dwProtocol = C.SCARD_PROTOCOL_T1
	case PROTOCOL_RAW:
		sendpci.dwProtocol = C.SCARD_PROTOCOL_RAW
	default:
		panic("unknown protocol")
	}
	sendpci.cbPciLength = C.sizeof_SCARD_IO_REQUEST

	var recv [C.MAX_BUFFER_SIZE_EXTENDED]byte
	var recvlen C.DWORD = C.DWORD(len(recv))

	r := C.SCardTransmit(card.handle, &sendpci, (*C.BYTE)(&cmd[0]), C.DWORD(len(cmd)), &recvpci, (*C.BYTE)(&recv[0]), &recvlen)
	if r != C.SCARD_S_SUCCESS {
		return nil, newError(r)
	}

	rsp := make([]byte, recvlen)
	copy(rsp, recv[0:recvlen])

	return rsp, nil
}

// wraps SCardControl
func (card *Card) Control(ctrl uint32, cmd []byte) ([]byte, error) {
	var recv [C.MAX_BUFFER_SIZE_EXTENDED]byte
	var recvlen C.DWORD
	var r C.LONG

	if len(cmd) == 0 {
		r = C.SCardControl(card.handle, C.DWORD(ctrl),
			(C.LPCVOID)(nil), 0,
			(C.LPVOID)(unsafe.Pointer(&recv[0])), C.DWORD(len(recv)), &recvlen)
	} else {
		r = C.SCardControl(card.handle, C.DWORD(ctrl),
			(C.LPCVOID)(unsafe.Pointer(&cmd[0])), C.DWORD(len(cmd)),
			(C.LPVOID)(unsafe.Pointer(&recv[0])), C.DWORD(len(recv)), &recvlen)
	}
	if r != C.SCARD_S_SUCCESS {
		return nil, newError(r)
	}

	rsp := make([]byte, recvlen)
	copy(rsp, recv[0:recvlen])

	return rsp, nil
}

// wraps SCardGetAttrib
func (card *Card) GetAttrib(id uint32) ([]byte, error) {
	var needed C.DWORD

	r := C.SCardGetAttrib(card.handle, C.DWORD(id), nil, &needed)
	if r != C.SCARD_S_SUCCESS {
		return nil, newError(r)
	}

	var attrib = make([]byte, needed)

	r = C.SCardGetAttrib(card.handle, C.DWORD(id), (*C.BYTE)(&attrib[0]), &needed)
	if r != C.SCARD_S_SUCCESS {
		return nil, newError(r)
	}

	return attrib[0:needed], nil
}

// wraps SCardSetAttrib
func (card *Card) SetAttrib(id uint32, data []byte) error {
	r := C.SCardSetAttrib(card.handle, C.DWORD(id), (*C.BYTE)(&data[0]), C.DWORD(len(data)))
	if r != C.SCARD_S_SUCCESS {
		return newError(r)
	}
	return nil
}

// SCardFreeMemory is not needed. We (hopefuly) never return buffers allocated by libpcsclite
<|endoftext|>


<commit_before>// Copyright 2012 Arne Roomann-Kurrik
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package twodee

import (
	"github.com/banthar/gl"
	"image"
	"image/png"
	"os"
)

type Node interface {
	AddChild(node Node)
	Parent() Node
	SetParent(Node)
	Draw()
	GlobalX() float32
	GlobalY() float32
}

type Element struct {
	Children []Node
	parent Node
	X float32
	Y float32
}

func (e *Element) AddChild(node Node) {
	node.SetParent(e)
	e.Children = append(e.Children, node)
}

func (e *Element) SetParent(node Node) {
	e.parent = node
}

func (e *Element) Parent() Node {
	return e.parent
}

func (e *Element) Draw() {
	for _, child := range e.Children {
		child.Draw()
	}
}

func (e *Element) GlobalX() float32 {
	if e.parent == nil {
		return e.X
	}
	return e.parent.GlobalX() + e.X
}

func (e *Element) GlobalY() float32 {
	if e.parent == nil {
		return e.Y
	}
	return e.parent.GlobalY() + e.Y
}

type Scene struct {
	Element
}

type Sprite struct {
	Element
	system  *System
	texture *Texture
	Width   int
	Height  int
	Frames  int
	Frame   int
}

func (s *System) NewSprite(name string, x float32, y float32, w int, h int, frames int) *Sprite {
	if frames <= 0 {
		frames = 1
	}
	sprite := &Sprite{
		system:  s,
		texture: s.Textures[name],
		Width:   w,
		Height:  h,
		Frame:   0,
		Frames:  frames,
	}
	sprite.X = x
	sprite.Y = y
	return sprite
}

func (s *Sprite) Draw() {
	s.Element.Draw()
	var (
		x1         float32 = s.GlobalX()
		y1         float32 = s.GlobalY()
		x2         float32 = x1 + float32(s.Width)
		y2         float32 = y1 + float32(s.Height)
		frame      int     = s.Frame % s.Frames
		framestep  float32 = 1.0 / float32(s.Frames)
		framestart float32 = framestep * float32(frame)
		framestop  float32 = framestep * float32(frame+1)
	)
	s.texture.Bind()
	gl.Begin(gl.QUADS)
	gl.TexCoord2f(framestop, 1)
	gl.Vertex2f(x1, y1)
	gl.TexCoord2f(framestop, 0)
	gl.Vertex2f(x2, y1)
	gl.TexCoord2f(framestart, 0)
	gl.Vertex2f(x2, y2)
	gl.TexCoord2f(framestart, 1)
	gl.Vertex2f(x1, y2)
	gl.End()
	s.texture.Unbind()
}

type Environment struct {
	Element
}

func (s *System) LoadEnvironment(name string, path string) (env *Environment, err error) {
	var (
		file   *os.File
		img    image.Image
		bounds image.Rectangle
		colors map[uint32]int
		sprite *Sprite
	)
	if file, err = os.Open(path); err != nil {
		return
	}
	defer file.Close()
	if img, err = png.Decode(file); err != nil {
		return
	}
	colors = make(map[uint32]int, 0)
	bounds = img.Bounds()
	env = &Environment{}
	for y := 0; y < bounds.Dy(); y++ {
		for x := 0; x < bounds.Dx(); x++ {
			r, g, b, _ := img.At(x, y).RGBA()
			index := (r << 8 + g) << 8 + b
			if _, exists := colors[index]; exists == false {
				colors[index] = len(colors)
			}
			sprite = s.NewSprite(name, float32(x * 32), float32(y * 32), 32, 32, 2)
			sprite.Frame = colors[index]
			env.AddChild(sprite)
		}
	}
	return
}
<commit_msg>Fixing texture coordinates<commit_after>// Copyright 2012 Arne Roomann-Kurrik
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package twodee

import (
	"github.com/banthar/gl"
	"image"
	"image/png"
	"os"
)

type Node interface {
	AddChild(node Node)
	Parent() Node
	SetParent(Node)
	Draw()
	GlobalX() float32
	GlobalY() float32
}

type Element struct {
	Children []Node
	parent Node
	X float32
	Y float32
}

func (e *Element) AddChild(node Node) {
	node.SetParent(e)
	e.Children = append(e.Children, node)
}

func (e *Element) SetParent(node Node) {
	e.parent = node
}

func (e *Element) Parent() Node {
	return e.parent
}

func (e *Element) Draw() {
	for _, child := range e.Children {
		child.Draw()
	}
}

func (e *Element) GlobalX() float32 {
	if e.parent == nil {
		return e.X
	}
	return e.parent.GlobalX() + e.X
}

func (e *Element) GlobalY() float32 {
	if e.parent == nil {
		return e.Y
	}
	return e.parent.GlobalY() + e.Y
}

type Scene struct {
	Element
}

type Sprite struct {
	Element
	system  *System
	texture *Texture
	Width   int
	Height  int
	Frames  int
	Frame   int
}

func (s *System) NewSprite(name string, x float32, y float32, w int, h int, frames int) *Sprite {
	if frames <= 0 {
		frames = 1
	}
	sprite := &Sprite{
		system:  s,
		texture: s.Textures[name],
		Width:   w,
		Height:  h,
		Frame:   0,
		Frames:  frames,
	}
	sprite.X = x
	sprite.Y = y
	return sprite
}

func (s *Sprite) Draw() {
	s.Element.Draw()
	var (
		x1         float32 = s.GlobalX()
		y1         float32 = s.GlobalY()
		x2         float32 = x1 + float32(s.Width)
		y2         float32 = y1 + float32(s.Height)
		frame      int     = s.Frame % s.Frames
		framestep  float32 = 1.0 / float32(s.Frames)
		framestart float32 = framestep * float32(frame)
		framestop  float32 = framestep * float32(frame+1)
	)
	s.texture.Bind()
	gl.Begin(gl.QUADS)
	gl.TexCoord2f(framestart, 1)
	gl.Vertex2f(x1, y1)
	gl.TexCoord2f(framestop, 1)
	gl.Vertex2f(x2, y1)
	gl.TexCoord2f(framestop, 0)
	gl.Vertex2f(x2, y2)
	gl.TexCoord2f(framestart, 0)
	gl.Vertex2f(x1, y2)
	gl.End()
	s.texture.Unbind()
}

type Environment struct {
	Element
}

func (s *System) LoadEnvironment(name string, path string) (env *Environment, err error) {
	var (
		file   *os.File
		img    image.Image
		bounds image.Rectangle
		colors map[uint32]int
		sprite *Sprite
	)
	if file, err = os.Open(path); err != nil {
		return
	}
	defer file.Close()
	if img, err = png.Decode(file); err != nil {
		return
	}
	colors = make(map[uint32]int, 0)
	bounds = img.Bounds()
	env = &Environment{}
	for y := 0; y < bounds.Dy(); y++ {
		for x := 0; x < bounds.Dx(); x++ {
			r, g, b, _ := img.At(x, y).RGBA()
			index := (r << 8 + g) << 8 + b
			if _, exists := colors[index]; exists == false {
				colors[index] = len(colors)
			}
			sprite = s.NewSprite(name, float32(x * 32), float32(y * 32), 32, 32, 2)
			sprite.Frame = colors[index]
			env.AddChild(sprite)
		}
	}
	return
}
<|endoftext|>


<commit_before>package koding

import (
	"fmt"
	"koding/db/mongodb"
	"strconv"

	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"

	aws "koding/kites/kloud/api/amazon"
	"koding/kites/kloud/klient"
	"koding/kites/kloud/machinestate"
	"koding/kites/kloud/protocol"
	"koding/kites/kloud/provider/amazon"

	"github.com/koding/kite"
	"github.com/koding/logging"
	"github.com/mitchellh/goamz/ec2"
)

// Checker checks various aspects of a machine. It is used for limiting certain
// aspects of a machine, such as the total allowed machine count, storage size
// and etc.
type Checker interface {
	// Total checks whether the user has reached the current plan's limit of
	// having a total number numbers of machines. It returns an error if the
	// limit is reached or an unexplained error happaned.
	Total() error

	// AlwaysOn checks whether the given machine has reached the current plans
	// always on limit
	AlwaysOn() error

	// Timeout checks whether the user has reached the current plan's
	// inactivity timeout.
	Timeout() error

	// Storage checks whether the user has reached the current plan's limit
	// total storage with the supplied wantStorage information. It returns an
	// error if the limit is reached or an unexplained error happaned.
	Storage(wantStorage int) error

	// AllowedInstances checks whether the given machine has the permisison to
	// create the given instance type
	AllowedInstances(wantInstance InstanceType) error
}

type PlanChecker struct {
	Api      *amazon.AmazonClient
	DB       *mongodb.MongoDB
	Machine  *protocol.Machine
	Provider *Provider
	Kite     *kite.Kite
	Username string
	Log      logging.Logger
}

func (p *PlanChecker) AllowedInstances(wantInstance InstanceType) error {
	plan, err := p.Provider.PlanFetcher(p.Machine)
	if err != nil {
		return err
	}

	allowedInstances := plan.Limits().AllowedInstances

	p.Log.Info("[%s] checking instance type. want: %s (plan: %s)",
		p.Machine.Id, wantInstance, plan)

	if _, ok := allowedInstances[wantInstance]; ok {
		return nil
	}

	return fmt.Errorf("not allowed to create instance type: %s", wantInstance)
}

func (p *PlanChecker) AlwaysOn() error {
	plan, err := p.Provider.PlanFetcher(p.Machine)
	if err != nil {
		return err
	}

	alwaysOnLimit := plan.Limits().AlwaysOn

	// get all alwaysOn machines that belongs to this user
	alwaysOnMachines := 0
	if err := p.DB.Run("jMachines", func(c *mgo.Collection) error {
		var err error
		alwaysOnMachines, err = c.Find(bson.M{
			"credential":    p.Machine.Username,
			"meta.alwaysOn": true,
		}).Count()

		return err
	}); err != nil && err != mgo.ErrNotFound {
		// if it's something else just return an error, needs to be fixed
		return err
	}

	p.Log.Info("[%s] checking alwaysOn limit. current alwaysOn count: %d (plan limit: %d, plan: %s)",
		p.Machine.Id, alwaysOnMachines, alwaysOnLimit, plan)
	// the user has still not reached the limit
	if alwaysOnMachines <= alwaysOnLimit {
		p.Log.Info("[%s] allowing user '%s'. current alwaysOn count: %d (plan limit: %d, plan: %s)",
			p.Machine.Id, p.Username, alwaysOnMachines, alwaysOnLimit, plan)
		return nil
	}

	p.Log.Info("[%s] denying user '%s'. current alwaysOn count: %d (plan limit: %d, plan: %s)",
		p.Machine.Id, p.Username, alwaysOnMachines, alwaysOnLimit, plan)
	return fmt.Errorf("total alwaysOn limit has been reached")
}

func (p *PlanChecker) Timeout() error {
	// Check klient state before rushing to AWS.
	klientRef, err := klient.Connect(p.Kite, p.Machine.QueryString)
	if err == kite.ErrNoKitesAvailable {
		p.Provider.startTimer(p.Machine)
		return err
	}

	// return if it's something else
	if err != nil {
		return err
	}

	defer klientRef.Close()

	if err = klientRef.Ping(); err != nil {
		return err
	}

	// now the klient is connected and we can ping it, stop the timer and
	// remove it from the list of inactive machines if it's still there.
	p.Provider.stopTimer(p.Machine)

	// get the usage directly from the klient, which is the most predictable source
	usg, err := klientRef.Usage()
	if err != nil {
		return err
	}

	plan, err := p.Provider.PlanFetcher(p.Machine)
	if err != nil {
		return err
	}

	// replace with the real and authenticated username
	p.Machine.Builder["username"] = klientRef.Username
	p.Username = klientRef.Username

	// get the timeout from the plan in which the user belongs to
	planTimeout := plan.Limits().Timeout

	p.Log.Debug("[%s] machine [%s] is inactive for %s (plan limit: %s, plan: %s).",
		p.Machine.Id, p.Machine.IpAddress, usg.InactiveDuration, planTimeout, plan)

	// It still have plenty of time to work, do not stop it
	if usg.InactiveDuration <= planTimeout {
		return nil
	}

	p.Log.Info("[%s] machine [%s] has reached current plan limit of %s (plan: %s). Shutting down...",
		p.Machine.Id, p.Machine.IpAddress, usg.InactiveDuration, plan)

	// lock so it doesn't interfere with others.
	p.Provider.Lock(p.Machine.Id)
	defer p.Provider.Unlock(p.Machine.Id)

	// mark our state as stopping so others know what we are doing
	stoppingReason := fmt.Sprintf("Stopping process started due inactivity of %.f minutes",
		planTimeout.Minutes())

	p.Provider.UpdateState(p.Machine.Id, stoppingReason, machinestate.Stopping)

	// Hasta la vista, baby!
	err = p.Provider.Stop(p.Machine)
	if err != nil {
		return err
	}

	// update to final state too
	stopReason := fmt.Sprintf("Stopped due inactivity of %.f minutes", planTimeout.Minutes())
	return p.Provider.UpdateState(p.Machine.Id, stopReason, machinestate.Stopped)
}

func (p *PlanChecker) Total() error {
	plan, err := p.Provider.PlanFetcher(p.Machine)
	if err != nil {
		return err
	}

	allowedMachines := plan.Limits().Total

	instances, err := p.userInstances()

	// no match, allow to create instance
	if err == aws.ErrNoInstances {
		p.Log.Info("[%s] allowing user '%s'. current machine count: %d (plan limit: %d, plan: %s)",
			p.Machine.Id, p.Username, len(instances), allowedMachines, plan)
		return nil
	}

	// if it's something else don't allow it until it's solved
	if err != nil {
		return err
	}

	go p.checkGhostMachines(instances)

	if len(instances) >= allowedMachines {
		p.Log.Info("[%s] denying user '%s'. current machine count: %d (plan limit: %d, plan: %s)",
			p.Machine.Id, p.Username, len(instances), allowedMachines, plan)

		return fmt.Errorf("total limit of %d machines has been reached", allowedMachines)
	}

	p.Log.Info("[%s] allowing user '%s'. current machine count: %d (plan limit: %d, plan: %s)",
		p.Machine.Id, p.Username, len(instances), allowedMachines, plan)

	return nil
}

func (p *PlanChecker) checkGhostMachines(instances []ec2.Instance) {
	for _, instance := range instances {
		for _, tag := range instance.Tags {
			if tag.Key != "koding-machineId" {
				continue
			}

			machineId := tag.Value

			// this is just for logging, so we don't care about handling
			// the error
			p.DB.Run("jMachines", func(c *mgo.Collection) error {
				n, err := c.FindId(bson.ObjectIdHex(machineId)).Count()
				if err != nil {
					return err
				}

				if n != 0 {
					return nil
				}

				p.Log.Warning("Detected a Ghost Machine in AWS! Instance id: %s", instance.InstanceId)
				return nil
			})
		}
	}
}

func (p *PlanChecker) Storage(wantStorage int) error {
	plan, err := p.Provider.PlanFetcher(p.Machine)
	if err != nil {
		return err
	}

	totalStorage := plan.Limits().Storage

	// no need for errors because instances will be empty in case of an error
	instances, _ := p.userInstances()

	// i hate for loops too, but unfortunaly the responses are always in form
	// of slices
	currentStorage := 0
	for _, instance := range instances {
		for _, blockDevice := range instance.BlockDevices {
			volumes, err := p.Api.Client.Volumes([]string{blockDevice.VolumeId}, ec2.NewFilter())
			if err != nil {
				return err
			}

			for _, volume := range volumes.Volumes {
				volumeStorage, err := strconv.Atoi(volume.Size)
				if err != nil {
					return err
				}

				currentStorage += volumeStorage
			}
		}
	}

	p.Log.Info("[%s] Checking storage. Current: %dGB. Want: %dGB (plan limit: %dGB, plan: %s)",
		p.Machine.Id, currentStorage, wantStorage, totalStorage, plan)

	if currentStorage+wantStorage > totalStorage {
		return fmt.Errorf("total storage limit has been reached. Can use %dGB of %dGB (plan: %s)",
			totalStorage-currentStorage, totalStorage, plan)
	}

	p.Log.Info("[%s] Allowing user '%s'. Current: %dGB. Want: %dGB (plan limit: %dGB, plan: %s)",
		p.Machine.Id, p.Username, currentStorage, wantStorage, totalStorage, plan)

	// allow to create storage
	return nil
}

func (p *PlanChecker) userInstances() ([]ec2.Instance, error) {
	filter := ec2.NewFilter()
	filter.Add("tag-value", p.Username)

	// Anything except "terminated" and "shutting-down"
	filter.Add("instance-state-name", "pending", "running", "stopping", "stopped")

	instances, err := p.Api.InstancesByFilter(filter)
	if err != nil {
		return nil, err
	}

	filtered := []ec2.Instance{}

	for _, instance := range instances {
		for _, tag := range instance.Tags {
			if tag.Key == "koding-user" && tag.Value == p.Username {
				for _, tag := range instance.Tags {
					if tag.Key == "koding-env" && tag.Value == p.Kite.Config.Environment {

						// now we have the instance that matches both the correct username
						// and environment
						filtered = append(filtered, instance)
					}
				}
			}
		}
	}

	// garbage collect it
	instances = nil
	return filtered, nil
}
<commit_msg>checker: add comment why we are doing manual filtering<commit_after>package koding

import (
	"fmt"
	"koding/db/mongodb"
	"strconv"

	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"

	aws "koding/kites/kloud/api/amazon"
	"koding/kites/kloud/klient"
	"koding/kites/kloud/machinestate"
	"koding/kites/kloud/protocol"
	"koding/kites/kloud/provider/amazon"

	"github.com/koding/kite"
	"github.com/koding/logging"
	"github.com/mitchellh/goamz/ec2"
)

// Checker checks various aspects of a machine. It is used for limiting certain
// aspects of a machine, such as the total allowed machine count, storage size
// and etc.
type Checker interface {
	// Total checks whether the user has reached the current plan's limit of
	// having a total number numbers of machines. It returns an error if the
	// limit is reached or an unexplained error happaned.
	Total() error

	// AlwaysOn checks whether the given machine has reached the current plans
	// always on limit
	AlwaysOn() error

	// Timeout checks whether the user has reached the current plan's
	// inactivity timeout.
	Timeout() error

	// Storage checks whether the user has reached the current plan's limit
	// total storage with the supplied wantStorage information. It returns an
	// error if the limit is reached or an unexplained error happaned.
	Storage(wantStorage int) error

	// AllowedInstances checks whether the given machine has the permisison to
	// create the given instance type
	AllowedInstances(wantInstance InstanceType) error
}

type PlanChecker struct {
	Api      *amazon.AmazonClient
	DB       *mongodb.MongoDB
	Machine  *protocol.Machine
	Provider *Provider
	Kite     *kite.Kite
	Username string
	Log      logging.Logger
}

func (p *PlanChecker) AllowedInstances(wantInstance InstanceType) error {
	plan, err := p.Provider.PlanFetcher(p.Machine)
	if err != nil {
		return err
	}

	allowedInstances := plan.Limits().AllowedInstances

	p.Log.Info("[%s] checking instance type. want: %s (plan: %s)",
		p.Machine.Id, wantInstance, plan)

	if _, ok := allowedInstances[wantInstance]; ok {
		return nil
	}

	return fmt.Errorf("not allowed to create instance type: %s", wantInstance)
}

func (p *PlanChecker) AlwaysOn() error {
	plan, err := p.Provider.PlanFetcher(p.Machine)
	if err != nil {
		return err
	}

	alwaysOnLimit := plan.Limits().AlwaysOn

	// get all alwaysOn machines that belongs to this user
	alwaysOnMachines := 0
	if err := p.DB.Run("jMachines", func(c *mgo.Collection) error {
		var err error
		alwaysOnMachines, err = c.Find(bson.M{
			"credential":    p.Machine.Username,
			"meta.alwaysOn": true,
		}).Count()

		return err
	}); err != nil && err != mgo.ErrNotFound {
		// if it's something else just return an error, needs to be fixed
		return err
	}

	p.Log.Info("[%s] checking alwaysOn limit. current alwaysOn count: %d (plan limit: %d, plan: %s)",
		p.Machine.Id, alwaysOnMachines, alwaysOnLimit, plan)
	// the user has still not reached the limit
	if alwaysOnMachines <= alwaysOnLimit {
		p.Log.Info("[%s] allowing user '%s'. current alwaysOn count: %d (plan limit: %d, plan: %s)",
			p.Machine.Id, p.Username, alwaysOnMachines, alwaysOnLimit, plan)
		return nil
	}

	p.Log.Info("[%s] denying user '%s'. current alwaysOn count: %d (plan limit: %d, plan: %s)",
		p.Machine.Id, p.Username, alwaysOnMachines, alwaysOnLimit, plan)
	return fmt.Errorf("total alwaysOn limit has been reached")
}

func (p *PlanChecker) Timeout() error {
	// Check klient state before rushing to AWS.
	klientRef, err := klient.Connect(p.Kite, p.Machine.QueryString)
	if err == kite.ErrNoKitesAvailable {
		p.Provider.startTimer(p.Machine)
		return err
	}

	// return if it's something else
	if err != nil {
		return err
	}

	defer klientRef.Close()

	if err = klientRef.Ping(); err != nil {
		return err
	}

	// now the klient is connected and we can ping it, stop the timer and
	// remove it from the list of inactive machines if it's still there.
	p.Provider.stopTimer(p.Machine)

	// get the usage directly from the klient, which is the most predictable source
	usg, err := klientRef.Usage()
	if err != nil {
		return err
	}

	plan, err := p.Provider.PlanFetcher(p.Machine)
	if err != nil {
		return err
	}

	// replace with the real and authenticated username
	p.Machine.Builder["username"] = klientRef.Username
	p.Username = klientRef.Username

	// get the timeout from the plan in which the user belongs to
	planTimeout := plan.Limits().Timeout

	p.Log.Debug("[%s] machine [%s] is inactive for %s (plan limit: %s, plan: %s).",
		p.Machine.Id, p.Machine.IpAddress, usg.InactiveDuration, planTimeout, plan)

	// It still have plenty of time to work, do not stop it
	if usg.InactiveDuration <= planTimeout {
		return nil
	}

	p.Log.Info("[%s] machine [%s] has reached current plan limit of %s (plan: %s). Shutting down...",
		p.Machine.Id, p.Machine.IpAddress, usg.InactiveDuration, plan)

	// lock so it doesn't interfere with others.
	p.Provider.Lock(p.Machine.Id)
	defer p.Provider.Unlock(p.Machine.Id)

	// mark our state as stopping so others know what we are doing
	stoppingReason := fmt.Sprintf("Stopping process started due inactivity of %.f minutes",
		planTimeout.Minutes())

	p.Provider.UpdateState(p.Machine.Id, stoppingReason, machinestate.Stopping)

	// Hasta la vista, baby!
	err = p.Provider.Stop(p.Machine)
	if err != nil {
		return err
	}

	// update to final state too
	stopReason := fmt.Sprintf("Stopped due inactivity of %.f minutes", planTimeout.Minutes())
	return p.Provider.UpdateState(p.Machine.Id, stopReason, machinestate.Stopped)
}

func (p *PlanChecker) Total() error {
	plan, err := p.Provider.PlanFetcher(p.Machine)
	if err != nil {
		return err
	}

	allowedMachines := plan.Limits().Total

	instances, err := p.userInstances()

	// no match, allow to create instance
	if err == aws.ErrNoInstances {
		p.Log.Info("[%s] allowing user '%s'. current machine count: %d (plan limit: %d, plan: %s)",
			p.Machine.Id, p.Username, len(instances), allowedMachines, plan)
		return nil
	}

	// if it's something else don't allow it until it's solved
	if err != nil {
		return err
	}

	go p.checkGhostMachines(instances)

	if len(instances) >= allowedMachines {
		p.Log.Info("[%s] denying user '%s'. current machine count: %d (plan limit: %d, plan: %s)",
			p.Machine.Id, p.Username, len(instances), allowedMachines, plan)

		return fmt.Errorf("total limit of %d machines has been reached", allowedMachines)
	}

	p.Log.Info("[%s] allowing user '%s'. current machine count: %d (plan limit: %d, plan: %s)",
		p.Machine.Id, p.Username, len(instances), allowedMachines, plan)

	return nil
}

func (p *PlanChecker) checkGhostMachines(instances []ec2.Instance) {
	for _, instance := range instances {
		for _, tag := range instance.Tags {
			if tag.Key != "koding-machineId" {
				continue
			}

			machineId := tag.Value

			// this is just for logging, so we don't care about handling
			// the error
			p.DB.Run("jMachines", func(c *mgo.Collection) error {
				n, err := c.FindId(bson.ObjectIdHex(machineId)).Count()
				if err != nil {
					return err
				}

				if n != 0 {
					return nil
				}

				p.Log.Warning("Detected a Ghost Machine in AWS! Instance id: %s", instance.InstanceId)
				return nil
			})
		}
	}
}

func (p *PlanChecker) Storage(wantStorage int) error {
	plan, err := p.Provider.PlanFetcher(p.Machine)
	if err != nil {
		return err
	}

	totalStorage := plan.Limits().Storage

	// no need for errors because instances will be empty in case of an error
	instances, _ := p.userInstances()

	// i hate for loops too, but unfortunaly the responses are always in form
	// of slices
	currentStorage := 0
	for _, instance := range instances {
		for _, blockDevice := range instance.BlockDevices {
			volumes, err := p.Api.Client.Volumes([]string{blockDevice.VolumeId}, ec2.NewFilter())
			if err != nil {
				return err
			}

			for _, volume := range volumes.Volumes {
				volumeStorage, err := strconv.Atoi(volume.Size)
				if err != nil {
					return err
				}

				currentStorage += volumeStorage
			}
		}
	}

	p.Log.Info("[%s] Checking storage. Current: %dGB. Want: %dGB (plan limit: %dGB, plan: %s)",
		p.Machine.Id, currentStorage, wantStorage, totalStorage, plan)

	if currentStorage+wantStorage > totalStorage {
		return fmt.Errorf("total storage limit has been reached. Can use %dGB of %dGB (plan: %s)",
			totalStorage-currentStorage, totalStorage, plan)
	}

	p.Log.Info("[%s] Allowing user '%s'. Current: %dGB. Want: %dGB (plan limit: %dGB, plan: %s)",
		p.Machine.Id, p.Username, currentStorage, wantStorage, totalStorage, plan)

	// allow to create storage
	return nil
}

func (p *PlanChecker) userInstances() ([]ec2.Instance, error) {
	filter := ec2.NewFilter()
	filter.Add("tag-value", p.Username)

	// Anything except "terminated" and "shutting-down"
	filter.Add("instance-state-name", "pending", "running", "stopping", "stopped")

	instances, err := p.Api.InstancesByFilter(filter)
	if err != nil {
		return nil, err
	}

	filtered := []ec2.Instance{}

	// we don't use filters because they are timing out for us due to high
	// instances count we have. However it seems the filter `tag-value` has an
	// index internally inside AWS so somewhot that one is not timing out.
	for _, instance := range instances {
		for _, tag := range instance.Tags {
			if tag.Key == "koding-user" && tag.Value == p.Username {
				for _, tag := range instance.Tags {
					if tag.Key == "koding-env" && tag.Value == p.Kite.Config.Environment {

						// now we have the instance that matches both the correct username
						// and environment
						filtered = append(filtered, instance)
					}
				}
			}
		}
	}

	// garbage collect it
	instances = nil
	return filtered, nil
}
<|endoftext|>


<commit_before>// this provides a simple datastore for kites just with get/set methods.
// mongodb has 24k number of collection limit in a single database
// http://stackoverflow.com/questions/9858393/limits-of-number-of-collections-in-databases
// thats why we have a single collection and use single index
// though instead of using a single collection  we can use different strategies, like
// multiple database, single collections
// multiple database, multiple collections
// etc... to make it a bit more performant.
// though mongodb has an auto sharding setup, http://docs.mongodb.org/manual/sharding/
// which should be considered first. or use another datastore like elasticsearch, cassandra etc.
// to handle the sharding on database level.
// thats why we only have one strategy only for now, to get the ball rolling.

package main

import (
	"flag"
	"koding/newkite/kite"
	"koding/db/mongodb/modelhelper"
)

var port = flag.String("port", "", "port to bind itself")

func main() {
	flag.Parse()

	options := &kite.Options{
		Kitename:    "datastore",
		Version:     "1",
		Port:        *port,
		Region:      "localhost",
		Environment: "development",
		PublicIP:    "127.0.0.1",
	}

	k := kite.New(options)

	k.HandleFunc("set", Set)
	k.HandleFunc("get", Get)

	k.Run()
	modelhelper.EnsureKeyValueIndexes()
}

func Set(r *kite.Request) (interface{}, error) {
	kv, err := r.Args.Array()
	if err != nil {
		return nil, err
	}

	keyValue := modelhelper.NewKeyValue(r.Username, r.RemoteKite.Name, r.RemoteKite.Environment, kv[0].(string))
	keyValue.Value = kv[1].(string)
	err = modelhelper.UpsertKeyValue(keyValue)

	result := true
	if err != nil {
		result = false
	}
	return result, err
}

func Get(r *kite.Request) (interface{}, error) {
	key, err := r.Args.String()
	if err != nil {
		return nil, err
	}

	kv, err := modelhelper.GetKeyValue(r.Username, r.RemoteKite.Name, r.RemoteKite.Environment, key)
	if err != nil{
		return err, nil
	}

	return kv.Value, nil
}
<commit_msg>kite: datastore kite: remove examples/datastore.go<commit_after><|endoftext|>


<commit_before>package es

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"time"
)

func NewBatchIndexer(addr string) *BatchIndexer {
	i := &BatchIndexer{
		Address:       addr,
		buf:           &bytes.Buffer{},
		doClose:       make(chan struct{}),
		doFlush:       make(chan chan struct{}),
		closed:        make(chan struct{}),
		docs:          make(chan *Doc),
		flushCount:    1000,
		flushDuration: 1 * time.Second,
	}
	go i.start()
	return i
}

type BatchIndexer struct {
	Address       string
	buf           *bytes.Buffer
	cnt           int
	docs          chan *Doc
	closed        chan struct{}
	doClose       chan struct{}
	doFlush       chan chan struct{}
	lastFlush     time.Time
	flushDuration time.Duration
	flushCount    int
}

func (i *BatchIndexer) Add(d *Doc) error {
	if d.Id == "" {
		return errors.New("ID must be set")
	}
	if d.Index == "" {
		return errors.New("Index must be set")
	}
	if d.Type == "" {
		return errors.New("Type must be set")
	}
	i.docs <- d
	return nil
}

func (i *BatchIndexer) Close() error {
	close(i.doClose)
	<-i.closed
	return nil
}

func (i *BatchIndexer) start() {
	defer close(i.closed)
	t := time.NewTicker(1 * time.Second)
	for {
		select {
		case d := <-i.docs:
			// add doc to buffer
			_, err := i.addInternal(d)
			if err != nil {
				log.Printf("err=%q", err)
			}
			i.checkFlush()
		case <-t.C:
			// index
			i.checkFlush()
		case c := <-i.doFlush:
			i.flush()
			c <- struct{}{}
		case <-i.doClose:
			// should stop indexing
			i.flush()
			return
		}
	}
}

// checkFlush calls flush if
// a) time since last flush > threshold
// or
// b) rows since last flush > threshold
func (i *BatchIndexer) checkFlush() bool {
	if time.Since(i.lastFlush) < i.flushDuration && i.cnt < i.flushCount {
		return false
	}
	i.flush()
	return true
}

func (i *BatchIndexer) Flush() {
	c := make(chan struct{})
	i.doFlush <- c
	<-c
}

func (i *BatchIndexer) flush() {
	if i.cnt == 0 || i.buf.Len() == 0 {
		return
	}
	defer func() {
		i.cnt = 0
	}()
	b := i.buf
	i.buf = &bytes.Buffer{}
	err := func() error {
		u := i.Address + "/_bulk"
		req, err := http.NewRequest("POST", u, b)
		if err != nil {
			return err
		}
		req.Header.Set("Content-Type", "application/json")
		rsp, err := http.DefaultClient.Do(req)
		if err != nil {
			return fmt.Errorf("error flushing: %s", err)
		}
		defer rsp.Body.Close()
		b, err := ioutil.ReadAll(rsp.Body)
		if err != nil {
			return err
		}
		if rsp.Status[0] != '2' {
			return fmt.Errorf("expected status 2xx, got %s: %s", rsp.Status, string(b))
		}
		return nil
	}()
	if err != nil {
		log.Printf("err=%q", err)
	}

	i.lastFlush = time.Now()
}

type indexDoc struct {
	Doc *Doc `json:"index"`
}

// add should NOT be called directly as it is not thread safe
// use the channel to add new documents
func (i *BatchIndexer) addInternal(doc *Doc) (int, error) {
	b, err := json.Marshal(&indexDoc{Doc: doc})
	if err != nil {
		return i.cnt, err
	}
	i.buf.Write(b)
	i.buf.Write([]byte("\n"))
	b, err = json.Marshal(doc.Source)
	if err != nil {
		return i.cnt, nil
	}
	i.buf.Write(b)
	i.buf.Write([]byte("\n"))
	i.cnt++
	return i.cnt, nil
}
<commit_msg>es: return errors on Flush<commit_after>package es

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"time"
)

func NewBatchIndexer(addr string) *BatchIndexer {
	i := &BatchIndexer{
		Address:       addr,
		buf:           &bytes.Buffer{},
		doClose:       make(chan struct{}),
		doFlush:       make(chan chan error),
		closed:        make(chan error),
		docs:          make(chan *Doc),
		flushCount:    1000,
		flushDuration: 1 * time.Second,
	}
	go i.start()
	return i
}

type BatchIndexer struct {
	Address       string
	buf           *bytes.Buffer
	cnt           int
	docs          chan *Doc
	closed        chan error
	doClose       chan struct{}
	doFlush       chan chan error
	lastFlush     time.Time
	flushDuration time.Duration
	flushCount    int
}

func (i *BatchIndexer) Add(d *Doc) error {
	if d.Id == "" {
		return errors.New("ID must be set")
	}
	if d.Index == "" {
		return errors.New("Index must be set")
	}
	if d.Type == "" {
		return errors.New("Type must be set")
	}
	i.docs <- d
	return nil
}

func (i *BatchIndexer) Close() error {
	close(i.doClose)
	<-i.closed
	return nil
}

// TODO: use context here
func (i *BatchIndexer) start() error {
	defer close(i.closed)
	t := time.NewTicker(1 * time.Second)
	for {
		select {
		case d := <-i.docs:
			// add doc to buffer
			_, err := i.addInternal(d)
			if err != nil {
				log.Printf("err=%q", err)
			}
			i.checkFlush()
		case <-t.C:
			// index
			i.checkFlush()
		case c := <-i.doFlush:
			c <- i.flush()
		case <-i.doClose:
			// should stop indexing
			return i.flush()
		}
	}
}

// checkFlush calls flush if
// a) time since last flush > threshold
// or
// b) rows since last flush > threshold
func (i *BatchIndexer) checkFlush() bool {
	if time.Since(i.lastFlush) < i.flushDuration && i.cnt < i.flushCount {
		return false
	}
	i.flush()
	return true
}

func (i *BatchIndexer) Flush() error {
	c := make(chan error)
	i.doFlush <- c
	return <-c
}

func (i *BatchIndexer) flush() error {
	if i.cnt == 0 || i.buf.Len() == 0 {
		return nil
	}
	defer func() {
		i.cnt = 0
	}()
	b := i.buf
	i.buf = &bytes.Buffer{}
	err := func() error {
		u := i.Address + "/_bulk"
		req, err := http.NewRequest("POST", u, b)
		if err != nil {
			return err
		}
		req.Header.Set("Content-Type", "application/json")
		rsp, err := http.DefaultClient.Do(req)
		if err != nil {
			return fmt.Errorf("error flushing: %s", err)
		}
		defer rsp.Body.Close()
		b, err := ioutil.ReadAll(rsp.Body)
		if err != nil {
			return err
		}
		if rsp.Status[0] != '2' {
			return fmt.Errorf("expected status 2xx, got %s: %s", rsp.Status, string(b))
		}
		return nil
	}()
	if err != nil {
		return err
	}

	i.lastFlush = time.Now()
	return nil
}

type indexDoc struct {
	Doc *Doc `json:"index"`
}

// add should NOT be called directly as it is not thread safe
// use the channel to add new documents
func (i *BatchIndexer) addInternal(doc *Doc) (int, error) {
	b, err := json.Marshal(&indexDoc{Doc: doc})
	if err != nil {
		return i.cnt, err
	}
	i.buf.Write(b)
	i.buf.Write([]byte("\n"))
	b, err = json.Marshal(doc.Source)
	if err != nil {
		return i.cnt, nil
	}
	i.buf.Write(b)
	i.buf.Write([]byte("\n"))
	i.cnt++
	return i.cnt, nil
}
<|endoftext|>