<commit_before>package main

import (
	"flag"
	"log"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatch"
	"github.com/buildkite/buildkite-cloudwatch-metrics-publisher/buildkite"
)

// Generates:
// Buildkite > RunningBuildsCount
// Buildkite > RunningJobsCount
// Buildkite > ScheduledBuildsCount
// Buildkite > ScheduledJobsCount
// Buildkite > (Queue) > RunningBuildsCount
// Buildkite > (Queue) > RunningJobsCount
// Buildkite > (Queue) > ScheduledBuildsCount
// Buildkite > (Queue) > ScheduledJobsCount
// Buildkite > (Pipeline) > RunningBuildsCount
// Buildkite > (Pipeline) > RunningJobsCount
// Buildkite > (Pipeline) > ScheduledBuildsCount
// Buildkite > (Pipeline) > ScheduledJobsCount

func main() {
	var (
		accessToken = flag.String("token", "", "A Buildkite API Access Token")
		orgSlug     = flag.String("org", "", "A Buildkite Organization Slug")
		interval    = flag.Duration("interval", 0, "Update metrics every interval, rather than once")
	)

	flag.Parse()

	if *accessToken == "" {
		log.Fatal("Must provide a value for -token")
	}

	if *orgSlug == "" {
		log.Fatal("Must provide a value for -org")
	}

	if err := runCollector(*orgSlug, *accessToken, time.Hour*24); err != nil {
		log.Fatal(err)
	}

	if *interval > 0 {
		for _ = range time.NewTicker(*interval).C {
			if err := runCollector(*orgSlug, *accessToken, time.Hour); err != nil {
				log.Println(err)
			}
		}
	}
}

func runCollector(orgSlug, accessToken string, historical time.Duration) error {
	svc := cloudwatch.New(session.New())

	log.Printf("Collecting buildkite metrics from org %s", orgSlug)
	result, err := collectResults(orgSlug, accessToken, historical)
	if err != nil {
		return err
	}

	log.Printf("Extracting cloudwatch metrics from results")
	metrics := result.extractMetricData()

	for _, chunk := range chunkMetricData(10, metrics) {
		log.Printf("Submitting chunk of %d metrics to Cloudwatch", len(chunk))
		if err := putMetricData(svc, chunk); err != nil {
			if err != nil {
				return err
			}
		}
	}

	return nil
}

func collectResults(orgSlug, accessToken string, historical time.Duration) (*Result, error) {
	var res *Result = &Result{
		Queues:    map[string]Counts{},
		Pipelines: map[string]Counts{},
	}

	// Algorithm:
	// Get Builds with finished_from = 24 hours ago
	// Build results with zero values for pipelines/queues
	// Get all running and scheduled builds, add to results

	builds, err := buildkite.Builds(&buildkite.BuildsInput{
		OrgSlug:      orgSlug,
		ApiToken:     accessToken,
		FinishedFrom: time.Now().UTC().Add(historical * -1),
	})
	if err != nil {
		return res, err
	}

	for _, queue := range builds.Queues() {
		res.Queues[queue] = Counts{}
	}

	for _, build := range builds {
		if _, ok := res.Pipelines[build.Pipeline.Name]; !ok {
			res.Pipelines[build.Pipeline.Name] = Counts{}
		}
	}

	states := []string{"scheduled", "running"}

	for _, state := range states {
		builds, err := buildkite.Builds(&buildkite.BuildsInput{
			OrgSlug:  orgSlug,
			ApiToken: accessToken,
			State:    state,
		})
		if err != nil {
			return res, err
		}

		for _, build := range builds {
			log.Printf("Adding build to stats (id=%q, pipeline=%q, branch=%q, state=%q)",
				build.ID, build.Pipeline.Name, build.Branch, build.State)

			res.Counts = res.Counts.addBuild(build)
			res.Pipelines[build.Pipeline.Name] = res.Pipelines[build.Pipeline.Name].addBuild(build)

			var buildQueues = map[string]int{}

			for _, job := range build.Jobs {
				log.Printf("Adding job to stats (id=%q, pipeline=%q, queue=%q, type=%q, state=%q)",
					job.ID, build.Pipeline.Name, job.Queue(), job.Type, job.State)

				res.Counts = res.Counts.addJob(job)
				res.Pipelines[build.Pipeline.Name] = res.Pipelines[build.Pipeline.Name].addJob(job)
				res.Queues[job.Queue()] = res.Queues[job.Queue()].addJob(job)
				buildQueues[job.Queue()]++
			}

			if len(buildQueues) > 0 {
				for queue := range buildQueues {
					log.Printf("Adding stats for build to queue %s", queue)
					res.Queues[queue] = res.Queues[queue].addBuild(build)
				}
			}
		}
	}

	return res, nil
}

type Counts struct {
	RunningBuilds, RunningJobs, ScheduledBuilds, ScheduledJobs int
}

func (c Counts) addBuild(build buildkite.Build) Counts {
	switch build.State {
	case "running":
		c.RunningBuilds++
	case "scheduled":
		c.ScheduledBuilds++
	}
	return c
}

func (c Counts) addJob(job buildkite.Job) Counts {
	switch job.State {
	case "running":
		c.RunningJobs++
	case "scheduled":
		c.ScheduledJobs++
	}
	return c
}

func (c Counts) asMetrics(dimensions []*cloudwatch.Dimension) []*cloudwatch.MetricDatum {
	return []*cloudwatch.MetricDatum{
		&cloudwatch.MetricDatum{
			MetricName: aws.String("RunningBuildsCount"),
			Dimensions: dimensions,
			Value:      aws.Float64(float64(c.RunningBuilds)),
			Unit:       aws.String("Count"),
		},
		&cloudwatch.MetricDatum{
			MetricName: aws.String("ScheduledBuildsCount"),
			Dimensions: dimensions,
			Value:      aws.Float64(float64(c.ScheduledBuilds)),
			Unit:       aws.String("Count"),
		},
		&cloudwatch.MetricDatum{
			MetricName: aws.String("RunningJobsCount"),
			Dimensions: dimensions,
			Value:      aws.Float64(float64(c.RunningJobs)),
			Unit:       aws.String("Count"),
		},
		&cloudwatch.MetricDatum{
			MetricName: aws.String("ScheduledJobsCount"),
			Dimensions: dimensions,
			Value:      aws.Float64(float64(c.ScheduledJobs)),
			Unit:       aws.String("Count"),
		},
	}
}

type Result struct {
	Counts
	Queues, Pipelines map[string]Counts
}

func (r *Result) extractMetricData() []*cloudwatch.MetricDatum {
	data := []*cloudwatch.MetricDatum{}
	data = append(data, r.Counts.asMetrics(nil)...)

	for name, c := range r.Queues {
		data = append(data, c.asMetrics([]*cloudwatch.Dimension{
			{Name: aws.String("Queue"), Value: aws.String(name)},
		})...)
	}

	for name, c := range r.Pipelines {
		data = append(data, c.asMetrics([]*cloudwatch.Dimension{
			{Name: aws.String("Pipeline"), Value: aws.String(name)},
		})...)
	}

	return data
}

func chunkMetricData(size int, data []*cloudwatch.MetricDatum) [][]*cloudwatch.MetricDatum {
	var chunks = [][]*cloudwatch.MetricDatum{}
	for i := 0; i < len(data); i += size {
		end := i + size
		if end > len(data) {
			end = len(data)
		}
		chunks = append(chunks, data[i:end])
	}
	return chunks
}

func putMetricData(svc *cloudwatch.CloudWatch, data []*cloudwatch.MetricDatum) error {
	_, err := svc.PutMetricData(&cloudwatch.PutMetricDataInput{
		MetricData: data,
		Namespace:  aws.String("Buildkite"),
	})
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>Refactored counts as a map rather than a struct<commit_after>package main

import (
	"flag"
	"log"
	"time"

	"github.com/99designs/go-buildkite/buildkite"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatch"
	"github.com/buildkite/buildkite-cloudwatch-metrics-publisher/buildkite"
)

// Generates:

// Buildkite > RunningBuildsCount
// Buildkite > RunningJobsCount
// Buildkite > ScheduledBuildsCount
// Buildkite > ScheduledJobsCount

// Buildkite > (Queue) > RunningBuildsCount
// Buildkite > (Queue) > RunningJobsCount
// Buildkite > (Queue) > ScheduledBuildsCount
// Buildkite > (Queue) > ScheduledJobsCount

// Buildkite > (Pipeline) > RunningBuildsCount
// Buildkite > (Pipeline) > RunningJobsCount
// Buildkite > (Pipeline) > ScheduledBuildsCount
// Buildkite > (Pipeline) > ScheduledJobsCount

func main() {
	var (
		accessToken = flag.String("token", "", "A Buildkite API Access Token")
		orgSlug     = flag.String("org", "", "A Buildkite Organization Slug")
		interval    = flag.Duration("interval", 0, "Update metrics every interval, rather than once")
	)

	flag.Parse()

	if *accessToken == "" {
		log.Fatal("Must provide a value for -token")
	}

	if *orgSlug == "" {
		log.Fatal("Must provide a value for -org")
	}

	if err := runCollector(*orgSlug, *accessToken, time.Hour*24); err != nil {
		log.Fatal(err)
	}

	if *interval > 0 {
		for _ = range time.NewTicker(*interval).C {
			if err := runCollector(*orgSlug, *accessToken, time.Hour); err != nil {
				log.Println(err)
			}
		}
	}
}

func runCollector(orgSlug, accessToken string, historical time.Duration) error {
	svc := cloudwatch.New(session.New())

	log.Printf("Collecting buildkite metrics from org %s", orgSlug)
	result, err := collectMetrics(orgSlug, accessToken, historical)
	if err != nil {
		return err
	}

	log.Printf("Extracting cloudwatch metrics from results")
	metrics := result.toMetrics()

	for _, chunk := range chunkMetricData(10, metrics) {
		log.Printf("Submitting chunk of %d metrics to Cloudwatch", len(chunk))
		if err := putMetricData(svc, chunk); err != nil {
			if err != nil {
				return err
			}
		}
	}

	return nil
}

const (
	runningBuildsCount   = "RunningBuildsCount"
	runningJobsCount     = "RunningJobsCount"
	scheduledBuildsCount = "ScheduledBuildsCount"
	scheduledJobsCount   = "ScheduledJobsCount"
)

type counts map[string]int

func newCounts() counts {
	return counts{
		runningBuildsCount:   0,
		scheduledBuildsCount: 0,
		runningJobsCount:     0,
		scheduledJobsCount:   0,
	}
}

func (c counts) toMetrics(dimensions []*cloudwatch.Dimension) []*cloudwatch.MetricDatum {
	m := []*cloudwatch.MetricDatum{}

	for k, v := range c {
		m = append(m, &cloudwatch.MetricDatum{
			MetricName: aws.String(k),
			Dimensions: dimensions,
			Value:      aws.Float64(float64(v)),
			Unit:       aws.String("Count"),
		})
	}

	return m
}

type result struct {
	totals            counts
	queues, pipelines map[string]counts
}

func (r *result) toMetrics() []*cloudwatch.MetricDatum {
	data := []*cloudwatch.MetricDatum{}
	data = append(data, r.totals.toMetrics(nil)...)

	for name, c := range r.queues {
		data = append(data, c.toMetrics([]*cloudwatch.Dimension{
			{Name: aws.String("Queue"), Value: aws.String(name)},
		})...)
	}

	for name, c := range r.pipelines {
		data = append(data, c.toMetrics([]*cloudwatch.Dimension{
			{Name: aws.String("Pipeline"), Value: aws.String(name)},
		})...)
	}

	return data
}

func collectMetrics(orgSlug, accessToken string, historical time.Duration) (*result, error) {
	totals := newCounts()
	queues := map[string]counts{}
	pipelines := map[string]counts{}

	// Algorithm:
	// Get Builds with finished_from = 24 hours ago
	// Build results with zero values for pipelines/queues
	// Get all running and scheduled builds, add to results

	builds, err := buildkite.Builds(&buildkite.BuildsInput{
		OrgSlug:      orgSlug,
		ApiToken:     accessToken,
		FinishedFrom: time.Now().UTC().Add(historical * -1),
	})
	if err != nil {
		return nil, err
	}

	for _, queue := range builds.Queues() {
		queues[queue] = newCounts()
	}

	for _, build := range builds {
		pipelines[build.Pipeline.Name] = newCounts()
	}

	states := []string{"scheduled", "running"}

	for _, state := range states {
		builds, err := buildkite.Builds(&buildkite.BuildsInput{
			OrgSlug:  orgSlug,
			ApiToken: accessToken,
			State:    state,
		})
		if err != nil {
			return nil, err
		}

		for _, build := range builds {
			log.Printf("Adding build to stats (id=%q, pipeline=%q, branch=%q, state=%q)",
				build.ID, build.Pipeline.Name, build.Branch, build.State)

			if _, ok := pipelines[build.Pipeline.Name]; !ok {
				pipelines[build.Pipeline.Name] = newCounts()
			}

			switch build.State {
			case "running":
				totals[runningBuildsCount]++
				pipelines[build.Pipeline.Name][runningBuildsCount]++

			case "scheduled":
				totals[scheduledBuildsCount]++
				pipelines[build.Pipeline.Name][scheduledBuildsCount]++
			}

			var buildQueues = map[string]int{}

			for _, job := range build.Jobs {
				log.Printf("Adding job to stats (id=%q, pipeline=%q, queue=%q, type=%q, state=%q)",
					job.ID, build.Pipeline.Name, job.Queue(), job.Type, job.State)

				if _, ok := queues[job.Queue()]; !ok {
					queues[job.Queue()] = newCounts()
				}

				switch job.State {
				case "running":
					totals[runningJobsCount]++
					queues[job.Queue()][runningJobsCount]++

				case "scheduled":
					totals[scheduledJobsCount]++
					queues[job.Queue()][scheduledJobsCount]++
				}

				buildQueues[job.Queue()]++
			}

			// add build metrics to queues
			if len(buildQueues) > 0 {
				for queue := range buildQueues {
					switch build.State {
					case "running":
						queues[queue][runningBuildsCount]++

					case "scheduled":
						queues[queue][scheduledBuildsCount]++
					}
				}
			}
		}
	}

	return &result{totals, queues, pipelines}, nil
}

func chunkMetricData(size int, data []*cloudwatch.MetricDatum) [][]*cloudwatch.MetricDatum {
	var chunks = [][]*cloudwatch.MetricDatum{}
	for i := 0; i < len(data); i += size {
		end := i + size
		if end > len(data) {
			end = len(data)
		}
		chunks = append(chunks, data[i:end])
	}
	return chunks
}

func putMetricData(svc *cloudwatch.CloudWatch, data []*cloudwatch.MetricDatum) error {
	_, err := svc.PutMetricData(&cloudwatch.PutMetricDataInput{
		MetricData: data,
		Namespace:  aws.String("Buildkite"),
	})
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package manifest_test

import (
	"cf/manifest"
	"generic"
	"github.com/stretchr/testify/assert"
	"strings"
	testassert "testhelpers/assert"
	"testing"
)

func TestManifestWithGlobalAndAppSpecificProperties(t *testing.T) {
	m, err := manifest.NewManifest("/some/path", generic.NewMap(map[string]interface{}{
		"instances": "3",
		"memory":    "512M",
		"applications": []interface{}{
			map[string]interface{}{
				"name":     "bitcoin-miner",
				"no-route": true,
			},
		},
	}))
	assert.NoError(t, err)

	apps := m.Applications
	assert.Equal(t, apps[0].Get("instances"), 3)
	assert.Equal(t, apps[0].Get("memory").(uint64), uint64(512))
	assert.True(t, apps[0].Get("no-route").(bool))
}

func TestManifestWithInvalidMemory(t *testing.T) {
	_, err := manifest.NewManifest("/some/path", generic.NewMap(map[string]interface{}{
		"instances": "3",
		"memory":    "512",
		"applications": []interface{}{
			map[string]interface{}{
				"name": "bitcoin-miner",
			},
		},
	}))

	assert.Error(t, err)
	assert.Contains(t, err.Error(), "memory")
}

func TestManifestWithTimeoutSetsHealthCheckTimeout(t *testing.T) {
	m, err := manifest.NewManifest("/some/path", generic.NewMap(map[string]interface{}{
		"applications": []interface{}{
			map[string]interface{}{
				"name":    "bitcoin-miner",
				"timeout": "360",
			},
		},
	}))

	assert.NoError(t, err)
	assert.Equal(t, m.Applications[0].Get("health_check_timeout"), 360)
	assert.False(t, m.Applications[0].Has("timeout"))
}

func TestManifestWithEmptyEnvVarIsInvalid(t *testing.T) {
	_, err := manifest.NewManifest("/some/path", generic.NewMap(map[string]interface{}{
		"env": map[string]interface{}{
			"bar": nil,
		},
		"applications": []interface{}{
			map[string]interface{}{
				"name": "bad app",
			},
		},
	}))

	assert.Error(t, err)
	assert.Contains(t, err.Error(), "env var 'bar' should not be null")
}

func TestManifestWithAbsolutePath(t *testing.T) {
	m, err := manifest.NewManifest("/some/path", generic.NewMap(map[string]interface{}{
		"applications": []interface{}{
			map[string]interface{}{
				"path": "/another/path-segment",
			},
		},
	}))

	assert.NoError(t, err)
	assert.Equal(t, m.Applications[0].Get("path"), "/another/path-segment")
}

func TestManifestWithRelativePath(t *testing.T) {
	m, err := manifest.NewManifest("/some/path", generic.NewMap(map[string]interface{}{
		"applications": []interface{}{
			map[string]interface{}{
				"path": "../another/path-segment",
			},
		},
	}))

	assert.NoError(t, err)
	assert.Equal(t, m.Applications[0].Get("path"), "/some/another/path-segment")
}

func TestParsingManifestWithNulls(t *testing.T) {
	_, errs := manifest.NewManifest("/some/path", generic.NewMap(map[string]interface{}{
		"applications": []interface{}{
			map[string]interface{}{
				"buildpack":  nil,
				"disk_quota": nil,
				"domain":     nil,
				"host":       nil,
				"name":       nil,
				"path":       nil,
				"stack":      nil,
				"memory":     nil,
				"instances":  nil,
				"timeout":    nil,
				"no-route":   nil,
				"services":   nil,
				"env":        nil,
			},
		},
	}))

	assert.Error(t, errs)
	errorSlice := strings.Split(errs.Error(), "\n")
	manifestKeys := []string{"buildpack", "disk_quota", "domain", "host", "name", "path", "stack",
		"memory", "instances", "timeout", "no-route", "services", "env"}

	for _, key := range manifestKeys {
		testassert.SliceContains(t, errorSlice, testassert.Lines{{key, "not be null"}})
	}
}

func TestParsingManifestWithPropertiesReturnsErrors(t *testing.T) {
	_, err := manifest.NewManifest("/some/path", generic.NewMap(map[string]interface{}{
		"applications": []interface{}{
			map[string]interface{}{
				"env": map[string]interface{}{
					"bar": "many-${foo}-are-cool",
				},
			},
		},
	}))

	assert.Error(t, err)
	assert.Contains(t, err.Error(), "Properties are not supported. Found property '${foo}'")
}

func TestParsingManifestWithNullCommand(t *testing.T) {
	m, err := manifest.NewManifest("/some/path", generic.NewMap(map[string]interface{}{
		"applications": []interface{}{
			map[string]interface{}{
				"command": nil,
			},
		},
	}))

	assert.NoError(t, err)
	assert.Equal(t, m.Applications[0].Get("command"), "")
}

func TestParsingEmptyManifestDoesNotSetCommand(t *testing.T) {
	m, err := manifest.NewManifest("/some/path", generic.NewMap(map[string]interface{}{
		"applications": []interface{}{
			map[string]interface{}{},
		},
	}))

	assert.NoError(t, err)
	assert.False(t, m.Applications[0].Has("command"))
}
<commit_msg>Fix manifest w/ absolute path test on windows<commit_after>package manifest_test

import (
	"cf/manifest"
	"generic"
	"github.com/stretchr/testify/assert"
	"runtime"
	"strings"
	testassert "testhelpers/assert"
	"testing"
)

func TestManifestWithGlobalAndAppSpecificProperties(t *testing.T) {
	m, err := manifest.NewManifest("/some/path", generic.NewMap(map[string]interface{}{
		"instances": "3",
		"memory":    "512M",
		"applications": []interface{}{
			map[string]interface{}{
				"name":     "bitcoin-miner",
				"no-route": true,
			},
		},
	}))
	assert.NoError(t, err)

	apps := m.Applications
	assert.Equal(t, apps[0].Get("instances"), 3)
	assert.Equal(t, apps[0].Get("memory").(uint64), uint64(512))
	assert.True(t, apps[0].Get("no-route").(bool))
}

func TestManifestWithInvalidMemory(t *testing.T) {
	_, err := manifest.NewManifest("/some/path", generic.NewMap(map[string]interface{}{
		"instances": "3",
		"memory":    "512",
		"applications": []interface{}{
			map[string]interface{}{
				"name": "bitcoin-miner",
			},
		},
	}))

	assert.Error(t, err)
	assert.Contains(t, err.Error(), "memory")
}

func TestManifestWithTimeoutSetsHealthCheckTimeout(t *testing.T) {
	m, err := manifest.NewManifest("/some/path", generic.NewMap(map[string]interface{}{
		"applications": []interface{}{
			map[string]interface{}{
				"name":    "bitcoin-miner",
				"timeout": "360",
			},
		},
	}))

	assert.NoError(t, err)
	assert.Equal(t, m.Applications[0].Get("health_check_timeout"), 360)
	assert.False(t, m.Applications[0].Has("timeout"))
}

func TestManifestWithEmptyEnvVarIsInvalid(t *testing.T) {
	_, err := manifest.NewManifest("/some/path", generic.NewMap(map[string]interface{}{
		"env": map[string]interface{}{
			"bar": nil,
		},
		"applications": []interface{}{
			map[string]interface{}{
				"name": "bad app",
			},
		},
	}))

	assert.Error(t, err)
	assert.Contains(t, err.Error(), "env var 'bar' should not be null")
}

func TestManifestWithAbsolutePath(t *testing.T) {
	if runtime.GOOS == "windows" {
		testManifestWithAbsolutePathOnWindows(t)
	} else {
		testManifestWithAbsolutePathOnPosix(t)
	}
}

func testManifestWithAbsolutePathOnPosix(t *testing.T) {
	m, err := manifest.NewManifest("/some/path", generic.NewMap(map[string]interface{}{
		"applications": []interface{}{
			map[string]interface{}{
				"path": "/another/path-segment",
			},
		},
	}))

	assert.NoError(t, err)
	assert.Equal(t, m.Applications[0].Get("path"), "/another/path-segment")
}

func testManifestWithAbsolutePathOnWindows(t *testing.T) {
	m, err := manifest.NewManifest(`C:\some\path`, generic.NewMap(map[string]interface{}{
		"applications": []interface{}{
			map[string]interface{}{
				"path": `C:\another\path`,
			},
		},
	}))

	assert.NoError(t, err)
	assert.Equal(t, m.Applications[0].Get("path"), `C:\another\path`)
}

func TestManifestWithRelativePath(t *testing.T) {
	m, err := manifest.NewManifest("/some/path", generic.NewMap(map[string]interface{}{
		"applications": []interface{}{
			map[string]interface{}{
				"path": "../another/path-segment",
			},
		},
	}))

	assert.NoError(t, err)
	assert.Equal(t, m.Applications[0].Get("path"), "/some/another/path-segment")
}

func TestParsingManifestWithNulls(t *testing.T) {
	_, errs := manifest.NewManifest("/some/path", generic.NewMap(map[string]interface{}{
		"applications": []interface{}{
			map[string]interface{}{
				"buildpack":  nil,
				"disk_quota": nil,
				"domain":     nil,
				"host":       nil,
				"name":       nil,
				"path":       nil,
				"stack":      nil,
				"memory":     nil,
				"instances":  nil,
				"timeout":    nil,
				"no-route":   nil,
				"services":   nil,
				"env":        nil,
			},
		},
	}))

	assert.Error(t, errs)
	errorSlice := strings.Split(errs.Error(), "\n")
	manifestKeys := []string{"buildpack", "disk_quota", "domain", "host", "name", "path", "stack",
		"memory", "instances", "timeout", "no-route", "services", "env"}

	for _, key := range manifestKeys {
		testassert.SliceContains(t, errorSlice, testassert.Lines{{key, "not be null"}})
	}
}

func TestParsingManifestWithPropertiesReturnsErrors(t *testing.T) {
	_, err := manifest.NewManifest("/some/path", generic.NewMap(map[string]interface{}{
		"applications": []interface{}{
			map[string]interface{}{
				"env": map[string]interface{}{
					"bar": "many-${foo}-are-cool",
				},
			},
		},
	}))

	assert.Error(t, err)
	assert.Contains(t, err.Error(), "Properties are not supported. Found property '${foo}'")
}

func TestParsingManifestWithNullCommand(t *testing.T) {
	m, err := manifest.NewManifest("/some/path", generic.NewMap(map[string]interface{}{
		"applications": []interface{}{
			map[string]interface{}{
				"command": nil,
			},
		},
	}))

	assert.NoError(t, err)
	assert.Equal(t, m.Applications[0].Get("command"), "")
}

func TestParsingEmptyManifestDoesNotSetCommand(t *testing.T) {
	m, err := manifest.NewManifest("/some/path", generic.NewMap(map[string]interface{}{
		"applications": []interface{}{
			map[string]interface{}{},
		},
	}))

	assert.NoError(t, err)
	assert.False(t, m.Applications[0].Has("command"))
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"strings"
	"time"

	"github.com/jingweno/travisarchive/db"
)

var (
	newBuildCrawlerInterval      = 10 * time.Second
	finishedBuildCrawlerInterval = 2 * time.Minute
)

type Crawler interface {
	Crawl()
}

func NewCrawler(travis *Travis, db *db.DB) []Crawler {
	return []Crawler{
		&NewBuildCrawler{travis, db, log.New(os.Stderr, "[NewBuildCrawler] ", log.LstdFlags)},
		&FinishedBuildCrawler{travis, db, log.New(os.Stderr, "[FinishedBuildCrawler] ", log.LstdFlags)},
	}
}

type NewBuildCrawler struct {
	Travis *Travis
	DB     *db.DB
	Logger *log.Logger
}

func (c *NewBuildCrawler) Crawl() {
	ch := time.Tick(newBuildCrawlerInterval)
	for _ = range ch {
		c.Logger.Println("crawling for new builds...")
		c.crawlNewBuilds()
	}
}

func (c *NewBuildCrawler) crawlNewBuilds() {
	repos, err := c.Travis.Repos()
	if err != nil {
		c.Logger.Println(err)
		return
	}

	newBuilds := []string{}
	for _, repo := range repos {
		updated, err := c.DB.Upsert("new_builds", db.Query{"lastbuildid": repo.LastBuildId}, repo)
		if err != nil {
			c.Logger.Println(err)
			continue
		}

		if updated {
			newBuilds = append(newBuilds, repo.Slug)
		}
	}

	c.Logger.Printf("harvested %d builds with %d new builds: %s\n", len(repos), len(newBuilds), strings.Join(newBuilds, ", "))
}

type FinishedBuildCrawler struct {
	Travis *Travis
	DB     *db.DB
	Logger *log.Logger
}

func (c *FinishedBuildCrawler) Crawl() {
	ch := time.Tick(finishedBuildCrawlerInterval)
	for _ = range ch {
		c.Logger.Println("crawling for finsihed builds...")
		c.crawlFinishedBuilds()
	}
}

func (c *FinishedBuildCrawler) crawlFinishedBuilds() {
	colNames, finishedBuilds, skippedBuilds := c.doCrawlFinishedBuilds()
	c.Logger.Printf("fetched %d builds with %d finsihed and %d skipped. Finsihed builds: %s\n", len(finishedBuilds)+len(skippedBuilds), len(finishedBuilds), len(skippedBuilds), strings.Join(finishedBuilds, ", "))

	err := c.ensureColIndexes(colNames)
	if err != nil {
		c.Logger.Println(err)
	}
}

func (c *FinishedBuildCrawler) doCrawlFinishedBuilds() (colNames map[string]string, finishedBuilds []string, skippedBuilds []string) {
	colNames = make(map[string]string)

	var (
		repo  *Repo
		query db.Query
	)
	//query = Query{"lastbuildstartedat": Query{"$gte": oneMinuteAgo()}}
	iter := c.DB.C("new_builds").Find(query).Sort("-lastbuildstartedat").Iter()
	for iter.Next(&repo) {
		build, err := c.crawlFinsihedBuild(repo)
		if err != nil {
			c.Logger.Println(err)
			skippedBuilds = append(skippedBuilds, repo.Slug)
			continue
		}

		colName, updated, err := c.upsertBuild(build)
		if err != nil {
			c.Logger.Println(err)
			continue
		}

		colNames[colName] = colName

		if updated {
			finishedBuilds = append(finishedBuilds, repo.Slug)
		}
	}

	if err := iter.Close(); err != nil {
		c.Logger.Println(err)
	}

	return
}

func (c *FinishedBuildCrawler) crawlFinsihedBuild(repo *Repo) (build *Build, err error) {
	build, err = c.Travis.Build(repo.LastBuildId)
	if err != nil {
		return
	}

	isFinished := !(build.FinishedAt == nil || build.StartedAt == nil)
	if !isFinished {
		err = fmt.Errorf("skipping build: %s - %d\n", repo.Slug, repo.LastBuildId)
		return
	}

	build.Repository = repo

	return
}

func (c *FinishedBuildCrawler) upsertBuild(build *Build) (colName string, updated bool, err error) {
	colName = buildColName(build.StartedAt)

	updated, err = c.DB.Upsert(colName, db.Query{"id": build.Id}, build)
	if err != nil {
		return
	}

	err = c.DB.C("new_builds").Remove(db.Query{"lastbuildid": build.Repository.LastBuildId})
	if err != nil {
		return
	}

	return
}

func (c *FinishedBuildCrawler) ensureColIndexes(colNames map[string]string) error {
	for _, colName := range colNames {
		c.Logger.Printf("ensuring index for collection %s\n", colName)
		err := c.DB.EnsureUniqueIndexKey(colName, "id")
		if err != nil {
			return err
		}
	}

	return nil
}

func buildColName(date *time.Time) string {
	buildDate := date.UTC().Format("2006_01_02")
	return fmt.Sprintf("builds_%s", buildDate)
}

func oneMinuteAgo() time.Time {
	now := time.Now()
	return now.Add(-1 * time.Minute).UTC()
}
<commit_msg>Sort it by the asscending order for lastbuildstartedat<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"strings"
	"time"

	"github.com/jingweno/travisarchive/db"
)

var (
	newBuildCrawlerInterval      = 10 * time.Second
	finishedBuildCrawlerInterval = 2 * time.Minute
)

type Crawler interface {
	Crawl()
}

func NewCrawler(travis *Travis, db *db.DB) []Crawler {
	return []Crawler{
		&NewBuildCrawler{travis, db, log.New(os.Stderr, "[NewBuildCrawler] ", log.LstdFlags)},
		&FinishedBuildCrawler{travis, db, log.New(os.Stderr, "[FinishedBuildCrawler] ", log.LstdFlags)},
	}
}

type NewBuildCrawler struct {
	Travis *Travis
	DB     *db.DB
	Logger *log.Logger
}

func (c *NewBuildCrawler) Crawl() {
	ch := time.Tick(newBuildCrawlerInterval)
	for _ = range ch {
		c.Logger.Println("crawling for new builds...")
		c.crawlNewBuilds()
	}
}

func (c *NewBuildCrawler) crawlNewBuilds() {
	repos, err := c.Travis.Repos()
	if err != nil {
		c.Logger.Println(err)
		return
	}

	newBuilds := []string{}
	for _, repo := range repos {
		updated, err := c.DB.Upsert("new_builds", db.Query{"lastbuildid": repo.LastBuildId}, repo)
		if err != nil {
			c.Logger.Println(err)
			continue
		}

		if updated {
			newBuilds = append(newBuilds, repo.Slug)
		}
	}

	c.Logger.Printf("harvested %d builds with %d new builds: %s\n", len(repos), len(newBuilds), strings.Join(newBuilds, ", "))
}

type FinishedBuildCrawler struct {
	Travis *Travis
	DB     *db.DB
	Logger *log.Logger
}

func (c *FinishedBuildCrawler) Crawl() {
	ch := time.Tick(finishedBuildCrawlerInterval)
	for _ = range ch {
		c.Logger.Println("crawling for finsihed builds...")
		c.crawlFinishedBuilds()
	}
}

func (c *FinishedBuildCrawler) crawlFinishedBuilds() {
	colNames, finishedBuilds, skippedBuilds := c.doCrawlFinishedBuilds()
	c.Logger.Printf("fetched %d builds with %d finsihed and %d skipped. Finsihed builds: %s\n", len(finishedBuilds)+len(skippedBuilds), len(finishedBuilds), len(skippedBuilds), strings.Join(finishedBuilds, ", "))

	err := c.ensureColIndexes(colNames)
	if err != nil {
		c.Logger.Println(err)
	}
}

func (c *FinishedBuildCrawler) doCrawlFinishedBuilds() (colNames map[string]string, finishedBuilds []string, skippedBuilds []string) {
	colNames = make(map[string]string)

	var (
		repo *Repo
		//query db.Query
	)
	//query = Query{"lastbuildstartedat": Query{"$gte": oneMinuteAgo()}}
	iter := c.DB.C("new_builds").Find(nil).Sort("-lastbuildstartedat").Iter()
	for iter.Next(&repo) {
		build, err := c.crawlFinsihedBuild(repo)
		if err != nil {
			c.Logger.Println(err)
			skippedBuilds = append(skippedBuilds, repo.Slug)
			continue
		}

		colName, updated, err := c.upsertBuild(build)
		if err != nil {
			c.Logger.Println(err)
			continue
		}

		colNames[colName] = colName

		if updated {
			finishedBuilds = append(finishedBuilds, repo.Slug)
		}
	}

	if err := iter.Close(); err != nil {
		c.Logger.Println(err)
	}

	return
}

func (c *FinishedBuildCrawler) crawlFinsihedBuild(repo *Repo) (build *Build, err error) {
	build, err = c.Travis.Build(repo.LastBuildId)
	if err != nil {
		return
	}

	isFinished := !(build.FinishedAt == nil || build.StartedAt == nil)
	if !isFinished {
		err = fmt.Errorf("skipping build: %s - %d\n", repo.Slug, repo.LastBuildId)
		return
	}

	build.Repository = repo

	return
}

func (c *FinishedBuildCrawler) upsertBuild(build *Build) (colName string, updated bool, err error) {
	colName = buildColName(build.StartedAt)

	updated, err = c.DB.Upsert(colName, db.Query{"id": build.Id}, build)
	if err != nil {
		return
	}

	err = c.DB.C("new_builds").Remove(db.Query{"lastbuildid": build.Repository.LastBuildId})
	if err != nil {
		return
	}

	return
}

func (c *FinishedBuildCrawler) ensureColIndexes(colNames map[string]string) error {
	for _, colName := range colNames {
		c.Logger.Printf("ensuring index for collection %s\n", colName)
		err := c.DB.EnsureUniqueIndexKey(colName, "id")
		if err != nil {
			return err
		}
	}

	return nil
}

func buildColName(date *time.Time) string {
	buildDate := date.UTC().Format("2006_01_02")
	return fmt.Sprintf("builds_%s", buildDate)
}

func oneMinuteAgo() time.Time {
	now := time.Now()
	return now.Add(-1 * time.Minute).UTC()
}
<|endoftext|>


<commit_before>// +build windows

package runcmd

var testdata2 = []cnl{
	{command: &Command{
		Logfile: "out.log",
	},
		name:    "",
		logfile: "out.log",
	},
	{command: &Command{
		CommandLine: `echo "home=%HOME%"`,
		UseEnv:      true,
	},
		name:    "cmd-c-echo-home-home",
		logfile: "cmd-c-echo-home-home",
	},
}
<commit_msg>fix windows test<commit_after>// +build windows

package runcmd

var testdata2 = []cnl{
	{command: &Command{
		Logfile: "out.log",
	},
		name:    "",
		logfile: "out.log",
	},
	{command: &Command{
		CommandLine: `echo "home=%HOME%"`,
		UseEnv:      true,
	},
		name:    "cmd-c-echo-home-home",
		logfile: "runcmd-cmd-c-echo-home-home.log",
	},
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/mgutz/ansi"
	"github.com/phrase/phraseapp-go/phraseapp"
)

var Debug bool

type LocaleFiles []*LocaleFile
type LocaleFile struct {
	Path, Name, Id, RFC, Tag, FileFormat string
}

func (localeFile *LocaleFile) RelPath() string {
	callerPath, _ := os.Getwd()
	relativePath, _ := filepath.Rel(callerPath, localeFile.Path)
	return relativePath
}

// PathComponents replacement for ugly slicing string logic on paths
type PathComponents struct {
	Path        string
	Separator   string
	Parts       []string
	GlobPattern string
	IsDir       bool
}

func (pc *PathComponents) isLocalePatternUsed() bool {
	return pc.isLocaleNameInPath() || pc.isLocaleCodeInPath()
}

func (pc *PathComponents) isLocaleNameInPath() bool {
	return strings.Contains(pc.Path, "<locale_name>")
}

func (pc *PathComponents) isLocaleCodeInPath() bool {
	return strings.Contains(pc.Path, "<locale_code>")
}

func (pc *PathComponents) isTagInPath() bool {
	return strings.Contains(pc.Path, "<tag>")
}

func (pc *PathComponents) isValidLocale(locale *phraseapp.Locale) (bool, error) {
	localePresent := (locale != nil)

	if !localePresent {
		return false, fmt.Errorf("Locale not set")
	}

	if pc.isLocaleCodeInPath() && (locale.Code == "") {
		return false, fmt.Errorf("Locale code is not set for Locale with Id: %s but locale_code is used in file name", locale.Id)
	}
	return true, nil
}

func ExtractPathComponents(userPath string) (*PathComponents, error) {
	pc := &PathComponents{Separator: string(os.PathSeparator)}
	pc.GlobPattern = extractGlobPattern(userPath)
	pc.Path = strings.TrimSpace(strings.TrimSuffix(userPath, pc.GlobPattern))
	pc.Parts = splitToParts(userPath, pc.Separator)

	isDir, err := isDir(userPath)
	if err != nil {
		return nil, err
	}

	pc.IsDir = isDir

	return pc, nil
}

func isDir(path string) (bool, error) {
	if strings.Contains(path, "<") {
		return false, nil
	}

	file, err := os.Open(path)
	if err != nil {
		return false, err
	}
	defer file.Close()
	stat, err := file.Stat()
	if err != nil {
		return false, err
	}
	switch mode := stat.Mode(); {
	case mode.IsDir():
		return true, nil
	}
	return false, nil
}

func extractGlobPattern(userPath string) string {
	if strings.HasSuffix(userPath, path.Join("**", "*")) {
		return "**/*"
	} else if strings.HasSuffix(userPath, "*") {
		return "*"
	} else {
		return ""
	}
}

func splitToParts(userPath, separator string) []string {
	split := strings.Split(userPath, separator)
	parts := []string{}
	for _, part := range split {
		if part != separator {
			parts = append(parts, part)
		}
	}
	return parts
}

// Locale to Path mapping
func CopyLocale(relPath string, localeFile *LocaleFile) *LocaleFile {
	newLocale := &LocaleFile{Path: relPath, Id: localeFile.Id, Name: localeFile.Name, Tag: localeFile.Tag, FileFormat: localeFile.FileFormat}
	return newLocale
}

func (localeFile *LocaleFile) Message() string {
	str := ""
	if Debug {
		if localeFile.Name != "" {
			str = fmt.Sprintf("%s Name: %s", str, localeFile.Name)
		}
		if localeFile.Id != "" {
			str = fmt.Sprintf("%s Id: %s", str, localeFile.Id)
		}
		if localeFile.RFC != "" {
			str = fmt.Sprintf("%s RFC5646: %s", str, localeFile.RFC)
		}
		if localeFile.Tag != "" {
			str = fmt.Sprintf("%s Tag: %s", str, localeFile.Tag)
		}
		if localeFile.FileFormat != "" {
			str = fmt.Sprintf("%s Format: %s", str, localeFile.FileFormat)
		}
	} else {
		str = fmt.Sprintf("%s", localeFile.Name)
	}
	return strings.TrimSpace(str)
}

// Locale placeholder logic <locale_name>
func (pc *PathComponents) ExpandPathsWithLocale(locales []*phraseapp.Locale, localeFile *LocaleFile) (LocaleFiles, error) {
	files := []*LocaleFile{}
	for _, remoteLocale := range locales {
		if localeFile.Id != "" && !(remoteLocale.Id == localeFile.Id || remoteLocale.Name == localeFile.Id) {
			continue
		}
		valid, err := pc.isValidLocale(remoteLocale)
		if err != nil {
			fmt.Println(err.Error())
		}
		if !valid {
			continue
		}

		localeFile := &LocaleFile{Name: remoteLocale.Name, Id: remoteLocale.Id, RFC: remoteLocale.Code, Tag: localeFile.Tag, FileFormat: localeFile.FileFormat}
		absPath, err := pc.filePath(localeFile)
		if err != nil {
			return nil, err
		}
		localeFile.Path = absPath
		files = append(files, localeFile)
	}
	return files, nil
}

// Locale logic
func localeForLocaleId(localeId string, locales []*phraseapp.Locale) *phraseapp.Locale {
	for _, locale := range locales {
		if locale.Id == localeId {
			return locale
		}
	}
	return nil
}

func (pc *PathComponents) filePath(localeFile *LocaleFile) (string, error) {
	absPath, err := filepath.Abs(pc.Path)
	if err != nil {
		return "", err
	}

	path := strings.Replace(absPath, "<locale_name>", localeFile.Name, -1)
	path = strings.Replace(path, "<locale_code>", localeFile.RFC, -1)
	path = strings.Replace(path, "<tag>", localeFile.Tag, -1)

	return path, nil
}

func Authenticate() error {
	defaultCredentials, err := ConfigDefaultCredentials()
	if err != nil {
		return err
	}
	phraseapp.RegisterAuthCredentials(defaultCredentials, defaultCredentials)
	return nil
}

func printErr(err error, msg string) {
	red := ansi.ColorCode("red+b:black")
	reset := ansi.ColorCode("reset")
	fmt.Fprintf(os.Stderr, "%sERROR: %s %s%s\n", red, err, msg, reset)
}

func sharedMessage(method string, localeFile *LocaleFile) {
	green := ansi.ColorCode("green+b:black")
	reset := ansi.ColorCode("reset")

	local := fmt.Sprint(green, localeFile.RelPath(), reset)

	if method == "pull" {
		remote := fmt.Sprint(green, localeFile.Message(), reset)
		fmt.Println("Downloaded", remote, "to", local)
	} else {
		fmt.Println("Uploaded", local, "successfully.")
	}
}

func contains(pathes []string, str string) bool {
	for _, item := range pathes {
		if str == item {
			return true
		}
	}
	return false
}

func RemoteLocales(projectId string) ([]*phraseapp.Locale, error) {
	page := 1
	locales, err := phraseapp.LocalesList(projectId, page, 25)
	if err != nil {
		return nil, err
	}
	result := locales
	for len(locales) == 25 {
		page = page + 1
		locales, err = phraseapp.LocalesList(projectId, page, 25)
		if err != nil {
			return nil, err
		}
		result = append(result, locales...)
	}
	return locales, nil
}
<commit_msg>returns result instead of locales<commit_after>package main

import (
	"fmt"
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/mgutz/ansi"
	"github.com/phrase/phraseapp-go/phraseapp"
)

var Debug bool

type LocaleFiles []*LocaleFile
type LocaleFile struct {
	Path, Name, Id, RFC, Tag, FileFormat string
}

func (localeFile *LocaleFile) RelPath() string {
	callerPath, _ := os.Getwd()
	relativePath, _ := filepath.Rel(callerPath, localeFile.Path)
	return relativePath
}

// PathComponents replacement for ugly slicing string logic on paths
type PathComponents struct {
	Path        string
	Separator   string
	Parts       []string
	GlobPattern string
	IsDir       bool
}

func (pc *PathComponents) isLocalePatternUsed() bool {
	return pc.isLocaleNameInPath() || pc.isLocaleCodeInPath()
}

func (pc *PathComponents) isLocaleNameInPath() bool {
	return strings.Contains(pc.Path, "<locale_name>")
}

func (pc *PathComponents) isLocaleCodeInPath() bool {
	return strings.Contains(pc.Path, "<locale_code>")
}

func (pc *PathComponents) isTagInPath() bool {
	return strings.Contains(pc.Path, "<tag>")
}

func (pc *PathComponents) isValidLocale(locale *phraseapp.Locale) (bool, error) {
	localePresent := (locale != nil)

	if !localePresent {
		return false, fmt.Errorf("Locale not set")
	}

	if pc.isLocaleCodeInPath() && (locale.Code == "") {
		return false, fmt.Errorf("Locale code is not set for Locale with Id: %s but locale_code is used in file name", locale.Id)
	}
	return true, nil
}

func ExtractPathComponents(userPath string) (*PathComponents, error) {
	pc := &PathComponents{Separator: string(os.PathSeparator)}
	pc.GlobPattern = extractGlobPattern(userPath)
	pc.Path = strings.TrimSpace(strings.TrimSuffix(userPath, pc.GlobPattern))
	pc.Parts = splitToParts(userPath, pc.Separator)

	isDir, err := isDir(userPath)
	if err != nil {
		return nil, err
	}

	pc.IsDir = isDir

	return pc, nil
}

func isDir(path string) (bool, error) {
	if strings.Contains(path, "<") {
		return false, nil
	}

	file, err := os.Open(path)
	if err != nil {
		return false, err
	}
	defer file.Close()
	stat, err := file.Stat()
	if err != nil {
		return false, err
	}
	switch mode := stat.Mode(); {
	case mode.IsDir():
		return true, nil
	}
	return false, nil
}

func extractGlobPattern(userPath string) string {
	if strings.HasSuffix(userPath, path.Join("**", "*")) {
		return "**/*"
	} else if strings.HasSuffix(userPath, "*") {
		return "*"
	} else {
		return ""
	}
}

func splitToParts(userPath, separator string) []string {
	split := strings.Split(userPath, separator)
	parts := []string{}
	for _, part := range split {
		if part != separator {
			parts = append(parts, part)
		}
	}
	return parts
}

// Locale to Path mapping
func CopyLocale(relPath string, localeFile *LocaleFile) *LocaleFile {
	newLocale := &LocaleFile{Path: relPath, Id: localeFile.Id, Name: localeFile.Name, Tag: localeFile.Tag, FileFormat: localeFile.FileFormat}
	return newLocale
}

func (localeFile *LocaleFile) Message() string {
	str := ""
	if Debug {
		if localeFile.Name != "" {
			str = fmt.Sprintf("%s Name: %s", str, localeFile.Name)
		}
		if localeFile.Id != "" {
			str = fmt.Sprintf("%s Id: %s", str, localeFile.Id)
		}
		if localeFile.RFC != "" {
			str = fmt.Sprintf("%s RFC5646: %s", str, localeFile.RFC)
		}
		if localeFile.Tag != "" {
			str = fmt.Sprintf("%s Tag: %s", str, localeFile.Tag)
		}
		if localeFile.FileFormat != "" {
			str = fmt.Sprintf("%s Format: %s", str, localeFile.FileFormat)
		}
	} else {
		str = fmt.Sprintf("%s", localeFile.Name)
	}
	return strings.TrimSpace(str)
}

// Locale placeholder logic <locale_name>
func (pc *PathComponents) ExpandPathsWithLocale(locales []*phraseapp.Locale, localeFile *LocaleFile) (LocaleFiles, error) {
	files := []*LocaleFile{}
	for _, remoteLocale := range locales {
		if localeFile.Id != "" && !(remoteLocale.Id == localeFile.Id || remoteLocale.Name == localeFile.Id) {
			continue
		}
		valid, err := pc.isValidLocale(remoteLocale)
		if err != nil {
			fmt.Println(err.Error())
		}
		if !valid {
			continue
		}

		localeFile := &LocaleFile{Name: remoteLocale.Name, Id: remoteLocale.Id, RFC: remoteLocale.Code, Tag: localeFile.Tag, FileFormat: localeFile.FileFormat}
		absPath, err := pc.filePath(localeFile)
		if err != nil {
			return nil, err
		}
		localeFile.Path = absPath
		files = append(files, localeFile)
	}
	return files, nil
}

// Locale logic
func localeForLocaleId(localeId string, locales []*phraseapp.Locale) *phraseapp.Locale {
	for _, locale := range locales {
		if locale.Id == localeId {
			return locale
		}
	}
	return nil
}

func (pc *PathComponents) filePath(localeFile *LocaleFile) (string, error) {
	absPath, err := filepath.Abs(pc.Path)
	if err != nil {
		return "", err
	}

	path := strings.Replace(absPath, "<locale_name>", localeFile.Name, -1)
	path = strings.Replace(path, "<locale_code>", localeFile.RFC, -1)
	path = strings.Replace(path, "<tag>", localeFile.Tag, -1)

	return path, nil
}

func Authenticate() error {
	defaultCredentials, err := ConfigDefaultCredentials()
	if err != nil {
		return err
	}
	phraseapp.RegisterAuthCredentials(defaultCredentials, defaultCredentials)
	return nil
}

func printErr(err error, msg string) {
	red := ansi.ColorCode("red+b:black")
	reset := ansi.ColorCode("reset")
	fmt.Fprintf(os.Stderr, "%sERROR: %s %s%s\n", red, err, msg, reset)
}

func sharedMessage(method string, localeFile *LocaleFile) {
	green := ansi.ColorCode("green+b:black")
	reset := ansi.ColorCode("reset")

	local := fmt.Sprint(green, localeFile.RelPath(), reset)

	if method == "pull" {
		remote := fmt.Sprint(green, localeFile.Message(), reset)
		fmt.Println("Downloaded", remote, "to", local)
	} else {
		fmt.Println("Uploaded", local, "successfully.")
	}
}

func contains(pathes []string, str string) bool {
	for _, item := range pathes {
		if str == item {
			return true
		}
	}
	return false
}

func RemoteLocales(projectId string) ([]*phraseapp.Locale, error) {
	page := 1
	locales, err := phraseapp.LocalesList(projectId, page, 25)
	if err != nil {
		return nil, err
	}
	result := locales
	for len(locales) == 25 {
		page = page + 1
		locales, err = phraseapp.LocalesList(projectId, page, 25)
		if err != nil {
			return nil, err
		}
		result = append(result, locales...)
	}
	return result, nil
}
<|endoftext|>


<commit_before>package database

import (
	"database/sql"
	"fmt"
	"log"

	_ "github.com/olt/libpq"

	"github.com/seadsystem/Backend/DB/landingzone/constants"
	"github.com/seadsystem/Backend/DB/landingzone/decoders"
)

type DB struct {
	conn *sql.DB
}

func New() (DB, error) {
	conn, err := sql.Open("postgres", fmt.Sprintf("host=%s user=%s dbname=%s password=%s port=%d sslmode=disable", constants.DB_SOCKET, constants.DB_USER, constants.DB_NAME, constants.DB_PASSWORD, constants.DB_PORT))
	return DB{conn}, err
}

func (db DB) InsertRaw(database_channel <-chan decoders.SeadPacket) {
	// Example code: https://github.com/olt/pq/blob/bulk/copy_test.go
	stmt, err := db.conn.Prepare("COPY data_raw (serial, type, data, time) FROM STDIN")
	if err != nil {
		log.Fatal(err)
	}

	for {
		log.Println("Waiting for data...")
		data := <-database_channel
		log.Println("Inserting data...")
		log.Printf("Data: %+v\n", data)
		_, err = stmt.Exec(data.Serial, data.Type, data.Data, data.Timestamp)
		if err != nil {
			log.Println(err)
		}
	}
}
<commit_msg>Added zero arg Exec to match example.<commit_after>package database

import (
	"database/sql"
	"fmt"
	"log"

	_ "github.com/olt/libpq"

	"github.com/seadsystem/Backend/DB/landingzone/constants"
	"github.com/seadsystem/Backend/DB/landingzone/decoders"
)

type DB struct {
	conn *sql.DB
}

func New() (DB, error) {
	conn, err := sql.Open("postgres", fmt.Sprintf("host=%s user=%s dbname=%s password=%s port=%d sslmode=disable", constants.DB_SOCKET, constants.DB_USER, constants.DB_NAME, constants.DB_PASSWORD, constants.DB_PORT))
	return DB{conn}, err
}

func (db DB) InsertRaw(database_channel <-chan decoders.SeadPacket) {
	// Example code: https://github.com/olt/pq/blob/bulk/copy_test.go
	stmt, err := db.conn.Prepare("COPY data_raw (serial, type, data, time) FROM STDIN")
	if err != nil {
		log.Fatal(err)
	}

	for {
		log.Println("Waiting for data...")
		data := <-database_channel
		log.Println("Inserting data...")
		log.Printf("Data: %+v\n", data)
		_, err = stmt.Exec(data.Serial, data.Type, data.Data, data.Timestamp)
		if err != nil {
			log.Println(err)
			continue
		}
		_, err = stmt.Exec()
		if err != nil {
			log.Println(err)
			continue
		}
	}
}
<|endoftext|>


<commit_before>package comparer

import (
	"github.com/petar/GoLLRB/llrb"
	"testing"
)

func TestMergeAdjacentBlocksAfter(t *testing.T) {
	const BLOCK_SIZE = 4

	mergeChan := make(chan BlockMatchResult)
	merger := &MatchMerger{}
	merger.StartMergeResultStream(mergeChan, BLOCK_SIZE)

	mergeChan <- BlockMatchResult{
		ComparisonOffset: 0,
		BlockIdx:         0,
	}

	mergeChan <- BlockMatchResult{
		ComparisonOffset: BLOCK_SIZE,
		BlockIdx:         1,
	}

	close(mergeChan)

	merged := merger.GetMergedBlocks()

	if len(merged) != 1 {
		t.Fatalf("Wrong number of blocks returned: %#v", merged)
	}

	if merged[0].EndBlock != 1 {
		t.Errorf("Wrong EndBlock, expected 1 got %#v", merged[0])
	}
}

func TestMergeAdjacentBlocksBefore(t *testing.T) {
	const BLOCK_SIZE = 4

	mergeChan := make(chan BlockMatchResult)
	merger := &MatchMerger{}
	merger.StartMergeResultStream(mergeChan, BLOCK_SIZE)

	mergeChan <- BlockMatchResult{
		ComparisonOffset: BLOCK_SIZE,
		BlockIdx:         1,
	}

	mergeChan <- BlockMatchResult{
		ComparisonOffset: 0,
		BlockIdx:         0,
	}

	close(mergeChan)

	merged := merger.GetMergedBlocks()

	if len(merged) != 1 {
		t.Fatalf("Wrong number of blocks returned: %#v", merged)
	}

	if merged[0].EndBlock != 1 {
		t.Errorf("Wrong EndBlock, expected 1 got %#v", merged[0])
	}

	// start and end
	if merger.startEndBlockMap.Len() != 2 {
		t.Errorf("Wrong number of entries in the map: %v", merger.startEndBlockMap.Len())
	}
}

func TestMergeAdjacentBlocksBetween(t *testing.T) {
	const BLOCK_SIZE = 4

	mergeChan := make(chan BlockMatchResult)
	merger := &MatchMerger{}
	merger.StartMergeResultStream(mergeChan, BLOCK_SIZE)

	mergeChan <- BlockMatchResult{
		ComparisonOffset: 2 * BLOCK_SIZE,
		BlockIdx:         2,
	}

	mergeChan <- BlockMatchResult{
		ComparisonOffset: 0,
		BlockIdx:         0,
	}

	// match in the center
	mergeChan <- BlockMatchResult{
		ComparisonOffset: BLOCK_SIZE,
		BlockIdx:         1,
	}

	close(mergeChan)

	merged := merger.GetMergedBlocks()

	if len(merged) != 1 {
		t.Fatalf("Wrong number of blocks returned: %#v", merged)
	}

	if merged[0].EndBlock != 2 {
		t.Errorf("Wrong EndBlock, expected 2 got %#v", merged[0])
	}
	if merged[0].StartBlock != 0 {
		t.Errorf("Wrong StartBlock, expected 0, got %#v", merged[0])
	}
	if merger.startEndBlockMap.Len() != 2 {
		t.Errorf("Wrong number of entries in the map: %v", merger.startEndBlockMap.Len())
	}
}

func TestMissingBlocksOffsetStart(t *testing.T) {
	b := BlockSpanList{
		{
			StartBlock: 2,
			EndBlock:   3,
		},
	}

	m := b.GetMissingBlocks(3)

	if len(m) != 1 {
		t.Fatalf("Wrong number of missing blocks: %v", len(m))
	}

	if m[0].StartBlock != 0 {
		t.Errorf("Missing block has wrong start: %v", m[0].StartBlock)
	}
	if m[0].EndBlock != 1 {
		t.Errorf("Missing block has wrong end: %v", m[0].EndBlock)
	}
}

func TestMissingCenterBlock(t *testing.T) {
	b := BlockSpanList{
		{
			StartBlock: 0,
			EndBlock:   0,
		},
		{
			StartBlock: 2,
			EndBlock:   3,
		},
	}

	m := b.GetMissingBlocks(3)

	if len(m) != 1 {
		t.Fatalf("Wrong number of missing blocks: %v", len(m))
	}

	if m[0].StartBlock != 1 {
		t.Errorf("Missing block has wrong start: %v", m[0].StartBlock)
	}
	if m[0].EndBlock != 1 {
		t.Errorf("Missing block has wrong end: %v", m[0].EndBlock)
	}
}

func TestMissingEndBlock(t *testing.T) {
	b := BlockSpanList{
		{
			StartBlock: 0,
			EndBlock:   1,
		},
	}

	m := b.GetMissingBlocks(3)

	if len(m) != 1 {
		t.Fatalf("Wrong number of missing blocks: %v", len(m))
	}

	if m[0].StartBlock != 2 {
		t.Errorf("Missing block has wrong start: %v", m[0].StartBlock)
	}
	if m[0].EndBlock != 3 {
		t.Errorf("Missing block has wrong end: %v", m[0].EndBlock)
	}
}

func TestDuplicatedReferenceBlocks(t *testing.T) {
	// Reference = AA
	// Local = A
	const BLOCK_SIZE = 4

	mergeChan := make(chan BlockMatchResult)
	merger := &MatchMerger{}
	merger.StartMergeResultStream(mergeChan, BLOCK_SIZE)

	// When we find multiple strong matches, we send each of them
	mergeChan <- BlockMatchResult{
		ComparisonOffset: 0,
		BlockIdx:         0,
	}

	mergeChan <- BlockMatchResult{
		ComparisonOffset: 0,
		BlockIdx:         1,
	}

	close(mergeChan)

	merged := merger.GetMergedBlocks()

	if len(merged) != 2 {
		t.Errorf("Duplicated blocks cannot be merged: %#v", merged)
	}

	missing := merged.GetMissingBlocks(1)

	if len(missing) > 0 {
		t.Errorf("There were no missing blocks: %#v", missing)
	}
}

func TestDuplicatedLocalBlocks(t *testing.T) {
	// Reference = A
	// Local = AA
	const BLOCK_SIZE = 4

	mergeChan := make(chan BlockMatchResult)
	merger := &MatchMerger{}
	merger.StartMergeResultStream(mergeChan, BLOCK_SIZE)

	// When we find multiple strong matches, we send each of them
	mergeChan <- BlockMatchResult{
		ComparisonOffset: 0,
		BlockIdx:         0,
	}

	mergeChan <- BlockMatchResult{
		ComparisonOffset: BLOCK_SIZE,
		BlockIdx:         0,
	}

	close(mergeChan)

	// We only need one of the matches in the resulting file
	merged := merger.GetMergedBlocks()

	if len(merged) != 1 {
		t.Errorf("Duplicated blocks cannot be merged: %#v", merged)
	}

	missing := merged.GetMissingBlocks(0)

	if len(missing) > 0 {
		t.Errorf("There were no missing blocks: %#v", missing)
	}
}

func TestDoublyDuplicatedBlocks(t *testing.T) {
	// Reference = AA
	// Local = AA
	const BLOCK_SIZE = 4

	mergeChan := make(chan BlockMatchResult)
	merger := &MatchMerger{}
	merger.StartMergeResultStream(mergeChan, BLOCK_SIZE)

	// When we find multiple strong matches, we send each of them
	mergeChan <- BlockMatchResult{
		ComparisonOffset: 0,
		BlockIdx:         0,
	}

	mergeChan <- BlockMatchResult{
		ComparisonOffset: 0,
		BlockIdx:         1,
	}

	// Second local match
	mergeChan <- BlockMatchResult{
		ComparisonOffset: BLOCK_SIZE,
		BlockIdx:         0,
	}

	mergeChan <- BlockMatchResult{
		ComparisonOffset: BLOCK_SIZE,
		BlockIdx:         1,
	}

	close(mergeChan)

	merged := merger.GetMergedBlocks()

	if len(merged) != 2 {
		t.Errorf("Duplicated blocks cannot be merged: %#v", merged)
	}

	missing := merged.GetMissingBlocks(1)

	if len(missing) > 0 {
		t.Errorf("There were no missing blocks: %#v", missing)
	}
}

func TestBlockWithinSpan(t *testing.T) {
	// catch the case where we're informed about a block,
	// after we've merged blocks around it, so that the start and end
	// are within a span, not bordering one
	const BLOCK_SIZE = 4

	mergeChan := make(chan BlockMatchResult)
	merger := &MatchMerger{}
	merger.StartMergeResultStream(mergeChan, BLOCK_SIZE)

	mergeChan <- BlockMatchResult{
		ComparisonOffset: 0,
		BlockIdx:         0,
	}

	mergeChan <- BlockMatchResult{
		ComparisonOffset: BLOCK_SIZE,
		BlockIdx:         1,
	}

	mergeChan <- BlockMatchResult{
		ComparisonOffset: 2 * BLOCK_SIZE,
		BlockIdx:         2,
	}

	// This one is a duplicate of an earlier one
	mergeChan <- BlockMatchResult{
		ComparisonOffset: BLOCK_SIZE,
		BlockIdx:         1,
	}

	close(mergeChan)

	merged := merger.GetMergedBlocks()

	if len(merged) != 1 {
		t.Fatalf("Wrong number of blocks returned: %#v", merged)
	}

	if merged[0].EndBlock != 2 {
		t.Errorf("Wrong EndBlock, expected 2 got %#v", merged[0])
	}

	// start and end
	if merger.startEndBlockMap.Len() != 2 {
		t.Errorf("Wrong number of entries in the map: %v", merger.startEndBlockMap.Len())
	}
}

// Just to test out usage of the LLRB interface and helpers
func TestLLRB(t *testing.T) {
	m := &MatchMerger{}
	m.startEndBlockMap = llrb.New()

	bm := m.startEndBlockMap

	bm.ReplaceOrInsert(
		BlockSpanStart(
			BlockSpan{
				StartBlock: 0,
				EndBlock:   10,
			},
		),
	)

	bm.ReplaceOrInsert(
		BlockSpanEnd(
			BlockSpan{
				StartBlock: 0,
				EndBlock:   10,
			},
		),
	)

	i := bm.Get(BlockSpanKey(10))

	var EndBlock uint
	switch j := i.(type) {
	case BlockSpanStart:
		EndBlock = j.EndBlock
	case BlockSpanEnd:
		EndBlock = j.EndBlock
	}

	if EndBlock != 10 {
		t.Fail()
	}
}
<commit_msg>Removing LLRB test<commit_after>package comparer

import (
	"testing"
)

func TestMergeAdjacentBlocksAfter(t *testing.T) {
	const BLOCK_SIZE = 4

	mergeChan := make(chan BlockMatchResult)
	merger := &MatchMerger{}
	merger.StartMergeResultStream(mergeChan, BLOCK_SIZE)

	mergeChan <- BlockMatchResult{
		ComparisonOffset: 0,
		BlockIdx:         0,
	}

	mergeChan <- BlockMatchResult{
		ComparisonOffset: BLOCK_SIZE,
		BlockIdx:         1,
	}

	close(mergeChan)

	merged := merger.GetMergedBlocks()

	if len(merged) != 1 {
		t.Fatalf("Wrong number of blocks returned: %#v", merged)
	}

	if merged[0].EndBlock != 1 {
		t.Errorf("Wrong EndBlock, expected 1 got %#v", merged[0])
	}
}

func TestMergeAdjacentBlocksBefore(t *testing.T) {
	const BLOCK_SIZE = 4

	mergeChan := make(chan BlockMatchResult)
	merger := &MatchMerger{}
	merger.StartMergeResultStream(mergeChan, BLOCK_SIZE)

	mergeChan <- BlockMatchResult{
		ComparisonOffset: BLOCK_SIZE,
		BlockIdx:         1,
	}

	mergeChan <- BlockMatchResult{
		ComparisonOffset: 0,
		BlockIdx:         0,
	}

	close(mergeChan)

	merged := merger.GetMergedBlocks()

	if len(merged) != 1 {
		t.Fatalf("Wrong number of blocks returned: %#v", merged)
	}

	if merged[0].EndBlock != 1 {
		t.Errorf("Wrong EndBlock, expected 1 got %#v", merged[0])
	}

	// start and end
	if merger.startEndBlockMap.Len() != 2 {
		t.Errorf("Wrong number of entries in the map: %v", merger.startEndBlockMap.Len())
	}
}

func TestMergeAdjacentBlocksBetween(t *testing.T) {
	const BLOCK_SIZE = 4

	mergeChan := make(chan BlockMatchResult)
	merger := &MatchMerger{}
	merger.StartMergeResultStream(mergeChan, BLOCK_SIZE)

	mergeChan <- BlockMatchResult{
		ComparisonOffset: 2 * BLOCK_SIZE,
		BlockIdx:         2,
	}

	mergeChan <- BlockMatchResult{
		ComparisonOffset: 0,
		BlockIdx:         0,
	}

	// match in the center
	mergeChan <- BlockMatchResult{
		ComparisonOffset: BLOCK_SIZE,
		BlockIdx:         1,
	}

	close(mergeChan)

	merged := merger.GetMergedBlocks()

	if len(merged) != 1 {
		t.Fatalf("Wrong number of blocks returned: %#v", merged)
	}

	if merged[0].EndBlock != 2 {
		t.Errorf("Wrong EndBlock, expected 2 got %#v", merged[0])
	}
	if merged[0].StartBlock != 0 {
		t.Errorf("Wrong StartBlock, expected 0, got %#v", merged[0])
	}
	if merger.startEndBlockMap.Len() != 2 {
		t.Errorf("Wrong number of entries in the map: %v", merger.startEndBlockMap.Len())
	}
}

func TestMissingBlocksOffsetStart(t *testing.T) {
	b := BlockSpanList{
		{
			StartBlock: 2,
			EndBlock:   3,
		},
	}

	m := b.GetMissingBlocks(3)

	if len(m) != 1 {
		t.Fatalf("Wrong number of missing blocks: %v", len(m))
	}

	if m[0].StartBlock != 0 {
		t.Errorf("Missing block has wrong start: %v", m[0].StartBlock)
	}
	if m[0].EndBlock != 1 {
		t.Errorf("Missing block has wrong end: %v", m[0].EndBlock)
	}
}

func TestMissingCenterBlock(t *testing.T) {
	b := BlockSpanList{
		{
			StartBlock: 0,
			EndBlock:   0,
		},
		{
			StartBlock: 2,
			EndBlock:   3,
		},
	}

	m := b.GetMissingBlocks(3)

	if len(m) != 1 {
		t.Fatalf("Wrong number of missing blocks: %v", len(m))
	}

	if m[0].StartBlock != 1 {
		t.Errorf("Missing block has wrong start: %v", m[0].StartBlock)
	}
	if m[0].EndBlock != 1 {
		t.Errorf("Missing block has wrong end: %v", m[0].EndBlock)
	}
}

func TestMissingEndBlock(t *testing.T) {
	b := BlockSpanList{
		{
			StartBlock: 0,
			EndBlock:   1,
		},
	}

	m := b.GetMissingBlocks(3)

	if len(m) != 1 {
		t.Fatalf("Wrong number of missing blocks: %v", len(m))
	}

	if m[0].StartBlock != 2 {
		t.Errorf("Missing block has wrong start: %v", m[0].StartBlock)
	}
	if m[0].EndBlock != 3 {
		t.Errorf("Missing block has wrong end: %v", m[0].EndBlock)
	}
}

func TestDuplicatedReferenceBlocks(t *testing.T) {
	// Reference = AA
	// Local = A
	const BLOCK_SIZE = 4

	mergeChan := make(chan BlockMatchResult)
	merger := &MatchMerger{}
	merger.StartMergeResultStream(mergeChan, BLOCK_SIZE)

	// When we find multiple strong matches, we send each of them
	mergeChan <- BlockMatchResult{
		ComparisonOffset: 0,
		BlockIdx:         0,
	}

	mergeChan <- BlockMatchResult{
		ComparisonOffset: 0,
		BlockIdx:         1,
	}

	close(mergeChan)

	merged := merger.GetMergedBlocks()

	if len(merged) != 2 {
		t.Errorf("Duplicated blocks cannot be merged: %#v", merged)
	}

	missing := merged.GetMissingBlocks(1)

	if len(missing) > 0 {
		t.Errorf("There were no missing blocks: %#v", missing)
	}
}

func TestDuplicatedLocalBlocks(t *testing.T) {
	// Reference = A
	// Local = AA
	const BLOCK_SIZE = 4

	mergeChan := make(chan BlockMatchResult)
	merger := &MatchMerger{}
	merger.StartMergeResultStream(mergeChan, BLOCK_SIZE)

	// When we find multiple strong matches, we send each of them
	mergeChan <- BlockMatchResult{
		ComparisonOffset: 0,
		BlockIdx:         0,
	}

	mergeChan <- BlockMatchResult{
		ComparisonOffset: BLOCK_SIZE,
		BlockIdx:         0,
	}

	close(mergeChan)

	// We only need one of the matches in the resulting file
	merged := merger.GetMergedBlocks()

	if len(merged) != 1 {
		t.Errorf("Duplicated blocks cannot be merged: %#v", merged)
	}

	missing := merged.GetMissingBlocks(0)

	if len(missing) > 0 {
		t.Errorf("There were no missing blocks: %#v", missing)
	}
}

func TestDoublyDuplicatedBlocks(t *testing.T) {
	// Reference = AA
	// Local = AA
	const BLOCK_SIZE = 4

	mergeChan := make(chan BlockMatchResult)
	merger := &MatchMerger{}
	merger.StartMergeResultStream(mergeChan, BLOCK_SIZE)

	// When we find multiple strong matches, we send each of them
	mergeChan <- BlockMatchResult{
		ComparisonOffset: 0,
		BlockIdx:         0,
	}

	mergeChan <- BlockMatchResult{
		ComparisonOffset: 0,
		BlockIdx:         1,
	}

	// Second local match
	mergeChan <- BlockMatchResult{
		ComparisonOffset: BLOCK_SIZE,
		BlockIdx:         0,
	}

	mergeChan <- BlockMatchResult{
		ComparisonOffset: BLOCK_SIZE,
		BlockIdx:         1,
	}

	close(mergeChan)

	merged := merger.GetMergedBlocks()

	if len(merged) != 2 {
		t.Errorf("Duplicated blocks cannot be merged: %#v", merged)
	}

	missing := merged.GetMissingBlocks(1)

	if len(missing) > 0 {
		t.Errorf("There were no missing blocks: %#v", missing)
	}
}

func TestBlockWithinSpan(t *testing.T) {
	// catch the case where we're informed about a block,
	// after we've merged blocks around it, so that the start and end
	// are within a span, not bordering one
	const BLOCK_SIZE = 4

	mergeChan := make(chan BlockMatchResult)
	merger := &MatchMerger{}
	merger.StartMergeResultStream(mergeChan, BLOCK_SIZE)

	mergeChan <- BlockMatchResult{
		ComparisonOffset: 0,
		BlockIdx:         0,
	}

	mergeChan <- BlockMatchResult{
		ComparisonOffset: BLOCK_SIZE,
		BlockIdx:         1,
	}

	mergeChan <- BlockMatchResult{
		ComparisonOffset: 2 * BLOCK_SIZE,
		BlockIdx:         2,
	}

	// This one is a duplicate of an earlier one
	mergeChan <- BlockMatchResult{
		ComparisonOffset: BLOCK_SIZE,
		BlockIdx:         1,
	}

	close(mergeChan)

	merged := merger.GetMergedBlocks()

	if len(merged) != 1 {
		t.Fatalf("Wrong number of blocks returned: %#v", merged)
	}

	if merged[0].EndBlock != 2 {
		t.Errorf("Wrong EndBlock, expected 2 got %#v", merged[0])
	}

	// start and end
	if merger.startEndBlockMap.Len() != 2 {
		t.Errorf("Wrong number of entries in the map: %v", merger.startEndBlockMap.Len())
	}
}
<|endoftext|>


<commit_before>package config

import (
	"io/ioutil"

	"gopkg.in/yaml.v2"
)

// Define Application Settings Structure
type CassabonConfig struct {
	Logging struct {
		Logdir   string // Log Directory
		Loglevel string // Level to log at.
	}
	Cassandra struct {
		Hosts []string // List of hostnames or IP addresses of Cassandra ring
		Port  int      // Cassandra port
	}
	Api struct {
		Address string // HTTP API listens on this address
		Port    int    // HTTP API listens on this port
	}
	Redis struct {
		Index RedisSettings // Settings for Redis Index
		Queue RedisSettings // Settings for Redis Queue
	}
	RedisQueue struct {
		Sentinel bool     // True if sentinel, false if standalone.
		Addr     []string // List of addresses in host:port format
		DB       int64    // Redis DB number for the index.
	}
	Carbon struct {
		Address  string // Address for Carbon Receiver to listen on
		Port     int    // Port for Carbon Receiver to listen on
		Protocol string // "tcp", "udp" or "both" are acceptable
	}
	Statsd struct {
		Host string // Host or IP address of statsd server
		Port int    // Port that statsd server listens on
	}
	Rollups map[string][]string // Map of regex and default rollups
}

// Redis struct for redis connection information
type RedisSettings struct {
	Sentinel bool     // True if sentinel, false if standalone.
	Addr     []string // List of addresses in host:port format
	DB       int64    // Redis DB number for the index.
}

// Get Rollup Settings
func ParseConfig(configFile string) CassabonConfig {
	// Load config file
	yamlConfig, err := ioutil.ReadFile(configFile)

	if err != nil {
		panic(err)
	}

	// Initialize config struct
	var config CassabonConfig

	// Unmarshal config file into config struct
	err = yaml.Unmarshal(yamlConfig, &config)

	if err != nil {
		panic(err)
	}

	// Send back config struct
	return config
}
<commit_msg>Removed redundant setting in config.<commit_after>package config

import (
	"io/ioutil"

	"gopkg.in/yaml.v2"
)

// Define Application Settings Structure
type CassabonConfig struct {
	Logging struct {
		Logdir   string // Log Directory
		Loglevel string // Level to log at.
	}
	Cassandra struct {
		Hosts []string // List of hostnames or IP addresses of Cassandra ring
		Port  int      // Cassandra port
	}
	Api struct {
		Address string // HTTP API listens on this address
		Port    int    // HTTP API listens on this port
	}
	Redis struct {
		Index RedisSettings // Settings for Redis Index
		Queue RedisSettings // Settings for Redis Queue
	}
	Carbon struct {
		Address  string // Address for Carbon Receiver to listen on
		Port     int    // Port for Carbon Receiver to listen on
		Protocol string // "tcp", "udp" or "both" are acceptable
	}
	Statsd struct {
		Host string // Host or IP address of statsd server
		Port int    // Port that statsd server listens on
	}
	Rollups map[string][]string // Map of regex and default rollups
}

// Redis struct for redis connection information
type RedisSettings struct {
	Sentinel bool     // True if sentinel, false if standalone.
	Addr     []string // List of addresses in host:port format
	DB       int64    // Redis DB number for the index.
}

// Get Rollup Settings
func ParseConfig(configFile string) CassabonConfig {
	// Load config file
	yamlConfig, err := ioutil.ReadFile(configFile)

	if err != nil {
		panic(err)
	}

	// Initialize config struct
	var config CassabonConfig

	// Unmarshal config file into config struct
	err = yaml.Unmarshal(yamlConfig, &config)

	if err != nil {
		panic(err)
	}

	// Send back config struct
	return config
}
<|endoftext|>


<commit_before>package forms

import (
	"log"

	"github.com/trumae/carcara/ws/action"
	"github.com/trumae/valente"
	"golang.org/x/net/websocket"
)

const htmlFormHome = `

Home

`

//FormHome example
type FormHome struct {
	valente.FormImpl
}

//Initialize inits the Home Form
func (form FormHome) Initialize(ws *websocket.Conn) valente.Form {
	log.Println("FormHome Initialize")

	action.Html(ws, "content", htmlFormHome)

	return form
}
<commit_msg>error in ref old package<commit_after>package forms

import (
	"log"

	"github.com/trumae/valente"
	"github.com/trumae/valente/action"
	"golang.org/x/net/websocket"
)

const htmlFormHome = `

Home

`

//FormHome example
type FormHome struct {
	valente.FormImpl
}

//Initialize inits the Home Form
func (form FormHome) Initialize(ws *websocket.Conn) valente.Form {
	log.Println("FormHome Initialize")

	action.Html(ws, "content", htmlFormHome)

	return form
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"

	"koding/kites/kloud/digitalocean"

	"github.com/koding/kite"
)

// Builder is used to create and provisiong a single image or machine for a
// given Provider.
type Builder interface {
	// Prepare is responsible of configuring the builder and validating the
	// given configuration prior Build.
	Prepare(...interface{}) error

	// Build is creating a image and a machine.
	Build(...interface{}) (interface{}, error)
}

type buildArgs struct {
	Provider     string
	SnapshotName string
	Credential   map[string]interface{}
	Builder      map[string]interface{}
}

var (
	defaultSnapshotName = "koding-klient-0.0.1"
	providers           = map[string]interface{}{
		"digitalocean": &digitalocean.DigitalOcean{},
	}
)

func (k *Kloud) build(r *kite.Request) (interface{}, error) {
	args := &buildArgs{}
	if err := r.Args.One().Unmarshal(args); err != nil {
		return nil, err
	}

	p, ok := providers[args.Provider]
	if !ok {
		return nil, errors.New("provider not supported")
	}

	provider, ok := p.(Builder)
	if !ok {
		return nil, errors.New("provider doesn't satisfy the builder interface.")
	}

	if err := provider.Prepare(args.Credential, args.Builder); err != nil {
		return nil, err
	}

	snapshotName := defaultSnapshotName
	if args.SnapshotName != "" {
		snapshotName = args.SnapshotName
	}

	signFunc := func() (string, error) {
		fmt.Println("running signFucn")
		return createKey(r.Username, k.KontrolURL, k.KontrolPrivateKey, k.KontrolPublicKey)
	}

	artifact, err := provider.Build(snapshotName, r.Username, signFunc)
	if err != nil {
		return nil, err
	}

	return artifact, nil
}
<commit_msg>kloud/builder: add machineName feature<commit_after>package main

import (
	"errors"
	"strconv"
	"time"

	"koding/kites/kloud/digitalocean"

	"github.com/koding/kite"
)

// Builder is used to create and provisiong a single image or machine for a
// given Provider.
type Builder interface {
	// Prepare is responsible of configuring the builder and validating the
	// given configuration prior Build.
	Prepare(...interface{}) error

	// Build is creating a image and a machine.
	Build(...interface{}) (interface{}, error)
}

type buildArgs struct {
	Provider     string
	SnapshotName string
	MachineName  string
	Credential   map[string]interface{}
	Builder      map[string]interface{}
}

var (
	defaultSnapshotName = "koding-klient-0.0.1"
	providers           = map[string]interface{}{
		"digitalocean": &digitalocean.DigitalOcean{},
	}
)

func (k *Kloud) build(r *kite.Request) (interface{}, error) {
	args := &buildArgs{}
	if err := r.Args.One().Unmarshal(args); err != nil {
		return nil, err
	}

	p, ok := providers[args.Provider]
	if !ok {
		return nil, errors.New("provider not supported")
	}

	provider, ok := p.(Builder)
	if !ok {
		return nil, errors.New("provider doesn't satisfy the builder interface.")
	}

	if err := provider.Prepare(args.Credential, args.Builder); err != nil {
		return nil, err
	}

	snapshotName := defaultSnapshotName
	if args.SnapshotName != "" {
		snapshotName = args.SnapshotName
	}

	signFunc := func() (string, error) {
		return createKey(r.Username, k.KontrolURL, k.KontrolPrivateKey, k.KontrolPublicKey)
	}

	machineName := r.Username + "-" + strconv.FormatInt(time.Now().UTC().UnixNano(), 10)
	if args.MachineName != "" {
		machineName = args.MachineName
	}

	artifact, err := provider.Build(snapshotName, machineName, signFunc)
	if err != nil {
		return nil, err
	}

	return artifact, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"koding/db/mongodb/modelhelper"
	"socialapi/config"
	algoliaapi "socialapi/workers/algoliaconnector/api"
	"socialapi/workers/api/handlers"
	"socialapi/workers/api/modules/account"
	"socialapi/workers/api/modules/channel"
	"socialapi/workers/api/modules/client"
	"socialapi/workers/api/modules/interaction"
	"socialapi/workers/api/modules/message"
	"socialapi/workers/api/modules/messagelist"
	"socialapi/workers/api/modules/notificationsetting"
	"socialapi/workers/api/modules/participant"
	"socialapi/workers/api/modules/pinnedactivity"
	"socialapi/workers/api/modules/popular"
	"socialapi/workers/api/modules/privatechannel"
	"socialapi/workers/api/modules/reply"
	collaboration "socialapi/workers/collaboration/api"
	"socialapi/workers/common/mux"
	mailapi "socialapi/workers/email/mailparse/api"
	"socialapi/workers/helper"
	topicmoderationapi "socialapi/workers/moderation/topic/api"
	notificationapi "socialapi/workers/notification/api"
	"socialapi/workers/payment"
	paymentapi "socialapi/workers/payment/api"
	permissionapi "socialapi/workers/permission/api"
	sitemapapi "socialapi/workers/sitemap/api"
	trollmodeapi "socialapi/workers/trollmode/api"

	"github.com/koding/runner"
)

var (
	Name = "SocialAPI"
)

func main() {
	r := runner.New(Name)
	if err := r.Init(); err != nil {
		fmt.Println(err)
		return
	}
	defer r.Close()

	// appConfig
	c := config.MustRead(r.Conf.Path)

	mc := mux.NewConfig(Name, r.Conf.Host, r.Conf.Port)
	mc.Debug = r.Conf.Debug
	m := mux.New(mc, r.Log, r.Metrics)

	// init redis
	redisConn := r.Bongo.MustGetRedisConn()

	m.SetRedis(redisConn)

	handlers.AddHandlers(m)
	permissionapi.AddHandlers(m)
	topicmoderationapi.AddHandlers(m)
	collaboration.AddHandlers(m)
	paymentapi.AddHandlers(m)
	notificationapi.AddHandlers(m)
	trollmodeapi.AddHandlers(m)
	sitemapapi.AddHandlers(m)
	mailapi.AddHandlers(m)
	algoliaapi.AddHandlers(m, r.Log)

	account.AddHandlers(m)
	channel.AddHandlers(m)
	client.AddHandlers(m)
	interaction.AddHandlers(m)
	message.AddHandlers(m)
	messagelist.AddHandlers(m)
	participant.AddHandlers(m)
	pinnedactivity.AddHandlers(m)
	popular.AddHandlers(m)
	privatechannel.AddHandlers(m)
	reply.AddHandlers(m)
	notificationsetting.AddHandlers(m)

	// init mongo connection
	modelhelper.Initialize(c.Mongo)
	defer modelhelper.Close()

	mmdb, err := helper.ReadGeoIPDB(c)
	if err != nil {
		r.Log.Critical("ip persisting wont work err: %s", err.Error())
	} else {
		defer mmdb.Close()
	}

	// set default values for dev env
	if r.Conf.Environment == "dev" {
		go setDefaults(r.Log)
	}

	payment.Initialize(c)

	m.Listen()
	// shutdown server
	defer m.Close()

	r.Listen()
	r.Wait()
}
<commit_msg>socialapi: add realtime apis into gateway<commit_after>package main

import (
	"fmt"
	"koding/db/mongodb/modelhelper"
	"socialapi/config"
	algoliaapi "socialapi/workers/algoliaconnector/api"
	"socialapi/workers/api/handlers"
	"socialapi/workers/api/modules/account"
	"socialapi/workers/api/modules/channel"
	"socialapi/workers/api/modules/client"
	"socialapi/workers/api/modules/interaction"
	"socialapi/workers/api/modules/message"
	"socialapi/workers/api/modules/messagelist"
	"socialapi/workers/api/modules/notificationsetting"
	"socialapi/workers/api/modules/participant"
	"socialapi/workers/api/modules/pinnedactivity"
	"socialapi/workers/api/modules/popular"
	"socialapi/workers/api/modules/privatechannel"
	"socialapi/workers/api/modules/reply"
	collaboration "socialapi/workers/collaboration/api"
	"socialapi/workers/common/mux"
	mailapi "socialapi/workers/email/mailparse/api"
	"socialapi/workers/helper"
	topicmoderationapi "socialapi/workers/moderation/topic/api"
	notificationapi "socialapi/workers/notification/api"
	"socialapi/workers/payment"
	paymentapi "socialapi/workers/payment/api"
	permissionapi "socialapi/workers/permission/api"
	realtimeapi "socialapi/workers/realtime/api"
	sitemapapi "socialapi/workers/sitemap/api"
	trollmodeapi "socialapi/workers/trollmode/api"

	"github.com/koding/runner"
)

var (
	Name = "SocialAPI"
)

func main() {
	r := runner.New(Name)
	if err := r.Init(); err != nil {
		fmt.Println(err)
		return
	}
	defer r.Close()

	// appConfig
	c := config.MustRead(r.Conf.Path)

	mc := mux.NewConfig(Name, r.Conf.Host, r.Conf.Port)
	mc.Debug = r.Conf.Debug
	m := mux.New(mc, r.Log, r.Metrics)

	// init redis
	redisConn := r.Bongo.MustGetRedisConn()

	m.SetRedis(redisConn)

	handlers.AddHandlers(m)
	permissionapi.AddHandlers(m)
	topicmoderationapi.AddHandlers(m)
	collaboration.AddHandlers(m)
	paymentapi.AddHandlers(m)
	notificationapi.AddHandlers(m)
	trollmodeapi.AddHandlers(m)
	sitemapapi.AddHandlers(m)
	mailapi.AddHandlers(m)
	algoliaapi.AddHandlers(m, r.Log)

	account.AddHandlers(m)
	channel.AddHandlers(m)
	client.AddHandlers(m)
	interaction.AddHandlers(m)
	message.AddHandlers(m)
	messagelist.AddHandlers(m)
	participant.AddHandlers(m)
	pinnedactivity.AddHandlers(m)
	popular.AddHandlers(m)
	privatechannel.AddHandlers(m)
	reply.AddHandlers(m)
	notificationsetting.AddHandlers(m)
	realtimeapi.AddHandlers(m)

	// init mongo connection
	modelhelper.Initialize(c.Mongo)
	defer modelhelper.Close()

	mmdb, err := helper.ReadGeoIPDB(c)
	if err != nil {
		r.Log.Critical("ip persisting wont work err: %s", err.Error())
	} else {
		defer mmdb.Close()
	}

	// set default values for dev env
	if r.Conf.Environment == "dev" {
		go setDefaults(r.Log)
	}

	payment.Initialize(c)

	m.Listen()
	// shutdown server
	defer m.Close()

	r.Listen()
	r.Wait()
}
<|endoftext|>


<commit_before>package dataapi

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"reflect"
	"strconv"
	"time"
)

type ErrorCode int
type Error struct {
	Message string
	Code    ErrorCode
}

const (
	AuthenticationError ErrorCode = iota + 1
)

func (e *Error) Error() string { return e.Message }

type Client struct {
	accessTokenData accessTokenData
	Opts            ClientOptions
}

type ClientOptionsStruct struct {
	OptEndpoint   string
	OptApiVersion string
	OptClientId   string
	OptUsername   string
	OptPassword   string
}

type ClientOptions interface {
	Endpoint() string
	ApiVersion() string
	ClientId() string
	Username() string
	Password() string
}

type RequestParameters map[string]interface{}

type Result struct {
	Error *ResultError
}

type ResultError struct {
	Message string `json:"message"`
	Code    int    `json:"code"`
}

type authenticationResult struct {
	Result
	SessionId     string      `json:"sessionId"`
	AccessToken   string      `json:"accessToken"`
	ExpiresInData interface{} `json:"expiresIn"`
	ExpiresIn     int         `json:"-"`
	Remember      bool        `json:"remember"`
}

type accessTokenData struct {
	authenticationResult
	startTime time.Time
}

func (d *accessTokenData) Normalize() {
	switch t := d.ExpiresInData.(type) {
	case string:
		d.ExpiresIn, _ = strconv.Atoi(t)
	case float64:
		d.ExpiresIn = int(t)
	}
}

func (o ClientOptionsStruct) Endpoint() string {
	return o.OptEndpoint
}

func (o ClientOptionsStruct) ApiVersion() string {
	return o.OptApiVersion
}

func (o ClientOptionsStruct) ClientId() string {
	return o.OptClientId
}

func (o ClientOptionsStruct) Username() string {
	return o.OptUsername
}

func (o ClientOptionsStruct) Password() string {
	return o.OptPassword
}

func NewClient(opts ClientOptions) Client {
	return Client{
		Opts: opts,
	}
}

func (a accessTokenData) isPrepared() bool {
	if a.AccessToken == "" {
		return false
	}

	if a.startTime.Add(time.Duration(a.ExpiresIn-10) * time.Second).Before(time.Now()) {
		return false
	}

	return true
}

func (c *Client) prepareAccessToken() error {
	if c.accessTokenData.isPrepared() {
		return nil
	}

	var data accessTokenData
	if c.accessTokenData.SessionId != "" {
		req, err := http.NewRequest("POST", c.Opts.Endpoint()+"/v1/token", nil)
		if err != nil {
			return err
		}

		client := &http.Client{}
		req.Header.Add("X-MT-Authorization", "MTAuth sessionId="+c.accessTokenData.SessionId)
		resp, err := client.Do(req)
		if err != nil {
			return err
		}

		defer resp.Body.Close()
		body, err := ioutil.ReadAll(resp.Body)

		data = accessTokenData{}
		err = json.Unmarshal(body, &data)
		if err != nil {
			return err
		}
		data.Normalize()

		if data.AccessToken == "" {
			c.accessTokenData = accessTokenData{}
			return c.prepareAccessToken()
		}
	} else {
		resp, err := http.PostForm(c.Opts.Endpoint()+"/v1/authentication",
			url.Values{"clientId": {c.Opts.ClientId()}, "username": {c.Opts.Username()}, "password": {c.Opts.Password()}})
		if err != nil {
			return err
		}

		defer resp.Body.Close()
		body, err := ioutil.ReadAll(resp.Body)

		data = accessTokenData{}
		err = json.Unmarshal(body, &data)
		if err != nil {
			return err
		}
		data.Normalize()

		if data.AccessToken == "" {
			c.accessTokenData = accessTokenData{}
			return &Error{
				Message: "Authentication error",
				Code:    AuthenticationError,
			}
		}
	}

	data.startTime = time.Now()
	c.accessTokenData = data

	return nil
}

func (c Client) requiresAccessToken() bool {
	return c.accessTokenData.AccessToken != "" || c.accessTokenData.SessionId != "" || c.Opts.Password() != ""
}

func marshal(v interface{}) ([]byte, error) {
	kind := reflect.TypeOf(v).Kind()
	if kind == reflect.Bool {
		return []byte("0"), nil
	} else if kind <= reflect.Float64 || kind == reflect.String {
		return []byte(fmt.Sprint(v)), nil
	} else {
		return json.Marshal(v)
	}
}

func isFileType(v interface{}) bool {
	return reflect.TypeOf(v) == reflect.TypeOf(&os.File{})
}

func (c *Client) SendRequest(method string, path string, params *RequestParameters, result interface{}) error {
	if c.requiresAccessToken() {
		err := c.prepareAccessToken()
		if err != nil {
			return err
		}
	}

	var requestBody *bytes.Buffer
	var writer *multipart.Writer
	queryString := ""
	if params != nil {
		if method == "GET" {
			if len(*params) != 0 {
				values := url.Values{}
				for k, v := range *params {
					data, err := marshal(v)
					if err != nil {
						return err
					}
					values.Add(k, string(data))
				}
				queryString = "?" + values.Encode()
			}
		} else {
			requestBody = &bytes.Buffer{}
			writer = multipart.NewWriter(requestBody)
			for k, v := range *params {
				if isFileType(v) {
					file := v.(*os.File)
					part, err := writer.CreateFormFile(k, filepath.Base(file.Name()))
					if err != nil {
						return err
					}

					_, err = io.Copy(part, file)
					if err != nil {
						return err
					}
				} else {
					data, err := marshal(v)
					if err != nil {
						return err
					}
					err = writer.WriteField(k, string(data))
					if err != nil {
						return err
					}
				}
			}
			writer.Close()
		}
	}

	requestUrl := c.Opts.Endpoint() + "/v" + c.Opts.ApiVersion() + path + queryString
	req, err := (func() (*http.Request, error) {
		if requestBody == nil {
			return http.NewRequest(method, requestUrl, nil)
		} else {
			return http.NewRequest(method, requestUrl, requestBody)
		}
	})()
	if err != nil {
		return err
	}

	client := &http.Client{}
	if c.requiresAccessToken() {
		req.Header.Add("X-MT-Authorization", "MTAuth accessToken="+c.accessTokenData.AccessToken)
	}
	if writer != nil {
		req.Header.Set("Content-Type", writer.FormDataContentType())
	}
	resp, err := client.Do(req)
	if err != nil {
		return err
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)

	err = json.Unmarshal(body, result)
	if err != nil {
		return err
	}

	errorField := reflect.ValueOf(result).Elem().FieldByName("Error")
	var resultError *ResultError
	resultError = errorField.Interface().(*ResultError)

	if resultError != nil && resultError.Code == 401 {
		var nilError *ResultError
		errorField.Set(reflect.ValueOf(nilError))

		c.accessTokenData.AccessToken = ""

		return c.SendRequest(method, requestUrl, params, result)
	}

	return nil
}
<commit_msg>ResultError is a kind of Error.<commit_after>package dataapi

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"reflect"
	"strconv"
	"time"
)

type ErrorCode int
type Error struct {
	Message string
	Code    ErrorCode
}

const (
	AuthenticationError ErrorCode = iota + 1
)

func (e *Error) Error() string { return e.Message }

type Client struct {
	accessTokenData accessTokenData
	Opts            ClientOptions
}

type ClientOptionsStruct struct {
	OptEndpoint   string
	OptApiVersion string
	OptClientId   string
	OptUsername   string
	OptPassword   string
}

type ClientOptions interface {
	Endpoint() string
	ApiVersion() string
	ClientId() string
	Username() string
	Password() string
}

type RequestParameters map[string]interface{}

type Result struct {
	Error *ResultError
}

type ResultError struct {
	Message string `json:"message"`
	Code    int    `json:"code"`
}

func (e *ResultError) Error() string { return e.Message }

type authenticationResult struct {
	Result
	SessionId     string      `json:"sessionId"`
	AccessToken   string      `json:"accessToken"`
	ExpiresInData interface{} `json:"expiresIn"`
	ExpiresIn     int         `json:"-"`
	Remember      bool        `json:"remember"`
}

type accessTokenData struct {
	authenticationResult
	startTime time.Time
}

func (d *accessTokenData) Normalize() {
	switch t := d.ExpiresInData.(type) {
	case string:
		d.ExpiresIn, _ = strconv.Atoi(t)
	case float64:
		d.ExpiresIn = int(t)
	}
}

func (o ClientOptionsStruct) Endpoint() string {
	return o.OptEndpoint
}

func (o ClientOptionsStruct) ApiVersion() string {
	return o.OptApiVersion
}

func (o ClientOptionsStruct) ClientId() string {
	return o.OptClientId
}

func (o ClientOptionsStruct) Username() string {
	return o.OptUsername
}

func (o ClientOptionsStruct) Password() string {
	return o.OptPassword
}

func NewClient(opts ClientOptions) Client {
	return Client{
		Opts: opts,
	}
}

func (a accessTokenData) isPrepared() bool {
	if a.AccessToken == "" {
		return false
	}

	if a.startTime.Add(time.Duration(a.ExpiresIn-10) * time.Second).Before(time.Now()) {
		return false
	}

	return true
}

func (c *Client) prepareAccessToken() error {
	if c.accessTokenData.isPrepared() {
		return nil
	}

	var data accessTokenData
	if c.accessTokenData.SessionId != "" {
		req, err := http.NewRequest("POST", c.Opts.Endpoint()+"/v1/token", nil)
		if err != nil {
			return err
		}

		client := &http.Client{}
		req.Header.Add("X-MT-Authorization", "MTAuth sessionId="+c.accessTokenData.SessionId)
		resp, err := client.Do(req)
		if err != nil {
			return err
		}

		defer resp.Body.Close()
		body, err := ioutil.ReadAll(resp.Body)

		data = accessTokenData{}
		err = json.Unmarshal(body, &data)
		if err != nil {
			return err
		}
		data.Normalize()

		if data.AccessToken == "" {
			c.accessTokenData = accessTokenData{}
			return c.prepareAccessToken()
		}
	} else {
		resp, err := http.PostForm(c.Opts.Endpoint()+"/v1/authentication",
			url.Values{"clientId": {c.Opts.ClientId()}, "username": {c.Opts.Username()}, "password": {c.Opts.Password()}})
		if err != nil {
			return err
		}

		defer resp.Body.Close()
		body, err := ioutil.ReadAll(resp.Body)

		data = accessTokenData{}
		err = json.Unmarshal(body, &data)
		if err != nil {
			return err
		}
		data.Normalize()

		if data.AccessToken == "" {
			c.accessTokenData = accessTokenData{}
			return &Error{
				Message: "Authentication error",
				Code:    AuthenticationError,
			}
		}
	}

	data.startTime = time.Now()
	c.accessTokenData = data

	return nil
}

func (c Client) requiresAccessToken() bool {
	return c.accessTokenData.AccessToken != "" || c.accessTokenData.SessionId != "" || c.Opts.Password() != ""
}

func marshal(v interface{}) ([]byte, error) {
	kind := reflect.TypeOf(v).Kind()
	if kind == reflect.Bool {
		return []byte("0"), nil
	} else if kind <= reflect.Float64 || kind == reflect.String {
		return []byte(fmt.Sprint(v)), nil
	} else {
		return json.Marshal(v)
	}
}

func isFileType(v interface{}) bool {
	return reflect.TypeOf(v) == reflect.TypeOf(&os.File{})
}

func (c *Client) SendRequest(method string, path string, params *RequestParameters, result interface{}) error {
	if c.requiresAccessToken() {
		err := c.prepareAccessToken()
		if err != nil {
			return err
		}
	}

	var requestBody *bytes.Buffer
	var writer *multipart.Writer
	queryString := ""
	if params != nil {
		if method == "GET" {
			if len(*params) != 0 {
				values := url.Values{}
				for k, v := range *params {
					data, err := marshal(v)
					if err != nil {
						return err
					}
					values.Add(k, string(data))
				}
				queryString = "?" + values.Encode()
			}
		} else {
			requestBody = &bytes.Buffer{}
			writer = multipart.NewWriter(requestBody)
			for k, v := range *params {
				if isFileType(v) {
					file := v.(*os.File)
					part, err := writer.CreateFormFile(k, filepath.Base(file.Name()))
					if err != nil {
						return err
					}

					_, err = io.Copy(part, file)
					if err != nil {
						return err
					}
				} else {
					data, err := marshal(v)
					if err != nil {
						return err
					}
					err = writer.WriteField(k, string(data))
					if err != nil {
						return err
					}
				}
			}
			writer.Close()
		}
	}

	requestUrl := c.Opts.Endpoint() + "/v" + c.Opts.ApiVersion() + path + queryString
	req, err := (func() (*http.Request, error) {
		if requestBody == nil {
			return http.NewRequest(method, requestUrl, nil)
		} else {
			return http.NewRequest(method, requestUrl, requestBody)
		}
	})()
	if err != nil {
		return err
	}

	client := &http.Client{}
	if c.requiresAccessToken() {
		req.Header.Add("X-MT-Authorization", "MTAuth accessToken="+c.accessTokenData.AccessToken)
	}
	if writer != nil {
		req.Header.Set("Content-Type", writer.FormDataContentType())
	}
	resp, err := client.Do(req)
	if err != nil {
		return err
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)

	err = json.Unmarshal(body, result)
	if err != nil {
		return err
	}

	errorField := reflect.ValueOf(result).Elem().FieldByName("Error")
	var resultError *ResultError
	resultError = errorField.Interface().(*ResultError)

	if resultError != nil && resultError.Code == 401 {
		var nilError *ResultError
		errorField.Set(reflect.ValueOf(nilError))

		c.accessTokenData.AccessToken = ""

		return c.SendRequest(method, requestUrl, params, result)
	}

	return nil
}
<|endoftext|>