<commit_before>package collector

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"net/http"
	"net/http/httputil"
	"net/url"
	"strconv"
	"strings"
	"time"
)

const (
	ScheduledJobsCount  = "ScheduledJobsCount"
	RunningJobsCount    = "RunningJobsCount"
	UnfinishedJobsCount = "UnfinishedJobsCount"
	WaitingJobsCount    = "WaitingJobsCount"
	IdleAgentCount      = "IdleAgentCount"
	BusyAgentCount      = "BusyAgentCount"
	TotalAgentCount     = "TotalAgentCount"
	BusyAgentPercentage = "BusyAgentPercentage"

	PollDurationHeader = `Buildkite-Agent-Metrics-Poll-Duration`
)

type Collector struct {
	Endpoint  string
	Token     string
	UserAgent string
	Queues    []string
	Quiet     bool
	Debug     bool
	DebugHttp bool
}

type Result struct {
	Totals       map[string]int
	Queues       map[string]map[string]int
	Org          string
	PollDuration time.Duration
}

type organizationResponse struct {
	Slug string `json:"slug"`
}

type metricsAgentsResponse struct {
	Idle  int `json:"idle"`
	Busy  int `json:"busy"`
	Total int `json:"total"`
}

type metricsJobsResponse struct {
	Scheduled int `json:"scheduled"`
	Running   int `json:"running"`
	Waiting   int `json:"waiting"`
	Total     int `json:"total"`
}

type queueMetricsResponse struct {
	Agents       metricsAgentsResponse `json:"agents"`
	Jobs         metricsJobsResponse   `json:"jobs"`
	Organization organizationResponse  `json:"organization"`
}

type allMetricsAgentsResponse struct {
	metricsAgentsResponse
	Queues map[string]metricsAgentsResponse `json:"queues"`
}

type allMetricsJobsResponse struct {
	metricsJobsResponse
	Queues map[string]metricsJobsResponse `json:"queues"`
}

type allMetricsResponse struct {
	Agents       allMetricsAgentsResponse `json:"agents"`
	Jobs         allMetricsJobsResponse   `json:"jobs"`
	Organization organizationResponse     `json:"organization"`
}

func (c *Collector) Collect() (*Result, error) {
	result := &Result{
		Totals: map[string]int{},
		Queues: map[string]map[string]int{},
	}

	if len(c.Queues) == 0 {
		log.Println("Collecting agent metrics for all queues")

		endpoint, err := url.Parse(c.Endpoint)
		if err != nil {
			return nil, err
		}

		endpoint.Path += "/metrics"

		req, err := http.NewRequest("GET", endpoint.String(), nil)
		if err != nil {
			return nil, err
		}

		req.Header.Set("User-Agent", c.UserAgent)
		req.Header.Set("Authorization", fmt.Sprintf("Token %s", c.Token))

		if c.DebugHttp {
			if dump, err := httputil.DumpRequest(req, true); err == nil {
				log.Printf("DEBUG request uri=%s\n%s\n", req.URL, dump)
			}
		}

		httpClient := &http.Client{
			Timeout: 5 * time.Second,
		}

		res, err := httpClient.Do(req)
		if err != nil {
			return nil, err
		}
		defer res.Body.Close()

		if c.DebugHttp {
			if dump, err := httputil.DumpResponse(res, true); err == nil {
				log.Printf("DEBUG response uri=%s\n%s\n", req.URL, dump)
			}
		}

		// Handle any errors
		if res.StatusCode != http.StatusOK {
			// If it's json response, show the error message
			if strings.HasPrefix(res.Header.Get("Content-Type"), "application/json") {
				var errStruct struct {
					Message string `json:"message"`
				}
				err := json.NewDecoder(res.Body).Decode(&errStruct)
				if err == nil {
					return nil, errors.New(errStruct.Message)
				} else {
					log.Printf("Failed to decode error: %v", err)
				}
			}

			return nil, fmt.Errorf("Request failed with %s (%d)", res.Status, res.StatusCode)
		}

		var allMetrics allMetricsResponse

		// Check if we get a poll duration header from server
		if pollSeconds := res.Header.Get(PollDurationHeader); pollSeconds != "" {
			pollSecondsInt, err := strconv.ParseInt(pollSeconds, 10, 64)
			if err != nil {
				log.Printf("Failed to parse %s header: %v", PollDurationHeader, err)
			} else {
				result.PollDuration = time.Duration(pollSecondsInt) * time.Second
			}
		}

		err = json.NewDecoder(res.Body).Decode(&allMetrics)
		if err != nil {
			return nil, err
		}

		if allMetrics.Organization.Slug == "" {
			return nil, fmt.Errorf("No organization slug was found in the metrics response")
		}

		log.Printf("Found organization %q", allMetrics.Organization.Slug)
		result.Org = allMetrics.Organization.Slug

		result.Totals[ScheduledJobsCount] = allMetrics.Jobs.Scheduled
		result.Totals[RunningJobsCount] = allMetrics.Jobs.Running
		result.Totals[UnfinishedJobsCount] = allMetrics.Jobs.Total
		result.Totals[WaitingJobsCount] = allMetrics.Jobs.Waiting
		result.Totals[IdleAgentCount] = allMetrics.Agents.Idle
		result.Totals[BusyAgentCount] = allMetrics.Agents.Busy
		result.Totals[TotalAgentCount] = allMetrics.Agents.Total
		result.Totals[BusyAgentPercentage] = busyAgentPercentage(allMetrics.Agents.metricsAgentsResponse)

		for queueName, queueJobMetrics := range allMetrics.Jobs.Queues {
			if _, ok := result.Queues[queueName]; !ok {
				result.Queues[queueName] = map[string]int{}
			}
			result.Queues[queueName][ScheduledJobsCount] = queueJobMetrics.Scheduled
			result.Queues[queueName][RunningJobsCount] = queueJobMetrics.Running
			result.Queues[queueName][UnfinishedJobsCount] = queueJobMetrics.Total
			result.Queues[queueName][WaitingJobsCount] = queueJobMetrics.Waiting
		}

		for queueName, queueAgentMetrics := range allMetrics.Agents.Queues {
			if _, ok := result.Queues[queueName]; !ok {
				result.Queues[queueName] = map[string]int{}
			}
			result.Queues[queueName][IdleAgentCount] = queueAgentMetrics.Idle
			result.Queues[queueName][BusyAgentCount] = queueAgentMetrics.Busy
			result.Queues[queueName][TotalAgentCount] = queueAgentMetrics.Total
			result.Queues[queueName][BusyAgentPercentage] = busyAgentPercentage(queueAgentMetrics)
		}
	} else {
		for _, queue := range c.Queues {
			log.Printf("Collecting agent metrics for queue '%s'", queue)

			endpoint, err := url.Parse(c.Endpoint)
			if err != nil {
				return nil, err
			}

			endpoint.Path += "/metrics/queue"
			endpoint.RawQuery = url.Values{"name": {queue}}.Encode()

			req, err := http.NewRequest("GET", endpoint.String(), nil)
			if err != nil {
				return nil, err
			}

			req.Header.Set("User-Agent", c.UserAgent)
			req.Header.Set("Authorization", fmt.Sprintf("Token %s", c.Token))

			if c.DebugHttp {
				if dump, err := httputil.DumpRequest(req, true); err == nil {
					log.Printf("DEBUG request uri=%s\n%s\n", req.URL, dump)
				}
			}

			res, err := http.DefaultClient.Do(req)
			if err != nil {
				return nil, err
			}

			if c.DebugHttp {
				if dump, err := httputil.DumpResponse(res, true); err == nil {
					log.Printf("DEBUG response uri=%s\n%s\n", req.URL, dump)
				}
			}

			var queueMetrics queueMetricsResponse
			defer res.Body.Close()
			err = json.NewDecoder(res.Body).Decode(&queueMetrics)
			if err != nil {
				return nil, err
			}

			if queueMetrics.Organization.Slug == "" {
				return nil, fmt.Errorf("No organization slug was found in the metrics response")
			}

			log.Printf("Found organization %q", queueMetrics.Organization.Slug)
			result.Org = queueMetrics.Organization.Slug

			result.Queues[queue] = map[string]int{
				ScheduledJobsCount:  queueMetrics.Jobs.Scheduled,
				RunningJobsCount:    queueMetrics.Jobs.Running,
				UnfinishedJobsCount: queueMetrics.Jobs.Total,
				WaitingJobsCount:    queueMetrics.Jobs.Waiting,
				IdleAgentCount:      queueMetrics.Agents.Idle,
				BusyAgentCount:      queueMetrics.Agents.Busy,
				TotalAgentCount:     queueMetrics.Agents.Total,
				BusyAgentPercentage: busyAgentPercentage(queueMetrics.Agents),
			}
		}
	}

	if !c.Quiet {
		result.Dump()
	}

	return result, nil
}

func busyAgentPercentage(agents metricsAgentsResponse) int {
	if agents.Total > 0 {
		return int(100 * agents.Busy / agents.Total)
	}
	return 0
}

func (r Result) Dump() {
	for name, c := range r.Totals {
		log.Printf("Buildkite > Org=%s > %s=%d", r.Org, name, c)
	}

	for name, c := range r.Queues {
		for k, v := range c {
			log.Printf("Buildkite > Org=%s > Queue=%s > %s=%d", r.Org, name, k, v)
		}
	}
}
<commit_msg>feat: increase http client timeout<commit_after>package collector

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"net/http"
	"net/http/httputil"
	"net/url"
	"strconv"
	"strings"
	"time"
)

const (
	ScheduledJobsCount  = "ScheduledJobsCount"
	RunningJobsCount    = "RunningJobsCount"
	UnfinishedJobsCount = "UnfinishedJobsCount"
	WaitingJobsCount    = "WaitingJobsCount"
	IdleAgentCount      = "IdleAgentCount"
	BusyAgentCount      = "BusyAgentCount"
	TotalAgentCount     = "TotalAgentCount"
	BusyAgentPercentage = "BusyAgentPercentage"

	PollDurationHeader = `Buildkite-Agent-Metrics-Poll-Duration`
)

type Collector struct {
	Endpoint  string
	Token     string
	UserAgent string
	Queues    []string
	Quiet     bool
	Debug     bool
	DebugHttp bool
}

type Result struct {
	Totals       map[string]int
	Queues       map[string]map[string]int
	Org          string
	PollDuration time.Duration
}

type organizationResponse struct {
	Slug string `json:"slug"`
}

type metricsAgentsResponse struct {
	Idle  int `json:"idle"`
	Busy  int `json:"busy"`
	Total int `json:"total"`
}

type metricsJobsResponse struct {
	Scheduled int `json:"scheduled"`
	Running   int `json:"running"`
	Waiting   int `json:"waiting"`
	Total     int `json:"total"`
}

type queueMetricsResponse struct {
	Agents       metricsAgentsResponse `json:"agents"`
	Jobs         metricsJobsResponse   `json:"jobs"`
	Organization organizationResponse  `json:"organization"`
}

type allMetricsAgentsResponse struct {
	metricsAgentsResponse
	Queues map[string]metricsAgentsResponse `json:"queues"`
}

type allMetricsJobsResponse struct {
	metricsJobsResponse
	Queues map[string]metricsJobsResponse `json:"queues"`
}

type allMetricsResponse struct {
	Agents       allMetricsAgentsResponse `json:"agents"`
	Jobs         allMetricsJobsResponse   `json:"jobs"`
	Organization organizationResponse     `json:"organization"`
}

func (c *Collector) Collect() (*Result, error) {
	result := &Result{
		Totals: map[string]int{},
		Queues: map[string]map[string]int{},
	}

	if len(c.Queues) == 0 {
		log.Println("Collecting agent metrics for all queues")

		endpoint, err := url.Parse(c.Endpoint)
		if err != nil {
			return nil, err
		}

		endpoint.Path += "/metrics"

		req, err := http.NewRequest("GET", endpoint.String(), nil)
		if err != nil {
			return nil, err
		}

		req.Header.Set("User-Agent", c.UserAgent)
		req.Header.Set("Authorization", fmt.Sprintf("Token %s", c.Token))

		if c.DebugHttp {
			if dump, err := httputil.DumpRequest(req, true); err == nil {
				log.Printf("DEBUG request uri=%s\n%s\n", req.URL, dump)
			}
		}

		httpClient := &http.Client{
			Timeout: 15 * time.Second,
		}

		res, err := httpClient.Do(req)
		if err != nil {
			return nil, err
		}
		defer res.Body.Close()

		if c.DebugHttp {
			if dump, err := httputil.DumpResponse(res, true); err == nil {
				log.Printf("DEBUG response uri=%s\n%s\n", req.URL, dump)
			}
		}

		// Handle any errors
		if res.StatusCode != http.StatusOK {
			// If it's json response, show the error message
			if strings.HasPrefix(res.Header.Get("Content-Type"), "application/json") {
				var errStruct struct {
					Message string `json:"message"`
				}
				err := json.NewDecoder(res.Body).Decode(&errStruct)
				if err == nil {
					return nil, errors.New(errStruct.Message)
				} else {
					log.Printf("Failed to decode error: %v", err)
				}
			}

			return nil, fmt.Errorf("Request failed with %s (%d)", res.Status, res.StatusCode)
		}

		var allMetrics allMetricsResponse

		// Check if we get a poll duration header from server
		if pollSeconds := res.Header.Get(PollDurationHeader); pollSeconds != "" {
			pollSecondsInt, err := strconv.ParseInt(pollSeconds, 10, 64)
			if err != nil {
				log.Printf("Failed to parse %s header: %v", PollDurationHeader, err)
			} else {
				result.PollDuration = time.Duration(pollSecondsInt) * time.Second
			}
		}

		err = json.NewDecoder(res.Body).Decode(&allMetrics)
		if err != nil {
			return nil, err
		}

		if allMetrics.Organization.Slug == "" {
			return nil, fmt.Errorf("No organization slug was found in the metrics response")
		}

		log.Printf("Found organization %q", allMetrics.Organization.Slug)
		result.Org = allMetrics.Organization.Slug

		result.Totals[ScheduledJobsCount] = allMetrics.Jobs.Scheduled
		result.Totals[RunningJobsCount] = allMetrics.Jobs.Running
		result.Totals[UnfinishedJobsCount] = allMetrics.Jobs.Total
		result.Totals[WaitingJobsCount] = allMetrics.Jobs.Waiting
		result.Totals[IdleAgentCount] = allMetrics.Agents.Idle
		result.Totals[BusyAgentCount] = allMetrics.Agents.Busy
		result.Totals[TotalAgentCount] = allMetrics.Agents.Total
		result.Totals[BusyAgentPercentage] = busyAgentPercentage(allMetrics.Agents.metricsAgentsResponse)

		for queueName, queueJobMetrics := range allMetrics.Jobs.Queues {
			if _, ok := result.Queues[queueName]; !ok {
				result.Queues[queueName] = map[string]int{}
			}
			result.Queues[queueName][ScheduledJobsCount] = queueJobMetrics.Scheduled
			result.Queues[queueName][RunningJobsCount] = queueJobMetrics.Running
			result.Queues[queueName][UnfinishedJobsCount] = queueJobMetrics.Total
			result.Queues[queueName][WaitingJobsCount] = queueJobMetrics.Waiting
		}

		for queueName, queueAgentMetrics := range allMetrics.Agents.Queues {
			if _, ok := result.Queues[queueName]; !ok {
				result.Queues[queueName] = map[string]int{}
			}
			result.Queues[queueName][IdleAgentCount] = queueAgentMetrics.Idle
			result.Queues[queueName][BusyAgentCount] = queueAgentMetrics.Busy
			result.Queues[queueName][TotalAgentCount] = queueAgentMetrics.Total
			result.Queues[queueName][BusyAgentPercentage] = busyAgentPercentage(queueAgentMetrics)
		}
	} else {
		for _, queue := range c.Queues {
			log.Printf("Collecting agent metrics for queue '%s'", queue)

			endpoint, err := url.Parse(c.Endpoint)
			if err != nil {
				return nil, err
			}

			endpoint.Path += "/metrics/queue"
			endpoint.RawQuery = url.Values{"name": {queue}}.Encode()

			req, err := http.NewRequest("GET", endpoint.String(), nil)
			if err != nil {
				return nil, err
			}

			req.Header.Set("User-Agent", c.UserAgent)
			req.Header.Set("Authorization", fmt.Sprintf("Token %s", c.Token))

			if c.DebugHttp {
				if dump, err := httputil.DumpRequest(req, true); err == nil {
					log.Printf("DEBUG request uri=%s\n%s\n", req.URL, dump)
				}
			}

			res, err := http.DefaultClient.Do(req)
			if err != nil {
				return nil, err
			}

			if c.DebugHttp {
				if dump, err := httputil.DumpResponse(res, true); err == nil {
					log.Printf("DEBUG response uri=%s\n%s\n", req.URL, dump)
				}
			}

			var queueMetrics queueMetricsResponse
			defer res.Body.Close()
			err = json.NewDecoder(res.Body).Decode(&queueMetrics)
			if err != nil {
				return nil, err
			}

			if queueMetrics.Organization.Slug == "" {
				return nil, fmt.Errorf("No organization slug was found in the metrics response")
			}

			log.Printf("Found organization %q", queueMetrics.Organization.Slug)
			result.Org = queueMetrics.Organization.Slug

			result.Queues[queue] = map[string]int{
				ScheduledJobsCount:  queueMetrics.Jobs.Scheduled,
				RunningJobsCount:    queueMetrics.Jobs.Running,
				UnfinishedJobsCount: queueMetrics.Jobs.Total,
				WaitingJobsCount:    queueMetrics.Jobs.Waiting,
				IdleAgentCount:      queueMetrics.Agents.Idle,
				BusyAgentCount:      queueMetrics.Agents.Busy,
				TotalAgentCount:     queueMetrics.Agents.Total,
				BusyAgentPercentage: busyAgentPercentage(queueMetrics.Agents),
			}
		}
	}

	if !c.Quiet {
		result.Dump()
	}

	return result, nil
}

func busyAgentPercentage(agents metricsAgentsResponse) int {
	if agents.Total > 0 {
		return int(100 * agents.Busy / agents.Total)
	}
	return 0
}

func (r Result) Dump() {
	for name, c := range r.Totals {
		log.Printf("Buildkite > Org=%s > %s=%d", r.Org, name, c)
	}

	for name, c := range r.Queues {
		for k, v := range c {
			log.Printf("Buildkite > Org=%s > Queue=%s > %s=%d", r.Org, name, k, v)
		}
	}
}
<|endoftext|>


<commit_before>package expr

import (
	"math"
	"strconv"
	"testing"

	"github.com/grafana/metrictank/api/models"
	"github.com/grafana/metrictank/schema"
	"github.com/grafana/metrictank/test"
)

func shiftInput(in []schema.Point, offset int) []schema.Point {
	out := getCopy(in)
	for i := range in {
		if offset < 0 {
			out[i].Ts -= uint32(offset * -1)
		} else {
			out[i].Ts += uint32(offset)
		}
	}
	return out
}

func TestTimeShiftSingle(t *testing.T) {
	offset := -600
	testTimeShift(
		"identity",
		[]models.Series{
			{
				Interval:   10,
				QueryPatt:  "a",
				Target:     "a",
				Datapoints: shiftInput(a, -offset), // shift forward to avoid underflow
			},
		},
		[]models.Series{
			{
				Interval:   10,
				QueryPatt:  "timeShift(a, \"10m\")",
				Datapoints: shiftInput(a, -offset*2),
			},
		},
		t,
		offset,
		"10m",
		true,
		false,
	)
}

func TestTimeShiftMultiple(t *testing.T) {
	offset := -600
	testTimeShift(
		"identity",
		[]models.Series{
			{
				Interval:   10,
				QueryPatt:  "a",
				Target:     "a",
				Datapoints: shiftInput(a, -offset), // shift forward to avoid underflow
			},
			{
				Interval:   10,
				QueryPatt:  "b",
				Target:     "b",
				Datapoints: shiftInput(b, -offset), // shift forward to avoid underflow
			},
		},
		[]models.Series{
			{
				Interval:   10,
				QueryPatt:  "timeShift(a, \"10m\")",
				Datapoints: shiftInput(a, -offset*2),
			},
			{
				Interval:   10,
				QueryPatt:  "timeShift(b, \"10m\")",
				Datapoints: shiftInput(b, -offset*2),
			},
		},
		t,
		offset,
		"10m",
		true,
		false,
	)
}

func TestTimeShiftPositive(t *testing.T) {
	offset := 600
	testTimeShift(
		"identity",
		[]models.Series{
			{
				Interval:   10,
				QueryPatt:  "a",
				Target:     "a",
				Datapoints: shiftInput(a, offset), // shift forward to avoid underflow
			},
		},
		[]models.Series{
			{
				Interval:   10,
				QueryPatt:  "timeShift(a, \"+10m\")",
				Datapoints: shiftInput(a, 0),
			},
		},
		t,
		offset,
		"+10m",
		true,
		false,
	)
}

func testTimeShift(name string, in []models.Series, out []models.Series, t *testing.T, expectedOffset int, shift string, resetEnd, alignDST bool) {
	f := NewTimeShift()
	f.(*FuncTimeShift).in = NewMock(in)
	f.(*FuncTimeShift).timeShift = shift
	f.(*FuncTimeShift).resetEnd = resetEnd
	f.(*FuncTimeShift).alignDST = alignDST

	// Much of the important logic is in the context
	context := Context{
		from: in[0].Datapoints[0].Ts,
		to:   in[0].Datapoints[len(in[0].Datapoints)-1].Ts,
	}
	newContext := f.Context(context)

	actualOffset := int(newContext.from) - int(context.from)
	if actualOffset != expectedOffset {
		t.Fatalf("case %q: Expected context offset = %d, got %d", name, expectedOffset, actualOffset)
	}

	gots, err := f.Exec(make(map[Req][]models.Series))
	if err != nil {
		t.Fatalf("case %q: err should be nil. got %q", name, err)
	}
	if len(gots) != len(out) {
		t.Fatalf("case %q: timeShift len output expected %d, got %d", name, len(out), len(gots))
	}
	for i, g := range gots {
		exp := out[i]
		if g.QueryPatt != exp.QueryPatt {
			t.Fatalf("case %q : expected target %q, got %q", name, exp.QueryPatt, g.QueryPatt)
		}
		if len(g.Datapoints) != len(exp.Datapoints) {
			t.Fatalf("case %q len output expected %d, got %d", name, len(exp.Datapoints), len(g.Datapoints))
		}
		for j, p := range g.Datapoints {
			bothNaN := math.IsNaN(p.Val) && math.IsNaN(exp.Datapoints[j].Val)
			if (bothNaN || p.Val == exp.Datapoints[j].Val) && p.Ts == exp.Datapoints[j].Ts {
				continue
			}
			t.Fatalf("case %q: output point %d - expected %v got %v", name, j, exp.Datapoints[j], p)
		}
	}
}

func BenchmarkTimeShift10k_1NoNulls(b *testing.B) {
	benchmarkTimeShift(b, 1, test.RandFloats10k, test.RandFloats10k)
}
func BenchmarkTimeShift10k_10NoNulls(b *testing.B) {
	benchmarkTimeShift(b, 10, test.RandFloats10k, test.RandFloats10k)
}
func BenchmarkTimeShift10k_100NoNulls(b *testing.B) {
	benchmarkTimeShift(b, 100, test.RandFloats10k, test.RandFloats10k)
}
func BenchmarkTimeShift10k_1000NoNulls(b *testing.B) {
	benchmarkTimeShift(b, 1000, test.RandFloats10k, test.RandFloats10k)
}

func BenchmarkTimeShift10k_1SomeSeriesHalfNulls(b *testing.B) {
	benchmarkTimeShift(b, 1, test.RandFloats10k, test.RandFloatsWithNulls10k)
}
func BenchmarkTimeShift10k_10SomeSeriesHalfNulls(b *testing.B) {
	benchmarkTimeShift(b, 10, test.RandFloats10k, test.RandFloatsWithNulls10k)
}
func BenchmarkTimeShift10k_100SomeSeriesHalfNulls(b *testing.B) {
	benchmarkTimeShift(b, 100, test.RandFloats10k, test.RandFloatsWithNulls10k)
}
func BenchmarkTimeShift10k_1000SomeSeriesHalfNulls(b *testing.B) {
	benchmarkTimeShift(b, 1000, test.RandFloats10k, test.RandFloatsWithNulls10k)
}

func BenchmarkTimeShift10k_1AllSeriesHalfNulls(b *testing.B) {
	benchmarkTimeShift(b, 1, test.RandFloatsWithNulls10k, test.RandFloatsWithNulls10k)
}
func BenchmarkTimeShift10k_10AllSeriesHalfNulls(b *testing.B) {
	benchmarkTimeShift(b, 10, test.RandFloatsWithNulls10k, test.RandFloatsWithNulls10k)
}
func BenchmarkTimeShift10k_100AllSeriesHalfNulls(b *testing.B) {
	benchmarkTimeShift(b, 100, test.RandFloatsWithNulls10k, test.RandFloatsWithNulls10k)
}
func BenchmarkTimeShift10k_1000AllSeriesHalfNulls(b *testing.B) {
	benchmarkTimeShift(b, 1000, test.RandFloatsWithNulls10k, test.RandFloatsWithNulls10k)
}

func benchmarkTimeShift(b *testing.B, numSeries int, fn0, fn1 func() []schema.Point) {
	var input []models.Series
	for i := 0; i < numSeries; i++ {
		series := models.Series{
			QueryPatt: strconv.Itoa(i),
		}
		if i%2 == 0 {
			series.Datapoints = fn0()
		} else {
			series.Datapoints = fn1()
		}
		input = append(input, series)
	}
	b.ResetTimer()
	dataMap := DataMap(make(map[Req][]models.Series))
	for i := 0; i < b.N; i++ {
		f := NewTimeShift()
		f.(*FuncTimeShift).in = NewMock(input)
		f.(*FuncTimeShift).timeShift = "-1h"
		got, err := f.Exec(dataMap)
		if err != nil {
			b.Fatalf("%s", err)
		}
		dataMap.Clean()
		results = got
	}
}
<commit_msg>Fix test cases<commit_after>package expr

import (
	"math"
	"strconv"
	"testing"

	"github.com/grafana/metrictank/api/models"
	"github.com/grafana/metrictank/schema"
	"github.com/grafana/metrictank/test"
)

func shiftInput(in []schema.Point, offset int) []schema.Point {
	out := getCopy(in)
	for i := range in {
		if offset < 0 {
			out[i].Ts -= uint32(offset * -1)
		} else {
			out[i].Ts += uint32(offset)
		}
	}
	return out
}

func TestTimeShiftSingle(t *testing.T) {
	offset := -600
	testTimeShift(
		"identity",
		[]models.Series{
			{
				Interval:   10,
				QueryPatt:  "a",
				Target:     "a",
				Datapoints: shiftInput(a, -offset), // shift forward to avoid underflow
			},
		},
		[]models.Series{
			{
				Interval:   10,
				QueryPatt:  "timeShift(a, \"-10m\")",
				Datapoints: shiftInput(a, -offset*2),
			},
		},
		t,
		offset,
		"10m",
		true,
		false,
	)
}

func TestTimeShiftMultiple(t *testing.T) {
	offset := -600
	testTimeShift(
		"identity",
		[]models.Series{
			{
				Interval:   10,
				QueryPatt:  "a",
				Target:     "a",
				Datapoints: shiftInput(a, -offset), // shift forward to avoid underflow
			},
			{
				Interval:   10,
				QueryPatt:  "b",
				Target:     "b",
				Datapoints: shiftInput(b, -offset), // shift forward to avoid underflow
			},
		},
		[]models.Series{
			{
				Interval:   10,
				QueryPatt:  "timeShift(a, \"-10m\")",
				Datapoints: shiftInput(a, -offset*2),
			},
			{
				Interval:   10,
				QueryPatt:  "timeShift(b, \"-10m\")",
				Datapoints: shiftInput(b, -offset*2),
			},
		},
		t,
		offset,
		"10m",
		true,
		false,
	)
}

func TestTimeShiftPositive(t *testing.T) {
	offset := 600
	testTimeShift(
		"identity",
		[]models.Series{
			{
				Interval:   10,
				QueryPatt:  "a",
				Target:     "a",
				Datapoints: shiftInput(a, offset), // shift forward to avoid underflow
			},
		},
		[]models.Series{
			{
				Interval:   10,
				QueryPatt:  "timeShift(a, \"+10m\")",
				Datapoints: shiftInput(a, 0),
			},
		},
		t,
		offset,
		"+10m",
		true,
		false,
	)
}

func testTimeShift(name string, in []models.Series, out []models.Series, t *testing.T, expectedOffset int, shift string, resetEnd, alignDST bool) {
	f := NewTimeShift()
	f.(*FuncTimeShift).in = NewMock(in)
	f.(*FuncTimeShift).timeShift = shift
	f.(*FuncTimeShift).resetEnd = resetEnd
	f.(*FuncTimeShift).alignDST = alignDST

	// Much of the important logic is in the context
	context := Context{
		from: in[0].Datapoints[0].Ts,
		to:   in[0].Datapoints[len(in[0].Datapoints)-1].Ts,
	}
	newContext := f.Context(context)

	actualOffset := int(newContext.from) - int(context.from)
	if actualOffset != expectedOffset {
		t.Fatalf("case %q: Expected context offset = %d, got %d", name, expectedOffset, actualOffset)
	}

	gots, err := f.Exec(make(map[Req][]models.Series))
	if err != nil {
		t.Fatalf("case %q: err should be nil. got %q", name, err)
	}
	if len(gots) != len(out) {
		t.Fatalf("case %q: timeShift len output expected %d, got %d", name, len(out), len(gots))
	}
	for i, g := range gots {
		exp := out[i]
		if g.QueryPatt != exp.QueryPatt {
			t.Fatalf("case %q : expected target %q, got %q", name, exp.QueryPatt, g.QueryPatt)
		}
		if len(g.Datapoints) != len(exp.Datapoints) {
			t.Fatalf("case %q len output expected %d, got %d", name, len(exp.Datapoints), len(g.Datapoints))
		}
		for j, p := range g.Datapoints {
			bothNaN := math.IsNaN(p.Val) && math.IsNaN(exp.Datapoints[j].Val)
			if (bothNaN || p.Val == exp.Datapoints[j].Val) && p.Ts == exp.Datapoints[j].Ts {
				continue
			}
			t.Fatalf("case %q: output point %d - expected %v got %v", name, j, exp.Datapoints[j], p)
		}
	}
}

func BenchmarkTimeShift10k_1NoNulls(b *testing.B) {
	benchmarkTimeShift(b, 1, test.RandFloats10k, test.RandFloats10k)
}
func BenchmarkTimeShift10k_10NoNulls(b *testing.B) {
	benchmarkTimeShift(b, 10, test.RandFloats10k, test.RandFloats10k)
}
func BenchmarkTimeShift10k_100NoNulls(b *testing.B) {
	benchmarkTimeShift(b, 100, test.RandFloats10k, test.RandFloats10k)
}
func BenchmarkTimeShift10k_1000NoNulls(b *testing.B) {
	benchmarkTimeShift(b, 1000, test.RandFloats10k, test.RandFloats10k)
}

func BenchmarkTimeShift10k_1SomeSeriesHalfNulls(b *testing.B) {
	benchmarkTimeShift(b, 1, test.RandFloats10k, test.RandFloatsWithNulls10k)
}
func BenchmarkTimeShift10k_10SomeSeriesHalfNulls(b *testing.B) {
	benchmarkTimeShift(b, 10, test.RandFloats10k, test.RandFloatsWithNulls10k)
}
func BenchmarkTimeShift10k_100SomeSeriesHalfNulls(b *testing.B) {
	benchmarkTimeShift(b, 100, test.RandFloats10k, test.RandFloatsWithNulls10k)
}
func BenchmarkTimeShift10k_1000SomeSeriesHalfNulls(b *testing.B) {
	benchmarkTimeShift(b, 1000, test.RandFloats10k, test.RandFloatsWithNulls10k)
}

func BenchmarkTimeShift10k_1AllSeriesHalfNulls(b *testing.B) {
	benchmarkTimeShift(b, 1, test.RandFloatsWithNulls10k, test.RandFloatsWithNulls10k)
}
func BenchmarkTimeShift10k_10AllSeriesHalfNulls(b *testing.B) {
	benchmarkTimeShift(b, 10, test.RandFloatsWithNulls10k, test.RandFloatsWithNulls10k)
}
func BenchmarkTimeShift10k_100AllSeriesHalfNulls(b *testing.B) {
	benchmarkTimeShift(b, 100, test.RandFloatsWithNulls10k, test.RandFloatsWithNulls10k)
}
func BenchmarkTimeShift10k_1000AllSeriesHalfNulls(b *testing.B) {
	benchmarkTimeShift(b, 1000, test.RandFloatsWithNulls10k, test.RandFloatsWithNulls10k)
}

func benchmarkTimeShift(b *testing.B, numSeries int, fn0, fn1 func() []schema.Point) {
	var input []models.Series
	for i := 0; i < numSeries; i++ {
		series := models.Series{
			QueryPatt: strconv.Itoa(i),
		}
		if i%2 == 0 {
			series.Datapoints = fn0()
		} else {
			series.Datapoints = fn1()
		}
		input = append(input, series)
	}
	b.ResetTimer()
	dataMap := DataMap(make(map[Req][]models.Series))
	for i := 0; i < b.N; i++ {
		f := NewTimeShift()
		f.(*FuncTimeShift).in = NewMock(input)
		f.(*FuncTimeShift).timeShift = "-1h"
		got, err := f.Exec(dataMap)
		if err != nil {
			b.Fatalf("%s", err)
		}
		dataMap.Clean()
		results = got
	}
}
<|endoftext|>


<commit_before><commit_msg>bugfix<commit_after><|endoftext|>


<commit_before>package app

import (
	"github.com/Sirupsen/logrus"
	dockerLib "github.com/fsouza/go-dockerclient"
	"github.com/swipely/iam-docker/src/docker"
	"github.com/swipely/iam-docker/src/http"
	"github.com/swipely/iam-docker/src/iam"
	netHTTP "net/http"
	"net/http/httputil"
	"time"
)

// New creates a new application with the given config.
func New(config *Config, dockerClient docker.RawClient, stsClient iam.STSClient, errorChan chan<- error) *App {
	return &App{
		Config:       config,
		DockerClient: dockerClient,
		STSClient:    stsClient,
		ErrorChan:    errorChan,
	}
}

// Run starts the application asynchronously.
func (app *App) Run() {
	log.Info("Running the app")

	containerStore := docker.NewContainerStore(app.DockerClient)
	credentialStore := iam.NewCredentialStore(app.STSClient)
	eventHandler := docker.NewEventHandler(app.Config.EventHandlers, containerStore, credentialStore)
	proxy := httputil.NewSingleHostReverseProxy(app.Config.MetaDataUpstream)
	handler := http.NewIAMHandler(proxy, containerStore, credentialStore)

	go app.containerSyncWorker(containerStore)
	go app.refreshCredentialWorker(credentialStore)
	go app.httpWorker(handler)
	go app.eventWorker(eventHandler)
}

func (app *App) containerSyncWorker(containerStore docker.ContainerStore) {
	timer := time.Tick(app.Config.DockerSyncPeriod)
	wlog := log.WithFields(logrus.Fields{"worker": "sync-containers"})
	wlog.Info("Starting")

	go app.syncRunningContainers(containerStore, wlog)
	for range timer {
		go app.syncRunningContainers(containerStore, wlog)
	}
}

func (app *App) refreshCredentialWorker(credentialStore iam.CredentialStore) {
	timer := time.Tick(app.Config.CredentialRefreshPeriod)
	wlog := log.WithFields(logrus.Fields{"worker": "refresh-credentials"})
	wlog.Info("Starting")

	for range timer {
		wlog.Debug("Refreshing credentials")
		go credentialStore.RefreshCredentials()
	}
}

func (app *App) httpWorker(handler netHTTP.Handler) {
	wlog := log.WithFields(logrus.Fields{"worker": "http"})
	server := netHTTP.Server{
		Addr:           app.Config.ListenAddr,
		Handler:        handler,
		ReadTimeout:    app.Config.ReadTimeout,
		WriteTimeout:   app.Config.WriteTimeout,
		MaxHeaderBytes: 1 << 20,
	}

	wlog.Info("Starting")
	err := server.ListenAndServe()
	wlog.WithFields(logrus.Fields{
		"error": err.Error(),
	}).Error("Failed to serve HTTP")
	app.ErrorChan <- err
}

func (app *App) eventWorker(eventHandler docker.EventHandler) {
	wlog := log.WithFields(logrus.Fields{"worker": "event-handler"})
	wlog.Info("Starting")
	for {
		wlog.Debug("Listening for Docker events")
		events := make(chan *dockerLib.APIEvents, app.Config.EventHandlers)
		err := app.DockerClient.AddEventListener(events)
		if err != nil {
			wlog.WithFields(logrus.Fields{
				"error": err.Error(),
			}).Error("Failed to add Docker event listener")
			app.ErrorChan <- err
			return
		}
		err = eventHandler.Listen(events)
		if err != nil {
			wlog.WithFields(logrus.Fields{
				"error": err.Error(),
			}).Warn("Exited")
		} else {
			wlog.Warn("Exited with no error")
		}
	}
}

func (app *App) syncRunningContainers(containerStore docker.ContainerStore, logger *logrus.Entry) {
	logger.Debug("Syncing containers")
	err := containerStore.SyncRunningContainers()
	if err != nil {
		logger.WithFields(logrus.Fields{
			"error": err.Error(),
		}).Warn("Failed syncing running containers")
	}
}
<commit_msg>Use logrus for HTTP logging<commit_after>package app

import (
	"github.com/Sirupsen/logrus"
	dockerLib "github.com/fsouza/go-dockerclient"
	"github.com/swipely/iam-docker/src/docker"
	"github.com/swipely/iam-docker/src/http"
	"github.com/swipely/iam-docker/src/iam"
	stdLog "log"
	netHTTP "net/http"
	"net/http/httputil"
	"time"
)

// New creates a new application with the given config.
func New(config *Config, dockerClient docker.RawClient, stsClient iam.STSClient, errorChan chan<- error) *App {
	return &App{
		Config:       config,
		DockerClient: dockerClient,
		STSClient:    stsClient,
		ErrorChan:    errorChan,
	}
}

// Run starts the application asynchronously.
func (app *App) Run() {
	log.Info("Running the app")

	containerStore := docker.NewContainerStore(app.DockerClient)
	credentialStore := iam.NewCredentialStore(app.STSClient)
	eventHandler := docker.NewEventHandler(app.Config.EventHandlers, containerStore, credentialStore)
	proxy := httputil.NewSingleHostReverseProxy(app.Config.MetaDataUpstream)
	handler := http.NewIAMHandler(proxy, containerStore, credentialStore)

	go app.containerSyncWorker(containerStore)
	go app.refreshCredentialWorker(credentialStore)
	go app.httpWorker(handler)
	go app.eventWorker(eventHandler)
}

func (app *App) containerSyncWorker(containerStore docker.ContainerStore) {
	timer := time.Tick(app.Config.DockerSyncPeriod)
	wlog := log.WithFields(logrus.Fields{"worker": "sync-containers"})
	wlog.Info("Starting")

	go app.syncRunningContainers(containerStore, wlog)
	for range timer {
		go app.syncRunningContainers(containerStore, wlog)
	}
}

func (app *App) refreshCredentialWorker(credentialStore iam.CredentialStore) {
	timer := time.Tick(app.Config.CredentialRefreshPeriod)
	wlog := log.WithFields(logrus.Fields{"worker": "refresh-credentials"})
	wlog.Info("Starting")

	for range timer {
		wlog.Debug("Refreshing credentials")
		go credentialStore.RefreshCredentials()
	}
}

func (app *App) httpWorker(handler netHTTP.Handler) {
	wlog := log.WithFields(logrus.Fields{"worker": "http"})
	writer := wlog.Logger.Writer()
	server := netHTTP.Server{
		Addr:           app.Config.ListenAddr,
		Handler:        handler,
		ReadTimeout:    app.Config.ReadTimeout,
		WriteTimeout:   app.Config.WriteTimeout,
		MaxHeaderBytes: 1 << 20,
		ErrorLog:       stdLog.New(writer, "", 0),
	}
	wlog.Info("Starting")
	err := server.ListenAndServe()
	wlog.WithFields(logrus.Fields{
		"error": err.Error(),
	}).Error("Failed to serve HTTP")
	_ = writer.Close()
	app.ErrorChan <- err
}

func (app *App) eventWorker(eventHandler docker.EventHandler) {
	wlog := log.WithFields(logrus.Fields{"worker": "event-handler"})
	wlog.Info("Starting")
	for {
		wlog.Debug("Listening for Docker events")
		events := make(chan *dockerLib.APIEvents, app.Config.EventHandlers)
		err := app.DockerClient.AddEventListener(events)
		if err != nil {
			wlog.WithFields(logrus.Fields{
				"error": err.Error(),
			}).Error("Failed to add Docker event listener")
			app.ErrorChan <- err
			return
		}
		err = eventHandler.Listen(events)
		if err != nil {
			wlog.WithFields(logrus.Fields{
				"error": err.Error(),
			}).Warn("Exited")
		} else {
			wlog.Warn("Exited with no error")
		}
	}
}

func (app *App) syncRunningContainers(containerStore docker.ContainerStore, logger *logrus.Entry) {
	logger.Debug("Syncing containers")
	err := containerStore.SyncRunningContainers()
	if err != nil {
		logger.WithFields(logrus.Fields{
			"error": err.Error(),
		}).Warn("Failed syncing running containers")
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2014 go-trello authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package trello

import (
	"encoding/json"
	"strings"
)

type Member struct {
	client     *Client
	Id         string `json:"id"`
	AvatarHash string `json:"avatarHash"`
	Bio        string `json:"bio"`
	BioData    struct {
		Emoji interface{} `json:"emoji,omitempty"`
	} `json:"bioData"`
	Confirmed                bool     `json:"confirmed"`
	FullName                 string   `json:"fullName"`
	IdPremOrgsAdmin          []string `json:"idPremOrgsAdmin"`
	Initials                 string   `json:"initials"`
	MemberType               string   `json:"memberType"`
	Products                 []int    `json:"products"`
	Status                   string   `json:"status"`
	Url                      string   `json:"url"`
	Username                 string   `json:"username"`
	AvatarSource             string   `json:"avatarSource"`
	Email                    string   `json:"email"`
	GravatarHash             string   `json:"gravatarHash"`
	IdBoards                 []string `json:"idBoards"`
	IdBoardsPinned           []string `json:"idBoardsPinned"`
	IdOrganizations          []string `json:"idOrganizations"`
	LoginTypes               string   `json:"loginTypes"`
	NewEmail                 string   `json:"newEmail"`
	OneTimeMessagesDismissed []string `json:"oneTimeMessagesDismissed"`
	Prefs                    struct {
		SendSummaries                 bool   `json:"sendSummaries"`
		MinutesBetweenSummaries       int    `json:"minutesBetweenSummaries"`
		MinutesBeforeDeadlineToNotify int    `json:"minutesBeforeDeadlineToNotify"`
		ColorBlind                    bool   `json:"colorBlind"`
		Locale                        string `json:"locale"`
	} `json:"prefs"`
	Trophies           []string `json:"trophies"`
	UploadedAvatarHash string   `json:"uploadedAvatarHash"`
	PremiumFeatures    []string `json:"premiumFeatures"`
}

func (c *Client) Member(nick string) (member *Member, err error) {
	body, err := c.Get("/members/" + nick)
	if err != nil {
		return
	}

	err = json.Unmarshal(body, &member)
	member.client = c
	return
}

func (m *Member) Boards(field ...string) (boards []Board, err error) {
	fields := ""
	if len(field) == 0 {
		fields = "all"
	} else {
		fields = strings.Join(field, ",")
	}

	body, err := m.client.Get("/members/" + m.Id + "/boards?fields=" + fields)
	if err != nil {
		return
	}

	err = json.Unmarshal(body, &boards)
	for i := range boards {
		boards[i].client = m.client
	}
	return
}

// TODO: Avatar sizes [170, 30]
func (m *Member) AvatarUrl() string {
	return "https://trello-avatars.s3.amazonaws.com/" + m.AvatarHash + "/170.png"
}
<commit_msg>Fixes member struct, `LoginTypes` changes to `[]string`<commit_after>/*
Copyright 2014 go-trello authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package trello

import (
	"encoding/json"
	"strings"
)

type Member struct {
	client     *Client
	Id         string `json:"id"`
	AvatarHash string `json:"avatarHash"`
	Bio        string `json:"bio"`
	BioData    struct {
		Emoji interface{} `json:"emoji,omitempty"`
	} `json:"bioData"`
	Confirmed                bool     `json:"confirmed"`
	FullName                 string   `json:"fullName"`
	IdPremOrgsAdmin          []string `json:"idPremOrgsAdmin"`
	Initials                 string   `json:"initials"`
	MemberType               string   `json:"memberType"`
	Products                 []int    `json:"products"`
	Status                   string   `json:"status"`
	Url                      string   `json:"url"`
	Username                 string   `json:"username"`
	AvatarSource             string   `json:"avatarSource"`
	Email                    string   `json:"email"`
	GravatarHash             string   `json:"gravatarHash"`
	IdBoards                 []string `json:"idBoards"`
	IdBoardsPinned           []string `json:"idBoardsPinned"`
	IdOrganizations          []string `json:"idOrganizations"`
	LoginTypes               []string `json:"loginTypes"`
	NewEmail                 string   `json:"newEmail"`
	OneTimeMessagesDismissed []string `json:"oneTimeMessagesDismissed"`
	Prefs                    struct {
		SendSummaries                 bool   `json:"sendSummaries"`
		MinutesBetweenSummaries       int    `json:"minutesBetweenSummaries"`
		MinutesBeforeDeadlineToNotify int    `json:"minutesBeforeDeadlineToNotify"`
		ColorBlind                    bool   `json:"colorBlind"`
		Locale                        string `json:"locale"`
	} `json:"prefs"`
	Trophies           []string `json:"trophies"`
	UploadedAvatarHash string   `json:"uploadedAvatarHash"`
	PremiumFeatures    []string `json:"premiumFeatures"`
}

func (c *Client) Member(nick string) (member *Member, err error) {
	body, err := c.Get("/members/" + nick)
	if err != nil {
		return
	}

	err = json.Unmarshal(body, &member)
	member.client = c
	return
}

func (m *Member) Boards(field ...string) (boards []Board, err error) {
	fields := ""
	if len(field) == 0 {
		fields = "all"
	} else {
		fields = strings.Join(field, ",")
	}

	body, err := m.client.Get("/members/" + m.Id + "/boards?fields=" + fields)
	if err != nil {
		return
	}

	err = json.Unmarshal(body, &boards)
	for i := range boards {
		boards[i].client = m.client
	}
	return
}

// TODO: Avatar sizes [170, 30]
func (m *Member) AvatarUrl() string {
	return "https://trello-avatars.s3.amazonaws.com/" + m.AvatarHash + "/170.png"
}
<|endoftext|>


<commit_before>// Copyright 2016 Mender Software AS
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.
package main

import (
	"bufio"
	"os"
	"strings"
	"time"

	"github.com/mendersoftware/log"
	"github.com/pkg/errors"
)

type Controller interface {
	Authorize() menderError
	Bootstrap() menderError
	GetCurrentImageID() string
	GetUpdatePollInterval() time.Duration
	HasUpgrade() (bool, menderError)
	CheckUpdate() (*UpdateResponse, menderError)

	UInstallCommitRebooter
	Updater
	StateRunner
}

const (
	defaultManifestFile = "/etc/build_mender"
	defaultKeyFile      = "mender-agent.pem"
	defaultDataStore    = "/var/lib/mender"
)

type MenderState int

// State transitions:
//
//            unknown
//               |
//               v
//             init
//               |
//               v
//          bootstrapped
//               |
//       +-------+-------------+
//       |                     |
//       v                     v
// fresh update         wait for update
//
// Any state can transition to MenderStateError

const (
	// initial state
	MenderStateInit MenderState = iota
	// client is bootstrapped, i.e. ready to go
	MenderStateBootstrapped
	// wait for new update
	MenderStateUpdateCheckWait
	// check update
	MenderStateUpdateCheck
	// update fetch
	MenderStateUpdateFetch
	// update install
	MenderStateUpdateInstall
	// commit needed
	MenderStateUpdateCommit
	// reboot
	MenderStateReboot
	// error
	MenderStateError
	// exit state
	MenderStateDone
)

type mender struct {
	UInstallCommitRebooter
	Updater
	env            BootEnvReadWriter
	state          State
	config         menderConfig
	manifestFile   string
	deviceKey      *Keystore
	forceBootstrap bool
}

type MenderPieces struct {
	updater Updater
	device  UInstallCommitRebooter
	env     BootEnvReadWriter
	store   Store
}

func NewMender(config menderConfig, pieces MenderPieces) *mender {

	ks := NewKeystore(pieces.store)
	if ks == nil {
		return nil
	}

	m := &mender{
		UInstallCommitRebooter: pieces.device,
		Updater:                pieces.updater,
		env:                    pieces.env,
		manifestFile:           defaultManifestFile,
		deviceKey:              ks,
		state:                  initState,
		config:                 config,
	}

	if err := m.deviceKey.Load(m.config.DeviceKey); err != nil && IsNoKeys(err) == false {
		log.Errorf("failed to load device keys: %s", err)
		return nil
	}

	return m
}

func (m mender) GetCurrentImageID() string {
	// This is where Yocto stores buid information
	manifest, err := os.Open(m.manifestFile)
	if err != nil {
		log.Error("Can not read current image id.")
		return ""
	}

	imageID := ""

	scanner := bufio.NewScanner(manifest)
	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())
		log.Debug("Read data from device manifest file: ", line)
		if strings.HasPrefix(line, "IMAGE_ID") {
			log.Debug("Found device id line: ", line)
			lineID := strings.Split(line, "=")
			if len(lineID) != 2 {
				log.Errorf("Broken device manifest file: (%v)", lineID)
				return ""
			}
			log.Debug("Current image id: ", strings.TrimSpace(lineID[1]))
			return strings.TrimSpace(lineID[1])
		}
	}
	if err := scanner.Err(); err != nil {
		log.Error(err)
	}
	return imageID
}

func (m *mender) HasUpgrade() (bool, menderError) {
	env, err := m.env.ReadEnv("upgrade_available")
	if err != nil {
		return false, NewFatalError(err)
	}
	upgradeAvailable := env["upgrade_available"]

	// we are after update
	if upgradeAvailable == "1" {
		return true, nil
	}
	return false, nil
}

func (m *mender) ForceBootstrap() {
	m.forceBootstrap = true
}

func (m *mender) needsBootstrap() bool {
	if m.forceBootstrap {
		return true
	}

	if m.deviceKey.Private() == nil {
		log.Debugf("needs keys")
		return true
	}

	return false
}

func (m *mender) Bootstrap() menderError {
	if !m.needsBootstrap() {
		return nil
	}

	return m.doBootstrap()
}

func (m *mender) doBootstrap() menderError {
	if m.deviceKey.Private() == nil {
		log.Infof("device keys not present, generating")
		if err := m.deviceKey.Generate(); err != nil {
			return NewFatalError(err)
		}

		if err := m.deviceKey.Save(m.config.DeviceKey); err != nil {
			log.Errorf("failed to save keys to %s: %s",
				m.config.DeviceKey, err)
			return NewFatalError(err)
		}
	}

	m.forceBootstrap = false

	return nil
}

// Check if new update is available. In case of errors, returns nil and error
// that occurred. If no update is available *UpdateResponse is nil, otherwise it
// contains update information.
func (m *mender) CheckUpdate() (*UpdateResponse, menderError) {
	currentImageID := m.GetCurrentImageID()
	//TODO: if currentImageID == "" {
	// 	return errors.New("")
	// }

	haveUpdate, err := m.Updater.GetScheduledUpdate(m.config.ServerURL, m.config.DeviceID)
	if err != nil {
		log.Error("Error receiving scheduled update data: ", err)
		return nil, NewTransientError(err)
	}

	log.Debug("Received correct response for update request.")

	update, ok := haveUpdate.(UpdateResponse)
	if !ok {
		return nil, NewTransientError(errors.Errorf("not an update response?"))
	}

	if update.Image.YoctoID == currentImageID {
		return nil, nil
	}
	return &update, nil
}

func (m mender) GetUpdatePollInterval() time.Duration {
	return time.Duration(m.config.PollIntervalSeconds) * time.Second
}

func (m *mender) SetState(s State) {
	log.Infof("Mender state: %v -> %v", m.state.Id(), s.Id())
	m.state = s
}

func (m *mender) GetState() State {
	return m.state
}
<commit_msg>mender: add Authorized and AuthorizeWait states<commit_after>// Copyright 2016 Mender Software AS
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.
package main

import (
	"bufio"
	"os"
	"strings"
	"time"

	"github.com/mendersoftware/log"
	"github.com/pkg/errors"
)

type Controller interface {
	Authorize() menderError
	Bootstrap() menderError
	GetCurrentImageID() string
	GetUpdatePollInterval() time.Duration
	HasUpgrade() (bool, menderError)
	CheckUpdate() (*UpdateResponse, menderError)

	UInstallCommitRebooter
	Updater
	StateRunner
}

const (
	defaultManifestFile = "/etc/build_mender"
	defaultKeyFile      = "mender-agent.pem"
	defaultDataStore    = "/var/lib/mender"
)

type MenderState int

// State transitions:
//
//            unknown
//               |
//               v
//             init
//               |
//               v
//          bootstrapped
//               |
//       +-------+-------------+
//       |                     |
//       v                     v
// fresh update         wait for update
//
// Any state can transition to MenderStateError

const (
	// initial state
	MenderStateInit MenderState = iota
	// client is bootstrapped, i.e. ready to go
	MenderStateBootstrapped
	// client has all authorization data available
	MenderStateAuthorized
	// wait before authorization attempt
	MenderStateAuthorizeWait
	// wait for new update
	MenderStateUpdateCheckWait
	// check update
	MenderStateUpdateCheck
	// update fetch
	MenderStateUpdateFetch
	// update install
	MenderStateUpdateInstall
	// commit needed
	MenderStateUpdateCommit
	// reboot
	MenderStateReboot
	// error
	MenderStateError
	// exit state
	MenderStateDone
)

type mender struct {
	UInstallCommitRebooter
	Updater
	env            BootEnvReadWriter
	state          State
	config         menderConfig
	manifestFile   string
	deviceKey      *Keystore
	forceBootstrap bool
}

type MenderPieces struct {
	updater Updater
	device  UInstallCommitRebooter
	env     BootEnvReadWriter
	store   Store
}

func NewMender(config menderConfig, pieces MenderPieces) *mender {

	ks := NewKeystore(pieces.store)
	if ks == nil {
		return nil
	}

	m := &mender{
		UInstallCommitRebooter: pieces.device,
		Updater:                pieces.updater,
		env:                    pieces.env,
		manifestFile:           defaultManifestFile,
		deviceKey:              ks,
		state:                  initState,
		config:                 config,
	}

	if err := m.deviceKey.Load(m.config.DeviceKey); err != nil && IsNoKeys(err) == false {
		log.Errorf("failed to load device keys: %s", err)
		return nil
	}

	return m
}

func (m mender) GetCurrentImageID() string {
	// This is where Yocto stores buid information
	manifest, err := os.Open(m.manifestFile)
	if err != nil {
		log.Error("Can not read current image id.")
		return ""
	}

	imageID := ""

	scanner := bufio.NewScanner(manifest)
	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())
		log.Debug("Read data from device manifest file: ", line)
		if strings.HasPrefix(line, "IMAGE_ID") {
			log.Debug("Found device id line: ", line)
			lineID := strings.Split(line, "=")
			if len(lineID) != 2 {
				log.Errorf("Broken device manifest file: (%v)", lineID)
				return ""
			}
			log.Debug("Current image id: ", strings.TrimSpace(lineID[1]))
			return strings.TrimSpace(lineID[1])
		}
	}
	if err := scanner.Err(); err != nil {
		log.Error(err)
	}
	return imageID
}

func (m *mender) HasUpgrade() (bool, menderError) {
	env, err := m.env.ReadEnv("upgrade_available")
	if err != nil {
		return false, NewFatalError(err)
	}
	upgradeAvailable := env["upgrade_available"]

	// we are after update
	if upgradeAvailable == "1" {
		return true, nil
	}
	return false, nil
}

func (m *mender) ForceBootstrap() {
	m.forceBootstrap = true
}

func (m *mender) needsBootstrap() bool {
	if m.forceBootstrap {
		return true
	}

	if m.deviceKey.Private() == nil {
		log.Debugf("needs keys")
		return true
	}

	return false
}

func (m *mender) Bootstrap() menderError {
	if !m.needsBootstrap() {
		return nil
	}

	return m.doBootstrap()
}

func (m *mender) doBootstrap() menderError {
	if m.deviceKey.Private() == nil {
		log.Infof("device keys not present, generating")
		if err := m.deviceKey.Generate(); err != nil {
			return NewFatalError(err)
		}

		if err := m.deviceKey.Save(m.config.DeviceKey); err != nil {
			log.Errorf("failed to save keys to %s: %s",
				m.config.DeviceKey, err)
			return NewFatalError(err)
		}
	}

	m.forceBootstrap = false

	return nil
}

// Check if new update is available. In case of errors, returns nil and error
// that occurred. If no update is available *UpdateResponse is nil, otherwise it
// contains update information.
func (m *mender) CheckUpdate() (*UpdateResponse, menderError) {
	currentImageID := m.GetCurrentImageID()
	//TODO: if currentImageID == "" {
	// 	return errors.New("")
	// }

	haveUpdate, err := m.Updater.GetScheduledUpdate(m.config.ServerURL, m.config.DeviceID)
	if err != nil {
		log.Error("Error receiving scheduled update data: ", err)
		return nil, NewTransientError(err)
	}

	log.Debug("Received correct response for update request.")

	update, ok := haveUpdate.(UpdateResponse)
	if !ok {
		return nil, NewTransientError(errors.Errorf("not an update response?"))
	}

	if update.Image.YoctoID == currentImageID {
		return nil, nil
	}
	return &update, nil
}

func (m mender) GetUpdatePollInterval() time.Duration {
	return time.Duration(m.config.PollIntervalSeconds) * time.Second
}

func (m *mender) SetState(s State) {
	log.Infof("Mender state: %v -> %v", m.state.Id(), s.Id())
	m.state = s
}

func (m *mender) GetState() State {
	return m.state
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Parse nodes.

package parse

import (
	"fmt"
	"strconv"
)

var textFormat = "%s" // Changed to "%q" in tests for better error messages.

// A Node is an element in the parse tree. The interface is trivial.
// The interface contains an unexported method so that only
// types local to this package can satisfy it.
type Node interface {
	Type() NodeType
	String() string
	StringAST() string
	Position() Pos // byte position of start of node in full original input string
	Check() error  // performs type checking for itself and sub-nodes
	Return() FuncType
	// Make sure only functions in this package can create Nodes.
	unexported()
}

// NodeType identifies the type of a parse tree node.
type NodeType int

// Pos represents a byte position in the original input text from which
// this template was parsed.
type Pos int

func (p Pos) Position() Pos {
	return p
}

// unexported keeps Node implementations local to the package.
// All implementations embed Pos, so this takes care of it.
func (Pos) unexported() {
}

// Type returns itself and provides an easy default implementation
// for embedding in a Node. Embedded in all non-trivial Nodes.
func (t NodeType) Type() NodeType {
	return t
}

const (
	NodeFunc   NodeType = iota // A function call.
	NodeBinary                 // Binary operator: math, logical, compare
	NodeUnary                  // Unary operator: !, -
	NodeString                 // A string constant.
	NodeNumber                 // A numerical constant.
)

// Nodes.

// FuncNode holds a function invocation.
type FuncNode struct {
	NodeType
	Pos
	Name string
	F    Func
	Args []Node
}

func newFunc(pos Pos, name string, f Func) *FuncNode {
	return &FuncNode{NodeType: NodeFunc, Pos: pos, Name: name, F: f}
}

func (c *FuncNode) append(arg Node) {
	c.Args = append(c.Args, arg)
}

func (c *FuncNode) String() string {
	s := c.Name + "("
	for i, arg := range c.Args {
		if i > 0 {
			s += ", "
		}
		s += arg.String()
	}
	s += ")"
	return s
}

func (c *FuncNode) StringAST() string {
	s := c.Name + "("
	for i, arg := range c.Args {
		if i > 0 {
			s += ", "
		}
		s += arg.StringAST()
	}
	s += ")"
	return s
}

func (c *FuncNode) Check() error {
	const errFuncType = "parse: bad argument type in %s, expected %s, got %s"
	if len(c.Args) < len(c.F.Args) {
		return fmt.Errorf("parse: not enough arguments for %s", c.Name)
	} else if len(c.Args) > len(c.F.Args) {
		return fmt.Errorf("parse: too many arguments for %s", c.Name)
	}
	for i, a := range c.Args {
		t := c.F.Args[i]
		switch a.(type) {
		case *NumberNode:
			if t != TYPE_NUMBER {
				return fmt.Errorf(errFuncType, c.Name, t, "number")
			}
		case *StringNode:
			if t != TYPE_STRING {
				return fmt.Errorf(errFuncType, c.Name, t, "string")
			}
		}
	}
	return nil
}

func (f *FuncNode) Return() FuncType { return f.F.Return }

// NumberNode holds a number: signed or unsigned integer or float.
// The value is parsed and stored under all the types that can represent the value.
// This simulates in a small amount of code the behavior of Go's ideal constants.
type NumberNode struct {
	NodeType
	Pos
	IsUint  bool    // Number has an unsigned integral value.
	IsFloat bool    // Number has a floating-point value.
	Uint64  uint64  // The unsigned integer value.
	Float64 float64 // The floating-point value.
	Text    string  // The original textual representation from the input.
}

func newNumber(pos Pos, text string) (*NumberNode, error) {
	n := &NumberNode{NodeType: NodeNumber, Pos: pos, Text: text}
	// Do integer test first so we get 0x123 etc.
	u, err := strconv.ParseUint(text, 0, 64) // will fail for -0.
	if err == nil {
		n.IsUint = true
		n.Uint64 = u
	}
	// If an integer extraction succeeded, promote the float.
	if n.IsUint {
		n.IsFloat = true
		n.Float64 = float64(n.Uint64)
	} else {
		f, err := strconv.ParseFloat(text, 64)
		if err == nil {
			n.IsFloat = true
			n.Float64 = f
			// If a floating-point extraction succeeded, extract the int if needed.
			if !n.IsUint && float64(uint64(f)) == f {
				n.IsUint = true
				n.Uint64 = uint64(f)
			}
		}
	}
	if !n.IsUint && !n.IsFloat {
		return nil, fmt.Errorf("illegal number syntax: %q", text)
	}
	return n, nil
}

func (n *NumberNode) String() string {
	return n.Text
}

func (n *NumberNode) StringAST() string {
	return n.String()
}

func (n *NumberNode) Check() error {
	return nil
}

func (n *NumberNode) Return() FuncType { return TYPE_NUMBER }

// StringNode holds a string constant. The value has been "unquoted".
type StringNode struct {
	NodeType
	Pos
	Quoted string // The original text of the string, with quotes.
	Text   string // The string, after quote processing.
}

func newString(pos Pos, orig, text string) *StringNode {
	return &StringNode{NodeType: NodeString, Pos: pos, Quoted: orig, Text: text}
}

func (s *StringNode) String() string {
	return s.Quoted
}

func (s *StringNode) StringAST() string {
	return s.String()
}

func (s *StringNode) Check() error {
	return nil
}

func (s *StringNode) Return() FuncType { return TYPE_STRING }

// BinaryNode holds two arguments and an operator.
type BinaryNode struct {
	NodeType
	Pos
	Args     [2]Node
	Operator item
	OpStr    string
}

func newBinary(operator item, arg1, arg2 Node) *BinaryNode {
	return &BinaryNode{NodeType: NodeBinary, Pos: operator.pos, Args: [2]Node{arg1, arg2}, Operator: operator, OpStr: operator.val}
}

func (b *BinaryNode) String() string {
	return fmt.Sprintf("%s %s %s", b.Args[0], b.Operator.val, b.Args[1])
}

func (b *BinaryNode) StringAST() string {
	return fmt.Sprintf("%s(%s, %s)", b.Operator.val, b.Args[0], b.Args[1])
}

func (b *BinaryNode) Check() error {
	t1 := b.Args[0].Return()
	t2 := b.Args[1].Return()
	if t1 == TYPE_SERIES && t2 == TYPE_SERIES {
		return fmt.Errorf("parse: type error in %s: at least one side must be a number", b)
	}
	if t1 != TYPE_NUMBER && t1 != TYPE_SERIES {
		return fmt.Errorf("parse: type error in %s", b)
	}
	if t2 != TYPE_NUMBER && t2 != TYPE_SERIES {
		return fmt.Errorf("parse: type error in %s", b)
	}
	if err := b.Args[0].Check(); err != nil {
		return err
	}
	return b.Args[1].Check()
}

func (b *BinaryNode) Return() FuncType { return TYPE_NUMBER }

// UnaryNode holds one argument and an operator.
type UnaryNode struct {
	NodeType
	Pos
	Arg      Node
	Operator item
	OpStr    string
}

func newUnary(operator item, arg Node) *UnaryNode {
	return &UnaryNode{NodeType: NodeUnary, Pos: operator.pos, Arg: arg, Operator: operator, OpStr: operator.val}
}

func (u *UnaryNode) String() string {
	return fmt.Sprintf("%s%s", u.Operator.val, u.Arg)
}

func (u *UnaryNode) StringAST() string {
	return fmt.Sprintf("%s(%s)", u.Operator.val, u.Arg)
}

func (u *UnaryNode) Check() error {
	if t := u.Arg.Return(); t != TYPE_NUMBER && t != TYPE_SERIES {
		return fmt.Errorf("parse: type error in %s, expected %s, got %s", u, "number", t)
	}
	return u.Arg.Check()
}

func (u *UnaryNode) Return() FuncType { return TYPE_NUMBER }
<commit_msg>Check embedded function type and arguments<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Parse nodes.

package parse

import (
	"fmt"
	"strconv"
)

var textFormat = "%s" // Changed to "%q" in tests for better error messages.

// A Node is an element in the parse tree. The interface is trivial.
// The interface contains an unexported method so that only
// types local to this package can satisfy it.
type Node interface {
	Type() NodeType
	String() string
	StringAST() string
	Position() Pos // byte position of start of node in full original input string
	Check() error  // performs type checking for itself and sub-nodes
	Return() FuncType
	// Make sure only functions in this package can create Nodes.
	unexported()
}

// NodeType identifies the type of a parse tree node.
type NodeType int

// Pos represents a byte position in the original input text from which
// this template was parsed.
type Pos int

func (p Pos) Position() Pos {
	return p
}

// unexported keeps Node implementations local to the package.
// All implementations embed Pos, so this takes care of it.
func (Pos) unexported() {
}

// Type returns itself and provides an easy default implementation
// for embedding in a Node. Embedded in all non-trivial Nodes.
func (t NodeType) Type() NodeType {
	return t
}

const (
	NodeFunc   NodeType = iota // A function call.
	NodeBinary                 // Binary operator: math, logical, compare
	NodeUnary                  // Unary operator: !, -
	NodeString                 // A string constant.
	NodeNumber                 // A numerical constant.
)

// Nodes.

// FuncNode holds a function invocation.
type FuncNode struct {
	NodeType
	Pos
	Name string
	F    Func
	Args []Node
}

func newFunc(pos Pos, name string, f Func) *FuncNode {
	return &FuncNode{NodeType: NodeFunc, Pos: pos, Name: name, F: f}
}

func (c *FuncNode) append(arg Node) {
	c.Args = append(c.Args, arg)
}

func (c *FuncNode) String() string {
	s := c.Name + "("
	for i, arg := range c.Args {
		if i > 0 {
			s += ", "
		}
		s += arg.String()
	}
	s += ")"
	return s
}

func (c *FuncNode) StringAST() string {
	s := c.Name + "("
	for i, arg := range c.Args {
		if i > 0 {
			s += ", "
		}
		s += arg.StringAST()
	}
	s += ")"
	return s
}

func (c *FuncNode) Check() error {
	const errFuncType = "parse: bad argument type in %s, expected %s, got %s"
	if len(c.Args) < len(c.F.Args) {
		return fmt.Errorf("parse: not enough arguments for %s", c.Name)
	} else if len(c.Args) > len(c.F.Args) {
		return fmt.Errorf("parse: too many arguments for %s", c.Name)
	}
	for i, a := range c.Args {
		t := c.F.Args[i]
		switch v := a.(type) {
		case *NumberNode:
			if t != TYPE_NUMBER {
				return fmt.Errorf(errFuncType, c.Name, t, "number")
			}
		case *StringNode:
			if t != TYPE_STRING {
				return fmt.Errorf(errFuncType, c.Name, t, "string")
			}
		case *FuncNode:
			if t != v.Return() {
				return fmt.Errorf(errFuncType, c.Name, t, v.Return())
			}
			if err := v.Check(); err != nil {
				return err
			}
		default:
			panic("unchecked")
		}
	}
	return nil
}

func (f *FuncNode) Return() FuncType { return f.F.Return }

// NumberNode holds a number: signed or unsigned integer or float.
// The value is parsed and stored under all the types that can represent the value.
// This simulates in a small amount of code the behavior of Go's ideal constants.
type NumberNode struct {
	NodeType
	Pos
	IsUint  bool    // Number has an unsigned integral value.
	IsFloat bool    // Number has a floating-point value.
	Uint64  uint64  // The unsigned integer value.
	Float64 float64 // The floating-point value.
	Text    string  // The original textual representation from the input.
}

func newNumber(pos Pos, text string) (*NumberNode, error) {
	n := &NumberNode{NodeType: NodeNumber, Pos: pos, Text: text}
	// Do integer test first so we get 0x123 etc.
	u, err := strconv.ParseUint(text, 0, 64) // will fail for -0.
	if err == nil {
		n.IsUint = true
		n.Uint64 = u
	}
	// If an integer extraction succeeded, promote the float.
	if n.IsUint {
		n.IsFloat = true
		n.Float64 = float64(n.Uint64)
	} else {
		f, err := strconv.ParseFloat(text, 64)
		if err == nil {
			n.IsFloat = true
			n.Float64 = f
			// If a floating-point extraction succeeded, extract the int if needed.
			if !n.IsUint && float64(uint64(f)) == f {
				n.IsUint = true
				n.Uint64 = uint64(f)
			}
		}
	}
	if !n.IsUint && !n.IsFloat {
		return nil, fmt.Errorf("illegal number syntax: %q", text)
	}
	return n, nil
}

func (n *NumberNode) String() string {
	return n.Text
}

func (n *NumberNode) StringAST() string {
	return n.String()
}

func (n *NumberNode) Check() error {
	return nil
}

func (n *NumberNode) Return() FuncType { return TYPE_NUMBER }

// StringNode holds a string constant. The value has been "unquoted".
type StringNode struct {
	NodeType
	Pos
	Quoted string // The original text of the string, with quotes.
	Text   string // The string, after quote processing.
}

func newString(pos Pos, orig, text string) *StringNode {
	return &StringNode{NodeType: NodeString, Pos: pos, Quoted: orig, Text: text}
}

func (s *StringNode) String() string {
	return s.Quoted
}

func (s *StringNode) StringAST() string {
	return s.String()
}

func (s *StringNode) Check() error {
	return nil
}

func (s *StringNode) Return() FuncType { return TYPE_STRING }

// BinaryNode holds two arguments and an operator.
type BinaryNode struct {
	NodeType
	Pos
	Args     [2]Node
	Operator item
	OpStr    string
}

func newBinary(operator item, arg1, arg2 Node) *BinaryNode {
	return &BinaryNode{NodeType: NodeBinary, Pos: operator.pos, Args: [2]Node{arg1, arg2}, Operator: operator, OpStr: operator.val}
}

func (b *BinaryNode) String() string {
	return fmt.Sprintf("%s %s %s", b.Args[0], b.Operator.val, b.Args[1])
}

func (b *BinaryNode) StringAST() string {
	return fmt.Sprintf("%s(%s, %s)", b.Operator.val, b.Args[0], b.Args[1])
}

func (b *BinaryNode) Check() error {
	t1 := b.Args[0].Return()
	t2 := b.Args[1].Return()
	if t1 == TYPE_SERIES && t2 == TYPE_SERIES {
		return fmt.Errorf("parse: type error in %s: at least one side must be a number", b)
	}
	if t1 != TYPE_NUMBER && t1 != TYPE_SERIES {
		return fmt.Errorf("parse: type error in %s", b)
	}
	if t2 != TYPE_NUMBER && t2 != TYPE_SERIES {
		return fmt.Errorf("parse: type error in %s", b)
	}
	if err := b.Args[0].Check(); err != nil {
		return err
	}
	return b.Args[1].Check()
}

func (b *BinaryNode) Return() FuncType { return TYPE_NUMBER }

// UnaryNode holds one argument and an operator.
type UnaryNode struct {
	NodeType
	Pos
	Arg      Node
	Operator item
	OpStr    string
}

func newUnary(operator item, arg Node) *UnaryNode {
	return &UnaryNode{NodeType: NodeUnary, Pos: operator.pos, Arg: arg, Operator: operator, OpStr: operator.val}
}

func (u *UnaryNode) String() string {
	return fmt.Sprintf("%s%s", u.Operator.val, u.Arg)
}

func (u *UnaryNode) StringAST() string {
	return fmt.Sprintf("%s(%s)", u.Operator.val, u.Arg)
}

func (u *UnaryNode) Check() error {
	if t := u.Arg.Return(); t != TYPE_NUMBER && t != TYPE_SERIES {
		return fmt.Errorf("parse: type error in %s, expected %s, got %s", u, "number", t)
	}
	return u.Arg.Check()
}

func (u *UnaryNode) Return() FuncType { return TYPE_NUMBER }
<|endoftext|>


<commit_before>package influx

import (
	"context"
	"encoding/json"
	"fmt"
  "bytes"

	"github.com/influxdata/chronograf"
)

func (c *Client) AllDB(ctx context.Context) ([]chronograf.Database, error) {
	databases, err := c.showDatabases(ctx)
	if err != nil {
		return nil, err
	}

	return databases, nil
}

func (c *Client) CreateDB(ctx context.Context, db *chronograf.Database) (*chronograf.Database, error) {
	_, err := c.Query(ctx, chronograf.Query{
		Command: fmt.Sprintf(`CREATE DATABASE "%s"`, db.Name),
	})
	if err != nil {
		return nil, err
	}

	res := &chronograf.Database{Name: db.Name}

	return res, nil
}

func (c *Client) DropDB(ctx context.Context, database string) error {
	_, err := c.Query(ctx, chronograf.Query{
		Command: fmt.Sprintf(`DROP DATABASE`),
		DB:      database,
	})
	if err != nil {
		return err
	}
	return nil
}

func (c *Client) AllRP(ctx context.Context, database string) ([]chronograf.RetentionPolicy, error) {
	retentionPolicies, err := c.showRetentionPolicies(ctx, database)
	if err != nil {
		return nil, err
	}

	return retentionPolicies, nil
}

func (c *Client) CreateRP(ctx context.Context, database string, rp *chronograf.RetentionPolicy) (*chronograf.RetentionPolicy, error) {
	_, err := c.Query(ctx, chronograf.Query{
		Command: fmt.Sprintf(`CREATE RETENTION POLICY "%s" DURATION "%s" REPLICATION "%s"`, rp.Name, rp.Duration, rp.Replication),
		DB:      database,
	})
	if err != nil {
		return nil, err
	}

	res := &chronograf.RetentionPolicy{
		Name:        rp.Name,
		Duration:    rp.Duration,
		Replication: rp.Replication,
	}

	return res, nil
}

func (c *Client) UpdateRP(ctx context.Context, database string, name string, rp *chronograf.RetentionPolicy) (*chronograf.RetentionPolicy, error) {
  var buffer bytes.Buffer
  buffer.WriteString("ALTER RETENTION POLICY")
  if (len(rp.Duration) > 0) {
    buffer.WriteString("DURATION " + rp.Duration)
  }

  _, err := c.Query(ctx, chronograf.Query{
    Command: buffer.String(),
    DB:      database,
    RP:      name,
  })
  if err != nil {
    return nil, err
  }

  // TODO: use actual information here
  res := &chronograf.RetentionPolicy{
    Name:        name,
  }

  return res, nil
}

func (c *Client) DropRP(ctx context.Context, database string, rp string) error {
	_, err := c.Query(ctx, chronograf.Query{
		Command: fmt.Sprintf(`DROP RETENTION POLICY`),
		DB:      database,
		RP:      rp,
	})
	if err != nil {
		return err
	}
	return nil
}

func (c *Client) showDatabases(ctx context.Context) ([]chronograf.Database, error) {
	res, err := c.Query(ctx, chronograf.Query{
		Command: `SHOW DATABASES`,
	})
	if err != nil {
		return nil, err
	}
	octets, err := res.MarshalJSON()
	if err != nil {
		return nil, err
	}

	results := showResults{}
	if err := json.Unmarshal(octets, &results); err != nil {
		return nil, err
	}

	return results.Databases(), nil
}

func (c *Client) showRetentionPolicies(ctx context.Context, name string) ([]chronograf.RetentionPolicy, error) {
	retentionPolicies, err := c.Query(ctx, chronograf.Query{
		Command: fmt.Sprintf(`SHOW RETENTION POLICIES`),
		DB:      name,
	})

	if err != nil {
		return nil, err
	}
	octets, err := retentionPolicies.MarshalJSON()
	if err != nil {
		return nil, err
	}

	results := showResults{}
	if err := json.Unmarshal(octets, &results); err != nil {
		return nil, err
	}

	return results.RetentionPolicies(), nil
}
<commit_msg>add options to alter rp<commit_after>package influx

import (
	"context"
	"encoding/json"
	"fmt"
  "bytes"

	"github.com/influxdata/chronograf"
)

func (c *Client) AllDB(ctx context.Context) ([]chronograf.Database, error) {
	databases, err := c.showDatabases(ctx)
	if err != nil {
		return nil, err
	}

	return databases, nil
}

func (c *Client) CreateDB(ctx context.Context, db *chronograf.Database) (*chronograf.Database, error) {
	_, err := c.Query(ctx, chronograf.Query{
		Command: fmt.Sprintf(`CREATE DATABASE "%s"`, db.Name),
	})
	if err != nil {
		return nil, err
	}

	res := &chronograf.Database{Name: db.Name}

	return res, nil
}

func (c *Client) DropDB(ctx context.Context, database string) error {
	_, err := c.Query(ctx, chronograf.Query{
		Command: fmt.Sprintf(`DROP DATABASE`),
		DB:      database,
	})
	if err != nil {
		return err
	}
	return nil
}

func (c *Client) AllRP(ctx context.Context, database string) ([]chronograf.RetentionPolicy, error) {
	retentionPolicies, err := c.showRetentionPolicies(ctx, database)
	if err != nil {
		return nil, err
	}

	return retentionPolicies, nil
}

func (c *Client) CreateRP(ctx context.Context, database string, rp *chronograf.RetentionPolicy) (*chronograf.RetentionPolicy, error) {
	_, err := c.Query(ctx, chronograf.Query{
		Command: fmt.Sprintf(`CREATE RETENTION POLICY "%s" DURATION "%s" REPLICATION "%s"`, rp.Name, rp.Duration, rp.Replication),
		DB:      database,
	})
	if err != nil {
		return nil, err
	}

	res := &chronograf.RetentionPolicy{
		Name:        rp.Name,
		Duration:    rp.Duration,
		Replication: rp.Replication,
	}

	return res, nil
}

func (c *Client) UpdateRP(ctx context.Context, database string, name string, rp *chronograf.RetentionPolicy) (*chronograf.RetentionPolicy, error) {
  var buffer bytes.Buffer
  buffer.WriteString("ALTER RETENTION POLICY")
  if (len(rp.Duration) > 0) {
    buffer.WriteString(" DURATION " + rp.Duration)
  }
  if (rp.Replication > 0) {
    buffer.WriteString(" REPLICATION " + fmt.Sprint(rp.Replication))
  }
  if (len(rp.ShardDuration) > 0) {
    buffer.WriteString(" SHARD DURATION " + rp.ShardDuration)
  }
  if (rp.Default == true) {
    buffer.WriteString(" DEFAULT")
  }

  _, err := c.Query(ctx, chronograf.Query{
    Command: buffer.String(),
    DB:      database,
    RP:      name,
  })
  if err != nil {
    return nil, err
  }

  // TODO: use actual information here
  res := &chronograf.RetentionPolicy{
    Name:        name,
  }

  return res, nil
}

func (c *Client) DropRP(ctx context.Context, database string, rp string) error {
	_, err := c.Query(ctx, chronograf.Query{
		Command: fmt.Sprintf(`DROP RETENTION POLICY`),
		DB:      database,
		RP:      rp,
	})
	if err != nil {
		return err
	}
	return nil
}

func (c *Client) showDatabases(ctx context.Context) ([]chronograf.Database, error) {
	res, err := c.Query(ctx, chronograf.Query{
		Command: `SHOW DATABASES`,
	})
	if err != nil {
		return nil, err
	}
	octets, err := res.MarshalJSON()
	if err != nil {
		return nil, err
	}

	results := showResults{}
	if err := json.Unmarshal(octets, &results); err != nil {
		return nil, err
	}

	return results.Databases(), nil
}

func (c *Client) showRetentionPolicies(ctx context.Context, name string) ([]chronograf.RetentionPolicy, error) {
	retentionPolicies, err := c.Query(ctx, chronograf.Query{
		Command: fmt.Sprintf(`SHOW RETENTION POLICIES`),
		DB:      name,
	})

	if err != nil {
		return nil, err
	}
	octets, err := retentionPolicies.MarshalJSON()
	if err != nil {
		return nil, err
	}

	results := showResults{}
	if err := json.Unmarshal(octets, &results); err != nil {
		return nil, err
	}

	return results.RetentionPolicies(), nil
}
<|endoftext|>


<commit_before>// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pilosa

import (
	"io/ioutil"
	"os"
	"path/filepath"

	"errors"
	"fmt"
	"github.com/gogo/protobuf/proto"
	"github.com/pilosa/pilosa/internal"
)

const (
	Mapping       = "mapping"
	ValueToRow    = "value-to-row"
	SingleRowBool = "single-row-boolean"
)

var ValidValueDestination = []string{Mapping, ValueToRow, SingleRowBool}

// InputDefinition represents a container for the data input definition.
type InputDefinition struct {
	name        string
	path        string
	index       string
	broadcaster Broadcaster
	frames      []InputFrame
	fields      []Field
}

// NewInputDefinition returns a new instance of InputDefinition.
func NewInputDefinition(path, index, name string) (*InputDefinition, error) {
	err := ValidateName(name)
	if err != nil {
		return nil, err
	}

	return &InputDefinition{
		path:  path,
		index: index,
		name:  name,
	}, nil
}

// Frames returns frames of the input definition was initialized with.
func (i *InputDefinition) Frames() []InputFrame { return i.frames }

// Fields returns fields of the input definition was initialized with.
func (i *InputDefinition) Fields() []Field { return i.fields }

// Open opens and initializes the InputDefinition from file.
func (i *InputDefinition) Open() error {
	if err := func() error {
		if err := os.MkdirAll(i.path, 0777); err != nil {
			return err
		}

		if err := i.loadMeta(); err != nil {
			return err
		}
		return nil
	}(); err != nil {
		return err
	}
	return nil
}

// LoadDefinition loads the protobuf format of a defition
func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error {
	// Copy metadata fields.
	i.name = pb.Name
	for _, fr := range pb.Frames {
		frameMeta := fr.Meta
		inputFrame := InputFrame{
			Name: fr.Name,
			Options: FrameOptions{
				RowLabel:       frameMeta.RowLabel,
				InverseEnabled: frameMeta.InverseEnabled,
				CacheSize:      frameMeta.CacheSize,
				CacheType:      frameMeta.CacheType,
				TimeQuantum:    TimeQuantum(frameMeta.TimeQuantum),
			},
		}
		i.frames = append(i.frames, inputFrame)
	}

	numPrimaryKey := 0
	countRowID := make(map[string]uint64)
	for _, field := range pb.Fields {
		var actions []Action
		for _, action := range field.Actions {
			if err := i.ValidateAction(action); err != nil {
				return err
			}
			if action.RowID != 0 && action.Frame != "" {
				val, ok := countRowID[action.Frame]
				if ok && val == action.RowID {
					return fmt.Errorf("duplicate rowID with other field: %v", action.RowID)
				} else {
					countRowID[action.Frame] = action.RowID
				}
			}
			actions = append(actions, Action{
				Frame:            action.Frame,
				ValueDestination: action.ValueDestination,
				ValueMap:         action.ValueMap,
				RowID:            &action.RowID,
			})
		}
		if field.PrimaryKey {
			numPrimaryKey += 1
		}

		if numPrimaryKey > 1 {
			return errors.New("duplicate primaryKey with other field")
		}

		inputField := Field{
			Name:       field.Name,
			PrimaryKey: field.PrimaryKey,
			Actions:    actions,
		}
		i.fields = append(i.fields, inputField)
	}

	return nil
}

func (i *InputDefinition) loadMeta() error {
	var pb internal.InputDefinition
	buf, err := ioutil.ReadFile(filepath.Join(i.path, i.name))
	if err != nil {
		return err
	}
	if err := proto.Unmarshal(buf, &pb); err != nil {
		return err
	}

	return i.LoadDefinition(&pb)
}

//saveMeta writes meta data for the input definition file.
func (i *InputDefinition) saveMeta() error {
	if err := os.MkdirAll(i.path, 0777); err != nil {
		return err
	}
	// Marshal metadata.
	var frames []*internal.Frame
	for _, fr := range i.frames {
		frameMeta := &internal.FrameMeta{
			RowLabel:       fr.Options.RowLabel,
			InverseEnabled: fr.Options.InverseEnabled,
			CacheType:      fr.Options.CacheType,
			CacheSize:      fr.Options.CacheSize,
			TimeQuantum:    string(fr.Options.TimeQuantum),
		}
		frame := &internal.Frame{Name: fr.Name, Meta: frameMeta}
		frames = append(frames, frame)
	}

	var fields []*internal.InputDefinitionField
	for _, field := range i.fields {
		var actions []*internal.Action
		for _, action := range field.Actions {
			actionMeta := &internal.Action{
				Frame:            action.Frame,
				ValueDestination: action.ValueDestination,
				ValueMap:         action.ValueMap,
				RowID:            convert(action.RowID),
			}
			actions = append(actions, actionMeta)
		}

		fieldMeta := &internal.InputDefinitionField{
			Name:       field.Name,
			PrimaryKey: field.PrimaryKey,
			Actions:    actions,
		}
		fields = append(fields, fieldMeta)
	}
	buf, err := proto.Marshal(&internal.InputDefinition{
		Name:   i.name,
		Frames: frames,
		Fields: fields,
	})
	if err != nil {
		return err
	}

	// Write to meta file.
	if err := ioutil.WriteFile(filepath.Join(i.path, i.name), buf, 0666); err != nil {
		return err
	}

	return nil
}

// Field descripes a single field mapping in the InputDefinition.
type Field struct {
	Name       string   `json:"name,omitempty"`
	PrimaryKey bool     `json:"primaryKey,omitempty"`
	Actions    []Action `json:"actions,omitempty"`
}

// Encode converts Field into its internal representation.
func (o *Field) Encode() (*internal.InputDefinitionField, error) {
	field := internal.InputDefinitionField{Name: o.Name, PrimaryKey: o.PrimaryKey}

	for _, action := range o.Actions {
		actionEncode, err := action.Encode()
		if err != nil {
			return nil, err
		}
		field.Actions = append(field.Actions, actionEncode)
	}
	return &field, nil
}

// Action descripes the mapping method for the field in the InputDefinition.
type Action struct {
	Frame            string            `json:"frame,omitempty"`
	ValueDestination string            `json:"valueDestination,omitempty"`
	ValueMap         map[string]uint64 `json:"valueMap,omitempty"`
	RowID            *uint64           `json:"rowID,omitempty"`
}

// Encode converts Action into its internal representation.
func (o *Action) Encode() (*internal.Action, error) {
	if o.RowID == nil && o.ValueDestination == "single-row-boolean" {
		return nil, errors.New("rowID required for single-row-boolean")
	}
	return &internal.Action{
		Frame:            o.Frame,
		ValueDestination: o.ValueDestination,
		ValueMap:         o.ValueMap,
		RowID:            convert(o.RowID),
	}, nil
}

func convert(x *uint64) uint64 {
	if x != nil {
		return *x
	}
	return 0
}

// InputFrame defines the frame used in the input definition.
type InputFrame struct {
	Name    string       `json:"name,omitempty"`
	Options FrameOptions `json:"options,omitempty"`
}

// InputDefinitionInfo the json message format to create an InputDefinition.
type InputDefinitionInfo struct {
	Frames []InputFrame `json:"frames"`
	Fields []Field      `json:"fields"`
}

// Encode converts InputDefinitionInfo into its internal representation.
func (i *InputDefinitionInfo) Encode() (*internal.InputDefinition, error) {
	var def internal.InputDefinition
	for _, f := range i.Frames {
		def.Frames = append(def.Frames, &internal.Frame{Name: f.Name, Meta: f.Options.Encode()})
	}
	for _, f := range i.Fields {
		fEncode, err := f.Encode()
		if err != nil {
			return nil, err
		}
		def.Fields = append(def.Fields, fEncode)
	}

	return &def, nil
}

func (i *InputDefinition) AddFrame(frame InputFrame) error {
	i.frames = append(i.frames, frame)
	if err := i.saveMeta(); err != nil {
		return err
	}
	return nil
}

func (i *InputDefinition) ValidateAction(action *internal.Action) error {
	if action.Frame == "" {
		return ErrFrameRequired
	}
	validValues := make(map[string]bool)
	for _, val := range ValidValueDestination {
		validValues[val] = true
	}
	if _, ok := validValues[action.ValueDestination]; !ok {
		return fmt.Errorf("invalid ValueDestination: %s", action.ValueDestination)
	}
	switch action.ValueDestination {
	case Mapping:
		if len(action.ValueMap) == 0 {
			return errors.New("valueMap required for map")
		}
	}
	return nil
}
<commit_msg>check SingleRowBool for duplicate rowID<commit_after>// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pilosa

import (
	"io/ioutil"
	"os"
	"path/filepath"

	"errors"
	"fmt"
	"github.com/gogo/protobuf/proto"
	"github.com/pilosa/pilosa/internal"
)

const (
	Mapping       = "mapping"
	ValueToRow    = "value-to-row"
	SingleRowBool = "single-row-boolean"
)

var ValidValueDestination = []string{Mapping, ValueToRow, SingleRowBool}

// InputDefinition represents a container for the data input definition.
type InputDefinition struct {
	name        string
	path        string
	index       string
	broadcaster Broadcaster
	frames      []InputFrame
	fields      []Field
}

// NewInputDefinition returns a new instance of InputDefinition.
func NewInputDefinition(path, index, name string) (*InputDefinition, error) {
	err := ValidateName(name)
	if err != nil {
		return nil, err
	}

	return &InputDefinition{
		path:  path,
		index: index,
		name:  name,
	}, nil
}

// Frames returns frames of the input definition was initialized with.
func (i *InputDefinition) Frames() []InputFrame { return i.frames }

// Fields returns fields of the input definition was initialized with.
func (i *InputDefinition) Fields() []Field { return i.fields }

// Open opens and initializes the InputDefinition from file.
func (i *InputDefinition) Open() error {
	if err := func() error {
		if err := os.MkdirAll(i.path, 0777); err != nil {
			return err
		}

		if err := i.loadMeta(); err != nil {
			return err
		}
		return nil
	}(); err != nil {
		return err
	}
	return nil
}

// LoadDefinition loads the protobuf format of a defition
func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error {
	// Copy metadata fields.
	i.name = pb.Name
	for _, fr := range pb.Frames {
		frameMeta := fr.Meta
		inputFrame := InputFrame{
			Name: fr.Name,
			Options: FrameOptions{
				RowLabel:       frameMeta.RowLabel,
				InverseEnabled: frameMeta.InverseEnabled,
				CacheSize:      frameMeta.CacheSize,
				CacheType:      frameMeta.CacheType,
				TimeQuantum:    TimeQuantum(frameMeta.TimeQuantum),
			},
		}
		i.frames = append(i.frames, inputFrame)
	}

	numPrimaryKey := 0
	countRowID := make(map[string]uint64)
	for _, field := range pb.Fields {
		var actions []Action
		for _, action := range field.Actions {
			if err := i.ValidateAction(action); err != nil {
				return err
			}
			if action.ValueDestination == SingleRowBool && action.Frame != "" {
				val, ok := countRowID[action.Frame]
				if ok && val == action.RowID {
					return fmt.Errorf("duplicate rowID with other field: %v", action.RowID)
				} else {
					countRowID[action.Frame] = action.RowID
				}
			}
			actions = append(actions, Action{
				Frame:            action.Frame,
				ValueDestination: action.ValueDestination,
				ValueMap:         action.ValueMap,
				RowID:            &action.RowID,
			})
		}
		if field.PrimaryKey {
			numPrimaryKey += 1
		}

		if numPrimaryKey > 1 {
			return errors.New("duplicate primaryKey with other field")
		}

		inputField := Field{
			Name:       field.Name,
			PrimaryKey: field.PrimaryKey,
			Actions:    actions,
		}
		i.fields = append(i.fields, inputField)
	}

	return nil
}

func (i *InputDefinition) loadMeta() error {
	var pb internal.InputDefinition
	buf, err := ioutil.ReadFile(filepath.Join(i.path, i.name))
	if err != nil {
		return err
	}
	if err := proto.Unmarshal(buf, &pb); err != nil {
		return err
	}

	return i.LoadDefinition(&pb)
}

//saveMeta writes meta data for the input definition file.
func (i *InputDefinition) saveMeta() error {
	if err := os.MkdirAll(i.path, 0777); err != nil {
		return err
	}
	// Marshal metadata.
	var frames []*internal.Frame
	for _, fr := range i.frames {
		frameMeta := &internal.FrameMeta{
			RowLabel:       fr.Options.RowLabel,
			InverseEnabled: fr.Options.InverseEnabled,
			CacheType:      fr.Options.CacheType,
			CacheSize:      fr.Options.CacheSize,
			TimeQuantum:    string(fr.Options.TimeQuantum),
		}
		frame := &internal.Frame{Name: fr.Name, Meta: frameMeta}
		frames = append(frames, frame)
	}

	var fields []*internal.InputDefinitionField
	for _, field := range i.fields {
		var actions []*internal.Action
		for _, action := range field.Actions {
			actionMeta := &internal.Action{
				Frame:            action.Frame,
				ValueDestination: action.ValueDestination,
				ValueMap:         action.ValueMap,
				RowID:            convert(action.RowID),
			}
			actions = append(actions, actionMeta)
		}

		fieldMeta := &internal.InputDefinitionField{
			Name:       field.Name,
			PrimaryKey: field.PrimaryKey,
			Actions:    actions,
		}
		fields = append(fields, fieldMeta)
	}
	buf, err := proto.Marshal(&internal.InputDefinition{
		Name:   i.name,
		Frames: frames,
		Fields: fields,
	})
	if err != nil {
		return err
	}

	// Write to meta file.
	if err := ioutil.WriteFile(filepath.Join(i.path, i.name), buf, 0666); err != nil {
		return err
	}

	return nil
}

// Field descripes a single field mapping in the InputDefinition.
type Field struct {
	Name       string   `json:"name,omitempty"`
	PrimaryKey bool     `json:"primaryKey,omitempty"`
	Actions    []Action `json:"actions,omitempty"`
}

// Encode converts Field into its internal representation.
func (o *Field) Encode() (*internal.InputDefinitionField, error) {
	field := internal.InputDefinitionField{Name: o.Name, PrimaryKey: o.PrimaryKey}

	for _, action := range o.Actions {
		actionEncode, err := action.Encode()
		if err != nil {
			return nil, err
		}
		field.Actions = append(field.Actions, actionEncode)
	}
	return &field, nil
}

// Action descripes the mapping method for the field in the InputDefinition.
type Action struct {
	Frame            string            `json:"frame,omitempty"`
	ValueDestination string            `json:"valueDestination,omitempty"`
	ValueMap         map[string]uint64 `json:"valueMap,omitempty"`
	RowID            *uint64           `json:"rowID,omitempty"`
}

// Encode converts Action into its internal representation.
func (o *Action) Encode() (*internal.Action, error) {
	if o.RowID == nil && o.ValueDestination == "single-row-boolean" {
		return nil, errors.New("rowID required for single-row-boolean")
	}
	return &internal.Action{
		Frame:            o.Frame,
		ValueDestination: o.ValueDestination,
		ValueMap:         o.ValueMap,
		RowID:            convert(o.RowID),
	}, nil
}

func convert(x *uint64) uint64 {
	if x != nil {
		return *x
	}
	return 0
}

// InputFrame defines the frame used in the input definition.
type InputFrame struct {
	Name    string       `json:"name,omitempty"`
	Options FrameOptions `json:"options,omitempty"`
}

// InputDefinitionInfo the json message format to create an InputDefinition.
type InputDefinitionInfo struct {
	Frames []InputFrame `json:"frames"`
	Fields []Field      `json:"fields"`
}

// Encode converts InputDefinitionInfo into its internal representation.
func (i *InputDefinitionInfo) Encode() (*internal.InputDefinition, error) {
	var def internal.InputDefinition
	for _, f := range i.Frames {
		def.Frames = append(def.Frames, &internal.Frame{Name: f.Name, Meta: f.Options.Encode()})
	}
	for _, f := range i.Fields {
		fEncode, err := f.Encode()
		if err != nil {
			return nil, err
		}
		def.Fields = append(def.Fields, fEncode)
	}

	return &def, nil
}

func (i *InputDefinition) AddFrame(frame InputFrame) error {
	i.frames = append(i.frames, frame)
	if err := i.saveMeta(); err != nil {
		return err
	}
	return nil
}

func (i *InputDefinition) ValidateAction(action *internal.Action) error {
	if action.Frame == "" {
		return ErrFrameRequired
	}
	validValues := make(map[string]bool)
	for _, val := range ValidValueDestination {
		validValues[val] = true
	}
	if _, ok := validValues[action.ValueDestination]; !ok {
		return fmt.Errorf("invalid ValueDestination: %s", action.ValueDestination)
	}
	switch action.ValueDestination {
	case Mapping:
		if len(action.ValueMap) == 0 {
			return errors.New("valueMap required for map")
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package mlmath

import (
	"errors"
	"math"
)

func Check_length(p *[]float64, q *[]float64) error {
	var err error = nil

	if len(*p) != len(*q) {
		err = errors.New("mymath: Input slices have not same length")
	}

	return err
}

func Euclide(p []float64, q []float64) float64 {

	res := 0.0

	for i, val := range p {
		res += (val - q[i]) * (val - q[i])
	}
	res = math.Sqrt(res)

	return res
}

func Manhatten(p []float64, q []float64) float64 {

	res := 0.0

	for i, val := range p {
		res += math.Abs(val - q[i])
	}

	return res
}

func KNN() {
}
<commit_msg>added type klabel<commit_after>package mlmath

import (
	"errors"
	"math"
)

type klabel struct {
	values []float64
	label  string
}

func Check_length(p *[]float64, q *[]float64) error {
	var err error = nil

	if len(*p) != len(*q) {
		err = errors.New("mymath: Input slices have not same length")
	}

	return err
}

func Euclide(p []float64, q []float64) float64 {

	res := 0.0

	for i, val := range p {
		res += (val - q[i]) * (val - q[i])
	}
	res = math.Sqrt(res)

	return res
}

func Manhatten(p []float64, q []float64) float64 {

	res := 0.0

	for i, val := range p {
		res += math.Abs(val - q[i])
	}

	return res
}

func KNN(in *[]float64, tdata *[]klabel) {

}
<|endoftext|>