<commit_before>package cloudwatch

import (
	"context"
	"errors"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/grafana/grafana/pkg/log"
	"github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/tsdb"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/request"
	"github.com/aws/aws-sdk-go/service/cloudwatch"
	"github.com/grafana/grafana/pkg/components/null"
	"github.com/grafana/grafana/pkg/components/simplejson"
	"github.com/grafana/grafana/pkg/metrics"
)

type CloudWatchExecutor struct {
	*models.DataSource
}

type DatasourceInfo struct {
	Profile       string
	Region        string
	AuthType      string
	AssumeRoleArn string
	Namespace     string

	AccessKey string
	SecretKey string
}

func NewCloudWatchExecutor(dsInfo *models.DataSource) (tsdb.TsdbQueryEndpoint, error) {
	return &CloudWatchExecutor{}, nil
}

var (
	plog               log.Logger
	standardStatistics map[string]bool
	aliasFormat        *regexp.Regexp
)

func init() {
	plog = log.New("tsdb.cloudwatch")
	tsdb.RegisterTsdbQueryEndpoint("cloudwatch", NewCloudWatchExecutor)
	standardStatistics = map[string]bool{
		"Average":     true,
		"Maximum":     true,
		"Minimum":     true,
		"Sum":         true,
		"SampleCount": true,
	}
	aliasFormat = regexp.MustCompile(`\{\{\s*(.+?)\s*\}\}`)
}

func (e *CloudWatchExecutor) Query(ctx context.Context, dsInfo *models.DataSource, queryContext *tsdb.TsdbQuery) (*tsdb.Response, error) {
	var result *tsdb.Response
	e.DataSource = dsInfo
	queryType := queryContext.Queries[0].Model.Get("type").MustString("")
	var err error

	switch queryType {
	case "metricFindQuery":
		result, err = e.executeMetricFindQuery(ctx, queryContext)
		break
	case "annotationQuery":
		result, err = e.executeAnnotationQuery(ctx, queryContext)
		break
	case "timeSeriesQuery":
		fallthrough
	default:
		result, err = e.executeTimeSeriesQuery(ctx, queryContext)
		break
	}

	return result, err
}

func (e *CloudWatchExecutor) executeTimeSeriesQuery(ctx context.Context, queryContext *tsdb.TsdbQuery) (*tsdb.Response, error) {
	result := &tsdb.Response{
		Results: make(map[string]*tsdb.QueryResult),
	}

	errCh := make(chan error, 1)
	resCh := make(chan *tsdb.QueryResult, 1)

	currentlyExecuting := 0
	for i, model := range queryContext.Queries {
		queryType := model.Model.Get("type").MustString()
		if queryType != "timeSeriesQuery" {
			continue
		}
		currentlyExecuting++
		go func(refId string, index int) {
			queryRes, err := e.executeQuery(ctx, queryContext.Queries[index].Model, queryContext)
			currentlyExecuting--
			if err != nil {
				errCh <- err
			} else {
				queryRes.RefId = refId
				resCh <- queryRes
			}
		}(model.RefId, i)
	}

	for currentlyExecuting != 0 {
		select {
		case res := <-resCh:
			result.Results[res.RefId] = res
		case err := <-errCh:
			return result, err
		case <-ctx.Done():
			return result, ctx.Err()
		}
	}

	return result, nil
}

func (e *CloudWatchExecutor) executeQuery(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.TsdbQuery) (*tsdb.QueryResult, error) {
	query, err := parseQuery(parameters)
	if err != nil {
		return nil, err
	}

	client, err := e.getClient(query.Region)
	if err != nil {
		return nil, err
	}

	startTime, err := queryContext.TimeRange.ParseFrom()
	if err != nil {
		return nil, err
	}

	endTime, err := queryContext.TimeRange.ParseTo()
	if err != nil {
		return nil, err
	}

	params := &cloudwatch.GetMetricStatisticsInput{
		Namespace:  aws.String(query.Namespace),
		MetricName: aws.String(query.MetricName),
		Dimensions: query.Dimensions,
		Period:     aws.Int64(int64(query.Period)),
		StartTime:  aws.Time(startTime),
		EndTime:    aws.Time(endTime),
	}
	if len(query.Statistics) > 0 {
		params.Statistics = query.Statistics
	}
	if len(query.ExtendedStatistics) > 0 {
		params.ExtendedStatistics = query.ExtendedStatistics
	}

	resp, err := client.GetMetricStatisticsWithContext(ctx, params, request.WithResponseReadTimeout(10*time.Second))
	if err != nil {
		return nil, err
	}
	metrics.M_Aws_CloudWatch_GetMetricStatistics.Inc()

	queryRes, err := parseResponse(resp, query)
	if err != nil {
		return nil, err
	}

	return queryRes, nil
}

func parseDimensions(model *simplejson.Json) ([]*cloudwatch.Dimension, error) {
	var result []*cloudwatch.Dimension

	for k, v := range model.Get("dimensions").MustMap() {
		kk := k
		if vv, ok := v.(string); ok {
			result = append(result, &cloudwatch.Dimension{
				Name:  &kk,
				Value: &vv,
			})
		} else {
			return nil, errors.New("failed to parse")
		}
	}

	sort.Slice(result, func(i, j int) bool {
		return *result[i].Name < *result[j].Name
	})
	return result, nil
}

func parseStatistics(model *simplejson.Json) ([]string, []string, error) {
	var statistics []string
	var extendedStatistics []string

	for _, s := range model.Get("statistics").MustArray() {
		if ss, ok := s.(string); ok {
			if _, isStandard := standardStatistics[ss]; isStandard {
				statistics = append(statistics, ss)
			} else {
				extendedStatistics = append(extendedStatistics, ss)
			}
		} else {
			return nil, nil, errors.New("failed to parse")
		}
	}

	return statistics, extendedStatistics, nil
}

func parseQuery(model *simplejson.Json) (*CloudWatchQuery, error) {
	region, err := model.Get("region").String()
	if err != nil {
		return nil, err
	}

	namespace, err := model.Get("namespace").String()
	if err != nil {
		return nil, err
	}

	metricName, err := model.Get("metricName").String()
	if err != nil {
		return nil, err
	}

	dimensions, err := parseDimensions(model)
	if err != nil {
		return nil, err
	}

	statistics, extendedStatistics, err := parseStatistics(model)
	if err != nil {
		return nil, err
	}

	p := model.Get("period").MustString("")
	if p == "" {
		if namespace == "AWS/EC2" {
			p = "300"
		} else {
			p = "60"
		}
	}

	period := 300
	if regexp.MustCompile(`^\d+$`).Match([]byte(p)) {
		period, err = strconv.Atoi(p)
		if err != nil {
			return nil, err
		}
	} else {
		d, err := time.ParseDuration(p)
		if err != nil {
			return nil, err
		}
		period = int(d.Seconds())
	}

	alias := model.Get("alias").MustString("{{metric}}_{{stat}}")

	return &CloudWatchQuery{
		Region:             region,
		Namespace:          namespace,
		MetricName:         metricName,
		Dimensions:         dimensions,
		Statistics:         aws.StringSlice(statistics),
		ExtendedStatistics: aws.StringSlice(extendedStatistics),
		Period:             period,
		Alias:              alias,
	}, nil
}

func formatAlias(query *CloudWatchQuery, stat string, dimensions map[string]string) string {
	data := map[string]string{}
	data["region"] = query.Region
	data["namespace"] = query.Namespace
	data["metric"] = query.MetricName
	data["stat"] = stat
	for k, v := range dimensions {
		data[k] = v
	}

	result := aliasFormat.ReplaceAllFunc([]byte(query.Alias), func(in []byte) []byte {
		labelName := strings.Replace(string(in), "{{", "", 1)
		labelName = strings.Replace(labelName, "}}", "", 1)
		labelName = strings.TrimSpace(labelName)
		if val, exists := data[labelName]; exists {
			return []byte(val)
		}

		return in
	})

	return string(result)
}

func parseResponse(resp *cloudwatch.GetMetricStatisticsOutput, query *CloudWatchQuery) (*tsdb.QueryResult, error) {
	queryRes := tsdb.NewQueryResult()

	var value float64
	for _, s := range append(query.Statistics, query.ExtendedStatistics...) {
		series := tsdb.TimeSeries{
			Tags: map[string]string{},
		}
		for _, d := range query.Dimensions {
			series.Tags[*d.Name] = *d.Value
		}
		series.Name = formatAlias(query, *s, series.Tags)

		lastTimestamp := make(map[string]time.Time)
		sort.Slice(resp.Datapoints, func(i, j int) bool {
			return (*resp.Datapoints[i].Timestamp).Before(*resp.Datapoints[j].Timestamp)
		})
		for _, v := range resp.Datapoints {
			switch *s {
			case "Average":
				value = *v.Average
			case "Maximum":
				value = *v.Maximum
			case "Minimum":
				value = *v.Minimum
			case "Sum":
				value = *v.Sum
			case "SampleCount":
				value = *v.SampleCount
			default:
				if strings.Index(*s, "p") == 0 && v.ExtendedStatistics[*s] != nil {
					value = *v.ExtendedStatistics[*s]
				}
			}

			// terminate gap of data points
			timestamp := *v.Timestamp
			if _, ok := lastTimestamp[*s]; ok {
				nextTimestampFromLast := lastTimestamp[*s].Add(time.Duration(query.Period) * time.Second)
				for timestamp.After(nextTimestampFromLast) {
					series.Points = append(series.Points, tsdb.NewTimePoint(null.FloatFromPtr(nil), float64(nextTimestampFromLast.Unix()*1000)))
					nextTimestampFromLast = nextTimestampFromLast.Add(time.Duration(query.Period) * time.Second)
				}
			}
			lastTimestamp[*s] = timestamp

			series.Points = append(series.Points, tsdb.NewTimePoint(null.FloatFrom(value), float64(timestamp.Unix()*1000)))
		}

		queryRes.Series = append(queryRes.Series, &series)
	}

	return queryRes, nil
}
<commit_msg>add debug log<commit_after>package cloudwatch

import (
	"context"
	"errors"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/grafana/grafana/pkg/log"
	"github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/setting"
	"github.com/grafana/grafana/pkg/tsdb"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/request"
	"github.com/aws/aws-sdk-go/service/cloudwatch"
	"github.com/grafana/grafana/pkg/components/null"
	"github.com/grafana/grafana/pkg/components/simplejson"
	"github.com/grafana/grafana/pkg/metrics"
)

type CloudWatchExecutor struct {
	*models.DataSource
}

type DatasourceInfo struct {
	Profile       string
	Region        string
	AuthType      string
	AssumeRoleArn string
	Namespace     string

	AccessKey string
	SecretKey string
}

func NewCloudWatchExecutor(dsInfo *models.DataSource) (tsdb.TsdbQueryEndpoint, error) {
	return &CloudWatchExecutor{}, nil
}

var (
	plog               log.Logger
	standardStatistics map[string]bool
	aliasFormat        *regexp.Regexp
)

func init() {
	plog = log.New("tsdb.cloudwatch")
	tsdb.RegisterTsdbQueryEndpoint("cloudwatch", NewCloudWatchExecutor)
	standardStatistics = map[string]bool{
		"Average":     true,
		"Maximum":     true,
		"Minimum":     true,
		"Sum":         true,
		"SampleCount": true,
	}
	aliasFormat = regexp.MustCompile(`\{\{\s*(.+?)\s*\}\}`)
}

func (e *CloudWatchExecutor) Query(ctx context.Context, dsInfo *models.DataSource, queryContext *tsdb.TsdbQuery) (*tsdb.Response, error) {
	var result *tsdb.Response
	e.DataSource = dsInfo
	queryType := queryContext.Queries[0].Model.Get("type").MustString("")
	var err error

	switch queryType {
	case "metricFindQuery":
		result, err = e.executeMetricFindQuery(ctx, queryContext)
		break
	case "annotationQuery":
		result, err = e.executeAnnotationQuery(ctx, queryContext)
		break
	case "timeSeriesQuery":
		fallthrough
	default:
		result, err = e.executeTimeSeriesQuery(ctx, queryContext)
		break
	}

	return result, err
}

func (e *CloudWatchExecutor) executeTimeSeriesQuery(ctx context.Context, queryContext *tsdb.TsdbQuery) (*tsdb.Response, error) {
	result := &tsdb.Response{
		Results: make(map[string]*tsdb.QueryResult),
	}

	errCh := make(chan error, 1)
	resCh := make(chan *tsdb.QueryResult, 1)

	currentlyExecuting := 0
	for i, model := range queryContext.Queries {
		queryType := model.Model.Get("type").MustString()
		if queryType != "timeSeriesQuery" {
			continue
		}
		currentlyExecuting++
		go func(refId string, index int) {
			queryRes, err := e.executeQuery(ctx, queryContext.Queries[index].Model, queryContext)
			currentlyExecuting--
			if err != nil {
				errCh <- err
			} else {
				queryRes.RefId = refId
				resCh <- queryRes
			}
		}(model.RefId, i)
	}

	for currentlyExecuting != 0 {
		select {
		case res := <-resCh:
			result.Results[res.RefId] = res
		case err := <-errCh:
			return result, err
		case <-ctx.Done():
			return result, ctx.Err()
		}
	}

	return result, nil
}

func (e *CloudWatchExecutor) executeQuery(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.TsdbQuery) (*tsdb.QueryResult, error) {
	query, err := parseQuery(parameters)
	if err != nil {
		return nil, err
	}

	client, err := e.getClient(query.Region)
	if err != nil {
		return nil, err
	}

	startTime, err := queryContext.TimeRange.ParseFrom()
	if err != nil {
		return nil, err
	}

	endTime, err := queryContext.TimeRange.ParseTo()
	if err != nil {
		return nil, err
	}

	params := &cloudwatch.GetMetricStatisticsInput{
		Namespace:  aws.String(query.Namespace),
		MetricName: aws.String(query.MetricName),
		Dimensions: query.Dimensions,
		Period:     aws.Int64(int64(query.Period)),
		StartTime:  aws.Time(startTime),
		EndTime:    aws.Time(endTime),
	}
	if len(query.Statistics) > 0 {
		params.Statistics = query.Statistics
	}
	if len(query.ExtendedStatistics) > 0 {
		params.ExtendedStatistics = query.ExtendedStatistics
	}

	if setting.Env == setting.DEV {
		plog.Debug("CloudWatch query", "raw query", params)
	}

	resp, err := client.GetMetricStatisticsWithContext(ctx, params, request.WithResponseReadTimeout(10*time.Second))
	if err != nil {
		return nil, err
	}
	metrics.M_Aws_CloudWatch_GetMetricStatistics.Inc()

	queryRes, err := parseResponse(resp, query)
	if err != nil {
		return nil, err
	}

	return queryRes, nil
}

func parseDimensions(model *simplejson.Json) ([]*cloudwatch.Dimension, error) {
	var result []*cloudwatch.Dimension

	for k, v := range model.Get("dimensions").MustMap() {
		kk := k
		if vv, ok := v.(string); ok {
			result = append(result, &cloudwatch.Dimension{
				Name:  &kk,
				Value: &vv,
			})
		} else {
			return nil, errors.New("failed to parse")
		}
	}

	sort.Slice(result, func(i, j int) bool {
		return *result[i].Name < *result[j].Name
	})
	return result, nil
}

func parseStatistics(model *simplejson.Json) ([]string, []string, error) {
	var statistics []string
	var extendedStatistics []string

	for _, s := range model.Get("statistics").MustArray() {
		if ss, ok := s.(string); ok {
			if _, isStandard := standardStatistics[ss]; isStandard {
				statistics = append(statistics, ss)
			} else {
				extendedStatistics = append(extendedStatistics, ss)
			}
		} else {
			return nil, nil, errors.New("failed to parse")
		}
	}

	return statistics, extendedStatistics, nil
}

func parseQuery(model *simplejson.Json) (*CloudWatchQuery, error) {
	region, err := model.Get("region").String()
	if err != nil {
		return nil, err
	}

	namespace, err := model.Get("namespace").String()
	if err != nil {
		return nil, err
	}

	metricName, err := model.Get("metricName").String()
	if err != nil {
		return nil, err
	}

	dimensions, err := parseDimensions(model)
	if err != nil {
		return nil, err
	}

	statistics, extendedStatistics, err := parseStatistics(model)
	if err != nil {
		return nil, err
	}

	p := model.Get("period").MustString("")
	if p == "" {
		if namespace == "AWS/EC2" {
			p = "300"
		} else {
			p = "60"
		}
	}

	period := 300
	if regexp.MustCompile(`^\d+$`).Match([]byte(p)) {
		period, err = strconv.Atoi(p)
		if err != nil {
			return nil, err
		}
	} else {
		d, err := time.ParseDuration(p)
		if err != nil {
			return nil, err
		}
		period = int(d.Seconds())
	}

	alias := model.Get("alias").MustString("{{metric}}_{{stat}}")

	return &CloudWatchQuery{
		Region:             region,
		Namespace:          namespace,
		MetricName:         metricName,
		Dimensions:         dimensions,
		Statistics:         aws.StringSlice(statistics),
		ExtendedStatistics: aws.StringSlice(extendedStatistics),
		Period:             period,
		Alias:              alias,
	}, nil
}

func formatAlias(query *CloudWatchQuery, stat string, dimensions map[string]string) string {
	data := map[string]string{}
	data["region"] = query.Region
	data["namespace"] = query.Namespace
	data["metric"] = query.MetricName
	data["stat"] = stat
	for k, v := range dimensions {
		data[k] = v
	}

	result := aliasFormat.ReplaceAllFunc([]byte(query.Alias), func(in []byte) []byte {
		labelName := strings.Replace(string(in), "{{", "", 1)
		labelName = strings.Replace(labelName, "}}", "", 1)
		labelName = strings.TrimSpace(labelName)
		if val, exists := data[labelName]; exists {
			return []byte(val)
		}

		return in
	})

	return string(result)
}

func parseResponse(resp *cloudwatch.GetMetricStatisticsOutput, query *CloudWatchQuery) (*tsdb.QueryResult, error) {
	queryRes := tsdb.NewQueryResult()

	var value float64
	for _, s := range append(query.Statistics, query.ExtendedStatistics...) {
		series := tsdb.TimeSeries{
			Tags: map[string]string{},
		}
		for _, d := range query.Dimensions {
			series.Tags[*d.Name] = *d.Value
		}
		series.Name = formatAlias(query, *s, series.Tags)

		lastTimestamp := make(map[string]time.Time)
		sort.Slice(resp.Datapoints, func(i, j int) bool {
			return (*resp.Datapoints[i].Timestamp).Before(*resp.Datapoints[j].Timestamp)
		})
		for _, v := range resp.Datapoints {
			switch *s {
			case "Average":
				value = *v.Average
			case "Maximum":
				value = *v.Maximum
			case "Minimum":
				value = *v.Minimum
			case "Sum":
				value = *v.Sum
			case "SampleCount":
				value = *v.SampleCount
			default:
				if strings.Index(*s, "p") == 0 && v.ExtendedStatistics[*s] != nil {
					value = *v.ExtendedStatistics[*s]
				}
			}

			// terminate gap of data points
			timestamp := *v.Timestamp
			if _, ok := lastTimestamp[*s]; ok {
				nextTimestampFromLast := lastTimestamp[*s].Add(time.Duration(query.Period) * time.Second)
				for timestamp.After(nextTimestampFromLast) {
					series.Points = append(series.Points, tsdb.NewTimePoint(null.FloatFromPtr(nil), float64(nextTimestampFromLast.Unix()*1000)))
					nextTimestampFromLast = nextTimestampFromLast.Add(time.Duration(query.Period) * time.Second)
				}
			}
			lastTimestamp[*s] = timestamp

			series.Points = append(series.Points, tsdb.NewTimePoint(null.FloatFrom(value), float64(timestamp.Unix()*1000)))
		}

		queryRes.Series = append(queryRes.Series, &series)
	}

	return queryRes, nil
}
<|endoftext|>


<commit_before>package transport

import (
	"fmt"
	"net/http"
	"net/http/httputil"

	"github.com/google/go-containerregistry/pkg/logs"
)

type logTransport struct {
	inner http.RoundTripper
}

// NewLogger returns a transport that logs requests and responses to
// github.com/google/go-containerregistry/pkg/logs.Debug.
func NewLogger(inner http.RoundTripper) http.RoundTripper {
	return &logTransport{inner}
}

func (t *logTransport) RoundTrip(in *http.Request) (out *http.Response, err error) {
	// Inspired by: github.com/motemen/go-loghttp
	logs.Debug.Printf("--> %s %s", in.Method, in.URL)
	b, err := httputil.DumpRequestOut(in, true)
	if err == nil {
		logs.Debug.Println(string(b))
	}
	out, err = t.inner.RoundTrip(in)
	if err != nil {
		logs.Debug.Printf("<-- %v %s", err, in.URL)
	}
	if out != nil {
		msg := fmt.Sprintf("<-- %d", out.StatusCode)
		if out.Request != nil {
			msg = fmt.Sprintf("%s %s", msg, out.Request.URL)
		}
		logs.Debug.Printf(msg)
		b, err := httputil.DumpResponse(out, true)
		if err == nil {
			logs.Debug.Println(string(b))
		}
	}
	return
}
<commit_msg>Fix logger transport (#615)<commit_after>package transport

import (
	"fmt"
	"net/http"
	"net/http/httputil"

	"github.com/google/go-containerregistry/pkg/logs"
)

type logTransport struct {
	inner http.RoundTripper
}

// NewLogger returns a transport that logs requests and responses to
// github.com/google/go-containerregistry/pkg/logs.Debug.
func NewLogger(inner http.RoundTripper) http.RoundTripper {
	return &logTransport{inner}
}

func (t *logTransport) RoundTrip(in *http.Request) (out *http.Response, err error) {
	// Inspired by: github.com/motemen/go-loghttp
	logs.Debug.Printf("--> %s %s", in.Method, in.URL)
	b, err := httputil.DumpRequestOut(in, true)
	if err == nil {
		logs.Debug.Println(string(b))
	}
	out, err = t.inner.RoundTrip(in)
	if err != nil {
		logs.Debug.Printf("<-- %v %s", err, in.URL)
	}
	if out != nil {
		msg := fmt.Sprintf("<-- %d", out.StatusCode)
		if out.Request != nil {
			msg = fmt.Sprintf("%s %s", msg, out.Request.URL)
		}
		logs.Debug.Print(msg)
		b, err := httputil.DumpResponse(out, true)
		if err == nil {
			logs.Debug.Println(string(b))
		}
	}
	return
}
<|endoftext|>


<commit_before>package starlark

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"

	"github.com/drone/drone-yaml/yaml"
	"github.com/drone/drone-yaml/yaml/pretty"

	"github.com/urfave/cli"
	"go.starlark.net/starlark"
	"go.starlark.net/starlarkstruct"
)

// Command exports the jsonnet command.
var Command = cli.Command{
	Name:      "starlark",
	Usage:     "generate .drone.yml from starlark",
	ArgsUsage: "[path/to/.drone.star]",
	Action: func(c *cli.Context) {
		if err := generate(c); err != nil {
			if err, ok := err.(*starlark.EvalError); ok {
				log.Fatalf("starlark evaluation error:\n%s", err.Backtrace())
			}
			log.Fatalln(err)
		}
	},
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "source",
			Usage: "Source file",
			Value: ".drone.star",
		},
		cli.StringFlag{
			Name:  "target",
			Usage: "target file",
			Value: ".drone.yml",
		},
		cli.BoolTFlag{
			Name:  "format",
			Usage: "Write output as formatted YAML",
		},
		cli.BoolFlag{
			Name:  "stdout",
			Usage: "Write output to stdout",
		},
		//
		// Drone Parameters
		//
		cli.StringFlag{
			Name:  "repo.name",
			Usage: "repository name",
		},
		cli.StringFlag{
			Name:  "repo.namespace",
			Usage: "repository namespace",
		},
		cli.StringFlag{
			Name:  "repo.slug",
			Usage: "repository slug",
		},

		cli.StringFlag{
			Name:  "build.event",
			Usage: "build event",
			Value: "push",
		},
		cli.StringFlag{
			Name:  "build.branch",
			Usage: "build branch",
			Value: "master",
		},
		cli.StringFlag{
			Name:  "build.source",
			Usage: "build source branch",
			Value: "master",
		},
		cli.StringFlag{
			Name:  "build.target",
			Usage: "build target branch",
			Value: "master",
		},
		cli.StringFlag{
			Name:  "build.ref",
			Usage: "build ref",
			Value: "refs/heads/master",
		},
		cli.StringFlag{
			Name:  "build.commit",
			Usage: "build commit sha",
		},
		cli.StringFlag{
			Name:  "build.message",
			Usage: "build commit message",
		},
	},
}

func generate(c *cli.Context) error {
	source := c.String("source")
	target := c.String("target")

	data, err := ioutil.ReadFile(source)
	if err != nil {
		return err
	}

	thread := &starlark.Thread{
		Name:  "drone",
		Print: func(_ *starlark.Thread, msg string) { fmt.Println(msg) },
		Load:  makeLoad(),
	}
	globals, err := starlark.ExecFile(thread, source, data, nil)
	if err != nil {
		return err
	}

	mainVal, ok := globals["main"]
	if !ok {
		return fmt.Errorf("no main function found")
	}
	main, ok := mainVal.(starlark.Callable)
	if !ok {
		return fmt.Errorf("main must be a function")
	}

	// TODO this needs to be flushed out.
	repo := starlark.StringDict{
		"name":      starlark.String(c.String("repo.name")),
		"namespace": starlark.String(c.String("repo.namespace")),
		"slug":      starlark.String(c.String("repo.slug")),
	}

	build := starlark.StringDict{
		"event":   starlark.String(c.String("build.event")),
		"branch":  starlark.String(c.String("build.branch")),
		"source":  starlark.String(c.String("build.source_branch")),
		"target":  starlark.String(c.String("build.target_branch")),
		"ref":     starlark.String(c.String("build.ref")),
		"commit":  starlark.String(c.String("build.commit")),
		"message": starlark.String(c.String("build.message")),
	}

	args := starlark.Tuple([]starlark.Value{
		starlarkstruct.FromStringDict(
			starlark.String("context"),
			starlark.StringDict{
				"repo":  starlarkstruct.FromStringDict(starlark.String("repo"), repo),
				"build": starlarkstruct.FromStringDict(starlark.String("build"), build),
			},
		),
	})
	mainVal, err = starlark.Call(thread, main, args, nil)
	if err != nil {
		return err
	}

	buf := new(bytes.Buffer)
	switch v := mainVal.(type) {
	case *starlark.List:
		for i := 0; i < v.Len(); i++ {
			item := v.Index(i)
			buf.WriteString("---\n")
			err = writeJSON(buf, item)
			if err != nil {
				return err
			}
			buf.WriteString("\n")
		}
	case *starlark.Dict:
		buf.WriteString("---\n")
		err = writeJSON(buf, v)
		if err != nil {
			return err
		}
	default:
		return fmt.Errorf("invalid return type (got a %s)", mainVal.Type())
	}

	// if the user disables pretty printing, the yaml is printed
	// to the console or written to the file in json format.
	if c.BoolT("format") == false {
		if c.Bool("stdout") {
			io.Copy(os.Stdout, buf)
			return nil
		}
		return ioutil.WriteFile(target, buf.Bytes(), 0644)
	}

	manifest, err := yaml.Parse(buf)
	if err != nil {
		return err
	}
	buf.Reset()
	pretty.Print(buf, manifest)

	// the user can optionally write the yaml to stdout. This
	// is useful for debugging purposes without mutating an
	// existing file.
	if c.Bool("stdout") {
		io.Copy(os.Stdout, buf)
		return nil
	}
	return ioutil.WriteFile(target, buf.Bytes(), 0644)
}

// Adapted from skycfg:
// https://github.com/stripe/skycfg/blob/eaa524101c2a0807c13ed5d2e52576fefc146ec3/internal/go/skycfg/json_write.go#L45
func writeJSON(out *bytes.Buffer, v starlark.Value) error {
	if marshaler, ok := v.(json.Marshaler); ok {
		jsonData, err := marshaler.MarshalJSON()
		if err != nil {
			return err
		}
		out.Write(jsonData)
		return nil
	}

	switch v := v.(type) {
	case starlark.NoneType:
		out.WriteString("null")
	case starlark.Bool:
		fmt.Fprintf(out, "%t", v)
	case starlark.Int:
		out.WriteString(v.String())
	case starlark.Float:
		fmt.Fprintf(out, "%g", v)
	case starlark.String:
		s := string(v)
		if goQuoteIsSafe(s) {
			fmt.Fprintf(out, "%q", s)
		} else {
			// vanishingly rare for text strings
			data, _ := json.Marshal(s)
			out.Write(data)
		}
	case starlark.Indexable: // Tuple, List
		out.WriteByte('[')
		for i, n := 0, starlark.Len(v); i < n; i++ {
			if i > 0 {
				out.WriteString(", ")
			}
			if err := writeJSON(out, v.Index(i)); err != nil {
				return err
			}
		}
		out.WriteByte(']')
	case *starlark.Dict:
		out.WriteByte('{')
		for i, itemPair := range v.Items() {
			key := itemPair[0]
			value := itemPair[1]
			if i > 0 {
				out.WriteString(", ")
			}
			if err := writeJSON(out, key); err != nil {
				return err
			}
			out.WriteString(": ")
			if err := writeJSON(out, value); err != nil {
				return err
			}
		}
		out.WriteByte('}')
	default:
		return fmt.Errorf("TypeError: value %s (type `%s') can't be converted to JSON.", v.String(), v.Type())
	}
	return nil
}

func goQuoteIsSafe(s string) bool {
	for _, r := range s {
		// JSON doesn't like Go's \xHH escapes for ASCII control codes,
		// nor its \UHHHHHHHH escapes for runes >16 bits.
		if r < 0x20 || r >= 0x10000 {
			return false
		}
	}
	return true
}

// https://github.com/google/starlark-go/blob/4eb76950c5f02ec5bcfd3ca898231a6543942fd9/repl/repl.go#L175
func makeLoad() func(thread *starlark.Thread, module string) (starlark.StringDict, error) {
	type entry struct {
		globals starlark.StringDict
		err     error
	}

	var cache = make(map[string]*entry)

	return func(thread *starlark.Thread, module string) (starlark.StringDict, error) {
		e, ok := cache[module]
		if e == nil {
			if ok {
				// request for package whose loading is in progress
				return nil, fmt.Errorf("cycle in load graph")
			}

			// Add a placeholder to indicate "load in progress".
			cache[module] = nil

			// Load it.
			thread := &starlark.Thread{Name: "exec " + module, Load: thread.Load}
			globals, err := starlark.ExecFile(thread, module, nil, nil)
			e = &entry{globals, err}

			// Update the cache.
			cache[module] = e
		}
		return e.globals, e.err
	}
}
<commit_msg>add more attributes to build context<commit_after>package starlark

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"

	"github.com/drone/drone-yaml/yaml"
	"github.com/drone/drone-yaml/yaml/pretty"

	"github.com/urfave/cli"
	"go.starlark.net/starlark"
	"go.starlark.net/starlarkstruct"
)

// Command exports the jsonnet command.
var Command = cli.Command{
	Name:      "starlark",
	Usage:     "generate .drone.yml from starlark",
	ArgsUsage: "[path/to/.drone.star]",
	Action: func(c *cli.Context) {
		if err := generate(c); err != nil {
			if err, ok := err.(*starlark.EvalError); ok {
				log.Fatalf("starlark evaluation error:\n%s", err.Backtrace())
			}
			log.Fatalln(err)
		}
	},
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "source",
			Usage: "Source file",
			Value: ".drone.star",
		},
		cli.StringFlag{
			Name:  "target",
			Usage: "target file",
			Value: ".drone.yml",
		},
		cli.BoolTFlag{
			Name:  "format",
			Usage: "Write output as formatted YAML",
		},
		cli.BoolFlag{
			Name:  "stdout",
			Usage: "Write output to stdout",
		},
		//
		// Drone Parameters
		//
		cli.StringFlag{
			Name:  "repo.name",
			Usage: "repository name",
		},
		cli.StringFlag{
			Name:  "repo.namespace",
			Usage: "repository namespace",
		},
		cli.StringFlag{
			Name:  "repo.slug",
			Usage: "repository slug",
		},

		cli.StringFlag{
			Name:  "build.event",
			Usage: "build event",
			Value: "push",
		},
		cli.StringFlag{
			Name:  "build.branch",
			Usage: "build branch",
			Value: "master",
		},
		cli.StringFlag{
			Name:  "build.source",
			Usage: "build source branch",
			Value: "master",
		},
		cli.StringFlag{
			Name:  "build.source_repo",
			Usage: "repo slug of source repository",
		},
		cli.StringFlag{
			Name:  "build.target",
			Usage: "build target branch",
			Value: "master",
		},
		cli.StringFlag{
			Name:  "build.ref",
			Usage: "build ref",
			Value: "refs/heads/master",
		},
		cli.StringFlag{
			Name:  "build.commit",
			Usage: "build commit sha",
		},
		cli.StringFlag{
			Name:  "build.message",
			Usage: "build commit message",
		},
		cli.StringFlag{
			Name:  "build.title",
			Usage: "build title",
		},
	},
}

func generate(c *cli.Context) error {
	source := c.String("source")
	target := c.String("target")

	data, err := ioutil.ReadFile(source)
	if err != nil {
		return err
	}

	thread := &starlark.Thread{
		Name:  "drone",
		Print: func(_ *starlark.Thread, msg string) { fmt.Println(msg) },
		Load:  makeLoad(),
	}
	globals, err := starlark.ExecFile(thread, source, data, nil)
	if err != nil {
		return err
	}

	mainVal, ok := globals["main"]
	if !ok {
		return fmt.Errorf("no main function found")
	}
	main, ok := mainVal.(starlark.Callable)
	if !ok {
		return fmt.Errorf("main must be a function")
	}

	// TODO this needs to be flushed out.
	repo := starlark.StringDict{
		"name":      starlark.String(c.String("repo.name")),
		"namespace": starlark.String(c.String("repo.namespace")),
		"slug":      starlark.String(c.String("repo.slug")),
	}

	build := starlark.StringDict{
		"event":       starlark.String(c.String("build.event")),
		"branch":      starlark.String(c.String("build.branch")),
		"source":      starlark.String(c.String("build.source_branch")),
		"source_repo": starlark.String(c.String("build.source_repo")),
		"target":      starlark.String(c.String("build.target_branch")),
		"ref":         starlark.String(c.String("build.ref")),
		"commit":      starlark.String(c.String("build.commit")),
		"message":     starlark.String(c.String("build.message")),
		"title":       starlark.String(c.String("build.title")),
	}

	args := starlark.Tuple([]starlark.Value{
		starlarkstruct.FromStringDict(
			starlark.String("context"),
			starlark.StringDict{
				"repo":  starlarkstruct.FromStringDict(starlark.String("repo"), repo),
				"build": starlarkstruct.FromStringDict(starlark.String("build"), build),
			},
		),
	})
	mainVal, err = starlark.Call(thread, main, args, nil)
	if err != nil {
		return err
	}

	buf := new(bytes.Buffer)
	switch v := mainVal.(type) {
	case *starlark.List:
		for i := 0; i < v.Len(); i++ {
			item := v.Index(i)
			buf.WriteString("---\n")
			err = writeJSON(buf, item)
			if err != nil {
				return err
			}
			buf.WriteString("\n")
		}
	case *starlark.Dict:
		buf.WriteString("---\n")
		err = writeJSON(buf, v)
		if err != nil {
			return err
		}
	default:
		return fmt.Errorf("invalid return type (got a %s)", mainVal.Type())
	}

	// if the user disables pretty printing, the yaml is printed
	// to the console or written to the file in json format.
	if c.BoolT("format") == false {
		if c.Bool("stdout") {
			io.Copy(os.Stdout, buf)
			return nil
		}
		return ioutil.WriteFile(target, buf.Bytes(), 0644)
	}

	manifest, err := yaml.Parse(buf)
	if err != nil {
		return err
	}
	buf.Reset()
	pretty.Print(buf, manifest)

	// the user can optionally write the yaml to stdout. This
	// is useful for debugging purposes without mutating an
	// existing file.
	if c.Bool("stdout") {
		io.Copy(os.Stdout, buf)
		return nil
	}
	return ioutil.WriteFile(target, buf.Bytes(), 0644)
}

// Adapted from skycfg:
// https://github.com/stripe/skycfg/blob/eaa524101c2a0807c13ed5d2e52576fefc146ec3/internal/go/skycfg/json_write.go#L45
func writeJSON(out *bytes.Buffer, v starlark.Value) error {
	if marshaler, ok := v.(json.Marshaler); ok {
		jsonData, err := marshaler.MarshalJSON()
		if err != nil {
			return err
		}
		out.Write(jsonData)
		return nil
	}

	switch v := v.(type) {
	case starlark.NoneType:
		out.WriteString("null")
	case starlark.Bool:
		fmt.Fprintf(out, "%t", v)
	case starlark.Int:
		out.WriteString(v.String())
	case starlark.Float:
		fmt.Fprintf(out, "%g", v)
	case starlark.String:
		s := string(v)
		if goQuoteIsSafe(s) {
			fmt.Fprintf(out, "%q", s)
		} else {
			// vanishingly rare for text strings
			data, _ := json.Marshal(s)
			out.Write(data)
		}
	case starlark.Indexable: // Tuple, List
		out.WriteByte('[')
		for i, n := 0, starlark.Len(v); i < n; i++ {
			if i > 0 {
				out.WriteString(", ")
			}
			if err := writeJSON(out, v.Index(i)); err != nil {
				return err
			}
		}
		out.WriteByte(']')
	case *starlark.Dict:
		out.WriteByte('{')
		for i, itemPair := range v.Items() {
			key := itemPair[0]
			value := itemPair[1]
			if i > 0 {
				out.WriteString(", ")
			}
			if err := writeJSON(out, key); err != nil {
				return err
			}
			out.WriteString(": ")
			if err := writeJSON(out, value); err != nil {
				return err
			}
		}
		out.WriteByte('}')
	default:
		return fmt.Errorf("TypeError: value %s (type `%s') can't be converted to JSON.", v.String(), v.Type())
	}
	return nil
}

func goQuoteIsSafe(s string) bool {
	for _, r := range s {
		// JSON doesn't like Go's \xHH escapes for ASCII control codes,
		// nor its \UHHHHHHHH escapes for runes >16 bits.
		if r < 0x20 || r >= 0x10000 {
			return false
		}
	}
	return true
}

// https://github.com/google/starlark-go/blob/4eb76950c5f02ec5bcfd3ca898231a6543942fd9/repl/repl.go#L175
func makeLoad() func(thread *starlark.Thread, module string) (starlark.StringDict, error) {
	type entry struct {
		globals starlark.StringDict
		err     error
	}

	var cache = make(map[string]*entry)

	return func(thread *starlark.Thread, module string) (starlark.StringDict, error) {
		e, ok := cache[module]
		if e == nil {
			if ok {
				// request for package whose loading is in progress
				return nil, fmt.Errorf("cycle in load graph")
			}

			// Add a placeholder to indicate "load in progress".
			cache[module] = nil

			// Load it.
			thread := &starlark.Thread{Name: "exec " + module, Load: thread.Load}
			globals, err := starlark.ExecFile(thread, module, nil, nil)
			e = &entry{globals, err}

			// Update the cache.
			cache[module] = e
		}
		return e.globals, e.err
	}
}
<|endoftext|>


<commit_before>package worker

import (
	"bufio"
	"bytes"
	"encoding/binary"
	"io"
	"net"
	"sync"
	"time"

	"github.com/cloudfoundry/gosigar"
)

// The agent of job server.
type agent struct {
	sync.Mutex
	conn      net.Conn
	rw        *bufio.ReadWriter
	worker    *Worker
	in        chan []byte
	net, addr string
}

// Create the agent of job server.
func newAgent(net, addr string, worker *Worker) (a *agent, err error) {
	a = &agent{
		net:    net,
		addr:   addr,
		worker: worker,
		in:     make(chan []byte, queueSize),
	}
	return
}

func (a *agent) Connect() (err error) {
	a.Lock()
	defer a.Unlock()
	a.conn, err = net.Dial(a.net, a.addr)
	if err != nil {
		return
	}
	a.rw = bufio.NewReadWriter(bufio.NewReader(a.conn),
		bufio.NewWriter(a.conn))
	go a.work()
	return
}

func (a *agent) work() {
	defer func() {
		if err := recover(); err != nil {
			a.worker.err(err.(error))
		}
	}()

	var inpack *inPack
	var l int
	var err error
	var data, leftdata []byte
	for {
		if data, err = a.read(); err != nil {
			if opErr, ok := err.(*net.OpError); ok {
				if opErr.Temporary() {
					continue
				} else {
					a.disconnect_error(err)
					// else - we're probably dc'ing due to a Close()

					break
				}

			} else if err == io.EOF {
				a.disconnect_error(err)
				break
			}
			a.worker.err(err)
			// If it is unexpected error and the connection wasn't
			// closed by Gearmand, the agent should close the conection
			// and reconnect to job server.
			a.Close()
			a.conn, err = net.Dial(a.net, a.addr)
			if err != nil {
				a.worker.err(err)
				break
			}
			a.rw = bufio.NewReadWriter(bufio.NewReader(a.conn),
				bufio.NewWriter(a.conn))
		}
		if len(leftdata) > 0 { // some data left for processing
			data = append(leftdata, data...)
		}
		if len(data) < minPacketLength { // not enough data
			leftdata = data
			continue
		}
		if inpack, l, err = decodeInPack(data); err != nil {
			//a.worker.err(err)
			leftdata = data
			continue
		}
		leftdata = nil
		inpack.a = a
		a.worker.in <- inpack
		if len(data) > l {
			leftdata = data[l:]
		}
	}
}

func (a *agent) disconnect_error(err error) {
	if a.conn != nil {
		err = &WorkerDisconnectError{
			err:   err,
			agent: a,
		}
		a.worker.err(err)
	}
}

func (a *agent) Close() {
	a.Lock()
	defer a.Unlock()
	if a.conn != nil {
		a.conn.Close()
		a.conn = nil
	}
}

func (a *agent) Grab() {
	avg := sigar.LoadAverage{}
	avg.Get()
	time.Sleep(avg.One * time.Millisecond)

	a.Lock()
	defer a.Unlock()
	a.grab()
}

func (a *agent) grab() {
	outpack := getOutPack()
	outpack.dataType = dtGrabJobUniq
	a.write(outpack)
}

func (a *agent) PreSleep() {
	a.Lock()
	defer a.Unlock()
	outpack := getOutPack()
	outpack.dataType = dtPreSleep
	a.write(outpack)
}

func (a *agent) reconnect() error {
	a.Lock()
	defer a.Unlock()
	conn, err := net.Dial(a.net, a.addr)
	if err != nil {
		return err
	}
	a.conn = conn
	a.rw = bufio.NewReadWriter(bufio.NewReader(a.conn),
		bufio.NewWriter(a.conn))
	a.grab()
	a.worker.reRegisterFuncsForAgent(a)

	go a.work()
	return nil
}

// read length bytes from the socket
func (a *agent) read() (data []byte, err error) {
	n := 0

	tmp := getBuffer(bufferSize)
	var buf bytes.Buffer

	// read the header so we can get the length of the data
	if n, err = a.rw.Read(tmp); err != nil {
		return
	}
	dl := int(binary.BigEndian.Uint32(tmp[8:12]))

	// write what we read so far
	buf.Write(tmp[:n])

	// read until we receive all the data
	for buf.Len() < dl+minPacketLength {
		if n, err = a.rw.Read(tmp); err != nil {
			return buf.Bytes(), err
		}

		buf.Write(tmp[:n])
	}

	return buf.Bytes(), err
}

// Internal write the encoded job.
func (a *agent) write(outpack *outPack) (err error) {
	var n int
	buf := outpack.Encode()
	for i := 0; i < len(buf); i += n {
		n, err = a.rw.Write(buf[i:])
		if err != nil {
			return err
		}
	}
	return a.rw.Flush()
}

// Write with lock
func (a *agent) Write(outpack *outPack) (err error) {
	a.Lock()
	defer a.Unlock()
	return a.write(outpack)
}
<commit_msg>Do not truncate load avg<commit_after>package worker

import (
	"bufio"
	"bytes"
	"encoding/binary"
	"io"
	"net"
	"sync"
	"time"

	"github.com/cloudfoundry/gosigar"
)

// The agent of job server.
type agent struct {
	sync.Mutex
	conn      net.Conn
	rw        *bufio.ReadWriter
	worker    *Worker
	in        chan []byte
	net, addr string
}

// Create the agent of job server.
func newAgent(net, addr string, worker *Worker) (a *agent, err error) {
	a = &agent{
		net:    net,
		addr:   addr,
		worker: worker,
		in:     make(chan []byte, queueSize),
	}
	return
}

func (a *agent) Connect() (err error) {
	a.Lock()
	defer a.Unlock()
	a.conn, err = net.Dial(a.net, a.addr)
	if err != nil {
		return
	}
	a.rw = bufio.NewReadWriter(bufio.NewReader(a.conn),
		bufio.NewWriter(a.conn))
	go a.work()
	return
}

func (a *agent) work() {
	defer func() {
		if err := recover(); err != nil {
			a.worker.err(err.(error))
		}
	}()

	var inpack *inPack
	var l int
	var err error
	var data, leftdata []byte
	for {
		if data, err = a.read(); err != nil {
			if opErr, ok := err.(*net.OpError); ok {
				if opErr.Temporary() {
					continue
				} else {
					a.disconnect_error(err)
					// else - we're probably dc'ing due to a Close()

					break
				}

			} else if err == io.EOF {
				a.disconnect_error(err)
				break
			}
			a.worker.err(err)
			// If it is unexpected error and the connection wasn't
			// closed by Gearmand, the agent should close the conection
			// and reconnect to job server.
			a.Close()
			a.conn, err = net.Dial(a.net, a.addr)
			if err != nil {
				a.worker.err(err)
				break
			}
			a.rw = bufio.NewReadWriter(bufio.NewReader(a.conn),
				bufio.NewWriter(a.conn))
		}
		if len(leftdata) > 0 { // some data left for processing
			data = append(leftdata, data...)
		}
		if len(data) < minPacketLength { // not enough data
			leftdata = data
			continue
		}
		if inpack, l, err = decodeInPack(data); err != nil {
			//a.worker.err(err)
			leftdata = data
			continue
		}
		leftdata = nil
		inpack.a = a
		a.worker.in <- inpack
		if len(data) > l {
			leftdata = data[l:]
		}
	}
}

func (a *agent) disconnect_error(err error) {
	if a.conn != nil {
		err = &WorkerDisconnectError{
			err:   err,
			agent: a,
		}
		a.worker.err(err)
	}
}

func (a *agent) Close() {
	a.Lock()
	defer a.Unlock()
	if a.conn != nil {
		a.conn.Close()
		a.conn = nil
	}
}

func (a *agent) Grab() {
	avg := sigar.LoadAverage{}
	avg.Get()
	time.Sleep(time.Duration(avg.One*1000) * time.Microsecond)

	a.Lock()
	defer a.Unlock()
	a.grab()
}

func (a *agent) grab() {
	outpack := getOutPack()
	outpack.dataType = dtGrabJobUniq
	a.write(outpack)
}

func (a *agent) PreSleep() {
	a.Lock()
	defer a.Unlock()
	outpack := getOutPack()
	outpack.dataType = dtPreSleep
	a.write(outpack)
}

func (a *agent) reconnect() error {
	a.Lock()
	defer a.Unlock()
	conn, err := net.Dial(a.net, a.addr)
	if err != nil {
		return err
	}
	a.conn = conn
	a.rw = bufio.NewReadWriter(bufio.NewReader(a.conn),
		bufio.NewWriter(a.conn))
	a.grab()
	a.worker.reRegisterFuncsForAgent(a)

	go a.work()
	return nil
}

// read length bytes from the socket
func (a *agent) read() (data []byte, err error) {
	n := 0

	tmp := getBuffer(bufferSize)
	var buf bytes.Buffer

	// read the header so we can get the length of the data
	if n, err = a.rw.Read(tmp); err != nil {
		return
	}
	dl := int(binary.BigEndian.Uint32(tmp[8:12]))

	// write what we read so far
	buf.Write(tmp[:n])

	// read until we receive all the data
	for buf.Len() < dl+minPacketLength {
		if n, err = a.rw.Read(tmp); err != nil {
			return buf.Bytes(), err
		}

		buf.Write(tmp[:n])
	}

	return buf.Bytes(), err
}

// Internal write the encoded job.
func (a *agent) write(outpack *outPack) (err error) {
	var n int
	buf := outpack.Encode()
	for i := 0; i < len(buf); i += n {
		n, err = a.rw.Write(buf[i:])
		if err != nil {
			return err
		}
	}
	return a.rw.Flush()
}

// Write with lock
func (a *agent) Write(outpack *outPack) (err error) {
	a.Lock()
	defer a.Unlock()
	return a.write(outpack)
}
<|endoftext|>


<commit_before>// Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package main

import (
	"go/importer"
	"go/types"
	"regexp"
)

//go:generate go run generate/std/main.go generate/std/pkgs.go
//go:generate gofmt -w std.go

type funcSign struct {
	params  []types.Type
	results []types.Type
}

type ifaceSign struct {
	name string
	t    *types.Interface

	funcs map[string]funcSign
}

type cache struct {
	done map[string]struct{}

	// key is importPath.typeName
	// TODO: do something about duplicates, especially to behave
	// deterministically if two keys map to equal ifaceSigns.
	stdIfaces map[string][]ifaceSign

	pkgIfaces map[string][]ifaceSign

	curPaths []string

	// TODO: avoid duplicates
	funcs []funcSign
}

func typesInit() error {
	c = &cache{
		done:      make(map[string]struct{}),
		stdIfaces: make(map[string][]ifaceSign),
		pkgIfaces: make(map[string][]ifaceSign),
	}
	imp := importer.Default()
	for path, names := range pkgs {
		c.done[path] = struct{}{}
		if len(names) == 0 {
			continue
		}
		pkg, err := imp.Import(path)
		if err != nil {
			return err
		}
		c.grabNames(pkg.Scope(), path, names)
	}
	c.grabNames(types.Universe, "", []string{"error"})
	return nil
}

func (c *cache) grabNames(scope *types.Scope, path string, names []string) {
	for _, name := range names {
		tn := scope.Lookup(name).(*types.TypeName)
		switch x := tn.Type().Underlying().(type) {
		case *types.Interface:
			c.addInterface(c.stdIfaces, path, name, x)
		case *types.Signature:
			c.addFunc(x)
		}
	}
}

func (c *cache) addInterface(m map[string][]ifaceSign, path, name string, iface *types.Interface) {
	ifsign := ifaceSign{
		name:  name,
		t:     iface,
		funcs: make(map[string]funcSign, iface.NumMethods()),
	}
	for i := 0; i < iface.NumMethods(); i++ {
		f := iface.Method(i)
		sign := f.Type().(*types.Signature)
		ifsign.funcs[f.Name()] = c.addFunc(sign)
	}
	m[path] = append(m[path], ifsign)
}

func (c *cache) addFunc(sign *types.Signature) funcSign {
	fsign := funcSign{
		params:  typeList(sign.Params()),
		results: typeList(sign.Results()),
	}
	c.funcs = append(c.funcs, fsign)
	return fsign
}

var exported = regexp.MustCompile(`^[A-Z]`)

func (c *cache) grabExported(scope *types.Scope, path string) {
	c.done[path] = struct{}{}
	for _, name := range scope.Names() {
		tn, ok := scope.Lookup(name).(*types.TypeName)
		if !ok {
			continue
		}
		if !exported.MatchString(tn.Name()) {
			continue
		}
		switch x := tn.Type().Underlying().(type) {
		case *types.Interface:
			if x.NumMethods() == 0 {
				continue
			}
			c.addInterface(c.pkgIfaces, path, name, x)
		case *types.Signature:
			c.addFunc(x)
		}
	}
}

func typeList(t *types.Tuple) []types.Type {
	var l []types.Type
	for i := 0; i < t.Len(); i++ {
		v := t.At(i)
		l = append(l, v.Type())
	}
	return l
}
<commit_msg>Deduplicate func types<commit_after>// Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package main

import (
	"bytes"
	"go/importer"
	"go/types"
	"io"
	"regexp"
)

//go:generate go run generate/std/main.go generate/std/pkgs.go
//go:generate gofmt -w std.go

type funcSign struct {
	params  []types.Type
	results []types.Type
}

func (fs funcSign) String() string {
	var b bytes.Buffer
	io.WriteString(&b, "[")
	for _, t := range fs.params {
		io.WriteString(&b, t.String())
		io.WriteString(&b, ",")
	}
	io.WriteString(&b, "] [")
	for _, t := range fs.results {
		io.WriteString(&b, t.String())
		io.WriteString(&b, ",")
	}
	io.WriteString(&b, "]")
	return b.String()
}

type ifaceSign struct {
	name string
	t    *types.Interface

	funcs map[string]funcSign
}

type cache struct {
	done map[string]struct{}

	// key is importPath.typeName
	// TODO: do something about duplicates, especially to behave
	// deterministically if two keys map to equal ifaceSigns.
	stdIfaces map[string][]ifaceSign
	pkgIfaces map[string][]ifaceSign

	curPaths []string

	funcs map[string]funcSign
}

func typesInit() error {
	c = &cache{
		done:      make(map[string]struct{}),
		stdIfaces: make(map[string][]ifaceSign),
		pkgIfaces: make(map[string][]ifaceSign),
		funcs:     make(map[string]funcSign),
	}
	imp := importer.Default()
	for path, names := range pkgs {
		c.done[path] = struct{}{}
		if len(names) == 0 {
			continue
		}
		pkg, err := imp.Import(path)
		if err != nil {
			return err
		}
		c.grabNames(pkg.Scope(), path, names)
	}
	c.grabNames(types.Universe, "", []string{"error"})
	return nil
}

func (c *cache) grabNames(scope *types.Scope, path string, names []string) {
	for _, name := range names {
		tn := scope.Lookup(name).(*types.TypeName)
		switch x := tn.Type().Underlying().(type) {
		case *types.Interface:
			c.addInterface(c.stdIfaces, path, name, x)
		case *types.Signature:
			c.addFunc(x)
		}
	}
}

func (c *cache) addInterface(m map[string][]ifaceSign, path, name string, iface *types.Interface) {
	ifsign := ifaceSign{
		name:  name,
		t:     iface,
		funcs: make(map[string]funcSign, iface.NumMethods()),
	}
	for i := 0; i < iface.NumMethods(); i++ {
		f := iface.Method(i)
		sign := f.Type().(*types.Signature)
		ifsign.funcs[f.Name()] = c.addFunc(sign)
	}
	m[path] = append(m[path], ifsign)
}

func (c *cache) addFunc(sign *types.Signature) funcSign {
	fsign := funcSign{
		params:  typeList(sign.Params()),
		results: typeList(sign.Results()),
	}
	s := fsign.String()
	if _, e := c.funcs[s]; !e {
		c.funcs[s] = fsign
	}
	return fsign
}

var exported = regexp.MustCompile(`^[A-Z]`)

func (c *cache) grabExported(scope *types.Scope, path string) {
	c.done[path] = struct{}{}
	for _, name := range scope.Names() {
		tn, ok := scope.Lookup(name).(*types.TypeName)
		if !ok {
			continue
		}
		if !exported.MatchString(tn.Name()) {
			continue
		}
		switch x := tn.Type().Underlying().(type) {
		case *types.Interface:
			if x.NumMethods() == 0 {
				continue
			}
			c.addInterface(c.pkgIfaces, path, name, x)
		case *types.Signature:
			c.addFunc(x)
		}
	}
}

func typeList(t *types.Tuple) []types.Type {
	var l []types.Type
	for i := 0; i < t.Len(); i++ {
		v := t.At(i)
		l = append(l, v.Type())
	}
	return l
}
<|endoftext|>


<commit_before>package application

import (
	"cf"
	"cf/terminal"
	"fmt"
	"github.com/cloudfoundry/loggregatorlib/logmessage"
	"regexp"
	"strings"
	"time"
)

const (
	TIMESTAMP_FORMAT = "2006-01-02T15:04:05.00-0700"
)

func logMessageOutput(msg *logmessage.Message) string {
	logHeader, coloredLogHeader := extractLogHeader(msg)
	logMsg := msg.GetLogMessage()
	logContent := extractLogContent(logMsg, logHeader)

	return fmt.Sprintf("%s%s", coloredLogHeader, logContent)
}

func extractLogHeader(msg *logmessage.Message) (logHeader, coloredLogHeader string) {
	logMsg := msg.GetLogMessage()
	sourceType := msg.GetShortSourceTypeName()
	sourceId := logMsg.GetSourceId()
	t := time.Unix(0, logMsg.GetTimestamp())
	timeFormat := TIMESTAMP_FORMAT
	timeString := t.Format(timeFormat)

	logHeader = fmt.Sprintf("%s [%s]", timeString, sourceType)
	coloredLogHeader = terminal.LogSysHeaderColor(logHeader)

	if logMsg.GetSourceType() == logmessage.LogMessage_WARDEN_CONTAINER {
		logHeader = fmt.Sprintf("%s [%s/%s]", timeString, sourceType, sourceId)
		coloredLogHeader = terminal.LogAppHeaderColor(logHeader)
	}

	// Calculate padding
	longestHeader := fmt.Sprintf("%s  [API] ", timeFormat)
	expectedHeaderLength := len(longestHeader)
	padding := strings.Repeat(" ", expectedHeaderLength-len(logHeader))

	logHeader = logHeader + padding
	coloredLogHeader = coloredLogHeader + padding

	return
}

func extractLogContent(logMsg *logmessage.LogMessage, logHeader string) (logContent string) {
	msgText := string(logMsg.GetMessage())
	reg, err := regexp.Compile("[\n\r]+$")
	if err == nil {
		msgText = reg.ReplaceAllString(msgText, "")
	}

	msgLines := strings.Split(msgText, "\n")
	padding := strings.Repeat(" ", len(logHeader))
	coloringFunc := terminal.LogStdoutColor
	logType := "OUT"

	if logMsg.GetMessageType() == logmessage.LogMessage_ERR {
		coloringFunc = terminal.LogStderrColor
		logType = "ERR"
	}

	logContent = fmt.Sprintf("%s %s", logType, msgLines[0])
	for _, msgLine := range msgLines[1:] {
		logContent = fmt.Sprintf("%s\n%s%s", logContent, padding, msgLine)
	}
	logContent = coloringFunc(logContent)

	return
}

func envVarFound(varName string, existingEnvVars map[string]string) (found bool) {
	for name, _ := range existingEnvVars {
		if name == varName {
			found = true
			return
		}
	}
	return
}

func coloredAppState(app cf.Application) string {
	appState := strings.ToLower(app.State)

	if app.RunningInstances == 0 {
		if appState == "stopped" {
			return appState
		} else {
			return terminal.CrashedColor(appState)
		}
	}

	if app.RunningInstances < app.Instances {
		return terminal.WarningColor(appState)
	}

	return appState
}

func coloredAppInstaces(app cf.Application) string {
	healthString := fmt.Sprintf("%d/%d", app.RunningInstances, app.Instances)

	if app.RunningInstances == 0 {
		if strings.ToLower(app.State) == "stopped" {
			return healthString
		} else {
			return terminal.CrashedColor(healthString)
		}
	}

	if app.RunningInstances < app.Instances {
		return terminal.WarningColor(healthString)
	}

	return healthString
}

func coloredInstanceState(instance cf.ApplicationInstance) (colored string) {
	state := string(instance.State)
	switch state {
	case "started", "running":
		colored = terminal.StartedColor("running")
	case "stopped":
		colored = terminal.StoppedColor("stopped")
	case "flapping":
		colored = terminal.WarningColor("crashing")
	case "starting":
		colored = terminal.AdvisoryColor("starting")
	default:
		colored = terminal.FailureColor(state)
	}

	return
}
<commit_msg>Make loggregator jobs have 5 spaces around the short source type name [#59295822]<commit_after>package application

import (
	"cf"
	"cf/terminal"
	"fmt"
	"github.com/cloudfoundry/loggregatorlib/logmessage"
	"regexp"
	"strings"
	"time"
)

const (
	TIMESTAMP_FORMAT = "2006-01-02T15:04:05.00-0700"
)

func logMessageOutput(msg *logmessage.Message) string {
	logHeader, coloredLogHeader := extractLogHeader(msg)
	logMsg := msg.GetLogMessage()
	logContent := extractLogContent(logMsg, logHeader)

	return fmt.Sprintf("%s%s", coloredLogHeader, logContent)
}

func extractLogHeader(msg *logmessage.Message) (logHeader, coloredLogHeader string) {
	logMsg := msg.GetLogMessage()
	sourceType := msg.GetShortSourceTypeName()
	sourceId := logMsg.GetSourceId()
	t := time.Unix(0, logMsg.GetTimestamp())
	timeFormat := TIMESTAMP_FORMAT
	timeString := t.Format(timeFormat)

	logHeader = fmt.Sprintf("%s [%s]", timeString, sourceType)
	coloredLogHeader = terminal.LogSysHeaderColor(logHeader)

	if logMsg.GetSourceType() == logmessage.LogMessage_WARDEN_CONTAINER {
		logHeader = fmt.Sprintf("%s [%s/%s]", timeString, sourceType, sourceId)
		coloredLogHeader = terminal.LogAppHeaderColor(logHeader)
	}

	// Calculate padding
	longestHeader := fmt.Sprintf("%s  [App/0]  ", timeFormat)
	expectedHeaderLength := len(longestHeader)
	padding := strings.Repeat(" ", expectedHeaderLength-len(logHeader))

	logHeader = logHeader + padding
	coloredLogHeader = coloredLogHeader + padding

	return
}

func extractLogContent(logMsg *logmessage.LogMessage, logHeader string) (logContent string) {
	msgText := string(logMsg.GetMessage())
	reg, err := regexp.Compile("[\n\r]+$")
	if err == nil {
		msgText = reg.ReplaceAllString(msgText, "")
	}

	msgLines := strings.Split(msgText, "\n")
	padding := strings.Repeat(" ", len(logHeader))
	coloringFunc := terminal.LogStdoutColor
	logType := "OUT"

	if logMsg.GetMessageType() == logmessage.LogMessage_ERR {
		coloringFunc = terminal.LogStderrColor
		logType = "ERR"
	}

	logContent = fmt.Sprintf("%s %s", logType, msgLines[0])
	for _, msgLine := range msgLines[1:] {
		logContent = fmt.Sprintf("%s\n%s%s", logContent, padding, msgLine)
	}
	logContent = coloringFunc(logContent)

	return
}

func envVarFound(varName string, existingEnvVars map[string]string) (found bool) {
	for name, _ := range existingEnvVars {
		if name == varName {
			found = true
			return
		}
	}
	return
}

func coloredAppState(app cf.Application) string {
	appState := strings.ToLower(app.State)

	if app.RunningInstances == 0 {
		if appState == "stopped" {
			return appState
		} else {
			return terminal.CrashedColor(appState)
		}
	}

	if app.RunningInstances < app.Instances {
		return terminal.WarningColor(appState)
	}

	return appState
}

func coloredAppInstaces(app cf.Application) string {
	healthString := fmt.Sprintf("%d/%d", app.RunningInstances, app.Instances)

	if app.RunningInstances == 0 {
		if strings.ToLower(app.State) == "stopped" {
			return healthString
		} else {
			return terminal.CrashedColor(healthString)
		}
	}

	if app.RunningInstances < app.Instances {
		return terminal.WarningColor(healthString)
	}

	return healthString
}

func coloredInstanceState(instance cf.ApplicationInstance) (colored string) {
	state := string(instance.State)
	switch state {
	case "started", "running":
		colored = terminal.StartedColor("running")
	case "stopped":
		colored = terminal.StoppedColor("stopped")
	case "flapping":
		colored = terminal.WarningColor("crashing")
	case "starting":
		colored = terminal.AdvisoryColor("starting")
	default:
		colored = terminal.FailureColor(state)
	}

	return
}
<|endoftext|>


<commit_before>// types.go

// This file contains the various types used by the API

package atlas

import (
	"log"
	"net/http"
)

// Client is the main struct holding state in an API client
type Client struct {
	config Config
	client *http.Client
	log    *log.Logger
	opts   map[string]string // Default, optional options
}

// Config is the main object when creating an API Client
type Config struct {
	APIKey       string
	DefaultProbe int
	AreaType     string
	AreaValue    string
	IsOneOff     bool
	PoolSize     int
	WantAF       string
	ProxyAuth    string
	Verbose      bool
	Tags         string
	Log          *log.Logger
}

// APIError is for errors returned by the RIPE API.
type APIError struct {
	Error struct {
		Status int    `json:"status"`
		Code   int    `json:"code"`
		Detail string `json:"detail"`
		Title  string `json:"title"`
		Errors []struct {
			Source struct {
				Pointer string
			} `json:"source"`
			Detail string
		} `json:"errors"`
	} `json:"error"`
}

// Key is holding the API key parameters
type Key struct {
	UUID      string `json:"uuid"`
	ValidFrom string `json:"valid_from"`
	ValidTo   string `json:"valid_to"`
	Enabled   bool
	IsActive  bool    `json:"is_active"`
	CreatedAt string  `json:"created_at"`
	Label     string  `json:"label"`
	Grants    []Grant `json:"grants"`
	Type      string  `json:"type"`
}

// Grant is the permission(s) associated with a key
type Grant struct {
	Permission string `json:"permission"`
	Target     struct {
		Type string `json:"type"`
		ID   string `json:"id"`
	} `json:"target"`
}

// Credits is holding credits data
type Credits struct {
	CurrentBalance            int    `json:"current_balance"`
	EstimatedDailyIncome      int    `json:"estimated_daily_income"`
	EstimatedDailyExpenditure int    `json:"estimated_daily_expenditure"`
	EstimatedDailyBalance     int    `json:"estimated_daily_balance"`
	CalculationTime           string `json:"calculation_time"`
	EstimatedRunoutSeconds    int    `json:"estimated_runout_seconds"`
	PastDayMeasurementResults int    `json:"past_day_measurement_results"`
	PastDayCreditsSpent       int    `json:"past_day_credits_spent"`
	IncomeItems               string `json:"income_items"`
	ExpenseItems              string `json:"expense_items"`
	Transactions              string `json:"transactions"`
}

// Probe is holding probe's data
type Probe struct {
	AddressV4      string `json:"address_v4"`
	AddressV6      string `json:"address_v6"`
	AsnV4          int    `json:"asn_v4"`
	AsnV6          int    `json:"asn_v6"`
	CountryCode    string `json:"country_code"`
	Description    string `json:"description"`
	FirstConnected int    `json:"first_connected"`
	Geometry       struct {
		Type        string    `json:"type"`
		Coordinates []float64 `json:"coordinates"`
	} `json:"geometry"`
	ID            int    `json:"id"`
	IsAnchor      bool   `json:"is_anchor"`
	IsPublic      bool   `json:"is_public"`
	LastConnected int    `json:"last_connected"`
	PrefixV4      string `json:"prefix_v4"`
	PrefixV6      string `json:"prefix_v6"`
	Status        struct {
		Since string `json:"since"`
		ID    int    `json:"id"`
		Name  string `json:"name"`
	} `json:"status"`
	StatusSince int `json:"status_since"`
	Tags        []struct {
		Name string `json:"name"`
		Slug string `json:"slug"`
	} `json:"tags"`
	Type string `json:"type"`
}

// Measurement is what we are working with
type Measurement struct {
	Af                    int                    `json:"af"`
	CreationTime          int                    `json:"creation_time"`
	Description           string                 `json:"description"`
	DestinationOptionSize interface{}            `json:"destination_option_size"`
	DontFragment          interface{}            `json:"dont_fragment"`
	DuplicateTimeout      interface{}            `json:"duplicate_timeout"`
	FirstHop              int                    `json:"first_hop"`
	Group                 string                 `json:"group"`
	GroupID               int                    `json:"group_id"`
	HopByHopOptionSize    interface{}            `json:"hop_by_hop_option_size"`
	ID                    int                    `json:"id"`
	InWifiGroup           bool                   `json:"in_wifi_group"`
	Interval              int                    `json:"interval"`
	IsAllScheduled        bool                   `json:"is_all_scheduled"`
	IsOneoff              bool                   `json:"is_oneoff"`
	IsPublic              bool                   `json:"is_public"`
	MaxHops               int                    `json:"max_hops"`
	PacketInterval        interface{}            `json:"packet_interval"`
	Packets               int                    `json:"packets"`
	Paris                 int                    `json:"paris"`
	ParticipantCount      int                    `json:"participant_count"`
	ParticipationRequests []ParticipationRequest `json:"participation_requests"`
	Port                  interface{}            `json:"port"`
	ProbesRequested       int                    `json:"probes_requested"`
	ProbesScheduled       int                    `json:"probes_scheduled"`
	Protocol              string                 `json:"protocol"`
	ResolveOnProbe        bool                   `json:"resolve_on_probe"`
	ResolvedIPs           []string               `json:"resolved_ips"`
	ResponseTimeout       int                    `json:"response_timeout"`
	Result                string                 `json:"result"`
	Size                  int                    `json:"size"`
	Spread                interface{}            `json:"spread"`
	StartTime             int                    `json:"start_time"`
	Status                struct {
		ID   int    `json:"id"`
		Name string `json:"name"`
	} `json:"status"`
	StopTime  int    `json:"stop_time"`
	Target    string `json:"target"`
	TargetASN int    `json:"target_asn"`
	TargetIP  string `json:"target_ip"`
	Type      string `json:"type"`
}

// ParticipationRequest allow you to add or remove probes from a measurement that
// was already created
type ParticipationRequest struct {
	Action        string `json:"action"`
	CreatedAt     int    `json:"created_at,omitempty"`
	ID            int    `json:"id,omitempty"`
	Self          string `json:"self,omitempty"`
	Measurement   string `json:"measurement,omitempty"`
	MeasurementID int    `json:"measurement_id,omitempty"`
	Requested     int    `json:"requested,omitempty"`
	Type          string `json:"type,omitempty"`
	Value         string `json:"value,omitempty"`
	Logs          string `json:"logs,omitempty"`
}

var (
	// ProbeTypes should be obvious
	ProbeTypes = []string{"area", "country", "prefix", "asn", "probes", "msm"}
	// AreaTypes should also be obvious
	AreaTypes = []string{"WW", "West", "North-Central", "South-Central", "North-East", "South-East"}
)

// MeasurementRequest contains the different measurement to create/view
type MeasurementRequest struct {
	// see below for definition
	Definitions []Definition `json:"definitions"`

	// requested set of probes
	Probes []ProbeSet `json:"probes"`
	//
	BillTo       int  `json:"bill_to,omitempty"`
	IsOneoff     bool `json:"is_oneoff,omitempty"`
	SkipDNSCheck bool `json:"skip_dns_check,omitempty"`
	Times        int  `json:"times,omitempty"`
	StartTime    int  `json:"start_time,omitempty"`
	StopTime     int  `json:"stop_time,omitempty"`
}

// ProbeSet is a set of probes obviously
type ProbeSet struct {
	Requested   int    `json:"requested"` // number of probes
	Type        string `json:"type"`      // area, country, prefix, asn, probes, msm
	Value       string `json:"value"`     // can be numeric or string
	TagsInclude string `json:"tags_include,omitempty"`
	TagsExclude string `json:"tags_exclude,omitempty"`
}

// Definition is used to create measurements
type Definition struct {
	// Required fields
	Description string `json:"description"`
	Type        string `json:"type"`
	AF          int    `json:"af"`

	// Required for all but "dns"
	Target string `json:"target,omitempty"`

	GroupID        int    `json:"group_id,omitempty"`
	Group          string `json:"group,omitempty"`
	InWifiGroup    bool   `json:"in_wifi_group,omitempty"`
	Spread         int    `json:"spread,omitempty"`
	Packets        int    `json:"packets,omitempty"`
	PacketInterval int    `json:"packet_interval,omitempty"`

	// Common parameters
	ExtraWait      int  `json:"extra_wait,omitempty"`
	IsOneoff       bool `json:"is_oneoff,omitempty"`
	IsPublic       bool `json:"is_public,omitempty"`
	ResolveOnProbe bool `json:"resolve_on_probe,omitempty"`

	// Default depends on type
	Interval int `json:"interval,omitempty"`

	// dns & traceroute parameters
	Protocol string `json:"protocol,omitempty"`

	// dns parameters
	QueryClass       string `json:"query_class,omitempty"`
	QueryType        string `json:"query_type,omitempty"`
	QueryArgument    string `json:"query_argument,omitempty"`
	Retry            int    `json:"retry,omitempty"`
	SetCDBit         bool   `json:"set_cd_bit,omitempty"`
	SetDOBit         bool   `json:"set_do_bit,omitempty"`
	SetNSIDBit       bool   `json:"set_nsid_bit,omitempty"`
	SetRDBit         bool   `json:"set_rd_bit,omitempty"`
	UDPPayloadSize   int    `json:"udp_payload_size,omitempty"`
	UseProbeResolver bool   `json:"use_probe_resolver"`

	// ping parameters
	//   none (see target)

	// traceroute parameters
	DestinationOptionSize int  `json:"destination_option_size,omitempty"`
	DontFragment          bool `json:"dont_fragment,omitempty"`
	DuplicateTimeout      int  `json:"duplicate_timeout,omitempty"`
	FirstHop              int  `json:"first_hop,omitempty"`
	HopByHopOptionSize    int  `json:"hop_by_hop_option_size,omitempty"`
	MaxHops               int  `json:"max_hops,omitempty"`
	Paris                 int  `json:"paris,omitempty"`

	// ntp parameters
	//   none (see target)

	// http parameters
	ExtendedTiming     bool   `json:"extended_timing,omitempty"`
	HeaderBytes        int    `json:"header_bytes,omitempty"`
	Method             string `json:"method,omitempty"`
	MoreExtendedTiming bool   `json:"more_extended_timing,omitempty"`
	Path               string `json:"path,omitempty"`
	QueryOptions       string `json:"query_options,omitempty"`
	UserAgent          string `json:"user_agent,omitempty"`
	Version            string `json:"version,omitempty"`

	// sslcert parameters
	//   none (see target)

	// sslcert & traceroute & http parameters
	Port int `json:"port,omitempty"`

	// ping & traceroute parameters
	Size int `json:"size,omitempty"`

	// wifi parameters
	AnonymousIdentity string `json:"anonymous_identity,omitempty"`
	Cert              string `json:"cert,omitempty"`
	EAP               string `json:"eap,omitempty"`
}
<commit_msg>Create a private "endpoint" field to help upcoming mocking.<commit_after>// types.go

// This file contains the various types used by the API

package atlas

import (
	"log"
	"net/http"
)

// Client is the main struct holding state in an API client
type Client struct {
	config Config
	client *http.Client
	log    *log.Logger
	opts   map[string]string // Default, optional options
}

// Config is the main object when creating an API Client
type Config struct {
	endpoint     string
	APIKey       string
	DefaultProbe int
	AreaType     string
	AreaValue    string
	IsOneOff     bool
	PoolSize     int
	WantAF       string
	ProxyAuth    string
	Verbose      bool
	Tags         string
	Log          *log.Logger
}

// APIError is for errors returned by the RIPE API.
type APIError struct {
	Error struct {
		Status int    `json:"status"`
		Code   int    `json:"code"`
		Detail string `json:"detail"`
		Title  string `json:"title"`
		Errors []struct {
			Source struct {
				Pointer string
			} `json:"source"`
			Detail string
		} `json:"errors"`
	} `json:"error"`
}

// Key is holding the API key parameters
type Key struct {
	UUID      string `json:"uuid"`
	ValidFrom string `json:"valid_from"`
	ValidTo   string `json:"valid_to"`
	Enabled   bool
	IsActive  bool    `json:"is_active"`
	CreatedAt string  `json:"created_at"`
	Label     string  `json:"label"`
	Grants    []Grant `json:"grants"`
	Type      string  `json:"type"`
}

// Grant is the permission(s) associated with a key
type Grant struct {
	Permission string `json:"permission"`
	Target     struct {
		Type string `json:"type"`
		ID   string `json:"id"`
	} `json:"target"`
}

// Credits is holding credits data
type Credits struct {
	CurrentBalance            int    `json:"current_balance"`
	EstimatedDailyIncome      int    `json:"estimated_daily_income"`
	EstimatedDailyExpenditure int    `json:"estimated_daily_expenditure"`
	EstimatedDailyBalance     int    `json:"estimated_daily_balance"`
	CalculationTime           string `json:"calculation_time"`
	EstimatedRunoutSeconds    int    `json:"estimated_runout_seconds"`
	PastDayMeasurementResults int    `json:"past_day_measurement_results"`
	PastDayCreditsSpent       int    `json:"past_day_credits_spent"`
	IncomeItems               string `json:"income_items"`
	ExpenseItems              string `json:"expense_items"`
	Transactions              string `json:"transactions"`
}

// Probe is holding probe's data
type Probe struct {
	AddressV4      string `json:"address_v4"`
	AddressV6      string `json:"address_v6"`
	AsnV4          int    `json:"asn_v4"`
	AsnV6          int    `json:"asn_v6"`
	CountryCode    string `json:"country_code"`
	Description    string `json:"description"`
	FirstConnected int    `json:"first_connected"`
	Geometry       struct {
		Type        string    `json:"type"`
		Coordinates []float64 `json:"coordinates"`
	} `json:"geometry"`
	ID            int    `json:"id"`
	IsAnchor      bool   `json:"is_anchor"`
	IsPublic      bool   `json:"is_public"`
	LastConnected int    `json:"last_connected"`
	PrefixV4      string `json:"prefix_v4"`
	PrefixV6      string `json:"prefix_v6"`
	Status        struct {
		Since string `json:"since"`
		ID    int    `json:"id"`
		Name  string `json:"name"`
	} `json:"status"`
	StatusSince int `json:"status_since"`
	Tags        []struct {
		Name string `json:"name"`
		Slug string `json:"slug"`
	} `json:"tags"`
	Type string `json:"type"`
}

// Measurement is what we are working with
type Measurement struct {
	Af                    int                    `json:"af"`
	CreationTime          int                    `json:"creation_time"`
	Description           string                 `json:"description"`
	DestinationOptionSize interface{}            `json:"destination_option_size"`
	DontFragment          interface{}            `json:"dont_fragment"`
	DuplicateTimeout      interface{}            `json:"duplicate_timeout"`
	FirstHop              int                    `json:"first_hop"`
	Group                 string                 `json:"group"`
	GroupID               int                    `json:"group_id"`
	HopByHopOptionSize    interface{}            `json:"hop_by_hop_option_size"`
	ID                    int                    `json:"id"`
	InWifiGroup           bool                   `json:"in_wifi_group"`
	Interval              int                    `json:"interval"`
	IsAllScheduled        bool                   `json:"is_all_scheduled"`
	IsOneoff              bool                   `json:"is_oneoff"`
	IsPublic              bool                   `json:"is_public"`
	MaxHops               int                    `json:"max_hops"`
	PacketInterval        interface{}            `json:"packet_interval"`
	Packets               int                    `json:"packets"`
	Paris                 int                    `json:"paris"`
	ParticipantCount      int                    `json:"participant_count"`
	ParticipationRequests []ParticipationRequest `json:"participation_requests"`
	Port                  interface{}            `json:"port"`
	ProbesRequested       int                    `json:"probes_requested"`
	ProbesScheduled       int                    `json:"probes_scheduled"`
	Protocol              string                 `json:"protocol"`
	ResolveOnProbe        bool                   `json:"resolve_on_probe"`
	ResolvedIPs           []string               `json:"resolved_ips"`
	ResponseTimeout       int                    `json:"response_timeout"`
	Result                string                 `json:"result"`
	Size                  int                    `json:"size"`
	Spread                interface{}            `json:"spread"`
	StartTime             int                    `json:"start_time"`
	Status                struct {
		ID   int    `json:"id"`
		Name string `json:"name"`
	} `json:"status"`
	StopTime  int    `json:"stop_time"`
	Target    string `json:"target"`
	TargetASN int    `json:"target_asn"`
	TargetIP  string `json:"target_ip"`
	Type      string `json:"type"`
}

// ParticipationRequest allow you to add or remove probes from a measurement that
// was already created
type ParticipationRequest struct {
	Action        string `json:"action"`
	CreatedAt     int    `json:"created_at,omitempty"`
	ID            int    `json:"id,omitempty"`
	Self          string `json:"self,omitempty"`
	Measurement   string `json:"measurement,omitempty"`
	MeasurementID int    `json:"measurement_id,omitempty"`
	Requested     int    `json:"requested,omitempty"`
	Type          string `json:"type,omitempty"`
	Value         string `json:"value,omitempty"`
	Logs          string `json:"logs,omitempty"`
}

var (
	// ProbeTypes should be obvious
	ProbeTypes = []string{"area", "country", "prefix", "asn", "probes", "msm"}
	// AreaTypes should also be obvious
	AreaTypes = []string{"WW", "West", "North-Central", "South-Central", "North-East", "South-East"}
)

// MeasurementRequest contains the different measurement to create/view
type MeasurementRequest struct {
	// see below for definition
	Definitions []Definition `json:"definitions"`

	// requested set of probes
	Probes []ProbeSet `json:"probes"`
	//
	BillTo       int  `json:"bill_to,omitempty"`
	IsOneoff     bool `json:"is_oneoff,omitempty"`
	SkipDNSCheck bool `json:"skip_dns_check,omitempty"`
	Times        int  `json:"times,omitempty"`
	StartTime    int  `json:"start_time,omitempty"`
	StopTime     int  `json:"stop_time,omitempty"`
}

// ProbeSet is a set of probes obviously
type ProbeSet struct {
	Requested   int    `json:"requested"` // number of probes
	Type        string `json:"type"`      // area, country, prefix, asn, probes, msm
	Value       string `json:"value"`     // can be numeric or string
	TagsInclude string `json:"tags_include,omitempty"`
	TagsExclude string `json:"tags_exclude,omitempty"`
}

// Definition is used to create measurements
type Definition struct {
	// Required fields
	Description string `json:"description"`
	Type        string `json:"type"`
	AF          int    `json:"af"`

	// Required for all but "dns"
	Target string `json:"target,omitempty"`

	GroupID        int    `json:"group_id,omitempty"`
	Group          string `json:"group,omitempty"`
	InWifiGroup    bool   `json:"in_wifi_group,omitempty"`
	Spread         int    `json:"spread,omitempty"`
	Packets        int    `json:"packets,omitempty"`
	PacketInterval int    `json:"packet_interval,omitempty"`

	// Common parameters
	ExtraWait      int  `json:"extra_wait,omitempty"`
	IsOneoff       bool `json:"is_oneoff,omitempty"`
	IsPublic       bool `json:"is_public,omitempty"`
	ResolveOnProbe bool `json:"resolve_on_probe,omitempty"`

	// Default depends on type
	Interval int `json:"interval,omitempty"`

	// dns & traceroute parameters
	Protocol string `json:"protocol,omitempty"`

	// dns parameters
	QueryClass       string `json:"query_class,omitempty"`
	QueryType        string `json:"query_type,omitempty"`
	QueryArgument    string `json:"query_argument,omitempty"`
	Retry            int    `json:"retry,omitempty"`
	SetCDBit         bool   `json:"set_cd_bit,omitempty"`
	SetDOBit         bool   `json:"set_do_bit,omitempty"`
	SetNSIDBit       bool   `json:"set_nsid_bit,omitempty"`
	SetRDBit         bool   `json:"set_rd_bit,omitempty"`
	UDPPayloadSize   int    `json:"udp_payload_size,omitempty"`
	UseProbeResolver bool   `json:"use_probe_resolver"`

	// ping parameters
	//   none (see target)

	// traceroute parameters
	DestinationOptionSize int  `json:"destination_option_size,omitempty"`
	DontFragment          bool `json:"dont_fragment,omitempty"`
	DuplicateTimeout      int  `json:"duplicate_timeout,omitempty"`
	FirstHop              int  `json:"first_hop,omitempty"`
	HopByHopOptionSize    int  `json:"hop_by_hop_option_size,omitempty"`
	MaxHops               int  `json:"max_hops,omitempty"`
	Paris                 int  `json:"paris,omitempty"`

	// ntp parameters
	//   none (see target)

	// http parameters
	ExtendedTiming     bool   `json:"extended_timing,omitempty"`
	HeaderBytes        int    `json:"header_bytes,omitempty"`
	Method             string `json:"method,omitempty"`
	MoreExtendedTiming bool   `json:"more_extended_timing,omitempty"`
	Path               string `json:"path,omitempty"`
	QueryOptions       string `json:"query_options,omitempty"`
	UserAgent          string `json:"user_agent,omitempty"`
	Version            string `json:"version,omitempty"`

	// sslcert parameters
	//   none (see target)

	// sslcert & traceroute & http parameters
	Port int `json:"port,omitempty"`

	// ping & traceroute parameters
	Size int `json:"size,omitempty"`

	// wifi parameters
	AnonymousIdentity string `json:"anonymous_identity,omitempty"`
	Cert              string `json:"cert,omitempty"`
	EAP               string `json:"eap,omitempty"`
}
<|endoftext|>


<commit_before>package hangoutjson

type ID struct {
	ID			string		`json:"ID"`
}

type ResponseHeader struct {
	Status			string		`json:"status"`
	DebugURL		string		`json:"debug_url"`
	RequestTraceID		string		`json:"request_trace_id"`
	CurrentServerTime	string		`json:"current_server_time"`
	BuildLabel		string		`json:"build_label"`
	ChangelistNumber	int		`json:"changelist_number"`
}

type ParticipantID struct {
	GaiaID			string		`json:"gaia_id"`
	ChatID			string		`json:"chat_id"`
}

type SelfReadState struct {
	ParticipantID		ID		`json:"participant_id"`
	LatestReadTimestamp	string		`json:"latest_read_timestamp"`
}

type DeliveryMedium struct {
	MediumType		string		`json:"medium_type"`
}

type DeliveryMediumOption []struct {
	DeliveryMedium		DeliveryMedium	`json:"delivery_medium"`
	CurrentDefault		bool		`json:"current_default"`
}
type SelfConversationState struct {
	SelfReadState		SelfReadState		`json:"self_read_state"`
	Status			string			`json:"status"`
	NotificationLevel	string			`json:"notification_level"`
	View			[]string		`json:"view"`
	InviterID		ID			`json:"inviter_id"`
	InviteTimestamp		string			`json:"invite_timestamp"`
	SortTimestamp		string			`json:"sort_timestamp"`
	ActiveTimestamp		string			`json:"active_timestamp"`
	DeliveryMediumOption	DeliveryMediumOption	`json:"delivery_medium_option"`
	IsGuest bool `json:"is_guest"`
}

type ParticipantDataT []struct {
	ID ParticipantID `json:"id"`
	FallbackName        string	`json:"fallback_name"`
	InvitationStatus    string	`json:"invitation_status"`
	ParticipantType     string	`json:"participant_type"`
	NewInvitationStatus string	`json:"new_invitation_status"`
}

type Conversation struct {
	ID                     ID `json:"id"`
	Type                   string `json:"type"`
	SelfConversationState  `json:"self_conversation_state"`
	ReadState              []SelfReadState `json:"read_state"`
	HasActiveHangout       bool            `json:"has_active_hangout"`
	OtrStatus              string          `json:"otr_status"`
	OtrToggle              string          `json:"otr_toggle"`
	CurrentParticipant     []ParticipantID `json:"current_participant"`
	ParticipantData        ParticipantDataT `json:"participant_data"`
	ForkOnExternalInvite   bool     `json:"fork_on_external_invite"`
	NetworkType            []string `json:"network_type"`
	ForceHistoryState      string   `json:"force_history_state"`
	GroupLinkSharingStatus string   `json:"group_link_sharing_status"`
}

type Formatting struct {
	Bold		bool	`json:"bold"`
	Italics		bool	`json:"italics"`
	Strikethrough	bool	`json:"strikethrough"`
	Underline	bool	`json:"underline"`
}

type Segment []struct {
	Type		string	`json:"type"`
	Text		string	`json:"text"`
	Formatting		`json:"formatting"`
}

type MessageContent struct {
	Segment		Segment	`json:"segment"`
}

type ChatMessage struct {
	MessageContent	MessageContent	`json:"message_content"`
}

type MembershipChange struct {
	Type		string		`json:"type"`
	ParticipantID	[]ParticipantID	`json:"participant_id"`
	LeaveReason	string		`json:"leave_reason"`
}

type SelfEventState struct {
	UserID			ID	`json:"user_id"`
	NotificationLevel	string	`json:"notification_level"`
}

type Event struct {
	ConversationID		ID			`json:"conversation_id"`
	SenderID		ParticipantID		`json:"sender_id"`
	Timestamp		string			`json:"timestamp"`
	SelfEventState		SelfEventState		`json:"self_event_state"`
	ChatMessage		ChatMessage		`json:"chat_message,omitempty"`
	EventID			string			`json:"event_id"`
	AdvancesSortTimestamp	bool			`json:"advances_sort_timestamp"`
	EventOtr		string			`json:"event_otr"`
	DeliveryMedium		DeliveryMedium		`json:"delivery_medium"`
	EventType		string			`json:"event_type"`
	EventVersion		string			`json:"event_version"`
	MembershipChange	MembershipChange	`json:"membership_change,omitempty"`
}

type ConversationState struct {
	ConversationID ID           `json:"conversation_id"`
	Conversation   Conversation `json:"conversation"`
	Event          []Event      `json:"event"`
}

type ConversationHeader struct {
	ConversationID    ID			`json:"conversation_id"`
	ResponseHeader    ResponseHeader	`json:"response_header"`
	ConversationState ConversationState	`json:"conversation_state"`
}

type Hangouts struct {
	ContinuationEndTimestamp	string			`json:"continuation_end_timestamp"`
	ConversationState		[]ConversationHeader	`json:"conversation_state"`
}

<commit_msg>Fix some spacing for clarity. Change ParticipantDataT to ParticipantData to match other self-named types.<commit_after>package hangoutjson

type ID struct {
	ID			string		`json:"ID"`
}

type ResponseHeader struct {
	Status			string		`json:"status"`
	DebugURL		string		`json:"debug_url"`
	RequestTraceID		string		`json:"request_trace_id"`
	CurrentServerTime	string		`json:"current_server_time"`
	BuildLabel		string		`json:"build_label"`
	ChangelistNumber	int		`json:"changelist_number"`
}

type ParticipantID struct {
	GaiaID			string		`json:"gaia_id"`
	ChatID			string		`json:"chat_id"`
}

type SelfReadState struct {
	ParticipantID		ID		`json:"participant_id"`
	LatestReadTimestamp	string		`json:"latest_read_timestamp"`
}

type DeliveryMedium struct {
	MediumType		string		`json:"medium_type"`
}

type DeliveryMediumOption []struct {
	DeliveryMedium		DeliveryMedium	`json:"delivery_medium"`
	CurrentDefault		bool		`json:"current_default"`
}
type SelfConversationState struct {
	SelfReadState		SelfReadState		`json:"self_read_state"`
	Status			string			`json:"status"`
	NotificationLevel	string			`json:"notification_level"`
	View			[]string		`json:"view"`
	InviterID		ID			`json:"inviter_id"`
	InviteTimestamp		string			`json:"invite_timestamp"`
	SortTimestamp		string			`json:"sort_timestamp"`
	ActiveTimestamp		string			`json:"active_timestamp"`
	DeliveryMediumOption	DeliveryMediumOption	`json:"delivery_medium_option"`
	IsGuest bool `json:"is_guest"`
}

type ParticipantData []struct {
	ID			ParticipantID	`json:"id"`
	FallbackName		string		`json:"fallback_name"`
	InvitationStatus	string		`json:"invitation_status"`
	ParticipantType		string		`json:"participant_type"`
	NewInvitationStatus	string		`json:"new_invitation_status"`
}

type Conversation struct {
	ID                     ID		`json:"id"`
	Type                   string		`json:"type"`
	SelfConversationState			`json:"self_conversation_state"`
	ReadState              []SelfReadState	`json:"read_state"`
	HasActiveHangout       bool		`json:"has_active_hangout"`
	OtrStatus              string		`json:"otr_status"`
	OtrToggle              string		`json:"otr_toggle"`
	CurrentParticipant     []ParticipantID	`json:"current_participant"`
	ParticipantData        ParticipantData	`json:"participant_data"`
	ForkOnExternalInvite   bool		`json:"fork_on_external_invite"`
	NetworkType            []string		`json:"network_type"`
	ForceHistoryState      string		`json:"force_history_state"`
	GroupLinkSharingStatus string		`json:"group_link_sharing_status"`
}

type Formatting struct {
	Bold		bool	`json:"bold"`
	Italics		bool	`json:"italics"`
	Strikethrough	bool	`json:"strikethrough"`
	Underline	bool	`json:"underline"`
}

type Segment []struct {
	Type		string	`json:"type"`
	Text		string	`json:"text"`
	Formatting		`json:"formatting"`
}

type MessageContent struct {
	Segment		Segment	`json:"segment"`
}

type ChatMessage struct {
	MessageContent	MessageContent	`json:"message_content"`
}

type MembershipChange struct {
	Type		string		`json:"type"`
	ParticipantID	[]ParticipantID	`json:"participant_id"`
	LeaveReason	string		`json:"leave_reason"`
}

type SelfEventState struct {
	UserID			ID	`json:"user_id"`
	NotificationLevel	string	`json:"notification_level"`
}

type Event struct {
	ConversationID		ID			`json:"conversation_id"`
	SenderID		ParticipantID		`json:"sender_id"`
	Timestamp		string			`json:"timestamp"`
	SelfEventState		SelfEventState		`json:"self_event_state"`
	ChatMessage		ChatMessage		`json:"chat_message,omitempty"`
	EventID			string			`json:"event_id"`
	AdvancesSortTimestamp	bool			`json:"advances_sort_timestamp"`
	EventOtr		string			`json:"event_otr"`
	DeliveryMedium		DeliveryMedium		`json:"delivery_medium"`
	EventType		string			`json:"event_type"`
	EventVersion		string			`json:"event_version"`
	MembershipChange	MembershipChange	`json:"membership_change,omitempty"`
}

type ConversationState struct {
	ConversationID ID           `json:"conversation_id"`
	Conversation   Conversation `json:"conversation"`
	Event          []Event      `json:"event"`
}

type ConversationHeader struct {
	ConversationID    ID			`json:"conversation_id"`
	ResponseHeader    ResponseHeader	`json:"response_header"`
	ConversationState ConversationState	`json:"conversation_state"`
}

type Hangouts struct {
	ContinuationEndTimestamp	string			`json:"continuation_end_timestamp"`
	ConversationState		[]ConversationHeader	`json:"conversation_state"`
}

<|endoftext|>


<commit_before>/*
 * Copyright 2015 Fabrício Godoy
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package http

import (
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/skarllot/raiqub"
)

const (
	DEFAULT_CORS_PREFLIGHT_METHOD = "OPTIONS"
	DEFAULT_CORS_MAX_AGE          = time.Hour * 24 / time.Second
	DEFAULT_CORS_METHODS          = "OPTIONS, GET, HEAD, POST, PUT, DELETE, TRACE, CONNECT"
	DEFAULT_CORS_ORIGIN           = "*"
)

type CORSHandler struct {
	PredicateOrigin raiqub.PredicateStringFunc
	Headers         []string
	ExposedHeaders  []string
}

func NewCORSHandler() *CORSHandler {
	return &CORSHandler{
		PredicateOrigin: raiqub.TrueForAll,
		Headers: []string{
			"Origin", "X-Requested-With", "Content-Type",
			"Accept", "Authorization",
		},
		ExposedHeaders: make([]string, 0),
	}
}

func (s *CORSHandler) CreateOptionsRoutes(routes Routes) Routes {
	list := make(Routes, 0, len(routes))
	hList := make(map[string]*CORSPreflight, len(routes))
	for _, v := range routes {
		preflight, ok := hList[v.Path]
		if !ok {
			preflight = &CORSPreflight{
				*s,
				make([]string, 0, 1),
				v.MustAuth,
			}
			hList[v.Path] = preflight
		}
		preflight.Methods = append(preflight.Methods, v.Method)
	}

	for k, v := range hList {
		list = append(list, Route{
			Name:       "",
			Method:     DEFAULT_CORS_PREFLIGHT_METHOD,
			Path:       k,
			MustAuth:   v.UseCredentials,
			ActionFunc: v.ServeHTTP,
		})
	}
	return list
}

type CORSPreflight struct {
	CORSHandler
	Methods        []string
	UseCredentials bool
}

func (s *CORSPreflight) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	origin := HttpHeader_Origin().GetReader(r.Header)
	status := http.StatusBadRequest
	defer func() {
		w.WriteHeader(status)
	}()

	if origin.Value != "" {
		if !s.PredicateOrigin(origin.Value) {
			status = http.StatusForbidden
			return
		}

		method := r.Header.Get("Access-Control-Request-Method")
		header := strings.Split(
			r.Header.Get("Access-Control-Request-Headers"),
			", ")

		if !raiqub.StringSlice(s.Methods).Exists(method) {
			return
		}

		if len(s.Headers) == 0 {
			HttpHeader_AccessControlAllowHeaders().
				SetWriter(w.Header())
		} else {
			if len(header) > 0 &&
				!raiqub.StringSlice(s.Headers).ExistsAllIgnoreCase(header) {
				return
			}
			HttpHeader_AccessControlAllowHeaders().
				SetValue(strings.Join(s.Headers, ", ")).
				SetWriter(w.Header())
		}

		HttpHeader_AccessControlAllowMethods().
			SetValue(strings.Join(s.Methods, ", ")).
			SetWriter(w.Header())
		origin.SetWriter(w.Header())
		HttpHeader_AccessControlAllowCredentials().
			SetValue(strconv.FormatBool(s.UseCredentials)).
			SetWriter(w.Header())
		// Optional
		HttpHeader_AccessControlMaxAge().
			SetValue(strconv.Itoa(int(DEFAULT_CORS_MAX_AGE))).
			SetWriter(w.Header())
		status = http.StatusOK
	} else {
		status = http.StatusNotFound
	}
}
<commit_msg>Changes to HTTP CORS implementation<commit_after>/*
 * Copyright 2015 Fabrício Godoy
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package http

import (
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/skarllot/raiqub"
)

const (
	DEFAULT_CORS_PREFLIGHT_METHOD = "OPTIONS"
	DEFAULT_CORS_MAX_AGE          = time.Hour * 24 / time.Second
	DEFAULT_CORS_METHODS          = "OPTIONS, GET, HEAD, POST, PUT, DELETE, TRACE, CONNECT"
	DEFAULT_CORS_ORIGIN           = "*"
)

// A CORSHandler allows to create a CORS-able API.
type CORSHandler struct {
	PredicateOrigin raiqub.PredicateStringFunc
	Headers         []string
	ExposedHeaders  []string
}

// NewCORSHandler creates a new CORSHandler with default values.
func NewCORSHandler() *CORSHandler {
	return &CORSHandler{
		PredicateOrigin: raiqub.TrueForAll,
		Headers: []string{
			"Origin", "X-Requested-With", "Content-Type",
			"Accept", "Authorization",
		},
		ExposedHeaders: make([]string, 0),
	}
}

// CreatePreflight creates HTTP routes that handles pre-flight requests.
func (s *CORSHandler) CreatePreflight(routes Routes) Routes {
	list := make(Routes, 0, len(routes))
	hList := make(map[string]*CORSPreflight, len(routes))
	for _, v := range routes {
		preflight, ok := hList[v.Path]
		if !ok {
			preflight = &CORSPreflight{
				*s,
				make([]string, 0, 1),
				v.MustAuth,
			}
			hList[v.Path] = preflight
		}
		preflight.Methods = append(preflight.Methods, v.Method)
	}

	for k, v := range hList {
		list = append(list, Route{
			Name:       "",
			Method:     DEFAULT_CORS_PREFLIGHT_METHOD,
			Path:       k,
			MustAuth:   v.UseCredentials,
			ActionFunc: v.ServeHTTP,
		})
	}
	return list
}

// A CORSPreflight represents a HTTP server that handles pre-flight requests.
type CORSPreflight struct {
	CORSHandler
	Methods        []string
	UseCredentials bool
}

// ServeHTTP handle a pre-flight request.
func (s *CORSPreflight) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	origin := HttpHeader_Origin().GetReader(r.Header)
	status := http.StatusBadRequest
	msg := ""
	defer func() {
		w.WriteHeader(status)
		w.Write([]byte(msg))
	}()

	if origin.Value != "" {
		if !s.PredicateOrigin(origin.Value) {
			status = http.StatusForbidden
			return
		}

		method := r.Header.Get("Access-Control-Request-Method")
		header := strings.Split(
			r.Header.Get("Access-Control-Request-Headers"),
			", ")
		if len(header) == 1 && header[0] == "" {
			header = []string{}
		}

		if !raiqub.StringSlice(s.Methods).Exists(method) {
			msg = "Method not allowed"
			return
		}

		if len(s.Headers) == 0 {
			HttpHeader_AccessControlAllowHeaders().
				SetWriter(w.Header())
		} else {
			if len(header) > 0 &&
				!raiqub.StringSlice(s.Headers).ExistsAllIgnoreCase(header) {
				msg = "Header not allowed"
				return
			}
			HttpHeader_AccessControlAllowHeaders().
				SetValue(strings.Join(s.Headers, ", ")).
				SetWriter(w.Header())
		}

		HttpHeader_AccessControlAllowMethods().
			SetValue(strings.Join(s.Methods, ", ")).
			SetWriter(w.Header())
		origin.SetWriter(w.Header())
		HttpHeader_AccessControlAllowCredentials().
			SetValue(strconv.FormatBool(s.UseCredentials)).
			SetWriter(w.Header())
		// Optional
		HttpHeader_AccessControlMaxAge().
			SetValue(strconv.Itoa(int(DEFAULT_CORS_MAX_AGE))).
			SetWriter(w.Header())
		status = http.StatusOK
	} else {
		status = http.StatusNotFound
	}
}

// A CORSMiddleware represents a HTTP middleware that handle HTTP headers for
// CORS-able API.
type CORSMiddleware struct {
	CORSHandler
	UseCredentials bool
}

// Handle is a HTTP handler for CORS-able API.
func (s *CORSMiddleware) Handle(next http.Handler) http.Handler {
	fn := func(w http.ResponseWriter, r *http.Request) {
		origin := HttpHeader_Origin().GetReader(r.Header)
		if r.Method != DEFAULT_CORS_PREFLIGHT_METHOD && origin.Value != "" {
			if !s.PredicateOrigin(origin.Value) {
				return
			}

			HttpHeader_AccessControlAllowOrigin().
				SetValue(origin.Value).
				SetWriter(w.Header())
			HttpHeader_AccessControlAllowCredentials().
				SetValue(strconv.FormatBool(s.UseCredentials)).
				SetWriter(w.Header())
			if len(s.Headers) > 0 {
				HttpHeader_AccessControlAllowHeaders().
					SetValue(strings.Join(s.Headers, ", ")).
					SetWriter(w.Header())
			} else {
				HttpHeader_AccessControlAllowHeaders().
					SetWriter(w.Header())
			}
		}
		next.ServeHTTP(w, r)
	}

	return http.HandlerFunc(fn)
}
<|endoftext|>


<commit_before>// Copyright 2019 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package rpcexplorer contains complied RPCExplorer web app.
//
// Linking to this package will add 7MB to your binary.
package rpcexplorer

// TODO(vadimsh): Implement public API.
<commit_msg>[rpcexplorer] Serve RPC Explorer static assets from Go.<commit_after>// Copyright 2019 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package rpcexplorer contains complied RPCExplorer web app.
//
// Linking to this package will add 7MB to your binary.
package rpcexplorer

import (
	"encoding/hex"
	"fmt"
	"net/http"
	"strings"
	"time"

	"go.chromium.org/luci/server/router"
	"go.chromium.org/luci/web/gowrappers/rpcexplorer/internal"
)

// Install adds routes to serve RPCExplorer web app from "/rpcexplorer".
func Install(r *router.Router) {
	r.GET("/rpcexplorer", router.MiddlewareChain{}, func(c *router.Context) {
		http.Redirect(c.Writer, c.Request, "/rpcexplorer/", http.StatusMovedPermanently)
	})

	r.GET("/rpcexplorer/*path", router.MiddlewareChain{}, func(c *router.Context) {
		path := strings.TrimPrefix(c.Params.ByName("path"), "/")
		if path == "" {
			path = "index.html"
		}

		hash := internal.GetAssetSHA256(path)
		if hash == nil {
			http.Error(c.Writer, "404 page not found", http.StatusNotFound)
			return
		}

		c.Writer.Header().Set("ETag", fmt.Sprintf("%q", hex.EncodeToString(hash)))
		http.ServeContent(
			c.Writer, c.Request, path, time.Time{},
			strings.NewReader(internal.GetAssetString(path)))
	})
}
<|endoftext|>