<commit_before>// The metrics package defines prometheus metric types and provides
// convenience methods to add accounting to various parts of the pipeline.
//
// When defining new operations or metrics, these are helpful values to track:
//  - things coming into or go out of the system: requests, files, tests, api calls.
//  - the success or error status of any of the above.
//  - the distribution of processing latency.
package metrics

import (
	"math"
	"net/http"
	"time"

	"github.com/prometheus/client_golang/prometheus"
)

func init() {
	// Register the metrics defined with Prometheus's default registry.
	prometheus.MustRegister(WorkerCount)
	prometheus.MustRegister(TaskCount)
	prometheus.MustRegister(BigQueryInsert)
	prometheus.MustRegister(DurationHistogram)
	prometheus.MustRegister(InsertionHistogram)
	prometheus.MustRegister(FileSizeHistogram)
}

var (
	// Counts the number of tasks processed by the pipeline.
	//
	// Provides metrics:
	//   etl_worker_count
	// Example usage:
	//   metrics.TaskCount.Inc() / .Dec()
	WorkerCount = prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "etl_worker_count",
		Help: "Number of active workers.",
	})

	// Counts the number of tasks processed by the pipeline.
	//
	// Provides metrics:
	//   etl_task_count{worker, status}
	// Example usage:
	//   metrics.TaskCount.WithLabelValues("ndt", "ok").Inc()
	TaskCount = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "etl_task_count",
			Help: "Number of tasks/archive files processed.",
		},
		// Worker type, e.g. ndt, sidestream, ptr, etc.
		[]string{"worker", "status"},
	)

	// Counts the number of into BigQuery insert operations.
	//
	// Provides metrics:
	//   etl_worker_bigquery_insert_total{worker, status}
	// Usage example:
	//   metrics.BigQueryInsert.WithLabelValues("ndt", "200").Inc()
	BigQueryInsert = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "etl_worker_bigquery_insert_total",
			Help: "Number of BigQuery insert operations.",
		},
		// Worker type, e.g. ndt, sidestream, ptr, etc.
		[]string{"worker", "status"},
	)

	// A histogram of bigquery insertion times. The buckets should use
	// periods that are intuitive for people.
	//
	// Provides metrics:
	//   etl_insertion_time_seconds_bucket{type="...", le="..."}
	//   ...
	//   etl_insertion_time_seconds_sum{type="..."}
	//   etl_insertion_time_seconds_count{type="..."}
	// Usage example:
	//   t := time.Now()
	//   // do some stuff.
	//   metrics.InsertionHistogram.WithLabelValues(
	//           "ndt_test", "ok").Observe(time.Since(t).Seconds())
	InsertionHistogram = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Name: "etl_insertion_time_seconds",
			Help: "Insertion time distributions.",
			Buckets: []float64{
				0.001, 0.003, 0.01, 0.03, 0.1, 0.2, 0.5, 1.0, 2.0,
				5.0, 10.0, 20.0, 50.0, 100.0, math.Inf(+1),
			},
		},
		// Worker type, e.g. ndt, sidestream, ptr, etc.
		[]string{"table", "status"},
	)

	// A histogram of worker processing times. The buckets should use
	// periods that are intuitive for people.
	//
	// Provides metrics:
	//   etl_worker_duration_seconds_bucket{worker="...", le="..."}
	//   ...
	//   etl_worker_duration_seconds_sum{worker="..."}
	//   etl_worker_duration_seconds_count{worker="..."}
	// Usage example:
	//   t := time.Now()
	//   // do some stuff.
	//   metrics.DurationHistogram.WithLabelValues(
	//           "ndt").Observe(time.Since(t).Seconds())
	DurationHistogram = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Name: "etl_worker_duration_seconds",
			Help: "Worker execution time distributions.",
			Buckets: []float64{
				0.001, 0.01, 0.1, 1.0, 2.0, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0,
				600.0, 1800.0, 3600.0, 7200.0, math.Inf(+1),
			},
		},
		// Worker type, e.g. ndt, sidestream, ptr, etc.
		// TODO(soltesz): support a status field based on HTTP status.
		[]string{"worker"},
	)

	FileSizeHistogram = prometheus.NewHistogram(
		prometheus.HistogramOpts{
			Name: "etl_web100_snaplog_file_size_bytes",
			Help: "Size of individual snaplog files.",
			Buckets: []float64{
				0,
				400000,     // 400k
				500000,     // 500k
				600000,     // 600k
				700000,     // 700k
				800000,     // 800k
				900000,     // 900k
				1000000,    // 1 mb
				1100000,    // 1.1 mb
				1200000,    // 1.2 mb
				1400000,    // 1.4 mb
				1600000,    // 1.6 mb
				1800000,    // 1.8 mb
				2000000,    // 2.0 mb
				2400000,    // 2.4 mb
				2800000,    // 2.8 mb
				3200000,    // 3.2 mb
				3600000,    // 3.6 mb
				4000000,    // 4 mb
				6000000,    // 6 mb
				8000000,    // 8 mb
				10000000,   // 10 mb
				20000000,   // 20
				40000000,   // 40
				80000000,   // 80
				100000000,  // 100 mb
				200000000,  // 200
				400000000,  // 400
				800000000,  // 800
				1000000000, // 1 gb
				math.Inf(+1),
			},
		},
	)
)

// DurationHandler wraps the call of an inner http.HandlerFunc and records the runtime.
func DurationHandler(name string, inner http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		t := time.Now()
		inner.ServeHTTP(w, r)
		// TODO(soltesz): collect success or failure status.
		DurationHistogram.WithLabelValues(name).Observe(time.Since(t).Seconds())
	}
}
<commit_msg>Add TODO for generic file size histogram.<commit_after>// The metrics package defines prometheus metric types and provides
// convenience methods to add accounting to various parts of the pipeline.
//
// When defining new operations or metrics, these are helpful values to track:
//  - things coming into or go out of the system: requests, files, tests, api calls.
//  - the success or error status of any of the above.
//  - the distribution of processing latency.
package metrics

import (
	"math"
	"net/http"
	"time"

	"github.com/prometheus/client_golang/prometheus"
)

func init() {
	// Register the metrics defined with Prometheus's default registry.
	prometheus.MustRegister(WorkerCount)
	prometheus.MustRegister(TaskCount)
	prometheus.MustRegister(BigQueryInsert)
	prometheus.MustRegister(DurationHistogram)
	prometheus.MustRegister(InsertionHistogram)
	prometheus.MustRegister(FileSizeHistogram)
}

var (
	// Counts the number of tasks processed by the pipeline.
	//
	// Provides metrics:
	//   etl_worker_count
	// Example usage:
	//   metrics.TaskCount.Inc() / .Dec()
	WorkerCount = prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "etl_worker_count",
		Help: "Number of active workers.",
	})

	// Counts the number of tasks processed by the pipeline.
	//
	// Provides metrics:
	//   etl_task_count{worker, status}
	// Example usage:
	//   metrics.TaskCount.WithLabelValues("ndt", "ok").Inc()
	TaskCount = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "etl_task_count",
			Help: "Number of tasks/archive files processed.",
		},
		// Worker type, e.g. ndt, sidestream, ptr, etc.
		[]string{"worker", "status"},
	)

	// Counts the number of into BigQuery insert operations.
	//
	// Provides metrics:
	//   etl_worker_bigquery_insert_total{worker, status}
	// Usage example:
	//   metrics.BigQueryInsert.WithLabelValues("ndt", "200").Inc()
	BigQueryInsert = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "etl_worker_bigquery_insert_total",
			Help: "Number of BigQuery insert operations.",
		},
		// Worker type, e.g. ndt, sidestream, ptr, etc.
		[]string{"worker", "status"},
	)

	// A histogram of bigquery insertion times. The buckets should use
	// periods that are intuitive for people.
	//
	// Provides metrics:
	//   etl_insertion_time_seconds_bucket{type="...", le="..."}
	//   ...
	//   etl_insertion_time_seconds_sum{type="..."}
	//   etl_insertion_time_seconds_count{type="..."}
	// Usage example:
	//   t := time.Now()
	//   // do some stuff.
	//   metrics.InsertionHistogram.WithLabelValues(
	//           "ndt_test", "ok").Observe(time.Since(t).Seconds())
	InsertionHistogram = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Name: "etl_insertion_time_seconds",
			Help: "Insertion time distributions.",
			Buckets: []float64{
				0.001, 0.003, 0.01, 0.03, 0.1, 0.2, 0.5, 1.0, 2.0,
				5.0, 10.0, 20.0, 50.0, 100.0, math.Inf(+1),
			},
		},
		// Worker type, e.g. ndt, sidestream, ptr, etc.
		[]string{"table", "status"},
	)

	// A histogram of worker processing times. The buckets should use
	// periods that are intuitive for people.
	//
	// Provides metrics:
	//   etl_worker_duration_seconds_bucket{worker="...", le="..."}
	//   ...
	//   etl_worker_duration_seconds_sum{worker="..."}
	//   etl_worker_duration_seconds_count{worker="..."}
	// Usage example:
	//   t := time.Now()
	//   // do some stuff.
	//   metrics.DurationHistogram.WithLabelValues(
	//           "ndt").Observe(time.Since(t).Seconds())
	DurationHistogram = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Name: "etl_worker_duration_seconds",
			Help: "Worker execution time distributions.",
			Buckets: []float64{
				0.001, 0.01, 0.1, 1.0, 2.0, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0,
				600.0, 1800.0, 3600.0, 7200.0, math.Inf(+1),
			},
		},
		// Worker type, e.g. ndt, sidestream, ptr, etc.
		// TODO(soltesz): support a status field based on HTTP status.
		[]string{"worker"},
	)

	// TODO(dev): generalize this metric for size of any file type.
	FileSizeHistogram = prometheus.NewHistogram(
		prometheus.HistogramOpts{
			Name: "etl_web100_snaplog_file_size_bytes",
			Help: "Size of individual snaplog files.",
			Buckets: []float64{
				0,
				400000,     // 400k
				500000,     // 500k
				600000,     // 600k
				700000,     // 700k
				800000,     // 800k
				900000,     // 900k
				1000000,    // 1 mb
				1100000,    // 1.1 mb
				1200000,    // 1.2 mb
				1400000,    // 1.4 mb
				1600000,    // 1.6 mb
				1800000,    // 1.8 mb
				2000000,    // 2.0 mb
				2400000,    // 2.4 mb
				2800000,    // 2.8 mb
				3200000,    // 3.2 mb
				3600000,    // 3.6 mb
				4000000,    // 4 mb
				6000000,    // 6 mb
				8000000,    // 8 mb
				10000000,   // 10 mb
				20000000,   // 20
				40000000,   // 40
				80000000,   // 80
				100000000,  // 100 mb
				200000000,  // 200
				400000000,  // 400
				800000000,  // 800
				1000000000, // 1 gb
				math.Inf(+1),
			},
		},
	)
)

// DurationHandler wraps the call of an inner http.HandlerFunc and records the runtime.
func DurationHandler(name string, inner http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		t := time.Now()
		inner.ServeHTTP(w, r)
		// TODO(soltesz): collect success or failure status.
		DurationHistogram.WithLabelValues(name).Observe(time.Since(t).Seconds())
	}
}
<|endoftext|>


<commit_before>package metrics

import (
	"time"

	"github.com/gkarlik/quark-go/system"
)

// Metric represents metric collected by the service
type Metric struct {
	Date   time.Time              // metric date - default: time.Now()
	Name   string                 // metric name
	Values map[string]interface{} // metric values
	Tags   map[string]string      // metric tags
}

// Reporter represents metrics reporter mechanism.
type Reporter interface {
	Report(ms ...Metric) error

	system.Disposer
}
<commit_msg>Add metric type.<commit_after>package metrics

import (
	"time"

	"github.com/gkarlik/quark-go/system"
)

const (
	// Counter is a cumulative metric that represents a single numerical value that only ever goes up.
	Counter int64 = iota
	// Gauge is a metric that represents a single numerical value that can arbitrarily go up and down.
	Gauge
	// Histogram samples observations (usually things like request durations or response sizes) and counts them in configurable buckets. It also provides a sum of all observed values.
	Histogram
	// Set supports counting unique occurences of events between flushes.
	Set
	// Summary samples observations (usually things like request durations and response sizes).
	Summary
	// Other type of metric.
	Other
)

// Metric represents metric collected by the service
type Metric struct {
	Date   time.Time              // metric date - default: time.Now()
	Type   int64                  // metric type
	Name   string                 // metric name
	Values map[string]interface{} // metric values
	Tags   map[string]string      // metric tags
}

// Reporter represents metrics reporter mechanism.
type Reporter interface {
	Report(ms ...Metric) error

	system.Disposer
}
<|endoftext|>


<commit_before>package metrics

import (
	"runtime"
	"time"
)

// sets how often the metrics Runtime will sample and
// emit stats.
var RuntimeMetricsSamplingInterval time.Duration = 30 * time.Second

// Runtime enters into a loop, sampling and outputing the runtime stats periodically.
// Usage:
//   func main() {
//     ...
//     go metrics.Runtime()
//     ...
//   }

func Runtime() {
	sampleEvery(RuntimeMetricsSamplingInterval)
}

func sampleEvery(t time.Duration) {
	c := time.Tick(t)
	for _ = range c {
		ReportRuntimeMetrics()
	}
}

// Runtime enters into a loop, sampling and outputing the runtime stats periodically.
func ReportRuntimeMetrics() {
	var memstats runtime.MemStats
	runtime.ReadMemStats(&memstats)
	numGoroutines := runtime.NumGoroutine()

	r := map[string]float64{
		"goroutine":                     float64(numGoroutines),
		"runtime.MemStats.Alloc":        float64(memstats.Alloc),
		"runtime.MemStats.Frees":        float64(memstats.Frees),
		"runtime.MemStats.HeapAlloc":    float64(memstats.HeapAlloc),
		"runtime.MemStats.HeapIdle":     float64(memstats.HeapIdle),
		"runtime.MemStats.HeapObjects":  float64(memstats.HeapObjects),
		"runtime.MemStats.HeapReleased": float64(memstats.HeapReleased),
		"runtime.MemStats.HeapSys":      float64(memstats.HeapSys),
		"runtime.MemStats.LastGC":       float64(memstats.LastGC),
		"runtime.MemStats.Lookups":      float64(memstats.Lookups),
		"runtime.MemStats.Mallocs":      float64(memstats.Mallocs),
		"runtime.MemStats.MCacheInuse":  float64(memstats.MCacheInuse),
		"runtime.MemStats.MCacheSys":    float64(memstats.MCacheSys),
		"runtime.MemStats.MSpanInuse":   float64(memstats.MSpanInuse),
		"runtime.MemStats.MSpanSys":     float64(memstats.MSpanSys),
		"runtime.MemStats.NextGC":       float64(memstats.NextGC),
		"runtime.MemStats.NumGC":        float64(memstats.NumGC),
		"runtime.MemStats.StackInuse":   float64(memstats.StackInuse),
	}

	for name, value := range r {
		Gauge(name, value, nil, 1.0)
	}
}
<commit_msg>Tiny cleanup in metrics/runtime<commit_after>package metrics

import (
	"runtime"
	"time"
)

// sets how often the metrics Runtime will sample and
// emit stats.
var RuntimeMetricsSamplingInterval time.Duration = 30 * time.Second

// Runtime enters into a loop, sampling and outputing the runtime stats periodically.
// Usage:
//   func main() {
//     ...
//     go metrics.Runtime()
//     ...
//   }
func Runtime() {
	c := time.Tick(RuntimeMetricsSamplingInterval)
	for _ = range c {
		ReportRuntimeMetrics()
	}
}

// Samples and reports current runtime status once.
func ReportRuntimeMetrics() {
	var memstats runtime.MemStats
	runtime.ReadMemStats(&memstats)
	numGoroutines := runtime.NumGoroutine()

	r := map[string]float64{
		"goroutine":                     float64(numGoroutines),
		"runtime.MemStats.Alloc":        float64(memstats.Alloc),
		"runtime.MemStats.Frees":        float64(memstats.Frees),
		"runtime.MemStats.HeapAlloc":    float64(memstats.HeapAlloc),
		"runtime.MemStats.HeapIdle":     float64(memstats.HeapIdle),
		"runtime.MemStats.HeapObjects":  float64(memstats.HeapObjects),
		"runtime.MemStats.HeapReleased": float64(memstats.HeapReleased),
		"runtime.MemStats.HeapSys":      float64(memstats.HeapSys),
		"runtime.MemStats.LastGC":       float64(memstats.LastGC),
		"runtime.MemStats.Lookups":      float64(memstats.Lookups),
		"runtime.MemStats.Mallocs":      float64(memstats.Mallocs),
		"runtime.MemStats.MCacheInuse":  float64(memstats.MCacheInuse),
		"runtime.MemStats.MCacheSys":    float64(memstats.MCacheSys),
		"runtime.MemStats.MSpanInuse":   float64(memstats.MSpanInuse),
		"runtime.MemStats.MSpanSys":     float64(memstats.MSpanSys),
		"runtime.MemStats.NextGC":       float64(memstats.NextGC),
		"runtime.MemStats.NumGC":        float64(memstats.NumGC),
		"runtime.MemStats.StackInuse":   float64(memstats.StackInuse),
	}

	for name, value := range r {
		Gauge(name, value, nil, 1.0)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"strings"
)
<commit_msg>not needed<commit_after><|endoftext|>


<commit_before>package main

import (
	"fmt"
	"koding/db/models"
	"koding/db/mongodb/modelhelper"
	"time"

	"labix.org/v2/mgo/bson"

	"github.com/crowdmob/goamz/aws"
	"github.com/crowdmob/goamz/cloudwatch"
	"github.com/jinzhu/now"
	"github.com/koding/redis"
)

var (
	AWS_NAMESPACE = "AWS/EC2"
	AWS_PERIOD    = 604800

	GB_TO_MB float64 = 1024

	rightNow = time.Now()

	auth aws.Auth

	PaidPlanMultiplier float64 = 2
)

type Cloudwatch struct {
	Name  string
	Limit float64
}

func (c *Cloudwatch) GetName() string {
	return c.Name
}

func (c *Cloudwatch) GetLimit() float64 {
	limit, _ := storage.GetLimit(c.GetName(), c.Limit)
	return limit
}

func isEmpty(s string) bool {
	return s == ""
}

func (c *Cloudwatch) GetAndSaveData(username string) error {
	userMachines, err := modelhelper.GetMachinesForUsername(username)
	if err != nil {
		return err
	}

	var sum float64

	for _, machine := range userMachines {
		meta, ok := machine.Meta.(bson.M)
		if !ok {
			Log.Error("queued machine has no `meta`", machine.ObjectId)
			continue
		}

		region, ok := meta["region"].(string)
		if !ok || isEmpty(region) {
			Log.Error("queued machine has no `region`", machine.ObjectId)
			continue
		}

		instanceId, ok := meta["instance_id"].(string)
		if !ok || isEmpty(instanceId) {
			Log.Error("queued machine has no `instanceId`", machine.ObjectId)
			continue
		}

		dimension := &cloudwatch.Dimension{
			Name:  "InstanceId",
			Value: instanceId,
		}

		cw, err := cloudwatch.NewCloudWatch(auth, aws.Regions[region].CloudWatchServicepoint)
		if err != nil {
			Log.Error("Failed to initialize cloudwatch client", err)
			continue
		}

		request := &cloudwatch.GetMetricStatisticsRequest{
			Dimensions: []cloudwatch.Dimension{*dimension},
			Statistics: []string{cloudwatch.StatisticDatapointSum},
			MetricName: c.GetName(),
			EndTime:    time.Now(),
			StartTime:  now.BeginningOfWeek(),
			Period:     AWS_PERIOD,
			Namespace:  AWS_NAMESPACE,
		}

		response, err := cw.GetMetricStatistics(request)
		if err != nil {
			Log.Error("Failed to get request for machine: %v", machine.ObjectId)
			continue
		}

		for _, raw := range response.GetMetricStatisticsResult.Datapoints {
			sum += raw.Sum / GB_TO_MB / GB_TO_MB
		}
	}

	Log.Debug("'%s' has used: %v '%s'", username, sum, c.Name)

	return storage.Save(c.Name, username, sum)
}

func (c *Cloudwatch) GetMachinesOverLimit() ([]*models.Machine, error) {
	usernames, err := storage.Range(c.Name, c.GetLimit())
	if err != nil {
		return nil, err
	}

	machines := []*models.Machine{}

	for _, username := range usernames {
		yes, err := exemptFromStopping(c.GetName(), username)
		if err != nil {
			return nil, err
		}

		if !yes {
			ms, err := modelhelper.GetMachinesForUsername(username)
			if err != nil {
				Log.Error(err.Error())
				continue
			}

			machines = append(machines, ms...)
		}
	}

	return machines, nil
}

func (c *Cloudwatch) IsUserOverLimit(username string) (*LimitResponse, error) {
	canStart := &LimitResponse{CanStart: true}

	value, err := storage.Get(c.GetName(), username)
	if err != nil && !isRedisRecordNil(err) {
		return nil, err
	}

	yes, err := exemptFromStopping(c.GetName(), username)
	if err != nil {
		return nil, err
	}

	if yes {
		return canStart, nil
	}

	planTitle, err := getPlanForUser(username)
	if err != nil {
		return nil, err
	}

	var limit float64

	switch planTitle {
	case FreePlan:
		limit = c.GetLimit()
	default:
		limit = c.GetLimit() * PaidPlanMultiplier
	}

	lr := &LimitResponse{
		CanStart:     limit >= value,
		AllowedUsage: limit,
		CurrentUsage: value,
		Reason:       fmt.Sprintf("%s overlimit", c.GetName()),
	}

	return lr, nil
}

func (c *Cloudwatch) RemoveUsername(username string) error {
	return storage.Remove(c.GetName(), username)
}

func isRedisRecordNil(err error) bool {
	return err != nil && err != redis.ErrNil
}
<commit_msg>vmwatcher: minor fixes<commit_after>package main

import (
	"fmt"
	"koding/db/models"
	"koding/db/mongodb/modelhelper"
	"time"

	"labix.org/v2/mgo/bson"

	"github.com/crowdmob/goamz/aws"
	"github.com/crowdmob/goamz/cloudwatch"
	"github.com/jinzhu/now"
	"github.com/koding/redis"
)

var (
	AWS_NAMESPACE = "AWS/EC2"
	AWS_PERIOD    = 604800

	GB_TO_MB float64 = 1024

	rightNow = time.Now()

	auth aws.Auth

	PaidPlanMultiplier float64 = 2
)

type Cloudwatch struct {
	Name  string
	Limit float64
}

func (c *Cloudwatch) GetName() string {
	return c.Name
}

func (c *Cloudwatch) GetLimit() float64 {
	limit, _ := storage.GetLimit(c.GetName(), c.Limit)
	return limit
}

func isEmpty(s string) bool {
	return s == ""
}

func (c *Cloudwatch) GetAndSaveData(username string) error {
	userMachines, err := modelhelper.GetMachinesForUsername(username)
	if err != nil {
		return err
	}

	var sum float64

	for _, machine := range userMachines {
		meta, ok := machine.Meta.(bson.M)
		if !ok {
			Log.Error("queued machine has no `meta`", machine.ObjectId)
			continue
		}

		region, ok := meta["region"].(string)
		if !ok || isEmpty(region) {
			Log.Error("queued machine has no `region`: %v", machine.ObjectId)
			continue
		}

		instanceId, ok := meta["instanceId"].(string)
		if !ok || isEmpty(instanceId) {
			Log.Error("queued machine has no `instanceId`: %v", machine.ObjectId)
			continue
		}

		dimension := &cloudwatch.Dimension{
			Name:  "InstanceId",
			Value: instanceId,
		}

		cw, err := cloudwatch.NewCloudWatch(auth, aws.Regions[region].CloudWatchServicepoint)
		if err != nil {
			Log.Error("Failed to initialize cloudwatch client", err)
			continue
		}

		request := &cloudwatch.GetMetricStatisticsRequest{
			Dimensions: []cloudwatch.Dimension{*dimension},
			Statistics: []string{cloudwatch.StatisticDatapointSum},
			MetricName: c.GetName(),
			EndTime:    time.Now(),
			StartTime:  now.BeginningOfWeek(),
			Period:     AWS_PERIOD,
			Namespace:  AWS_NAMESPACE,
		}

		response, err := cw.GetMetricStatistics(request)
		if err != nil {
			Log.Error("Failed to get request for machine: %v", machine.ObjectId)
			continue
		}

		for _, raw := range response.GetMetricStatisticsResult.Datapoints {
			sum += raw.Sum / GB_TO_MB / GB_TO_MB
		}
	}

	Log.Debug("'%s' has used: %v '%s'", username, sum, c.Name)

	return storage.Save(c.Name, username, sum)
}

func (c *Cloudwatch) GetMachinesOverLimit() ([]*models.Machine, error) {
	usernames, err := storage.Range(c.Name, c.GetLimit())
	if err != nil {
		return nil, err
	}

	machines := []*models.Machine{}

	for _, username := range usernames {
		yes, err := exemptFromStopping(c.GetName(), username)
		if err != nil {
			return nil, err
		}

		if !yes {
			ms, err := modelhelper.GetMachinesForUsername(username)
			if err != nil {
				Log.Error(err.Error())
				continue
			}

			machines = append(machines, ms...)
		}
	}

	return machines, nil
}

func (c *Cloudwatch) IsUserOverLimit(username string) (*LimitResponse, error) {
	canStart := &LimitResponse{CanStart: true}

	value, err := storage.Get(c.GetName(), username)
	if err != nil && !isRedisRecordNil(err) {
		return nil, err
	}

	yes, err := exemptFromStopping(c.GetName(), username)
	if err != nil {
		return nil, err
	}

	if yes {
		return canStart, nil
	}

	planTitle, err := getPlanForUser(username)
	if err != nil {
		return nil, err
	}

	var limit float64

	switch planTitle {
	case FreePlan:
		limit = c.GetLimit()
	default:
		limit = c.GetLimit() * PaidPlanMultiplier
	}

	lr := &LimitResponse{
		CanStart:     limit >= value,
		AllowedUsage: limit,
		CurrentUsage: value,
		Reason:       fmt.Sprintf("%s overlimit", c.GetName()),
	}

	return lr, nil
}

func (c *Cloudwatch) RemoveUsername(username string) error {
	return storage.Remove(c.GetName(), username)
}

func isRedisRecordNil(err error) bool {
	return err != nil && err == redis.ErrNil
}
<|endoftext|>


<commit_before>package models

type PinRequest struct {
	MessageId int64  `json:"messageId"`
	GroupName string `json:"groupName"`
	AccountId int64  `json:"accountId"`
}
<commit_msg>Social: add constructor function<commit_after>package models

type PinRequest struct {
	MessageId int64  `json:"messageId"`
	GroupName string `json:"groupName"`
	AccountId int64  `json:"accountId"`
}

func NewPinRequest() *PinRequest {
	return &PinRequest{
		GroupName: Channel_KODING_NAME,
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2021 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package simplifier

import (
	"sync"

	"vitess.io/vitess/go/sync2"
	"vitess.io/vitess/go/vt/log"
	"vitess.io/vitess/go/vt/sqlparser"
	"vitess.io/vitess/go/vt/vtgate/semantics"
)

// SimplifyStatement simplifies the AST of a query. It basically iteratevely prunes leaves of the AST, as long as the pruning
// continues to return true from the `test` function.
func SimplifyStatement(
	in sqlparser.SelectStatement,
	currentDB string,
	si semantics.SchemaInformation,
	testF func(sqlparser.SelectStatement) bool,
) sqlparser.SelectStatement {
	var semTable *semantics.SemTable
	var err error
	{
		// Since our semantic analysis changes the AST, we clone it first, so we have a pristine AST to play with
		clone := sqlparser.CloneSelectStatement(in)
		semTable, err = semantics.Analyze(clone, currentDB, si)
		if err != nil {
			return nil
		}
	}

	test := func(s sqlparser.SelectStatement) bool {
		// Since our semantic analysis changes the AST, we clone it first, so we have a pristine AST to play with
		return testF(sqlparser.CloneSelectStatement(s))
	}

	// we start by removing one table at a time, and see if we still have an interesting plan
	for idx := range semTable.Tables {
		clone := sqlparser.CloneSelectStatement(in)
		if err != nil {
			panic(err) // this should never happen
		}
		searchedTS := semantics.SingleTableSet(idx)
		simplified := removeTable(clone, searchedTS, semTable)
		if simplified && test(clone) {
			name, _ := semTable.Tables[idx].Name()
			log.Errorf("removed table %s", name)
			return SimplifyStatement(clone, currentDB, si, test)
		}
	}

	// now let's try to simplify * expressions
	if simplifyStarExpr(in, test) {
		return SimplifyStatement(in, currentDB, si, test)
	}

	// if we get here, we couldn't find a simpler query by just removing one table,
	// we try to remove select expressions next
	ch := make(chan expressionCursor)
	findExpressions(in, ch)
	for cursor := range ch {
		// first - let's try to remove the expression
		if cursor.remove() {
			if test(in) {
				log.Errorf("removed expression: %s", sqlparser.String(cursor.expr))
				cursor.abort()
				return SimplifyStatement(in, currentDB, si, test)
			}
			cursor.restore()
		}

		// ok, we seem to need this expression. let's see if we can find a simpler version
		s := &shrinker{orig: cursor.expr}
		newExpr := s.Next()
		for newExpr != nil {
			cursor.replace(newExpr)
			if test(in) {
				log.Errorf("simplified expression: %s -> %s", sqlparser.String(cursor.expr), sqlparser.String(newExpr))
				cursor.abort()
				return SimplifyStatement(in, currentDB, si, test)
			}
			newExpr = s.Next()
		}
		// if we get here, we failed to simplify this expression,
		// so we put back in the original expression
		cursor.restore()
		cursor.wg.Done()
	}

	return in
}

func simplifyStarExpr(in sqlparser.SelectStatement, test func(sqlparser.SelectStatement) bool) bool {
	simplified := false
	sqlparser.Rewrite(in, func(cursor *sqlparser.Cursor) bool {
		se, ok := cursor.Node().(*sqlparser.StarExpr)
		if !ok {
			return true
		}
		cursor.Replace(&sqlparser.AliasedExpr{
			Expr: sqlparser.NewIntLiteral("0"),
		})
		if test(in) {
			log.Errorf("replaced star with literal")
			simplified = true
			return false
		}
		cursor.Replace(se)

		return true
	}, nil)
	return simplified
}

// removeTable removes the table with the given index from the select statement, which includes the FROM clause
// but also all expressions and predicates that depend on the table
func removeTable(clone sqlparser.SelectStatement, searchedTS semantics.TableSet, inner *semantics.SemTable) bool {
	simplified := false
	sqlparser.Rewrite(clone, func(cursor *sqlparser.Cursor) bool {
		switch node := cursor.Node().(type) {
		case *sqlparser.JoinTableExpr:
			lft, ok := node.LeftExpr.(*sqlparser.AliasedTableExpr)
			if ok {
				ts := inner.TableSetFor(lft)
				if ts == searchedTS {
					cursor.Replace(node.RightExpr)
					simplified = true
				}
			}
			rgt, ok := node.RightExpr.(*sqlparser.AliasedTableExpr)
			if ok {
				ts := inner.TableSetFor(rgt)
				if ts == searchedTS {
					cursor.Replace(node.LeftExpr)
					simplified = true
				}
			}
		case *sqlparser.Select:
			if len(node.From) == 1 {
				return true
			}
			for i, tbl := range node.From {
				lft, ok := tbl.(*sqlparser.AliasedTableExpr)
				if ok {
					ts := inner.TableSetFor(lft)
					if ts == searchedTS {
						node.From = append(node.From[:i], node.From[i+1:]...)
						simplified = true
						return true
					}
				}
			}
		case *sqlparser.Where:
			exprs := sqlparser.SplitAndExpression(nil, node.Expr)
			var newPredicate sqlparser.Expr
			for _, expr := range exprs {
				if !inner.RecursiveDeps(expr).IsOverlapping(searchedTS) {
					newPredicate = sqlparser.AndExpressions(newPredicate, expr)
				}
			}
			node.Expr = newPredicate
		case sqlparser.SelectExprs:
			_, isSel := cursor.Parent().(*sqlparser.Select)
			if !isSel {
				return true
			}

			var newExprs sqlparser.SelectExprs
			for _, ae := range node {
				expr, ok := ae.(*sqlparser.AliasedExpr)
				if !ok {
					newExprs = append(newExprs, ae)
					continue
				}
				if !inner.RecursiveDeps(expr.Expr).IsOverlapping(searchedTS) || sqlparser.ContainsAggregation(expr.Expr) {
					newExprs = append(newExprs, ae)
				}
			}
			cursor.Replace(newExprs)
		case sqlparser.GroupBy:
			var newExprs sqlparser.GroupBy
			for _, expr := range node {
				if !inner.RecursiveDeps(expr).IsOverlapping(searchedTS) || sqlparser.ContainsAggregation(expr) {
					newExprs = append(newExprs, expr)
				}
			}
			cursor.Replace(newExprs)
		case sqlparser.OrderBy:
			var newExprs sqlparser.OrderBy
			for _, expr := range node {
				if !inner.RecursiveDeps(expr.Expr).IsOverlapping(searchedTS) || sqlparser.ContainsAggregation(expr.Expr) {
					newExprs = append(newExprs, expr)
				}
			}

			cursor.Replace(newExprs)
		}
		return true
	}, nil)
	return simplified
}

type expressionCursor struct {
	expr        sqlparser.Expr
	replace     func(replaceWith sqlparser.Expr)
	remove      func() bool
	restore     func()
	wg          *sync.WaitGroup
	abortMarker *sync2.AtomicBool
}

func (i expressionCursor) abort() {
	i.abortMarker.Set(true)
}

func newCursorItem(
	ch chan<- expressionCursor,
	expr sqlparser.Expr,
	abort *sync2.AtomicBool,
	replace func(replaceWith sqlparser.Expr),
	remove func() bool,
	restore func(),
) {
	wg := &sync.WaitGroup{}
	wg.Add(1)
	ch <- expressionCursor{
		expr:        expr,
		replace:     replace,
		remove:      remove,
		restore:     restore,
		wg:          wg,
		abortMarker: abort,
	}
	wg.Wait()
}

func findExpressions(clone sqlparser.SelectStatement, ch chan<- expressionCursor) {
	abort := &sync2.AtomicBool{}
	go func() {
		sqlparser.Rewrite(clone, func(cursor *sqlparser.Cursor) bool {
			switch node := cursor.Node().(type) {
			case sqlparser.SelectExprs:
				_, isSel := cursor.Parent().(*sqlparser.Select)
				if !isSel {
					return true
				}
				for idx := 0; idx < len(node); idx++ {
					ae := node[idx]
					expr, ok := ae.(*sqlparser.AliasedExpr)
					if !ok {
						continue
					}
					removed := false
					original := sqlparser.CloneExpr(expr.Expr)
					newCursorItem(
						ch, expr.Expr, abort,
						/*replace*/ func(replaceWith sqlparser.Expr) {
							if removed {
								panic("cant replace after remove without restore")
							}
							expr.Expr = replaceWith
						},
						/*remove*/ func() bool {
							if removed {
								panic("can't remove twice, silly")
							}
							if len(node) == 1 {
								// can't remove the last expressions - we'd end up with an empty SELECT clause
								return false
							}
							withoutElement := append(node[:idx], node[idx+1:]...)
							cursor.Replace(withoutElement)
							node = withoutElement
							removed = true
							return true
						},
						/*restore*/ func() {
							if removed {
								front := make(sqlparser.SelectExprs, idx)
								copy(front, node[:idx])
								back := make(sqlparser.SelectExprs, len(node)-idx)
								copy(back, node[idx:])
								frontWithRestoredExpr := append(front, ae)
								node = append(frontWithRestoredExpr, back...)
								cursor.Replace(node)
								removed = false
								return
							}
							expr.Expr = original
						},
					)
					if abort.Get() {
						close(ch)
						return false
					}
				}
			case *sqlparser.Where:
				exprs := sqlparser.SplitAndExpression(nil, node.Expr)
				set := func(input []sqlparser.Expr) {
					node.Expr = sqlparser.AndExpressions(input...)
					exprs = input
				}
				if !visitExpressions(exprs, set, ch, abort) {
					return false
				}
			case *sqlparser.JoinTableExpr:

			case *sqlparser.JoinCondition:
				join, ok := cursor.Parent().(*sqlparser.JoinTableExpr)
				if !ok {
					return true
				}
				if join.Join != sqlparser.NormalJoinType || node.Using != nil {
					return false
				}
				exprs := sqlparser.SplitAndExpression(nil, node.On)
				set := func(input []sqlparser.Expr) {
					node.On = sqlparser.AndExpressions(input...)
					exprs = input
				}
				if !visitExpressions(exprs, set, ch, abort) {
					return false
				}
			case sqlparser.GroupBy:
				set := func(input []sqlparser.Expr) {
					node = input
					cursor.Replace(node)
				}
				if !visitExpressions(node, set, ch, abort) {
					return false
				}
			case sqlparser.OrderBy:
				for idx := 0; idx < len(node); idx++ {
					order := node[idx]
					removed := false
					original := sqlparser.CloneExpr(order.Expr)
					newCursorItem(
						ch, order.Expr, abort,
						/*replace*/ func(replaceWith sqlparser.Expr) {
							if removed {
								panic("cant replace after remove without restore")
							}
							order.Expr = replaceWith
						},
						/*remove*/ func() bool {
							if removed {
								panic("can't remove twice, silly")
							}
							withoutElement := append(node[:idx], node[idx+1:]...)
							cursor.Replace(withoutElement)
							node = withoutElement
							removed = true
							return true
						},
						/*restore*/ func() {
							if removed {
								front := make(sqlparser.OrderBy, idx)
								copy(front, node[:idx])
								back := make(sqlparser.OrderBy, len(node)-idx)
								copy(back, node[idx:])
								frontWithRestoredExpr := append(front, order)
								node = append(frontWithRestoredExpr, back...)
								cursor.Replace(node)
								removed = false
								return
							}
							order.Expr = original
						},
					)
					if abort.Get() {
						close(ch)
						return false
					}
				}
			case *sqlparser.Limit:
				if node.Offset != nil {
					original := node.Offset
					newCursorItem(ch, node.Offset, abort,
						/*replace*/ func(replaceWith sqlparser.Expr) {
							node.Offset = replaceWith
						},
						/*remove*/ func() bool {
							node.Offset = nil
							return true
						},
						/*restore*/ func() {
							node.Offset = original
						})
				}
				if node.Rowcount != nil {
					original := node.Rowcount
					newCursorItem(ch, node.Rowcount, abort,
						/*replace*/ func(replaceWith sqlparser.Expr) {
							node.Rowcount = replaceWith
						},
						/*remove*/ func() bool {
							// removing Rowcount is an invalid op
							return false
						},
						/*restore*/ func() {
							node.Rowcount = original
						})
				}

			}
			return true
		}, nil)
		close(ch)
	}()
}

// visitExpressions allows the cursor to visit all expressions in a slice,
// and can replace or remove items and restore the slice.
func visitExpressions(
	exprs []sqlparser.Expr,
	set func(input []sqlparser.Expr),
	ch chan<- expressionCursor,
	abort *sync2.AtomicBool,
) bool {
	for idx := 0; idx < len(exprs); idx++ {
		expr := exprs[idx]
		removed := false
		newCursorItem(ch, expr, abort,
			func(replaceWith sqlparser.Expr) {
				if removed {
					panic("cant replace after remove without restore")
				}
				exprs[idx] = replaceWith
				set(exprs)
			},
			/*remove*/ func() bool {
				if removed {
					panic("can't remove twice, silly")
				}
				exprs = append(exprs[:idx], exprs[idx+1:]...)
				set(exprs)
				removed = true
				return true
			},
			/*restore*/ func() {
				if removed {
					front := make([]sqlparser.Expr, idx)
					copy(front, exprs[:idx])
					back := make([]sqlparser.Expr, len(exprs)-idx)
					copy(back, exprs[idx:])
					frontWithRestoredExpr := append(front, expr)
					exprs = append(frontWithRestoredExpr, back...)
					set(exprs)
					removed = false
					return
				}
				exprs[idx] = expr
				set(exprs)
			})
		if abort.Get() {
			return false
		}
	}
	return true
}
<commit_msg>refactor: extracted snippet that was used three times<commit_after>/*
Copyright 2021 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package simplifier

import (
	"sync"

	"vitess.io/vitess/go/sync2"
	"vitess.io/vitess/go/vt/log"
	"vitess.io/vitess/go/vt/sqlparser"
	"vitess.io/vitess/go/vt/vtgate/semantics"
)

// SimplifyStatement simplifies the AST of a query. It basically iteratevely prunes leaves of the AST, as long as the pruning
// continues to return true from the `test` function.
func SimplifyStatement(
	in sqlparser.SelectStatement,
	currentDB string,
	si semantics.SchemaInformation,
	testF func(sqlparser.SelectStatement) bool,
) sqlparser.SelectStatement {
	var semTable *semantics.SemTable
	var err error
	{
		// Since our semantic analysis changes the AST, we clone it first, so we have a pristine AST to play with
		clone := sqlparser.CloneSelectStatement(in)
		semTable, err = semantics.Analyze(clone, currentDB, si)
		if err != nil {
			return nil
		}
	}

	test := func(s sqlparser.SelectStatement) bool {
		// Since our semantic analysis changes the AST, we clone it first, so we have a pristine AST to play with
		return testF(sqlparser.CloneSelectStatement(s))
	}

	// we start by removing one table at a time, and see if we still have an interesting plan
	for idx := range semTable.Tables {
		clone := sqlparser.CloneSelectStatement(in)
		if err != nil {
			panic(err) // this should never happen
		}
		searchedTS := semantics.SingleTableSet(idx)
		simplified := removeTable(clone, searchedTS, semTable)
		if simplified && test(clone) {
			name, _ := semTable.Tables[idx].Name()
			log.Errorf("removed table %s", name)
			return SimplifyStatement(clone, currentDB, si, test)
		}
	}

	// now let's try to simplify * expressions
	if simplifyStarExpr(in, test) {
		return SimplifyStatement(in, currentDB, si, test)
	}

	// if we get here, we couldn't find a simpler query by just removing one table,
	// we try to remove select expressions next
	ch := make(chan expressionCursor)
	findExpressions(in, ch)
	for cursor := range ch {
		// first - let's try to remove the expression
		if cursor.remove() {
			if test(in) {
				log.Errorf("removed expression: %s", sqlparser.String(cursor.expr))
				cursor.abort()
				return SimplifyStatement(in, currentDB, si, test)
			}
			cursor.restore()
		}

		// ok, we seem to need this expression. let's see if we can find a simpler version
		s := &shrinker{orig: cursor.expr}
		newExpr := s.Next()
		for newExpr != nil {
			cursor.replace(newExpr)
			if test(in) {
				log.Errorf("simplified expression: %s -> %s", sqlparser.String(cursor.expr), sqlparser.String(newExpr))
				cursor.abort()
				return SimplifyStatement(in, currentDB, si, test)
			}
			newExpr = s.Next()
		}
		// if we get here, we failed to simplify this expression,
		// so we put back in the original expression
		cursor.restore()
		cursor.wg.Done()
	}

	return in
}

func simplifyStarExpr(in sqlparser.SelectStatement, test func(sqlparser.SelectStatement) bool) bool {
	simplified := false
	sqlparser.Rewrite(in, func(cursor *sqlparser.Cursor) bool {
		se, ok := cursor.Node().(*sqlparser.StarExpr)
		if !ok {
			return true
		}
		cursor.Replace(&sqlparser.AliasedExpr{
			Expr: sqlparser.NewIntLiteral("0"),
		})
		if test(in) {
			log.Errorf("replaced star with literal")
			simplified = true
			return false
		}
		cursor.Replace(se)

		return true
	}, nil)
	return simplified
}

// removeTable removes the table with the given index from the select statement, which includes the FROM clause
// but also all expressions and predicates that depend on the table
func removeTable(clone sqlparser.SelectStatement, searchedTS semantics.TableSet, inner *semantics.SemTable) bool {
	simplified := false
	shouldKeepExpr := func(expr sqlparser.Expr) bool {
		return !inner.RecursiveDeps(expr).IsOverlapping(searchedTS) || sqlparser.ContainsAggregation(expr)
	}
	sqlparser.Rewrite(clone, func(cursor *sqlparser.Cursor) bool {
		switch node := cursor.Node().(type) {
		case *sqlparser.JoinTableExpr:
			lft, ok := node.LeftExpr.(*sqlparser.AliasedTableExpr)
			if ok {
				ts := inner.TableSetFor(lft)
				if ts == searchedTS {
					cursor.Replace(node.RightExpr)
					simplified = true
				}
			}
			rgt, ok := node.RightExpr.(*sqlparser.AliasedTableExpr)
			if ok {
				ts := inner.TableSetFor(rgt)
				if ts == searchedTS {
					cursor.Replace(node.LeftExpr)
					simplified = true
				}
			}
		case *sqlparser.Select:
			if len(node.From) == 1 {
				return true
			}
			for i, tbl := range node.From {
				lft, ok := tbl.(*sqlparser.AliasedTableExpr)
				if ok {
					ts := inner.TableSetFor(lft)
					if ts == searchedTS {
						node.From = append(node.From[:i], node.From[i+1:]...)
						simplified = true
						return true
					}
				}
			}
		case *sqlparser.Where:
			exprs := sqlparser.SplitAndExpression(nil, node.Expr)
			var newPredicate sqlparser.Expr
			for _, expr := range exprs {
				if !inner.RecursiveDeps(expr).IsOverlapping(searchedTS) {
					newPredicate = sqlparser.AndExpressions(newPredicate, expr)
				}
			}
			node.Expr = newPredicate
		case sqlparser.SelectExprs:
			_, isSel := cursor.Parent().(*sqlparser.Select)
			if !isSel {
				return true
			}

			var newExprs sqlparser.SelectExprs
			for _, ae := range node {
				expr, ok := ae.(*sqlparser.AliasedExpr)
				if !ok {
					newExprs = append(newExprs, ae)
					continue
				}
				if shouldKeepExpr(expr.Expr) {
					newExprs = append(newExprs, ae)
				}
			}
			cursor.Replace(newExprs)
		case sqlparser.GroupBy:
			var newExprs sqlparser.GroupBy
			for _, expr := range node {
				if shouldKeepExpr(expr) {
					newExprs = append(newExprs, expr)
				}
			}
			cursor.Replace(newExprs)
		case sqlparser.OrderBy:
			var newExprs sqlparser.OrderBy
			for _, expr := range node {
				if shouldKeepExpr(expr.Expr) {
					newExprs = append(newExprs, expr)
				}
			}

			cursor.Replace(newExprs)
		}
		return true
	}, nil)
	return simplified
}

type expressionCursor struct {
	expr        sqlparser.Expr
	replace     func(replaceWith sqlparser.Expr)
	remove      func() bool
	restore     func()
	wg          *sync.WaitGroup
	abortMarker *sync2.AtomicBool
}

func (i expressionCursor) abort() {
	i.abortMarker.Set(true)
}

func newCursorItem(
	ch chan<- expressionCursor,
	expr sqlparser.Expr,
	abort *sync2.AtomicBool,
	replace func(replaceWith sqlparser.Expr),
	remove func() bool,
	restore func(),
) {
	wg := &sync.WaitGroup{}
	wg.Add(1)
	ch <- expressionCursor{
		expr:        expr,
		replace:     replace,
		remove:      remove,
		restore:     restore,
		wg:          wg,
		abortMarker: abort,
	}
	wg.Wait()
}

func findExpressions(clone sqlparser.SelectStatement, ch chan<- expressionCursor) {
	abort := &sync2.AtomicBool{}
	go func() {
		sqlparser.Rewrite(clone, func(cursor *sqlparser.Cursor) bool {
			switch node := cursor.Node().(type) {
			case sqlparser.SelectExprs:
				_, isSel := cursor.Parent().(*sqlparser.Select)
				if !isSel {
					return true
				}
				for idx := 0; idx < len(node); idx++ {
					ae := node[idx]
					expr, ok := ae.(*sqlparser.AliasedExpr)
					if !ok {
						continue
					}
					removed := false
					original := sqlparser.CloneExpr(expr.Expr)
					newCursorItem(
						ch, expr.Expr, abort,
						/*replace*/ func(replaceWith sqlparser.Expr) {
							if removed {
								panic("cant replace after remove without restore")
							}
							expr.Expr = replaceWith
						},
						/*remove*/ func() bool {
							if removed {
								panic("can't remove twice, silly")
							}
							if len(node) == 1 {
								// can't remove the last expressions - we'd end up with an empty SELECT clause
								return false
							}
							withoutElement := append(node[:idx], node[idx+1:]...)
							cursor.Replace(withoutElement)
							node = withoutElement
							removed = true
							return true
						},
						/*restore*/ func() {
							if removed {
								front := make(sqlparser.SelectExprs, idx)
								copy(front, node[:idx])
								back := make(sqlparser.SelectExprs, len(node)-idx)
								copy(back, node[idx:])
								frontWithRestoredExpr := append(front, ae)
								node = append(frontWithRestoredExpr, back...)
								cursor.Replace(node)
								removed = false
								return
							}
							expr.Expr = original
						},
					)
					if abort.Get() {
						close(ch)
						return false
					}
				}
			case *sqlparser.Where:
				exprs := sqlparser.SplitAndExpression(nil, node.Expr)
				set := func(input []sqlparser.Expr) {
					node.Expr = sqlparser.AndExpressions(input...)
					exprs = input
				}
				if !visitExpressions(exprs, set, ch, abort) {
					return false
				}
			case *sqlparser.JoinTableExpr:

			case *sqlparser.JoinCondition:
				join, ok := cursor.Parent().(*sqlparser.JoinTableExpr)
				if !ok {
					return true
				}
				if join.Join != sqlparser.NormalJoinType || node.Using != nil {
					return false
				}
				exprs := sqlparser.SplitAndExpression(nil, node.On)
				set := func(input []sqlparser.Expr) {
					node.On = sqlparser.AndExpressions(input...)
					exprs = input
				}
				if !visitExpressions(exprs, set, ch, abort) {
					return false
				}
			case sqlparser.GroupBy:
				set := func(input []sqlparser.Expr) {
					node = input
					cursor.Replace(node)
				}
				if !visitExpressions(node, set, ch, abort) {
					return false
				}
			case sqlparser.OrderBy:
				for idx := 0; idx < len(node); idx++ {
					order := node[idx]
					removed := false
					original := sqlparser.CloneExpr(order.Expr)
					newCursorItem(
						ch, order.Expr, abort,
						/*replace*/ func(replaceWith sqlparser.Expr) {
							if removed {
								panic("cant replace after remove without restore")
							}
							order.Expr = replaceWith
						},
						/*remove*/ func() bool {
							if removed {
								panic("can't remove twice, silly")
							}
							withoutElement := append(node[:idx], node[idx+1:]...)
							cursor.Replace(withoutElement)
							node = withoutElement
							removed = true
							return true
						},
						/*restore*/ func() {
							if removed {
								front := make(sqlparser.OrderBy, idx)
								copy(front, node[:idx])
								back := make(sqlparser.OrderBy, len(node)-idx)
								copy(back, node[idx:])
								frontWithRestoredExpr := append(front, order)
								node = append(frontWithRestoredExpr, back...)
								cursor.Replace(node)
								removed = false
								return
							}
							order.Expr = original
						},
					)
					if abort.Get() {
						close(ch)
						return false
					}
				}
			case *sqlparser.Limit:
				if node.Offset != nil {
					original := node.Offset
					newCursorItem(ch, node.Offset, abort,
						/*replace*/ func(replaceWith sqlparser.Expr) {
							node.Offset = replaceWith
						},
						/*remove*/ func() bool {
							node.Offset = nil
							return true
						},
						/*restore*/ func() {
							node.Offset = original
						})
				}
				if node.Rowcount != nil {
					original := node.Rowcount
					newCursorItem(ch, node.Rowcount, abort,
						/*replace*/ func(replaceWith sqlparser.Expr) {
							node.Rowcount = replaceWith
						},
						/*remove*/ func() bool {
							// removing Rowcount is an invalid op
							return false
						},
						/*restore*/ func() {
							node.Rowcount = original
						})
				}

			}
			return true
		}, nil)
		close(ch)
	}()
}

// visitExpressions allows the cursor to visit all expressions in a slice,
// and can replace or remove items and restore the slice.
func visitExpressions(
	exprs []sqlparser.Expr,
	set func(input []sqlparser.Expr),
	ch chan<- expressionCursor,
	abort *sync2.AtomicBool,
) bool {
	for idx := 0; idx < len(exprs); idx++ {
		expr := exprs[idx]
		removed := false
		newCursorItem(ch, expr, abort,
			func(replaceWith sqlparser.Expr) {
				if removed {
					panic("cant replace after remove without restore")
				}
				exprs[idx] = replaceWith
				set(exprs)
			},
			/*remove*/ func() bool {
				if removed {
					panic("can't remove twice, silly")
				}
				exprs = append(exprs[:idx], exprs[idx+1:]...)
				set(exprs)
				removed = true
				return true
			},
			/*restore*/ func() {
				if removed {
					front := make([]sqlparser.Expr, idx)
					copy(front, exprs[:idx])
					back := make([]sqlparser.Expr, len(exprs)-idx)
					copy(back, exprs[idx:])
					frontWithRestoredExpr := append(front, expr)
					exprs = append(frontWithRestoredExpr, back...)
					set(exprs)
					removed = false
					return
				}
				exprs[idx] = expr
				set(exprs)
			})
		if abort.Get() {
			return false
		}
	}
	return true
}
<|endoftext|>


<commit_before>// +build docker

package cockroach

import (
	"database/sql"
	"testing"

	"github.com/cayleygraph/cayley/graph"
	"github.com/cayleygraph/cayley/graph/sql/sqltest"
	"github.com/cayleygraph/cayley/internal/dock"
	"github.com/lib/pq"
)

func makeCockroach(t testing.TB) (string, graph.Options, func()) {
	var conf dock.Config

	conf.Image = "cockroachdb/cockroach:v1.1.2"
	conf.Cmd = []string{"start", "--insecure"}

	addr, closer := dock.RunAndWait(t, conf, "26257", func(addr string) bool {
		conn, err := pq.Open(`postgresql://root@` + addr + `?sslmode=disable`)
		if err != nil {
			return false
		}
		conn.Close()
		return true
	})
	addr = `postgresql://root@` + addr
	db, err := sql.Open(driverName, addr+`?sslmode=disable`)
	if err != nil {
		closer()
		t.Fatal(err)
	}
	defer db.Close()
	const dbName = "cayley"
	if _, err = db.Exec("CREATE DATABASE " + dbName); err != nil {
		closer()
		t.Fatal(err)
	}
	addr = addr + `/` + dbName + `?sslmode=disable`
	return addr, nil, func() {
		closer()
	}
}

var conf = &sqltest.Config{
	TimeRound: true,
}

func TestCockroach(t *testing.T) {
	sqltest.TestAll(t, Type, makeCockroach, conf)
}

func BenchmarkCockroach(t *testing.B) {
	sqltest.BenchmarkAll(t, Type, makeCockroach, conf)
}
<commit_msg>Bump cockroachdb Docker tag for testing to the latest version, v1.1.5.<commit_after>// +build docker

package cockroach

import (
	"database/sql"
	"testing"

	"github.com/cayleygraph/cayley/graph"
	"github.com/cayleygraph/cayley/graph/sql/sqltest"
	"github.com/cayleygraph/cayley/internal/dock"
	"github.com/lib/pq"
)

func makeCockroach(t testing.TB) (string, graph.Options, func()) {
	var conf dock.Config

	conf.Image = "cockroachdb/cockroach:v1.1.5"
	conf.Cmd = []string{"start", "--insecure"}

	addr, closer := dock.RunAndWait(t, conf, "26257", func(addr string) bool {
		conn, err := pq.Open(`postgresql://root@` + addr + `?sslmode=disable`)
		if err != nil {
			return false
		}
		conn.Close()
		return true
	})
	addr = `postgresql://root@` + addr
	db, err := sql.Open(driverName, addr+`?sslmode=disable`)
	if err != nil {
		closer()
		t.Fatal(err)
	}
	defer db.Close()
	const dbName = "cayley"
	if _, err = db.Exec("CREATE DATABASE " + dbName); err != nil {
		closer()
		t.Fatal(err)
	}
	addr = addr + `/` + dbName + `?sslmode=disable`
	return addr, nil, func() {
		closer()
	}
}

var conf = &sqltest.Config{
	TimeRound: true,
}

func TestCockroach(t *testing.T) {
	sqltest.TestAll(t, Type, makeCockroach, conf)
}

func BenchmarkCockroach(t *testing.B) {
	sqltest.BenchmarkAll(t, Type, makeCockroach, conf)
}
<|endoftext|>


<commit_before>package handlers

import (
	"encoding/json"
	"net/http"
	"time"

	"github.com/cloudfoundry-incubator/receptor"
	"github.com/cloudfoundry-incubator/runtime-schema/cc_messages"
	"github.com/cloudfoundry-incubator/runtime-schema/metric"
	"github.com/cloudfoundry-incubator/stager/backend"
	"github.com/cloudfoundry-incubator/stager/cc_client"
	"github.com/pivotal-golang/clock"
	"github.com/pivotal-golang/lager"
)

const (
	// Metrics
	stagingSuccessCounter  = metric.Counter("StagingRequestsSucceeded")
	stagingSuccessDuration = metric.Duration("StagingRequestSucceededDuration")
	stagingFailureCounter  = metric.Counter("StagingRequestsFailed")
	stagingFailureDuration = metric.Duration("StagingRequestFailedDuration")
)

type CompletionHandler interface {
	StagingComplete(resp http.ResponseWriter, req *http.Request)
}

type completionHandler struct {
	ccClient cc_client.CcClient
	backends map[string]backend.Backend
	logger   lager.Logger
	clock    clock.Clock
}

func NewStagingCompletionHandler(logger lager.Logger, ccClient cc_client.CcClient, backends map[string]backend.Backend, clock clock.Clock) CompletionHandler {
	return &completionHandler{
		ccClient: ccClient,
		backends: backends,
		logger:   logger.Session("completion-handler"),
		clock:    clock,
	}
}

func (handler *completionHandler) StagingComplete(res http.ResponseWriter, req *http.Request) {
	taskGuid := req.FormValue(":staging_guid")
	logger := handler.logger.Session("task-complete-callback-received", lager.Data{
		"guid": taskGuid,
	})

	var task receptor.TaskResponse
	err := json.NewDecoder(req.Body).Decode(&task)
	if err != nil {
		handler.logger.Error("parsing-incoming-task-failed", err)
		res.WriteHeader(http.StatusBadRequest)
		return
	}

	if taskGuid != task.TaskGuid {
		handler.logger.Error("task-guid-mismatch", err, lager.Data{"body-task-guid": task.TaskGuid})
		res.WriteHeader(http.StatusBadRequest)
		return
	}

	var annotation cc_messages.StagingTaskAnnotation
	err = json.Unmarshal([]byte(task.Annotation), &annotation)
	if err != nil {
		res.WriteHeader(http.StatusBadRequest)
		logger.Error("parsing-annotation-failed", err)
		return
	}

	backend := handler.backends[annotation.Lifecycle]
	if backend == nil {
		res.WriteHeader(http.StatusNotFound)
		logger.Error("get-staging-response-failed-backend-not-found", err)
		return
	}

	response, err := backend.BuildStagingResponse(task)
	if err != nil {
		res.WriteHeader(http.StatusBadRequest)
		logger.Error("get-staging-response-failed", err)
		return
	}

	responseJson, err := json.Marshal(response)
	if err != nil {
		res.WriteHeader(http.StatusBadRequest)
		logger.Error("get-staging-response-failed", err)
		return
	}

	if responseJson == nil {
		res.WriteHeader(http.StatusNotFound)
		res.Write([]byte("Unknown task domain"))
		return
	}

	logger.Info("posting-staging-complete", lager.Data{
		"payload": responseJson,
	})

	err = handler.ccClient.StagingComplete(taskGuid, responseJson, logger)
	if err != nil {
		logger.Error("cc-staging-complete-failed", err)
		if responseErr, ok := err.(*cc_client.BadResponseError); ok {
			res.WriteHeader(responseErr.StatusCode)
		} else {
			res.WriteHeader(http.StatusServiceUnavailable)
		}
		return
	}

	handler.reportMetrics(task)

	logger.Info("posted-staging-complete")
	res.WriteHeader(http.StatusOK)
}

func (handler *completionHandler) reportMetrics(task receptor.TaskResponse) {
	duration := handler.clock.Now().Sub(time.Unix(0, task.CreatedAt))
	if task.Failed {
		stagingFailureCounter.Increment()
		stagingFailureDuration.Send(duration)
	} else {
		stagingSuccessDuration.Send(duration)
		stagingSuccessCounter.Increment()
	}
}
<commit_msg>Correct logging<commit_after>package handlers

import (
	"encoding/json"
	"net/http"
	"time"

	"github.com/cloudfoundry-incubator/receptor"
	"github.com/cloudfoundry-incubator/runtime-schema/cc_messages"
	"github.com/cloudfoundry-incubator/runtime-schema/metric"
	"github.com/cloudfoundry-incubator/stager/backend"
	"github.com/cloudfoundry-incubator/stager/cc_client"
	"github.com/pivotal-golang/clock"
	"github.com/pivotal-golang/lager"
)

const (
	// Metrics
	stagingSuccessCounter  = metric.Counter("StagingRequestsSucceeded")
	stagingSuccessDuration = metric.Duration("StagingRequestSucceededDuration")
	stagingFailureCounter  = metric.Counter("StagingRequestsFailed")
	stagingFailureDuration = metric.Duration("StagingRequestFailedDuration")
)

type CompletionHandler interface {
	StagingComplete(resp http.ResponseWriter, req *http.Request)
}

type completionHandler struct {
	ccClient cc_client.CcClient
	backends map[string]backend.Backend
	logger   lager.Logger
	clock    clock.Clock
}

func NewStagingCompletionHandler(logger lager.Logger, ccClient cc_client.CcClient, backends map[string]backend.Backend, clock clock.Clock) CompletionHandler {
	return &completionHandler{
		ccClient: ccClient,
		backends: backends,
		logger:   logger.Session("completion-handler"),
		clock:    clock,
	}
}

func (handler *completionHandler) StagingComplete(res http.ResponseWriter, req *http.Request) {
	taskGuid := req.FormValue(":staging_guid")
	logger := handler.logger.Session("task-complete-callback-received", lager.Data{
		"guid": taskGuid,
	})

	var task receptor.TaskResponse
	err := json.NewDecoder(req.Body).Decode(&task)
	if err != nil {
		handler.logger.Error("parsing-incoming-task-failed", err)
		res.WriteHeader(http.StatusBadRequest)
		return
	}

	if taskGuid != task.TaskGuid {
		logger.Error("task-guid-mismatch", err, lager.Data{"body-task-guid": task.TaskGuid})
		res.WriteHeader(http.StatusBadRequest)
		return
	}

	var annotation cc_messages.StagingTaskAnnotation
	err = json.Unmarshal([]byte(task.Annotation), &annotation)
	if err != nil {
		res.WriteHeader(http.StatusBadRequest)
		logger.Error("parsing-annotation-failed", err)
		return
	}

	backend := handler.backends[annotation.Lifecycle]
	if backend == nil {
		res.WriteHeader(http.StatusNotFound)
		logger.Error("get-staging-response-failed-backend-not-found", err)
		return
	}

	response, err := backend.BuildStagingResponse(task)
	if err != nil {
		res.WriteHeader(http.StatusBadRequest)
		logger.Error("get-staging-response-failed", err)
		return
	}

	responseJson, err := json.Marshal(response)
	if err != nil {
		res.WriteHeader(http.StatusBadRequest)
		logger.Error("get-staging-response-failed", err)
		return
	}

	if responseJson == nil {
		res.WriteHeader(http.StatusNotFound)
		res.Write([]byte("Unknown task domain"))
		return
	}

	logger.Info("posting-staging-complete", lager.Data{
		"payload": responseJson,
	})

	err = handler.ccClient.StagingComplete(taskGuid, responseJson, logger)
	if err != nil {
		logger.Error("cc-staging-complete-failed", err)
		if responseErr, ok := err.(*cc_client.BadResponseError); ok {
			res.WriteHeader(responseErr.StatusCode)
		} else {
			res.WriteHeader(http.StatusServiceUnavailable)
		}
		return
	}

	handler.reportMetrics(task)

	logger.Info("posted-staging-complete")
	res.WriteHeader(http.StatusOK)
}

func (handler *completionHandler) reportMetrics(task receptor.TaskResponse) {
	duration := handler.clock.Now().Sub(time.Unix(0, task.CreatedAt))
	if task.Failed {
		stagingFailureCounter.Increment()
		stagingFailureDuration.Send(duration)
	} else {
		stagingSuccessDuration.Send(duration)
		stagingSuccessCounter.Increment()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"net"
	"dns"
	"dns/resolver"
	"os"
	"flag"
	"fmt"
)

var Usage = func() {
	fmt.Fprintf(os.Stderr, "Usage: %s [@server] [qtype] [qclass] [name ...]\n", os.Args[0])
	flag.PrintDefaults()
}

func main() {
	var dnssec *bool = flag.Bool("dnssec", false, "Set the DO (DNSSEC OK) bit and set the bufsize to 4096")
	nameserver := "127.0.0.1"       // Default nameserver
	qtype := uint16(dns.TypeA)      // Default qtype
	qclass := uint16(dns.ClassINET) // Default qclass
	var qname []string

	flag.Parse()

	if *dnssec {
		/* */
	}
FLAGS:
	for i := 0; i < flag.NArg(); i++ {
		// If it starts with @ it is a nameserver
		if flag.Arg(i)[0] == '@' {
			nameserver = flag.Arg(i)
			continue FLAGS
		}
		// If it looks like a class, it is a class
		for k, v := range dns.Class_str {
			if v == flag.Arg(i) {
				qclass = k
				continue FLAGS
			}
		}
		// And if it looks like type, it is a type
		for k, v := range dns.Rr_str {
			if v == flag.Arg(i) {
				qtype = k
				continue FLAGS
			}
		}
		// Anything else is a qname
		qname = append(qname, flag.Arg(i))
	}
	r := new(resolver.Resolver)
        r.Timeout = 2
        r.Attempts = 1

	qr := resolver.NewQuerier(r)
	// @server may be a name, resolv that
	var err os.Error
	_, addr, err := net.LookupHost(nameserver)
        if err == nil {
                r.Servers = addr
        } else {
                r.Servers = []string{nameserver}
        }

	m := new(dns.Msg)
	m.Question = make([]dns.Question, 1)
	for _, v := range qname {
		m.Question[0] = dns.Question{v, qtype, qclass}
		qr <- resolver.DnsMsg{m, nil}
		in := <-qr
		if in.Dns != nil {
			fmt.Printf("%v\n", in.Dns)
		}
	}
	qr <- resolver.DnsMsg{nil, nil}
	<-qr
}
<commit_msg>Add 'q' a dig-like program (only 100 lines of code)<commit_after>package main

import (
	"net"
	"dns"
	"dns/resolver"
	"os"
	"flag"
	"fmt"
        "strings"
)

var Usage = func() {
	fmt.Fprintf(os.Stderr, "Usage: %s [@server] [qtype] [qclass] [name ...]\n", os.Args[0])
	flag.PrintDefaults()
}

func main() {
	var dnssec *bool = flag.Bool("dnssec", false, "Set the DO (DNSSEC OK) bit and set the bufsize to 4096")
	nameserver := "127.0.0.1"       // Default nameserver
	qtype := uint16(dns.TypeA)      // Default qtype
	qclass := uint16(dns.ClassINET) // Default qclass
	var qname []string

	flag.Parse()

	if *dnssec {
		/* */
	}
FLAGS:
	for i := 0; i < flag.NArg(); i++ {
		// If it starts with @ it is a nameserver
		if flag.Arg(i)[0] == '@' {
			nameserver = flag.Arg(i)
			continue FLAGS
		}
		// If it looks like a class, it is a class
		for k, v := range dns.Class_str {
			if v == strings.ToUpper(flag.Arg(i)) {
				qclass = k
				continue FLAGS
			}
		}
		// And if it looks like type, it is a type
		for k, v := range dns.Rr_str {
			if v == strings.ToUpper(flag.Arg(i)) {
				qtype = k
				continue FLAGS
			}
		}
		// Anything else is a qname
		qname = append(qname, flag.Arg(i))
	}
	r := new(resolver.Resolver)
        r.Timeout = 2
        r.Attempts = 1

	qr := resolver.NewQuerier(r)
	// @server may be a name, resolv that
	var err os.Error
	_, addr, err := net.LookupHost(string([]byte(nameserver)[1:]))  //chop off @
        if err == nil {
                r.Servers = addr
        } else {
                r.Servers = []string{nameserver}
        }

	m := new(dns.Msg)
	m.Question = make([]dns.Question, 1)
	for _, v := range qname {
		m.Question[0] = dns.Question{v, qtype, qclass}
		qr <- resolver.DnsMsg{m, nil}
		in := <-qr
		if in.Dns != nil {
			fmt.Printf("%v\n", in.Dns)
		}
	}
	qr <- resolver.DnsMsg{nil, nil}
	<-qr
}
<|endoftext|>


<commit_before>// +build freebsd linux

package configs

import "fmt"

// Gets the root uid for the process on host which could be non-zero
// when user namespaces are enabled.
func (c Config) HostUID() (int, error) {
	if c.Namespaces.Contains(NEWUSER) {
		if c.UidMappings == nil {
			return -1, fmt.Errorf("User namespaces enabled, but no user mappings found.")
		}
		id, found := c.hostIDFromMapping(0, c.UidMappings)
		if !found {
			return -1, fmt.Errorf("User namespaces enabled, but no root user mapping found.")
		}
		return id, nil
	}
	// Return default root uid 0
	return 0, nil
}

// Gets the root uid for the process on host which could be non-zero
// when user namespaces are enabled.
func (c Config) HostGID() (int, error) {
	if c.Namespaces.Contains(NEWUSER) {
		if c.GidMappings == nil {
			return -1, fmt.Errorf("User namespaces enabled, but no gid mappings found.")
		}
		id, found := c.hostIDFromMapping(0, c.GidMappings)
		if !found {
			return -1, fmt.Errorf("User namespaces enabled, but no root user mapping found.")
		}
		return id, nil
	}
	// Return default root uid 0
	return 0, nil
}

// Utility function that gets a host ID for a container ID from user namespace map
// if that ID is present in the map.
func (c Config) hostIDFromMapping(containerID int, uMap []IDMap) (int, bool) {
	for _, m := range uMap {
		if (containerID >= m.ContainerID) && (containerID <= (m.ContainerID + m.Size - 1)) {
			hostID := m.HostID + (containerID - m.ContainerID)
			return hostID, true
		}
	}
	return -1, false
}
<commit_msg>change uid to gid in func HostGID<commit_after>// +build freebsd linux

package configs

import "fmt"

// Gets the root uid for the process on host which could be non-zero
// when user namespaces are enabled.
func (c Config) HostUID() (int, error) {
	if c.Namespaces.Contains(NEWUSER) {
		if c.UidMappings == nil {
			return -1, fmt.Errorf("User namespaces enabled, but no user mappings found.")
		}
		id, found := c.hostIDFromMapping(0, c.UidMappings)
		if !found {
			return -1, fmt.Errorf("User namespaces enabled, but no root user mapping found.")
		}
		return id, nil
	}
	// Return default root uid 0
	return 0, nil
}

// Gets the root gid for the process on host which could be non-zero
// when user namespaces are enabled.
func (c Config) HostGID() (int, error) {
	if c.Namespaces.Contains(NEWUSER) {
		if c.GidMappings == nil {
			return -1, fmt.Errorf("User namespaces enabled, but no gid mappings found.")
		}
		id, found := c.hostIDFromMapping(0, c.GidMappings)
		if !found {
			return -1, fmt.Errorf("User namespaces enabled, but no root group mapping found.")
		}
		return id, nil
	}
	// Return default root gid 0
	return 0, nil
}

// Utility function that gets a host ID for a container ID from user namespace map
// if that ID is present in the map.
func (c Config) hostIDFromMapping(containerID int, uMap []IDMap) (int, bool) {
	for _, m := range uMap {
		if (containerID >= m.ContainerID) && (containerID <= (m.ContainerID + m.Size - 1)) {
			hostID := m.HostID + (containerID - m.ContainerID)
			return hostID, true
		}
	}
	return -1, false
}
<|endoftext|>


<commit_before>// Copyright 2018 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package libhttpserver

import (
	"net/http"
	"strings"
)

type contentTypeOverridingResponseWriter struct {
	original http.ResponseWriter
}

var _ http.ResponseWriter = (*contentTypeOverridingResponseWriter)(nil)

func newContentTypeOverridingResponseWriter(
	original http.ResponseWriter) *contentTypeOverridingResponseWriter {
	return &contentTypeOverridingResponseWriter{
		original: original,
	}
}

func (w *contentTypeOverridingResponseWriter) overrideMimeType(
	mimeType string) (newMimeType string) {
	// Send text/plain for all HTML and JS files to avoid them being executed
	// by the frontend WebView.
	ty := strings.ToLower(mimeType)
	switch {
	case strings.Contains(ty, "javascript") ||
		strings.Contains(ty, "xml") ||
		strings.Contains(ty, "html"):
		return "text/plain"
	case strings.HasPrefix(ty, "audio/") ||
		strings.HasPrefix(ty, "image/") ||
		strings.HasPrefix(ty, "video/") ||
		ty == "application/pdf":
		return ty
	default:
		return "text/plain"
	}
}

func (w *contentTypeOverridingResponseWriter) override() {
	t := w.original.Header().Get("Content-Type")
	if len(t) > 0 {
		w.original.Header().Set("Content-Type", w.overrideMimeType(t))
	}
	w.original.Header().Set("X-Content-Type-Options", "nosniff")
}

func (w *contentTypeOverridingResponseWriter) Header() http.Header {
	return w.original.Header()
}

func (w *contentTypeOverridingResponseWriter) WriteHeader(statusCode int) {
	w.override()
	w.original.WriteHeader(statusCode)
}

func (w *contentTypeOverridingResponseWriter) Write(data []byte) (int, error) {
	w.override()
	return w.original.Write(data)
}

var additionalMimeTypes = map[string]string{
	".go":    "text/plain",
	".py":    "text/plain",
	".zsh":   "text/plain",
	".fish":  "text/plain",
	".cs":    "text/plain",
	".rb":    "text/plain",
	".m":     "text/plain",
	".mm":    "text/plain",
	".swift": "text/plain",
	".flow":  "text/plain",
	".php":   "text/plain",
	".pl":    "text/plain",
	".sh":    "text/plain",
	".js":    "text/plain",
	".json":  "text/plain",
	".sql":   "text/plain",
	".rs":    "text/plain",
	".xml":   "text/plain",
	".tex":   "text/plain",
	".pub":   "text/plain",
}
<commit_msg>libhttpserver: Add comment<commit_after>// Copyright 2018 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package libhttpserver

import (
	"net/http"
	"strings"
)

type contentTypeOverridingResponseWriter struct {
	original http.ResponseWriter
}

var _ http.ResponseWriter = (*contentTypeOverridingResponseWriter)(nil)

func newContentTypeOverridingResponseWriter(
	original http.ResponseWriter) *contentTypeOverridingResponseWriter {
	return &contentTypeOverridingResponseWriter{
		original: original,
	}
}

func (w *contentTypeOverridingResponseWriter) overrideMimeType(
	mimeType string) (newMimeType string) {
	// Send text/plain for all HTML and JS files to avoid them being executed
	// by the frontend WebView.
	ty := strings.ToLower(mimeType)
	switch {
	// First reject anything containing javascript/xml/html.
	case strings.Contains(ty, "javascript") ||
		strings.Contains(ty, "xml") ||
		strings.Contains(ty, "html"):
		return "text/plain"
	// Pass multimedia types through, and pdf too.
	case strings.HasPrefix(ty, "audio/") ||
		strings.HasPrefix(ty, "image/") ||
		strings.HasPrefix(ty, "video/") ||
		ty == "application/pdf":
		return ty
	// Otherwise text/plain.
	default:
		return "text/plain"
	}
}

func (w *contentTypeOverridingResponseWriter) override() {
	t := w.original.Header().Get("Content-Type")
	if len(t) > 0 {
		w.original.Header().Set("Content-Type", w.overrideMimeType(t))
	}
	w.original.Header().Set("X-Content-Type-Options", "nosniff")
}

func (w *contentTypeOverridingResponseWriter) Header() http.Header {
	return w.original.Header()
}

func (w *contentTypeOverridingResponseWriter) WriteHeader(statusCode int) {
	w.override()
	w.original.WriteHeader(statusCode)
}

func (w *contentTypeOverridingResponseWriter) Write(data []byte) (int, error) {
	w.override()
	return w.original.Write(data)
}

var additionalMimeTypes = map[string]string{
	".go":    "text/plain",
	".py":    "text/plain",
	".zsh":   "text/plain",
	".fish":  "text/plain",
	".cs":    "text/plain",
	".rb":    "text/plain",
	".m":     "text/plain",
	".mm":    "text/plain",
	".swift": "text/plain",
	".flow":  "text/plain",
	".php":   "text/plain",
	".pl":    "text/plain",
	".sh":    "text/plain",
	".js":    "text/plain",
	".json":  "text/plain",
	".sql":   "text/plain",
	".rs":    "text/plain",
	".xml":   "text/plain",
	".tex":   "text/plain",
	".pub":   "text/plain",
}
<|endoftext|>


<commit_before>package api

import (
	//	"gopkg.in/mgo.v2"
	"appengine/aetest"
	"appengine/datastore"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/ErikBjare/Futarchio/src/db"
	"io/ioutil"
	"log"
	"net/http"
	"testing"
	"time"
)

func init() {
	_, err := http.Get("http://localhost:8080/api/0/init")
	if err != nil {
		log.Fatal(err)
	}
}

func getAuthkey() (string, error) {
	client := &http.Client{}
	bodybuf := &bytes.Buffer{}
	bodybuf.Write([]byte("{\"username\": \"erb\", \"password\": \"password\"}"))
	resp, err := client.Post("http://localhost:8080/api/0/auth", "application/json", bodybuf)
	if err != nil {
		return "", err
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}

	msg := db.Auth{}
	err = json.Unmarshal(body, &msg)
	if err != nil {
		return "", err
	}

	return msg.Key, nil
}

func TestAuth(t *testing.T) {
	authkey, err := getAuthkey()
	if err != nil {
		t.Fatal(err)
	}

	// Apparently required to allow the datastore time to be able to store Auth
	// TODO: Should probably be removed once Memcache is implemented
	time.Sleep(time.Second)

	body, err := getBody("http://localhost:8080/api/0/users", authkey)
	if err != nil {
		t.Fatal(err)
	}
	// The following comment line can be useful for debugging by printing body
	//log.Println(string(body))

	var users []db.User
	err = json.Unmarshal(body, &users)
	if err != nil {
		t.Error(err)
	}

	body, err = getBody("http://localhost:8080/api/0/users/me", authkey)
	if err != nil {
		t.Fatal(err)
	}
	// The following comment line can be useful for debugging by printing body
	//log.Println(string(body))

	var user db.User
	err = json.Unmarshal(body, &user)
	if err != nil {
		t.Error(err)
	}
}

func getBody(url string, authkey string) ([]byte, error) {
	client := &http.Client{}
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Add("Authorization", authkey)
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		err := errors.New(fmt.Sprintf("Status code was not 200, was %d with message: %s", resp.StatusCode, resp.Status))
		return nil, err
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	return body, err
}

func TestUsers(t *testing.T) {
	c, err := aetest.NewContext(nil)
	if err != nil {
		t.Fatal(err)
	}
	defer c.Close()

	key := datastore.NewKey(c, "User", "erb", 0, nil)
	user1 := db.NewUser("erb", "secretpassword", "Erik", "erik@bjareho.lt")
	_, err = datastore.Put(c, key, &user1)
	if err != nil {
		t.Fatal(err)
	}

	var user2 db.User
	err = datastore.Get(c, key, &user2)
	if err != nil {
		t.Error(err)
	}

	var users []db.User
	q := datastore.NewQuery("User").Filter("Email =", "erik@bjareho.lt")
	keys, err := q.GetAll(c, &users)
	if err != nil {
		t.Error(err)
	}

	if len(keys) == 0 {
		t.Error("Couldn't find Erik in database")
	} else if len(keys) > 1 {
		t.Error("More than one user with email erik@bjareho.lt in database")
	}
}
<commit_msg>Readded sleep<commit_after>package api

import (
	//	"gopkg.in/mgo.v2"
	"appengine/aetest"
	"appengine/datastore"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/ErikBjare/Futarchio/src/db"
	"io/ioutil"
	"log"
	"net/http"
	"testing"
	"time"
)

func init() {
	_, err := http.Get("http://localhost:8080/api/0/init")
	if err != nil {
		log.Fatal(err)
	}

	// TODO: Should probably be removed once Memcache is implemented
	time.Sleep(time.Second)
}

func getAuthkey() (string, error) {
	client := &http.Client{}
	bodybuf := &bytes.Buffer{}
	bodybuf.Write([]byte("{\"username\": \"erb\", \"password\": \"password\"}"))
	resp, err := client.Post("http://localhost:8080/api/0/auth", "application/json", bodybuf)
	if err != nil {
		return "", err
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}

	msg := db.Auth{}
	err = json.Unmarshal(body, &msg)
	if err != nil {
		return "", err
	}

	return msg.Key, nil
}

func TestAuth(t *testing.T) {
	authkey, err := getAuthkey()
	if err != nil {
		t.Fatal(err)
	}

	// Apparently required to allow the datastore time to be able to store Auth
	// TODO: Should probably be removed once Memcache is implemented
	time.Sleep(time.Second)

	body, err := getBody("http://localhost:8080/api/0/users", authkey)
	if err != nil {
		t.Fatal(err)
	}
	// The following comment line can be useful for debugging by printing body
	//log.Println(string(body))

	var users []db.User
	err = json.Unmarshal(body, &users)
	if err != nil {
		t.Error(err)
	}

	body, err = getBody("http://localhost:8080/api/0/users/me", authkey)
	if err != nil {
		t.Fatal(err)
	}
	// The following comment line can be useful for debugging by printing body
	//log.Println(string(body))

	var user db.User
	err = json.Unmarshal(body, &user)
	if err != nil {
		t.Error(err)
	}
}

func getBody(url string, authkey string) ([]byte, error) {
	client := &http.Client{}
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Add("Authorization", authkey)
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		err := errors.New(fmt.Sprintf("Status code was not 200, was %d with message: %s", resp.StatusCode, resp.Status))
		return nil, err
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	return body, err
}

func TestUsers(t *testing.T) {
	c, err := aetest.NewContext(nil)
	if err != nil {
		t.Fatal(err)
	}
	defer c.Close()

	key := datastore.NewKey(c, "User", "erb", 0, nil)
	user1 := db.NewUser("erb", "secretpassword", "Erik", "erik@bjareho.lt")
	_, err = datastore.Put(c, key, &user1)
	if err != nil {
		t.Fatal(err)
	}

	var user2 db.User
	err = datastore.Get(c, key, &user2)
	if err != nil {
		t.Error(err)
	}

	var users []db.User
	q := datastore.NewQuery("User").Filter("Email =", "erik@bjareho.lt")
	keys, err := q.GetAll(c, &users)
	if err != nil {
		t.Error(err)
	}

	if len(keys) == 0 {
		t.Error("Couldn't find Erik in database")
	} else if len(keys) > 1 {
		t.Error("More than one user with email erik@bjareho.lt in database")
	}
}
<|endoftext|>


<commit_before>package action

import (
	"errors"

	boshsettings "github.com/cloudfoundry/bosh-agent/settings"
	bosherr "github.com/cloudfoundry/bosh-utils/errors"
)

type RemovePersistentDiskAction struct {
	settingsService boshsettings.Service
}

func NewRemovePersistentDiskAction(settingsService boshsettings.Service) RemovePersistentDiskAction {
	return RemovePersistentDiskAction{
		settingsService: settingsService,
	}
}

func (a RemovePersistentDiskAction) Run(diskCID string) (interface{}, error) {
	err := a.settingsService.LoadSettings()
	if err != nil {
		return nil, bosherr.WrapError(err, "Refreshing the settings")
	}

	if err := a.settingsService.RemovePersistentDiskSettings(diskCID); err != nil {
		return "", bosherr.WrapError(err, "Removing persistent disk hints")
	}

	return map[string]string{}, nil
}

func (a RemovePersistentDiskAction) IsAsynchronous(_ ProtocolVersion) bool {
	return false
}

func (a RemovePersistentDiskAction) IsPersistent() bool {
	return false
}

func (a RemovePersistentDiskAction) IsLoggable() bool {
	return true
}

func (a RemovePersistentDiskAction) Resume() (interface{}, error) {
	return nil, errors.New("not supported")
}

func (a RemovePersistentDiskAction) Cancel() error {
	return errors.New("not supported")
}
<commit_msg>Made remove_persistent_disk asyncronous.<commit_after>package action

import (
	"errors"

	boshsettings "github.com/cloudfoundry/bosh-agent/settings"
	bosherr "github.com/cloudfoundry/bosh-utils/errors"
)

type RemovePersistentDiskAction struct {
	settingsService boshsettings.Service
}

func NewRemovePersistentDiskAction(settingsService boshsettings.Service) RemovePersistentDiskAction {
	return RemovePersistentDiskAction{
		settingsService: settingsService,
	}
}

func (a RemovePersistentDiskAction) Run(diskCID string) (interface{}, error) {
	err := a.settingsService.LoadSettings()
	if err != nil {
		return nil, bosherr.WrapError(err, "Refreshing the settings")
	}

	if err := a.settingsService.RemovePersistentDiskSettings(diskCID); err != nil {
		return "", bosherr.WrapError(err, "Removing persistent disk hints")
	}

	return map[string]string{}, nil
}

func (a RemovePersistentDiskAction) IsAsynchronous(_ ProtocolVersion) bool {
	return true
}

func (a RemovePersistentDiskAction) IsPersistent() bool {
	return false
}

func (a RemovePersistentDiskAction) IsLoggable() bool {
	return true
}

func (a RemovePersistentDiskAction) Resume() (interface{}, error) {
	return nil, errors.New("not supported")
}

func (a RemovePersistentDiskAction) Cancel() error {
	return errors.New("not supported")
}
<|endoftext|>