<commit_before>// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"time"

	workflowclientSet "github.com/argoproj/argo/pkg/client/clientset/versioned"
	workflowinformers "github.com/argoproj/argo/pkg/client/informers/externalversions"
	"github.com/kubeflow/pipelines/backend/src/agent/persistence/client"
	"github.com/kubeflow/pipelines/backend/src/common/util"
	swfclientset "github.com/kubeflow/pipelines/backend/src/crd/pkg/client/clientset/versioned"
	swfinformers "github.com/kubeflow/pipelines/backend/src/crd/pkg/client/informers/externalversions"
	"github.com/kubeflow/pipelines/backend/src/crd/pkg/signals"
	log "github.com/sirupsen/logrus"
	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
	"k8s.io/client-go/tools/clientcmd"
)

var (
	masterURL                   string
	kubeconfig                  string
	initializeTimeout           time.Duration
	timeout                     time.Duration
	mlPipelineAPIServerName     string
	mlPipelineAPIServerPort     string
	mlPipelineAPIServerBasePath string
	mlPipelineServiceHttpPort   string
	mlPipelineServiceGRPCPort   string
)

const (
	kubeconfigFlagName                  = "kubeconfig"
	masterFlagName                      = "master"
	initializationTimeoutFlagName       = "initializeTimeout"
	timeoutFlagName                     = "timeout"
	mlPipelineAPIServerBasePathFlagName = "mlPipelineAPIServerBasePath"
	mlPipelineAPIServerNameFlagName     = "mlPipelineAPIServerName"
	mlPipelineAPIServerHttpPortFlagName = "mlPipelineServiceHttpPort"
	mlPipelineAPIServerGRPCPortFlagName = "mlPipelineServiceGRPCPort"
)

func main() {
	flag.Parse()

	// set up signals so we handle the first shutdown signal gracefully
	stopCh := signals.SetupSignalHandler()

	cfg, err := clientcmd.BuildConfigFromFlags(masterURL, kubeconfig)
	if err != nil {
		log.Fatalf("Error building kubeconfig: %s", err.Error())
	}

	swfClient, err := swfclientset.NewForConfig(cfg)
	if err != nil {
		log.Fatalf("Error building schedule clientset: %s", err.Error())
	}

	workflowClient, err := workflowclientSet.NewForConfig(cfg)
	if err != nil {
		log.Fatalf("Error building workflow clientset: %s", err.Error())
	}

	swfInformerFactory := swfinformers.NewSharedInformerFactory(swfClient, time.Second*30)
	workflowInformerFactory := workflowinformers.NewSharedInformerFactory(workflowClient, time.Second*30)

	pipelineClient, err := client.NewPipelineClient(
		initializeTimeout,
		timeout,
		mlPipelineAPIServerBasePath,
		mlPipelineAPIServerName,
		mlPipelineServiceHttpPort,
		mlPipelineServiceGRPCPort)
	if err != nil {
		log.Fatalf("Error creating ML pipeline API Server client: %v", err)
	}

	controller := NewPersistenceAgent(
		swfInformerFactory,
		workflowInformerFactory,
		pipelineClient,
		util.NewRealTime())

	go swfInformerFactory.Start(stopCh)
	go workflowInformerFactory.Start(stopCh)

	if err = controller.Run(2, stopCh); err != nil {
		log.Fatalf("Error running controller: %s", err.Error())
	}
}

func init() {
	flag.StringVar(&kubeconfig, kubeconfigFlagName, "", "Path to a kubeconfig. Only required if out-of-cluster.")
	flag.StringVar(&masterURL, masterFlagName, "", "The address of the Kubernetes API server. Overrides any value in kubeconfig. Only required if out-of-cluster.")
	flag.DurationVar(&initializeTimeout, initializationTimeoutFlagName, 2*time.Minute, "Duration to wait for initialization of the ML pipeline API server.")
	flag.DurationVar(&timeout, timeoutFlagName, 1*time.Minute, "Duration to wait for calls to complete.")
	flag.StringVar(&mlPipelineAPIServerName, mlPipelineAPIServerNameFlagName, "ml-pipeline", "Name of the ML pipeline API server.")
	flag.StringVar(&mlPipelineServiceHttpPort, mlPipelineAPIServerHttpPortFlagName, "8888", "Http Port of the ML pipeline API server.")
	flag.StringVar(&mlPipelineServiceGRPCPort, mlPipelineAPIServerGRPCPortFlagName, "8887", "GRPC Port of the ML pipeline API server.")
	flag.StringVar(&mlPipelineAPIServerBasePath, mlPipelineAPIServerBasePathFlagName,
		"/apis/v1beta1", "The base path for the ML pipeline API server.")
}
<commit_msg>Allow to specify informers namespace in persistence agent (#901)<commit_after>// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"time"

	workflowclientSet "github.com/argoproj/argo/pkg/client/clientset/versioned"
	workflowinformers "github.com/argoproj/argo/pkg/client/informers/externalversions"
	"github.com/kubeflow/pipelines/backend/src/agent/persistence/client"
	"github.com/kubeflow/pipelines/backend/src/common/util"
	swfclientset "github.com/kubeflow/pipelines/backend/src/crd/pkg/client/clientset/versioned"
	swfinformers "github.com/kubeflow/pipelines/backend/src/crd/pkg/client/informers/externalversions"
	"github.com/kubeflow/pipelines/backend/src/crd/pkg/signals"
	log "github.com/sirupsen/logrus"
	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
	"k8s.io/client-go/tools/clientcmd"
)

var (
	masterURL                   string
	kubeconfig                  string
	initializeTimeout           time.Duration
	timeout                     time.Duration
	mlPipelineAPIServerName     string
	mlPipelineAPIServerPort     string
	mlPipelineAPIServerBasePath string
	mlPipelineServiceHttpPort   string
	mlPipelineServiceGRPCPort   string
	namespace                   string
)

const (
	kubeconfigFlagName                  = "kubeconfig"
	masterFlagName                      = "master"
	initializationTimeoutFlagName       = "initializeTimeout"
	timeoutFlagName                     = "timeout"
	mlPipelineAPIServerBasePathFlagName = "mlPipelineAPIServerBasePath"
	mlPipelineAPIServerNameFlagName     = "mlPipelineAPIServerName"
	mlPipelineAPIServerHttpPortFlagName = "mlPipelineServiceHttpPort"
	mlPipelineAPIServerGRPCPortFlagName = "mlPipelineServiceGRPCPort"
	namespaceFlagName                   = "namespace"
)

func main() {
	flag.Parse()

	// set up signals so we handle the first shutdown signal gracefully
	stopCh := signals.SetupSignalHandler()

	cfg, err := clientcmd.BuildConfigFromFlags(masterURL, kubeconfig)
	if err != nil {
		log.Fatalf("Error building kubeconfig: %s", err.Error())
	}

	swfClient, err := swfclientset.NewForConfig(cfg)
	if err != nil {
		log.Fatalf("Error building schedule clientset: %s", err.Error())
	}

	workflowClient, err := workflowclientSet.NewForConfig(cfg)
	if err != nil {
		log.Fatalf("Error building workflow clientset: %s", err.Error())
	}

	var swfInformerFactory swfinformers.SharedInformerFactory
	var workflowInformerFactory workflowinformers.SharedInformerFactory
	if namespace == "" {
		swfInformerFactory = swfinformers.NewSharedInformerFactory(swfClient, time.Second*30)
		workflowInformerFactory = workflowinformers.NewSharedInformerFactory(workflowClient, time.Second*30)
	} else {
		swfInformerFactory = swfinformers.NewFilteredSharedInformerFactory(swfClient, time.Second*30, namespace, nil)
		workflowInformerFactory = workflowinformers.NewFilteredSharedInformerFactory(workflowClient, time.Second*30, namespace, nil)
	}

	pipelineClient, err := client.NewPipelineClient(
		initializeTimeout,
		timeout,
		mlPipelineAPIServerBasePath,
		mlPipelineAPIServerName,
		mlPipelineServiceHttpPort,
		mlPipelineServiceGRPCPort)
	if err != nil {
		log.Fatalf("Error creating ML pipeline API Server client: %v", err)
	}

	controller := NewPersistenceAgent(
		swfInformerFactory,
		workflowInformerFactory,
		pipelineClient,
		util.NewRealTime())

	go swfInformerFactory.Start(stopCh)
	go workflowInformerFactory.Start(stopCh)

	if err = controller.Run(2, stopCh); err != nil {
		log.Fatalf("Error running controller: %s", err.Error())
	}
}

func init() {
	flag.StringVar(&kubeconfig, kubeconfigFlagName, "", "Path to a kubeconfig. Only required if out-of-cluster.")
	flag.StringVar(&masterURL, masterFlagName, "", "The address of the Kubernetes API server. Overrides any value in kubeconfig. Only required if out-of-cluster.")
	flag.DurationVar(&initializeTimeout, initializationTimeoutFlagName, 2*time.Minute, "Duration to wait for initialization of the ML pipeline API server.")
	flag.DurationVar(&timeout, timeoutFlagName, 1*time.Minute, "Duration to wait for calls to complete.")
	flag.StringVar(&mlPipelineAPIServerName, mlPipelineAPIServerNameFlagName, "ml-pipeline", "Name of the ML pipeline API server.")
	flag.StringVar(&mlPipelineServiceHttpPort, mlPipelineAPIServerHttpPortFlagName, "8888", "Http Port of the ML pipeline API server.")
	flag.StringVar(&mlPipelineServiceGRPCPort, mlPipelineAPIServerGRPCPortFlagName, "8887", "GRPC Port of the ML pipeline API server.")
	flag.StringVar(&mlPipelineAPIServerBasePath, mlPipelineAPIServerBasePathFlagName,
		"/apis/v1beta1", "The base path for the ML pipeline API server.")
	flag.StringVar(&namespace, namespaceFlagName, "", "The namespace name used for Kubernetes informers to obtain the listers.")
}
<|endoftext|>


<commit_before>package metrics

import (
	"context"
	"net/http"

	"github.com/projecteru2/core/cluster"
	log "github.com/sirupsen/logrus"
)

// ResourceMiddleware to make sure update resource correct
func (m *Metrics) ResourceMiddleware(cluster cluster.Cluster) func(http.Handler) http.Handler {
	return func(h http.Handler) http.Handler {
		ctx, cancel := context.WithTimeout(context.Background(), m.Config.GlobalTimeout)
		defer cancel()
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			pods, err := cluster.ListPods(ctx)
			if err != nil {
				log.Errorf("[ResourceMiddleware] List pods err %v", err)
				// Nothing to do here, pods will be nil
			}
			for _, pod := range pods {
				nodes, err := cluster.ListPodNodes(ctx, pod.Name, nil, true)
				if err != nil {
					log.Errorf("[ResourceMiddleware] List pod %s nodes err %v", pod.Name, err)
					continue
				}
				for _, node := range nodes {
					m.SendNodeInfo(node)
				}
			}
			h.ServeHTTP(w, r)
		})
	}
}
<commit_msg>metrics: Fix ctx scope. (#216)<commit_after>package metrics

import (
	"context"
	"net/http"

	"github.com/projecteru2/core/cluster"
	log "github.com/sirupsen/logrus"
)

// ResourceMiddleware to make sure update resource correct
func (m *Metrics) ResourceMiddleware(cluster cluster.Cluster) func(http.Handler) http.Handler {
	return func(h http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			ctx, cancel := context.WithTimeout(context.Background(), m.Config.GlobalTimeout)
			defer cancel()
			pods, err := cluster.ListPods(ctx)
			if err != nil {
				log.Errorf("[ResourceMiddleware] List pods err %v", err)
				// Nothing to do here, pods will be nil
			}
			for _, pod := range pods {
				nodes, err := cluster.ListPodNodes(ctx, pod.Name, nil, true)
				if err != nil {
					log.Errorf("[ResourceMiddleware] List pod %s nodes err %v", pod.Name, err)
					continue
				}
				for _, node := range nodes {
					m.SendNodeInfo(node)
				}
			}
			h.ServeHTTP(w, r)
		})
	}
}
<|endoftext|>


<commit_before>//go:generate mockgen -destination mock/cloudwatchiface.go github.com/aws/aws-sdk-go/service/cloudwatch/cloudwatchiface CloudWatchAPI

// Package metrics fetches CloudWatch metrics for a function.
package metrics

import (
	"fmt"
	"sync"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/cloudwatch"
	"github.com/aws/aws-sdk-go/service/cloudwatch/cloudwatchiface"
)

// MetricCollector a wrapper all metrics for a specific function.
type MetricCollector struct {
	Metrics      []string
	Collected    int
	FunctionName string
	Service      cloudwatchiface.CloudWatchAPI
	StartDate    time.Time
	EndDate      time.Time
}

// Metric represents a CloudWatch metric with a given name and value.
type Metric struct {
	Name  string
	Value []*cloudwatch.Datapoint
}

// Collect builds the collector pipeline
func (mc *MetricCollector) Collect() <-chan Metric {
	return mc.collect(mc.gen())
}

// collect starts a new cloudwatch session and requests the key metrics.
func (mc *MetricCollector) collect(in <-chan string) <-chan Metric {
	var wg sync.WaitGroup
	out := make(chan Metric)

	for name := range in {
		wg.Add(1)
		name := name

		go func() {
			defer wg.Done()

			m := &Metric{Name: name}

			params := &cloudwatch.GetMetricStatisticsInput{
				StartTime:  aws.Time(mc.StartDate),
				EndTime:    aws.Time(mc.EndDate),
				MetricName: aws.String(m.Name),
				Namespace:  aws.String("AWS/Lambda"),
				Period:     aws.Int64(int64(period(mc.StartDate, mc.EndDate).Seconds())),
				Statistics: []*string{
					aws.String("Sum"),
				},
				Dimensions: []*cloudwatch.Dimension{
					{
						Name:  aws.String("FunctionName"),
						Value: aws.String(mc.FunctionName),
					},
				},
				Unit: aws.String(unit(name)),
			}

			resp, err := mc.Service.GetMetricStatistics(params)

			if err != nil {
				// TODO: refactor so that errors are reported in cmd
				fmt.Println(err.Error())
				return
			}

			m.Value = resp.Datapoints

			out <- *m
		}()
	}

	go func() {
		wg.Wait()
		close(out)
	}()

	return out
}

// gen generates the key metric structs and returns a channel pipeline.
func (mc *MetricCollector) gen() <-chan string {
	out := make(chan string, len(mc.Metrics))
	for _, n := range mc.Metrics {
		out <- n
	}
	close(out)
	return out
}

// period returns the resolution of metrics.
func period(start, end time.Time) time.Duration {
	switch n := end.Sub(start).Hours(); {
	case n > 24:
		return time.Hour * 24
	default:
		return time.Hour
	}
}

// unit for metric name.
func unit(name string) string {
	switch name {
	case "Duration":
		return "Milliseconds"
	default:
		return "Count"
	}
}
<commit_msg>refactor chan send in metrics<commit_after>//go:generate mockgen -destination mock/cloudwatchiface.go github.com/aws/aws-sdk-go/service/cloudwatch/cloudwatchiface CloudWatchAPI

// Package metrics fetches CloudWatch metrics for a function.
package metrics

import (
	"fmt"
	"sync"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/cloudwatch"
	"github.com/aws/aws-sdk-go/service/cloudwatch/cloudwatchiface"
)

// MetricCollector a wrapper all metrics for a specific function.
type MetricCollector struct {
	Metrics      []string
	Collected    int
	FunctionName string
	Service      cloudwatchiface.CloudWatchAPI
	StartDate    time.Time
	EndDate      time.Time
}

// Metric represents a CloudWatch metric with a given name and value.
type Metric struct {
	Name  string
	Value []*cloudwatch.Datapoint
}

// Collect builds the collector pipeline
func (mc *MetricCollector) Collect() <-chan Metric {
	return mc.collect(mc.gen())
}

// collect starts a new cloudwatch session and requests the key metrics.
func (mc *MetricCollector) collect(in <-chan string) <-chan Metric {
	var wg sync.WaitGroup
	out := make(chan Metric)

	for name := range in {
		wg.Add(1)
		name := name

		go func() {
			defer wg.Done()

			params := &cloudwatch.GetMetricStatisticsInput{
				StartTime:  aws.Time(mc.StartDate),
				EndTime:    aws.Time(mc.EndDate),
				MetricName: aws.String(name),
				Namespace:  aws.String("AWS/Lambda"),
				Period:     aws.Int64(int64(period(mc.StartDate, mc.EndDate).Seconds())),
				Statistics: []*string{
					aws.String("Sum"),
				},
				Dimensions: []*cloudwatch.Dimension{
					{
						Name:  aws.String("FunctionName"),
						Value: aws.String(mc.FunctionName),
					},
				},
				Unit: aws.String(unit(name)),
			}

			res, err := mc.Service.GetMetricStatistics(params)

			if err != nil {
				// TODO: refactor so that errors are reported in cmd
				fmt.Println(err.Error())
				return
			}

			out <- Metric{
				Name:  name,
				Value: res.Datapoints,
			}
		}()
	}

	go func() {
		wg.Wait()
		close(out)
	}()

	return out
}

// gen generates the key metric structs and returns a channel pipeline.
func (mc *MetricCollector) gen() <-chan string {
	out := make(chan string, len(mc.Metrics))
	for _, n := range mc.Metrics {
		out <- n
	}
	close(out)
	return out
}

// period returns the resolution of metrics.
func period(start, end time.Time) time.Duration {
	switch n := end.Sub(start).Hours(); {
	case n > 24:
		return time.Hour * 24
	default:
		return time.Hour
	}
}

// unit for metric name.
func unit(name string) string {
	switch name {
	case "Duration":
		return "Milliseconds"
	default:
		return "Count"
	}
}
<|endoftext|>


<commit_before>package metrics

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"

	log "github.com/Sirupsen/logrus"
)

// PrometheusMetrics is a struct to push metrics to Prometheus
type PrometheusMetrics struct {
	Instance       string
	PushgatewayURL string
	Metrics        map[string]*Metric
}

// Metric is a Prometheus Metric
type Metric struct {
	Name   string
	Events []*Event
	Type   string
}

// Event is a Prometheus Metric Event
type Event struct {
	Name   string
	Labels map[string]string
	Value  string
}

// NewMetrics returns a new metrics struct
func NewMetrics(instance, pushgatewayURL string) *PrometheusMetrics {
	return &PrometheusMetrics{
		Instance:       instance,
		PushgatewayURL: pushgatewayURL,
		Metrics:        make(map[string]*Metric),
	}
}

// String formats an event for printing
func (e *Event) String() string {
	var labels []string
	for l, v := range e.Labels {
		labels = append(labels, fmt.Sprintf("%s=\"%s\"", l, v))
	}
	return fmt.Sprintf("%s{%s} %s", e.Name, strings.Join(labels, ","), e.Value)
}

// Equals checks if two Events refer to the same Prometheus event
func (e *Event) Equals(newEvent *Event) bool {
	if e.Name == newEvent.Name {
		return false
	}

	if e.Labels["volume"] != newEvent.Labels["volume"] {
		return false
	}

	return true
}

// UpdateEvent adds an event, or updates it if the event already exists
func (m *Metric) UpdateEvent(event *Event) {
	event.Name = m.Name
	var found bool
	for _, e := range m.Events {
		if e.Equals(event) {
			e = event
			found = true
			break
		}
	}
	if !found {
		m.Events = append(m.Events, event)
	}
}

// NewMetric adds a new metric if it doesn't exist yet
// or returns the existing matching metric otherwise
func (p *PrometheusMetrics) NewMetric(name, mType string) (m *Metric) {
	m, ok := p.Metrics[name]
	if !ok {
		m = &Metric{
			Name: name,
			Type: mType,
		}
		p.Metrics[name] = m
	}
	return
}

// GetMetrics returns a map of existing metrics
func (p *PrometheusMetrics) GetMetrics() (err error) {
	if p.PushgatewayURL == "" {
		log.Debug("No Pushgateway URL specified, not retrieving metrics")
		return
	}
	log.Debug("Retrieving existing metrics")
	url := p.PushgatewayURL + "/metrics"
	resp, err := http.Get(url)
	if err != nil {
		err = fmt.Errorf("failed to get existing metrics from Prometheus: %v", err)
		return
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		err = fmt.Errorf("failed to read HTTP response: %v", err)
		return
	}
	log.Debug(string(body))
	for _, l := range strings.Split(string(body), "\n") {
		if strings.HasPrefix(l, "# TYPE ") {
			lSplit := strings.Fields(l)
			mName := lSplit[2]
			mType := lSplit[3]
			m, ok := p.Metrics[mName]
			if !ok {
				m = &Metric{
					Name: mName,
				}
			}
			m.Type = mType
			continue
		}
		e := parseEvent(l)
		if e == nil {
			continue
		}
		if e.Labels["instance"] != p.Instance {
			log.WithFields(log.Fields{
				"event":    e.Name,
				"instance": e.Labels["instance"],
			}).Debug("Ignoring event from wrong instance")
			continue
		}
		log.WithFields(log.Fields{
			"event":  e.Name,
			"value":  e.Value,
			"labels": e.Labels,
		}).Debug("Found event")
		m, ok := p.Metrics[e.Name]
		if !ok {
			m = &Metric{
				Name: e.Name,
			}
			p.Metrics[e.Name] = m
		}
		m.Events = append(m.Events, e)
	}

	return
}

func parseEvent(line string) (event *Event) {
	// Filter out metrics and comments
	if !strings.HasPrefix(line, "conplicity") {
		return
	}

	spaceSplit := strings.Fields(line)
	BraceSplit := strings.Split(spaceSplit[0], "{")
	name := BraceSplit[0]
	labelsStr := strings.TrimSuffix(strings.TrimPrefix(spaceSplit[0], fmt.Sprintf("%s{", name)), "}")
	labels := make(map[string]string)
	for _, l := range strings.Split(labelsStr, ",") {
		lParse := strings.Split(l, "=")
		lName := lParse[0]
		lValue := strings.TrimSuffix(strings.TrimPrefix(l, fmt.Sprintf("%s=\"", lName)), "\"")
		labels[lName] = lValue
	}
	event = &Event{
		Name:   name,
		Value:  spaceSplit[1],
		Labels: labels,
	}
	return
}

// Push sends metrics to a Prometheus push gateway
func (p *PrometheusMetrics) Push() (err error) {
	if p.PushgatewayURL == "" {
		log.Debug("No Pushgateway URL specified, not pushing metrics")
		return
	}
	metrics := p.Metrics
	url := p.PushgatewayURL + "/metrics/job/conplicity/instance/" + p.Instance

	var data string
	for _, m := range metrics {
		if m.Type != "" {
			data += fmt.Sprintf("# TYPE %s %s\n", m.Name, m.Type)
		}
		for _, e := range m.Events {
			data += fmt.Sprintf("%s\n", e)
		}
	}
	data += "\n"

	log.WithFields(log.Fields{
		"data": data,
		"url":  url,
	}).Debug("Sending metrics to Prometheus Pushgateway")

	req, err := http.NewRequest("PUT", url, bytes.NewBufferString(data))
	if err != nil {
		err = fmt.Errorf("failed to create HTTP request: %v", err)
		return
	}

	req.Header.Set("Content-Type", "text/plain; version=0.0.4")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		err = fmt.Errorf("failed to get HTTP response: %v", err)
		return
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		err = fmt.Errorf("failed to read HTTP response: %v", err)
		return
	}

	log.WithFields(log.Fields{
		"resp": string(body),
	}).Debug("Received Prometheus response")

	return
}
<commit_msg>Force metric type<commit_after>package metrics

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"

	log "github.com/Sirupsen/logrus"
)

// PrometheusMetrics is a struct to push metrics to Prometheus
type PrometheusMetrics struct {
	Instance       string
	PushgatewayURL string
	Metrics        map[string]*Metric
}

// Metric is a Prometheus Metric
type Metric struct {
	Name   string
	Events []*Event
	Type   string
}

// Event is a Prometheus Metric Event
type Event struct {
	Name   string
	Labels map[string]string
	Value  string
}

// NewMetrics returns a new metrics struct
func NewMetrics(instance, pushgatewayURL string) *PrometheusMetrics {
	return &PrometheusMetrics{
		Instance:       instance,
		PushgatewayURL: pushgatewayURL,
		Metrics:        make(map[string]*Metric),
	}
}

// String formats an event for printing
func (e *Event) String() string {
	var labels []string
	for l, v := range e.Labels {
		labels = append(labels, fmt.Sprintf("%s=\"%s\"", l, v))
	}
	return fmt.Sprintf("%s{%s} %s", e.Name, strings.Join(labels, ","), e.Value)
}

// Equals checks if two Events refer to the same Prometheus event
func (e *Event) Equals(newEvent *Event) bool {
	if e.Name == newEvent.Name {
		return false
	}

	if e.Labels["volume"] != newEvent.Labels["volume"] {
		return false
	}

	return true
}

// UpdateEvent adds an event, or updates it if the event already exists
func (m *Metric) UpdateEvent(event *Event) {
	event.Name = m.Name
	var found bool
	for _, e := range m.Events {
		if e.Equals(event) {
			e = event
			found = true
			break
		}
	}
	if !found {
		m.Events = append(m.Events, event)
	}
}

// NewMetric adds a new metric if it doesn't exist yet
// or returns the existing matching metric otherwise
func (p *PrometheusMetrics) NewMetric(name, mType string) (m *Metric) {
	m, ok := p.Metrics[name]
	if !ok {
		m = &Metric{
			Name: name,
		}
		p.Metrics[name] = m
	}
	m.Type = mType
	return
}

// GetMetrics returns a map of existing metrics
func (p *PrometheusMetrics) GetMetrics() (err error) {
	if p.PushgatewayURL == "" {
		log.Debug("No Pushgateway URL specified, not retrieving metrics")
		return
	}
	log.Debug("Retrieving existing metrics")
	url := p.PushgatewayURL + "/metrics"
	resp, err := http.Get(url)
	if err != nil {
		err = fmt.Errorf("failed to get existing metrics from Prometheus: %v", err)
		return
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		err = fmt.Errorf("failed to read HTTP response: %v", err)
		return
	}
	log.Debug(string(body))
	for _, l := range strings.Split(string(body), "\n") {
		if strings.HasPrefix(l, "# TYPE ") {
			lSplit := strings.Fields(l)
			mName := lSplit[2]
			mType := lSplit[3]
			m, ok := p.Metrics[mName]
			if !ok {
				m = &Metric{
					Name: mName,
				}
			}
			m.Type = mType
			continue
		}
		e := parseEvent(l)
		if e == nil {
			continue
		}
		if e.Labels["instance"] != p.Instance {
			log.WithFields(log.Fields{
				"event":    e.Name,
				"instance": e.Labels["instance"],
			}).Debug("Ignoring event from wrong instance")
			continue
		}
		log.WithFields(log.Fields{
			"event":  e.Name,
			"value":  e.Value,
			"labels": e.Labels,
		}).Debug("Found event")
		m, ok := p.Metrics[e.Name]
		if !ok {
			m = &Metric{
				Name: e.Name,
			}
			p.Metrics[e.Name] = m
		}
		m.Events = append(m.Events, e)
	}

	return
}

func parseEvent(line string) (event *Event) {
	// Filter out metrics and comments
	if !strings.HasPrefix(line, "conplicity") {
		return
	}

	spaceSplit := strings.Fields(line)
	BraceSplit := strings.Split(spaceSplit[0], "{")
	name := BraceSplit[0]
	labelsStr := strings.TrimSuffix(strings.TrimPrefix(spaceSplit[0], fmt.Sprintf("%s{", name)), "}")
	labels := make(map[string]string)
	for _, l := range strings.Split(labelsStr, ",") {
		lParse := strings.Split(l, "=")
		lName := lParse[0]
		lValue := strings.TrimSuffix(strings.TrimPrefix(l, fmt.Sprintf("%s=\"", lName)), "\"")
		labels[lName] = lValue
	}
	event = &Event{
		Name:   name,
		Value:  spaceSplit[1],
		Labels: labels,
	}
	return
}

// Push sends metrics to a Prometheus push gateway
func (p *PrometheusMetrics) Push() (err error) {
	if p.PushgatewayURL == "" {
		log.Debug("No Pushgateway URL specified, not pushing metrics")
		return
	}
	metrics := p.Metrics
	url := p.PushgatewayURL + "/metrics/job/conplicity/instance/" + p.Instance

	var data string
	for _, m := range metrics {
		if m.Type != "" {
			data += fmt.Sprintf("# TYPE %s %s\n", m.Name, m.Type)
		}
		for _, e := range m.Events {
			data += fmt.Sprintf("%s\n", e)
		}
	}
	data += "\n"

	log.WithFields(log.Fields{
		"data": data,
		"url":  url,
	}).Debug("Sending metrics to Prometheus Pushgateway")

	req, err := http.NewRequest("PUT", url, bytes.NewBufferString(data))
	if err != nil {
		err = fmt.Errorf("failed to create HTTP request: %v", err)
		return
	}

	req.Header.Set("Content-Type", "text/plain; version=0.0.4")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		err = fmt.Errorf("failed to get HTTP response: %v", err)
		return
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		err = fmt.Errorf("failed to read HTTP response: %v", err)
		return
	}

	log.WithFields(log.Fields{
		"resp": string(body),
	}).Debug("Received Prometheus response")

	return
}
<|endoftext|>


<commit_before>// The metrics package defines prometheus metric types and provides
// convenience methods to add accounting to various parts of the pipeline.
//
// When defining new operations or metrics, these are helpful values to track:
//  - things coming into or go out of the system: requests, files, tests, api calls.
//  - the success or error status of any of the above.
//  - the distribution of processing latency.
package metrics

import (
	"math"
	"net/http"
	"time"

	"github.com/prometheus/client_golang/prometheus"
)

func init() {
	// Register the metrics defined with Prometheus's default registry.
	prometheus.MustRegister(WorkerCount)
	prometheus.MustRegister(WorkerState)
	prometheus.MustRegister(TaskCount)
	prometheus.MustRegister(TestCount)
	prometheus.MustRegister(ErrorCount)
	prometheus.MustRegister(GCSRetryCount)
	prometheus.MustRegister(BigQueryInsert)
	prometheus.MustRegister(DurationHistogram)
	prometheus.MustRegister(InsertionHistogram)
	prometheus.MustRegister(FileSizeHistogram)
}

// TODO
// Want a goroutine that monitors the workers, and updates metrics to indicate how long the
// workers have been working, and perhaps what their state is.
// How about a gauge, broken down by state?  The state transitions will be triggered by the
// worker code.
//

var (
	// Counts the number of tasks processed by the pipeline.
	//
	// Provides metrics:
	//   etl_worker_count
	// Example usage:
	//   metrics.WorkerCount.Inc() / .Dec()
	WorkerCount = prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "etl_worker_count",
		Help: "Number of active workers.",
	})

	// Counts the number of tasks processed by the pipeline.
	//
	// Provides metrics:
	//   etl_worker_count{state}
	// Example usage:
	//   metrics.WorkerState.WithLabelValues("flush").Inc() / .Dec()
	WorkerState = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: "etl_worker_state",
		Help: "Number of workers in different states.",
	},
		// Worker state, e.g. create task, read, parse, insert
		[]string{"state"},
	)

	// Counts the number of tasks processed by the pipeline.
	//
	// Provides metrics:
	//   etl_task_count{package, status}
	// Example usage:
	//   metrics.TaskCount.WithLabelValues("Task", "ok").Inc()
	TaskCount = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "etl_task_count",
			Help: "Number of tasks/archive files processed.",
		},
		// Go package or filename, and Status
		[]string{"package", "status"},
	)

	// Counts the number of tests processed by the parsers..
	//
	// Provides metrics:
	//   etl_test_count{table, suffix, filetype, status}
	// Example usage:
	// metrics.TestCount.WithLabelValues(
	//	tt.Inserter.TableBase(), tt.Inserter.TableSuffix(), "s2c", "ok").Inc()
	// TODO(2017) Remove suffix field, and use a more scalable solution, perhaps bigquery
	// table to store operations metrics.
	TestCount = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "etl_test_count",
			Help: "Number of tests processed.",
		},
		// ndt/pt/ss, s2c/c2s/meta, ok/reject/error/
		[]string{"table", "suffix", "filetype", "status"},
	)

	// Counts the all types of errors.
	//
	// Provides metrics:
	//   etl_error_count{parser, kind}
	// Example usage:
	//   metrics.ErrorCount.WithLabelValues("NDT", "insert").Inc()
	ErrorCount = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "etl_error_count",
			Help: "Number of errors.",
		},
		// Parser type, error description.
		[]string{"parser", "kind"},
	)

	// Counts of anomolies in tests.  Generally these are non-terminal, so the
	// test also shows up in TestCount with status ok.
	//
	// Provides metrics:
	//   etl_funny_tests{table, filetype, anomoly}
	// Example usage:
	// metrics.FunnyTests.WithLabelValues(
	//	tt.Inserter.TableBase(), "s2c", "<16KB").Inc()
	FunnyTests = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "etl_funny_tests",
			Help: "Counts of anomolous tests.",
		},
		// ndt/pt/ss, s2c/c2s/meta, big/small/corrupt/
		[]string{"table", "filetype", "anomoly"},
	)

	// Counts the number of retries on GCS read operations.
	//
	// Provides metrics:
	//   etl_gcs_retry_count{type}
	// Example usage:
	// metrics.GCSRetryCount.WithLabelValues(
	//	TableName(), retries, "ok").Inc()
	GCSRetryCount = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "etl_gcs_retry_count",
			Help: "Number of retries on GCS reads.",
		},
		// open/read/zip, num_retries, ok/error/
		[]string{"phase", "retries", "status"},
	)

	// Counts the number of into BigQuery insert operations.
	//
	// Provides metrics:
	//   etl_worker_bigquery_insert_total{table, status}
	// Usage example:
	//   metrics.BigQueryInsert.WithLabelValues("ndt", "200").Inc()
	BigQueryInsert = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "etl_worker_bigquery_insert_total",
			Help: "Number of BigQuery insert operations.",
		},
		// Worker type, e.g. ndt, sidestream, ptr, etc.
		[]string{"table", "status"},
	)

	// A histogram of bigquery insertion times. The buckets should use
	// periods that are intuitive for people.
	//
	// Provides metrics:
	//   etl_insertion_time_seconds_bucket{type="...", le="..."}
	//   ...
	//   etl_insertion_time_seconds_sum{type="..."}
	//   etl_insertion_time_seconds_count{type="..."}
	// Usage example:
	//   t := time.Now()
	//   // do some stuff.
	//   metrics.InsertionHistogram.WithLabelValues(
	//           "ndt_test", "ok").Observe(time.Since(t).Seconds())
	InsertionHistogram = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Name: "etl_insertion_time_seconds",
			Help: "Insertion time distributions.",
			Buckets: []float64{
				0.001, 0.003, 0.01, 0.03, 0.1, 0.2, 0.5, 1.0, 2.0,
				5.0, 10.0, 20.0, 50.0, 100.0, math.Inf(+1),
			},
		},
		// Worker type, e.g. ndt, sidestream, ptr, etc.
		[]string{"table", "status"},
	)

	// A histogram of worker processing times. The buckets should use
	// periods that are intuitive for people.
	//
	// Provides metrics:
	//   etl_worker_duration_seconds_bucket{worker="...", le="..."}
	//   ...
	//   etl_worker_duration_seconds_sum{worker="..."}
	//   etl_worker_duration_seconds_count{worker="..."}
	// Usage example:
	//   t := time.Now()
	//   // do some stuff.
	//   metrics.DurationHistogram.WithLabelValues(
	//           "ndt").Observe(time.Since(t).Seconds())
	DurationHistogram = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Name: "etl_worker_duration_seconds",
			Help: "Worker execution time distributions.",
			Buckets: []float64{
				0.001, 0.01, 0.1, 1.0, 2.0, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0,
				600.0, 1800.0, 3600.0, 7200.0, math.Inf(+1),
			},
		},
		// Worker type, e.g. ndt, sidestream, ptr, etc.
		// TODO(soltesz): support a status field based on HTTP status.
		[]string{"worker"},
	)

	// TODO(dev): generalize this metric for size of any file type.
	FileSizeHistogram = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Name: "etl_web100_snaplog_file_size_bytes",
			Help: "Size of individual snaplog files.",
			Buckets: []float64{
				0,
				400000,     // 400k
				500000,     // 500k
				600000,     // 600k
				700000,     // 700k
				800000,     // 800k
				900000,     // 900k
				1000000,    // 1 mb
				1100000,    // 1.1 mb
				1200000,    // 1.2 mb
				1400000,    // 1.4 mb
				1600000,    // 1.6 mb
				1800000,    // 1.8 mb
				2000000,    // 2.0 mb
				2400000,    // 2.4 mb
				2800000,    // 2.8 mb
				3200000,    // 3.2 mb
				3600000,    // 3.6 mb
				4000000,    // 4 mb
				6000000,    // 6 mb
				8000000,    // 8 mb
				10000000,   // 10 mb
				20000000,   // 20
				40000000,   // 40
				80000000,   // 80
				100000000,  // 100 mb
				200000000,  // 200
				400000000,  // 400
				800000000,  // 800
				1000000000, // 1 gb
				math.Inf(+1),
			},
		},
		[]string{"range"},
	)
)

// DurationHandler wraps the call of an inner http.HandlerFunc and records the runtime.
func DurationHandler(name string, inner http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		t := time.Now()
		inner.ServeHTTP(w, r)
		// TODO(soltesz): collect success or failure status.
		DurationHistogram.WithLabelValues(name).Observe(time.Since(t).Seconds())
	}
}
<commit_msg>parser -> table<commit_after>// The metrics package defines prometheus metric types and provides
// convenience methods to add accounting to various parts of the pipeline.
//
// When defining new operations or metrics, these are helpful values to track:
//  - things coming into or go out of the system: requests, files, tests, api calls.
//  - the success or error status of any of the above.
//  - the distribution of processing latency.
package metrics

import (
	"math"
	"net/http"
	"time"

	"github.com/prometheus/client_golang/prometheus"
)

func init() {
	// Register the metrics defined with Prometheus's default registry.
	prometheus.MustRegister(WorkerCount)
	prometheus.MustRegister(WorkerState)
	prometheus.MustRegister(TaskCount)
	prometheus.MustRegister(TestCount)
	prometheus.MustRegister(ErrorCount)
	prometheus.MustRegister(GCSRetryCount)
	prometheus.MustRegister(BigQueryInsert)
	prometheus.MustRegister(DurationHistogram)
	prometheus.MustRegister(InsertionHistogram)
	prometheus.MustRegister(FileSizeHistogram)
}

// TODO
// Want a goroutine that monitors the workers, and updates metrics to indicate how long the
// workers have been working, and perhaps what their state is.
// How about a gauge, broken down by state?  The state transitions will be triggered by the
// worker code.
//

var (
	// Counts the number of tasks processed by the pipeline.
	//
	// Provides metrics:
	//   etl_worker_count
	// Example usage:
	//   metrics.WorkerCount.Inc() / .Dec()
	WorkerCount = prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "etl_worker_count",
		Help: "Number of active workers.",
	})

	// Counts the number of tasks processed by the pipeline.
	//
	// Provides metrics:
	//   etl_worker_count{state}
	// Example usage:
	//   metrics.WorkerState.WithLabelValues("flush").Inc() / .Dec()
	WorkerState = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: "etl_worker_state",
		Help: "Number of workers in different states.",
	},
		// Worker state, e.g. create task, read, parse, insert
		[]string{"state"},
	)

	// Counts the number of tasks processed by the pipeline.
	//
	// Provides metrics:
	//   etl_task_count{package, status}
	// Example usage:
	//   metrics.TaskCount.WithLabelValues("Task", "ok").Inc()
	TaskCount = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "etl_task_count",
			Help: "Number of tasks/archive files processed.",
		},
		// Go package or filename, and Status
		[]string{"package", "status"},
	)

	// Counts the number of tests processed by the parsers..
	//
	// Provides metrics:
	//   etl_test_count{table, suffix, filetype, status}
	// Example usage:
	// metrics.TestCount.WithLabelValues(
	//	tt.Inserter.TableBase(), tt.Inserter.TableSuffix(), "s2c", "ok").Inc()
	// TODO(2017) Remove suffix field, and use a more scalable solution, perhaps bigquery
	// table to store operations metrics.
	TestCount = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "etl_test_count",
			Help: "Number of tests processed.",
		},
		// ndt/pt/ss, s2c/c2s/meta, ok/reject/error/
		[]string{"table", "suffix", "filetype", "status"},
	)

	// Counts the all types of errors.
	//
	// Provides metrics:
	//   etl_error_count{table, kind}
	// Example usage:
	//   metrics.ErrorCount.WithLabelValues(TableName(), "insert").Inc()
	ErrorCount = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "etl_error_count",
			Help: "Number of errors.",
		},
		// Parser type, error description.
		[]string{"table", "kind"},
	)

	// Counts of anomolies in tests.  Generally these are non-terminal, so the
	// test also shows up in TestCount with status ok.
	//
	// Provides metrics:
	//   etl_funny_tests{table, filetype, anomoly}
	// Example usage:
	// metrics.FunnyTests.WithLabelValues(
	//	tt.Inserter.TableBase(), "s2c", "<16KB").Inc()
	FunnyTests = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "etl_funny_tests",
			Help: "Counts of anomolous tests.",
		},
		// ndt/pt/ss, s2c/c2s/meta, big/small/corrupt/
		[]string{"table", "filetype", "anomoly"},
	)

	// Counts the number of retries on GCS read operations.
	//
	// Provides metrics:
	//   etl_gcs_retry_count{type}
	// Example usage:
	// metrics.GCSRetryCount.WithLabelValues(
	//	TableName(), retries, "ok").Inc()
	GCSRetryCount = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "etl_gcs_retry_count",
			Help: "Number of retries on GCS reads.",
		},
		// open/read/zip, num_retries, ok/error/
		[]string{"phase", "retries", "status"},
	)

	// Counts the number of into BigQuery insert operations.
	//
	// Provides metrics:
	//   etl_worker_bigquery_insert_total{table, status}
	// Usage example:
	//   metrics.BigQueryInsert.WithLabelValues("ndt", "200").Inc()
	BigQueryInsert = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "etl_worker_bigquery_insert_total",
			Help: "Number of BigQuery insert operations.",
		},
		// Worker type, e.g. ndt, sidestream, ptr, etc.
		[]string{"table", "status"},
	)

	// A histogram of bigquery insertion times. The buckets should use
	// periods that are intuitive for people.
	//
	// Provides metrics:
	//   etl_insertion_time_seconds_bucket{type="...", le="..."}
	//   ...
	//   etl_insertion_time_seconds_sum{type="..."}
	//   etl_insertion_time_seconds_count{type="..."}
	// Usage example:
	//   t := time.Now()
	//   // do some stuff.
	//   metrics.InsertionHistogram.WithLabelValues(
	//           "ndt_test", "ok").Observe(time.Since(t).Seconds())
	InsertionHistogram = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Name: "etl_insertion_time_seconds",
			Help: "Insertion time distributions.",
			Buckets: []float64{
				0.001, 0.003, 0.01, 0.03, 0.1, 0.2, 0.5, 1.0, 2.0,
				5.0, 10.0, 20.0, 50.0, 100.0, math.Inf(+1),
			},
		},
		// Worker type, e.g. ndt, sidestream, ptr, etc.
		[]string{"table", "status"},
	)

	// A histogram of worker processing times. The buckets should use
	// periods that are intuitive for people.
	//
	// Provides metrics:
	//   etl_worker_duration_seconds_bucket{worker="...", le="..."}
	//   ...
	//   etl_worker_duration_seconds_sum{worker="..."}
	//   etl_worker_duration_seconds_count{worker="..."}
	// Usage example:
	//   t := time.Now()
	//   // do some stuff.
	//   metrics.DurationHistogram.WithLabelValues(
	//           "ndt").Observe(time.Since(t).Seconds())
	DurationHistogram = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Name: "etl_worker_duration_seconds",
			Help: "Worker execution time distributions.",
			Buckets: []float64{
				0.001, 0.01, 0.1, 1.0, 2.0, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0,
				600.0, 1800.0, 3600.0, 7200.0, math.Inf(+1),
			},
		},
		// Worker type, e.g. ndt, sidestream, ptr, etc.
		// TODO(soltesz): support a status field based on HTTP status.
		[]string{"worker"},
	)

	// TODO(dev): generalize this metric for size of any file type.
	FileSizeHistogram = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Name: "etl_web100_snaplog_file_size_bytes",
			Help: "Size of individual snaplog files.",
			Buckets: []float64{
				0,
				400000,     // 400k
				500000,     // 500k
				600000,     // 600k
				700000,     // 700k
				800000,     // 800k
				900000,     // 900k
				1000000,    // 1 mb
				1100000,    // 1.1 mb
				1200000,    // 1.2 mb
				1400000,    // 1.4 mb
				1600000,    // 1.6 mb
				1800000,    // 1.8 mb
				2000000,    // 2.0 mb
				2400000,    // 2.4 mb
				2800000,    // 2.8 mb
				3200000,    // 3.2 mb
				3600000,    // 3.6 mb
				4000000,    // 4 mb
				6000000,    // 6 mb
				8000000,    // 8 mb
				10000000,   // 10 mb
				20000000,   // 20
				40000000,   // 40
				80000000,   // 80
				100000000,  // 100 mb
				200000000,  // 200
				400000000,  // 400
				800000000,  // 800
				1000000000, // 1 gb
				math.Inf(+1),
			},
		},
		[]string{"range"},
	)
)

// DurationHandler wraps the call of an inner http.HandlerFunc and records the runtime.
func DurationHandler(name string, inner http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		t := time.Now()
		inner.ServeHTTP(w, r)
		// TODO(soltesz): collect success or failure status.
		DurationHistogram.WithLabelValues(name).Observe(time.Since(t).Seconds())
	}
}
<|endoftext|>


<commit_before>package middleware

import (
	"github.com/gin-gonic/gin"
	"http"
	"net/url"
	"strings"

	"github.com/techjanitor/pram-post/config"
)

var validSites = map[string]bool{}

func init() {

	// add valid sites to map
	for _, site := range config.Settings.CORS.Sites {
		validSites[site] = true
	}

}

// CORS will set the headers for Cross-origin resource sharing
func CORS() gin.HandlerFunc {
	return func(c *gin.Context) {

		req := c.Request
		res := c.Writer
		origin := req.Header.Get("Origin")

		// Set origin header from sites config
		if isAllowedSite(origin) {
			res.Header().Set("Access-Control-Allow-Origin", origin)
		} else {
			res.Header().Set("Access-Control-Allow-Origin", "")
		}

		if req.Method == "OPTIONS" {

			// Add allowed method header
			res.Header().Set("Access-Control-Allow-Methods", "POST")

			c.AbortWithStatus(http.StatusOK)

		} else {

			c.Next()

		}

	}
}

// Check if origin is allowed
func isAllowedSite(host string) bool {

	// Get the host from the origin
	parsed, err := url.Parse(host)
	if err != nil {
		return false
	}

	if validSites[strings.ToLower(parsed.Host)] {
		return true
	}

	return false

}
<commit_msg>add CORS middleware<commit_after>package middleware

import (
	"github.com/gin-gonic/gin"
	"net/http"
	"net/url"
	"strings"

	"github.com/techjanitor/pram-post/config"
)

var validSites = map[string]bool{}

func init() {

	// add valid sites to map
	for _, site := range config.Settings.CORS.Sites {
		validSites[site] = true
	}

}

// CORS will set the headers for Cross-origin resource sharing
func CORS() gin.HandlerFunc {
	return func(c *gin.Context) {

		req := c.Request
		res := c.Writer
		origin := req.Header.Get("Origin")

		// Set origin header from sites config
		if isAllowedSite(origin) {
			res.Header().Set("Access-Control-Allow-Origin", origin)
		} else {
			res.Header().Set("Access-Control-Allow-Origin", "")
		}

		if req.Method == "OPTIONS" {

			// Add allowed method header
			res.Header().Set("Access-Control-Allow-Methods", "POST")

			c.AbortWithStatus(http.StatusOK)

		} else {

			c.Next()

		}

	}
}

// Check if origin is allowed
func isAllowedSite(host string) bool {

	// Get the host from the origin
	parsed, err := url.Parse(host)
	if err != nil {
		return false
	}

	if validSites[strings.ToLower(parsed.Host)] {
		return true
	}

	return false

}
<|endoftext|>


<commit_before>package middleware

import (
	"net/http"
	"time"

	"github.com/kyani-inc/go-utils/ip"
	"github.com/kyani-inc/logger"

	"github.com/labstack/echo"
)

func Echo(log logger.Client, prefixes ...interface{}) echo.MiddlewareFunc {
	return func(next echo.HandlerFunc) echo.HandlerFunc {
		return func(c *echo.Context) error {
			start := time.Now()

			// Get a single IP Address of the connecting party
			addr := ip.Client(c.Request())

			if err := next(c); err != nil {
				c.Error(err)
			}

			latency := time.Since(start)
			// Create the prefix for the logger
			prefix := makePrefixes(c, prefixes...)
			log.Infof("%s%v %s %s %v %s \"%s\"", prefix, c.Response().Status(), http.StatusText(c.Response().Status()), c.Request().Method, latency, c.Request().URL.Path, addr)

			return nil
		}
	}
}
<commit_msg>use kyani/echo as temp fix to broken deps in echo<commit_after>package middleware

import (
	"net/http"
	"time"

	"github.com/kyani-inc/go-utils/ip"
	"github.com/kyani-inc/logger"

	"github.com/kyani-inc/echo"
)

func Echo(log logger.Client, prefixes ...interface{}) echo.MiddlewareFunc {
	return func(next echo.HandlerFunc) echo.HandlerFunc {
		return func(c *echo.Context) error {
			start := time.Now()

			// Get a single IP Address of the connecting party
			addr := ip.Client(c.Request())

			if err := next(c); err != nil {
				c.Error(err)
			}

			latency := time.Since(start)
			// Create the prefix for the logger
			prefix := makePrefixes(c, prefixes...)
			log.Infof("%s%v %s %s %v %s \"%s\"", prefix, c.Response().Status(), http.StatusText(c.Response().Status()), c.Request().Method, latency, c.Request().URL.Path, addr)

			return nil
		}
	}
}
<|endoftext|>


<commit_before>package goMiddlewareChain

import (
	"fmt"
	"net/http"
	"testing"

	"net/http/httptest"

	"encoding/json"

	"reflect"

	"github.com/julienschmidt/httprouter"
)

// response handler which write response to writer
var responseHandler = func(response *Response, writer http.ResponseWriter, _ *http.Request, _ httprouter.Params) {
	bodyMap := map[string]interface{}{"data": response.Data}
	bodyJSON, _ := json.Marshal(bodyMap)
	fmt.Fprintf(writer, "%s", bodyJSON)
}

func TestMain(t *testing.T) {

	var tests = []struct {
		handler         Handler
		restrictHandler RestrictHandler
		expectedData    interface{}
	}{
		// Test 0:
		// Check if chain works
		{
			handler: func(response *Response, _ *http.Request, _ httprouter.Params) {
				response.Data = "data"
			},
			expectedData: "data",
		},
		// Test 1:
		// Check if restrictedChain works with restriction true
		{
			handler: func(response *Response, _ *http.Request, _ httprouter.Params) {
				response.Data = "data"
			},
			restrictHandler: func(response *Response, request *http.Request, _ httprouter.Params) bool {
				return true
			},
			expectedData: "data",
		},
	}

	// run all tests
	for index, test := range tests {
		// recorder and request to simulate test
		recorder := httptest.NewRecorder()
		request, err := http.NewRequest("POST", "", nil)

		// build chain
		if reflect.ValueOf(test.restrictHandler).IsNil() {
			handlerChain := RequestChainHandler(responseHandler, test.handler)
			handlerChain(recorder, request, nil)
		} else {
			restrictedHandleChain := RestrictedRequestChainHandler(test.restrictHandler, responseHandler, test.handler)
			restrictedHandleChain(recorder, request, nil)
		}

		// check result
		var responseBody map[string]interface{}
		json.Unmarshal([]byte(recorder.Body.String()), &responseBody)
		if err != nil || responseBody["data"] != test.expectedData {
			t.Errorf("Test %v failed: expected data is not equals response.data; expected: %v; response: %v", index, test.expectedData, responseBody["data"])
		}
	}
}
<commit_msg>test for contextHandler<commit_after>package goMiddlewareChain

import (
	"context"
	"fmt"
	"net/http"
	"testing"

	"net/http/httptest"

	"encoding/json"

	"reflect"

	"github.com/julienschmidt/httprouter"
)

// response handler which write response to writer
var responseHandler = func(response *Response, writer http.ResponseWriter, _ *http.Request, _ httprouter.Params) {
	bodyMap := map[string]interface{}{"data": response.Data}
	bodyJSON, _ := json.Marshal(bodyMap)
	fmt.Fprintf(writer, "%s", bodyJSON)
}

func TestMain(t *testing.T) {

	var tests = []struct {
		handler         Handler
		restrictHandler RestrictHandler
		expectedData    interface{}
	}{
		// Test 0:
		// Check if chain works
		{
			handler: func(response *Response, _ *http.Request, _ httprouter.Params) {
				response.Data = "data"
			},
			expectedData: "data",
		},
		// Test 1:
		// Check if restrictedChain works with restriction true
		{
			handler: func(response *Response, _ *http.Request, _ httprouter.Params) {
				response.Data = "data"
			},
			restrictHandler: func(response *Response, request *http.Request, _ httprouter.Params) bool {
				return true
			},
			expectedData: "data",
		},
	}

	// run all tests
	for index, test := range tests {
		// recorder and request to simulate test
		recorder := httptest.NewRecorder()
		request, err := http.NewRequest("POST", "", nil)

		// build chain
		if reflect.ValueOf(test.restrictHandler).IsNil() {
			handlerChain := RequestChainHandler(responseHandler, test.handler)
			handlerChain(recorder, request, nil)
		} else {
			restrictedHandleChain := RestrictedRequestChainHandler(test.restrictHandler, responseHandler, test.handler)
			restrictedHandleChain(recorder, request, nil)
		}

		// check result
		var responseBody map[string]interface{}
		json.Unmarshal([]byte(recorder.Body.String()), &responseBody)
		if err != nil || responseBody["data"] != test.expectedData {
			t.Errorf("Test %v failed: expected data is not equals response.data; expected: %v; response: %v", index, test.expectedData, responseBody["data"])
		}
	}
}

func TestContextHandler(t *testing.T) {

	var key = struct {
		key string
	}{
		key: "key",
	}

	var tests = []struct {
		handler         []ContextHandler
		restrictHandler RestrictHandler
		expectedData    interface{}
	}{
		// Test 0:
		// Check if chain works
		{
			handler: []ContextHandler{
				func(ctx context.Context, response *Response, _ *http.Request, _ httprouter.Params) context.Context {
					return context.WithValue(ctx, key, "data")
				},
				func(ctx context.Context, response *Response, _ *http.Request, _ httprouter.Params) context.Context {
					response.Data = ctx.Value(key).(string)
					return ctx
				},
			},
			expectedData: "data",
		},
	}

	// run all tests
	for index, test := range tests {
		// recorder and request to simulate test
		recorder := httptest.NewRecorder()
		request, err := http.NewRequest("POST", "", nil)

		// build chain
		handlerChain := RequestChainContextHandler(responseHandler, test.handler...)
		handlerChain(recorder, request, nil)

		// check result
		var responseBody map[string]interface{}
		json.Unmarshal([]byte(recorder.Body.String()), &responseBody)
		if err != nil || responseBody["data"] != test.expectedData {
			t.Errorf("Test %v failed: expected data is not equals response.data; expected: %v; response: %v", index, test.expectedData, responseBody["data"])
		}
	}
}
<|endoftext|>


<commit_before>package migrations

import (
	"crypto/md5"
	"database/sql"
	"fmt"
	"log"
	"strings"
	"time"
)

const createMigrationsSql = `
  CREATE TABLE migrations (idx INTEGER PRIMARY KEY NOT NULL, md5 UUID NOT NULL, statement VARCHAR NOT NULL, created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL)
`

func New(migrations ...interface{}) *Migrations {
	return &Migrations{steps: migrations}
}

type Migrations struct {
	Logger *log.Logger
	steps  []interface{}
}

func (list Migrations) Execute(db *sql.DB) error {
	tx, e := db.Begin()
	if e != nil {
		return e
	}

	e = list.ExecuteTx(tx)

	if e != nil {
		tx.Rollback()
		return e
	}
	return tx.Commit()
}

func (list Migrations) ExecuteTx(tx *sql.Tx) error {
	_, e := list.setup(tx)
	if e != nil {
		return e
	}

	return func() error {
		for idx, i := range list.steps {
			m, e := newMigration(idx+1, i)
			if e != nil {
				return e
			}
			m.Logger = list.Logger
			_, e = m.Execute(tx)
			if e != nil {
				return e
			}
		}
		return nil
	}()
}

func newMigration(idx int, statement interface{}) (*Migration, error) {
	m := &Migration{Idx: idx}

	switch casted := statement.(type) {
	case string:
		m.Statement = casted
	case fmt.Stringer:
		m.Statement = casted.String()
	default:
		return nil, fmt.Errorf("type %T not supported", casted)
	}
	return m, nil
}

type Migration struct {
	Idx       int
	Statement string
	*log.Logger
}

const errorMigrationsDoesNotExist = `pq: relation "migrations" does not exist`

func (list Migrations) setup(tx *sql.Tx) (sql.Result, error) {
	row := tx.QueryRow("SELECT COUNT(1) FROM pg_tables WHERE schemaname = $1 AND tablename = $2", "public", "migrations")
	var cnt int
	e := row.Scan(&cnt)
	if e != nil {
		return nil, e
	}

	if cnt == 0 {
		return tx.Exec(createMigrationsSql)
	}
	return nil, nil
}

func (m *Migration) log(t string) {
	if m.Logger != nil {
		out := []string{}
		lines := strings.Split(strings.TrimSpace(m.Statement), "\n")
		for _, l := range lines {
			out = append(out, strings.TrimSpace(l))
		}
		m.Printf(t+": migration %d %q %q", m.Idx, m.checksum(), strings.Join(strings.Fields(strings.Join(out, " ")), " "))
	}
}

func (m *Migration) checksum() string {
	return fmt.Sprintf("%x", md5.Sum([]byte(m.Statement)))
}

func (m *Migration) Execute(tx *sql.Tx) (sql.Result, error) {
	rows, e := tx.Query("SELECT md5 FROM migrations where idx = $1", m.Idx)
	if e != nil {
		return nil, e
	}
	defer rows.Close()
	for rows.Next() {
		var cs string
		e = rows.Scan(&cs)
		if e != nil {
			return nil, e
		}
		cs = strings.Replace(cs, "-", "", -1)
		if cs == m.checksum() {
			m.log("SKIP")
			return nil, nil
		} else {
			return nil, fmt.Errorf("migration %d (new checksum %s) already exists with checksum %q", m.Idx, cs, m.checksum())
		}
	}
	m.log("EXEC")
	res, e := tx.Exec(m.Statement)
	if e != nil {
		return nil, e
	}
	_, e = tx.Exec("INSERT INTO migrations (idx, md5, statement, created_at) VALUES ($1, $2, $3, $4)", m.Idx, m.checksum(), m.Statement, time.Now().UTC().Format(time.RFC3339Nano))
	if e != nil {
		return nil, e
	}
	return res, e
}
<commit_msg>accept interface for Logger<commit_after>package migrations

import (
	"crypto/md5"
	"database/sql"
	"fmt"
	"strings"
	"time"
)

const createMigrationsSql = `
  CREATE TABLE migrations (idx INTEGER PRIMARY KEY NOT NULL, md5 UUID NOT NULL, statement VARCHAR NOT NULL, created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL)
`

func New(migrations ...interface{}) *Migrations {
	return &Migrations{steps: migrations}
}

type logger interface {
	Printf(string, ...interface{})
}

type Migrations struct {
	Logger logger
	steps  []interface{}
}

func (list Migrations) Execute(db *sql.DB) error {
	tx, e := db.Begin()
	if e != nil {
		return e
	}

	e = list.ExecuteTx(tx)

	if e != nil {
		tx.Rollback()
		return e
	}
	return tx.Commit()
}

func (list Migrations) ExecuteTx(tx *sql.Tx) error {
	_, e := list.setup(tx)
	if e != nil {
		return e
	}

	return func() error {
		for idx, i := range list.steps {
			m, e := newMigration(idx+1, i)
			if e != nil {
				return e
			}
			m.Logger = list.Logger
			_, e = m.Execute(tx)
			if e != nil {
				return e
			}
		}
		return nil
	}()
}

func newMigration(idx int, statement interface{}) (*Migration, error) {
	m := &Migration{Idx: idx}

	switch casted := statement.(type) {
	case string:
		m.Statement = casted
	case fmt.Stringer:
		m.Statement = casted.String()
	default:
		return nil, fmt.Errorf("type %T not supported", casted)
	}
	return m, nil
}

type Migration struct {
	Idx       int
	Statement string
	Logger    logger
}

const errorMigrationsDoesNotExist = `pq: relation "migrations" does not exist`

func (list Migrations) setup(tx *sql.Tx) (sql.Result, error) {
	row := tx.QueryRow("SELECT COUNT(1) FROM pg_tables WHERE schemaname = $1 AND tablename = $2", "public", "migrations")
	var cnt int
	e := row.Scan(&cnt)
	if e != nil {
		return nil, e
	}

	if cnt == 0 {
		return tx.Exec(createMigrationsSql)
	}
	return nil, nil
}

func (m *Migration) log(t string) {
	if m.Logger != nil {
		out := []string{}
		lines := strings.Split(strings.TrimSpace(m.Statement), "\n")
		for _, l := range lines {
			out = append(out, strings.TrimSpace(l))
		}
		m.Logger.Printf(t+": migration %d %q %q", m.Idx, m.checksum(), strings.Join(strings.Fields(strings.Join(out, " ")), " "))
	}
}

func (m *Migration) checksum() string {
	return fmt.Sprintf("%x", md5.Sum([]byte(m.Statement)))
}

func (m *Migration) Execute(tx *sql.Tx) (sql.Result, error) {
	rows, e := tx.Query("SELECT md5 FROM migrations where idx = $1", m.Idx)
	if e != nil {
		return nil, e
	}
	defer rows.Close()
	for rows.Next() {
		var cs string
		e = rows.Scan(&cs)
		if e != nil {
			return nil, e
		}
		cs = strings.Replace(cs, "-", "", -1)
		if cs == m.checksum() {
			m.log("SKIP")
			return nil, nil
		} else {
			return nil, fmt.Errorf("migration %d (new checksum %s) already exists with checksum %q", m.Idx, cs, m.checksum())
		}
	}
	m.log("EXEC")
	res, e := tx.Exec(m.Statement)
	if e != nil {
		return nil, e
	}
	_, e = tx.Exec("INSERT INTO migrations (idx, md5, statement, created_at) VALUES ($1, $2, $3, $4)", m.Idx, m.checksum(), m.Statement, time.Now().UTC().Format(time.RFC3339Nano))
	if e != nil {
		return nil, e
	}
	return res, e
}
<|endoftext|>


<commit_before>package quotas_test

import(
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
	. "github.com/pivotal-cf-experimental/cf-test-helpers/cf"
)

var _ = Describe("CF Quota commands", func() {
	It("can Create, Read, Update, and Delete quotas", func() {
		AsUser(context.AdminUserContext(), func() {
			Eventually(Cf("create-quota",
				"quota-name-goes-here",
				"-m", "512M",
			), 5.0).Should(Exit(0))

			Eventually(Cf("quota", "quota-name-goes-here"), 5.0).Should(Say("512M"))

			quotaOutput := Cf("quotas")
			Eventually(quotaOutput, 5).Should(Say("quota-name-goes-here"))

			Eventually(Cf("update-quota",
				"quota-name-goes-here",
				"-m", "513M",
			), 5).Should(Exit(0))

			Eventually(Cf("quotas")).Should(Say("513M"))

			Eventually(Cf("delete-quota",
				"quota-name-goes-here",
				"-f,",
			)).Should(Exit(0))

			Eventually(Cf("quotas")).ShouldNot(Say("quota-name-goes-here"))
		})
	})
})
<commit_msg>Make assertions based on STDOUT<commit_after>package quotas_test

import(
	"github.com/nu7hatch/gouuid"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/pivotal-cf-experimental/cf-test-helpers/cf"
)

var assertionTimeout = 10.0

var _ = Describe("CF Quota commands", func() {
	It("can Create, Read, Update, and Delete quotas", func() {
		AsUser(context.AdminUserContext(), func() {
			quotaBytes, err := uuid.NewV4()
			Expect(err).ToNot(HaveOccurred())
			quotaName := quotaBytes.String()

			Eventually(Cf("create-quota",
				quotaName,
				"-m", "512M",
			), assertionTimeout).Should(Say("OK"))

			Eventually(Cf("quota", quotaName), assertionTimeout).Should(Say("512M"))

			quotaOutput := Cf("quotas")
			Eventually(quotaOutput, assertionTimeout).Should(Say(quotaName))

			Eventually(Cf("update-quota",
				quotaName,
				"-m", "513M",
			), assertionTimeout).Should(Say("OK"))

			Eventually(Cf("quotas"), assertionTimeout).Should(Say("513M"))

			Eventually(Cf("delete-quota",
				quotaName,
				"-f",
			), assertionTimeout).Should(Say("OK"))

			Eventually(Cf("quotas"), assertionTimeout).ShouldNot(Say(quotaName))
		})
	})
})
<|endoftext|>


<commit_before>package models

import (
	"testing"
)

func TestHasRecordTypeName(t *testing.T) {
	x := &RecordConfig{
		Type: "A",
		Name: "@",
	}
	dc := DomainConfig{}
	if dc.HasRecordTypeName("A", "@") {
		t.Errorf("%v: expected (%v) got (%v)\n", dc.Records, false, true)
	}
	dc.Records = append(dc.Records, x)
	if !dc.HasRecordTypeName("A", "@") {
		t.Errorf("%v: expected (%v) got (%v)\n", dc.Records, true, false)
	}
	if dc.HasRecordTypeName("AAAA", "@") {
		t.Errorf("%v: expected (%v) got (%v)\n", dc.Records, false, true)
	}
}

func TestRR(t *testing.T) {
	experiment := RecordConfig{
		Type:         "A",
		Name:         "foo",
		Target:       "1.2.3.4",
		TTL:          0,
		NameFQDN:     "foo.example.com",
		MxPreference: 0,
	}
	expected := "foo.example.com.\t300\tIN\tA\t1.2.3.4"
	found := experiment.ToRR().String()
	if found != expected {
		t.Errorf("RR expected (%#v) got (%#v)\n", expected, found)
	}

	experiment = RecordConfig{
		Type:     "CAA",
		Name:     "@",
		Target:   "mailto:test@example.com",
		TTL:      300,
		NameFQDN: "example.com",
		CaaTag:   "iodef",
		CaaFlags: 1,
	}
	expected = "example.com.\t300\tIN\tCAA\t1 iodef \"mailto:test@example.com\""
	found = experiment.ToRR().String()
	if found != expected {
		t.Errorf("RR expected (%#v) got (%#v)\n", expected, found)
	}
}
<commit_msg>Fix build<commit_after>package models

import (
	"testing"
)

func TestHasRecordTypeName(t *testing.T) {
	x := &RecordConfig{
		Type: "A",
		Name: "@",
	}
	dc := DomainConfig{}
	if dc.HasRecordTypeName("A", "@") {
		t.Errorf("%v: expected (%v) got (%v)\n", dc.Records, false, true)
	}
	dc.Records = append(dc.Records, x)
	if !dc.HasRecordTypeName("A", "@") {
		t.Errorf("%v: expected (%v) got (%v)\n", dc.Records, true, false)
	}
	if dc.HasRecordTypeName("AAAA", "@") {
		t.Errorf("%v: expected (%v) got (%v)\n", dc.Records, false, true)
	}
}

func TestRR(t *testing.T) {
	experiment := RecordConfig{
		Type:         "A",
		Name:         "foo",
		Target:       "1.2.3.4",
		TTL:          0,
		NameFQDN:     "foo.example.com",
		MxPreference: 0,
	}
	expected := "foo.example.com.\t300\tIN\tA\t1.2.3.4"
	found := experiment.ToRR().String()
	if found != expected {
		t.Errorf("RR expected (%#v) got (%#v)\n", expected, found)
	}

	experiment = RecordConfig{
		Type:     "CAA",
		Name:     "@",
		Target:   "mailto:test@example.com",
		TTL:      300,
		NameFQDN: "example.com",
		CaaTag:   "iodef",
		CaaFlag: 1,
	}
	expected = "example.com.\t300\tIN\tCAA\t1 iodef \"mailto:test@example.com\""
	found = experiment.ToRR().String()
	if found != expected {
		t.Errorf("RR expected (%#v) got (%#v)\n", expected, found)
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The LUCI Authors. All rights reserved.
// Use of this source code is governed under the Apache License, Version 2.0
// that can be found in the LICENSE file.

package isolatedclient

import (
	"bytes"
	"encoding/base64"
	"errors"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"strings"

	"golang.org/x/net/context"

	isolateservice "github.com/luci/luci-go/common/api/isolate/isolateservice/v1"
	"github.com/luci/luci-go/common/isolated"
	"github.com/luci/luci-go/common/lhttp"
	"github.com/luci/luci-go/common/retry"
	"github.com/luci/luci-go/common/runtime/tracer"
)

// DefaultNamespace is the namespace that should be used with the New function.
const DefaultNamespace = "default-gzip"

// Source is a generator method to return source data. A generated Source must
// be Closed before the generator is called again.
type Source func() (io.ReadCloser, error)

// NewBytesSource returns a Source implementation that reads from the supplied
// byte slice.
func NewBytesSource(d []byte) Source {
	return func() (io.ReadCloser, error) {
		return ioutil.NopCloser(bytes.NewReader(d)), nil
	}
}

// PushState is per-item state passed from IsolateServer.Contains() to
// IsolateServer.Push().
//
// Its content is implementation specific.
type PushState struct {
	status    isolateservice.HandlersEndpointsV1PreuploadStatus
	digest    isolated.HexDigest
	size      int64
	uploaded  bool
	finalized bool
}

// FetchGCS is a function type of a handler for retrieving specified content from GCS and storing the response in the provided destination buffer.
type FetchGCS func(context.Context, *Client, isolateservice.HandlersEndpointsV1RetrievedContent, io.WriteSeeker) error

// Client is a client to an isolated server.
type Client struct {
	// All the members are immutable.
	retryFactory retry.Factory
	url          string
	namespace    string

	authClient      *http.Client // client that sends auth tokens
	anonClient      *http.Client // client that does NOT send auth tokens
	fetchGCSHandler FetchGCS     // function which fetches url from GCS and stores results in provided destination
}

// New returns a new IsolateServer client.
//
// 'authClient' must implement authentication sufficient to talk to Isolate server
// (OAuth tokens with 'email' scope).
//
// 'anonClient' must be a functional http.Client.
//
// If either client is nil, it will use http.DefaultClient (which will not work
// on Classic AppEngine!).
//
// If you're unsure which namespace to use, use the DefaultNamespace constant.
//
// If fetchGCSHandler is nil, the fetchGCSDefault function will be used to retrieve
// files from GCS.
func New(anonClient, authClient *http.Client, host, namespace string, rFn retry.Factory, fetchGCSHandler FetchGCS) *Client {
	if anonClient == nil {
		anonClient = http.DefaultClient
	}
	if authClient == nil {
		authClient = http.DefaultClient
	}
	if fetchGCSHandler == nil {
		fetchGCSHandler = fetchGCSDefault
	}
	i := &Client{
		retryFactory:    rFn,
		url:             strings.TrimRight(host, "/"),
		namespace:       namespace,
		authClient:      authClient,
		anonClient:      anonClient,
		fetchGCSHandler: fetchGCSHandler,
	}
	tracer.NewPID(i, "isolatedclient:"+i.url)
	return i
}

// ServerCapabilities returns the server details.
func (i *Client) ServerCapabilities(c context.Context) (*isolateservice.HandlersEndpointsV1ServerDetails, error) {
	out := &isolateservice.HandlersEndpointsV1ServerDetails{}
	if err := i.postJSON(c, "/api/isolateservice/v1/server_details", nil, map[string]string{}, out); err != nil {
		return nil, err
	}
	return out, nil
}

// Contains looks up cache presence on the server of multiple items.
//
// The returned list is in the same order as 'items', with entries nil for
// items that were present.
func (i *Client) Contains(c context.Context, items []*isolateservice.HandlersEndpointsV1Digest) (out []*PushState, err error) {
	end := tracer.Span(i, "contains", tracer.Args{"number": len(items)})
	defer func() { end(tracer.Args{"err": err}) }()
	in := isolateservice.HandlersEndpointsV1DigestCollection{Items: items, Namespace: &isolateservice.HandlersEndpointsV1Namespace{}}
	in.Namespace.Namespace = i.namespace
	data := &isolateservice.HandlersEndpointsV1UrlCollection{}
	if err = i.postJSON(c, "/api/isolateservice/v1/preupload", nil, in, data); err != nil {
		return nil, err
	}
	out = make([]*PushState, len(items))
	for _, e := range data.Items {
		index := int(e.Index)
		out[index] = &PushState{
			status: *e,
			digest: isolated.HexDigest(items[index].Digest),
			size:   items[index].Size,
		}
	}
	return out, nil
}

// Push pushed a missing item, as reported by Contains(), to the server.
func (i *Client) Push(c context.Context, state *PushState, source Source) (err error) {
	// This push operation may be a retry after failed finalization call below,
	// no need to reupload contents in that case.
	if !state.uploaded {
		// PUT file to uploadURL.
		if err = i.doPush(c, state, source); err != nil {
			log.Printf("doPush(%s) failed: %s\n%#v", state.digest, err, state)
			return
		}
		state.uploaded = true
	}

	// Optionally notify the server that it's done.
	if state.status.GsUploadUrl != "" {
		end := tracer.Span(i, "finalize", nil)
		defer func() { end(tracer.Args{"err": err}) }()
		// TODO(vadimsh): Calculate MD5 or CRC32C sum while uploading a file and
		// send it to isolated server. That way isolate server can verify that
		// the data safely reached Google Storage (GS provides MD5 and CRC32C of
		// stored files).
		in := isolateservice.HandlersEndpointsV1FinalizeRequest{UploadTicket: state.status.UploadTicket}
		headers := map[string]string{"Cache-Control": "public, max-age=31536000"}
		if err = i.postJSON(c, "/api/isolateservice/v1/finalize_gs_upload", headers, in, nil); err != nil {
			log.Printf("Push(%s) (finalize) failed: %s\n%#v", state.digest, err, state)
			return
		}
	}
	state.finalized = true
	return
}

// Fetch downloads an item from the server.
func (i *Client) Fetch(c context.Context, item *isolateservice.HandlersEndpointsV1Digest, dest io.WriteSeeker) error {
	// Perform initial request.
	url := i.url + "/api/isolateservice/v1/retrieve"
	in := &isolateservice.HandlersEndpointsV1RetrieveRequest{
		Digest: item.Digest,
		Namespace: &isolateservice.HandlersEndpointsV1Namespace{
			DigestHash: "sha-1",
			Namespace:  i.namespace,
		},
		Offset: 0,
	}
	var out isolateservice.HandlersEndpointsV1RetrievedContent
	if _, err := lhttp.PostJSON(c, i.retryFactory, i.authClient, url, nil, in, &out); err != nil {
		return err
	}

	// Handle DB items.
	if out.Content != "" {
		decoded, err := base64.StdEncoding.DecodeString(out.Content)
		if err != nil {
			return err
		}
		decompressor := isolated.GetDecompressor(bytes.NewReader(decoded))
		defer decompressor.Close()
		_, err = io.Copy(dest, decompressor)
		return err
	}

	// Handle GCS items.
	return i.fetchGCSHandler(c, i, out, dest)
}

// fetchGCSDefault uses the provided HandlersEndpointsV1RetrievedContent response to download content from GCS to the provided dest.
func fetchGCSDefault(c context.Context, i *Client, content isolateservice.HandlersEndpointsV1RetrievedContent, dest io.WriteSeeker) error {
	rgen := func() (*http.Request, error) {
		return http.NewRequest("GET", content.Url, nil)
	}
	handler := func(resp *http.Response) error {
		defer resp.Body.Close()
		decompressor := isolated.GetDecompressor(resp.Body)
		defer decompressor.Close()
		if _, err := dest.Seek(0, io.SeekStart); err != nil {
			return err
		}
		_, err := io.Copy(dest, decompressor)
		return err
	}
	_, err := lhttp.NewRequest(c, i.authClient, i.retryFactory, rgen, handler, nil)()
	return err
}

// postJSON does authenticated POST request.
func (i *Client) postJSON(c context.Context, resource string, headers map[string]string, in, out interface{}) error {
	if len(resource) == 0 || resource[0] != '/' {
		return errors.New("resource must start with '/'")
	}
	_, err := lhttp.PostJSON(c, i.retryFactory, i.authClient, i.url+resource, headers, in, out)
	return err
}

func (i *Client) doPush(c context.Context, state *PushState, source Source) (err error) {
	useDB := state.status.GsUploadUrl == ""
	end := tracer.Span(i, "push", tracer.Args{"useDB": useDB, "size": state.size})
	defer func() { end(tracer.Args{"err": err}) }()
	if useDB {
		src, err := source()
		if err != nil {
			return err
		}
		defer src.Close()

		err = i.doPushDB(c, state, src)
	} else {
		err = i.doPushGCS(c, state, source)
	}
	if err != nil {
		tracer.CounterAdd(i, "bytesUploaded", float64(state.size))
	}
	return err
}

func (i *Client) doPushDB(c context.Context, state *PushState, reader io.Reader) error {
	buf := bytes.Buffer{}
	compressor := isolated.GetCompressor(&buf)
	if _, err := io.Copy(compressor, reader); err != nil {
		return err
	}
	if err := compressor.Close(); err != nil {
		return err
	}
	in := &isolateservice.HandlersEndpointsV1StorageRequest{
		UploadTicket: state.status.UploadTicket,
		Content:      base64.StdEncoding.EncodeToString(buf.Bytes()),
	}
	return i.postJSON(c, "/api/isolateservice/v1/store_inline", nil, in, nil)
}

func (i *Client) doPushGCS(c context.Context, state *PushState, source Source) (err error) {
	// GsUploadUrl is signed Google Storage URL that doesn't require additional
	// authentication. In fact, using authClient causes HTTP 403 because
	// authClient's tokens don't have Cloud Storage OAuth scope. Use anonymous
	// client instead.
	req := lhttp.NewRequest(c, i.anonClient, i.retryFactory, func() (*http.Request, error) {
		src, err := source()
		if err != nil {
			return nil, err
		}

		request, err := http.NewRequest("PUT", state.status.GsUploadUrl, nil)
		if err != nil {
			src.Close()
			return nil, err
		}
		request.Body = newCompressed(src)
		request.Header.Set("Content-Type", "application/octet-stream")
		return request, nil
	}, func(resp *http.Response) error {
		_, err4 := io.Copy(ioutil.Discard, resp.Body)
		err5 := resp.Body.Close()
		if err4 != nil {
			return err4
		}
		return err5
	}, nil)
	_, err = req()
	return
}

// compressed is an io.ReadCloser that transparently compresses source data in
// a separate goroutine.
type compressed struct {
	pr  *io.PipeReader
	src io.ReadCloser
}

func (c *compressed) Read(data []byte) (int, error) {
	return c.pr.Read(data)
}

func (c *compressed) Close() error {
	err := c.pr.Close()
	if err1 := c.src.Close(); err == nil {
		err = err1
	}
	return err
}

func newCompressed(src io.ReadCloser) io.ReadCloser {
	pr, pw := io.Pipe()
	go func() {
		// The compressor itself is not thread safe.
		compressor := isolated.GetCompressor(pw)

		buf := make([]byte, 4096)
		if _, err := io.CopyBuffer(compressor, src, buf); err != nil {
			pw.CloseWithError(err)
			return
		}
		pw.CloseWithError(compressor.Close())
	}()

	return &compressed{pr, src}
}
<commit_msg>Make common/isolatedclient importable on GAE.<commit_after>// Copyright 2015 The LUCI Authors. All rights reserved.
// Use of this source code is governed under the Apache License, Version 2.0
// that can be found in the LICENSE file.

package isolatedclient

import (
	"bytes"
	"encoding/base64"
	"errors"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"

	"golang.org/x/net/context"

	isolateservice "github.com/luci/luci-go/common/api/isolate/isolateservice/v1"
	"github.com/luci/luci-go/common/isolated"
	"github.com/luci/luci-go/common/lhttp"
	"github.com/luci/luci-go/common/retry"
	"github.com/luci/luci-go/common/runtime/tracer"
)

// DefaultNamespace is the namespace that should be used with the New function.
const DefaultNamespace = "default-gzip"

// Source is a generator method to return source data. A generated Source must
// be Closed before the generator is called again.
type Source func() (io.ReadCloser, error)

// NewBytesSource returns a Source implementation that reads from the supplied
// byte slice.
func NewBytesSource(d []byte) Source {
	return func() (io.ReadCloser, error) {
		return ioutil.NopCloser(bytes.NewReader(d)), nil
	}
}

// PushState is per-item state passed from IsolateServer.Contains() to
// IsolateServer.Push().
//
// Its content is implementation specific.
type PushState struct {
	status    isolateservice.HandlersEndpointsV1PreuploadStatus
	digest    isolated.HexDigest
	size      int64
	uploaded  bool
	finalized bool
}

// FetchGCS is a function type of a handler for retrieving specified content from GCS and storing the response in the provided destination buffer.
type FetchGCS func(context.Context, *Client, isolateservice.HandlersEndpointsV1RetrievedContent, io.WriteSeeker) error

// Client is a client to an isolated server.
type Client struct {
	// All the members are immutable.
	retryFactory retry.Factory
	url          string
	namespace    string

	authClient      *http.Client // client that sends auth tokens
	anonClient      *http.Client // client that does NOT send auth tokens
	fetchGCSHandler FetchGCS     // function which fetches url from GCS and stores results in provided destination
}

// New returns a new IsolateServer client.
//
// 'authClient' must implement authentication sufficient to talk to Isolate server
// (OAuth tokens with 'email' scope).
//
// 'anonClient' must be a functional http.Client.
//
// If either client is nil, it will use http.DefaultClient (which will not work
// on Classic AppEngine!).
//
// If you're unsure which namespace to use, use the DefaultNamespace constant.
//
// If fetchGCSHandler is nil, the fetchGCSDefault function will be used to retrieve
// files from GCS.
func New(anonClient, authClient *http.Client, host, namespace string, rFn retry.Factory, fetchGCSHandler FetchGCS) *Client {
	if anonClient == nil {
		anonClient = http.DefaultClient
	}
	if authClient == nil {
		authClient = http.DefaultClient
	}
	if fetchGCSHandler == nil {
		fetchGCSHandler = fetchGCSDefault
	}
	i := &Client{
		retryFactory:    rFn,
		url:             strings.TrimRight(host, "/"),
		namespace:       namespace,
		authClient:      authClient,
		anonClient:      anonClient,
		fetchGCSHandler: fetchGCSHandler,
	}
	tracer.NewPID(i, "isolatedclient:"+i.url)
	return i
}

// ServerCapabilities returns the server details.
func (i *Client) ServerCapabilities(c context.Context) (*isolateservice.HandlersEndpointsV1ServerDetails, error) {
	out := &isolateservice.HandlersEndpointsV1ServerDetails{}
	if err := i.postJSON(c, "/api/isolateservice/v1/server_details", nil, map[string]string{}, out); err != nil {
		return nil, err
	}
	return out, nil
}

// Contains looks up cache presence on the server of multiple items.
//
// The returned list is in the same order as 'items', with entries nil for
// items that were present.
func (i *Client) Contains(c context.Context, items []*isolateservice.HandlersEndpointsV1Digest) (out []*PushState, err error) {
	end := tracer.Span(i, "contains", tracer.Args{"number": len(items)})
	defer func() { end(tracer.Args{"err": err}) }()
	in := isolateservice.HandlersEndpointsV1DigestCollection{Items: items, Namespace: &isolateservice.HandlersEndpointsV1Namespace{}}
	in.Namespace.Namespace = i.namespace
	data := &isolateservice.HandlersEndpointsV1UrlCollection{}
	if err = i.postJSON(c, "/api/isolateservice/v1/preupload", nil, in, data); err != nil {
		return nil, err
	}
	out = make([]*PushState, len(items))
	for _, e := range data.Items {
		index := int(e.Index)
		out[index] = &PushState{
			status: *e,
			digest: isolated.HexDigest(items[index].Digest),
			size:   items[index].Size,
		}
	}
	return out, nil
}

// Push pushed a missing item, as reported by Contains(), to the server.
func (i *Client) Push(c context.Context, state *PushState, source Source) (err error) {
	// This push operation may be a retry after failed finalization call below,
	// no need to reupload contents in that case.
	if !state.uploaded {
		// PUT file to uploadURL.
		if err = i.doPush(c, state, source); err != nil {
			log.Printf("doPush(%s) failed: %s\n%#v", state.digest, err, state)
			return
		}
		state.uploaded = true
	}

	// Optionally notify the server that it's done.
	if state.status.GsUploadUrl != "" {
		end := tracer.Span(i, "finalize", nil)
		defer func() { end(tracer.Args{"err": err}) }()
		// TODO(vadimsh): Calculate MD5 or CRC32C sum while uploading a file and
		// send it to isolated server. That way isolate server can verify that
		// the data safely reached Google Storage (GS provides MD5 and CRC32C of
		// stored files).
		in := isolateservice.HandlersEndpointsV1FinalizeRequest{UploadTicket: state.status.UploadTicket}
		headers := map[string]string{"Cache-Control": "public, max-age=31536000"}
		if err = i.postJSON(c, "/api/isolateservice/v1/finalize_gs_upload", headers, in, nil); err != nil {
			log.Printf("Push(%s) (finalize) failed: %s\n%#v", state.digest, err, state)
			return
		}
	}
	state.finalized = true
	return
}

// Fetch downloads an item from the server.
func (i *Client) Fetch(c context.Context, item *isolateservice.HandlersEndpointsV1Digest, dest io.WriteSeeker) error {
	// Perform initial request.
	url := i.url + "/api/isolateservice/v1/retrieve"
	in := &isolateservice.HandlersEndpointsV1RetrieveRequest{
		Digest: item.Digest,
		Namespace: &isolateservice.HandlersEndpointsV1Namespace{
			DigestHash: "sha-1",
			Namespace:  i.namespace,
		},
		Offset: 0,
	}
	var out isolateservice.HandlersEndpointsV1RetrievedContent
	if _, err := lhttp.PostJSON(c, i.retryFactory, i.authClient, url, nil, in, &out); err != nil {
		return err
	}

	// Handle DB items.
	if out.Content != "" {
		decoded, err := base64.StdEncoding.DecodeString(out.Content)
		if err != nil {
			return err
		}
		decompressor := isolated.GetDecompressor(bytes.NewReader(decoded))
		defer decompressor.Close()
		_, err = io.Copy(dest, decompressor)
		return err
	}

	// Handle GCS items.
	return i.fetchGCSHandler(c, i, out, dest)
}

// fetchGCSDefault uses the provided HandlersEndpointsV1RetrievedContent response to download content from GCS to the provided dest.
func fetchGCSDefault(c context.Context, i *Client, content isolateservice.HandlersEndpointsV1RetrievedContent, dest io.WriteSeeker) error {
	rgen := func() (*http.Request, error) {
		return http.NewRequest("GET", content.Url, nil)
	}
	handler := func(resp *http.Response) error {
		defer resp.Body.Close()
		decompressor := isolated.GetDecompressor(resp.Body)
		defer decompressor.Close()
		if _, err := dest.Seek(0, os.SEEK_SET); err != nil {
			return err
		}
		_, err := io.Copy(dest, decompressor)
		return err
	}
	_, err := lhttp.NewRequest(c, i.authClient, i.retryFactory, rgen, handler, nil)()
	return err
}

// postJSON does authenticated POST request.
func (i *Client) postJSON(c context.Context, resource string, headers map[string]string, in, out interface{}) error {
	if len(resource) == 0 || resource[0] != '/' {
		return errors.New("resource must start with '/'")
	}
	_, err := lhttp.PostJSON(c, i.retryFactory, i.authClient, i.url+resource, headers, in, out)
	return err
}

func (i *Client) doPush(c context.Context, state *PushState, source Source) (err error) {
	useDB := state.status.GsUploadUrl == ""
	end := tracer.Span(i, "push", tracer.Args{"useDB": useDB, "size": state.size})
	defer func() { end(tracer.Args{"err": err}) }()
	if useDB {
		src, err := source()
		if err != nil {
			return err
		}
		defer src.Close()

		err = i.doPushDB(c, state, src)
	} else {
		err = i.doPushGCS(c, state, source)
	}
	if err != nil {
		tracer.CounterAdd(i, "bytesUploaded", float64(state.size))
	}
	return err
}

func (i *Client) doPushDB(c context.Context, state *PushState, reader io.Reader) error {
	buf := bytes.Buffer{}
	compressor := isolated.GetCompressor(&buf)
	if _, err := io.Copy(compressor, reader); err != nil {
		return err
	}
	if err := compressor.Close(); err != nil {
		return err
	}
	in := &isolateservice.HandlersEndpointsV1StorageRequest{
		UploadTicket: state.status.UploadTicket,
		Content:      base64.StdEncoding.EncodeToString(buf.Bytes()),
	}
	return i.postJSON(c, "/api/isolateservice/v1/store_inline", nil, in, nil)
}

func (i *Client) doPushGCS(c context.Context, state *PushState, source Source) (err error) {
	// GsUploadUrl is signed Google Storage URL that doesn't require additional
	// authentication. In fact, using authClient causes HTTP 403 because
	// authClient's tokens don't have Cloud Storage OAuth scope. Use anonymous
	// client instead.
	req := lhttp.NewRequest(c, i.anonClient, i.retryFactory, func() (*http.Request, error) {
		src, err := source()
		if err != nil {
			return nil, err
		}

		request, err := http.NewRequest("PUT", state.status.GsUploadUrl, nil)
		if err != nil {
			src.Close()
			return nil, err
		}
		request.Body = newCompressed(src)
		request.Header.Set("Content-Type", "application/octet-stream")
		return request, nil
	}, func(resp *http.Response) error {
		_, err4 := io.Copy(ioutil.Discard, resp.Body)
		err5 := resp.Body.Close()
		if err4 != nil {
			return err4
		}
		return err5
	}, nil)
	_, err = req()
	return
}

// compressed is an io.ReadCloser that transparently compresses source data in
// a separate goroutine.
type compressed struct {
	pr  *io.PipeReader
	src io.ReadCloser
}

func (c *compressed) Read(data []byte) (int, error) {
	return c.pr.Read(data)
}

func (c *compressed) Close() error {
	err := c.pr.Close()
	if err1 := c.src.Close(); err == nil {
		err = err1
	}
	return err
}

func newCompressed(src io.ReadCloser) io.ReadCloser {
	pr, pw := io.Pipe()
	go func() {
		// The compressor itself is not thread safe.
		compressor := isolated.GetCompressor(pw)

		buf := make([]byte, 4096)
		if _, err := io.CopyBuffer(compressor, src, buf); err != nil {
			pw.CloseWithError(err)
			return
		}
		pw.CloseWithError(compressor.Close())
	}()

	return &compressed{pr, src}
}
<|endoftext|>


<commit_before>package models

import (
	"gopkg.in/yaml.v2"
	"io/ioutil"
)

type Setting struct {
	Database struct {
		Host       string
		DbName     string
		TokenTable string
		UserTable  string
	}
	Ssl struct {
		Key         string
		Sertificate string
	}
	Router struct {
		Register string
		Login    string
		Validate string
	}
}

var Set Setting

func LoadSettings() error {
	text, err := ioutil.ReadFile("./settings.yml")
	if err != nil {
		return err
	}

	if err = yaml.Unmarshal(text, &Set); err != nil {
		return err
	}

	return nil
}
<commit_msg>Rename Sertifiace to Certificate.<commit_after>package models

import (
	"gopkg.in/yaml.v2"
	"io/ioutil"
)

type Setting struct {
	Database struct {
		Host       string
		DbName     string
		TokenTable string
		UserTable  string
	}
	Ssl struct {
		Key         string
		Certificate string
	}
	Router struct {
		Register string
		Login    string
		Validate string
	}
}

var Set Setting

func LoadSettings() error {
	text, err := ioutil.ReadFile("./settings.yml")
	if err != nil {
		return err
	}

	if err = yaml.Unmarshal(text, &Set); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>/*
 Copyright 2019 Padduck, LLC
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at
  	http://www.apache.org/licenses/LICENSE-2.0
  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
*/

package models

import (
	"github.com/pufferpanel/pufferpanel/v2"
	"gopkg.in/go-playground/validator.v9"
	"net/url"
)

type UserView struct {
	Username string `json:"username,omitempty"`
	Email    string `json:"email,omitempty"`
	//ONLY SHOW WHEN COPYING
	Password string `json:"password,omitempty"`
}

func FromUser(model *User) *UserView {
	return &UserView{
		Username: model.Username,
		Email:    model.Email,
	}
}

func FromUsers(users *Users) []*UserView {
	result := make([]*UserView, len(*users))

	for k, v := range *users {
		result[k] = FromUser(v)
	}

	return result
}

func (model *UserView) CopyToModel(newModel *User) {
	if model.Username != "" {
		newModel.Username = model.Username
	}

	if model.Email != "" {
		newModel.Email = model.Email
	}

	if model.Password != "" {
		_ = newModel.SetPassword(model.Password)
	}
}

func (model *UserView) Valid(allowEmpty bool) error {
	validate := validator.New()

	if !allowEmpty && validate.Var(model.Username, "required") != nil {
		return pufferpanel.ErrFieldRequired("username")
	}

	if validate.Var(model.Username, "optional|printascii") != nil {
		return pufferpanel.ErrFieldMustBePrintable("username")
	}

	testName := url.QueryEscape(model.Username)
	if testName != model.Username {
		return pufferpanel.ErrFieldHasURICharacters("username")
	}

	if !allowEmpty && validate.Var(model.Email, "required") != nil {
		return pufferpanel.ErrFieldRequired("email")
	}

	if validate.Var(model.Email, "optional|email") != nil {
		return pufferpanel.ErrFieldNotEmail("email")
	}

	return nil
}
<commit_msg>Fix validator<commit_after>/*
 Copyright 2019 Padduck, LLC
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at
  	http://www.apache.org/licenses/LICENSE-2.0
  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
*/

package models

import (
	"github.com/pufferpanel/pufferpanel/v2"
	"gopkg.in/go-playground/validator.v9"
	"net/url"
)

type UserView struct {
	Username string `json:"username,omitempty"`
	Email    string `json:"email,omitempty"`
	//ONLY SHOW WHEN COPYING
	Password string `json:"password,omitempty"`
}

func FromUser(model *User) *UserView {
	return &UserView{
		Username: model.Username,
		Email:    model.Email,
	}
}

func FromUsers(users *Users) []*UserView {
	result := make([]*UserView, len(*users))

	for k, v := range *users {
		result[k] = FromUser(v)
	}

	return result
}

func (model *UserView) CopyToModel(newModel *User) {
	if model.Username != "" {
		newModel.Username = model.Username
	}

	if model.Email != "" {
		newModel.Email = model.Email
	}

	if model.Password != "" {
		_ = newModel.SetPassword(model.Password)
	}
}

func (model *UserView) Valid(allowEmpty bool) error {
	validate := validator.New()

	if !allowEmpty && validate.Var(model.Username, "required") != nil {
		return pufferpanel.ErrFieldRequired("username")
	}

	if validate.Var(model.Username, "omitempty|printascii") != nil {
		return pufferpanel.ErrFieldMustBePrintable("username")
	}

	testName := url.QueryEscape(model.Username)
	if testName != model.Username {
		return pufferpanel.ErrFieldHasURICharacters("username")
	}

	if !allowEmpty && validate.Var(model.Email, "required") != nil {
		return pufferpanel.ErrFieldRequired("email")
	}

	if validate.Var(model.Email, "omitempty|email") != nil {
		return pufferpanel.ErrFieldNotEmail("email")
	}

	return nil
}
<|endoftext|>