<commit_before>package metrics

import (
	"fmt"
	"time"

	"github.com/pachyderm/pachyderm/v2/src/enterprise"
	"github.com/pachyderm/pachyderm/v2/src/internal/clientsdk"
	"github.com/pachyderm/pachyderm/v2/src/internal/config"
	"github.com/pachyderm/pachyderm/v2/src/internal/errors"
	"github.com/pachyderm/pachyderm/v2/src/internal/serviceenv"
	"github.com/pachyderm/pachyderm/v2/src/internal/uuid"
	"github.com/pachyderm/pachyderm/v2/src/pfs"
	"github.com/pachyderm/pachyderm/v2/src/pps"
	enterprisemetrics "github.com/pachyderm/pachyderm/v2/src/server/enterprise/metrics"
	"github.com/pachyderm/pachyderm/v2/src/version"

	log "github.com/sirupsen/logrus"
	"golang.org/x/net/context"
	"google.golang.org/grpc/metadata"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	kube "k8s.io/client-go/kubernetes"
)

//Reporter is used to submit user & cluster metrics to segment
type Reporter struct {
	router    *router
	clusterID string
	env       serviceenv.ServiceEnv
}

// NewReporter creates a new reporter and kicks off the loop to report cluster
// metrics
func NewReporter(env serviceenv.ServiceEnv) *Reporter {
	var r *router
	if env.Config().MetricsEndpoint != "" {
		r = newRouter(env.Config().MetricsEndpoint)
	} else {
		r = newRouter()
	}
	reporter := &Reporter{
		router:    r,
		clusterID: env.ClusterID(),
		env:       env,
	}
	go reporter.reportClusterMetrics()
	return reporter
}

//ReportUserAction pushes the action into a queue for reporting,
// and reports the start, finish, and error conditions
func ReportUserAction(ctx context.Context, r *Reporter, action string) func(time.Time, error) {
	if r == nil {
		// This happens when stubbing out metrics for testing, e.g. src/server/pfs/server/server_test.go
		return func(time.Time, error) {}
	}
	// If we report nil, segment sees it, but mixpanel omits the field
	r.reportUserAction(ctx, fmt.Sprintf("%vStarted", action), 1)
	return func(start time.Time, err error) {
		if err == nil {
			r.reportUserAction(ctx, fmt.Sprintf("%vFinished", action), time.Since(start).Seconds())
		} else {
			r.reportUserAction(ctx, fmt.Sprintf("%vErrored", action), err.Error())
		}
	}
}

func getKeyFromMD(md metadata.MD, key string) (string, error) {
	if md[key] != nil && len(md[key]) > 0 {
		return md[key][0], nil
	}
	return "", errors.Errorf("error extracting userid from metadata. userid is empty")
}

func (r *Reporter) reportUserAction(ctx context.Context, action string, value interface{}) {
	md, ok := metadata.FromIncomingContext(ctx)
	if ok {
		// metadata API downcases all the key names
		userID, err := getKeyFromMD(md, "userid")
		if err != nil {
			// The FUSE client will never have a userID, so normal usage will produce a lot of these errors
			return
		}
		prefix, err := getKeyFromMD(md, "prefix")
		if err != nil {
			// metrics errors are non fatal
			return
		}
		r.router.reportUserMetricsToSegment(
			userID,
			prefix,
			action,
			value,
			r.clusterID,
		)
	}
}

// Helper method called by (Start|Finish)ReportAndFlushUserAction. Like those
// functions, it is used by the pachctl binary and runs on users' machines
func reportAndFlushUserAction(action string, value interface{}) func() {
	metricsDone := make(chan struct{})
	go func() {
		defer close(metricsDone)
		cfg, _ := config.Read(false, false)
		if cfg == nil || cfg.UserID == "" || !cfg.V2.Metrics {
			return
		}
		client := newSegmentClient()
		defer client.Close()
		reportUserMetricsToSegment(client, cfg.UserID, "user", action, value, "")
	}()
	return func() {
		select {
		case <-metricsDone:
			return
		case <-time.After(time.Second * 5):
			return
		}
	}
}

// StartReportAndFlushUserAction immediately reports the metric but does
// not block execution. It returns a wait function which waits or times
// out after 5s.
// It is used by the pachctl binary and runs on users' machines
func StartReportAndFlushUserAction(action string, value interface{}) func() {
	return reportAndFlushUserAction(fmt.Sprintf("%vStarted", action), value)
}

// FinishReportAndFlushUserAction immediately reports the metric but does
// not block execution. It returns a wait function which waits or times
// out after 5s.
// It is used by the pachctl binary and runs on users' machines
func FinishReportAndFlushUserAction(action string, err error, start time.Time) func() {
	var wait func()
	if err != nil {
		wait = reportAndFlushUserAction(fmt.Sprintf("%vErrored", action), err)
	} else {
		wait = reportAndFlushUserAction(fmt.Sprintf("%vFinished", action), time.Since(start).Seconds())
	}
	return wait
}

func (r *Reporter) reportClusterMetrics() {
	for {
		time.Sleep(reportingInterval)
		metrics := &Metrics{}
		r.internalMetrics(metrics)
		externalMetrics(r.env.GetKubeClient(), metrics)
		metrics.ClusterID = r.clusterID
		metrics.PodID = uuid.NewWithoutDashes()
		metrics.Version = version.PrettyPrintVersion(version.Version)
		r.router.reportClusterMetricsToSegment(metrics)
	}
}

func externalMetrics(kubeClient *kube.Clientset, metrics *Metrics) error {
	nodeList, err := kubeClient.CoreV1().Nodes().List(metav1.ListOptions{})
	if err != nil {
		return errors.Wrapf(err, "externalMetrics: unable to retrieve node list from k8s")
	}
	metrics.Nodes = int64(len(nodeList.Items))
	return nil
}

func pfsInputMetrics(pfsInput *pps.PFSInput, metrics *Metrics) {
	metrics.InputPfs++
	if pfsInput.Commit != "" {
		metrics.InputCommit++
	}
	if pfsInput.JoinOn != "" {
		metrics.InputJoinOn++
	}
	if pfsInput.OuterJoin {
		metrics.InputOuterJoin++
	}
	if pfsInput.Lazy {
		metrics.InputLazy++
	}
	if pfsInput.EmptyFiles {
		metrics.InputEmptyFiles++
	}
	if pfsInput.S3 {
		metrics.InputS3++
	}
	if pfsInput.Trigger != nil {
		metrics.InputTrigger++
	}
}

func inputMetrics(input *pps.Input, metrics *Metrics) {
	if input.Join != nil {
		metrics.InputJoin++
		for _, item := range input.Join {
			if item.Pfs != nil {
				pfsInputMetrics(item.Pfs, metrics)
			} else {
				inputMetrics(item, metrics)
			}
		}
	}
	if input.Group != nil {
		metrics.InputGroup++
		for _, item := range input.Group {
			if item.Pfs != nil {
				pfsInputMetrics(item.Pfs, metrics)
			} else {
				inputMetrics(item, metrics)
			}
		}
	}
	if input.Cross != nil {
		metrics.InputCross++
		for _, item := range input.Cross {
			if item.Pfs != nil {
				pfsInputMetrics(item.Pfs, metrics)
			} else {
				inputMetrics(item, metrics)
			}
		}
	}
	if input.Union != nil {
		metrics.InputUnion++
		for _, item := range input.Union {
			if item.Pfs != nil {
				pfsInputMetrics(item.Pfs, metrics)
			} else {
				inputMetrics(item, metrics)
			}
		}
	}
	if input.Cron != nil {
		metrics.InputCron++
	}
	if input.Pfs != nil {
		pfsInputMetrics(input.Pfs, metrics)
	}
}

func (r *Reporter) internalMetrics(metrics *Metrics) {
	// We should not return due to an error
	// Activation code
	ctx, cf := context.WithCancel(context.Background())
	defer cf()
	enterpriseState, err := r.env.EnterpriseServer().GetState(ctx, &enterprise.GetStateRequest{})
	if err == nil {
		metrics.ActivationCode = enterpriseState.ActivationCode
	}
	metrics.EnterpriseFailures = enterprisemetrics.GetEnterpriseFailures()

	// Pipeline info
	if err := func() error {
		lpClient, err := r.env.GetPachClient(ctx).PpsAPIClient.ListPipeline(ctx, &pps.ListPipelineRequest{Details: false})
		if err != nil {
			return err
		}
		pipelineInfos, err := clientsdk.ListPipelineInfo(lpClient)
		if err != nil {
			return err
		}
		metrics.Pipelines = int64(len(pipelineInfos)) // Number of pipelines
		for _, pi := range pipelineInfos {
			if pi.Details.ParallelismSpec != nil {
				if metrics.MaxParallelism < pi.Details.ParallelismSpec.Constant {
					metrics.MaxParallelism = pi.Details.ParallelismSpec.Constant
				}
				if metrics.MinParallelism > pi.Details.ParallelismSpec.Constant {
					metrics.MinParallelism = pi.Details.ParallelismSpec.Constant
				}
				metrics.NumParallelism++
			}
			if pi.Details.Egress != nil {
				metrics.CfgEgress++
			}
			if pi.JobCounts != nil {
				var cnt int64 = 0
				for _, c := range pi.JobCounts {
					cnt += int64(c)
				}
				if metrics.Jobs < cnt {
					metrics.Jobs = cnt
				}
			}
			if details := pi.Details; details != nil {
				if details.ParallelismSpec != nil {
					if metrics.MaxParallelism < details.ParallelismSpec.Constant {
						metrics.MaxParallelism = details.ParallelismSpec.Constant
					}
					if metrics.MinParallelism > details.ParallelismSpec.Constant {
						metrics.MinParallelism = details.ParallelismSpec.Constant
					}
					metrics.NumParallelism++
				}
				if details.Egress != nil {
					metrics.CfgEgress++
				}
				if details.ResourceRequests != nil {
					if details.ResourceRequests.Cpu != 0 {
						metrics.ResourceCpuReq += details.ResourceRequests.Cpu
						if metrics.ResourceCpuReqMax < details.ResourceRequests.Cpu {
							metrics.ResourceCpuReqMax = details.ResourceRequests.Cpu
						}
					}
					if details.ResourceRequests.Memory != "" {
						metrics.ResourceMemReq += (details.ResourceRequests.Memory + " ")
					}
					if details.ResourceRequests.Gpu != nil {
						metrics.ResourceGpuReq += details.ResourceRequests.Gpu.Number
						if metrics.ResourceGpuReqMax < details.ResourceRequests.Gpu.Number {
							metrics.ResourceGpuReqMax = details.ResourceRequests.Gpu.Number
						}
					}
					if details.ResourceRequests.Disk != "" {
						metrics.ResourceDiskReq += (details.ResourceRequests.Disk + " ")
					}
				}
				if details.ResourceLimits != nil {
					if details.ResourceLimits.Cpu != 0 {
						metrics.ResourceCpuLimit += details.ResourceLimits.Cpu
						if metrics.ResourceCpuLimitMax < details.ResourceLimits.Cpu {
							metrics.ResourceCpuLimitMax = details.ResourceLimits.Cpu
						}
					}
					if details.ResourceLimits.Memory != "" {
						metrics.ResourceMemLimit += (details.ResourceLimits.Memory + " ")
					}
					if details.ResourceLimits.Gpu != nil {
						metrics.ResourceGpuLimit += details.ResourceLimits.Gpu.Number
						if metrics.ResourceGpuLimitMax < details.ResourceLimits.Gpu.Number {
							metrics.ResourceGpuLimitMax = details.ResourceLimits.Gpu.Number
						}
					}
					if details.ResourceLimits.Disk != "" {
						metrics.ResourceDiskLimit += (details.ResourceLimits.Disk + " ")
					}
				}
				if details.Input != nil {
					inputMetrics(details.Input, metrics)
				}
				if details.Service != nil {
					metrics.CfgServices++
				}
				if details.Spout != nil {
					metrics.PpsSpout++
					if details.Spout.Service != nil {
						metrics.PpsSpoutService++
					}
				}
				if details.S3Out {
					metrics.CfgS3Gateway++
				}
				if details.Transform != nil {
					if details.Transform.ErrCmd != nil {
						metrics.CfgErrcmd++
					}
				}
				if details.TFJob != nil {
					metrics.CfgTfjob++
				}
			}
		}
		return nil
	}(); err != nil {
		log.Errorf("Error getting pipeline metrics: %v", err)
	}

	if err := func() error {
		rClient, err := r.env.GetPachClient(ctx).PfsAPIClient.ListRepo(ctx, &pfs.ListRepoRequest{})
		if err != nil {
			return err
		}
		repoInfos, err := clientsdk.ListRepoInfo(rClient)
		if err != nil {
			return err
		}
		var sz, mbranch uint64 = 0, 0
		for _, ri := range repoInfos {
			sz += uint64(ri.SizeBytesUpperBound)
			if mbranch < uint64(len(ri.Branches)) {
				mbranch = uint64(len(ri.Branches))
			}
		}
		metrics.Repos = int64(len(repoInfos))
		metrics.Bytes = sz
		metrics.MaxBranches = mbranch
		return nil
	}(); err != nil {
		log.Errorf("Error getting repos: %v", err)
	}
	//log.Infof("Metrics logged: %v", metrics)
}
<commit_msg>Fix metrics pipeline details (#6547)<commit_after>package metrics

import (
	"fmt"
	"time"

	"github.com/pachyderm/pachyderm/v2/src/enterprise"
	"github.com/pachyderm/pachyderm/v2/src/internal/clientsdk"
	"github.com/pachyderm/pachyderm/v2/src/internal/config"
	"github.com/pachyderm/pachyderm/v2/src/internal/errors"
	"github.com/pachyderm/pachyderm/v2/src/internal/serviceenv"
	"github.com/pachyderm/pachyderm/v2/src/internal/uuid"
	"github.com/pachyderm/pachyderm/v2/src/pfs"
	"github.com/pachyderm/pachyderm/v2/src/pps"
	enterprisemetrics "github.com/pachyderm/pachyderm/v2/src/server/enterprise/metrics"
	"github.com/pachyderm/pachyderm/v2/src/version"

	log "github.com/sirupsen/logrus"
	"golang.org/x/net/context"
	"google.golang.org/grpc/metadata"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	kube "k8s.io/client-go/kubernetes"
)

//Reporter is used to submit user & cluster metrics to segment
type Reporter struct {
	router    *router
	clusterID string
	env       serviceenv.ServiceEnv
}

// NewReporter creates a new reporter and kicks off the loop to report cluster
// metrics
func NewReporter(env serviceenv.ServiceEnv) *Reporter {
	var r *router
	if env.Config().MetricsEndpoint != "" {
		r = newRouter(env.Config().MetricsEndpoint)
	} else {
		r = newRouter()
	}
	reporter := &Reporter{
		router:    r,
		clusterID: env.ClusterID(),
		env:       env,
	}
	go reporter.reportClusterMetrics()
	return reporter
}

//ReportUserAction pushes the action into a queue for reporting,
// and reports the start, finish, and error conditions
func ReportUserAction(ctx context.Context, r *Reporter, action string) func(time.Time, error) {
	if r == nil {
		// This happens when stubbing out metrics for testing, e.g. src/server/pfs/server/server_test.go
		return func(time.Time, error) {}
	}
	// If we report nil, segment sees it, but mixpanel omits the field
	r.reportUserAction(ctx, fmt.Sprintf("%vStarted", action), 1)
	return func(start time.Time, err error) {
		if err == nil {
			r.reportUserAction(ctx, fmt.Sprintf("%vFinished", action), time.Since(start).Seconds())
		} else {
			r.reportUserAction(ctx, fmt.Sprintf("%vErrored", action), err.Error())
		}
	}
}

func getKeyFromMD(md metadata.MD, key string) (string, error) {
	if md[key] != nil && len(md[key]) > 0 {
		return md[key][0], nil
	}
	return "", errors.Errorf("error extracting userid from metadata. userid is empty")
}

func (r *Reporter) reportUserAction(ctx context.Context, action string, value interface{}) {
	md, ok := metadata.FromIncomingContext(ctx)
	if ok {
		// metadata API downcases all the key names
		userID, err := getKeyFromMD(md, "userid")
		if err != nil {
			// The FUSE client will never have a userID, so normal usage will produce a lot of these errors
			return
		}
		prefix, err := getKeyFromMD(md, "prefix")
		if err != nil {
			// metrics errors are non fatal
			return
		}
		r.router.reportUserMetricsToSegment(
			userID,
			prefix,
			action,
			value,
			r.clusterID,
		)
	}
}

// Helper method called by (Start|Finish)ReportAndFlushUserAction. Like those
// functions, it is used by the pachctl binary and runs on users' machines
func reportAndFlushUserAction(action string, value interface{}) func() {
	metricsDone := make(chan struct{})
	go func() {
		defer close(metricsDone)
		cfg, _ := config.Read(false, false)
		if cfg == nil || cfg.UserID == "" || !cfg.V2.Metrics {
			return
		}
		client := newSegmentClient()
		defer client.Close()
		reportUserMetricsToSegment(client, cfg.UserID, "user", action, value, "")
	}()
	return func() {
		select {
		case <-metricsDone:
			return
		case <-time.After(time.Second * 5):
			return
		}
	}
}

// StartReportAndFlushUserAction immediately reports the metric but does
// not block execution. It returns a wait function which waits or times
// out after 5s.
// It is used by the pachctl binary and runs on users' machines
func StartReportAndFlushUserAction(action string, value interface{}) func() {
	return reportAndFlushUserAction(fmt.Sprintf("%vStarted", action), value)
}

// FinishReportAndFlushUserAction immediately reports the metric but does
// not block execution. It returns a wait function which waits or times
// out after 5s.
// It is used by the pachctl binary and runs on users' machines
func FinishReportAndFlushUserAction(action string, err error, start time.Time) func() {
	var wait func()
	if err != nil {
		wait = reportAndFlushUserAction(fmt.Sprintf("%vErrored", action), err)
	} else {
		wait = reportAndFlushUserAction(fmt.Sprintf("%vFinished", action), time.Since(start).Seconds())
	}
	return wait
}

func (r *Reporter) reportClusterMetrics() {
	for {
		time.Sleep(reportingInterval)
		metrics := &Metrics{}
		r.internalMetrics(metrics)
		externalMetrics(r.env.GetKubeClient(), metrics)
		metrics.ClusterID = r.clusterID
		metrics.PodID = uuid.NewWithoutDashes()
		metrics.Version = version.PrettyPrintVersion(version.Version)
		r.router.reportClusterMetricsToSegment(metrics)
	}
}

func externalMetrics(kubeClient *kube.Clientset, metrics *Metrics) error {
	nodeList, err := kubeClient.CoreV1().Nodes().List(metav1.ListOptions{})
	if err != nil {
		return errors.Wrapf(err, "externalMetrics: unable to retrieve node list from k8s")
	}
	metrics.Nodes = int64(len(nodeList.Items))
	return nil
}

func pfsInputMetrics(pfsInput *pps.PFSInput, metrics *Metrics) {
	metrics.InputPfs++
	if pfsInput.Commit != "" {
		metrics.InputCommit++
	}
	if pfsInput.JoinOn != "" {
		metrics.InputJoinOn++
	}
	if pfsInput.OuterJoin {
		metrics.InputOuterJoin++
	}
	if pfsInput.Lazy {
		metrics.InputLazy++
	}
	if pfsInput.EmptyFiles {
		metrics.InputEmptyFiles++
	}
	if pfsInput.S3 {
		metrics.InputS3++
	}
	if pfsInput.Trigger != nil {
		metrics.InputTrigger++
	}
}

func inputMetrics(input *pps.Input, metrics *Metrics) {
	if input.Join != nil {
		metrics.InputJoin++
		for _, item := range input.Join {
			if item.Pfs != nil {
				pfsInputMetrics(item.Pfs, metrics)
			} else {
				inputMetrics(item, metrics)
			}
		}
	}
	if input.Group != nil {
		metrics.InputGroup++
		for _, item := range input.Group {
			if item.Pfs != nil {
				pfsInputMetrics(item.Pfs, metrics)
			} else {
				inputMetrics(item, metrics)
			}
		}
	}
	if input.Cross != nil {
		metrics.InputCross++
		for _, item := range input.Cross {
			if item.Pfs != nil {
				pfsInputMetrics(item.Pfs, metrics)
			} else {
				inputMetrics(item, metrics)
			}
		}
	}
	if input.Union != nil {
		metrics.InputUnion++
		for _, item := range input.Union {
			if item.Pfs != nil {
				pfsInputMetrics(item.Pfs, metrics)
			} else {
				inputMetrics(item, metrics)
			}
		}
	}
	if input.Cron != nil {
		metrics.InputCron++
	}
	if input.Pfs != nil {
		pfsInputMetrics(input.Pfs, metrics)
	}
}

func (r *Reporter) internalMetrics(metrics *Metrics) {
	// We should not return due to an error
	// Activation code
	ctx, cf := context.WithCancel(context.Background())
	defer cf()
	enterpriseState, err := r.env.EnterpriseServer().GetState(ctx, &enterprise.GetStateRequest{})
	if err == nil {
		metrics.ActivationCode = enterpriseState.ActivationCode
	}
	metrics.EnterpriseFailures = enterprisemetrics.GetEnterpriseFailures()

	// Pipeline info
	if err := func() error {
		lpClient, err := r.env.GetPachClient(ctx).PpsAPIClient.ListPipeline(ctx, &pps.ListPipelineRequest{Details: false})
		if err != nil {
			return err
		}
		pipelineInfos, err := clientsdk.ListPipelineInfo(lpClient)
		if err != nil {
			return err
		}
		metrics.Pipelines = int64(len(pipelineInfos)) // Number of pipelines
		for _, pi := range pipelineInfos {
			if pi.JobCounts != nil {
				var cnt int64 = 0
				for _, c := range pi.JobCounts {
					cnt += int64(c)
				}
				if metrics.Jobs < cnt {
					metrics.Jobs = cnt
				}
			}
			if details := pi.Details; details != nil {
				if details.ParallelismSpec != nil {
					if metrics.MaxParallelism < details.ParallelismSpec.Constant {
						metrics.MaxParallelism = details.ParallelismSpec.Constant
					}
					if metrics.MinParallelism > details.ParallelismSpec.Constant {
						metrics.MinParallelism = details.ParallelismSpec.Constant
					}
					metrics.NumParallelism++
				}
				if details.Egress != nil {
					metrics.CfgEgress++
				}
				if details.ResourceRequests != nil {
					if details.ResourceRequests.Cpu != 0 {
						metrics.ResourceCpuReq += details.ResourceRequests.Cpu
						if metrics.ResourceCpuReqMax < details.ResourceRequests.Cpu {
							metrics.ResourceCpuReqMax = details.ResourceRequests.Cpu
						}
					}
					if details.ResourceRequests.Memory != "" {
						metrics.ResourceMemReq += (details.ResourceRequests.Memory + " ")
					}
					if details.ResourceRequests.Gpu != nil {
						metrics.ResourceGpuReq += details.ResourceRequests.Gpu.Number
						if metrics.ResourceGpuReqMax < details.ResourceRequests.Gpu.Number {
							metrics.ResourceGpuReqMax = details.ResourceRequests.Gpu.Number
						}
					}
					if details.ResourceRequests.Disk != "" {
						metrics.ResourceDiskReq += (details.ResourceRequests.Disk + " ")
					}
				}
				if details.ResourceLimits != nil {
					if details.ResourceLimits.Cpu != 0 {
						metrics.ResourceCpuLimit += details.ResourceLimits.Cpu
						if metrics.ResourceCpuLimitMax < details.ResourceLimits.Cpu {
							metrics.ResourceCpuLimitMax = details.ResourceLimits.Cpu
						}
					}
					if details.ResourceLimits.Memory != "" {
						metrics.ResourceMemLimit += (details.ResourceLimits.Memory + " ")
					}
					if details.ResourceLimits.Gpu != nil {
						metrics.ResourceGpuLimit += details.ResourceLimits.Gpu.Number
						if metrics.ResourceGpuLimitMax < details.ResourceLimits.Gpu.Number {
							metrics.ResourceGpuLimitMax = details.ResourceLimits.Gpu.Number
						}
					}
					if details.ResourceLimits.Disk != "" {
						metrics.ResourceDiskLimit += (details.ResourceLimits.Disk + " ")
					}
				}
				if details.Input != nil {
					inputMetrics(details.Input, metrics)
				}
				if details.Service != nil {
					metrics.CfgServices++
				}
				if details.Spout != nil {
					metrics.PpsSpout++
					if details.Spout.Service != nil {
						metrics.PpsSpoutService++
					}
				}
				if details.S3Out {
					metrics.CfgS3Gateway++
				}
				if details.Transform != nil {
					if details.Transform.ErrCmd != nil {
						metrics.CfgErrcmd++
					}
				}
				if details.TFJob != nil {
					metrics.CfgTfjob++
				}
			}
		}
		return nil
	}(); err != nil {
		log.Errorf("Error getting pipeline metrics: %v", err)
	}

	if err := func() error {
		rClient, err := r.env.GetPachClient(ctx).PfsAPIClient.ListRepo(ctx, &pfs.ListRepoRequest{})
		if err != nil {
			return err
		}
		repoInfos, err := clientsdk.ListRepoInfo(rClient)
		if err != nil {
			return err
		}
		var sz, mbranch uint64 = 0, 0
		for _, ri := range repoInfos {
			sz += uint64(ri.SizeBytesUpperBound)
			if mbranch < uint64(len(ri.Branches)) {
				mbranch = uint64(len(ri.Branches))
			}
		}
		metrics.Repos = int64(len(repoInfos))
		metrics.Bytes = sz
		metrics.MaxBranches = mbranch
		return nil
	}(); err != nil {
		log.Errorf("Error getting repos: %v", err)
	}
	//log.Infof("Metrics logged: %v", metrics)
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package main

import (
	"encoding/json"
	"fmt"
	"net/http"

	"golang.org/x/net/context"

	"cocoon/commands"
	"cocoon/db"

	"io/ioutil"

	"strings"

	"google.golang.org/appengine"

	"google.golang.org/appengine/user"
)

func init() {
	http.HandleFunc("/api/get-authentication-status", getAuthenticationStatus)
	registerRPC("/api/create-agent", commands.CreateAgent)
	registerRPC("/api/authorize-agent", commands.AuthorizeAgent)
	registerRPC("/api/get-status", commands.GetStatus)
	registerRPC("/api/refresh-github-commits", commands.RefreshGithubCommits)
	registerRPC("/api/refresh-travis-status", commands.RefreshTravisStatus)
	registerRPC("/api/reserve-task", commands.ReserveTask)
	registerRPC("/api/update-task-status", commands.UpdateTaskStatus)
}

func registerRPC(path string, handler func(cocoon *db.Cocoon, inputJSON []byte) (interface{}, error)) {
	http.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
		ctx := appengine.NewContext(r)
		cocoon := db.NewCocoon(ctx)
		ctx, err := getAuthenticatedContext(ctx, r)

		if err != nil {
			serveUnauthorizedAccess(w, err)
			return
		}

		bytes, err := ioutil.ReadAll(r.Body)
		if err != nil {
			serveError(w, err)
			return
		}

		response, err := handler(cocoon, bytes)
		if err != nil {
			serveError(w, err)
			return
		}

		outputData, err := json.Marshal(response)
		if err != nil {
			serveError(w, err)
			return
		}

		w.Write(outputData)
	})
}

func authenticateAgent(ctx context.Context, agentID string, agentAuthToken string) (*db.Agent, error) {
	cocoon := db.NewCocoon(ctx)
	agent, err := cocoon.GetAgentByAuthToken(agentID, agentAuthToken)

	if err != nil {
		return nil, err
	}

	return agent, nil
}

func serveError(w http.ResponseWriter, err error) {
	http.Error(w, fmt.Sprintf("%v", err), http.StatusInternalServerError)
}

func serveUnauthorizedAccess(w http.ResponseWriter, err error) {
	http.Error(w, fmt.Sprintf("Authentication/authorization error: %v", err), http.StatusUnauthorized)
}

func getAuthenticatedContext(ctx context.Context, r *http.Request) (context.Context, error) {
	agentAuthToken := r.Header.Get("Agent-Auth-Token")
	isCron := r.Header.Get("X-Appengine-Cron") == "true"
	if agentAuthToken != "" {
		// Authenticate as an agent
		agentID := r.Header.Get("Agent-ID")
		agent, err := authenticateAgent(ctx, agentID, agentAuthToken)

		if err != nil {
			return nil, fmt.Errorf("Invalid agent: %v", agentID)
		}

		return context.WithValue(ctx, "agent", agent), nil
	} else if isCron {
		// Authenticate cron requests
		return ctx, nil
	} else {
		// Authenticate as Google account
		user := user.Current(ctx)

		if user == nil {
			return nil, fmt.Errorf("User not signed in")
		}

		if !strings.HasSuffix(user.Email, "@google.com") {
			return nil, fmt.Errorf("Only @google.com users are authorized")
		}

		return ctx, nil
	}
}

func getAuthenticationStatus(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)
	ctx, err := getAuthenticatedContext(ctx, r)

	var response map[string]interface{}

	if err == nil {
		response = map[string]interface{}{
			"Status": "OK",
		}
	} else {
		loginURL, _ := user.LoginURL(ctx, "/")
		response = map[string]interface{}{
			"LoginURL": loginURL,
		}
	}

	outputData, _ := json.Marshal(response)
	w.Write(outputData)
}
<commit_msg>log errors; more paranoia (#3)<commit_after>// Copyright (c) 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package main

import (
	"encoding/json"
	"fmt"
	"net/http"

	"golang.org/x/net/context"

	"cocoon/commands"
	"cocoon/db"

	"io/ioutil"

	"strings"

	"google.golang.org/appengine"

	"google.golang.org/appengine/log"
	"google.golang.org/appengine/user"
)

func init() {
	registerRPC("/api/create-agent", commands.CreateAgent)
	registerRPC("/api/authorize-agent", commands.AuthorizeAgent)
	registerRPC("/api/get-status", commands.GetStatus)
	registerRPC("/api/refresh-github-commits", commands.RefreshGithubCommits)
	registerRPC("/api/refresh-travis-status", commands.RefreshTravisStatus)
	registerRPC("/api/reserve-task", commands.ReserveTask)
	registerRPC("/api/update-task-status", commands.UpdateTaskStatus)

	// IMPORTANT: This is the ONLY path that does NOT require authentication. Do
	//            not copy this pattern. Use registerRPC instead.
	http.HandleFunc("/api/get-authentication-status", getAuthenticationStatus)
}

func registerRPC(path string, handler func(cocoon *db.Cocoon, inputJSON []byte) (interface{}, error)) {
	http.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
		ctx := appengine.NewContext(r)
		cocoon := db.NewCocoon(ctx)
		ctx, err := getAuthenticatedContext(ctx, r)

		if err != nil {
			serveUnauthorizedAccess(w, err)
			return
		}

		bytes, err := ioutil.ReadAll(r.Body)
		if err != nil {
			serveError(ctx, w, r, err)
			return
		}

		response, err := handler(cocoon, bytes)
		if err != nil {
			serveError(ctx, w, r, err)
			return
		}

		outputData, err := json.Marshal(response)
		if err != nil {
			serveError(ctx, w, r, err)
			return
		}

		w.Write(outputData)
	})
}

func authenticateAgent(ctx context.Context, agentID string, agentAuthToken string) (*db.Agent, error) {
	cocoon := db.NewCocoon(ctx)
	agent, err := cocoon.GetAgentByAuthToken(agentID, agentAuthToken)

	if err != nil {
		return nil, err
	}

	return agent, nil
}

func serveError(ctx context.Context, w http.ResponseWriter, r *http.Request, err error) {
	errorMessage := fmt.Sprintf("[%v] %v", r.URL, err)
	log.Errorf(ctx, "%v", errorMessage)
	http.Error(w, errorMessage, http.StatusInternalServerError)
}

func serveUnauthorizedAccess(w http.ResponseWriter, err error) {
	http.Error(w, fmt.Sprintf("Authentication/authorization error: %v", err), http.StatusUnauthorized)
}

func getAuthenticatedContext(ctx context.Context, r *http.Request) (context.Context, error) {
	agentAuthToken := r.Header.Get("Agent-Auth-Token")
	isCron := r.Header.Get("X-Appengine-Cron") == "true"
	if agentAuthToken != "" {
		// Authenticate as an agent. Note that it could simulaneously be cron and
		// agent, or Google account and agent.
		agentID := r.Header.Get("Agent-ID")
		agent, err := authenticateAgent(ctx, agentID, agentAuthToken)

		if err != nil {
			return nil, fmt.Errorf("Invalid agent: %v", agentID)
		}

		return context.WithValue(ctx, "agent", agent), nil
	} else if isCron {
		// Authenticate cron requests that are not agents.
		return ctx, nil
	} else {
		// Authenticate as Google account. Note that it could be both a Google
		// account and agent.
		user := user.Current(ctx)

		if user == nil {
			return nil, fmt.Errorf("User not signed in")
		}

		if !strings.HasSuffix(user.Email, "@google.com") {
			return nil, fmt.Errorf("Only @google.com users are authorized")
		}

		return ctx, nil
	}
}

func getAuthenticationStatus(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)
	ctx, err := getAuthenticatedContext(ctx, r)

	var response map[string]interface{}

	if err == nil {
		response = map[string]interface{}{
			"Status": "OK",
		}
	} else {
		loginURL, _ := user.LoginURL(ctx, "/")
		response = map[string]interface{}{
			"LoginURL": loginURL,
		}
	}

	outputData, _ := json.Marshal(response)
	w.Write(outputData)
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package main

import (
	"encoding/json"
	"fmt"
	"net/http"

	"golang.org/x/net/context"

	"cocoon/commands"
	"cocoon/db"

	"io/ioutil"

	"strings"

	"google.golang.org/appengine"

	"google.golang.org/appengine/log"
	"google.golang.org/appengine/user"
)

func init() {
	registerRPC("/api/create-agent", commands.CreateAgent)
	registerRPC("/api/authorize-agent", commands.AuthorizeAgent)
	registerRPC("/api/get-status", commands.GetStatus)
	registerRPC("/api/refresh-github-commits", commands.RefreshGithubCommits)
	registerRPC("/api/refresh-travis-status", commands.RefreshTravisStatus)
	registerRPC("/api/reserve-task", commands.ReserveTask)
	registerRPC("/api/update-task-status", commands.UpdateTaskStatus)

	// IMPORTANT: This is the ONLY path that does NOT require authentication. Do
	//            not copy this pattern. Use registerRPC instead.
	http.HandleFunc("/api/get-authentication-status", getAuthenticationStatus)
}

func registerRPC(path string, handler func(cocoon *db.Cocoon, inputJSON []byte) (interface{}, error)) {
	http.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
		ctx := appengine.NewContext(r)
		cocoon := db.NewCocoon(ctx)
		ctx, err := getAuthenticatedContext(ctx, r)

		if err != nil {
			serveUnauthorizedAccess(w, err)
			return
		}

		bytes, err := ioutil.ReadAll(r.Body)
		if err != nil {
			serveError(ctx, w, r, err)
			return
		}

		response, err := handler(cocoon, bytes)
		if err != nil {
			serveError(ctx, w, r, err)
			return
		}

		outputData, err := json.Marshal(response)
		if err != nil {
			serveError(ctx, w, r, err)
			return
		}

		w.Write(outputData)
	})
}

func authenticateAgent(ctx context.Context, agentID string, agentAuthToken string) (*db.Agent, error) {
	cocoon := db.NewCocoon(ctx)
	agent, err := cocoon.GetAgentByAuthToken(agentID, agentAuthToken)

	if err != nil {
		return nil, err
	}

	return agent, nil
}

func serveError(ctx context.Context, w http.ResponseWriter, r *http.Request, err error) {
	errorMessage := fmt.Sprintf("[%v] %v", r.URL, err)
	log.Errorf(ctx, "%v", errorMessage)
	http.Error(w, errorMessage, http.StatusInternalServerError)
}

func serveUnauthorizedAccess(w http.ResponseWriter, err error) {
	http.Error(w, fmt.Sprintf("Authentication/authorization error: %v", err), http.StatusUnauthorized)
}

func getAuthenticatedContext(ctx context.Context, r *http.Request) (context.Context, error) {
	agentAuthToken := r.Header.Get("Agent-Auth-Token")
	isCron := r.Header.Get("X-Appengine-Cron") == "true"
	if agentAuthToken != "" {
		// Authenticate as an agent. Note that it could simulaneously be cron and
		// agent, or Google account and agent.
		agentID := r.Header.Get("Agent-ID")
		agent, err := authenticateAgent(ctx, agentID, agentAuthToken)

		if err != nil {
			return nil, fmt.Errorf("Invalid agent: %v", agentID)
		}

		return context.WithValue(ctx, "agent", agent), nil
	} else if isCron {
		// Authenticate cron requests that are not agents.
		return ctx, nil
	} else {
		// Authenticate as Google account. Note that it could be both a Google
		// account and agent.
		user := user.Current(ctx)

		if user == nil {
			return nil, fmt.Errorf("User not signed in")
		}

		if !strings.HasSuffix(user.Email, "@google.com") {
			return nil, fmt.Errorf("Only @google.com users are authorized")
		}

		return ctx, nil
	}
}

func getAuthenticationStatus(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)
	ctx, err := getAuthenticatedContext(ctx, r)

	var response map[string]interface{}

	if err == nil {
		response = map[string]interface{}{
			"Status": "OK",
		}
	} else {
		loginURL, _ := user.LoginURL(ctx, "/")
		response = map[string]interface{}{
			"LoginURL": loginURL,
		}
	}

	outputData, _ := json.Marshal(response)
	w.Write(outputData)
}
<commit_msg>/api/get-authentication-status succeeds despite errors (#5)<commit_after>// Copyright (c) 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package main

import (
	"encoding/json"
	"fmt"
	"net/http"

	"golang.org/x/net/context"

	"cocoon/commands"
	"cocoon/db"

	"io/ioutil"

	"strings"

	"google.golang.org/appengine"

	"google.golang.org/appengine/log"
	"google.golang.org/appengine/user"
)

func init() {
	registerRPC("/api/create-agent", commands.CreateAgent)
	registerRPC("/api/authorize-agent", commands.AuthorizeAgent)
	registerRPC("/api/get-status", commands.GetStatus)
	registerRPC("/api/refresh-github-commits", commands.RefreshGithubCommits)
	registerRPC("/api/refresh-travis-status", commands.RefreshTravisStatus)
	registerRPC("/api/reserve-task", commands.ReserveTask)
	registerRPC("/api/update-task-status", commands.UpdateTaskStatus)

	// IMPORTANT: This is the ONLY path that does NOT require authentication. Do
	//            not copy this pattern. Use registerRPC instead.
	http.HandleFunc("/api/get-authentication-status", getAuthenticationStatus)
}

func registerRPC(path string, handler func(cocoon *db.Cocoon, inputJSON []byte) (interface{}, error)) {
	http.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
		ctx := appengine.NewContext(r)
		cocoon := db.NewCocoon(ctx)
		ctx, err := getAuthenticatedContext(ctx, r)

		if err != nil {
			serveUnauthorizedAccess(w, err)
			return
		}

		bytes, err := ioutil.ReadAll(r.Body)
		if err != nil {
			serveError(ctx, w, r, err)
			return
		}

		response, err := handler(cocoon, bytes)
		if err != nil {
			serveError(ctx, w, r, err)
			return
		}

		outputData, err := json.Marshal(response)
		if err != nil {
			serveError(ctx, w, r, err)
			return
		}

		w.Write(outputData)
	})
}

func authenticateAgent(ctx context.Context, agentID string, agentAuthToken string) (*db.Agent, error) {
	cocoon := db.NewCocoon(ctx)
	agent, err := cocoon.GetAgentByAuthToken(agentID, agentAuthToken)

	if err != nil {
		return nil, err
	}

	return agent, nil
}

func serveError(ctx context.Context, w http.ResponseWriter, r *http.Request, err error) {
	errorMessage := fmt.Sprintf("[%v] %v", r.URL, err)
	log.Errorf(ctx, "%v", errorMessage)
	http.Error(w, errorMessage, http.StatusInternalServerError)
}

func serveUnauthorizedAccess(w http.ResponseWriter, err error) {
	http.Error(w, fmt.Sprintf("Authentication/authorization error: %v", err), http.StatusUnauthorized)
}

func getAuthenticatedContext(ctx context.Context, r *http.Request) (context.Context, error) {
	agentAuthToken := r.Header.Get("Agent-Auth-Token")
	isCron := r.Header.Get("X-Appengine-Cron") == "true"
	if agentAuthToken != "" {
		// Authenticate as an agent. Note that it could simulaneously be cron and
		// agent, or Google account and agent.
		agentID := r.Header.Get("Agent-ID")
		agent, err := authenticateAgent(ctx, agentID, agentAuthToken)

		if err != nil {
			return nil, fmt.Errorf("Invalid agent: %v", agentID)
		}

		return context.WithValue(ctx, "agent", agent), nil
	} else if isCron {
		// Authenticate cron requests that are not agents.
		return ctx, nil
	} else {
		// Authenticate as Google account. Note that it could be both a Google
		// account and agent.
		user := user.Current(ctx)

		if user == nil {
			return nil, fmt.Errorf("User not signed in")
		}

		if !strings.HasSuffix(user.Email, "@google.com") {
			return nil, fmt.Errorf("Only @google.com users are authorized")
		}

		return ctx, nil
	}
}

func getAuthenticationStatus(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)

	// Ignore returned context. This request must succeed in the presence of
	// errors.
	_, err := getAuthenticatedContext(ctx, r)

	var response map[string]interface{}

	if err == nil {
		response = map[string]interface{}{
			"Status": "OK",
		}
	} else {
		loginURL, _ := user.LoginURL(ctx, "/")
		response = map[string]interface{}{
			"LoginURL": loginURL,
		}
	}

	outputData, _ := json.Marshal(response)
	w.Write(outputData)
}
<|endoftext|>


<commit_before>package passhash

import (
	"net"
	"time"
)

// AuditType represents the type of Audit that's logged
type AuditType uint

// UserID is the ID of the user being authenticated
type UserID uint64

const (
	// AuthnSucceeded means authorization succeeded
	AuthnSucceeded AuditType = iota + 1
	// AuthnFailed means authorization failed
	AuthnFailed
	// UpgradedKdf means the key derivation function was updated
	UpgradedKdf
)

// Log is an audit log entry
type Log struct {
	UserID UserID
	Time   time.Time
	Type   AuditType
	IP     net.IP
}

// AuditLogger is an interface for storing Specs with an audit trail
type AuditLogger interface {
	// Log logs an authorization action/result
	Log(UserID, AuditType, net.IP)
	// LastN gets the last N logs for a user
	LastN(userID UserID, n int) []Log
	// LastNWithTypes gets the last N logs for a user with the specified types
	LastNWithTypes(userID UserID, n int, auditTypes ...AuditType) []Log
}

// DummyAuditLogger is a dummy AuditLogger. e.g. it doesn't track any audit logs
type DummyAuditLogger struct{}

// Log doesn't actually do anything
func (al *DummyAuditLogger) Log(UserID, AuditType, net.IP) {}

// LastN doesn't actually do anything
func (al *DummyAuditLogger) LastN(userID UserID, n int) []Log {
	return []Log{}
}

// LastNWithTypes doesn't actually do anything
func (al *DummyAuditLogger) LastNWithTypes(userID UserID, n int, auditTypes ...AuditType) []Log {
	return []Log{}
}

// MemoryAuditLogger is an AuditLogger that stores all of it's logs in memory
// It is not recommended that you use this AuditLogger in production since the logs are not persisted and concurrent access is not supported
type MemoryAuditLogger struct {
	allLogs map[UserID][]Log
}

// Log will log the AuditLog in memory
func (al *MemoryAuditLogger) Log(userID UserID, at AuditType, ip net.IP) {
	if al.allLogs == nil {
		al.allLogs = make(map[UserID][]Log)
	}
	entry := al.allLogs[userID]
	if entry == nil {
		entry = []Log{}
	}
	al.allLogs[userID] = append(entry, Log{UserID: userID, Time: time.Now(), Type: at, IP: ip})
}

// LastN gets the last N logs for a user
func (al *MemoryAuditLogger) LastN(userID UserID, n int) []Log {
	origLogs := al.allLogs[userID]
	logs := make([]Log, 0, n)
	if n >= len(origLogs) {
		return origLogs
	}
	for _, log := range origLogs[len(origLogs)-n:] {
		logs = append(logs, log)
	}
	return logs
}

// LastNWithTypes gets the last N logs for a user with the specified types
func (al *MemoryAuditLogger) LastNWithTypes(userID UserID, n int, auditTypes ...AuditType) (logs []Log) {
	origLogs := al.allLogs[userID]
	logs = make([]Log, 0, n)
	for i := len(origLogs) - 1; i >= 0; i-- {
		log := al.allLogs[userID][i]
		for _, auditType := range auditTypes {
			if log.Type == auditType {
				logs = append(logs, log)
				continue
			}
		}
		if len(logs) >= n {
			return
		}
	}
	return
}
<commit_msg>Use variadic version of append for efficiency<commit_after>package passhash

import (
	"net"
	"time"
)

// AuditType represents the type of Audit that's logged
type AuditType uint

// UserID is the ID of the user being authenticated
type UserID uint64

const (
	// AuthnSucceeded means authorization succeeded
	AuthnSucceeded AuditType = iota + 1
	// AuthnFailed means authorization failed
	AuthnFailed
	// UpgradedKdf means the key derivation function was updated
	UpgradedKdf
)

// Log is an audit log entry
type Log struct {
	UserID UserID
	Time   time.Time
	Type   AuditType
	IP     net.IP
}

// AuditLogger is an interface for storing Specs with an audit trail
type AuditLogger interface {
	// Log logs an authorization action/result
	Log(UserID, AuditType, net.IP)
	// LastN gets the last N logs for a user
	LastN(userID UserID, n int) []Log
	// LastNWithTypes gets the last N logs for a user with the specified types
	LastNWithTypes(userID UserID, n int, auditTypes ...AuditType) []Log
}

// DummyAuditLogger is a dummy AuditLogger. e.g. it doesn't track any audit logs
type DummyAuditLogger struct{}

// Log doesn't actually do anything
func (al *DummyAuditLogger) Log(UserID, AuditType, net.IP) {}

// LastN doesn't actually do anything
func (al *DummyAuditLogger) LastN(userID UserID, n int) []Log {
	return []Log{}
}

// LastNWithTypes doesn't actually do anything
func (al *DummyAuditLogger) LastNWithTypes(userID UserID, n int, auditTypes ...AuditType) []Log {
	return []Log{}
}

// MemoryAuditLogger is an AuditLogger that stores all of it's logs in memory
// It is not recommended that you use this AuditLogger in production since the logs are not persisted and concurrent access is not supported
type MemoryAuditLogger struct {
	allLogs map[UserID][]Log
}

// Log will log the AuditLog in memory
func (al *MemoryAuditLogger) Log(userID UserID, at AuditType, ip net.IP) {
	if al.allLogs == nil {
		al.allLogs = make(map[UserID][]Log)
	}
	entry := al.allLogs[userID]
	if entry == nil {
		entry = []Log{}
	}
	al.allLogs[userID] = append(entry, Log{UserID: userID, Time: time.Now(), Type: at, IP: ip})
}

// LastN gets the last N logs for a user
func (al *MemoryAuditLogger) LastN(userID UserID, n int) []Log {
	origLogs := al.allLogs[userID]
	logs := make([]Log, 0, n)
	if n >= len(origLogs) {
		return origLogs
	}
	return append(logs, origLogs[len(origLogs)-n:]...)
}

// LastNWithTypes gets the last N logs for a user with the specified types
func (al *MemoryAuditLogger) LastNWithTypes(userID UserID, n int, auditTypes ...AuditType) (logs []Log) {
	origLogs := al.allLogs[userID]
	logs = make([]Log, 0, n)
	for i := len(origLogs) - 1; i >= 0; i-- {
		log := al.allLogs[userID][i]
		for _, auditType := range auditTypes {
			if log.Type == auditType {
				logs = append(logs, log)
				continue
			}
		}
		if len(logs) >= n {
			return
		}
	}
	return
}
<|endoftext|>


<commit_before>package auth

import (
	"encoding/base64"
	"fmt"
	"time"

	"github.com/synapse-garden/sg-proto/store"

	"github.com/boltdb/bolt"
	"github.com/pkg/errors"
	uuid "github.com/satori/go.uuid"
)

type TokenType int

const (
	BearerType TokenType = iota
	RefreshType

	Expiration = 5 * time.Minute
)

func (t TokenType) String() string {
	return tokenNames[t]
}

func ValidType(t string) bool {
	return tokenTypes[t]
}

var (
	SessionBucket = store.Bucket("sessions")
	RefreshBucket = store.Bucket("refresh")

	tokenNames = map[TokenType]string{
		BearerType: "Bearer",
	}

	tokenTypes = map[string]bool{
		"Bearer": true,
	}
)

type Token []byte

type ErrMissingSession []byte

func (e ErrMissingSession) Error() string {
	return fmt.Sprintf("no such session %#q", string(e))
}

type ErrTokenExpired []byte

func (e ErrTokenExpired) Error() string {
	return fmt.Sprintf("session %#q expired", string(e))
}

// Session is a client login session.
type Session struct {
	Token        Token         `json:"token"`
	ExpiresIn    time.Duration `json:"expires_in"`
	Expiration   time.Time     `json:"expires_at,omitempty"`
	TokenType    TokenType     `json:"token_type"`
	RefreshToken Token         `json:"refresh_token"`
}

func DecodeToken(tStr string) (Token, error) {
	tBytes, err := base64.StdEncoding.DecodeString(tStr)
	if err != nil {
		return nil, errors.Wrapf(err, "invalid token %#q", tStr)
	}
	return Token(tBytes), nil
}

func NewToken(t TokenType) Token {
	switch t {
	case BearerType:
		return uuid.NewV4().Bytes()
	case RefreshType:
		return uuid.NewV4().Bytes()
	}

	return nil
}

func RefreshIfValid(
	s *Session,
	expires time.Time,
	validFor time.Duration,
) func(*bolt.Tx) error {
	return func(tx *bolt.Tx) error {
		err := store.CheckExists(
			RefreshBucket,
			s.RefreshToken,
		)(tx)
		switch {
		case store.IsMissing(err):
			return ErrMissingSession(s.RefreshToken)
		case err != nil:
			return err
		}
		return Refresh(s, expires, validFor)(tx)
	}
}

func Refresh(
	s *Session,
	expires time.Time,
	validFor time.Duration,
) func(*bolt.Tx) error {
	return func(tx *bolt.Tx) (err error) {
		var (
			oldExpiresIn  = s.ExpiresIn
			oldExpiration = s.Expiration
		)
		s.Expiration = expires
		s.ExpiresIn = validFor

		defer func() {
			if err != nil {
				s.ExpiresIn = oldExpiresIn
				s.Expiration = oldExpiration
			}
		}()

		return store.Marshal(SessionBucket, s, s.Token)(tx)
	}
}

func CheckRefresh(t Token) func(*bolt.Tx) error {
	return store.CheckExists(RefreshBucket, t)
}

// CheckToken attempts to load the given Token's Session from the
// Sessions bucket.  If it was missing, it returns ErrMissingSession.
// If it was expired, it returns ErrTokenExpired.  This functionality
// should be used to branch logic into a RefreshIfValid.  The REST API
// user should not be trusted with the knowledge that a given token ever
// existed.  From the REST API user's point of view, an expired session
// with an invalid refresh token simply does not exist.
func CheckToken(t Token) func(*bolt.Tx) error {
	return func(tx *bolt.Tx) error {
		var (
			now      = time.Now().UTC()
			existent = new(Session)
		)
		err := store.Unmarshal(SessionBucket, existent, t)(tx)

		switch {
		case store.IsMissing(err):
			// The session was not found.
			return ErrMissingSession(t)
		case err != nil:
			// There was an unknown error.
			return err
		case existent.Expiration.Before(now):
			// The token has already expired.
			return ErrTokenExpired(t)
		}

		return nil
	}
}

// NewSession prepares and assigns values to the given Session, and
// stores them in the database, or returns any error.
func NewSession(
	s *Session,
	expiration time.Time,
	validFor time.Duration,
	token, refresh Token,
	userID string,
) func(*bolt.Tx) error {
	return func(tx *bolt.Tx) (err error) {
		var (
			kind = BearerType

			oldToken, oldRefresh = s.Token, s.RefreshToken
			oldExpiration        = s.Expiration
			oldExpiresAt         = s.ExpiresIn
			oldType              = s.TokenType
		)

		s.Token, s.RefreshToken = token, refresh
		s.TokenType = kind
		s.Expiration, s.ExpiresIn = expiration, validFor

		defer func() {
			if err != nil {
				s.Token = oldToken
				s.RefreshToken = oldRefresh
				s.Expiration = oldExpiration
				s.ExpiresIn = oldExpiresAt
				s.TokenType = oldType
			}
		}()

		err = store.Marshal(SessionBucket, s, s.Token)(tx)
		if err != nil {
			return
		}

		err = SaveContext(&Context{
			Token:        s.Token,
			RefreshToken: s.RefreshToken,
			UserID:       userID,
		})(tx)
		if err != nil {
			return
		}

		return store.Put(RefreshBucket, s.RefreshToken, nil)(tx)
	}
}

func DeleteToken(t Token) func(*bolt.Tx) error {
	return func(tx *bolt.Tx) error {
		err := store.Delete(SessionBucket, t)(tx)
		if store.IsMissing(err) {
			return ErrMissingSession(t)
		}

		return err
	}
}

func DeleteSession(s *Session) func(*bolt.Tx) error {
	return func(tx *bolt.Tx) error {
		err := store.Delete(SessionBucket, s.Token)(tx)
		if err != nil && !store.IsMissing(err) {
			return err
		}

		err = store.Delete(RefreshBucket, s.RefreshToken)(tx)
		if err != nil && !store.IsMissing(err) {
			return err
		}
		return nil
	}
}
<commit_msg>Add auth.Token Admin type const, String method<commit_after>package auth

import (
	"encoding/base64"
	"fmt"
	"time"

	"github.com/synapse-garden/sg-proto/store"

	"github.com/boltdb/bolt"
	"github.com/pkg/errors"
	uuid "github.com/satori/go.uuid"
)

type TokenType int

const (
	BearerType TokenType = iota
	AdminType
	RefreshType

	Expiration = 5 * time.Minute
)

func (t TokenType) String() string {
	return tokenNames[t]
}

func ValidType(t string) bool {
	return tokenTypes[t]
}

var (
	SessionBucket = store.Bucket("sessions")
	RefreshBucket = store.Bucket("refresh")

	tokenNames = map[TokenType]string{
		BearerType:  "Bearer",
		RefreshType: "Refresh",
		AdminType:   "Admin",
	}

	tokenTypes = map[string]bool{
		"Bearer":  true,
		"Refresh": true,
		"Admin":   true,
	}
)

type Token []byte

func (t Token) String() string {
	return base64.StdEncoding.EncodeToString([]byte(t))
}

type ErrMissingSession []byte

func (e ErrMissingSession) Error() string {
	return fmt.Sprintf("no such session %#q", string(e))
}

type ErrTokenExpired []byte

func (e ErrTokenExpired) Error() string {
	return fmt.Sprintf("session %#q expired", string(e))
}

// Session is a client login session.
type Session struct {
	Token        Token         `json:"token"`
	ExpiresIn    time.Duration `json:"expires_in"`
	Expiration   time.Time     `json:"expires_at,omitempty"`
	TokenType    TokenType     `json:"token_type"`
	RefreshToken Token         `json:"refresh_token"`
}

func DecodeToken(tStr string) (Token, error) {
	tBytes, err := base64.StdEncoding.DecodeString(tStr)
	if err != nil {
		return nil, errors.Wrapf(err, "invalid token %#q", tStr)
	}
	return Token(tBytes), nil
}

func NewToken(t TokenType) Token {
	switch t {
	case BearerType:
		return uuid.NewV4().Bytes()
	case RefreshType:
		return uuid.NewV4().Bytes()
	}

	return nil
}

func RefreshIfValid(
	s *Session,
	expires time.Time,
	validFor time.Duration,
) func(*bolt.Tx) error {
	return func(tx *bolt.Tx) error {
		err := store.CheckExists(
			RefreshBucket,
			s.RefreshToken,
		)(tx)
		switch {
		case store.IsMissing(err):
			return ErrMissingSession(s.RefreshToken)
		case err != nil:
			return err
		}
		return Refresh(s, expires, validFor)(tx)
	}
}

func Refresh(
	s *Session,
	expires time.Time,
	validFor time.Duration,
) func(*bolt.Tx) error {
	return func(tx *bolt.Tx) (err error) {
		var (
			oldExpiresIn  = s.ExpiresIn
			oldExpiration = s.Expiration
		)
		s.Expiration = expires
		s.ExpiresIn = validFor

		defer func() {
			if err != nil {
				s.ExpiresIn = oldExpiresIn
				s.Expiration = oldExpiration
			}
		}()

		return store.Marshal(SessionBucket, s, s.Token)(tx)
	}
}

func CheckRefresh(t Token) func(*bolt.Tx) error {
	return store.CheckExists(RefreshBucket, t)
}

// CheckToken attempts to load the given Token's Session from the
// Sessions bucket.  If it was missing, it returns ErrMissingSession.
// If it was expired, it returns ErrTokenExpired.  This functionality
// should be used to branch logic into a RefreshIfValid.  The REST API
// user should not be trusted with the knowledge that a given token ever
// existed.  From the REST API user's point of view, an expired session
// with an invalid refresh token simply does not exist.
func CheckToken(t Token) func(*bolt.Tx) error {
	return func(tx *bolt.Tx) error {
		var (
			now      = time.Now().UTC()
			existent = new(Session)
		)
		err := store.Unmarshal(SessionBucket, existent, t)(tx)

		switch {
		case store.IsMissing(err):
			// The session was not found.
			return ErrMissingSession(t)
		case err != nil:
			// There was an unknown error.
			return err
		case existent.Expiration.Before(now):
			// The token has already expired.
			return ErrTokenExpired(t)
		}

		return nil
	}
}

// NewSession prepares and assigns values to the given Session, and
// stores them in the database, or returns any error.
func NewSession(
	s *Session,
	expiration time.Time,
	validFor time.Duration,
	token, refresh Token,
	userID string,
) func(*bolt.Tx) error {
	return func(tx *bolt.Tx) (err error) {
		var (
			kind = BearerType

			oldToken, oldRefresh = s.Token, s.RefreshToken
			oldExpiration        = s.Expiration
			oldExpiresAt         = s.ExpiresIn
			oldType              = s.TokenType
		)

		s.Token, s.RefreshToken = token, refresh
		s.TokenType = kind
		s.Expiration, s.ExpiresIn = expiration, validFor

		defer func() {
			if err != nil {
				s.Token = oldToken
				s.RefreshToken = oldRefresh
				s.Expiration = oldExpiration
				s.ExpiresIn = oldExpiresAt
				s.TokenType = oldType
			}
		}()

		err = store.Marshal(SessionBucket, s, s.Token)(tx)
		if err != nil {
			return
		}

		err = SaveContext(&Context{
			Token:        s.Token,
			RefreshToken: s.RefreshToken,
			UserID:       userID,
		})(tx)
		if err != nil {
			return
		}

		return store.Put(RefreshBucket, s.RefreshToken, nil)(tx)
	}
}

func DeleteToken(t Token) func(*bolt.Tx) error {
	return func(tx *bolt.Tx) error {
		err := store.Delete(SessionBucket, t)(tx)
		if store.IsMissing(err) {
			return ErrMissingSession(t)
		}

		return err
	}
}

func DeleteSession(s *Session) func(*bolt.Tx) error {
	return func(tx *bolt.Tx) error {
		err := store.Delete(SessionBucket, s.Token)(tx)
		if err != nil && !store.IsMissing(err) {
			return err
		}

		err = store.Delete(RefreshBucket, s.RefreshToken)(tx)
		if err != nil && !store.IsMissing(err) {
			return err
		}
		return nil
	}
}
<|endoftext|>


<commit_before>package swift

import (
	"flag"
	"path/filepath"
	"io/ioutil"
	"log"
	"github.com/pengzj/swift/internal"
	"github.com/pengzj/swift/server"
	"encoding/json"
	"github.com/pengzj/swift/logger"
)

func (app *Application) init()  {
	app.loadDefaultConfig()
}

func (app *Application) loadDefaultConfig()  {
	var serverId string
	flag.StringVar(&serverId, "serverId", "", "input correct serverId")
	flag.Parse()

	var serverType = "";

	if serverId  == "" {
		serverType = SERVER_MASTER
	}

	var internalServers []internal.Server

	var filePath string
	var  s *server.Server

	filePath = filepath.Join(app.configPath, "master.json")
	in, err := ioutil.ReadFile(filePath)
	if err != nil {
		log.Fatal(err)
	}
	err = json.Unmarshal(in, &s)
	if err != nil {
		log.Fatal(err)
	}
	s.Type = SERVER_MASTER
	s.IsMaster = true
	s.Frontend = false

	internalServers = append(internalServers, internal.Server{
		Type: s.Type,
		Id: s.Id,
		Host:s.Host,
		Port:s.Port,
	})

	isExist := false

	if serverType == SERVER_MASTER {
		app.server = s

		isExist = true
	}

	filePath = filepath.Join(app.configPath, "servers.json")
	var servers []*server.Server
	in, err = ioutil.ReadFile(filePath)
	if err != nil {
		log.Fatal(err)
	}
	err = json.Unmarshal(in, &servers)
	if err != nil {
		log.Fatal(err)
	}

	for _, s :=range servers {
		if s.Id == serverId {
			app.server = s
			isExist = true
		}

		internalServers = append(internalServers, internal.Server{
			Type: s.Type,
			Id: s.Id,
			Host:s.Host,
			Port:s.Port,
		})

	}

	if isExist == false {
		log.Fatal(serverId + " not  exists")
	}

	secretKeyPath := filepath.Join(app.configPath, "secret.key")
	in, err = ioutil.ReadFile(secretKeyPath)
	if err != nil {
		log.Fatal(err)
	}

	internal.SetSecretKey(string(in))

	internal.PutServers(internalServers)

	if len(app.logPath) == 0 {
		panic("logPath is not aviable")
	}

	logger.SetFile(filepath.Join(app.logPath, app.server.Id + ".log"))
}<commit_msg>fix the reading file content without removing tail<commit_after>package swift

import (
	"flag"
	"path/filepath"
	"io/ioutil"
	"log"
	"github.com/pengzj/swift/internal"
	"github.com/pengzj/swift/server"
	"encoding/json"
	"github.com/pengzj/swift/logger"
)

func (app *Application) init()  {
	app.loadDefaultConfig()
}

func (app *Application) loadDefaultConfig()  {
	var serverId string
	flag.StringVar(&serverId, "serverId", "", "input correct serverId")
	flag.Parse()

	var serverType = "";

	if serverId  == "" {
		serverType = SERVER_MASTER
	}

	var internalServers []internal.Server

	var filePath string
	var  s *server.Server

	filePath = filepath.Join(app.configPath, "master.json")
	in, err := ioutil.ReadFile(filePath)
	if err != nil {
		log.Fatal(err)
	}
	err = json.Unmarshal(in, &s)
	if err != nil {
		log.Fatal(err)
	}
	s.Type = SERVER_MASTER
	s.IsMaster = true
	s.Frontend = false

	internalServers = append(internalServers, internal.Server{
		Type: s.Type,
		Id: s.Id,
		Host:s.Host,
		Port:s.Port,
	})

	isExist := false

	if serverType == SERVER_MASTER {
		app.server = s

		isExist = true
	}

	filePath = filepath.Join(app.configPath, "servers.json")
	var servers []*server.Server
	in, err = ioutil.ReadFile(filePath)
	if err != nil {
		log.Fatal(err)
	}
	err = json.Unmarshal(in, &servers)
	if err != nil {
		log.Fatal(err)
	}

	for _, s :=range servers {
		if s.Id == serverId {
			app.server = s
			isExist = true
		}

		internalServers = append(internalServers, internal.Server{
			Type: s.Type,
			Id: s.Id,
			Host:s.Host,
			Port:s.Port,
		})

	}

	if isExist == false {
		log.Fatal(serverId + " not  exists")
	}

	secretKeyPath := filepath.Join(app.configPath, "secret.key")
	in, err = ioutil.ReadFile(secretKeyPath)
	if err != nil {
		log.Fatal(err)
	}

	internal.SetSecretKey(string(in[:len(in)-1]))

	internal.PutServers(internalServers)

	if len(app.logPath) == 0 {
		panic("logPath is not aviable")
	}

	logger.SetFile(filepath.Join(app.logPath, app.server.Id + ".log"))
}<|endoftext|>


<commit_before>package cheshire

import (
	"bufio"
	"code.google.com/p/go.net/websocket"
	// "github.com/trendrr/cheshire-golang/dynmap"
	"encoding/json"
	"io"
	"log"
	"net/http"
	"sync"
)

type WebsocketConnection struct {
	conn       *websocket.Conn
	writerLock sync.Mutex
}

func (conn WebsocketConnection) Write(response *Response) (int, error) {
	json, err := json.Marshal(response)
	if err != nil {
		//TODO: uhh, do something..
		log.Print(err)
	}
	// log.Println("writing ", string(json))
	defer conn.writerLock.Unlock()
	conn.writerLock.Lock()
	bytes, err := conn.conn.Write(json)
	return bytes, err
}

type WebsocketController struct {
	Conf         *ControllerConfig
	Handler      websocket.Handler
	serverConfig *ServerConfig
}

func (wc WebsocketController) Config() *ControllerConfig {
	return wc.Conf
}
func (wc WebsocketController) HandleRequest(*Request, Writer) {
	//do nothing, this should never be called.
}

func NewWebsocketController(route string, config *ServerConfig) WebsocketController {
	var wc = new(WebsocketController)
	wc.Conf = NewControllerConfig(route)
	wc.serverConfig = config
	wc.Handler = websocket.Handler(func(ws *websocket.Conn) { wc.HandleWCConnection(ws) }) //use anon function because a method is impossible
	return *wc
}

// implements the HttpHijacker interface so we can handle the request directly.
func (this WebsocketController) HttpHijack(writer http.ResponseWriter, req *http.Request) {
	this.Handler.ServeHTTP(writer, req)
}

func (wc WebsocketController) HandleWCConnection(ws *websocket.Conn) {
	// Uhh, guessing we are already in a go routine..
	log.Print("CONNECT!")

	defer ws.Close()
	// log.Print("CONNECT!")
	// conn.writer = bufio.NewWriter(conn.conn)

	dec := json.NewDecoder(bufio.NewReader(ws))
	conn := WebsocketConnection{conn: ws}
	for {
		var req Request
		err := dec.Decode(&req)

		if err == io.EOF {
			log.Print(err)
			break
		} else if err != nil {
			log.Print(err)
			break
		}
		log.Print(req)
		log.Print(req.Uri())
		controller := wc.serverConfig.Router.Match(req.Method(), req.Uri())

		log.Print("GOT CONTROLLER ")
		log.Print(controller)

		go controller.HandleRequest(&req, conn)
	}
	log.Print("DISCONNECT!")
}
<commit_msg>clean up websocket writer and controller<commit_after>package cheshire

import (
	"bufio"
	"code.google.com/p/go.net/websocket"
	"encoding/json"
	"io"
	"log"
	"net/http"
	"sync"
)

type WebsocketWriter struct {
	conn       *websocket.Conn
	writerLock sync.Mutex
}

func (this *WebsocketWriter) Write(response *Response) (int, error) {
	json, err := json.Marshal(response)
	if err != nil {
		//TODO: uhh, do something..
		log.Print(err)
	}
	// log.Println("writing ", string(json))
	defer this.writerLock.Unlock()
	this.writerLock.Lock()
	bytes, err := this.conn.Write(json)
	return bytes, err
}

type WebsocketController struct {
	Conf         *ControllerConfig
	Handler      websocket.Handler
	serverConfig *ServerConfig
}

func (wc WebsocketController) Config() *ControllerConfig {
	return wc.Conf
}
func (wc WebsocketController) HandleRequest(*Request, Writer) {
	//do nothing, this should never be called.
}

func NewWebsocketController(route string, config *ServerConfig) WebsocketController {
	var wc = new(WebsocketController)
	wc.Conf = NewControllerConfig(route)
	wc.serverConfig = config
	wc.Handler = websocket.Handler(func(ws *websocket.Conn) { wc.HandleWCConnection(ws) }) //use anon function because a method is impossible
	return *wc
}

// implements the HttpHijacker interface so we can handle the request directly.
func (this *WebsocketController) HttpHijack(writer http.ResponseWriter, req *http.Request) {
	this.Handler.ServeHTTP(writer, req)
}

func (this *WebsocketController) HandleWCConnection(ws *websocket.Conn) {
	// Uhh, guessing we are already in a go routine..
	log.Print("CONNECT!")

	defer ws.Close()
	// log.Print("CONNECT!")
	// conn.writer = bufio.NewWriter(conn.conn)

	dec := json.NewDecoder(bufio.NewReader(ws))
	writer := &WebsocketWriter{conn: ws}
	for {
		var req Request
		err := dec.Decode(&req)

		if err == io.EOF {
			log.Print(err)
			break
		} else if err != nil {
			log.Print(err)
			break
		}
		log.Print(req)
		log.Print(req.Uri())
		controller := this.serverConfig.Router.Match(req.Method(), req.Uri())

		log.Print("GOT CONTROLLER ")
		log.Print(controller)

		go controller.HandleRequest(&req, writer)
	}
	log.Print("DISCONNECT!")
}
<|endoftext|>


<commit_before>// Copyright 2016 laosj Author @songtianyi. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	//"github.com/songtianyi/laosj/downloader"
	"github.com/songtianyi/laosj/spider"
	//"github.com/songtianyi/rrframework/config"
	//"github.com/songtianyi/rrframework/connector/redis"
)

func main() {
	s, err := spider.CreateSpiderFromUrl("http://www.douban.com/group/haixiuzu/discussion")
	if err != nil {
		fmt.Println(err)
	}
	rs, _ := s.GetAttr("div.grid-16-8.clearfix>div.article>div>table.olt>tbody>tr>td.title>a", "href")
	fmt.Println(rs)
}
<commit_msg>haixiuzu test pass<commit_after>// Copyright 2016 laosj Author @songtianyi. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"github.com/songtianyi/laosj/downloader"
	"github.com/songtianyi/laosj/spider"
	//"github.com/songtianyi/rrframework/config"
	"github.com/songtianyi/rrframework/storage"
	"github.com/songtianyi/rrframework/logs"
	"github.com/songtianyi/rrframework/connector/redis"
)

func main() {
	url := "http://www.douban.com/group/haixiuzu/discussion"
	d := &downloader.Downloader{
		ConcurrencyLimit: 10,
		UrlChannelFactor: 10,
		RedisConnStr: "10.19.147.75:6379",
		SourceQueue: "DATA:IMAGE:HAIXIUZU",
		Store: rrstorage.CreateLocalDiskStorage("/data/sexx/haixiuzu/"),
	}
	err, rc := rrredis.GetRedisClient("10.19.147.75:6379")
	if err != nil {
		panic(err)
	}
	go func() {
		d.Start()
	}()

	for {
		s, err := spider.CreateSpiderFromUrl(url)
		if err != nil {
			logs.Debug(err)
			continue
		}
		rs, _ := s.GetAttr("div.grid-16-8.clearfix>div.article>div>table.olt>tbody>tr>td.title>a", "href")
		for _, v := range rs {
			s01, err := spider.CreateSpiderFromUrl(v)
			if err != nil {
				logs.Error(err)
				continue
			}
			rs01, _ := s01.GetAttr("div.grid-16-8.clearfix>div.article>div.topic-content.clearfix>div.topic-doc>div#link-report>div.topic-content>div.topic-figure.cc>img", "src")
			for _, vv := range rs01 {
				if _, err := rc.RPush("DATA:IMAGE:HAIXIUZU", vv); err != nil {
					logs.Error(err)
				}
			}
		}
		rs1, _ := s.GetAttr("div.grid-16-8.clearfix>div.article>div.paginator>span.next>a", "href")
		if len(rs1) != 1 {
			break
		}
		url = rs1[0]
	}
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"net/http"
)

// PageData is used by the index template to populate things and stuff
type PageData struct {
	MainTable  []TableRow `json:"mainTable"`
	TimeSeries graphData  `json:"timeSeries"`
	FreqDist   graphData  `json:"freqDist"`
}

// TableRow contains data used in the top table
type TableRow struct {
	Label string `json:"label"`
	Num   []int  `json:"num"`
}

func handlerQuery(w http.ResponseWriter, r *http.Request) {
	p := params(r)

	JSON{
		Status: http.StatusOK,
		Data: PageData{
			MainTable:  createMainTableData(p),
			TimeSeries: graphTimeSeries(db.getResults(p)),
			FreqDist:   graphFreqDist(db.getResults(p)),
		},
	}.write(w)
}

func handlerListSets(w http.ResponseWriter, r *http.Request) {
	res, err := db.getSetList(params(r))
	if err != nil {
		JSON{Status: http.StatusInternalServerError, Data: err}.write(w)
	} else {
		JSON{Status: http.StatusOK, Data: res}.write(w)
	}
}

func handlerListMachines(w http.ResponseWriter, r *http.Request) {
	res, err := db.getMachineList(params(r))
	if err != nil {
		JSON{Status: http.StatusInternalServerError, Data: err}.write(w)
	} else {
		JSON{Status: http.StatusOK, Data: res}.write(w)
	}
}

func handlerDataRange(w http.ResponseWriter, r *http.Request) {
	f, l, err := db.getDataRange()
	if err != nil {
		log.Println("handlerDataRange:", err.Error())
	}

	JSON{Status: http.StatusOK, Data: map[string]int64{"first": f.Unix(), "last": l.Unix()}}.write(w)
}
<commit_msg>since things are broken I'll leave this here and fix it later<commit_after>package main

import (
	"log"
	"net/http"

	jweb "github.com/nboughton/go-utils/json/web"
)

// PageData is used by the index template to populate things and stuff
type PageData struct {
	MainTable  []TableRow `json:"mainTable"`
	TimeSeries graphData  `json:"timeSeries"`
	FreqDist   graphData  `json:"freqDist"`
}

// TableRow contains data used in the top table
type TableRow struct {
	Label string `json:"label"`
	Num   []int  `json:"num"`
}

func handlerQuery(w http.ResponseWriter, r *http.Request) {
	p := params(r)

	jweb.New(http.StatusOK,
		PageData{
			MainTable:  createMainTableData(p),
			TimeSeries: graphTimeSeries(db.getResults(p)),
			FreqDist:   graphFreqDist(db.getResults(p)),
		},
	).Write(w)
}

func handlerListSets(w http.ResponseWriter, r *http.Request) {
	res, err := db.getSetList(params(r))
	if err != nil {
		jweb.New(http.StatusInternalServerError, err).Write(w)
	} else {
		jweb.New(http.StatusOK, res).Write(w)
	}
}

func handlerListMachines(w http.ResponseWriter, r *http.Request) {
	res, err := db.getMachineList(params(r))
	if err != nil {
		jweb.New(http.StatusInternalServerError, err).Write(w)
	} else {
		jweb.New(http.StatusOK, res).Write(w)
	}
}

func handlerDataRange(w http.ResponseWriter, r *http.Request) {
	f, l, err := db.getDataRange()
	if err != nil {
		log.Println("handlerDataRange:", err.Error())
	}

	jweb.New(http.StatusOK, map[string]int64{"first": f.Unix(), "last": l.Unix()}).Write(w)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"io/ioutil"
	"net/http"
	"strconv"

	"github.com/bitly/go-simplejson"
	"github.com/gin-gonic/gin"
)

// Charge handler will do the charging by adding Server Key into header
func Charge(c *gin.Context) {
	// Encode server key using base 64 string
	authorization := base64.StdEncoding.EncodeToString([]byte(VTServerKey + ":"))

	// HTTP client
	client := http.DefaultClient
	var URL = SnapURL
	if EnableProduction {
		URL = SnapURLProduction
	}
	request, err := http.NewRequest("POST", URL+"/transactions", c.Request.Body)

	if err != nil {
		c.JSON(http.StatusBadRequest, gin.H{"status_code": "400", "status_message": "Bad Request"})
	} else {
		request.Header.Add("Content-Type", "application/json")
		request.Header.Add("Accept", "application/json")
		request.Header.Add("Authorization", "Basic "+authorization)
		response, _ := client.Do(request)

		responseBody, _ := ioutil.ReadAll(response.Body)
		var respObj interface{}
		json.Unmarshal(responseBody, &respObj)
		c.JSON(http.StatusOK, respObj)
	}
}

// ChargeWithInstallment will do the charging with added installment
func ChargeWithInstallment(c *gin.Context) {
	// Encode server key using base 64 string
	authorization := base64.StdEncoding.EncodeToString([]byte(VTServerKey + ":"))

	// HTTP client
	client := http.DefaultClient
	var URL = SnapURL
	if EnableProduction {
		URL = SnapURLProduction
	}

	installment := getInstallmentData()
	whitelist := getWhitelistBin()
	requestBody, _ := ioutil.ReadAll(c.Request.Body)
	requestJSON, _ := simplejson.NewJson(requestBody)
	creditCard := requestJSON.Get("credit_card")
	if creditCard == nil {
		requestJSON.Set("credit_card", CreditCard{Installment: installment, WhitelistBins: whitelist})
	} else {
		creditCard.Set("installment", installment)
		creditCard.Set("whitelist_bins", whitelist)
		requestJSON.Set("credit_card", creditCard)
	}
	requestJSONMarshaled, _ := requestJSON.MarshalJSON()
	requestObj := bytes.NewReader(requestJSONMarshaled)
	request, err := http.NewRequest("POST", URL+"/transactions", requestObj)

	if err != nil {
		c.JSON(http.StatusBadRequest, gin.H{"status_code": "400", "status_message": "Bad Request"})
	} else {
		request.Header.Add("Content-Type", "application/json")
		request.Header.Add("Accept", "application/json")
		request.Header.Add("Authorization", "Basic "+authorization)
		response, _ := client.Do(request)

		responseBody, _ := ioutil.ReadAll(response.Body)
		var respObj interface{}
		json.Unmarshal(responseBody, &respObj)

		c.JSON(http.StatusOK, respObj)
	}

}

// GetCardsEndpoint handles get cards action
func GetCardsEndpoint(c *gin.Context) {
	id := c.Param("id")
	limitQuery := c.Query("limit")
	offsetQuery := c.Query("offset")

	var limit int64 = 10
	var offset int64

	if limitQuery != "" {
		limit1, _ := strconv.ParseInt(limitQuery, 10, 32)
		limit = limit1
	}
	if offsetQuery != "" {
		offset1, _ := strconv.ParseInt(offsetQuery, 0, 32)
		offset = offset1
	}

	cards := GetCards(id, int(limit), int(offset))

	if len(cards) > 0 {
		c.JSON(http.StatusOK, cards)
	} else {
		c.String(http.StatusOK, "There's no saved card for that ID")
	}
}

// SaveCardsEndpoint handles save card
func SaveCardsEndpoint(c *gin.Context) {
	id := c.Param("id")
	var arrayCard []Card
	requestBody, err := ioutil.ReadAll(c.Request.Body)
	if err != nil {
		c.String(http.StatusBadRequest, "Wrong data format")
	} else {
		err1 := json.Unmarshal(requestBody, &arrayCard)
		if err1 != nil {
			c.String(http.StatusBadRequest, "Wrong data format")
		} else {
			err2 := SaveCards(id, arrayCard)
			if err2 != nil {
				c.JSON(http.StatusInternalServerError, gin.H{"message": err2.Error(), "status": "failed", "code": http.StatusInternalServerError})
			} else {
				c.JSON(http.StatusOK, gin.H{"message": "Card is saved", "status": "success", "code": http.StatusCreated})
			}
		}
	}
}

func getInstallmentData() Installment {
	return Installment{Required: false, Terms: Terms{BNI: []int{3, 6, 12}, Mandiri: []int{3, 6, 12}, BCA: []int{3, 6, 12}, CIMB: []int{3, 6, 12}, Offline: []int{3, 6, 12}}}
}

func getWhitelistBin() []string {
	return []string{"481111", "521111"}
}
<commit_msg>Rollback<commit_after>package main

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"io/ioutil"
	"net/http"
	"strconv"

	"github.com/bitly/go-simplejson"
	"github.com/gin-gonic/gin"
)

// Charge handler will do the charging by adding Server Key into header
func Charge(c *gin.Context) {
	// Encode server key using base 64 string
	authorization := base64.StdEncoding.EncodeToString([]byte(VTServerKey + ":"))

	// HTTP client
	client := http.DefaultClient
	var URL = SnapURL
	if EnableProduction {
		URL = SnapURLProduction
	}
	request, err := http.NewRequest("POST", URL+"/transactions", c.Request.Body)

	if err != nil {
		c.JSON(http.StatusBadRequest, gin.H{"status_code": "400", "status_message": "Bad Request"})
	} else {
		request.Header.Add("Content-Type", "application/json")
		request.Header.Add("Accept", "application/json")
		request.Header.Add("Authorization", "Basic "+authorization)
		response, _ := client.Do(request)

		responseBody, _ := ioutil.ReadAll(response.Body)
		var respObj interface{}
		json.Unmarshal(responseBody, &respObj)
		c.JSON(http.StatusOK, respObj)
	}
}

// ChargeWithInstallment will do the charging with added installment
func ChargeWithInstallment(c *gin.Context) {
	// Encode server key using base 64 string
	authorization := base64.StdEncoding.EncodeToString([]byte(VTServerKey + ":"))

	// HTTP client
	client := http.DefaultClient
	var URL = SnapURL
	if EnableProduction {
		URL = SnapURLProduction
	}

	installment := getInstallmentData()
	whitelist := getWhitelistBin()
	requestBody, _ := ioutil.ReadAll(c.Request.Body)
	requestJSON, _ := simplejson.NewJson(requestBody)
	creditCard := requestJSON.Get("credit_card")
	if creditCard == nil {
		requestJSON.Set("credit_card", CreditCard{Installment: installment, WhitelistBins: whitelist})
	} else {
		creditCard.Set("installment", installment)
		creditCard.Set("whitelist_bins", whitelist)
		requestJSON.Set("credit_card", creditCard)
	}
	requestJSONMarshaled, _ := requestJSON.MarshalJSON()
	requestObj := bytes.NewReader(requestJSONMarshaled)
	request, err := http.NewRequest("POST", URL+"/transactions", requestObj)

	if err != nil {
		c.JSON(http.StatusBadRequest, gin.H{"status_code": "400", "status_message": "Bad Request"})
	} else {
		request.Header.Add("Content-Type", "application/json")
		request.Header.Add("Accept", "application/json")
		request.Header.Add("Authorization", "Basic "+authorization)
		response, _ := client.Do(request)

		responseBody, _ := ioutil.ReadAll(response.Body)
		var respObj interface{}
		json.Unmarshal(responseBody, &respObj)

		c.JSON(http.StatusOK, respObj)
	}

}

// GetCardsEndpoint handles get cards action
func GetCardsEndpoint(c *gin.Context) {
	id := c.Param("id")
	limitQuery := c.Query("limit")
	offsetQuery := c.Query("offset")

	var limit int64 = 10
	var offset int64

	if limitQuery != "" {
		limit1, _ := strconv.ParseInt(limitQuery, 10, 32)
		limit = limit1
	}
	if offsetQuery != "" {
		offset1, _ := strconv.ParseInt(offsetQuery, 0, 32)
		offset = offset1
	}

	cards := GetCards(id, int(limit), int(offset))

	if len(cards) > 0 {
		c.JSON(http.StatusOK, cards)
	} else {
		c.String(http.StatusOK, "There's no saved card for that ID")
	}
}

// SaveCardsEndpoint handles save card
func SaveCardsEndpoint(c *gin.Context) {
	id := c.Param("id")
	var arrayCard []Card
	requestBody, err := ioutil.ReadAll(c.Request.Body)
	if err != nil {
		c.String(http.StatusBadRequest, "Wrong data format")
	} else {
		err1 := json.Unmarshal(requestBody, &arrayCard)
		if err1 != nil {
			c.String(http.StatusBadRequest, "Wrong data format")
		} else {
			err2 := SaveCards(id, arrayCard)
			if err2 != nil {
				c.String(http.StatusInternalServerError, err2.Error())
			} else {
				c.String(http.StatusOK, "Card is saved")
			}
		}
	}
}

func getInstallmentData() Installment {
	return Installment{Required: false, Terms: Terms{BNI: []int{3, 6, 12}, Mandiri: []int{3, 6, 12}, BCA: []int{3, 6, 12}, CIMB: []int{3, 6, 12}, Offline: []int{3, 6, 12}}}
}

func getWhitelistBin() []string {
	return []string{"481111", "521111"}
}
<|endoftext|>


<commit_before>package goskeleton

import (
	"bytes"
	"fmt"
	"github.com/codegangsta/inject"
	"github.com/gin-gonic/gin"
	"log"
	"net/http/httptest"
	"reflect"
	"strings"
)

const (
	CTX_KEY_REQUEST_INJECTOR = "REQUEST_INJECTOR"
)

func GetInjector(c *gin.Context) inject.Injector {
	var injector inject.Injector
	requestInjector, err := c.Get(CTX_KEY_REQUEST_INJECTOR)
	if err == nil && requestInjector != nil {
		injector = requestInjector.(inject.Injector)
	}
	return injector
}

//
// 将 injector 注入至 当前请求的 gin.Context 中
//
func injectorHandler(globalInjector inject.Injector) func(*gin.Context) {
	return func(c *gin.Context) {
		requestInjector := inject.New()
		requestInjector.SetParent(globalInjector)

		requestInjector.Map(c)

		c.Set(CTX_KEY_REQUEST_INJECTOR, requestInjector)
	}
}

// 生成一个处理器，用于调用我们的自定义处理器
func wrapperCustomHandler(customHandler interface{}, formValue interface{}) func(c *gin.Context) {

	if customHandler == nil {
		panic("rawHandler was nil")
	}

	if reflect.ValueOf(customHandler).Kind() != reflect.Func {
		panic("rawHandler is not a reflect.Func")
	}

	return func(c *gin.Context) {
		requestInjector := c.MustGet(CTX_KEY_REQUEST_INJECTOR)
		if requestInjector != nil {

			injector := requestInjector.(inject.Injector)
			if injector != nil {

				if formValue != nil {
					newForm := reflect.New(reflect.TypeOf(formValue))
					//					fmt.Println("newForm.Interface()", newForm.Interface())
					//					fmt.Println("newForm.Addr().Interface()", newForm.Addr().Interface())
					newForm.Elem().Set(reflect.ValueOf(formValue))

					if strings.HasPrefix(c.Request.Header.Get("Content-Type"), "multipart/form-data") {
						c.Request.ParseMultipartForm(2 << 20)
						c.Request.Header.Set("Content-Type", gin.MIMEPOSTForm)
					}

					if c.Bind(newForm.Interface()) {
						//						fmt.Println("c.Bind succeed")

						injector.Map(newForm.Elem().Interface())
					} else {
						log.Println("c.Bind failed")
					}

				}

				returnValues, err := injector.Invoke(customHandler)
				if err != nil {
					panic(err)
				}

				if len(returnValues) > 0 {
					// TODO 处理返回值
				}
			}
		} else {
			// 否则继续处理下一个
			c.Next()
		}

	}
}

// 用于记录HTTP 请求/响应内容，以便于开发调试
func RecordHttpHandler(c *gin.Context) {
	wrapper := &responseWriter{
		recorder: httptest.NewRecorder(),
	}
	wrapper.ResponseWriter = c.Writer

	oldWriter := c.Writer

	// 重写 wrapper
	c.Writer = wrapper
	c.Next()
	c.Writer = oldWriter

	// 文本类型的响应，输出其内容
	contentType := wrapper.ResponseWriter.Header().Get("Content-Type")
	contentType = strings.ToLower(contentType)

	if strings.HasPrefix(contentType, "text") ||
		strings.Contains(contentType, "javascript") ||
		strings.Contains(contentType, "json") {

		req := c.Request

		buf := bytes.NewBuffer(make([]byte, 0, 512))

		req.ParseMultipartForm(32 << 20)
		req.ParseForm()

		buf.WriteString(fmt.Sprintf("%s\n", req.RemoteAddr))

		buf.WriteString(fmt.Sprintf("%s %s\n", req.Method, req.RequestURI))
		if len(req.UserAgent()) > 0 {
			buf.WriteString(fmt.Sprintf("User-Agent: %s\n", req.UserAgent()))
		}

		if len(req.Form) > 0 {
			buf.WriteString("\n解析的参数:\n")
			for k, v := range req.Form {
				buf.WriteString("\t")
				buf.WriteString(k)
				buf.WriteString("=")
				buf.WriteString(strings.Join(v, ","))
				buf.WriteString("\n")
			}
		}

		if req.MultipartForm != nil {
			if len(req.MultipartForm.Value) > 0 {
				buf.WriteString("\nMultipartForm:\n")
				for k, v := range req.MultipartForm.Value {
					buf.WriteString("\t")
					buf.WriteString(k)
					buf.WriteString("=")
					buf.WriteString(strings.Join(v, ","))
					buf.WriteString("\n")
				}
			}

			if len(req.MultipartForm.File) > 0 {

				buf.WriteString("MultipartFile:\n")
				for k, v := range req.MultipartForm.File {
					if v != nil {
						buf.WriteString("\t")
						buf.WriteString(k)
						buf.WriteString("=")
						for i, e := range v {
							if i > 0 {
								buf.WriteString(",")
							}
							buf.WriteString("filename:" + e.Filename)

						}

						buf.WriteString("\n")
					}
				}
			}
		}

		buf.WriteString("\nResponseBody:\n")
		buf.Write(wrapper.recorder.Body.Bytes())

		fmt.Println(buf.String())
	}

}
<commit_msg>add RecordHttp variable<commit_after>package goskeleton

import (
	"bytes"
	"fmt"
	"github.com/codegangsta/inject"
	"github.com/gin-gonic/gin"
	"log"
	"net/http/httptest"
	"reflect"
	"strings"
)

const (
	CTX_KEY_REQUEST_INJECTOR = "REQUEST_INJECTOR"
)

func GetInjector(c *gin.Context) inject.Injector {
	var injector inject.Injector
	requestInjector, err := c.Get(CTX_KEY_REQUEST_INJECTOR)
	if err == nil && requestInjector != nil {
		injector = requestInjector.(inject.Injector)
	}
	return injector
}

//
// 将 injector 注入至 当前请求的 gin.Context 中
//
func injectorHandler(globalInjector inject.Injector) func(*gin.Context) {
	return func(c *gin.Context) {
		requestInjector := inject.New()
		requestInjector.SetParent(globalInjector)

		requestInjector.Map(c)

		c.Set(CTX_KEY_REQUEST_INJECTOR, requestInjector)
	}
}

// 生成一个处理器，用于调用我们的自定义处理器
func wrapperCustomHandler(customHandler interface{}, formValue interface{}) func(c *gin.Context) {

	if customHandler == nil {
		panic("rawHandler was nil")
	}

	if reflect.ValueOf(customHandler).Kind() != reflect.Func {
		panic("rawHandler is not a reflect.Func")
	}

	return func(c *gin.Context) {
		requestInjector := c.MustGet(CTX_KEY_REQUEST_INJECTOR)
		if requestInjector != nil {

			injector := requestInjector.(inject.Injector)
			if injector != nil {

				if formValue != nil {
					newForm := reflect.New(reflect.TypeOf(formValue))
					//					fmt.Println("newForm.Interface()", newForm.Interface())
					//					fmt.Println("newForm.Addr().Interface()", newForm.Addr().Interface())
					newForm.Elem().Set(reflect.ValueOf(formValue))

					if strings.HasPrefix(c.Request.Header.Get("Content-Type"), "multipart/form-data") {
						c.Request.ParseMultipartForm(2 << 20)
						c.Request.Header.Set("Content-Type", gin.MIMEPOSTForm)
					}

					if c.Bind(newForm.Interface()) {
						//						fmt.Println("c.Bind succeed")

						injector.Map(newForm.Elem().Interface())
					} else {
						log.Println("c.Bind failed")
					}

				}

				returnValues, err := injector.Invoke(customHandler)
				if err != nil {
					panic(err)
				}

				if len(returnValues) > 0 {
					// TODO 处理返回值
				}
			}
		} else {
			// 否则继续处理下一个
			c.Next()
		}

	}
}

var RecordHttp bool

// 用于记录HTTP 请求/响应内容，以便于开发调试
func RecordHttpHandler(c *gin.Context) {

	if !RecordHttp {
		c.Next()
		return
	}

	wrapper := &responseWriter{
		recorder: httptest.NewRecorder(),
	}
	wrapper.ResponseWriter = c.Writer

	oldWriter := c.Writer

	// 重写 wrapper
	c.Writer = wrapper
	c.Next()
	c.Writer = oldWriter

	// 文本类型的响应，输出其内容
	contentType := wrapper.ResponseWriter.Header().Get("Content-Type")
	contentType = strings.ToLower(contentType)

	if strings.HasPrefix(contentType, "text") ||
		strings.Contains(contentType, "javascript") ||
		strings.Contains(contentType, "json") {

		req := c.Request

		buf := bytes.NewBuffer(make([]byte, 0, 512))

		req.ParseMultipartForm(32 << 20)
		req.ParseForm()

		buf.WriteString(fmt.Sprintf("%s\n", req.RemoteAddr))

		buf.WriteString(fmt.Sprintf("%s %s\n", req.Method, req.RequestURI))
		if len(req.UserAgent()) > 0 {
			buf.WriteString(fmt.Sprintf("User-Agent: %s\n", req.UserAgent()))
		}

		if len(req.Form) > 0 {
			buf.WriteString("\n解析的参数:\n")
			for k, v := range req.Form {
				buf.WriteString("\t")
				buf.WriteString(k)
				buf.WriteString("=")
				buf.WriteString(strings.Join(v, ","))
				buf.WriteString("\n")
			}
		}

		if req.MultipartForm != nil {
			if len(req.MultipartForm.Value) > 0 {
				buf.WriteString("\nMultipartForm:\n")
				for k, v := range req.MultipartForm.Value {
					buf.WriteString("\t")
					buf.WriteString(k)
					buf.WriteString("=")
					buf.WriteString(strings.Join(v, ","))
					buf.WriteString("\n")
				}
			}

			if len(req.MultipartForm.File) > 0 {

				buf.WriteString("MultipartFile:\n")
				for k, v := range req.MultipartForm.File {
					if v != nil {
						buf.WriteString("\t")
						buf.WriteString(k)
						buf.WriteString("=")
						for i, e := range v {
							if i > 0 {
								buf.WriteString(",")
							}
							buf.WriteString("filename:" + e.Filename)

						}

						buf.WriteString("\n")
					}
				}
			}
		}

		buf.WriteString("\nResponseBody:\n")
		buf.Write(wrapper.recorder.Body.Bytes())

		fmt.Println(buf.String())
	}

}
<|endoftext|>