<commit_before>package lfs

import (
	"fmt"
	"os"
	"path/filepath"
)

// Uploadable describes a file that can be uploaded.
type Uploadable struct {
	oid      string
	OidPath  string
	Filename string
	size     int64
	object   *objectResource
}

// NewUploadable builds the Uploadable from the given information.
// "filename" can be empty if a raw object is pushed (see "object-id" flag in push command)/
func NewUploadable(oid, filename string) (*Uploadable, *WrappedError) {
	localMediaPath, err := LocalMediaPath(oid)
	if err != nil {
		return nil, Errorf(err, "Error uploading file %s (%s)", filename, oid)
	}

	pathForStats := localMediaPath
	if len(filename) > 0 {
		if err := ensureFile(filename, localMediaPath); err != nil {
			return nil, Errorf(err, "Error uploading file %s (%s)", filename, oid)
		}
		pathForStats = filename
	}

	fi, err := os.Stat(pathForStats)
	if err != nil {
		return nil, Errorf(err, "Error uploading file %s (%s)", filename, oid)
	}

	return &Uploadable{oid: oid, OidPath: localMediaPath, Filename: filename, size: fi.Size()}, nil
}

func (u *Uploadable) Check() (*objectResource, *WrappedError) {
	return UploadCheck(u.OidPath)
}

func (u *Uploadable) Transfer(cb CopyCallback) *WrappedError {
	wcb := func(total, read int64, current int) error {
		cb(total, read, current)
		return nil
	}

	return UploadObject(u.object, wcb)
}

func (u *Uploadable) Object() *objectResource {
	return u.object
}

func (u *Uploadable) Oid() string {
	return u.oid
}

func (u *Uploadable) Size() int64 {
	return u.size
}

func (u *Uploadable) Name() string {
	return u.Filename
}

func (u *Uploadable) SetObject(o *objectResource) {
	u.object = o
}

// NewUploadQueue builds an UploadQueue, allowing `workers` concurrent uploads.
func NewUploadQueue(workers, files int) *TransferQueue {
	q := newTransferQueue(workers, files)
	q.transferKind = "upload"
	return q
}

// ensureFile makes sure that the cleanPath exists before pushing it.  If it
// does not exist, it attempts to clean it by reading the file at smudgePath.
func ensureFile(smudgePath, cleanPath string) error {
	if _, err := os.Stat(cleanPath); err == nil {
		return nil
	}

	expectedOid := filepath.Base(cleanPath)
	localPath := filepath.Join(LocalWorkingDir, smudgePath)
	file, err := os.Open(localPath)
	if err != nil {
		return err
	}

	defer file.Close()

	stat, err := file.Stat()
	if err != nil {
		return err
	}

	cleaned, err := PointerClean(file, stat.Size(), nil)
	if err != nil {
		return err
	}

	cleaned.Close()

	if expectedOid != cleaned.Oid {
		return fmt.Errorf("Expected %s to have an OID of %s, got %s", smudgePath, expectedOid, cleaned.Oid)
	}

	return nil
}
<commit_msg>Rename `pathForStats` to `statsPath`<commit_after>package lfs

import (
	"fmt"
	"os"
	"path/filepath"
)

// Uploadable describes a file that can be uploaded.
type Uploadable struct {
	oid      string
	OidPath  string
	Filename string
	size     int64
	object   *objectResource
}

// NewUploadable builds the Uploadable from the given information.
// "filename" can be empty if a raw object is pushed (see "object-id" flag in push command)/
func NewUploadable(oid, filename string) (*Uploadable, *WrappedError) {
	localMediaPath, err := LocalMediaPath(oid)
	if err != nil {
		return nil, Errorf(err, "Error uploading file %s (%s)", filename, oid)
	}

	statsPath := localMediaPath
	if len(filename) > 0 {
		if err := ensureFile(filename, localMediaPath); err != nil {
			return nil, Errorf(err, "Error uploading file %s (%s)", filename, oid)
		}
		statsPath = filename
	}

	fi, err := os.Stat(statsPath)
	if err != nil {
		return nil, Errorf(err, "Error uploading file %s (%s)", filename, oid)
	}

	return &Uploadable{oid: oid, OidPath: localMediaPath, Filename: filename, size: fi.Size()}, nil
}

func (u *Uploadable) Check() (*objectResource, *WrappedError) {
	return UploadCheck(u.OidPath)
}

func (u *Uploadable) Transfer(cb CopyCallback) *WrappedError {
	wcb := func(total, read int64, current int) error {
		cb(total, read, current)
		return nil
	}

	return UploadObject(u.object, wcb)
}

func (u *Uploadable) Object() *objectResource {
	return u.object
}

func (u *Uploadable) Oid() string {
	return u.oid
}

func (u *Uploadable) Size() int64 {
	return u.size
}

func (u *Uploadable) Name() string {
	return u.Filename
}

func (u *Uploadable) SetObject(o *objectResource) {
	u.object = o
}

// NewUploadQueue builds an UploadQueue, allowing `workers` concurrent uploads.
func NewUploadQueue(workers, files int) *TransferQueue {
	q := newTransferQueue(workers, files)
	q.transferKind = "upload"
	return q
}

// ensureFile makes sure that the cleanPath exists before pushing it.  If it
// does not exist, it attempts to clean it by reading the file at smudgePath.
func ensureFile(smudgePath, cleanPath string) error {
	if _, err := os.Stat(cleanPath); err == nil {
		return nil
	}

	expectedOid := filepath.Base(cleanPath)
	localPath := filepath.Join(LocalWorkingDir, smudgePath)
	file, err := os.Open(localPath)
	if err != nil {
		return err
	}

	defer file.Close()

	stat, err := file.Stat()
	if err != nil {
		return err
	}

	cleaned, err := PointerClean(file, stat.Size(), nil)
	if err != nil {
		return err
	}

	cleaned.Close()

	if expectedOid != cleaned.Oid {
		return fmt.Errorf("Expected %s to have an OID of %s, got %s", smudgePath, expectedOid, cleaned.Oid)
	}

	return nil
}
<|endoftext|>


<commit_before>// +build lambdabinary

package sparta

import (
	"context"
	"encoding/json"
	"fmt"
	"os"
	"reflect"
	"strings"

	awsLambdaGo "github.com/aws/aws-lambda-go/lambda"
	awsLambdaContext "github.com/aws/aws-lambda-go/lambdacontext"
	cloudformationResources "github.com/mweagle/Sparta/aws/cloudformation/resources"
	"github.com/sirupsen/logrus"
)

// https://blog.cloudflare.com/setting-go-variables-at-compile-time/
// StampedServiceName is the serviceName stamped into this binary
var StampedServiceName string

// awsLambdaFunctionName returns the name of the function, that includes
// a prefix that was stamped into the binary via a linker flag in provision.go
func awsLambdaFunctionName(serviceName string,
	internalFunctionName string) string {
	// Ok, so we need to scope the functionname with the StackName, otherwise
	// there will be collisions in the account. So how to publish
	// the stack name into the awsbinary?
	// How about
	// Linker flags would be nice...sparta.StampedServiceName ?
	awsLambdaName := fmt.Sprintf("%s-%s",
		StampedServiceName,
		internalFunctionName)
	return sanitizedName(awsLambdaName)
}

func takesContext(handler reflect.Type) bool {
	handlerTakesContext := false
	if handler.NumIn() > 0 {
		contextType := reflect.TypeOf((*context.Context)(nil)).Elem()
		argumentType := handler.In(0)
		handlerTakesContext = argumentType.Implements(contextType)
	}
	return handlerTakesContext
}

func tappedHandler(handlerSymbol interface{},
	logger *logrus.Logger) interface{} {

	// Tap the call chain to inject the context params...
	handler := reflect.ValueOf(handlerSymbol)
	handlerType := reflect.TypeOf(handlerSymbol)
	takesContext := takesContext(handlerType)

	return func(ctx context.Context, msg json.RawMessage) (interface{}, error) {
		ctx = context.WithValue(ctx, ContextKeyLogger, logger)

		// Create the entry logger that has some context information
		var logrusEntry *logrus.Entry
		lambdaContext, lambdaContextOk := awsLambdaContext.FromContext(ctx)
		if lambdaContextOk {
			logrusEntry = logrus.
				NewEntry(logger).
				WithFields(logrus.Fields{
					"reqID": lambdaContext.AwsRequestID,
					"arn":   lambdaContext.InvokedFunctionArn,
				})
		} else {
			logrusEntry = logrus.
				NewEntry(logger).
				WithFields(logrus.Fields{})
		}
		ctx = context.WithValue(ctx, ContextKeyRequestLogger, logrusEntry)

		// construct arguments
		var args []reflect.Value
		if takesContext {
			args = append(args, reflect.ValueOf(ctx))
		}
		if (handlerType.NumIn() == 1 && !takesContext) ||
			handlerType.NumIn() == 2 {
			eventType := handlerType.In(handlerType.NumIn() - 1)
			event := reflect.New(eventType)
			unmarshalErr := json.Unmarshal(msg, event.Interface())
			if unmarshalErr != nil {
				return nil, unmarshalErr
			}
			args = append(args, event.Elem())
		}
		response := handler.Call(args)

		// If the user function
		// convert return values into (interface{}, error)
		var err error
		if len(response) > 0 {
			if errVal, ok := response[len(response)-1].Interface().(error); ok {
				err = errVal
			}
		}
		var val interface{}
		if len(response) > 1 {
			val = response[0].Interface()
		}
		return val, err
	}
}

// Execute creates an HTTP listener to dispatch execution. Typically
// called via Main() via command line arguments.
func Execute(serviceName string,
	lambdaAWSInfos []*LambdaAWSInfo,
	port int,
	parentProcessPID int,
	logger *logrus.Logger) error {

	// Find the function name based on the dispatch
	// https://docs.aws.amazon.com/lambda/latest/dg/current-supported-versions.html
	requestedLambdaFunctionName := os.Getenv("AWS_LAMBDA_FUNCTION_NAME")
	// Log any info when we start up...
	platformLogSysInfo(requestedLambdaFunctionName, logger)

	/*
		There are three types of targets:
			- User functions
			- User custom resources
			- Sparta custom resources
	*/
	// Based on the environment variable, setup the proper listener...
	lambdaFunctionName := ""
	var handlerSymbol interface{}
	knownNames := []string{}
	for _, eachLambdaInfo := range lambdaAWSInfos {
		lambdaFunctionName = awsLambdaFunctionName(serviceName,
			eachLambdaInfo.lambdaFunctionName())

		knownNames = append(knownNames, lambdaFunctionName)
		if requestedLambdaFunctionName == lambdaFunctionName {
			handlerSymbol = eachLambdaInfo.handlerSymbol
		}
		// User defined custom resource handler?
		for _, eachCustomResource := range eachLambdaInfo.customResources {
			lambdaFunctionName = awsLambdaFunctionName(serviceName,
				eachCustomResource.userFunctionName)
			knownNames = append(knownNames, lambdaFunctionName)
			if requestedLambdaFunctionName == lambdaFunctionName {
				handlerSymbol = eachCustomResource.handlerSymbol
			}
		}
		if handlerSymbol != nil {
			break
		}
	}
	if handlerSymbol == nil {
		// So the sanitized name is what we use to dispatch. For
		// that we need a reverse lookup?
		customResourceTypes := []string{
			cloudformationResources.HelloWorld,
			cloudformationResources.S3LambdaEventSource,
			cloudformationResources.SNSLambdaEventSource,
			cloudformationResources.SESLambdaEventSource,
			cloudformationResources.CloudWatchLogsLambdaEventSource,
			cloudformationResources.ZipToS3Bucket,
		}
		customResourceType := ""
		for _, eachCustomResourceType := range customResourceTypes {
			lambdaFunctionName = awsLambdaFunctionName(serviceName,
				eachCustomResourceType)

			logger.WithFields(logrus.Fields{
				"customResourceType": eachCustomResourceType,
				"computedLambdaName": lambdaFunctionName,
			}).Debug("Checking custom resource")

			knownNames = append(knownNames, lambdaFunctionName)

			if requestedLambdaFunctionName == lambdaFunctionName {
				customResourceType = eachCustomResourceType
				break
			}
		}
		if customResourceType != "" {
			customHandler := cloudformationResources.NewCustomResourceLambdaHandler(customResourceType,
				logger)
			if customHandler != nil {
				handlerSymbol = customHandler
			}
		}
	}

	if handlerSymbol == nil {
		errorMessage := fmt.Errorf("No handler found for lambdaName: %s. Known: %s",
			requestedLambdaFunctionName,
			strings.Join(knownNames, ","))
		logger.Error(errorMessage)
		return errorMessage
	}

	// Startup our version...
	tappedHandler := tappedHandler(handlerSymbol, logger)
	awsLambdaGo.Start(tappedHandler)
	return nil
}
<commit_msg>Fix comment to satisfy Go prechecks<commit_after>// +build lambdabinary

package sparta

import (
	"context"
	"encoding/json"
	"fmt"
	"os"
	"reflect"
	"strings"

	awsLambdaGo "github.com/aws/aws-lambda-go/lambda"
	awsLambdaContext "github.com/aws/aws-lambda-go/lambdacontext"
	cloudformationResources "github.com/mweagle/Sparta/aws/cloudformation/resources"
	"github.com/sirupsen/logrus"
)

// StampedServiceName is the name stamp
// https://blog.cloudflare.com/setting-go-variables-at-compile-time/
// StampedServiceName is the serviceName stamped into this binary
var StampedServiceName string

// awsLambdaFunctionName returns the name of the function, that includes
// a prefix that was stamped into the binary via a linker flag in provision.go
func awsLambdaFunctionName(serviceName string,
	internalFunctionName string) string {
	// Ok, so we need to scope the functionname with the StackName, otherwise
	// there will be collisions in the account. So how to publish
	// the stack name into the awsbinary?
	// How about
	// Linker flags would be nice...sparta.StampedServiceName ?
	awsLambdaName := fmt.Sprintf("%s-%s",
		StampedServiceName,
		internalFunctionName)
	return sanitizedName(awsLambdaName)
}

func takesContext(handler reflect.Type) bool {
	handlerTakesContext := false
	if handler.NumIn() > 0 {
		contextType := reflect.TypeOf((*context.Context)(nil)).Elem()
		argumentType := handler.In(0)
		handlerTakesContext = argumentType.Implements(contextType)
	}
	return handlerTakesContext
}

func tappedHandler(handlerSymbol interface{},
	logger *logrus.Logger) interface{} {

	// Tap the call chain to inject the context params...
	handler := reflect.ValueOf(handlerSymbol)
	handlerType := reflect.TypeOf(handlerSymbol)
	takesContext := takesContext(handlerType)

	return func(ctx context.Context, msg json.RawMessage) (interface{}, error) {
		ctx = context.WithValue(ctx, ContextKeyLogger, logger)

		// Create the entry logger that has some context information
		var logrusEntry *logrus.Entry
		lambdaContext, lambdaContextOk := awsLambdaContext.FromContext(ctx)
		if lambdaContextOk {
			logrusEntry = logrus.
				NewEntry(logger).
				WithFields(logrus.Fields{
					"reqID": lambdaContext.AwsRequestID,
					"arn":   lambdaContext.InvokedFunctionArn,
				})
		} else {
			logrusEntry = logrus.
				NewEntry(logger).
				WithFields(logrus.Fields{})
		}
		ctx = context.WithValue(ctx, ContextKeyRequestLogger, logrusEntry)

		// construct arguments
		var args []reflect.Value
		if takesContext {
			args = append(args, reflect.ValueOf(ctx))
		}
		if (handlerType.NumIn() == 1 && !takesContext) ||
			handlerType.NumIn() == 2 {
			eventType := handlerType.In(handlerType.NumIn() - 1)
			event := reflect.New(eventType)
			unmarshalErr := json.Unmarshal(msg, event.Interface())
			if unmarshalErr != nil {
				return nil, unmarshalErr
			}
			args = append(args, event.Elem())
		}
		response := handler.Call(args)

		// If the user function
		// convert return values into (interface{}, error)
		var err error
		if len(response) > 0 {
			if errVal, ok := response[len(response)-1].Interface().(error); ok {
				err = errVal
			}
		}
		var val interface{}
		if len(response) > 1 {
			val = response[0].Interface()
		}
		return val, err
	}
}

// Execute creates an HTTP listener to dispatch execution. Typically
// called via Main() via command line arguments.
func Execute(serviceName string,
	lambdaAWSInfos []*LambdaAWSInfo,
	port int,
	parentProcessPID int,
	logger *logrus.Logger) error {

	// Find the function name based on the dispatch
	// https://docs.aws.amazon.com/lambda/latest/dg/current-supported-versions.html
	requestedLambdaFunctionName := os.Getenv("AWS_LAMBDA_FUNCTION_NAME")
	// Log any info when we start up...
	platformLogSysInfo(requestedLambdaFunctionName, logger)

	/*
		There are three types of targets:
			- User functions
			- User custom resources
			- Sparta custom resources
	*/
	// Based on the environment variable, setup the proper listener...
	lambdaFunctionName := ""
	var handlerSymbol interface{}
	knownNames := []string{}
	for _, eachLambdaInfo := range lambdaAWSInfos {
		lambdaFunctionName = awsLambdaFunctionName(serviceName,
			eachLambdaInfo.lambdaFunctionName())

		knownNames = append(knownNames, lambdaFunctionName)
		if requestedLambdaFunctionName == lambdaFunctionName {
			handlerSymbol = eachLambdaInfo.handlerSymbol
		}
		// User defined custom resource handler?
		for _, eachCustomResource := range eachLambdaInfo.customResources {
			lambdaFunctionName = awsLambdaFunctionName(serviceName,
				eachCustomResource.userFunctionName)
			knownNames = append(knownNames, lambdaFunctionName)
			if requestedLambdaFunctionName == lambdaFunctionName {
				handlerSymbol = eachCustomResource.handlerSymbol
			}
		}
		if handlerSymbol != nil {
			break
		}
	}
	if handlerSymbol == nil {
		// So the sanitized name is what we use to dispatch. For
		// that we need a reverse lookup?
		customResourceTypes := []string{
			cloudformationResources.HelloWorld,
			cloudformationResources.S3LambdaEventSource,
			cloudformationResources.SNSLambdaEventSource,
			cloudformationResources.SESLambdaEventSource,
			cloudformationResources.CloudWatchLogsLambdaEventSource,
			cloudformationResources.ZipToS3Bucket,
		}
		customResourceType := ""
		for _, eachCustomResourceType := range customResourceTypes {
			lambdaFunctionName = awsLambdaFunctionName(serviceName,
				eachCustomResourceType)

			logger.WithFields(logrus.Fields{
				"customResourceType": eachCustomResourceType,
				"computedLambdaName": lambdaFunctionName,
			}).Debug("Checking custom resource")

			knownNames = append(knownNames, lambdaFunctionName)

			if requestedLambdaFunctionName == lambdaFunctionName {
				customResourceType = eachCustomResourceType
				break
			}
		}
		if customResourceType != "" {
			customHandler := cloudformationResources.NewCustomResourceLambdaHandler(customResourceType,
				logger)
			if customHandler != nil {
				handlerSymbol = customHandler
			}
		}
	}

	if handlerSymbol == nil {
		errorMessage := fmt.Errorf("No handler found for lambdaName: %s. Known: %s",
			requestedLambdaFunctionName,
			strings.Join(knownNames, ","))
		logger.Error(errorMessage)
		return errorMessage
	}

	// Startup our version...
	tappedHandler := tappedHandler(handlerSymbol, logger)
	awsLambdaGo.Start(tappedHandler)
	return nil
}
<|endoftext|>


<commit_before>package sack

import (
	"bufio"
	"fmt"
	"github.com/codegangsta/cli"
	. "github.com/tj/go-debug"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"
)

func executeCmd(term string, path string, flags string) []string {
	var debug = Debug("sack:search")

	debug("executeCmd:agCmd: %v", agCmd)
	var lines []string
	_, err := exec.LookPath(agCmd)
	if err == nil {
		debug("executing Ag search")
		lines = agSearch(term, path, flags)
	} else {
		debug("executing Grep search")
		lines = grepSearch(term, path, flags)
	}

	return lines
}

func agSearch(term string, path string, flags string) []string {
	bin := getPath(agCmd)

	// Blows up if flags == "" without this conditional
	var cmdOut []byte
	var err error
	if flags == "" {
		cmdOut, err = exec.Command(bin, term, path).Output()
	} else {
		cmdOut, err = exec.Command(bin, flags, term, path).Output()
	}

	var debug = Debug("sack:search")
	debug("agSearch:bin: %v, flags: %v, term: %v, path: %v", bin, flags, term, path)
	debug("agSearch:Exec: %v %v %v %v", bin, flags, term, path)
	debug("agSearch:cmdOut: %v", outputLines(cmdOut))
	debug("agSearch:err: %v", err)

	checkCmd(err)

	return outputLines(cmdOut)
}

func getPath(bin string) string {
	agBin, err := exec.LookPath(bin)
	var debug = Debug("sack:search")
	debug("getPath:agBin: %v", agBin)
	debug("getPath:err: %v", err)
	check(err)
	return agBin
}

func grepSearch(term string, path string, flags string) []string {
	bin := getPath(grepCmd)

	flag1 := "-ir"
	flag2 := "--line-number"

	cmdOut, err := exec.Command(bin, flag1, flag2, term, path).Output()
	checkCmd(err)

	return outputLines(cmdOut)
}

func outputLines(b []byte) []string {
	return strings.Split(string(b), "\n")
}

type agLine struct {
	file    string
	line    string
	content string
}

func setTermPath(c *cli.Context) (string, string) {
	argLen := len(c.Args())
	var term string
	var path string
	var debug = Debug("sack:search")
	debug("setTermPath:argLen: %v", argLen)
	switch argLen {
	case 0:
		panic(1)
	case 1:
		term = c.Args()[0]
		path, _ = os.Getwd()
	case 2:
		term = c.Args()[0]
		path = c.Args()[1]
	default:
		term = c.Args()[0]
		path = c.Args()[1]
	}
	return term, path
}

func search(c *cli.Context) {

	term, searchPath := setTermPath(c)

	lines := executeCmd(term, searchPath, c.String("flags"))

	if len(lines) == 0 {

		var debug = Debug("sack:search:error")
		debug("nolines:lines: %v", lines)

		os.Exit(1)
		return
	}

	t := []byte(term)
	err := ioutil.WriteFile(termPath, t, 0644)
	check(err)

	f, err := os.Create(filePath)
	check(err)
	defer f.Close()

	fmt.Print(header)

	for i, line := range lines {

		if line == "" {
			break
		}

		lp := splitLine(line)
		l := agLine{file: lp[0], line: lp[1], content: lp[2]}

		s := displayLines(term, i, l.line, l.file, l.content)
		fmt.Println(s)
		o := fmt.Sprint(l.line, " ", l.file, " ", l.content, "\n")

		w := bufio.NewWriter(f)
		_, err := w.WriteString(o)
		check(err)
		w.Flush()
	}
}
<commit_msg>Truncate long lines<commit_after>package sack

import (
	"bufio"
	"fmt"
	"github.com/codegangsta/cli"
	. "github.com/tj/go-debug"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"
)

func executeCmd(term string, path string, flags string) []string {
	var debug = Debug("sack:search")

	debug("executeCmd:agCmd: %v", agCmd)
	var lines []string
	_, err := exec.LookPath(agCmd)
	if err == nil {
		debug("executing Ag search")
		lines = agSearch(term, path, flags)
	} else {
		debug("executing Grep search")
		lines = grepSearch(term, path, flags)
	}

	return lines
}

func agSearch(term string, path string, flags string) []string {
	bin := getPath(agCmd)

	// Blows up if flags == "" without this conditional
	var cmdOut []byte
	var err error
	if flags == "" {
		cmdOut, err = exec.Command(bin, term, path).Output()
	} else {
		cmdOut, err = exec.Command(bin, flags, term, path).Output()
	}

	var debug = Debug("sack:search")
	debug("agSearch:bin: %v, flags: %v, term: %v, path: %v", bin, flags, term, path)
	debug("agSearch:Exec: %v %v %v %v", bin, flags, term, path)
	debug("agSearch:cmdOut: %v", outputLines(cmdOut))
	debug("agSearch:err: %v", err)

	checkCmd(err)

	return outputLines(cmdOut)
}

func getPath(bin string) string {
	agBin, err := exec.LookPath(bin)
	var debug = Debug("sack:search")
	debug("getPath:agBin: %v", agBin)
	debug("getPath:err: %v", err)
	check(err)
	return agBin
}

func grepSearch(term string, path string, flags string) []string {
	bin := getPath(grepCmd)

	flag1 := "-ir"
	flag2 := "--line-number"

	cmdOut, err := exec.Command(bin, flag1, flag2, term, path).Output()
	checkCmd(err)

	return outputLines(cmdOut)
}

func outputLines(b []byte) []string {
	return strings.Split(string(b), "\n")
}

type agLine struct {
	file    string
	line    string
	content string
}

func setTermPath(c *cli.Context) (string, string) {
	argLen := len(c.Args())
	var term string
	var path string
	var debug = Debug("sack:search")
	debug("setTermPath:argLen: %v", argLen)
	switch argLen {
	case 0:
		panic(1)
	case 1:
		term = c.Args()[0]
		path, _ = os.Getwd()
	case 2:
		term = c.Args()[0]
		path = c.Args()[1]
	default:
		term = c.Args()[0]
		path = c.Args()[1]
	}
	return term, path
}

func truncateLine(line string) string {
	// Lines longer than maxLength are often junk results
	actualContentLen := len(line)
	maxLength := 200
	var contentLen int
	if actualContentLen > maxLength {
		contentLen = maxLength
	} else {
		contentLen = len(line)
	}

	return line[:contentLen]
}

func search(c *cli.Context) {

	term, searchPath := setTermPath(c)

	lines := executeCmd(term, searchPath, c.String("flags"))

	if len(lines) == 0 {

		var debug = Debug("sack:search:error")
		debug("nolines:lines: %v", lines)

		os.Exit(1)
		return
	}

	t := []byte(term)
	err := ioutil.WriteFile(termPath, t, 0644)
	check(err)

	f, err := os.Create(filePath)
	check(err)
	defer f.Close()

	fmt.Print(header)

	for i, line := range lines {

		if line == "" {
			break
		}

		lp := splitLine(line)

		l := agLine{file: lp[0], line: lp[1], content: truncateLine(lp[2])}

		s := displayLines(term, i, l.line, l.file, l.content)
		fmt.Println(s)
		o := fmt.Sprint(l.line, " ", l.file, " ", l.content, "\n")

		w := bufio.NewWriter(f)
		_, err := w.WriteString(o)
		check(err)
		w.Flush()
	}
}
<|endoftext|>


<commit_before>/*
   Nging is a toolbox for webmasters
   Copyright (C) 2018-present  Wenhui Shen <swh@admpub.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.
*/

package manager

import (
	"errors"
	"strings"

	"github.com/admpub/nging/v3/application/handler"
	"github.com/admpub/nging/v3/application/library/config"
	"github.com/admpub/nging/v3/application/registry/settings"
	"github.com/webx-top/com"
	"github.com/webx-top/echo"
)

func Settings(ctx echo.Context) error {
	//panic(echo.Dump(settings.ConfigAsStore(), false))
	var err error
	var errs []string
	group := ctx.Form(`group`, `base`)
	var groups []string
	if len(group) > 0 {
		groups = append(groups, group)
	}
	if ctx.IsPost() {
		err = configPost(ctx, groups...)
		if err != nil {
			goto END
		}
		err = settings.RunHookPost(ctx, groups...)
		if err != nil {
			goto END
		}
		if len(groups) > 0 {
			if com.InSlice(`base`, groups) {
				config.DefaultConfig.SetDebug(ctx.Formx(`base[debug][value]`).Bool())
			}
			err = config.DefaultConfig.Settings.SetConfigs(ctx, groups...)
		} else {
			err = config.DefaultConfig.Settings.Init(ctx)
		}
		if err != nil {
			goto END
		}
		handler.SendOk(ctx, ctx.T(`操作成功`))
		return ctx.Redirect(handler.URLFor(`/manager/settings?group=` + group))
	}

END:
	if _err := configGet(ctx, groups...); _err != nil {
		errs = append(errs, _err.Error())
	}
	if _err := settings.RunHookGet(ctx, groups...); _err != nil {
		errs = append(errs, _err.Error())
	}
	if err != nil {
		errs = append(errs, err.Error())
	}
	if len(errs) > 0 {
		err = errors.New(strings.Join(errs, "\n"))
	}
	ctx.Set(`config`, config.DefaultConfig)
	ctx.Set(`settings`, settings.Settings())
	ctx.Set(`group`, group)
	ctx.SetFunc(`hasConfigGroup`, settings.ConfigHasGroup)
	ctx.SetFunc(`hasConfigKey`, settings.ConfigHasKey)
	return ctx.Render(`/manager/settings`, handler.Err(ctx, err))
}
<commit_msg>update<commit_after>/*
   Nging is a toolbox for webmasters
   Copyright (C) 2018-present  Wenhui Shen <swh@admpub.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.
*/

package manager

import (
	"github.com/admpub/nging/v3/application/handler"
	"github.com/admpub/nging/v3/application/library/common"
	"github.com/admpub/nging/v3/application/library/config"
	"github.com/admpub/nging/v3/application/registry/settings"
	"github.com/webx-top/com"
	"github.com/webx-top/echo"
)

func Settings(ctx echo.Context) error {
	//panic(echo.Dump(settings.ConfigAsStore(), false))
	errs := common.NewErrors()
	group := ctx.Form(`group`, `base`)
	var groups []string
	if len(group) > 0 {
		groups = append(groups, group)
	}
	if ctx.IsPost() {
		err := configPost(ctx, groups...)
		if err != nil {
			errs.Add(err)
		}
		err = settings.RunHookPost(ctx, groups...)
		if err != nil {
			errs.Add(err)
		}
		if len(groups) > 0 {
			if com.InSlice(`base`, groups) {
				config.DefaultConfig.SetDebug(ctx.Formx(`base[debug][value]`).Bool())
			}
			err = config.DefaultConfig.Settings.SetConfigs(ctx, groups...)
		} else {
			err = config.DefaultConfig.Settings.Init(ctx)
		}
		if err != nil {
			errs.Add(err)
		}
		err = errs.ToError()
		if err != nil {
			goto END
		}
		handler.SendOk(ctx, ctx.T(`操作成功`))
		return ctx.Redirect(handler.URLFor(`/manager/settings?group=` + group))
	}

END:
	if _err := configGet(ctx, groups...); _err != nil {
		errs.Add(_err)
	}
	if _err := settings.RunHookGet(ctx, groups...); _err != nil {
		errs.Add(_err)
	}
	ctx.Set(`config`, config.DefaultConfig)
	ctx.Set(`settings`, settings.Settings())
	ctx.Set(`group`, group)
	ctx.SetFunc(`hasConfigGroup`, settings.ConfigHasGroup)
	ctx.SetFunc(`hasConfigKey`, settings.ConfigHasKey)
	return ctx.Render(`/manager/settings`, handler.Err(ctx, errs.ToError()))
}
<|endoftext|>


<commit_before>package core

import (
	"errors"
	"fmt"
	"math/rand"
	"sync"
	"time"

	config "github.com/jbenet/go-ipfs/config"
	host "github.com/jbenet/go-ipfs/p2p/host"
	inet "github.com/jbenet/go-ipfs/p2p/net"
	peer "github.com/jbenet/go-ipfs/p2p/peer"
	dht "github.com/jbenet/go-ipfs/routing/dht"
	lgbl "github.com/jbenet/go-ipfs/util/eventlog/loggables"
	math2 "github.com/jbenet/go-ipfs/util/math2"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
	ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"
)

const (
	period                               = 30 * time.Second // how often to check connection status
	connectiontimeout      time.Duration = period / 3       // duration to wait when attempting to connect
	recoveryThreshold                    = 4                // attempt to bootstrap if connection count falls below this value
	numDHTBootstrapQueries               = 10               // number of DHT queries to execute
)

func superviseConnections(parent context.Context,
	h host.Host,
	route *dht.IpfsDHT, // TODO depend on abstract interface for testing purposes
	store peer.Peerstore,
	peers []*config.BootstrapPeer) error {

	for {
		ctx, _ := context.WithTimeout(parent, connectiontimeout)
		// TODO get config from disk so |peers| always reflects the latest
		// information
		if err := bootstrap(ctx, h, route, store, peers); err != nil {
			log.Error(err)
		}
		select {
		case <-parent.Done():
			return parent.Err()
		case <-time.Tick(period):
		}
	}
	return nil
}

func bootstrap(ctx context.Context,
	h host.Host,
	r *dht.IpfsDHT,
	ps peer.Peerstore,
	boots []*config.BootstrapPeer) error {

	connectedPeers := h.Network().Peers()
	if len(connectedPeers) >= recoveryThreshold {
		log.Event(ctx, "bootstrapSkip", h.ID())
		log.Debugf("%s bootstrap skipped -- connected to %d (> %d) nodes",
			h.ID(), len(connectedPeers), recoveryThreshold)

		return nil
	}
	numCxnsToCreate := recoveryThreshold - len(connectedPeers)

	log.Event(ctx, "bootstrapStart", h.ID())
	log.Debugf("%s bootstrapping to %d more nodes", h.ID(), numCxnsToCreate)

	var bootstrapPeers []peer.PeerInfo
	for _, bootstrap := range boots {
		p, err := toPeer(bootstrap)
		if err != nil {
			log.Event(ctx, "bootstrapError", h.ID(), lgbl.Error(err))
			log.Errorf("%s bootstrap error: %s", h.ID(), err)
			return err
		}
		bootstrapPeers = append(bootstrapPeers, p)
	}

	var notConnected []peer.PeerInfo
	for _, p := range bootstrapPeers {
		if h.Network().Connectedness(p.ID) != inet.Connected {
			notConnected = append(notConnected, p)
		}
	}

	if len(notConnected) < 1 {
		s := "must bootstrap to %d more nodes, but already connected to all candidates"
		err := fmt.Errorf(s, numCxnsToCreate)
		log.Event(ctx, "bootstrapError", h.ID(), lgbl.Error(err))
		log.Errorf("%s bootstrap error: %s", h.ID(), err)
		return err
	}

	var randomSubset = randomSubsetOfPeers(notConnected, numCxnsToCreate)

	log.Debugf("%s bootstrapping to %d nodes: %s", h.ID(), numCxnsToCreate, randomSubset)
	if err := connect(ctx, ps, r, randomSubset); err != nil {
		log.Event(ctx, "bootstrapError", h.ID(), lgbl.Error(err))
		log.Errorf("%s bootstrap error: %s", h.ID(), err)
		return err
	}
	return nil
}

func connect(ctx context.Context, ps peer.Peerstore, r *dht.IpfsDHT, peers []peer.PeerInfo) error {
	if len(peers) < 1 {
		return errors.New("bootstrap set empty")
	}

	var wg sync.WaitGroup
	for _, p := range peers {

		// performed asynchronously because when performed synchronously, if
		// one `Connect` call hangs, subsequent calls are more likely to
		// fail/abort due to an expiring context.

		wg.Add(1)
		go func(p peer.PeerInfo) {
			defer wg.Done()
			log.Event(ctx, "bootstrapDial", r.LocalPeer(), p.ID)
			log.Debugf("%s bootstrapping to %s", r.LocalPeer(), p.ID)

			ps.AddAddresses(p.ID, p.Addrs)
			err := r.Connect(ctx, p.ID)
			if err != nil {
				log.Event(ctx, "bootstrapFailed", p.ID)
				log.Criticalf("failed to bootstrap with %v", p.ID)
				return
			}
			log.Event(ctx, "bootstrapSuccess", p.ID)
			log.Infof("bootstrapped with %v", p.ID)
		}(p)
	}
	wg.Wait()
	if err := r.Bootstrap(ctx, numDHTBootstrapQueries); err != nil {
		return err
	}
	return nil
}

func toPeer(bootstrap *config.BootstrapPeer) (p peer.PeerInfo, err error) {
	id, err := peer.IDB58Decode(bootstrap.PeerID)
	if err != nil {
		return
	}
	maddr, err := ma.NewMultiaddr(bootstrap.Address)
	if err != nil {
		return
	}
	p = peer.PeerInfo{
		ID:    id,
		Addrs: []ma.Multiaddr{maddr},
	}
	return
}

func randomSubsetOfPeers(in []peer.PeerInfo, max int) []peer.PeerInfo {
	n := math2.IntMin(max, len(in))
	var out []peer.PeerInfo
	for _, val := range rand.Perm(n) {
		out = append(out, in[val])
	}
	return out
}
<commit_msg>adjust const<commit_after>package core

import (
	"errors"
	"fmt"
	"math/rand"
	"sync"
	"time"

	config "github.com/jbenet/go-ipfs/config"
	host "github.com/jbenet/go-ipfs/p2p/host"
	inet "github.com/jbenet/go-ipfs/p2p/net"
	peer "github.com/jbenet/go-ipfs/p2p/peer"
	dht "github.com/jbenet/go-ipfs/routing/dht"
	lgbl "github.com/jbenet/go-ipfs/util/eventlog/loggables"
	math2 "github.com/jbenet/go-ipfs/util/math2"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
	ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"
)

const (
	period                               = 30 * time.Second // how often to check connection status
	connectiontimeout      time.Duration = period / 3       // duration to wait when attempting to connect
	recoveryThreshold                    = 4                // attempt to bootstrap if connection count falls below this value
	numDHTBootstrapQueries               = 15               // number of DHT queries to execute
)

func superviseConnections(parent context.Context,
	h host.Host,
	route *dht.IpfsDHT, // TODO depend on abstract interface for testing purposes
	store peer.Peerstore,
	peers []*config.BootstrapPeer) error {

	for {
		ctx, _ := context.WithTimeout(parent, connectiontimeout)
		// TODO get config from disk so |peers| always reflects the latest
		// information
		if err := bootstrap(ctx, h, route, store, peers); err != nil {
			log.Error(err)
		}
		select {
		case <-parent.Done():
			return parent.Err()
		case <-time.Tick(period):
		}
	}
	return nil
}

func bootstrap(ctx context.Context,
	h host.Host,
	r *dht.IpfsDHT,
	ps peer.Peerstore,
	boots []*config.BootstrapPeer) error {

	connectedPeers := h.Network().Peers()
	if len(connectedPeers) >= recoveryThreshold {
		log.Event(ctx, "bootstrapSkip", h.ID())
		log.Debugf("%s bootstrap skipped -- connected to %d (> %d) nodes",
			h.ID(), len(connectedPeers), recoveryThreshold)

		return nil
	}
	numCxnsToCreate := recoveryThreshold - len(connectedPeers)

	log.Event(ctx, "bootstrapStart", h.ID())
	log.Debugf("%s bootstrapping to %d more nodes", h.ID(), numCxnsToCreate)

	var bootstrapPeers []peer.PeerInfo
	for _, bootstrap := range boots {
		p, err := toPeer(bootstrap)
		if err != nil {
			log.Event(ctx, "bootstrapError", h.ID(), lgbl.Error(err))
			log.Errorf("%s bootstrap error: %s", h.ID(), err)
			return err
		}
		bootstrapPeers = append(bootstrapPeers, p)
	}

	var notConnected []peer.PeerInfo
	for _, p := range bootstrapPeers {
		if h.Network().Connectedness(p.ID) != inet.Connected {
			notConnected = append(notConnected, p)
		}
	}

	if len(notConnected) < 1 {
		s := "must bootstrap to %d more nodes, but already connected to all candidates"
		err := fmt.Errorf(s, numCxnsToCreate)
		log.Event(ctx, "bootstrapError", h.ID(), lgbl.Error(err))
		log.Errorf("%s bootstrap error: %s", h.ID(), err)
		return err
	}

	var randomSubset = randomSubsetOfPeers(notConnected, numCxnsToCreate)

	log.Debugf("%s bootstrapping to %d nodes: %s", h.ID(), numCxnsToCreate, randomSubset)
	if err := connect(ctx, ps, r, randomSubset); err != nil {
		log.Event(ctx, "bootstrapError", h.ID(), lgbl.Error(err))
		log.Errorf("%s bootstrap error: %s", h.ID(), err)
		return err
	}
	return nil
}

func connect(ctx context.Context, ps peer.Peerstore, r *dht.IpfsDHT, peers []peer.PeerInfo) error {
	if len(peers) < 1 {
		return errors.New("bootstrap set empty")
	}

	var wg sync.WaitGroup
	for _, p := range peers {

		// performed asynchronously because when performed synchronously, if
		// one `Connect` call hangs, subsequent calls are more likely to
		// fail/abort due to an expiring context.

		wg.Add(1)
		go func(p peer.PeerInfo) {
			defer wg.Done()
			log.Event(ctx, "bootstrapDial", r.LocalPeer(), p.ID)
			log.Debugf("%s bootstrapping to %s", r.LocalPeer(), p.ID)

			ps.AddAddresses(p.ID, p.Addrs)
			err := r.Connect(ctx, p.ID)
			if err != nil {
				log.Event(ctx, "bootstrapFailed", p.ID)
				log.Criticalf("failed to bootstrap with %v", p.ID)
				return
			}
			log.Event(ctx, "bootstrapSuccess", p.ID)
			log.Infof("bootstrapped with %v", p.ID)
		}(p)
	}
	wg.Wait()
	if err := r.Bootstrap(ctx, numDHTBootstrapQueries); err != nil {
		return err
	}
	return nil
}

func toPeer(bootstrap *config.BootstrapPeer) (p peer.PeerInfo, err error) {
	id, err := peer.IDB58Decode(bootstrap.PeerID)
	if err != nil {
		return
	}
	maddr, err := ma.NewMultiaddr(bootstrap.Address)
	if err != nil {
		return
	}
	p = peer.PeerInfo{
		ID:    id,
		Addrs: []ma.Multiaddr{maddr},
	}
	return
}

func randomSubsetOfPeers(in []peer.PeerInfo, max int) []peer.PeerInfo {
	n := math2.IntMin(max, len(in))
	var out []peer.PeerInfo
	for _, val := range rand.Perm(n) {
		out = append(out, in[val])
	}
	return out
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/APTrust/exchange/context"
	dpn_models "github.com/APTrust/exchange/dpn/models"
	"github.com/APTrust/exchange/dpn/network"
	"github.com/APTrust/exchange/models"
	"net/url"
	"os"
	"time"
)

var InstitutionIdMap map[string]int

// dpn_sync syncs data in our local DPN registry by pulling data about
// bags, replication requests, etc. from other nodes. See printUsage().
func main() {
	pathToConfigFile := parseCommandLine()
	config, err := models.LoadConfigFile(pathToConfigFile)
	if err != nil {
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}
	_context := context.NewContext(config)
	err = initInstitutionIdMap(_context)
	if err != nil {
		_context.MessageLog.Error(err.Error())
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}
	err = syncToPharos(_context)
	if err != nil {
		_context.MessageLog.Error(err.Error())
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}
}

func initInstitutionIdMap(ctx *context.Context) error {
	params := url.Values{}
	params.Add("page", "1")
	params.Add("per_page", "100")
	resp := ctx.PharosClient.InstitutionList(params)
	ctx.MessageLog.Info(resp.Request.URL.Opaque)
	if resp.Error != nil {
		return resp.Error
	}
	InstitutionIdMap = make(map[string]int)
	for _, inst := range resp.Institutions() {
		if inst.DPNUUID != "" {
			InstitutionIdMap[inst.DPNUUID] = inst.Id
		}
	}
	return nil
}

// getLatestTimestamp returns the latest UpdatedAt timestamp
// from the DPN bags table in Pharos.
func getLatestTimestamp(ctx *context.Context) (time.Time, error) {
	params := url.Values{}
	params.Add("sort", "dpn_upated_at DESC")
	resp := ctx.PharosClient.DPNBagList(params)
	ctx.MessageLog.Info(resp.Request.URL.Opaque)
	if resp.Error != nil {
		return time.Time{}, resp.Error
	}
	return resp.DPNBag().DPNUpdatedAt, nil
}

func syncToPharos(ctx *context.Context) error {
	timestamp, err := getLatestTimestamp(ctx)
	if err != nil {
		return err
	}
	dpnClient, err := network.NewDPNRestClient(
		ctx.Config.DPN.RestClient.LocalServiceURL,
		ctx.Config.DPN.DPNAPIVersion,
		ctx.Config.DPN.RestClient.LocalAuthToken,
		ctx.Config.DPN.LocalNode,
		ctx.Config.DPN)
	if err != nil {
		return fmt.Errorf("Error creating local DPN REST client: %v", err)
	}
	params := url.Values{}
	params.Add("after", timestamp.Format(time.RFC3339))
	params.Add("ingest_node", ctx.Config.DPN.LocalNode) // only bags we ingested
	params.Add("page", "1")
	params.Add("page_size", "100")

	for {
		resp := dpnClient.DPNBagList(params)
		ctx.MessageLog.Info(resp.Request.URL.Opaque)
		if resp.Error != nil {
			return resp.Error
		}
		for _, dpnBag := range resp.Bags() {
			// Quit early if this happens. It shouldn't.
			if InstitutionIdMap[dpnBag.Member] == 0 {
				return fmt.Errorf("Pharos has no institution record for DPN member %s",
					dpnBag.Member)
			}
			existingBag := getExistingPharosDPNBag(ctx, dpnBag.UUID)
			existingBagId := 0
			if existingBag != nil {
				existingBagId = existingBag.Id
			}
			updatedBag := convertToPharos(dpnBag, existingBagId)
			saveResponse := ctx.PharosClient.DPNBagSave(updatedBag)
			if saveResponse.Error != nil {
				ctx.MessageLog.Error("Error saving DPN Bag %s to Pharos: %v",
					dpnBag.UUID, saveResponse.Error)
			} else {
				ctx.MessageLog.Info("Saved DPN Bag %s with id %d",
					dpnBag.UUID, saveResponse.DPNBag().Id)
			}
		}
		if !resp.HasNextPage() {
			break
		} else {
			params = resp.ParamsForNextPage()
		}
	}

	return nil
}

func convertToPharos(dpnBag *dpn_models.DPNBag, existingBagId int) *models.PharosDPNBag {
	pharosDPNBag := &models.PharosDPNBag{
		Id: existingBagId,
	}
	pharosDPNBag.InstitutionId = InstitutionIdMap[dpnBag.Member]
	pharosDPNBag.ObjectIdentifier = dpnBag.LocalId
	pharosDPNBag.DPNIdentifier = dpnBag.UUID
	pharosDPNBag.DPNSize = dpnBag.Size
	if len(dpnBag.ReplicatingNodes) > 2 {
		pharosDPNBag.Node3 = dpnBag.ReplicatingNodes[2]
	}
	if len(dpnBag.ReplicatingNodes) > 1 {
		pharosDPNBag.Node2 = dpnBag.ReplicatingNodes[1]
	}
	if len(dpnBag.ReplicatingNodes) > 0 {
		pharosDPNBag.Node1 = dpnBag.ReplicatingNodes[0]
	}
	pharosDPNBag.DPNCreatedAt = dpnBag.CreatedAt
	pharosDPNBag.DPNUpdatedAt = dpnBag.UpdatedAt
	return pharosDPNBag
}

func getExistingPharosDPNBag(ctx *context.Context, dpnUUID string) *models.PharosDPNBag {
	params := url.Values{}
	params.Add("dpn_identifier", dpnUUID)
	params.Add("page", "1")
	params.Add("page_size", "10")
	resp := ctx.PharosClient.DPNBagList(params)
	ctx.MessageLog.Info(resp.Request.URL.Opaque)
	if resp.Error != nil {
		// Quit here, so we don't corrupt the DPNBags table.
		// We don't want to insert a bag with a DPN UUID that's already
		// in the table.
		ctx.MessageLog.Error(resp.Error.Error())
		fmt.Fprintf(os.Stderr, resp.Error.Error())
		os.Exit(1)
	}
	count := len(resp.DPNBags())
	if count > 1 {
		// Again: quit. Duplicate records need to be fixed
		// before we proceed.
		ctx.MessageLog.Error("Fatal: Found %d records for DPN Bag %s", count, dpnUUID)
		os.Exit(1)
	}
	if count == 0 {
		return nil
	}
	return resp.DPNBag()
}

// See if you can figure out from the function name what this does.
func parseCommandLine() (configFile string) {
	var pathToConfigFile string
	flag.StringVar(&pathToConfigFile, "config", "", "Path to APTrust config file")
	flag.Parse()
	if pathToConfigFile == "" {
		printUsage()
		os.Exit(1)
	}
	return pathToConfigFile
}

// Tell the user about the program.
func printUsage() {
	message := `
dpn_pharos_sync syncs data from our local DPN registry to Pharos.

Usage: dpn_sync -config=<absolute path to APTrust config file>

Param -config is required.
`
	fmt.Println(message)
}
<commit_msg>Fixed sort param, improved logging<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/APTrust/exchange/context"
	dpn_models "github.com/APTrust/exchange/dpn/models"
	"github.com/APTrust/exchange/dpn/network"
	"github.com/APTrust/exchange/models"
	"net/url"
	"os"
	"time"
)

var InstitutionIdMap map[string]int

// dpn_sync syncs data in our local DPN registry by pulling data about
// bags, replication requests, etc. from other nodes. See printUsage().
func main() {
	pathToConfigFile := parseCommandLine()
	config, err := models.LoadConfigFile(pathToConfigFile)
	if err != nil {
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}
	_context := context.NewContext(config)
	err = initInstitutionIdMap(_context)
	if err != nil {
		_context.MessageLog.Error(err.Error())
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}
	err = syncToPharos(_context)
	if err != nil {
		_context.MessageLog.Error(err.Error())
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}
}

func initInstitutionIdMap(ctx *context.Context) error {
	ctx.MessageLog.Info("Caching institutions")
	params := url.Values{}
	params.Add("page", "1")
	params.Add("per_page", "100")
	resp := ctx.PharosClient.InstitutionList(params)
	ctx.MessageLog.Info(resp.Request.URL.Opaque)
	if resp.Error != nil {
		return resp.Error
	}
	InstitutionIdMap = make(map[string]int)
	for _, inst := range resp.Institutions() {
		if inst.DPNUUID != "" {
			InstitutionIdMap[inst.DPNUUID] = inst.Id
		}
	}
	return nil
}

// getLatestTimestamp returns the latest UpdatedAt timestamp
// from the DPN bags table in Pharos.
func getLatestTimestamp(ctx *context.Context) (time.Time, error) {
	ctx.MessageLog.Info("Getting latest timestamp from Pharos")
	params := url.Values{}
	params.Add("sort", "dpn_updated_at DESC")
	resp := ctx.PharosClient.DPNBagList(params)
	ctx.MessageLog.Info(resp.Request.URL.Opaque)
	if resp.Error != nil {
		return time.Time{}, resp.Error
	}
	return resp.DPNBag().DPNUpdatedAt, nil
}

func syncToPharos(ctx *context.Context) error {
	timestamp, err := getLatestTimestamp(ctx)
	if err != nil {
		return err
	}
	ctx.MessageLog.Info("Most recent DPN bag has update timestamp of %s", timestamp.Format(time.RFC3339))

	dpnClient, err := network.NewDPNRestClient(
		ctx.Config.DPN.RestClient.LocalServiceURL,
		ctx.Config.DPN.DPNAPIVersion,
		ctx.Config.DPN.RestClient.LocalAuthToken,
		ctx.Config.DPN.LocalNode,
		ctx.Config.DPN)
	if err != nil {
		return fmt.Errorf("Error creating local DPN REST client: %v", err)
	}
	ctx.MessageLog.Info("Set up DPN client for %s", ctx.Config.DPN.RestClient.LocalServiceURL)

	params := url.Values{}
	params.Add("after", timestamp.Format(time.RFC3339))
	params.Add("ingest_node", ctx.Config.DPN.LocalNode) // only bags we ingested
	params.Add("page", "1")
	params.Add("page_size", "100")

	ctx.MessageLog.Info("Checking for bags updated since %s", timestamp.Format(time.RFC3339))
	for {
		resp := dpnClient.DPNBagList(params)
		ctx.MessageLog.Info("%s", resp.Request.URL.String())

		// VERBOSE LOGGING
		// ctx.MessageLog.Info("%v", resp.Request)
		// ctx.MessageLog.Info("%v", resp.Response)
		// data, _ := resp.RawResponseData()
		// ctx.MessageLog.Info(string(data))
		// END VERBOSE LOGGING

		if resp.Error != nil {
			return resp.Error
		}
		ctx.MessageLog.Info("Request returned %d bags", len(resp.Bags()))
		for _, dpnBag := range resp.Bags() {
			// Quit early if this happens. It shouldn't.
			if InstitutionIdMap[dpnBag.Member] == 0 {
				return fmt.Errorf("Pharos has no institution record for DPN member %s",
					dpnBag.Member)
			}
			existingBag := getExistingPharosDPNBag(ctx, dpnBag.UUID)
			existingBagId := 0
			if existingBag != nil {
				existingBagId = existingBag.Id
			}
			updatedBag := convertToPharos(dpnBag, existingBagId)
			saveResponse := ctx.PharosClient.DPNBagSave(updatedBag)
			if saveResponse.Error != nil {
				ctx.MessageLog.Error("Error saving DPN Bag %s to Pharos: %v",
					dpnBag.UUID, saveResponse.Error)
			} else {
				ctx.MessageLog.Info("Saved DPN Bag %s with id %d",
					dpnBag.UUID, saveResponse.DPNBag().Id)
			}
		}
		if !resp.HasNextPage() {
			break
		} else {
			params = resp.ParamsForNextPage()
		}
	}

	return nil
}

func convertToPharos(dpnBag *dpn_models.DPNBag, existingBagId int) *models.PharosDPNBag {
	pharosDPNBag := &models.PharosDPNBag{
		Id: existingBagId,
	}
	pharosDPNBag.InstitutionId = InstitutionIdMap[dpnBag.Member]
	pharosDPNBag.ObjectIdentifier = dpnBag.LocalId
	pharosDPNBag.DPNIdentifier = dpnBag.UUID
	pharosDPNBag.DPNSize = dpnBag.Size
	if len(dpnBag.ReplicatingNodes) > 2 {
		pharosDPNBag.Node3 = dpnBag.ReplicatingNodes[2]
	}
	if len(dpnBag.ReplicatingNodes) > 1 {
		pharosDPNBag.Node2 = dpnBag.ReplicatingNodes[1]
	}
	if len(dpnBag.ReplicatingNodes) > 0 {
		pharosDPNBag.Node1 = dpnBag.ReplicatingNodes[0]
	}
	pharosDPNBag.DPNCreatedAt = dpnBag.CreatedAt
	pharosDPNBag.DPNUpdatedAt = dpnBag.UpdatedAt
	return pharosDPNBag
}

func getExistingPharosDPNBag(ctx *context.Context, dpnUUID string) *models.PharosDPNBag {
	params := url.Values{}
	params.Add("dpn_identifier", dpnUUID)
	params.Add("page", "1")
	params.Add("page_size", "10")
	resp := ctx.PharosClient.DPNBagList(params)
	ctx.MessageLog.Info(resp.Request.URL.Opaque)
	if resp.Error != nil {
		// Quit here, so we don't corrupt the DPNBags table.
		// We don't want to insert a bag with a DPN UUID that's already
		// in the table.
		ctx.MessageLog.Error(resp.Error.Error())
		fmt.Fprintf(os.Stderr, resp.Error.Error())
		os.Exit(1)
	}
	count := len(resp.DPNBags())
	if count > 1 {
		// Again: quit. Duplicate records need to be fixed
		// before we proceed.
		ctx.MessageLog.Error("Fatal: Found %d records for DPN Bag %s", count, dpnUUID)
		os.Exit(1)
	}
	if count == 0 {
		return nil
	}
	return resp.DPNBag()
}

// See if you can figure out from the function name what this does.
func parseCommandLine() (configFile string) {
	var pathToConfigFile string
	flag.StringVar(&pathToConfigFile, "config", "", "Path to APTrust config file")
	flag.Parse()
	if pathToConfigFile == "" {
		printUsage()
		os.Exit(1)
	}
	return pathToConfigFile
}

// Tell the user about the program.
func printUsage() {
	message := `
dpn_pharos_sync syncs data from our local DPN registry to Pharos.

Usage: dpn_sync -config=<absolute path to APTrust config file>

Param -config is required.
`
	fmt.Println(message)
}
<|endoftext|>


<commit_before>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package net

// parsePort parses service as a decimal interger and returns the
// corresponding value as port. It is the caller's responsibility to
// parse service as a non-decimal integer when needsLookup is true.
//
// Some system resolvers will return a valid port number when given a number
// over 65536 (see https://golang.org/issues/11715). Alas, the parser
// can't bail early on numbers > 65536. Therefore reasonably large/small
// numbers are parsed in full and rejected if invalid.
func parsePort(service string) (port int, needsLookup bool) {
	if service == "" {
		// Lock in the legacy behavior that an empty string
		// means port 0. See golang.org/issue/13610.
		return 0, false
	}
	const (
		max    = uint32(1<<32 - 1)
		cutoff = uint32(1 << 30)
	)
	neg := false
	if service[0] == '+' {
		service = service[1:]
	} else if service[0] == '-' {
		neg = true
		service = service[1:]
	}
	var n uint32
	for _, d := range service {
		if '0' <= d && d <= '9' {
			d -= '0'
		} else {
			return 0, true
		}
		if n >= cutoff {
			n = max
			break
		}
		n *= 10
		nn := n + uint32(d)
		if nn < n || nn > max {
			n = max
			break
		}
		n = nn
	}
	if !neg && n >= cutoff {
		port = int(cutoff - 1)
	} else if neg && n > cutoff {
		port = int(cutoff)
	} else {
		port = int(n)
	}
	if neg {
		port = -port
	}
	return port, false
}
<commit_msg>net: fix typo in parsePort documentation<commit_after>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package net

// parsePort parses service as a decimal integer and returns the
// corresponding value as port. It is the caller's responsibility to
// parse service as a non-decimal integer when needsLookup is true.
//
// Some system resolvers will return a valid port number when given a number
// over 65536 (see https://golang.org/issues/11715). Alas, the parser
// can't bail early on numbers > 65536. Therefore reasonably large/small
// numbers are parsed in full and rejected if invalid.
func parsePort(service string) (port int, needsLookup bool) {
	if service == "" {
		// Lock in the legacy behavior that an empty string
		// means port 0. See golang.org/issue/13610.
		return 0, false
	}
	const (
		max    = uint32(1<<32 - 1)
		cutoff = uint32(1 << 30)
	)
	neg := false
	if service[0] == '+' {
		service = service[1:]
	} else if service[0] == '-' {
		neg = true
		service = service[1:]
	}
	var n uint32
	for _, d := range service {
		if '0' <= d && d <= '9' {
			d -= '0'
		} else {
			return 0, true
		}
		if n >= cutoff {
			n = max
			break
		}
		n *= 10
		nn := n + uint32(d)
		if nn < n || nn > max {
			n = max
			break
		}
		n = nn
	}
	if !neg && n >= cutoff {
		port = int(cutoff - 1)
	} else if neg && n > cutoff {
		port = int(cutoff)
	} else {
		port = int(n)
	}
	if neg {
		port = -port
	}
	return port, false
}
<|endoftext|>


<commit_before>package smtpapi

import (
	"bytes"
	"encoding/json"
	"fmt"
	"unicode/utf16"
)

const Version = "0.4.0"

// SMTPAPIHeader will be used to set up X-SMTPAPI params
type SMTPAPIHeader struct {
	To         []string            `json:"to,omitempty"`
	Sub        map[string][]string `json:"sub,omitempty"`
	Section    map[string]string   `json:"section,omitempty"`
	Category   []string            `json:"category,omitempty"`
	UniqueArgs map[string]string   `json:"unique_args,omitempty"`
	Filters    map[string]Filter   `json:"filters,omitempty"`
	ASMGroupID int                 `json:"asm_group_id,omitempty"`
	SendAt     int64               `json:"send_at,omitempty"`
	SendEachAt []int64             `json:"send_each_at,omitempty"`
	IpPool     string              `json:"ip_pool,omitempty"`
	BatchID    string              `json:"batch_id"`
}

// Filter represents an App/Filter and its settings
type Filter struct {
	Settings map[string]string `json:"settings,omitempty"`
}

// NewSMTPAPIHeader creates a new header struct
func NewSMTPAPIHeader() *SMTPAPIHeader {
	return &SMTPAPIHeader{}
}

// AddTo appends a single email to the To header
func (h *SMTPAPIHeader) AddTo(email string) {
	h.To = append(h.To, email)
}

// AddTos appends multiple emails to the To header
func (h *SMTPAPIHeader) AddTos(emails []string) {
	for i := 0; i < len(emails); i++ {
		h.AddTo(emails[i])
	}
}

// SetTos sets the value of the To header
func (h *SMTPAPIHeader) SetTos(emails []string) {
	h.To = emails
}

// AddSubstitution adds a new substitution to a specific key
func (h *SMTPAPIHeader) AddSubstitution(key, sub string) {
	if h.Sub == nil {
		h.Sub = make(map[string][]string)
	}
	h.Sub[key] = append(h.Sub[key], sub)
}

// AddSubstitutions adds a multiple substitutions to a specific key
func (h *SMTPAPIHeader) AddSubstitutions(key string, subs []string) {
	for i := 0; i < len(subs); i++ {
		h.AddSubstitution(key, subs[i])
	}
}

// SetSubstitutions sets the value of the substitutions on the Sub header
func (h *SMTPAPIHeader) SetSubstitutions(sub map[string][]string) {
	h.Sub = sub
}

// AddSection sets the value for a specific section
func (h *SMTPAPIHeader) AddSection(section, value string) {
	if h.Section == nil {
		h.Section = make(map[string]string)
	}
	h.Section[section] = value
}

// SetSections sets the value for the Section header
func (h *SMTPAPIHeader) SetSections(sections map[string]string) {
	h.Section = sections
}

// AddCategory adds a new category to the Category header
func (h *SMTPAPIHeader) AddCategory(category string) {
	h.Category = append(h.Category, category)
}

// AddCategories adds multiple categories to the Category header
func (h *SMTPAPIHeader) AddCategories(categories []string) {
	for i := 0; i < len(categories); i++ {
		h.AddCategory(categories[i])
	}
}

// SetCategories will set the value of the Categories field
func (h *SMTPAPIHeader) SetCategories(categories []string) {
	h.Category = categories
}

// SetASMGroupID will set the value of the ASMGroupID field
func (h *SMTPAPIHeader) SetASMGroupID(groupID int) {
	h.ASMGroupID = groupID
}

// AddUniqueArg will set the value of a specific argument
func (h *SMTPAPIHeader) AddUniqueArg(arg, value string) {
	if h.UniqueArgs == nil {
		h.UniqueArgs = make(map[string]string)
	}
	h.UniqueArgs[arg] = value
}

// SetUniqueArgs will set the value of the Unique_args header
func (h *SMTPAPIHeader) SetUniqueArgs(args map[string]string) {
	h.UniqueArgs = args
}

// AddFilter will set the specific setting for a filter
func (h *SMTPAPIHeader) AddFilter(filter, setting, value string) {
	if h.Filters == nil {
		h.Filters = make(map[string]Filter)
	}
	if _, ok := h.Filters[filter]; !ok {
		h.Filters[filter] = Filter{
			Settings: make(map[string]string),
		}
	}
	h.Filters[filter].Settings[setting] = value
}

// SetFilter takes in a Filter struct with predetermined settings and sets it for such Filter key
func (h *SMTPAPIHeader) SetFilter(filter string, value *Filter) {
	if h.Filters == nil {
		h.Filters = make(map[string]Filter)
	}
	h.Filters[filter] = *value
}

// SetSendAt takes in a timestamp which determines when the email will be sent
func (h *SMTPAPIHeader) SetSendAt(sendAt int64) {
	h.SendAt = sendAt
}

// AddSendEachAt takes in a timestamp and pushes it into a list Must match length of To emails
func (h *SMTPAPIHeader) AddSendEachAt(sendEachAt int64) {
	h.SendEachAt = append(h.SendEachAt, sendEachAt)
}

// SetSendEachAt takes an array of timestamps. Must match length of To emails
func (h *SMTPAPIHeader) SetSendEachAt(sendEachAt []int64) {
	h.SendEachAt = sendEachAt
}

// SetIpPool takes a strings and sets the IpPool field
func (h *SMTPAPIHeader) SetIpPool(ipPool string) {
	h.IpPool = ipPool
}

// Unicode escape
func escapeUnicode(input string) string {
	//var buffer bytes.Buffer
	buffer := bytes.NewBufferString("")
	for _, r := range input {
		if r > 65535 {
			// surrogate pair
			var r1, r2 = utf16.EncodeRune(r)
			var s = fmt.Sprintf("\\u%x\\u%x", r1, r2)
			buffer.WriteString(s)
		} else if r > 127 {
			var s = fmt.Sprintf("\\u%04x", r)
			buffer.WriteString(s)
		} else {
			var s = fmt.Sprintf("%c", r)
			buffer.WriteString(s)
		}
	}
	return buffer.String()
}

// JSONString returns the representation of the Header
func (h *SMTPAPIHeader) JSONString() (string, error) {
	headers, e := json.Marshal(h)
	return escapeUnicode(string(headers)), e
}
<commit_msg>Update smtpapi.go<commit_after>package smtpapi

import (
	"bytes"
	"encoding/json"
	"fmt"
	"unicode/utf16"
)

const Version = "0.4.0"

// SMTPAPIHeader will be used to set up X-SMTPAPI params
type SMTPAPIHeader struct {
	To         []string            `json:"to,omitempty"`
	Sub        map[string][]string `json:"sub,omitempty"`
	Section    map[string]string   `json:"section,omitempty"`
	Category   []string            `json:"category,omitempty"`
	UniqueArgs map[string]string   `json:"unique_args,omitempty"`
	Filters    map[string]Filter   `json:"filters,omitempty"`
	ASMGroupID int                 `json:"asm_group_id,omitempty"`
	SendAt     int64               `json:"send_at,omitempty"`
	SendEachAt []int64             `json:"send_each_at,omitempty"`
	IpPool     string              `json:"ip_pool,omitempty"`
	BatchID    string              `json:"batch_id,omitempty"`
}

// Filter represents an App/Filter and its settings
type Filter struct {
	Settings map[string]string `json:"settings,omitempty"`
}

// NewSMTPAPIHeader creates a new header struct
func NewSMTPAPIHeader() *SMTPAPIHeader {
	return &SMTPAPIHeader{}
}

// AddTo appends a single email to the To header
func (h *SMTPAPIHeader) AddTo(email string) {
	h.To = append(h.To, email)
}

// AddTos appends multiple emails to the To header
func (h *SMTPAPIHeader) AddTos(emails []string) {
	for i := 0; i < len(emails); i++ {
		h.AddTo(emails[i])
	}
}

// SetTos sets the value of the To header
func (h *SMTPAPIHeader) SetTos(emails []string) {
	h.To = emails
}

// AddSubstitution adds a new substitution to a specific key
func (h *SMTPAPIHeader) AddSubstitution(key, sub string) {
	if h.Sub == nil {
		h.Sub = make(map[string][]string)
	}
	h.Sub[key] = append(h.Sub[key], sub)
}

// AddSubstitutions adds a multiple substitutions to a specific key
func (h *SMTPAPIHeader) AddSubstitutions(key string, subs []string) {
	for i := 0; i < len(subs); i++ {
		h.AddSubstitution(key, subs[i])
	}
}

// SetSubstitutions sets the value of the substitutions on the Sub header
func (h *SMTPAPIHeader) SetSubstitutions(sub map[string][]string) {
	h.Sub = sub
}

// AddSection sets the value for a specific section
func (h *SMTPAPIHeader) AddSection(section, value string) {
	if h.Section == nil {
		h.Section = make(map[string]string)
	}
	h.Section[section] = value
}

// SetSections sets the value for the Section header
func (h *SMTPAPIHeader) SetSections(sections map[string]string) {
	h.Section = sections
}

// AddCategory adds a new category to the Category header
func (h *SMTPAPIHeader) AddCategory(category string) {
	h.Category = append(h.Category, category)
}

// AddCategories adds multiple categories to the Category header
func (h *SMTPAPIHeader) AddCategories(categories []string) {
	for i := 0; i < len(categories); i++ {
		h.AddCategory(categories[i])
	}
}

// SetCategories will set the value of the Categories field
func (h *SMTPAPIHeader) SetCategories(categories []string) {
	h.Category = categories
}

// SetASMGroupID will set the value of the ASMGroupID field
func (h *SMTPAPIHeader) SetASMGroupID(groupID int) {
	h.ASMGroupID = groupID
}

// AddUniqueArg will set the value of a specific argument
func (h *SMTPAPIHeader) AddUniqueArg(arg, value string) {
	if h.UniqueArgs == nil {
		h.UniqueArgs = make(map[string]string)
	}
	h.UniqueArgs[arg] = value
}

// SetUniqueArgs will set the value of the Unique_args header
func (h *SMTPAPIHeader) SetUniqueArgs(args map[string]string) {
	h.UniqueArgs = args
}

// AddFilter will set the specific setting for a filter
func (h *SMTPAPIHeader) AddFilter(filter, setting, value string) {
	if h.Filters == nil {
		h.Filters = make(map[string]Filter)
	}
	if _, ok := h.Filters[filter]; !ok {
		h.Filters[filter] = Filter{
			Settings: make(map[string]string),
		}
	}
	h.Filters[filter].Settings[setting] = value
}

// SetFilter takes in a Filter struct with predetermined settings and sets it for such Filter key
func (h *SMTPAPIHeader) SetFilter(filter string, value *Filter) {
	if h.Filters == nil {
		h.Filters = make(map[string]Filter)
	}
	h.Filters[filter] = *value
}

// SetSendAt takes in a timestamp which determines when the email will be sent
func (h *SMTPAPIHeader) SetSendAt(sendAt int64) {
	h.SendAt = sendAt
}

// AddSendEachAt takes in a timestamp and pushes it into a list Must match length of To emails
func (h *SMTPAPIHeader) AddSendEachAt(sendEachAt int64) {
	h.SendEachAt = append(h.SendEachAt, sendEachAt)
}

// SetSendEachAt takes an array of timestamps. Must match length of To emails
func (h *SMTPAPIHeader) SetSendEachAt(sendEachAt []int64) {
	h.SendEachAt = sendEachAt
}

// SetIpPool takes a strings and sets the IpPool field
func (h *SMTPAPIHeader) SetIpPool(ipPool string) {
	h.IpPool = ipPool
}

// Unicode escape
func escapeUnicode(input string) string {
	//var buffer bytes.Buffer
	buffer := bytes.NewBufferString("")
	for _, r := range input {
		if r > 65535 {
			// surrogate pair
			var r1, r2 = utf16.EncodeRune(r)
			var s = fmt.Sprintf("\\u%x\\u%x", r1, r2)
			buffer.WriteString(s)
		} else if r > 127 {
			var s = fmt.Sprintf("\\u%04x", r)
			buffer.WriteString(s)
		} else {
			var s = fmt.Sprintf("%c", r)
			buffer.WriteString(s)
		}
	}
	return buffer.String()
}

// JSONString returns the representation of the Header
func (h *SMTPAPIHeader) JSONString() (string, error) {
	headers, e := json.Marshal(h)
	return escapeUnicode(string(headers)), e
}
<|endoftext|>


<commit_before>package sched

import (
	"bytes"
	"encoding/gob"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"os"
	"sync"
	"time"

	"github.com/StackExchange/scollector/opentsdb"
	"github.com/StackExchange/tsaf/conf"
	"github.com/StackExchange/tsaf/expr"
)

type Schedule struct {
	sync.Mutex

	Conf          *conf.Conf
	Freq          time.Duration
	Status        map[AlertKey]*State
	Notifications map[AlertKey]map[string]time.Time

	cache     *opentsdb.Cache
	runStates map[AlertKey]Status
	nc        chan interface{}
}

func (s *Schedule) MarshalJSON() ([]byte, error) {
	t := struct {
		Alerts map[string]*conf.Alert
		Freq   time.Duration
		Status map[string]*State
	}{
		s.Conf.Alerts,
		s.Freq,
		make(map[string]*State),
	}
	for k, v := range s.Status {
		if v.Last().Status < ST_WARN {
			continue
		}
		t.Status[k.String()] = v
	}
	return json.Marshal(&t)
}

var DefaultSched = &Schedule{
	Freq: time.Minute * 5,
}

// Loads a configuration into the default schedule
func Load(c *conf.Conf) {
	DefaultSched.Load(c)
}

// Runs the default schedule.
func Run() error {
	return DefaultSched.Run()
}

func (s *Schedule) Load(c *conf.Conf) {
	s.Conf = c
	s.RestoreState()
}

// Restores notification and alert state from the file on disk.
func (s *Schedule) RestoreState() {
	s.Lock()
	defer s.Unlock()
	s.cache = opentsdb.NewCache(s.Conf.TsdbHost)
	s.Notifications = nil
	s.Status = make(map[AlertKey]*State)
	f, err := os.Open(s.Conf.StateFile)
	if err != nil {
		log.Println(err)
		return
	}
	dec := gob.NewDecoder(f)
	notifications := make(map[AlertKey]map[string]time.Time)
	if err := dec.Decode(&notifications); err != nil {
		log.Println(err)
		return
	}
	for ak, ns := range notifications {
		for name, t := range ns {
			n, present := s.Conf.Notifications[name]
			if !present {
				log.Println("sched: notification not present during restore:", name)
				continue
			}
			_, present = s.Conf.Alerts[ak.Name]
			if !present {
				log.Println("sched: alert not present during restore:", ak.Name)
				continue
			}
			s.AddNotification(ak, n, t)
		}
	}
	for {
		var ak AlertKey
		var st State
		if err := dec.Decode(&ak); err == io.EOF {
			break
		} else if err != nil {
			log.Println(err)
			return
		}
		if err := dec.Decode(&st); err != nil {
			log.Println(err)
			return
		}
		if _, present := s.Conf.Alerts[ak.Name]; !present {
			log.Println("sched: alert no longer present, ignoring:", ak)
			continue
		}
		s.Status[ak] = &st
	}
}

func (s *Schedule) Save() {
	s.Lock()
	defer s.Unlock()
	f, err := os.Create(s.Conf.StateFile)
	if err != nil {
		log.Println(err)
		return
	}
	enc := gob.NewEncoder(f)
	if err := enc.Encode(s.Notifications); err != nil {
		log.Println(err)
		return
	}
	for k, v := range s.Status {
		enc.Encode(k)
		enc.Encode(v)
	}
	if err := f.Close(); err != nil {
		log.Println(err)
		return
	}
	log.Println("sched: wrote state to", s.Conf.StateFile)
}

func (s *Schedule) Run() error {
	go func() {
		for {
			time.Sleep(time.Minute)
			s.Save()
		}
	}()
	s.nc = make(chan interface{}, 1)
	go s.Poll()
	for {
		wait := time.After(s.Freq)
		if s.Freq < time.Second {
			return fmt.Errorf("sched: frequency must be > 1 second")
		}
		if s.Conf == nil {
			return fmt.Errorf("sched: nil configuration")
		}
		start := time.Now()
		s.Check()
		fmt.Printf("run at %v took %v\n", start, time.Since(start))
		<-wait
	}
}

// Poll dispatches notification checks when needed.
func (s *Schedule) Poll() {
	var timeout time.Duration
	for {
		// Wait for one of these two.
		select {
		case <-time.After(timeout):
		case <-s.nc:
		}
		timeout = s.CheckNotifications()
	}
}

// CheckNotifications processes past notification events. It returns the
func (s *Schedule) CheckNotifications() time.Duration {
	s.Lock()
	defer s.Unlock()
	timeout := time.Hour
	notifications := s.Notifications
	s.Notifications = nil
	for ak, ns := range notifications {
		for name, t := range ns {
			n, present := s.Conf.Notifications[name]
			if !present {
				continue
			}
			remaining := t.Add(n.Timeout).Sub(time.Now())
			if remaining > 0 {
				if remaining < timeout {
					timeout = remaining
				}
				s.AddNotification(ak, n, t)
				continue
			}
			st, present := s.Status[ak]
			if !present {
				continue
			}
			a, present := s.Conf.Alerts[ak.Name]
			if !present {
				continue
			}
			s.Notify(st, a, n)
			if n.Timeout < timeout {
				timeout = n.Timeout
			}
		}
	}
	return timeout
}

func (s *Schedule) Check() {
	s.Lock()
	defer s.Unlock()
	s.runStates = make(map[AlertKey]Status)
	s.cache = opentsdb.NewCache(s.Conf.TsdbHost)
	for _, a := range s.Conf.Alerts {
		s.CheckAlert(a)
	}
	changed := false
	for ak, status := range s.runStates {
		state := s.Status[ak]
		change := state.Append(status)
		if change {
			changed = true
		}
		if status > ST_NORM {
			var subject = new(bytes.Buffer)
			a := s.Conf.Alerts[ak.Name]
			if err := s.ExecuteSubject(subject, a, state); err != nil {
				log.Println(err)
			}
			state.Subject = subject.String()
			if change {
				notify := func(notifications map[string]*conf.Notification) {
					for _, n := range notifications {
						s.Notify(state, a, n)
					}
				}
				switch status {
				case ST_CRIT:
					notify(a.CritNotification)
				case ST_WARN:
					notify(a.WarnNotification)
				}
			}
		}
	}
	if changed {
		s.nc <- true
	}
}

func (s *Schedule) CheckAlert(a *conf.Alert) {
	crits := s.CheckExpr(a, a.Crit, true, nil)
	s.CheckExpr(a, a.Warn, false, crits)
}

func (s *Schedule) CheckExpr(a *conf.Alert, e *expr.Expr, isCrit bool, ignore []AlertKey) (alerts []AlertKey) {
	if e == nil {
		return
	}
	results, _, err := e.Execute(s.cache, nil)
	if err != nil {
		// todo: do something here?
		log.Println(err)
		return
	}
Loop:
	for _, r := range results {
		if a.Squelched(r.Group) {
			continue
		}
		ak := AlertKey{a.Name, r.Group.String()}
		for _, v := range ignore {
			if ak == v {
				continue Loop
			}
		}
		state := s.Status[ak]
		if state == nil {
			state = &State{
				Group: r.Group,
			}
			s.Status[ak] = state
		}
		state.Computations = r.Computations
		status := ST_NORM
		if r.Value.(expr.Number) != 0 {
			state.Expr = e.String()
			alerts = append(alerts, ak)
			if isCrit {
				status = ST_CRIT
			} else {
				status = ST_WARN
			}
		}
		if status > s.runStates[ak] {
			s.runStates[ak] = status
		}
	}
	return
}

func (s *Schedule) Notify(st *State, a *conf.Alert, n *conf.Notification) {
	if len(n.Email) > 0 {
		go s.Email(a, n, st)
	}
	if n.Post != nil {
		go s.Post(a, n, st)
	}
	if n.Get != nil {
		go s.Get(a, n, st)
	}
	if n.Print {
		go s.Print(a, n, st)
	}
	if n.Next == nil {
		return
	}
	s.AddNotification(AlertKey{Name: a.Name, Group: st.Group.String()}, n, time.Now().UTC())
}

func (s *Schedule) AddNotification(ak AlertKey, n *conf.Notification, started time.Time) {
	if s.Notifications == nil {
		s.Notifications = make(map[AlertKey]map[string]time.Time)
	}
	if s.Notifications[ak] == nil {
		s.Notifications[ak] = make(map[string]time.Time)
	}
	stn := s.Notifications[ak]
	// Prevent duplicate notifications restarting each other.
	if _, present := stn[n.Name]; !present {
		stn[n.Name] = started
	}
}

type AlertKey struct {
	Name  string
	Group string
}

func (a AlertKey) String() string {
	return a.Name + a.Group
}

type State struct {
	// Most recent event last.
	History      []Event
	Touched      time.Time
	Expr         string
	Group        opentsdb.TagSet
	Computations expr.Computations
	Subject      string
}

func (s *Schedule) Acknowledge(ak AlertKey) {
	s.Lock()
	delete(s.Notifications, ak)
	s.Unlock()
}

func (s *State) Touch() {
	s.Touched = time.Now().UTC()
}

// Appends status to the history if the status is different than the latest
// status. Returns true if state was changed.
func (s *State) Append(status Status) bool {
	s.Touch()
	if len(s.History) == 0 || s.Last().Status != status {
		s.History = append(s.History, Event{status, time.Now().UTC()})
		return true
	}
	return false
}

func (s *State) Last() Event {
	if len(s.History) == 0 {
		return Event{}
	}
	return s.History[len(s.History)-1]
}

type Event struct {
	Status Status
	Time   time.Time // embedding this breaks JSON encoding
}

type Status int

const (
	ST_UNKNOWN Status = iota
	ST_NORM
	ST_WARN
	ST_CRIT
)

func (s Status) String() string {
	switch s {
	case ST_NORM:
		return "normal"
	case ST_WARN:
		return "warning"
	case ST_CRIT:
		return "critical"
	default:
		return "unknown"
	}
}
<commit_msg>Better status names and API<commit_after>package sched

import (
	"bytes"
	"encoding/gob"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"os"
	"sync"
	"time"

	"github.com/StackExchange/scollector/opentsdb"
	"github.com/StackExchange/tsaf/conf"
	"github.com/StackExchange/tsaf/expr"
)

type Schedule struct {
	sync.Mutex

	Conf          *conf.Conf
	Freq          time.Duration
	Status        map[AlertKey]*State
	Notifications map[AlertKey]map[string]time.Time

	cache     *opentsdb.Cache
	runStates map[AlertKey]Status
	nc        chan interface{}
}

func (s *Schedule) MarshalJSON() ([]byte, error) {
	t := struct {
		Alerts map[string]*conf.Alert
		Freq   time.Duration
		Status map[string]*State
	}{
		s.Conf.Alerts,
		s.Freq,
		make(map[string]*State),
	}
	for k, v := range s.Status {
		if v.Last().Status < stWarning {
			continue
		}
		t.Status[k.String()] = v
	}
	return json.Marshal(&t)
}

var DefaultSched = &Schedule{
	Freq: time.Minute * 5,
}

// Loads a configuration into the default schedule
func Load(c *conf.Conf) {
	DefaultSched.Load(c)
}

// Runs the default schedule.
func Run() error {
	return DefaultSched.Run()
}

func (s *Schedule) Load(c *conf.Conf) {
	s.Conf = c
	s.RestoreState()
}

// Restores notification and alert state from the file on disk.
func (s *Schedule) RestoreState() {
	s.Lock()
	defer s.Unlock()
	s.cache = opentsdb.NewCache(s.Conf.TsdbHost)
	s.Notifications = nil
	s.Status = make(map[AlertKey]*State)
	f, err := os.Open(s.Conf.StateFile)
	if err != nil {
		log.Println(err)
		return
	}
	dec := gob.NewDecoder(f)
	notifications := make(map[AlertKey]map[string]time.Time)
	if err := dec.Decode(&notifications); err != nil {
		log.Println(err)
		return
	}
	for ak, ns := range notifications {
		for name, t := range ns {
			n, present := s.Conf.Notifications[name]
			if !present {
				log.Println("sched: notification not present during restore:", name)
				continue
			}
			_, present = s.Conf.Alerts[ak.Name]
			if !present {
				log.Println("sched: alert not present during restore:", ak.Name)
				continue
			}
			s.AddNotification(ak, n, t)
		}
	}
	for {
		var ak AlertKey
		var st State
		if err := dec.Decode(&ak); err == io.EOF {
			break
		} else if err != nil {
			log.Println(err)
			return
		}
		if err := dec.Decode(&st); err != nil {
			log.Println(err)
			return
		}
		if _, present := s.Conf.Alerts[ak.Name]; !present {
			log.Println("sched: alert no longer present, ignoring:", ak)
			continue
		}
		s.Status[ak] = &st
	}
}

func (s *Schedule) Save() {
	s.Lock()
	defer s.Unlock()
	f, err := os.Create(s.Conf.StateFile)
	if err != nil {
		log.Println(err)
		return
	}
	enc := gob.NewEncoder(f)
	if err := enc.Encode(s.Notifications); err != nil {
		log.Println(err)
		return
	}
	for k, v := range s.Status {
		enc.Encode(k)
		enc.Encode(v)
	}
	if err := f.Close(); err != nil {
		log.Println(err)
		return
	}
	log.Println("sched: wrote state to", s.Conf.StateFile)
}

func (s *Schedule) Run() error {
	go func() {
		for {
			time.Sleep(time.Minute)
			s.Save()
		}
	}()
	s.nc = make(chan interface{}, 1)
	go s.Poll()
	for {
		wait := time.After(s.Freq)
		if s.Freq < time.Second {
			return fmt.Errorf("sched: frequency must be > 1 second")
		}
		if s.Conf == nil {
			return fmt.Errorf("sched: nil configuration")
		}
		start := time.Now()
		s.Check()
		fmt.Printf("run at %v took %v\n", start, time.Since(start))
		<-wait
	}
}

// Poll dispatches notification checks when needed.
func (s *Schedule) Poll() {
	var timeout time.Duration
	for {
		// Wait for one of these two.
		select {
		case <-time.After(timeout):
		case <-s.nc:
		}
		timeout = s.CheckNotifications()
	}
}

// CheckNotifications processes past notification events. It returns the
func (s *Schedule) CheckNotifications() time.Duration {
	s.Lock()
	defer s.Unlock()
	timeout := time.Hour
	notifications := s.Notifications
	s.Notifications = nil
	for ak, ns := range notifications {
		for name, t := range ns {
			n, present := s.Conf.Notifications[name]
			if !present {
				continue
			}
			remaining := t.Add(n.Timeout).Sub(time.Now())
			if remaining > 0 {
				if remaining < timeout {
					timeout = remaining
				}
				s.AddNotification(ak, n, t)
				continue
			}
			st, present := s.Status[ak]
			if !present {
				continue
			}
			a, present := s.Conf.Alerts[ak.Name]
			if !present {
				continue
			}
			s.Notify(st, a, n)
			if n.Timeout < timeout {
				timeout = n.Timeout
			}
		}
	}
	return timeout
}

func (s *Schedule) Check() {
	s.Lock()
	defer s.Unlock()
	s.runStates = make(map[AlertKey]Status)
	s.cache = opentsdb.NewCache(s.Conf.TsdbHost)
	for _, a := range s.Conf.Alerts {
		s.CheckAlert(a)
	}
	changed := false
	for ak, status := range s.runStates {
		state := s.Status[ak]
		change := state.Append(status)
		if change {
			changed = true
		}
		if status > stNormal {
			var subject = new(bytes.Buffer)
			a := s.Conf.Alerts[ak.Name]
			if err := s.ExecuteSubject(subject, a, state); err != nil {
				log.Println(err)
			}
			state.Subject = subject.String()
			if change {
				notify := func(notifications map[string]*conf.Notification) {
					for _, n := range notifications {
						s.Notify(state, a, n)
					}
				}
				switch status {
				case stCritical:
					notify(a.CritNotification)
				case stWarning:
					notify(a.WarnNotification)
				}
			}
		}
	}
	if changed {
		s.nc <- true
	}
}

func (s *Schedule) CheckAlert(a *conf.Alert) {
	crits := s.CheckExpr(a, a.Crit, stCritical, nil)
	s.CheckExpr(a, a.Warn, stWarning, crits)
}

func (s *Schedule) CheckExpr(a *conf.Alert, e *expr.Expr, status Status, ignore []AlertKey) (alerts []AlertKey) {
	if e == nil {
		return
	}
	results, _, err := e.Execute(s.cache, nil)
	if err != nil {
		// todo: do something here?
		log.Println(err)
		return
	}
Loop:
	for _, r := range results {
		if a.Squelched(r.Group) {
			continue
		}
		ak := AlertKey{a.Name, r.Group.String()}
		for _, v := range ignore {
			if ak == v {
				continue Loop
			}
		}
		state := s.Status[ak]
		if state == nil {
			state = &State{
				Group: r.Group,
			}
			s.Status[ak] = state
		}
		state.Computations = r.Computations
		if r.Value.(expr.Number) != 0 {
			state.Expr = e.String()
			alerts = append(alerts, ak)
		} else {
			status = stNormal
		}
		if status > s.runStates[ak] {
			s.runStates[ak] = status
		}
	}
	return
}

func (s *Schedule) Notify(st *State, a *conf.Alert, n *conf.Notification) {
	if len(n.Email) > 0 {
		go s.Email(a, n, st)
	}
	if n.Post != nil {
		go s.Post(a, n, st)
	}
	if n.Get != nil {
		go s.Get(a, n, st)
	}
	if n.Print {
		go s.Print(a, n, st)
	}
	if n.Next == nil {
		return
	}
	s.AddNotification(AlertKey{Name: a.Name, Group: st.Group.String()}, n, time.Now().UTC())
}

func (s *Schedule) AddNotification(ak AlertKey, n *conf.Notification, started time.Time) {
	if s.Notifications == nil {
		s.Notifications = make(map[AlertKey]map[string]time.Time)
	}
	if s.Notifications[ak] == nil {
		s.Notifications[ak] = make(map[string]time.Time)
	}
	stn := s.Notifications[ak]
	// Prevent duplicate notifications restarting each other.
	if _, present := stn[n.Name]; !present {
		stn[n.Name] = started
	}
}

type AlertKey struct {
	Name  string
	Group string
}

func (a AlertKey) String() string {
	return a.Name + a.Group
}

type State struct {
	// Most recent event last.
	History      []Event
	Touched      time.Time
	Expr         string
	Group        opentsdb.TagSet
	Computations expr.Computations
	Subject      string
}

func (s *Schedule) Acknowledge(ak AlertKey) {
	s.Lock()
	delete(s.Notifications, ak)
	s.Unlock()
}

func (s *State) Touch() {
	s.Touched = time.Now().UTC()
}

// Appends status to the history if the status is different than the latest
// status. Returns true if state was changed.
func (s *State) Append(status Status) bool {
	s.Touch()
	if len(s.History) == 0 || s.Last().Status != status {
		s.History = append(s.History, Event{status, time.Now().UTC()})
		return true
	}
	return false
}

func (s *State) Last() Event {
	if len(s.History) == 0 {
		return Event{}
	}
	return s.History[len(s.History)-1]
}

type Event struct {
	Status Status
	Time   time.Time // embedding this breaks JSON encoding
}

type Status int

const (
	stUnknown Status = iota
	stNormal
	stWarning
	stCritical
)

func (s Status) String() string {
	switch s {
	case stNormal:
		return "normal"
	case stWarning:
		return "warning"
	case stCritical:
		return "critical"
	default:
		return "unknown"
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 xeipuuv ( https://github.com/xeipuuv )
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// author           xeipuuv
// author-github    https://github.com/xeipuuv
// author-mail      xeipuuv@gmail.com
//
// repository-name  gojsonschema
// repository-desc  An implementation of JSON Schema, based on IETF's draft v4 - Go language.
//
// description      (Unit) Tests for schema validation.
//
// created          16-06-2013

package gojsonschema

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"

	"github.com/stretchr/testify/assert"
)

const displayErrorMessages = false

const circularReference = `{
	"type": "object",
	"properties": {
		"games": {
			"type": "array",
			"items": {
				"$ref": "#/definitions/game"
			}
		}
	},
	"definitions": {
		"game": {
			"type": "object",
			"properties": {
				"winner": {
					"$ref": "#/definitions/player"
				},
				"loser": {
					"$ref": "#/definitions/player"
				}
			}
		},
		"player": {
			"type": "object",
			"properties": {
				"user": {
					"$ref": "#/definitions/user"
				},
				"game": {
					"$ref": "#/definitions/game"
				}
			}
		},
		"user": {
			"type": "object",
			"properties": {
				"fullName": {
					"type": "string"
				}
			}
		}
	}
}`

func TestCircularReference(t *testing.T) {
	loader := NewStringLoader(circularReference)
	// call the target function
	_, err := NewSchema(loader)
	if err != nil {
		t.Errorf("Got error: %s", err.Error())
	}
}

// From http://json-schema.org/examples.html
const simpleSchema = `{
  "title": "Example Schema",
  "type": "object",
  "properties": {
    "firstName": {
      "type": "string"
    },
    "lastName": {
      "type": "string"
    },
    "age": {
      "description": "Age in years",
      "type": "integer",
      "minimum": 0
    }
  },
  "required": ["firstName", "lastName"]
}`

func TestLoaders(t *testing.T) {
	// setup reader loader
	reader := bytes.NewBufferString(simpleSchema)
	readerLoader, wrappedReader := NewReaderLoader(reader)

	// drain reader
	by, err := ioutil.ReadAll(wrappedReader)
	assert.Nil(t, err)
	assert.Equal(t, simpleSchema, string(by))

	// setup writer loaders
	writer := &bytes.Buffer{}
	writerLoader, wrappedWriter := NewWriterLoader(writer)

	// fill writer
	n, err := io.WriteString(wrappedWriter, simpleSchema)
	assert.Nil(t, err)
	assert.Equal(t, n, len(simpleSchema))

	loaders := []JSONLoader{
		NewStringLoader(simpleSchema),
		readerLoader,
		writerLoader,
	}

	for _, l := range loaders {
		_, err := NewSchema(l)
		assert.Nil(t, err, "loader: %T", l)
	}
}

const invalidPattern = `{
  "title": "Example Pattern",
  "type": "object",
  "properties": {
    "invalid": {
      "type": "string",
      "pattern": 99999
    }
  }
}`

func TestLoadersWithInvalidPattern(t *testing.T) {
	// setup reader loader
	reader := bytes.NewBufferString(invalidPattern)
	readerLoader, wrappedReader := NewReaderLoader(reader)

	// drain reader
	by, err := ioutil.ReadAll(wrappedReader)
	assert.Nil(t, err)
	assert.Equal(t, invalidPattern, string(by))

	// setup writer loaders
	writer := &bytes.Buffer{}
	writerLoader, wrappedWriter := NewWriterLoader(writer)

	// fill writer
	n, err := io.WriteString(wrappedWriter, invalidPattern)
	assert.Nil(t, err)
	assert.Equal(t, n, len(invalidPattern))

	loaders := []JSONLoader{
		NewStringLoader(invalidPattern),
		readerLoader,
		writerLoader,
	}

	for _, l := range loaders {
		_, err := NewSchema(l)
		assert.NotNil(t, err, "expected error loading invalid pattern: %T", l)
	}
}

const refPropertySchema = `{
	"$id" : "http://localhost/schema.json",
	"properties" : {
		"$id" : {
			"$id": "http://localhost/foo.json"
		},
		"$ref" : {
			"const": {
				"$ref" : "hello.world"
			}
		},
		"const" : {
			"$ref" : "#/definitions/$ref"
		}
	},
	"definitions" : {
		"$ref" : {
			"const": {
				"$ref" : "hello.world"
			}
		}
	},
	"dependencies" : {
		"$ref" : [ "const" ],
		"const" : [ "$ref" ]
	}
}`

func TestRefProperty(t *testing.T) {
	schemaLoader := NewStringLoader(refPropertySchema)
	documentLoader := NewStringLoader(`{
		"$ref" : { "$ref" : "hello.world" },
		"const" : { "$ref" : "hello.world" }
		}`)
	// call the target function
	s, err := NewSchema(schemaLoader)
	if err != nil {
		t.Errorf("Got error: %s", err.Error())
	}
	result, err := s.Validate(documentLoader)
	if err != nil {
		t.Errorf("Got error: %s", err.Error())
	}
	if !result.Valid() {
		for _, err := range result.Errors() {
			fmt.Println(err.String())
		}
		t.Errorf("Got invalid validation result.")
	}
}

func TestFragmentLoader(t *testing.T) {
	wd, err := os.Getwd()

	if err != nil {
		panic(err.Error())
	}

	fileName := filepath.Join(wd, "testdata", "extra", "fragment_schema.json")

	schemaLoader := NewReferenceLoader("file://" + fileName + "#/definitions/x")
	schema, err := NewSchema(schemaLoader)

	if err != nil {
		t.Errorf("Encountered error while loading schema: %s", err.Error())
	}

	validDocument := NewStringLoader(`5`)
	invalidDocument := NewStringLoader(`"a"`)

	result, err := schema.Validate(validDocument)

	if assert.Nil(t, err, "Unexpected error while validating document: %T", err) {
		if !result.Valid() {
			t.Errorf("Got invalid validation result.")
		}
	}

	result, err = schema.Validate(invalidDocument)

	if assert.Nil(t, err, "Unexpected error while validating document: %T", err) {
		if len(result.Errors()) != 1 || result.Errors()[0].Type() != "invalid_type" {
			t.Errorf("Got invalid validation result.")
		}
	}
}

// Inspired by http://json-schema.org/latest/json-schema-core.html#rfc.section.8.2.3
const locationIndependentSchema = `{
  "definitions": {
    "A": {
      "$id": "#foo"
    },
    "B": {
      "$id": "http://example.com/other.json",
      "definitions": {
        "X": {
          "$id": "#bar",
          "allOf": [false]
        },
        "Y": {
          "$id": "t/inner.json"
        }
      }
    },
    "C": {
			"$id" : "#frag",
      "$ref": "http://example.com/other.json#bar"
    }
  },
  "$ref": "#frag"
}`

func TestLocationIndependentIdentifier(t *testing.T) {
	schemaLoader := NewStringLoader(locationIndependentSchema)
	documentLoader := NewStringLoader(`{}`)

	s, err := NewSchema(schemaLoader)
	if err != nil {
		t.Errorf("Got error: %s", err.Error())
	}

	result, err := s.Validate(documentLoader)
	if err != nil {
		t.Errorf("Got error: %s", err.Error())
	}

	if len(result.Errors()) != 2 || result.Errors()[0].Type() != "number_not" || result.Errors()[1].Type() != "number_all_of" {
		t.Errorf("Got invalid validation result.")
	}
}

const incorrectRefSchema = `{
  "$ref" : "#/fail"
}`

func TestIncorrectRef(t *testing.T) {

	schemaLoader := NewStringLoader(incorrectRefSchema)
	s, err := NewSchema(schemaLoader)

	assert.Nil(t, s)
	assert.Equal(t, "Object has no key 'fail'", err.Error())
}
<commit_msg>Fix tests on Windows<commit_after>// Copyright 2015 xeipuuv ( https://github.com/xeipuuv )
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// author           xeipuuv
// author-github    https://github.com/xeipuuv
// author-mail      xeipuuv@gmail.com
//
// repository-name  gojsonschema
// repository-desc  An implementation of JSON Schema, based on IETF's draft v4 - Go language.
//
// description      (Unit) Tests for schema validation.
//
// created          16-06-2013

package gojsonschema

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"

	"github.com/stretchr/testify/assert"
)

const displayErrorMessages = false

const circularReference = `{
	"type": "object",
	"properties": {
		"games": {
			"type": "array",
			"items": {
				"$ref": "#/definitions/game"
			}
		}
	},
	"definitions": {
		"game": {
			"type": "object",
			"properties": {
				"winner": {
					"$ref": "#/definitions/player"
				},
				"loser": {
					"$ref": "#/definitions/player"
				}
			}
		},
		"player": {
			"type": "object",
			"properties": {
				"user": {
					"$ref": "#/definitions/user"
				},
				"game": {
					"$ref": "#/definitions/game"
				}
			}
		},
		"user": {
			"type": "object",
			"properties": {
				"fullName": {
					"type": "string"
				}
			}
		}
	}
}`

func TestCircularReference(t *testing.T) {
	loader := NewStringLoader(circularReference)
	// call the target function
	_, err := NewSchema(loader)
	if err != nil {
		t.Errorf("Got error: %s", err.Error())
	}
}

// From http://json-schema.org/examples.html
const simpleSchema = `{
  "title": "Example Schema",
  "type": "object",
  "properties": {
    "firstName": {
      "type": "string"
    },
    "lastName": {
      "type": "string"
    },
    "age": {
      "description": "Age in years",
      "type": "integer",
      "minimum": 0
    }
  },
  "required": ["firstName", "lastName"]
}`

func TestLoaders(t *testing.T) {
	// setup reader loader
	reader := bytes.NewBufferString(simpleSchema)
	readerLoader, wrappedReader := NewReaderLoader(reader)

	// drain reader
	by, err := ioutil.ReadAll(wrappedReader)
	assert.Nil(t, err)
	assert.Equal(t, simpleSchema, string(by))

	// setup writer loaders
	writer := &bytes.Buffer{}
	writerLoader, wrappedWriter := NewWriterLoader(writer)

	// fill writer
	n, err := io.WriteString(wrappedWriter, simpleSchema)
	assert.Nil(t, err)
	assert.Equal(t, n, len(simpleSchema))

	loaders := []JSONLoader{
		NewStringLoader(simpleSchema),
		readerLoader,
		writerLoader,
	}

	for _, l := range loaders {
		_, err := NewSchema(l)
		assert.Nil(t, err, "loader: %T", l)
	}
}

const invalidPattern = `{
  "title": "Example Pattern",
  "type": "object",
  "properties": {
    "invalid": {
      "type": "string",
      "pattern": 99999
    }
  }
}`

func TestLoadersWithInvalidPattern(t *testing.T) {
	// setup reader loader
	reader := bytes.NewBufferString(invalidPattern)
	readerLoader, wrappedReader := NewReaderLoader(reader)

	// drain reader
	by, err := ioutil.ReadAll(wrappedReader)
	assert.Nil(t, err)
	assert.Equal(t, invalidPattern, string(by))

	// setup writer loaders
	writer := &bytes.Buffer{}
	writerLoader, wrappedWriter := NewWriterLoader(writer)

	// fill writer
	n, err := io.WriteString(wrappedWriter, invalidPattern)
	assert.Nil(t, err)
	assert.Equal(t, n, len(invalidPattern))

	loaders := []JSONLoader{
		NewStringLoader(invalidPattern),
		readerLoader,
		writerLoader,
	}

	for _, l := range loaders {
		_, err := NewSchema(l)
		assert.NotNil(t, err, "expected error loading invalid pattern: %T", l)
	}
}

const refPropertySchema = `{
	"$id" : "http://localhost/schema.json",
	"properties" : {
		"$id" : {
			"$id": "http://localhost/foo.json"
		},
		"$ref" : {
			"const": {
				"$ref" : "hello.world"
			}
		},
		"const" : {
			"$ref" : "#/definitions/$ref"
		}
	},
	"definitions" : {
		"$ref" : {
			"const": {
				"$ref" : "hello.world"
			}
		}
	},
	"dependencies" : {
		"$ref" : [ "const" ],
		"const" : [ "$ref" ]
	}
}`

func TestRefProperty(t *testing.T) {
	schemaLoader := NewStringLoader(refPropertySchema)
	documentLoader := NewStringLoader(`{
		"$ref" : { "$ref" : "hello.world" },
		"const" : { "$ref" : "hello.world" }
		}`)
	// call the target function
	s, err := NewSchema(schemaLoader)
	if err != nil {
		t.Errorf("Got error: %s", err.Error())
	}
	result, err := s.Validate(documentLoader)
	if err != nil {
		t.Errorf("Got error: %s", err.Error())
	}
	if !result.Valid() {
		for _, err := range result.Errors() {
			fmt.Println(err.String())
		}
		t.Errorf("Got invalid validation result.")
	}
}

func TestFragmentLoader(t *testing.T) {
	wd, err := os.Getwd()

	if err != nil {
		panic(err.Error())
	}

	fileName := filepath.Join(wd, "testdata", "extra", "fragment_schema.json")

	schemaLoader := NewReferenceLoader("file://" + filepath.ToSlash(fileName) + "#/definitions/x")
	schema, err := NewSchema(schemaLoader)

	if err != nil {
		t.Errorf("Encountered error while loading schema: %s", err.Error())
	}

	validDocument := NewStringLoader(`5`)
	invalidDocument := NewStringLoader(`"a"`)

	result, err := schema.Validate(validDocument)

	if assert.Nil(t, err, "Unexpected error while validating document: %T", err) {
		if !result.Valid() {
			t.Errorf("Got invalid validation result.")
		}
	}

	result, err = schema.Validate(invalidDocument)

	if assert.Nil(t, err, "Unexpected error while validating document: %T", err) {
		if len(result.Errors()) != 1 || result.Errors()[0].Type() != "invalid_type" {
			t.Errorf("Got invalid validation result.")
		}
	}
}

// Inspired by http://json-schema.org/latest/json-schema-core.html#rfc.section.8.2.3
const locationIndependentSchema = `{
  "definitions": {
    "A": {
      "$id": "#foo"
    },
    "B": {
      "$id": "http://example.com/other.json",
      "definitions": {
        "X": {
          "$id": "#bar",
          "allOf": [false]
        },
        "Y": {
          "$id": "t/inner.json"
        }
      }
    },
    "C": {
			"$id" : "#frag",
      "$ref": "http://example.com/other.json#bar"
    }
  },
  "$ref": "#frag"
}`

func TestLocationIndependentIdentifier(t *testing.T) {
	schemaLoader := NewStringLoader(locationIndependentSchema)
	documentLoader := NewStringLoader(`{}`)

	s, err := NewSchema(schemaLoader)
	if err != nil {
		t.Errorf("Got error: %s", err.Error())
	}

	result, err := s.Validate(documentLoader)
	if err != nil {
		t.Errorf("Got error: %s", err.Error())
	}

	if len(result.Errors()) != 2 || result.Errors()[0].Type() != "number_not" || result.Errors()[1].Type() != "number_all_of" {
		t.Errorf("Got invalid validation result.")
	}
}

const incorrectRefSchema = `{
  "$ref" : "#/fail"
}`

func TestIncorrectRef(t *testing.T) {

	schemaLoader := NewStringLoader(incorrectRefSchema)
	s, err := NewSchema(schemaLoader)

	assert.Nil(t, s)
	assert.Equal(t, "Object has no key 'fail'", err.Error())
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"os"

	xero "github.com/thisissoon/go-xero"
	xauth "github.com/thisissoon/go-xero/authorizers/go-oauth"
)

func main() {
	pemfile := flag.String("pemfile", "", "Xero API PEM file location")
	token := flag.String("token", "", "Xero API Token")
	flag.Parse()
	f, err := os.Open(*pemfile)
	if err != nil {
		log.Fatal(err)
	}
	pk, err := xero.PrivateKey(f)
	if err != nil {
		log.Fatal(err)
	}
	client := xero.New(xauth.New(*token, pk))
	for i, contacts, err := client.Contacts(); err != io.EOF; i, contacts, err = i.Next() {
		if err != nil {
			log.Fatal(err)
		}
		for i := range contacts {
			fmt.Println(contacts[i].Name)
		}
	}
}
<commit_msg>Updated contacts example of with singal contact request<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"os"

	xero "github.com/thisissoon/go-xero"
	xauth "github.com/thisissoon/go-xero/authorizers/go-oauth"
)

func main() {
	pemfile := flag.String("pemfile", "", "Xero API PEM file location")
	token := flag.String("token", "", "Xero API Token")
	flag.Parse()
	f, err := os.Open(*pemfile)
	if err != nil {
		log.Fatal(err)
	}
	pk, err := xero.PrivateKey(f)
	if err != nil {
		log.Fatal(err)
	}
	client := xero.New(xauth.New(*token, pk))
	// Iteration
	fmt.Println("Contact Iteration")
	fmt.Println("-----------------")
	for i, contacts, err := client.Contacts(); err != io.EOF; i, contacts, err = i.Next() {
		if err != nil {
			log.Fatal(err)
		}
		for i := range contacts {
			fmt.Println(contacts[i].ContactID, contacts[i].Name)
		}
	}
	// Get Singular Contact
	fmt.Println("Single Contact")
	fmt.Println("--------------")
	contact, err := client.Contact("847d9131-ac1e-4819-af61-b3f1cf854009")
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(contact.ContactID, contact.Name)
}
<|endoftext|>