<commit_before>package synq

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/request"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
)

// These Amazon Web Services credentials are provided to the AWS SDK, which is
// used to upload content in multiple parts. There is no IAM user with these
// credentials; they are supplied because the AWS SDK requires some credentials
// to attempt to start uploading. This package replaces the AWS SDK's request
// signing method with its own method.
const multipartUploadAwsAccessKeyId = "AAAAAAAAAAAAAAAAAAAA"
const multipartUploadAwsSecretAccessKey = "ssssssssssssssssssssssssssssssssssssssss"

// TODO(mastensg): Determine region from bucket, or /v1/video/uploader
const multipartUploadS3BucketRegion = "us-east-1"

// UploaderSignatureUrlFormat is a printf format string which is used when
// signing multipart upload requests.
// TODO(mastensg): Determine this format (or at least prefix) at runtime, from
// the Synq HTTP API.
const UploaderSignatureUrlFormat = "https://uploader.synq.fm/uploader/signature/%s?token=%s"

// UploaderSignatureRequest is the request that is sent when using the
// embeddable web uploader's request signing service.
type UploaderSignatureRequest struct {
	Headers string `json:"headers"`
}

// UploaderSignatureResponse is the response that is received when using the
// embeddable web uploader's request signing service.
type UploaderSignatureResponse struct {
	Signature string `json:"signature"`
}

// UploaderSignature uses the backend of the embeddable web uploader to sign
// multipart upload requests.
func UploaderSignature(url_fmt, video_id, token, headers string) ([]byte, error) {
	url := fmt.Sprintf(url_fmt, video_id, token)

	// construct request body
	reqStruct := UploaderSignatureRequest{Headers: headers}
	reqBody, err := json.Marshal(reqStruct)
	if err != nil {
		return nil, err
	}

	// perform request
	resp, err := http.Post(url, "application/json", bytes.NewReader(reqBody))
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	// read response
	respBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	// parse response
	respStruct := UploaderSignatureResponse{}
	err = json.Unmarshal(respBody, &respStruct)
	if err != nil {
		return nil, err
	}

	return []byte(respStruct.Signature), nil
}

// tokenOfUploaderURL parses an uploader URL string, and returns its token
// parameter.
//
// Example:
//         const u = "https://uploader.synq.fm/uploader/" +
//         "00000000000000000000000000000000" +
//         "?token=11111111111111111111111111111111"
//
//         token, err := tokenOfUploaderURL(u)
//         if err != nil {
//                 log.Fatal(err)
//         }
//         fmt.Println(token) // prints 11111111111111111111111111111111
func tokenOfUploaderURL(uploaderURL string) (string, error) {
	u, err := url.Parse(uploaderURL)
	if err != nil {
		return "", err
	}

	values, err := url.ParseQuery(u.RawQuery)
	if err != nil {
		return "", err
	}

	token := values.Get("token")
	if token == "" {
		return "", errors.New("Found no token parameter in URL.")
	}

	return token, nil
}

// MultipartUploadSigner returns a function that can be added to an s3 client's
// list of handlers. The function will take over signing of requests from
// aws-sdk-go.
//
// The signer function uses SYNQ's embeddable web uploader's remote procedure
// to sign requests.
//
// This function is used by the internal multipartUpload function.
//
// Example:
//         // AWS session.
//         sess := session.Must(session.NewSession())
//
//         // S3 service client.
//         svc := s3.New(sess)
//
//         // Signer function. Determine the arguments somehow.
//         signer := MultipartUploadSigner(acl, awsAccessKeyId, bucket, contentType, key, token, video_id)
//
//         // Register handler as the last handler of the signing phase.
//         svc.Handlers.Sign.PushBack(signer)
//
//         // S3 requests are now signed by signer().
func MultipartUploadSigner(acl, awsAccessKeyId, bucket, contentType, key, token, video_id string) func(r *request.Request) {
	signer := func(r *request.Request) {
		hr := r.HTTPRequest

		// rewrite the X-Amz-Date header into the format that
		// https://uploader.synq.fm/uploader/signature expects
		{
			x_amz_date_in := hr.Header.Get("X-Amz-Date")
			if x_amz_date_in == "" {
				return // TODO(mastensg): how to report errors from handlers?
			}
			x_amz_date_t, err := time.Parse("20060102T150405Z", x_amz_date_in)
			if err != nil {
				return // TODO(mastensg): how to report errors from handlers?
			}
			x_amz_date := x_amz_date_t.Format("Mon, 2 Jan 2006 15:04:05 MST")
			delete(hr.Header, "X-Amz-Date") // TODO(mastensg): enough to just set and not delete?
			hr.Header.Set("X-Amz-Date", x_amz_date)
		}

		x_amz_date := hr.Header.Get("X-Amz-Date")

		// construct "headers" string to send to
		// https://uploader.synq.fm/uploader/signature
		headers := ""
		if hr.URL.RawQuery == "uploads=" {
			// Initiate multi-part upload

			// TODO(mastensg): parameterize bucket name, content-type, acl
			headers = fmt.Sprintf("%s\n\n%s\n\n%s\nx-amz-date:%s\n/synqfm%s",
				hr.Method,
				"video/mp4",
				"x-amz-acl:public-read",
				x_amz_date,
				hr.URL.Path+"?uploads",
			)
		} else if hr.Method == "PUT" {
			// Upload one part

			// TODO(mastensg): parameterize bucket name
			headers = fmt.Sprintf("%s\n\n%s\n\nx-amz-date:%s\n/synqfm%s",
				hr.Method,
				"",
				x_amz_date,
				hr.URL.Path+"?" + hr.URL.RawQuery,
			)
		} else if hr.Method == "POST" {
			// Finish multi-part upload

			// TODO(mastensg): parameterize bucket name, content-type(?)
			headers = fmt.Sprintf("%s\n\n%s\n\nx-amz-date:%s\n/synqfm%s",
				hr.Method,
				"application/xml; charset=UTF-8",
				x_amz_date,
				hr.URL.Path+"?"+hr.URL.RawQuery,
			)

			// TODO(mastensg): the content-type header set by
			// aws-sdk-go is not exactly the one expected by
			// uploader/signature, maybe
			hr.Header.Set("Content-Type", "application/xml; charset=UTF-8")
		} else {
			// Unknown request type
			return // TODO(mastensg): how to report errors from handlers?
		}

		signature, err := UploaderSignature(UploaderSignatureUrlFormat, video_id, token, headers)
		if err != nil {
			return // TODO(mastensg): how to report errors from handlers?
		}

		// rewrite authorization header(s)
		delete(hr.Header, "X-Amz-Content-Sha256") // TODO(mastensg): can this be left in?
		delete(hr.Header, "Authorization")
		authorization := fmt.Sprintf("AWS %s:%s", awsAccessKeyId, signature)
		hr.Header.Set("Authorization", authorization)
	}

	return signer
}

// multipartUpload uploads a file as the video's original_file.
// This procedure will use Amazon S3's Multipart Upload API:
// http://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html
//
// This is the internal function to make uploads, which is called by the public
// MultipartUpload. This function uses s3manager from aws-sdk-go to manage the
// process of uploading in multiple parts. In particular, this will start
// several goroutines that will upload parts concurrently.
func multipartUpload(body io.Reader, acl, awsAccessKeyId, bucket, contentType, key, uploaderURL, video_id string) (*s3manager.UploadOutput, error) {
	token, err := tokenOfUploaderURL(uploaderURL)
	if err != nil {
		return nil, err
	}

	// credentials
	provider := credentials.StaticProvider{}
	provider.Value.AccessKeyID = multipartUploadAwsAccessKeyId
	provider.Value.SecretAccessKey = multipartUploadAwsSecretAccessKey
	credentials := credentials.NewCredentials(&provider)

	// session
	sess, err := session.NewSession(&aws.Config{
		Credentials: credentials,
		Region:      aws.String(multipartUploadS3BucketRegion),
	})
	if err != nil {
		return nil, err
	}

	svc := s3.New(sess)

	// sign handler
	signer := makeMultipartUploadSigner(acl, awsAccessKeyId, bucket, contentType, key, token, video_id)
	svc.Handlers.Sign.PushBack(signer)

	// s3manager uploader
	uploader := s3manager.NewUploaderWithClient(svc)

	// upload parameters
	uploadInput := &s3manager.UploadInput{
		ACL:         &acl,
		Body:        body,
		Bucket:      &bucket,
		ContentType: &contentType,
		Key:         &key,
	}

	return uploader.Upload(uploadInput)
}
<commit_msg>synq/multipart_upload: multipartUpload: call MultipartUploadSigner<commit_after>package synq

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/request"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
)

// These Amazon Web Services credentials are provided to the AWS SDK, which is
// used to upload content in multiple parts. There is no IAM user with these
// credentials; they are supplied because the AWS SDK requires some credentials
// to attempt to start uploading. This package replaces the AWS SDK's request
// signing method with its own method.
const multipartUploadAwsAccessKeyId = "AAAAAAAAAAAAAAAAAAAA"
const multipartUploadAwsSecretAccessKey = "ssssssssssssssssssssssssssssssssssssssss"

// TODO(mastensg): Determine region from bucket, or /v1/video/uploader
const multipartUploadS3BucketRegion = "us-east-1"

// UploaderSignatureUrlFormat is a printf format string which is used when
// signing multipart upload requests.
// TODO(mastensg): Determine this format (or at least prefix) at runtime, from
// the Synq HTTP API.
const UploaderSignatureUrlFormat = "https://uploader.synq.fm/uploader/signature/%s?token=%s"

// UploaderSignatureRequest is the request that is sent when using the
// embeddable web uploader's request signing service.
type UploaderSignatureRequest struct {
	Headers string `json:"headers"`
}

// UploaderSignatureResponse is the response that is received when using the
// embeddable web uploader's request signing service.
type UploaderSignatureResponse struct {
	Signature string `json:"signature"`
}

// UploaderSignature uses the backend of the embeddable web uploader to sign
// multipart upload requests.
func UploaderSignature(url_fmt, video_id, token, headers string) ([]byte, error) {
	url := fmt.Sprintf(url_fmt, video_id, token)

	// construct request body
	reqStruct := UploaderSignatureRequest{Headers: headers}
	reqBody, err := json.Marshal(reqStruct)
	if err != nil {
		return nil, err
	}

	// perform request
	resp, err := http.Post(url, "application/json", bytes.NewReader(reqBody))
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	// read response
	respBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	// parse response
	respStruct := UploaderSignatureResponse{}
	err = json.Unmarshal(respBody, &respStruct)
	if err != nil {
		return nil, err
	}

	return []byte(respStruct.Signature), nil
}

// tokenOfUploaderURL parses an uploader URL string, and returns its token
// parameter.
//
// Example:
//         const u = "https://uploader.synq.fm/uploader/" +
//         "00000000000000000000000000000000" +
//         "?token=11111111111111111111111111111111"
//
//         token, err := tokenOfUploaderURL(u)
//         if err != nil {
//                 log.Fatal(err)
//         }
//         fmt.Println(token) // prints 11111111111111111111111111111111
func tokenOfUploaderURL(uploaderURL string) (string, error) {
	u, err := url.Parse(uploaderURL)
	if err != nil {
		return "", err
	}

	values, err := url.ParseQuery(u.RawQuery)
	if err != nil {
		return "", err
	}

	token := values.Get("token")
	if token == "" {
		return "", errors.New("Found no token parameter in URL.")
	}

	return token, nil
}

// MultipartUploadSigner returns a function that can be added to an s3 client's
// list of handlers. The function will take over signing of requests from
// aws-sdk-go.
//
// The signer function uses SYNQ's embeddable web uploader's remote procedure
// to sign requests.
//
// This function is used by the internal multipartUpload function.
//
// Example:
//         // AWS session.
//         sess := session.Must(session.NewSession())
//
//         // S3 service client.
//         svc := s3.New(sess)
//
//         // Signer function. Determine the arguments somehow.
//         signer := MultipartUploadSigner(acl, awsAccessKeyId, bucket, contentType, key, token, video_id)
//
//         // Register handler as the last handler of the signing phase.
//         svc.Handlers.Sign.PushBack(signer)
//
//         // S3 requests are now signed by signer().
func MultipartUploadSigner(acl, awsAccessKeyId, bucket, contentType, key, token, video_id string) func(r *request.Request) {
	signer := func(r *request.Request) {
		hr := r.HTTPRequest

		// rewrite the X-Amz-Date header into the format that
		// https://uploader.synq.fm/uploader/signature expects
		{
			x_amz_date_in := hr.Header.Get("X-Amz-Date")
			if x_amz_date_in == "" {
				return // TODO(mastensg): how to report errors from handlers?
			}
			x_amz_date_t, err := time.Parse("20060102T150405Z", x_amz_date_in)
			if err != nil {
				return // TODO(mastensg): how to report errors from handlers?
			}
			x_amz_date := x_amz_date_t.Format("Mon, 2 Jan 2006 15:04:05 MST")
			delete(hr.Header, "X-Amz-Date") // TODO(mastensg): enough to just set and not delete?
			hr.Header.Set("X-Amz-Date", x_amz_date)
		}

		x_amz_date := hr.Header.Get("X-Amz-Date")

		// construct "headers" string to send to
		// https://uploader.synq.fm/uploader/signature
		headers := ""
		if hr.URL.RawQuery == "uploads=" {
			// Initiate multi-part upload

			// TODO(mastensg): parameterize bucket name, content-type, acl
			headers = fmt.Sprintf("%s\n\n%s\n\n%s\nx-amz-date:%s\n/synqfm%s",
				hr.Method,
				"video/mp4",
				"x-amz-acl:public-read",
				x_amz_date,
				hr.URL.Path+"?uploads",
			)
		} else if hr.Method == "PUT" {
			// Upload one part

			// TODO(mastensg): parameterize bucket name
			headers = fmt.Sprintf("%s\n\n%s\n\nx-amz-date:%s\n/synqfm%s",
				hr.Method,
				"",
				x_amz_date,
				hr.URL.Path+"?" + hr.URL.RawQuery,
			)
		} else if hr.Method == "POST" {
			// Finish multi-part upload

			// TODO(mastensg): parameterize bucket name, content-type(?)
			headers = fmt.Sprintf("%s\n\n%s\n\nx-amz-date:%s\n/synqfm%s",
				hr.Method,
				"application/xml; charset=UTF-8",
				x_amz_date,
				hr.URL.Path+"?"+hr.URL.RawQuery,
			)

			// TODO(mastensg): the content-type header set by
			// aws-sdk-go is not exactly the one expected by
			// uploader/signature, maybe
			hr.Header.Set("Content-Type", "application/xml; charset=UTF-8")
		} else {
			// Unknown request type
			return // TODO(mastensg): how to report errors from handlers?
		}

		signature, err := UploaderSignature(UploaderSignatureUrlFormat, video_id, token, headers)
		if err != nil {
			return // TODO(mastensg): how to report errors from handlers?
		}

		// rewrite authorization header(s)
		delete(hr.Header, "X-Amz-Content-Sha256") // TODO(mastensg): can this be left in?
		delete(hr.Header, "Authorization")
		authorization := fmt.Sprintf("AWS %s:%s", awsAccessKeyId, signature)
		hr.Header.Set("Authorization", authorization)
	}

	return signer
}

// multipartUpload uploads a file as the video's original_file.
// This procedure will use Amazon S3's Multipart Upload API:
// http://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html
//
// This is the internal function to make uploads, which is called by the public
// MultipartUpload. This function uses s3manager from aws-sdk-go to manage the
// process of uploading in multiple parts. In particular, this will start
// several goroutines that will upload parts concurrently.
func multipartUpload(body io.Reader, acl, awsAccessKeyId, bucket, contentType, key, uploaderURL, video_id string) (*s3manager.UploadOutput, error) {
	token, err := tokenOfUploaderURL(uploaderURL)
	if err != nil {
		return nil, err
	}

	// credentials
	provider := credentials.StaticProvider{}
	provider.Value.AccessKeyID = multipartUploadAwsAccessKeyId
	provider.Value.SecretAccessKey = multipartUploadAwsSecretAccessKey
	credentials := credentials.NewCredentials(&provider)

	// session
	sess, err := session.NewSession(&aws.Config{
		Credentials: credentials,
		Region:      aws.String(multipartUploadS3BucketRegion),
	})
	if err != nil {
		return nil, err
	}

	svc := s3.New(sess)

	// sign handler
	signer := MultipartUploadSigner(acl, awsAccessKeyId, bucket, contentType, key, token, video_id)
	svc.Handlers.Sign.PushBack(signer)

	// s3manager uploader
	uploader := s3manager.NewUploaderWithClient(svc)

	// upload parameters
	uploadInput := &s3manager.UploadInput{
		ACL:         &acl,
		Body:        body,
		Bucket:      &bucket,
		ContentType: &contentType,
		Key:         &key,
	}

	return uploader.Upload(uploadInput)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"github.com/BytemarkHosting/bytemark-client/cmd/bytemark/util"
	"github.com/BytemarkHosting/bytemark-client/lib"
	"github.com/BytemarkHosting/bytemark-client/lib/brain"
	"github.com/BytemarkHosting/bytemark-client/util/log"
	"github.com/urfave/cli"
	"net"
)

// Context is a wrapper around urfave/cli.Context which provides easy access to
// the next unused argument and can have various bytemarky types attached to it
// in order to keep code DRY
type Context struct {
	Context            *cli.Context
	Account            *lib.Account
	Authed             bool
	Definitions        *lib.Definitions
	Group              *brain.Group
	User               *brain.User
	UserName           *string
	VirtualMachine     *brain.VirtualMachine
	VirtualMachineName *lib.VirtualMachineName

	currentArgIndex int
}

// Reset replaces the Context with a blank one (keeping the cli.Context)
func (c *Context) Reset() {
	*c = Context{
		Context: c.Context,
	}
}

// args returns all the args that were passed to the Context (i.e. all the args passed to this (sub)command)
func (c *Context) args() cli.Args {
	return c.Context.Args()
}

// Args returns all the unused arguments
func (c *Context) Args() []string {
	return c.args()[c.currentArgIndex:]
}

// NextArg returns the next unused argument, and marks it as used.
func (c *Context) NextArg() (string, error) {
	if len(c.args()) <= c.currentArgIndex {
		return "", c.Help("not enough arguments were specified")
	}
	arg := c.args()[c.currentArgIndex]
	c.currentArgIndex++
	return arg, nil
}

// Help creates a UsageDisplayedError that will output the issue and a message to consult the documentation
func (c *Context) Help(whatsyourproblem string) (err error) {
	return util.UsageDisplayedError{TheProblem: whatsyourproblem, Command: c.Context.Command.FullName()}
}

// flags below

// Bool returns the value of the named flag as a bool
func (c *Context) Bool(flagname string) bool {
	return c.Context.Bool(flagname)
}

// Discs returns the discs passed along as the named flag.
// I can't imagine why I'd ever name a disc flag anything other than --disc, but the flexibility is there just in case.
func (c *Context) Discs(flagname string) []brain.Disc {
	disc, ok := c.Context.Generic(flagname).(*util.DiscSpecFlag)
	if ok {
		return []brain.Disc(*disc)
	}
	return []brain.Disc{}
}

// FileName returns the name of the file given by the named flag
func (c *Context) FileName(flagname string) string {
	file, ok := c.Context.Generic(flagname).(*util.FileFlag)
	if ok {
		return file.FileName
	}
	return ""
}

// FileContents returns the contents of the file given by the named flag.
func (c *Context) FileContents(flagname string) string {
	file, ok := c.Context.Generic(flagname).(*util.FileFlag)
	if ok {
		return file.Value
	}
	return ""
}

func (c *Context) GroupName(flagname string) lib.GroupName {
	gpNameFlag, ok := c.Context.Generic(flagname).(*GroupNameFlag)
	if !ok {
		return lib.GroupName{}
	}
	return lib.GroupName(*gpNameFlag)
}

// Int returns the value of the named flag as an int
func (c *Context) Int(flagname string) int {
	return c.Context.Int(flagname)
}

// IPs returns the ips passed along as the named flag.
func (c *Context) IPs(flagname string) []net.IP {
	ips, ok := c.Context.Generic(flagname).(*util.IPFlag)
	if ok {
		return []net.IP(*ips)
	}
	return []net.IP{}
}

// String returns the value of the named flag as a string
func (c *Context) String(flagname string) string {
	return c.Context.String(flagname)
}

// Size returns the value of the named SizeSpecFlag as an int in megabytes
func (c *Context) Size(flagname string) int {
	size, ok := c.Context.Generic(flagname).(*util.SizeSpecFlag)
	if ok {
		return int(*size)
	}
	return 0
}

// IfNotMarshalJSON checks to see if the json flag was set, and outputs obj as a JSON object if so.
// if not, runs fn
func (c *Context) IfNotMarshalJSON(obj interface{}, fn func() error) error {
	if c.Bool("json") {
		js, err := json.MarshalIndent(obj, "", "    ")
		if err != nil {
			return err
		}
		log.Output(string(js))
		return nil
	}
	return fn()
}
<commit_msg>Add documentation comment to Context.GroupName<commit_after>package main

import (
	"encoding/json"
	"github.com/BytemarkHosting/bytemark-client/cmd/bytemark/util"
	"github.com/BytemarkHosting/bytemark-client/lib"
	"github.com/BytemarkHosting/bytemark-client/lib/brain"
	"github.com/BytemarkHosting/bytemark-client/util/log"
	"github.com/urfave/cli"
	"net"
)

// Context is a wrapper around urfave/cli.Context which provides easy access to
// the next unused argument and can have various bytemarky types attached to it
// in order to keep code DRY
type Context struct {
	Context            *cli.Context
	Account            *lib.Account
	Authed             bool
	Definitions        *lib.Definitions
	Group              *brain.Group
	User               *brain.User
	UserName           *string
	VirtualMachine     *brain.VirtualMachine
	VirtualMachineName *lib.VirtualMachineName

	currentArgIndex int
}

// Reset replaces the Context with a blank one (keeping the cli.Context)
func (c *Context) Reset() {
	*c = Context{
		Context: c.Context,
	}
}

// args returns all the args that were passed to the Context (i.e. all the args passed to this (sub)command)
func (c *Context) args() cli.Args {
	return c.Context.Args()
}

// Args returns all the unused arguments
func (c *Context) Args() []string {
	return c.args()[c.currentArgIndex:]
}

// NextArg returns the next unused argument, and marks it as used.
func (c *Context) NextArg() (string, error) {
	if len(c.args()) <= c.currentArgIndex {
		return "", c.Help("not enough arguments were specified")
	}
	arg := c.args()[c.currentArgIndex]
	c.currentArgIndex++
	return arg, nil
}

// Help creates a UsageDisplayedError that will output the issue and a message to consult the documentation
func (c *Context) Help(whatsyourproblem string) (err error) {
	return util.UsageDisplayedError{TheProblem: whatsyourproblem, Command: c.Context.Command.FullName()}
}

// flags below

// Bool returns the value of the named flag as a bool
func (c *Context) Bool(flagname string) bool {
	return c.Context.Bool(flagname)
}

// Discs returns the discs passed along as the named flag.
// I can't imagine why I'd ever name a disc flag anything other than --disc, but the flexibility is there just in case.
func (c *Context) Discs(flagname string) []brain.Disc {
	disc, ok := c.Context.Generic(flagname).(*util.DiscSpecFlag)
	if ok {
		return []brain.Disc(*disc)
	}
	return []brain.Disc{}
}

// FileName returns the name of the file given by the named flag
func (c *Context) FileName(flagname string) string {
	file, ok := c.Context.Generic(flagname).(*util.FileFlag)
	if ok {
		return file.FileName
	}
	return ""
}

// FileContents returns the contents of the file given by the named flag.
func (c *Context) FileContents(flagname string) string {
	file, ok := c.Context.Generic(flagname).(*util.FileFlag)
	if ok {
		return file.Value
	}
	return ""
}

// GroupName returns the named flag as a lib.GroupName
func (c *Context) GroupName(flagname string) lib.GroupName {
	gpNameFlag, ok := c.Context.Generic(flagname).(*GroupNameFlag)
	if !ok {
		return lib.GroupName{}
	}
	return lib.GroupName(*gpNameFlag)
}

// Int returns the value of the named flag as an int
func (c *Context) Int(flagname string) int {
	return c.Context.Int(flagname)
}

// IPs returns the ips passed along as the named flag.
func (c *Context) IPs(flagname string) []net.IP {
	ips, ok := c.Context.Generic(flagname).(*util.IPFlag)
	if ok {
		return []net.IP(*ips)
	}
	return []net.IP{}
}

// String returns the value of the named flag as a string
func (c *Context) String(flagname string) string {
	return c.Context.String(flagname)
}

// Size returns the value of the named SizeSpecFlag as an int in megabytes
func (c *Context) Size(flagname string) int {
	size, ok := c.Context.Generic(flagname).(*util.SizeSpecFlag)
	if ok {
		return int(*size)
	}
	return 0
}

// IfNotMarshalJSON checks to see if the json flag was set, and outputs obj as a JSON object if so.
// if not, runs fn
func (c *Context) IfNotMarshalJSON(obj interface{}, fn func() error) error {
	if c.Bool("json") {
		js, err := json.MarshalIndent(obj, "", "    ")
		if err != nil {
			return err
		}
		log.Output(string(js))
		return nil
	}
	return fn()
}
<|endoftext|>


<commit_before>package restapi

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
)

type TestHelloWorldEndpoint struct{}

func (api TestHelloWorldEndpoint) HelloWorld(names []string) (string, error) {
	// handle 0-to-Many qs names
	defaultName := "World"
	name := defaultName
	if len(names) != 0 {
		name = strings.Join(names, ", ")
	}

	msg := fmt.Sprintf("Hello %s!", name)
	return msg, nil
}

func TestHelloWorldHandler(t *testing.T) {
	tests := []struct {
		description        string
		hwEndpoint         *TestHelloWorldEndpoint
		url                string
		expectedStatusCode int
		message            string
	}{
		{
			description:        "successful query",
			hwEndpoint:         &TestHelloWorldEndpoint{},
			url:                "/hw?name=DUDE",
			expectedStatusCode: 200,
			message:            "Hello DUDE!",
		},
		{
			description:        "successful query",
			hwEndpoint:         &TestHelloWorldEndpoint{},
			url:                "/hw",
			expectedStatusCode: 200,
			message:            "Hello World!",
		},
	}

	for _, hw := range tests {
		app := &RestAPI{hw: hw.hwEndpoint}
		req, err := http.NewRequest("GET", hw.url, nil)
		if err != nil {

		}

		w := httptest.NewRecorder()
		app.HelloWorldHandler(w, req)

		// TODO: travis test forcing error
		// if hw.expectedStatusCode != w.Code {
		if hw.expectedStatusCode == w.Code {
			t.Errorf("handler returned wrong status code: got %v want %v",
				w.Code, hw.expectedStatusCode)
		}

		assert.Equal(t, hw.message, w.Body.String(), hw.description)

	}
}
<commit_msg>revert force fail changes<commit_after>package restapi

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
)

type TestHelloWorldEndpoint struct{}

func (api TestHelloWorldEndpoint) HelloWorld(names []string) (string, error) {
	// handle 0-to-Many qs names
	defaultName := "World"
	name := defaultName
	if len(names) != 0 {
		name = strings.Join(names, ", ")
	}

	msg := fmt.Sprintf("Hello %s!", name)
	return msg, nil
}

func TestHelloWorldHandler(t *testing.T) {
	tests := []struct {
		description        string
		hwEndpoint         *TestHelloWorldEndpoint
		url                string
		expectedStatusCode int
		message            string
	}{
		{
			description:        "successful query",
			hwEndpoint:         &TestHelloWorldEndpoint{},
			url:                "/hw?name=DUDE",
			expectedStatusCode: 200,
			message:            "Hello DUDE!",
		},
		{
			description:        "successful query",
			hwEndpoint:         &TestHelloWorldEndpoint{},
			url:                "/hw",
			expectedStatusCode: 200,
			message:            "Hello World!",
		},
	}

	for _, hw := range tests {
		app := &RestAPI{hw: hw.hwEndpoint}
		req, err := http.NewRequest("GET", hw.url, nil)
		if err != nil {

		}

		w := httptest.NewRecorder()
		app.HelloWorldHandler(w, req)

		if hw.expectedStatusCode != w.Code {
			t.Errorf("handler returned wrong status code: got %v want %v",
				w.Code, hw.expectedStatusCode)
		}

		assert.Equal(t, hw.message, w.Body.String(), hw.description)

	}
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"math/rand"
	"net/http"
	"os"
	"testing"
)

func BenchmarkWriteBatchRandom(b *testing.B) {
	ctx := context.Background()

	bd := new(BadgerAdapter)
	bd.Init("/tmp/bench-tmp")
	defer bd.Close()

	rd := new(RocksDBAdapter)
	rd.Init("/tmp/bench-tmp")
	defer rd.Close()

	batchSize := 1000
	valSizes := []int{100, 1000, 10000, 100000}

	for i := 0; i < 2; i++ {
		var db Database
		name := "badger"
		db = bd
		if i == 1 {
			db = rd
			name = "rocksdb"
		}
		for _, vsz := range valSizes {
			b.Run(fmt.Sprintf("db=%s valuesize=%d", name, vsz), func(b *testing.B) {
				b.RunParallel(func(pb *testing.PB) {
					keys := make([][]byte, batchSize)
					vals := make([][]byte, batchSize)
					for pb.Next() {
						for j := 0; j < batchSize; j++ {
							keys[j] = []byte(fmt.Sprintf("%016d", rand.Int()))
							vals[j] = make([]byte, vsz)
							rand.Read(vals[j])
						}
						db.BatchPut(ctx, keys, vals)
					}
				})
			})
		}
	}
}

func TestMain(m *testing.M) {
	// call flag.Parse() here if TestMain uses flags
	go http.ListenAndServe(":8080", nil)
	os.Exit(m.Run())
}
<commit_msg>Add benchmark for iteration<commit_after>package main

import (
	"context"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/http"
	"os"
	"runtime/pprof"
	"testing"
	"time"

	"github.com/dgraph-io/badger/badger"
	"github.com/dgraph-io/badger/value"
	"github.com/dgraph-io/badger/y"
)

func writeBatch(bdb *badger.DB, max int) int {
	entries := make([]value.Entry, 0, 100)
	for i := 0; i < 10000; i++ {
		v := make([]byte, 10)
		rand.Read(v)
		e := value.Entry{
			Key:   []byte(fmt.Sprintf("%016d", rand.Int()%max)),
			Value: v,
		}
		entries = append(entries, e)
	}
	y.Check(bdb.Write(context.Background(), entries))
	return len(entries)
}

func BenchmarkIterate(b *testing.B) {
	opt := badger.DefaultOptions
	opt.Verbose = true
	dir, err := ioutil.TempDir("tmp", "badger")
	Check(err)
	opt.Dir = dir
	bdb := badger.NewDB(&opt)

	nw := 10000000
	for written := 0; written < nw; {
		written += writeBatch(bdb, nw*10)
	}
	bdb.Close()
	b.Log("Sleeping for 10 seconds to allow compaction.")
	time.Sleep(time.Second)

	opt.DoNotCompact = true
	bdb = badger.NewDB(&opt)
	b.ResetTimer()

	f, err := os.Create("cpu.prof")
	if err != nil {
		b.Fatalf("Error: %v", err)
	}
	pprof.StartCPUProfile(f)
	defer pprof.StopCPUProfile()

	b.Run(fmt.Sprintf("writes=%d", nw), func(b *testing.B) {
		for j := 0; j < b.N; j++ {
			itr := bdb.NewIterator(context.Background())
			var count int
			for itr.SeekToFirst(); itr.Valid(); itr.Next() {
				count++
			}
			b.Logf("[%d] Counted %d keys\n", j, count)
		}
	})
	// bdb.Close()
}

func BenchmarkWriteBatchRandom(b *testing.B) {
	ctx := context.Background()

	bd := new(BadgerAdapter)
	bd.Init("/tmp/bench-tmp")
	defer bd.Close()

	rd := new(RocksDBAdapter)
	rd.Init("/tmp/bench-tmp")
	defer rd.Close()

	batchSize := 1000
	valSizes := []int{100, 1000, 10000, 100000}

	for i := 0; i < 2; i++ {
		var db Database
		name := "badger"
		db = bd
		if i == 1 {
			db = rd
			name = "rocksdb"
		}
		for _, vsz := range valSizes {
			b.Run(fmt.Sprintf("db=%s valuesize=%d", name, vsz), func(b *testing.B) {
				b.RunParallel(func(pb *testing.PB) {
					keys := make([][]byte, batchSize)
					vals := make([][]byte, batchSize)
					for pb.Next() {
						for j := 0; j < batchSize; j++ {
							keys[j] = []byte(fmt.Sprintf("%016d", rand.Int()))
							vals[j] = make([]byte, vsz)
							rand.Read(vals[j])
						}
						db.BatchPut(ctx, keys, vals)
					}
				})
			})
		}
	}
}

func TestMain(m *testing.M) {
	// call flag.Parse() here if TestMain uses flags
	go http.ListenAndServe(":8080", nil)
	os.Exit(m.Run())
}
<|endoftext|>


<commit_before>// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"strconv"
	"strings"
	"text/template"

	"github.com/google/syzkaller/pkg/compiler"
	"github.com/google/syzkaller/pkg/osutil"
)

func extract(info *compiler.ConstInfo, cc string, args []string, addSource string, declarePrintf bool) (map[string]uint64, map[string]bool, error) {
	data := &CompileData{
		AddSource:     addSource,
		Defines:       info.Defines,
		Includes:      info.Includes,
		Values:        info.Consts,
		DeclarePrintf: declarePrintf,
	}
	undeclared := make(map[string]bool)
	bin, out, err := compile(cc, args, data)
	if err != nil {
		// Some consts and syscall numbers are not defined on some archs.
		// Figure out from compiler output undefined consts,
		// and try to compile again without them.
		valMap := make(map[string]bool)
		for _, val := range info.Consts {
			valMap[val] = true
		}
		for _, errMsg := range []string{
			"error: ‘([a-zA-Z0-9_]+)’ undeclared",
			"error: '([a-zA-Z0-9_]+)' undeclared",
			"note: in expansion of macro ‘([a-zA-Z0-9_]+)’",
			"error: use of undeclared identifier '([a-zA-Z0-9_]+)'",
		} {
			re := regexp.MustCompile(errMsg)
			matches := re.FindAllSubmatch(out, -1)
			for _, match := range matches {
				val := string(match[1])
				if valMap[val] {
					undeclared[val] = true
				}
			}
		}
		data.Values = nil
		for _, v := range info.Consts {
			if undeclared[v] {
				continue
			}
			data.Values = append(data.Values, v)
		}
		bin, out, err = compile(cc, args, data)
		if err != nil {
			return nil, nil, fmt.Errorf("failed to run compiler: %v\n%v", err, string(out))
		}
	}
	defer os.Remove(bin)

	out, err = osutil.Command(bin).CombinedOutput()
	if err != nil {
		return nil, nil, fmt.Errorf("failed to run flags binary: %v\n%v", err, string(out))
	}
	flagVals := strings.Split(string(out), " ")
	if len(out) == 0 {
		flagVals = nil
	}
	if len(flagVals) != len(data.Values) {
		return nil, nil, fmt.Errorf("fetched wrong number of values %v, want != %v",
			len(flagVals), len(data.Values))
	}
	res := make(map[string]uint64)
	for i, name := range data.Values {
		val := flagVals[i]
		n, err := strconv.ParseUint(val, 10, 64)
		if err != nil {
			return nil, nil, fmt.Errorf("failed to parse value: %v (%v)", err, val)
		}
		res[name] = n
	}
	return res, undeclared, nil
}

type CompileData struct {
	AddSource     string
	Defines       map[string]string
	Includes      []string
	Values        []string
	DeclarePrintf bool
}

func compile(cc string, args []string, data *CompileData) (bin string, out []byte, err error) {
	srcFile, err := ioutil.TempFile("", "")
	if err != nil {
		return "", nil, fmt.Errorf("failed to create temp file: %v", err)
	}
	srcFile.Close()
	os.Remove(srcFile.Name())
	srcName := srcFile.Name() + ".c"
	defer os.Remove(srcName)
	src := new(bytes.Buffer)
	if err := srcTemplate.Execute(src, data); err != nil {
		return "", nil, fmt.Errorf("failed to generate source: %v", err)
	}
	if err := ioutil.WriteFile(srcName, src.Bytes(), 0600); err != nil {
		return "", nil, fmt.Errorf("failed to write source file: %v", err)
	}

	binFile, err := ioutil.TempFile("", "")
	if err != nil {
		return "", nil, fmt.Errorf("failed to create temp file: %v", err)
	}
	binFile.Close()

	args = append(args, []string{
		srcName,
		"-o", binFile.Name(),
		"-w",
	}...)
	out, err = osutil.Command(cc, args...).CombinedOutput()
	if err != nil {
		os.Remove(binFile.Name())
		return "", out, err
	}
	return binFile.Name(), nil, nil
}

var srcTemplate = template.Must(template.New("").Parse(`
{{range $incl := $.Includes}}
#include <{{$incl}}>
{{end}}

{{range $name, $val := $.Defines}}
#ifndef {{$name}}
#	define {{$name}} {{$val}}
#endif
{{end}}

{{.AddSource}}

{{if .DeclarePrintf}}
int printf(const char *format, ...);
{{end}}

int main() {
	int i;
	unsigned long long vals[] = {
		{{range $val := $.Values}}(unsigned long long){{$val}},{{end}}
	};
	for (i = 0; i < sizeof(vals)/sizeof(vals[0]); i++) {
		if (i != 0)
			printf(" ");
		printf("%llu", vals[i]);
	}
	return 0;
}
`))
<commit_msg>sys/syz-extract: fix linux/arm<commit_after>// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"strconv"
	"strings"
	"text/template"

	"github.com/google/syzkaller/pkg/compiler"
	"github.com/google/syzkaller/pkg/osutil"
)

func extract(info *compiler.ConstInfo, cc string, args []string, addSource string, declarePrintf bool) (map[string]uint64, map[string]bool, error) {
	data := &CompileData{
		AddSource:     addSource,
		Defines:       info.Defines,
		Includes:      info.Includes,
		Values:        info.Consts,
		DeclarePrintf: declarePrintf,
	}
	undeclared := make(map[string]bool)
	bin, out, err := compile(cc, args, data)
	if err != nil {
		// Some consts and syscall numbers are not defined on some archs.
		// Figure out from compiler output undefined consts,
		// and try to compile again without them.
		valMap := make(map[string]bool)
		for _, val := range info.Consts {
			valMap[val] = true
		}
		for _, errMsg := range []string{
			"error: ‘([a-zA-Z0-9_]+)’ undeclared",
			"error: '([a-zA-Z0-9_]+)' undeclared",
			"note: in expansion of macro ‘([a-zA-Z0-9_]+)’",
			"error: use of undeclared identifier '([a-zA-Z0-9_]+)'",
		} {
			re := regexp.MustCompile(errMsg)
			matches := re.FindAllSubmatch(out, -1)
			for _, match := range matches {
				val := string(match[1])
				if valMap[val] {
					undeclared[val] = true
				}
			}
		}
		data.Values = nil
		for _, v := range info.Consts {
			if undeclared[v] {
				continue
			}
			data.Values = append(data.Values, v)
		}
		bin, out, err = compile(cc, args, data)
		if err != nil {
			return nil, nil, fmt.Errorf("failed to run compiler: %v\n%v", err, string(out))
		}
	}
	defer os.Remove(bin)

	out, err = osutil.Command(bin).CombinedOutput()
	if err != nil {
		return nil, nil, fmt.Errorf("failed to run flags binary: %v\n%v", err, string(out))
	}
	flagVals := strings.Split(string(out), " ")
	if len(out) == 0 {
		flagVals = nil
	}
	if len(flagVals) != len(data.Values) {
		return nil, nil, fmt.Errorf("fetched wrong number of values %v, want != %v",
			len(flagVals), len(data.Values))
	}
	res := make(map[string]uint64)
	for i, name := range data.Values {
		val := flagVals[i]
		n, err := strconv.ParseUint(val, 10, 64)
		if err != nil {
			return nil, nil, fmt.Errorf("failed to parse value: %v (%v)", err, val)
		}
		res[name] = n
	}
	return res, undeclared, nil
}

type CompileData struct {
	AddSource     string
	Defines       map[string]string
	Includes      []string
	Values        []string
	DeclarePrintf bool
}

func compile(cc string, args []string, data *CompileData) (bin string, out []byte, err error) {
	srcFile, err := ioutil.TempFile("", "")
	if err != nil {
		return "", nil, fmt.Errorf("failed to create temp file: %v", err)
	}
	srcFile.Close()
	os.Remove(srcFile.Name())
	srcName := srcFile.Name() + ".c"
	defer os.Remove(srcName)
	src := new(bytes.Buffer)
	if err := srcTemplate.Execute(src, data); err != nil {
		return "", nil, fmt.Errorf("failed to generate source: %v", err)
	}
	if err := ioutil.WriteFile(srcName, src.Bytes(), 0600); err != nil {
		return "", nil, fmt.Errorf("failed to write source file: %v", err)
	}

	binFile, err := ioutil.TempFile("", "")
	if err != nil {
		return "", nil, fmt.Errorf("failed to create temp file: %v", err)
	}
	binFile.Close()

	args = append(args, []string{
		srcName,
		"-o", binFile.Name(),
		"-w",
	}...)
	out, err = osutil.Command(cc, args...).CombinedOutput()
	if err != nil {
		os.Remove(binFile.Name())
		return "", out, err
	}
	return binFile.Name(), nil, nil
}

var srcTemplate = template.Must(template.New("").Parse(`
#define __asm__(...)

{{range $incl := $.Includes}}
#include <{{$incl}}>
{{end}}

{{range $name, $val := $.Defines}}
#ifndef {{$name}}
#	define {{$name}} {{$val}}
#endif
{{end}}

{{.AddSource}}

{{if .DeclarePrintf}}
int printf(const char *format, ...);
{{end}}

int main() {
	int i;
	unsigned long long vals[] = {
		{{range $val := $.Values}}(unsigned long long){{$val}},{{end}}
	};
	for (i = 0; i < sizeof(vals)/sizeof(vals[0]); i++) {
		if (i != 0)
			printf(" ");
		printf("%llu", vals[i]);
	}
	return 0;
}
`))
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"syscall"

	"github.com/lomik/carbon-clickhouse/carbon"
	"github.com/lomik/carbon-clickhouse/helper/RowBinary"
	"github.com/lomik/zapwriter"
	"go.uber.org/zap"

	_ "net/http/pprof"
)

// Version of carbon-clickhouse
const Version = "0.11.0.custom.3"

func httpServe(addr string) (func(), error) {
	tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
	if err != nil {
		return nil, err
	}

	listener, err := net.ListenTCP("tcp", tcpAddr)
	if err != nil {
		return nil, err
	}

	go http.Serve(listener, nil)
	return func() { listener.Close() }, nil
}

func main() {
	var err error

	/* CONFIG start */

	configFile := flag.String("config", "/etc/carbon-clickhouse/carbon-clickhouse.conf", "Filename of config")
	printDefaultConfig := flag.Bool("config-print-default", false, "Print default config")
	checkConfig := flag.Bool("check-config", false, "Check config and exit")
	printVersion := flag.Bool("version", false, "Print version")
	cat := flag.String("cat", "", "Print RowBinary file in TabSeparated format")
	bincat := flag.String("recover", "", "Read all good records from corrupted data file. Write binary data to stdout")

	flag.Parse()

	if *printVersion {
		fmt.Print(Version)
		return
	}

	if *cat != "" {
		reader, err := RowBinary.NewReader(*cat, false)
		if err != nil {
			log.Fatal(err)
		}

		for {
			metric, err := reader.ReadRecord()
			if err != nil {
				if err == io.EOF {
					return
				}
				log.Fatal(err)
			}

			fmt.Printf("%s\t%#v\t%d\t%s\t%d\n",
				string(metric),
				reader.Value(),
				reader.Timestamp(),
				reader.DaysString(),
				reader.Version(),
			)
		}
	}

	if *bincat != "" {
		reader, err := RowBinary.NewReader(*bincat, false)
		if err != nil {
			log.Fatal(err)
		}

		io.Copy(os.Stdout, reader)
		return
	}

	if *printDefaultConfig {
		if err = carbon.PrintDefaultConfig(); err != nil {
			log.Fatal(err)
		}
		return
	}

	app := carbon.New(*configFile)

	if err = app.ParseConfig(); err != nil {
		log.Fatal(err)
	}

	// config parsed successfully. Exit in check-only mode
	if *checkConfig {
		return
	}

	cfg := app.Config

	if err = zapwriter.ApplyConfig(cfg.Logging); err != nil {
		log.Fatal(err)
	}

	mainLogger := zapwriter.Logger("main")

	/* CONFIG end */

	// pprof
	if cfg.Pprof.Enabled {
		_, err = httpServe(cfg.Pprof.Listen)
		if err != nil {
			mainLogger.Fatal("pprof listen failed", zap.Error(err))
		}
	}

	if err = app.Start(); err != nil {
		mainLogger.Fatal("app start failed", zap.Error(err))
	} else {
		mainLogger.Info("app started")
	}

	go func() {
		c := make(chan os.Signal, 1)
		signal.Notify(c, syscall.SIGUSR1, syscall.SIGUSR2, syscall.SIGHUP)

		for {
			s := <-c
			switch s {
			case syscall.SIGUSR1:
				mainLogger.Info("SIGUSR1 received. Clear tree cache")
				app.Reset()
			case syscall.SIGUSR2:
				mainLogger.Info("SIGUSR2 received. Ignoring")
			case syscall.SIGHUP:
				mainLogger.Info("SIGHUP received. Ignoring")
			}
		}
	}()

	app.Loop()

	mainLogger.Info("app stopped")
}
<commit_msg>AD-12849 remove version for upstream<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"syscall"

	"github.com/lomik/carbon-clickhouse/carbon"
	"github.com/lomik/carbon-clickhouse/helper/RowBinary"
	"github.com/lomik/zapwriter"
	"go.uber.org/zap"

	_ "net/http/pprof"
)

// Version of carbon-clickhouse
const Version = "0.11.0"

func httpServe(addr string) (func(), error) {
	tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
	if err != nil {
		return nil, err
	}

	listener, err := net.ListenTCP("tcp", tcpAddr)
	if err != nil {
		return nil, err
	}

	go http.Serve(listener, nil)
	return func() { listener.Close() }, nil
}

func main() {
	var err error

	/* CONFIG start */

	configFile := flag.String("config", "/etc/carbon-clickhouse/carbon-clickhouse.conf", "Filename of config")
	printDefaultConfig := flag.Bool("config-print-default", false, "Print default config")
	checkConfig := flag.Bool("check-config", false, "Check config and exit")
	printVersion := flag.Bool("version", false, "Print version")
	cat := flag.String("cat", "", "Print RowBinary file in TabSeparated format")
	bincat := flag.String("recover", "", "Read all good records from corrupted data file. Write binary data to stdout")

	flag.Parse()

	if *printVersion {
		fmt.Print(Version)
		return
	}

	if *cat != "" {
		reader, err := RowBinary.NewReader(*cat, false)
		if err != nil {
			log.Fatal(err)
		}

		for {
			metric, err := reader.ReadRecord()
			if err != nil {
				if err == io.EOF {
					return
				}
				log.Fatal(err)
			}

			fmt.Printf("%s\t%#v\t%d\t%s\t%d\n",
				string(metric),
				reader.Value(),
				reader.Timestamp(),
				reader.DaysString(),
				reader.Version(),
			)
		}
	}

	if *bincat != "" {
		reader, err := RowBinary.NewReader(*bincat, false)
		if err != nil {
			log.Fatal(err)
		}

		io.Copy(os.Stdout, reader)
		return
	}

	if *printDefaultConfig {
		if err = carbon.PrintDefaultConfig(); err != nil {
			log.Fatal(err)
		}
		return
	}

	app := carbon.New(*configFile)

	if err = app.ParseConfig(); err != nil {
		log.Fatal(err)
	}

	// config parsed successfully. Exit in check-only mode
	if *checkConfig {
		return
	}

	cfg := app.Config

	if err = zapwriter.ApplyConfig(cfg.Logging); err != nil {
		log.Fatal(err)
	}

	mainLogger := zapwriter.Logger("main")

	/* CONFIG end */

	// pprof
	if cfg.Pprof.Enabled {
		_, err = httpServe(cfg.Pprof.Listen)
		if err != nil {
			mainLogger.Fatal("pprof listen failed", zap.Error(err))
		}
	}

	if err = app.Start(); err != nil {
		mainLogger.Fatal("app start failed", zap.Error(err))
	} else {
		mainLogger.Info("app started")
	}

	go func() {
		c := make(chan os.Signal, 1)
		signal.Notify(c, syscall.SIGUSR1, syscall.SIGUSR2, syscall.SIGHUP)

		for {
			s := <-c
			switch s {
			case syscall.SIGUSR1:
				mainLogger.Info("SIGUSR1 received. Clear tree cache")
				app.Reset()
			case syscall.SIGUSR2:
				mainLogger.Info("SIGUSR2 received. Ignoring")
			case syscall.SIGHUP:
				mainLogger.Info("SIGHUP received. Ignoring")
			}
		}
	}()

	app.Loop()

	mainLogger.Info("app stopped")
}
<|endoftext|>


<commit_before>// Copyright 2014-2015 Canopy Services, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main

import (
    "encoding/json"
    "fmt"
    "net/http"
    "github.com/gorilla/mux"
)

type CanopySimService struct {
    droneCnt int64
}

var gService = CanopySimService{}

func main() {
    r := mux.NewRouter().StrictSlash(false)

    r.HandleFunc("/drones_started", DronesStartedHandler)

    fmt.Println("Starting server on :8383")
    http.ListenAndServe(":8383", r)
}

/*
 * Takes payload:
 *  {
 *      "cnt" : 1,
 *      "testname" : "dev02.canopy.link:myTest"
 *  }
 */
func DronesStartedHandler(w http.ResponseWriter, r *http.Request) {
    inPayload, err := ReadAndDecodeRequestBody(r)
    if err != nil {
        fmt.Fprintf(w, "{\"error\" : \"%s\"}\n", err.Error())
        return
    }

    cnt_f64, ok := inPayload["cnt"].(float64)
    if !ok {
        fmt.Fprintf(w, "{\"error\" : \"Expected integer field \\\"cnt\\\"\"}\n")
        return
    }
    cnt := int64(cnt_f64)

    testname, ok := inPayload["testname"].(string)
    if !ok {
        fmt.Fprintf(w, "{\"error\" : \"Expected string field \\\"testname\\\"\"}\n")
        return
    }
    fmt.Println(inPayload, cnt, testname)
    gService.droneCnt += cnt
    fmt.Fprintf(w, "{\"drone_cnt\" : %d}\n", gService.droneCnt)
}

func ReadAndDecodeRequestBody(r *http.Request) (map[string]interface{}, error) {
    var out map[string]interface{}
    decoder := json.NewDecoder(r.Body)
    err := decoder.Decode(&out)
    if err != nil {
        return nil, fmt.Errorf("Error decoding body"+ err.Error())
    }
    return out, nil
}

<commit_msg>Add support for multiple tests<commit_after>// Copyright 2014-2015 Canopy Services, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main

import (
    "encoding/json"
    "fmt"
    "net/http"
    "github.com/gorilla/mux"
)

type CanopySimTest struct {
    droneCnt int64
    testname string
}

type CanopySimService struct {
    tests map[string]*CanopySimTest
}

var gService = CanopySimService{
    tests: map[string]*CanopySimTest{},
}

func main() {
    r := mux.NewRouter().StrictSlash(false)

    r.HandleFunc("/drones_started", DronesStartedHandler)

    fmt.Println("Starting server on :8383")
    http.ListenAndServe(":8383", r)
}

/*
 * Takes payload:
 *  {
 *      "cnt" : 1,
 *      "testname" : "dev02.canopy.link:myTest"
 *  }
 */
func DronesStartedHandler(w http.ResponseWriter, r *http.Request) {
    // Decode payload
    inPayload, err := ReadAndDecodeRequestBody(r)
    if err != nil {
        fmt.Fprintf(w, "{\"error\" : \"%s\"}\n", err.Error())
        return
    }

    // Read fields
    cnt_f64, ok := inPayload["cnt"].(float64)
    if !ok {
        fmt.Fprintf(w, "{\"error\" : \"Expected integer field \\\"cnt\\\"\"}\n")
        return
    }
    cnt := int64(cnt_f64)

    testname, ok := inPayload["testname"].(string)
    if !ok {
        fmt.Fprintf(w, "{\"error\" : \"Expected string field \\\"testname\\\"\"}\n")
        return
    }

    // Create test if necessary
    test, ok := gService.tests[testname]
    if !ok {
        fmt.Println("creating test", testname)
        test = &CanopySimTest{
            testname: testname,
        }
        gService.tests[testname] = test
    }
    test.droneCnt += cnt
    fmt.Fprintf(w, "{\"testname\" : \"%s\", \"drone_cnt\" : %d}\n", test.testname, test.droneCnt)
}

func ReadAndDecodeRequestBody(r *http.Request) (map[string]interface{}, error) {
    var out map[string]interface{}
    decoder := json.NewDecoder(r.Body)
    err := decoder.Decode(&out)
    if err != nil {
        return nil, fmt.Errorf("Error decoding body"+ err.Error())
    }
    return out, nil
}

<|endoftext|>


<commit_before>package aphgrpc

import (
	"fmt"
	"math"
	"net/http"
	"strings"
	"time"

	"gopkg.in/mgutz/dat.v2/dat"
	"gopkg.in/mgutz/dat.v2/sqlx-runner"

	"github.com/dictyBase/go-genproto/dictybaseapis/api/jsonapi"
	"github.com/fatih/structs"
	"github.com/golang/protobuf/proto"
	"github.com/golang/protobuf/ptypes"
	"github.com/golang/protobuf/ptypes/timestamp"
	"github.com/grpc-ecosystem/grpc-gateway/runtime"
	context "golang.org/x/net/context"
	"google.golang.org/grpc/metadata"
)

const (
	DefaultPagenum  int64 = 1
	DefaultPagesize int64 = 10
)

// JSONAPIParamsInfo interface should be implement by all grpc-gateway services
// that supports JSON API specifications.
type JSONAPIParamsInfo interface {
	// Relationships that could be included
	AllowedInclude() []string
	// Attribute fields that are allowed
	AllowedFields() []string
	// Filter fields that are allowed
	AllowedFilter() []string
	// FilterToColumns provides mapping between filter and storage columns
	FilterToColumns() map[string]string
	// RequiredAttrs are the mandatory attributes for creating a new resource
	RequiredAttrs() []string
}

// JSONAPIResource interface provides information about HTTP resource. All
// grpc-gateway services that supports JSONAPI should implement this interface.
type JSONAPIResource interface {
	//GetResourceName returns canonical resource name
	GetResourceName() string
	// GetBaseURL returns the base url with the scheme
	GetBaseURL() string
	// GetPrefix returns the path that could be appended to base url
	GetPathPrefix() string
}

func NullToTime(nt dat.NullTime) *timestamp.Timestamp {
	var ts *timestamp.Timestamp
	if nt.Valid {
		ts, _ := ptypes.TimestampProto(nt.Time)
		return ts
	}
	return ts
}

func ProtoTimeStamp(ts *timestamp.Timestamp) time.Time {
	t, _ := ptypes.Timestamp(ts)
	return t
}

func TimestampProto(t time.Time) *timestamp.Timestamp {
	ts, _ := ptypes.TimestampProto(t)
	return ts
}

func NullToString(s dat.NullString) string {
	if s.Valid {
		return s.String
	}
	return ""
}

func NullToInt64(i dat.NullInt64) int64 {
	if i.Valid {
		return i.Int64
	}
	var i64 int64
	return i64
}

// GetTotalPageNum calculate total no of pages from total no. records and page size
func GetTotalPageNum(record, pagesize int64) int64 {
	total := int64(math.Floor(float64(record) / float64(pagesize)))
	if math.Mod(float64(record), float64(pagesize)) > 0 {
		total += 1
	}
	return total
}

// GenPaginatedLinks generates paginated resource links
// from various page properties.
func GenPaginatedLinks(url string, lastpage, pagenum, pagesize int64) map[string]string {
	links := make(map[string]string)
	links["self"] = AppendPaginationParams(url, pagenum, pagesize)
	links["first"] = AppendPaginationParams(url, 1, pagesize)
	if pagenum != 1 {
		links["previous"] = AppendPaginationParams(url, pagenum-1, pagesize)
	}
	links["last"] = AppendPaginationParams(url, lastpage, pagesize)
	if pagenum != lastpage {
		links["next"] = AppendPaginationParams(url, pagenum+1, pagesize)
	}
	return links
}

func GenBaseLink(rs JSONAPIResource) string {
	return fmt.Sprintf(
		"%s/%s",
		strings.Trim(rs.GetBaseURL(), "/"),
		strings.Trim(rs.GetPathPrefix(), "/"),
	)
}

func GenSingleResourceLink(rs JSONAPIResource, id int64) string {
	return fmt.Sprintf(
		"%s/%d",
		GenBaseLink(rs),
		id,
	)
}

func GenMultiResourceLink(rs JSONAPIResource) string {
	return GenBaseLink(rs)
}

func AppendPaginationParams(url string, pagenum, pagesize int64) string {
	return fmt.Sprintf("%s?pagenum=%d&pagesize=%d", url, pagenum, pagesize)
}

func GenPaginatedResourceLink(rs JSONAPIResource, pagenum, pagesize int64) string {
	return fmt.Sprintf(
		"%s?pagenum=%d&pagesize=%d",
		GenBaseLink(rs),
		pagenum,
		pagesize,
	)
}

func GenSelfRelationshipLink(rs JSONAPIResource, rel string, id int64) string {
	return fmt.Sprintf(
		"%s/%d/relationships/%s",
		GenBaseLink(rs),
		id,
		rel,
	)
}

func GenRelatedRelationshipLink(rs JSONAPIResource, rel string, id int64) string {
	return fmt.Sprintf(
		"%s/%d/%s",
		GenBaseLink(rs),
		id,
		rel,
	)
}

//GetDefinedTagsWithValue check for fields that are initialized and returns a map
//with the tag and their values
func GetDefinedTagsWithValue(i interface{}, key string) map[string]interface{} {
	m := make(map[string]interface{})
	s := structs.New(i)
	for _, f := range s.Fields() {
		if !f.IsZero() {
			m[f.Tag(key)] = f.Value()
		}
	}
	return m
}

//GetDefinedTags check for fields that are initialized and returns a slice of
//their matching tag values
func GetDefinedTags(i interface{}, tag string) []string {
	var v []string
	s := structs.New(i)
	for _, f := range s.Fields() {
		if !f.IsZero() {
			v = append(v, f.Tag(tag))
		}
	}
	return v
}

// HandleCreateResponse modifies the grpc gateway filter which adds the JSON API header and
// modifies the http status response for POST request
func HandleCreateResponse(ctx context.Context, w http.ResponseWriter, resp proto.Message) error {
	w.Header().Set("Content-Type", "application/vnd.api+json")
	md, ok := runtime.ServerMetadataFromContext(ctx)
	if ok {
		trMD := md.TrailerMD
		if _, ok := trMD["method"]; ok {
			switch trMD["method"][0] {
			case "POST":
				w.WriteHeader(http.StatusCreated)
			case "POST_NO_CONTENT":
				w.WriteHeader(http.StatusNoContent)
			}
		}
	}
	return nil
}

type Service struct {
	Dbh             *runner.DB
	PathPrefix      string
	Include         []string
	IncludeStr      string
	FieldsToColumns map[string]string
	FieldsStr       string
	Resource        string
	BaseURL         string
	FilToColumns    map[string]string
	FilterStr       string
	Params          *JSONAPIParams
	ListMethod      bool
	ReqAttrs        []string
}

func (s *Service) RequiredAttrs() []string {
	return s.ReqAttrs
}

func (s *Service) IsListMethod() bool {
	return s.ListMethod
}

func (s *Service) FilterToColumns() map[string]string {
	return s.FilToColumns
}

func (s *Service) AllowedFilter() []string {
	var f []string
	for k, _ := range s.FilterToColumns() {
		f = append(f, k)
	}
	return f
}

func (s *Service) AllowedInclude() []string {
	return s.Include
}

func (s *Service) AllowedFields() []string {
	var f []string
	for k, _ := range s.FieldsToColumns {
		f = append(f, k)
	}
	return f
}

func (s *Service) GetResourceName() string {
	return s.Resource
}

func (s *Service) GetBaseURL() string {
	return s.BaseURL
}

func (s *Service) GetPathPrefix() string {
	return s.PathPrefix
}

func (s *Service) SetBaseURL(ctx context.Context) error {
	md, ok := metadata.FromIncomingContext(ctx)
	if !ok {
		return ErrRetrieveMetadata
	}
	if slice, ok := md["x-forwarded-host"]; ok {
		s.BaseURL = fmt.Sprintf("http://%s", slice[0])
	}
	return nil
}

func (s *Service) MapFieldsToColumns(fields []string) []string {
	var columns []string
	for _, v := range fields {
		columns = append(columns, s.FieldsToColumns[v])
	}
	return columns
}

func (s *Service) GetCount(table string) (int64, error) {
	var count int64
	err := s.Dbh.Select("COUNT(*)").From(table).QueryScalar(&count)
	return count, err
}

func (s *Service) GetAllFilteredCount(table string) (int64, error) {
	var count int64
	err := s.Dbh.Select("COUNT(*)").
		From(table).
		Scope(
			FilterToWhereClause(s, s.Params.Filters),
			FilterToBindValue(s.Params.Filters)...,
		).QueryScalar(&count)
	return count, err
}

// GetRelatedPagination generates JSONAPI pagination links for relation resources
func (s *Service) GetRelatedPagination(id, record, pagenum, pagesize int64, relation string) (*jsonapi.PaginationLinks, int64) {
	pages := GetTotalPageNum(record, pagesize)
	baseLink := s.GenCollResourceRelSelfLink(id, relation)
	pageLinks := GenPaginatedLinks(baseLink, pages, pagenum, pagesize)
	jsapiLinks := &jsonapi.PaginationLinks{
		Self:  pageLinks["self"],
		Last:  pageLinks["last"],
		First: pageLinks["first"],
	}
	if _, ok := pageLinks["previous"]; ok {
		jsapiLinks.Prev = pageLinks["previous"]
	}
	if _, ok := pageLinks["next"]; ok {
		jsapiLinks.Next = pageLinks["next"]
	}
	return jsapiLinks, pages
}

// GetPagination generates JSONAPI pagination links along with fields, include and filter query parameters
func (s *Service) GetPagination(record, pagenum, pagesize int64) (*jsonapi.PaginationLinks, int64) {
	pages := GetTotalPageNum(record, pagesize)
	baseLink := s.GenCollResourceSelfLink()
	pageLinks := GenPaginatedLinks(baseLink, pages, pagenum, pagesize)
	pageType := []string{"self", "last", "first", "previous", "next"}

	if s.Params != nil {
		params := s.Params
		switch {
		case params.HasFields && params.HasInclude && params.HasFilter:
			for _, v := range pageType {
				if _, ok := pageLinks[v]; ok {
					pageLinks[v] += fmt.Sprintf("&fields=%s&include=%s&filter=%s", s.FieldsStr, s.IncludeStr, s.FilterStr)
				}
			}
		case params.HasFields && params.HasInclude:
			for _, v := range pageType {
				if _, ok := pageLinks[v]; ok {
					pageLinks[v] += fmt.Sprintf("&fields=%s&include=%s", s.FieldsStr, s.IncludeStr)
				}
			}
		case params.HasFields && params.HasFilter:
			for _, v := range pageType {
				if _, ok := pageLinks[v]; ok {
					pageLinks[v] += fmt.Sprintf("&fields=%s&filter=%s", s.FieldsStr, s.FilterStr)
				}
			}
		case params.HasInclude && params.HasFilter:
			for _, v := range pageType {
				if _, ok := pageLinks[v]; ok {
					pageLinks[v] += fmt.Sprintf("&include=%s&filter=%s", s.IncludeStr, s.FilterStr)
				}
			}
		case params.HasInclude:
			for _, v := range pageType {
				if _, ok := pageLinks[v]; ok {
					pageLinks[v] += fmt.Sprintf("&include=%s", s.IncludeStr)
				}
			}
		case params.HasFilter:
			for _, v := range pageType {
				if _, ok := pageLinks[v]; ok {
					pageLinks[v] += fmt.Sprintf("&filter=%s", s.FilterStr)
				}
			}
		case params.HasFields:
			for _, v := range pageType {
				if _, ok := pageLinks[v]; ok {
					pageLinks[v] += fmt.Sprintf("&fields=%s", s.FieldsStr)
				}
			}
		}
	}
	jsapiLinks := &jsonapi.PaginationLinks{
		Self:  pageLinks["self"],
		Last:  pageLinks["last"],
		First: pageLinks["first"],
	}
	if _, ok := pageLinks["previous"]; ok {
		jsapiLinks.Prev = pageLinks["previous"]
	}
	if _, ok := pageLinks["next"]; ok {
		jsapiLinks.Next = pageLinks["next"]
	}
	return jsapiLinks, pages
}

func (s *Service) GenCollResourceRelSelfLink(id int64, relation string) string {
	return fmt.Sprintf(
		"%s/%d/%s",
		GenMultiResourceLink(s),
		id,
		relation,
	)
}

func (s *Service) GenCollResourceSelfLink() string {
	link := GenMultiResourceLink(s)
	if s.Params == nil {
		return link
	}
	params := s.Params
	switch {
	case params.HasFields && params.HasFilter && params.HasInclude:
		link += fmt.Sprintf("?fields=%s&include=%s&filter=%s", s.FieldsStr, s.IncludeStr, s.FilterStr)
	case params.HasFields && params.HasFilter:
		link += fmt.Sprintf("?fields=%s&filter=%s", s.FieldsStr, s.FilterStr)
	case params.HasFields && params.HasInclude:
		link += fmt.Sprintf("?fields=%s&include=%s", s.FieldsStr, s.IncludeStr)
	case params.HasFilter && params.HasInclude:
		link += fmt.Sprintf("?filter=%s&include=%s", s.FilterStr, s.IncludeStr)
	case params.HasInclude:
		link += fmt.Sprintf("?include=%s", s.IncludeStr)
	case params.HasFilter:
		link += fmt.Sprintf("?filter=%s", s.FilterStr)
	case params.HasFields:
		link += fmt.Sprintf("?fields=%s", s.FieldsStr)
	}
	return link
}

func (s *Service) GenResourceSelfLink(id int64) string {
	links := GenSingleResourceLink(s, id)
	if !s.IsListMethod() && s.Params != nil {
		params := s.Params
		switch {
		case params.HasFields && params.HasInclude:
			links += fmt.Sprintf("?fields=%s&include=%s", s.FieldsStr, s.IncludeStr)
		case params.HasFields:
			links += fmt.Sprintf("?fields=%s", s.FieldsStr)
		case params.HasInclude:
			links += fmt.Sprintf("?include=%s", s.IncludeStr)
		}
	}
	return links
}
<commit_msg>Added grpc metadata probing for skipping of http links<commit_after>package aphgrpc

import (
	"fmt"
	"math"
	"net/http"
	"strings"
	"time"

	"gopkg.in/mgutz/dat.v2/dat"
	"gopkg.in/mgutz/dat.v2/sqlx-runner"

	"github.com/dictyBase/go-genproto/dictybaseapis/api/jsonapi"
	"github.com/fatih/structs"
	"github.com/golang/protobuf/proto"
	"github.com/golang/protobuf/ptypes"
	"github.com/golang/protobuf/ptypes/timestamp"
	"github.com/grpc-ecosystem/grpc-gateway/runtime"
	context "golang.org/x/net/context"
	"google.golang.org/grpc/metadata"
)

const (
	DefaultPagenum  int64 = 1
	DefaultPagesize int64 = 10
)

// JSONAPIParamsInfo interface should be implement by all grpc-gateway services
// that supports JSON API specifications.
type JSONAPIParamsInfo interface {
	// Relationships that could be included
	AllowedInclude() []string
	// Attribute fields that are allowed
	AllowedFields() []string
	// Filter fields that are allowed
	AllowedFilter() []string
	// FilterToColumns provides mapping between filter and storage columns
	FilterToColumns() map[string]string
	// RequiredAttrs are the mandatory attributes for creating a new resource
	RequiredAttrs() []string
}

// JSONAPIResource interface provides information about HTTP resource. All
// grpc-gateway services that supports JSONAPI should implement this interface.
type JSONAPIResource interface {
	//GetResourceName returns canonical resource name
	GetResourceName() string
	// GetBaseURL returns the base url with the scheme
	GetBaseURL() string
	// GetPrefix returns the path that could be appended to base url
	GetPathPrefix() string
}

func NullToTime(nt dat.NullTime) *timestamp.Timestamp {
	var ts *timestamp.Timestamp
	if nt.Valid {
		ts, _ := ptypes.TimestampProto(nt.Time)
		return ts
	}
	return ts
}

func ProtoTimeStamp(ts *timestamp.Timestamp) time.Time {
	t, _ := ptypes.Timestamp(ts)
	return t
}

func TimestampProto(t time.Time) *timestamp.Timestamp {
	ts, _ := ptypes.TimestampProto(t)
	return ts
}

func NullToString(s dat.NullString) string {
	if s.Valid {
		return s.String
	}
	return ""
}

func NullToInt64(i dat.NullInt64) int64 {
	if i.Valid {
		return i.Int64
	}
	var i64 int64
	return i64
}

// GetTotalPageNum calculate total no of pages from total no. records and page size
func GetTotalPageNum(record, pagesize int64) int64 {
	total := int64(math.Floor(float64(record) / float64(pagesize)))
	if math.Mod(float64(record), float64(pagesize)) > 0 {
		total += 1
	}
	return total
}

// GenPaginatedLinks generates paginated resource links
// from various page properties.
func GenPaginatedLinks(url string, lastpage, pagenum, pagesize int64) map[string]string {
	links := make(map[string]string)
	links["self"] = AppendPaginationParams(url, pagenum, pagesize)
	links["first"] = AppendPaginationParams(url, 1, pagesize)
	if pagenum != 1 {
		links["previous"] = AppendPaginationParams(url, pagenum-1, pagesize)
	}
	links["last"] = AppendPaginationParams(url, lastpage, pagesize)
	if pagenum != lastpage {
		links["next"] = AppendPaginationParams(url, pagenum+1, pagesize)
	}
	return links
}

func GenBaseLink(rs JSONAPIResource) string {
	return fmt.Sprintf(
		"%s/%s",
		strings.Trim(rs.GetBaseURL(), "/"),
		strings.Trim(rs.GetPathPrefix(), "/"),
	)
}

func GenSingleResourceLink(rs JSONAPIResource, id int64) string {
	return fmt.Sprintf(
		"%s/%d",
		GenBaseLink(rs),
		id,
	)
}

func GenMultiResourceLink(rs JSONAPIResource) string {
	return GenBaseLink(rs)
}

func AppendPaginationParams(url string, pagenum, pagesize int64) string {
	return fmt.Sprintf("%s?pagenum=%d&pagesize=%d", url, pagenum, pagesize)
}

func GenPaginatedResourceLink(rs JSONAPIResource, pagenum, pagesize int64) string {
	return fmt.Sprintf(
		"%s?pagenum=%d&pagesize=%d",
		GenBaseLink(rs),
		pagenum,
		pagesize,
	)
}

func GenSelfRelationshipLink(rs JSONAPIResource, rel string, id int64) string {
	return fmt.Sprintf(
		"%s/%d/relationships/%s",
		GenBaseLink(rs),
		id,
		rel,
	)
}

func GenRelatedRelationshipLink(rs JSONAPIResource, rel string, id int64) string {
	return fmt.Sprintf(
		"%s/%d/%s",
		GenBaseLink(rs),
		id,
		rel,
	)
}

//GetDefinedTagsWithValue check for fields that are initialized and returns a map
//with the tag and their values
func GetDefinedTagsWithValue(i interface{}, key string) map[string]interface{} {
	m := make(map[string]interface{})
	s := structs.New(i)
	for _, f := range s.Fields() {
		if !f.IsZero() {
			m[f.Tag(key)] = f.Value()
		}
	}
	return m
}

//GetDefinedTags check for fields that are initialized and returns a slice of
//their matching tag values
func GetDefinedTags(i interface{}, tag string) []string {
	var v []string
	s := structs.New(i)
	for _, f := range s.Fields() {
		if !f.IsZero() {
			v = append(v, f.Tag(tag))
		}
	}
	return v
}

// HandleCreateResponse modifies the grpc gateway filter which adds the JSON API header and
// modifies the http status response for POST request
func HandleCreateResponse(ctx context.Context, w http.ResponseWriter, resp proto.Message) error {
	w.Header().Set("Content-Type", "application/vnd.api+json")
	md, ok := runtime.ServerMetadataFromContext(ctx)
	if ok {
		trMD := md.TrailerMD
		if _, ok := trMD["method"]; ok {
			switch trMD["method"][0] {
			case "POST":
				w.WriteHeader(http.StatusCreated)
			case "POST_NO_CONTENT":
				w.WriteHeader(http.StatusNoContent)
			}
		}
	}
	return nil
}

// SkipHTTPLinks looks up the context for the presence of gprc metadata
// for skipping the generation of HTTP links
func SkipHTTPLinks(ctx context.Context) bool {
	md, ok := metadata.FromIncomingContext(ctx)
	if !ok {
		return false
	}
	if _, ok := md["skip-http-links"]; ok {
		return true
	}
	return false
}

type Service struct {
	Dbh             *runner.DB
	PathPrefix      string
	Include         []string
	IncludeStr      string
	FieldsToColumns map[string]string
	FieldsStr       string
	Resource        string
	BaseURL         string
	FilToColumns    map[string]string
	FilterStr       string
	Params          *JSONAPIParams
	ListMethod      bool
	ReqAttrs        []string
}

func (s *Service) RequiredAttrs() []string {
	return s.ReqAttrs
}

func (s *Service) IsListMethod() bool {
	return s.ListMethod
}

func (s *Service) FilterToColumns() map[string]string {
	return s.FilToColumns
}

func (s *Service) AllowedFilter() []string {
	var f []string
	for k, _ := range s.FilterToColumns() {
		f = append(f, k)
	}
	return f
}

func (s *Service) AllowedInclude() []string {
	return s.Include
}

func (s *Service) AllowedFields() []string {
	var f []string
	for k, _ := range s.FieldsToColumns {
		f = append(f, k)
	}
	return f
}

func (s *Service) GetResourceName() string {
	return s.Resource
}

func (s *Service) GetBaseURL() string {
	return s.BaseURL
}

func (s *Service) GetPathPrefix() string {
	return s.PathPrefix
}

func (s *Service) SetBaseURL(ctx context.Context) error {
	md, ok := metadata.FromIncomingContext(ctx)
	if !ok {
		return ErrRetrieveMetadata
	}
	if slice, ok := md["x-forwarded-host"]; ok {
		s.BaseURL = fmt.Sprintf("http://%s", slice[0])
	}
	return nil
}

func (s *Service) MapFieldsToColumns(fields []string) []string {
	var columns []string
	for _, v := range fields {
		columns = append(columns, s.FieldsToColumns[v])
	}
	return columns
}

func (s *Service) GetCount(table string) (int64, error) {
	var count int64
	err := s.Dbh.Select("COUNT(*)").From(table).QueryScalar(&count)
	return count, err
}

func (s *Service) GetAllFilteredCount(table string) (int64, error) {
	var count int64
	err := s.Dbh.Select("COUNT(*)").
		From(table).
		Scope(
			FilterToWhereClause(s, s.Params.Filters),
			FilterToBindValue(s.Params.Filters)...,
		).QueryScalar(&count)
	return count, err
}

// GetRelatedPagination generates JSONAPI pagination links for relation resources
func (s *Service) GetRelatedPagination(id, record, pagenum, pagesize int64, relation string) (*jsonapi.PaginationLinks, int64) {
	pages := GetTotalPageNum(record, pagesize)
	baseLink := s.GenCollResourceRelSelfLink(id, relation)
	pageLinks := GenPaginatedLinks(baseLink, pages, pagenum, pagesize)
	jsapiLinks := &jsonapi.PaginationLinks{
		Self:  pageLinks["self"],
		Last:  pageLinks["last"],
		First: pageLinks["first"],
	}
	if _, ok := pageLinks["previous"]; ok {
		jsapiLinks.Prev = pageLinks["previous"]
	}
	if _, ok := pageLinks["next"]; ok {
		jsapiLinks.Next = pageLinks["next"]
	}
	return jsapiLinks, pages
}

// GetPagination generates JSONAPI pagination links along with fields, include and filter query parameters
func (s *Service) GetPagination(record, pagenum, pagesize int64) (*jsonapi.PaginationLinks, int64) {
	pages := GetTotalPageNum(record, pagesize)
	baseLink := s.GenCollResourceSelfLink()
	pageLinks := GenPaginatedLinks(baseLink, pages, pagenum, pagesize)
	pageType := []string{"self", "last", "first", "previous", "next"}

	if s.Params != nil {
		params := s.Params
		switch {
		case params.HasFields && params.HasInclude && params.HasFilter:
			for _, v := range pageType {
				if _, ok := pageLinks[v]; ok {
					pageLinks[v] += fmt.Sprintf("&fields=%s&include=%s&filter=%s", s.FieldsStr, s.IncludeStr, s.FilterStr)
				}
			}
		case params.HasFields && params.HasInclude:
			for _, v := range pageType {
				if _, ok := pageLinks[v]; ok {
					pageLinks[v] += fmt.Sprintf("&fields=%s&include=%s", s.FieldsStr, s.IncludeStr)
				}
			}
		case params.HasFields && params.HasFilter:
			for _, v := range pageType {
				if _, ok := pageLinks[v]; ok {
					pageLinks[v] += fmt.Sprintf("&fields=%s&filter=%s", s.FieldsStr, s.FilterStr)
				}
			}
		case params.HasInclude && params.HasFilter:
			for _, v := range pageType {
				if _, ok := pageLinks[v]; ok {
					pageLinks[v] += fmt.Sprintf("&include=%s&filter=%s", s.IncludeStr, s.FilterStr)
				}
			}
		case params.HasInclude:
			for _, v := range pageType {
				if _, ok := pageLinks[v]; ok {
					pageLinks[v] += fmt.Sprintf("&include=%s", s.IncludeStr)
				}
			}
		case params.HasFilter:
			for _, v := range pageType {
				if _, ok := pageLinks[v]; ok {
					pageLinks[v] += fmt.Sprintf("&filter=%s", s.FilterStr)
				}
			}
		case params.HasFields:
			for _, v := range pageType {
				if _, ok := pageLinks[v]; ok {
					pageLinks[v] += fmt.Sprintf("&fields=%s", s.FieldsStr)
				}
			}
		}
	}
	jsapiLinks := &jsonapi.PaginationLinks{
		Self:  pageLinks["self"],
		Last:  pageLinks["last"],
		First: pageLinks["first"],
	}
	if _, ok := pageLinks["previous"]; ok {
		jsapiLinks.Prev = pageLinks["previous"]
	}
	if _, ok := pageLinks["next"]; ok {
		jsapiLinks.Next = pageLinks["next"]
	}
	return jsapiLinks, pages
}

func (s *Service) GenCollResourceRelSelfLink(id int64, relation string) string {
	return fmt.Sprintf(
		"%s/%d/%s",
		GenMultiResourceLink(s),
		id,
		relation,
	)
}

func (s *Service) GenCollResourceSelfLink() string {
	link := GenMultiResourceLink(s)
	if s.Params == nil {
		return link
	}
	params := s.Params
	switch {
	case params.HasFields && params.HasFilter && params.HasInclude:
		link += fmt.Sprintf("?fields=%s&include=%s&filter=%s", s.FieldsStr, s.IncludeStr, s.FilterStr)
	case params.HasFields && params.HasFilter:
		link += fmt.Sprintf("?fields=%s&filter=%s", s.FieldsStr, s.FilterStr)
	case params.HasFields && params.HasInclude:
		link += fmt.Sprintf("?fields=%s&include=%s", s.FieldsStr, s.IncludeStr)
	case params.HasFilter && params.HasInclude:
		link += fmt.Sprintf("?filter=%s&include=%s", s.FilterStr, s.IncludeStr)
	case params.HasInclude:
		link += fmt.Sprintf("?include=%s", s.IncludeStr)
	case params.HasFilter:
		link += fmt.Sprintf("?filter=%s", s.FilterStr)
	case params.HasFields:
		link += fmt.Sprintf("?fields=%s", s.FieldsStr)
	}
	return link
}

func (s *Service) GenResourceSelfLink(id int64) string {
	links := GenSingleResourceLink(s, id)
	if !s.IsListMethod() && s.Params != nil {
		params := s.Params
		switch {
		case params.HasFields && params.HasInclude:
			links += fmt.Sprintf("?fields=%s&include=%s", s.FieldsStr, s.IncludeStr)
		case params.HasFields:
			links += fmt.Sprintf("?fields=%s", s.FieldsStr)
		case params.HasInclude:
			links += fmt.Sprintf("?include=%s", s.IncludeStr)
		}
	}
	return links
}
<|endoftext|>


<commit_before>package command

import (
	"fmt"
	"errors"

	"github.com/dnaeon/gru/task"

	"code.google.com/p/go-uuid/uuid"
	"github.com/codegangsta/cli"
)

func NewRunCommand() cli.Command {
	cmd := cli.Command{
		Name: "run",
		Usage: "send task to minion(s)",
		Action: execRunCommand,
		Flags: []cli.Flag{
			cli.BoolFlag{
				Name: "is-concurrent",
				Usage: "flag task as concurrent",
			},
			cli.Flag{
				Name: "with-classifier",
				Value: "",
				Usage: "match minions with given classifier pattern",
			},
		},
	}

	return cmd
}

// Executes the "run" command
func execRunCommand(c *cli.Context) {
	if len(c.Args()) < 1 {
		displayError(errors.New("Must provide command to run"), 64)
	}

	client := newEtcdMinionClientFromFlags(c)

	cFlag := c.String("with-classifier")
	minions, err := parseClassifierPattern(client, cFlag)

	if err != nil {
		displayError(err, 1)
	}

	numMinions := len(minions)
	if numMinions == 0 {
		displayError(errors.New("No minion(s) found"), 1)
	}

	fmt.Printf("Found %d minion(s) for task processing\n", numMinions)

	// The first argument is the command and anything else
	// that follows is considered as task arguments
	args := c.Args()
	isConcurrent := c.Bool("is-concurrent")
	taskCommand := args[0]
	taskArgs := args[1:]
	t := task.New(taskCommand, taskArgs...)
	t.IsConcurrent = isConcurrent

	failed := 0
	for i, minion := range minions {
		fmt.Printf("[%d/%d] Submitting task to minion %s\r", i + 1, numMinions, minion)
		err = client.MinionSubmitTask(minion, t)
		if err != nil {
			failed += 1
			fmt.Printf("\nFailed to submit task to %s: %s\n", minion, err)
		}
	}
	fmt.Println()

	fmt.Printf("Task submitted to %d minion(s), %d of which has failed\n", numMinions, failed)
	fmt.Printf("Task results can be retrieved by using this task id: %s\n", t.TaskID)
}
<commit_msg>gructl: typo fixes<commit_after>package command

import (
	"fmt"
	"errors"

	"github.com/dnaeon/gru/task"

	"github.com/codegangsta/cli"
)

func NewRunCommand() cli.Command {
	cmd := cli.Command{
		Name: "run",
		Usage: "send task to minion(s)",
		Action: execRunCommand,
		Flags: []cli.Flag{
			cli.BoolFlag{
				Name: "is-concurrent",
				Usage: "flag task as concurrent",
			},
			cli.StringFlag{
				Name: "with-classifier",
				Value: "",
				Usage: "match minions with given classifier pattern",
			},
		},
	}

	return cmd
}

// Executes the "run" command
func execRunCommand(c *cli.Context) {
	if len(c.Args()) < 1 {
		displayError(errors.New("Must provide command to run"), 64)
	}

	client := newEtcdMinionClientFromFlags(c)

	cFlag := c.String("with-classifier")
	minions, err := parseClassifierPattern(client, cFlag)

	if err != nil {
		displayError(err, 1)
	}

	numMinions := len(minions)
	if numMinions == 0 {
		displayError(errors.New("No minion(s) found"), 1)
	}

	fmt.Printf("Found %d minion(s) for task processing\n", numMinions)

	// The first argument is the command and anything else
	// that follows is considered as task arguments
	args := c.Args()
	isConcurrent := c.Bool("is-concurrent")
	taskCommand := args[0]
	taskArgs := args[1:]
	t := task.New(taskCommand, taskArgs...)
	t.IsConcurrent = isConcurrent

	failed := 0
	for i, minion := range minions {
		fmt.Printf("[%d/%d] Submitting task to minion %s\r", i + 1, numMinions, minion)
		err = client.MinionSubmitTask(minion, t)
		if err != nil {
			failed += 1
			fmt.Printf("\nFailed to submit task to %s: %s\n", minion, err)
		}
	}
	fmt.Println()

	fmt.Printf("Task submitted to %d minion(s), %d of which has failed\n", numMinions, failed)
	fmt.Printf("Task results can be retrieved by using this task id: %s\n", t.TaskID)
}
<|endoftext|>


<commit_before>package main

import (
	_ "expvar"
	"flag"
	"http"
	_ "http/pprof"
	"log"
	"net"
	"os"

	"chunkymonkey"
	"chunkymonkey/gamerules"
	"chunkymonkey/worldstore"
)

var addr = flag.String(
	"addr", ":25565",
	"Serves on the given address:port.")

var httpAddr = flag.String(
	"http_addr", ":25566",
	"Serves HTTP diagnostics on the given address:port.")

var blockDefs = flag.String(
	"blocks", "blocks.json",
	"The JSON file containing block type definitions.")

var itemDefs = flag.String(
	"items", "items.json",
	"The JSON file containing item type definitions.")

var recipeDefs = flag.String(
	"recipes", "recipes.json",
	"The JSON file containing recipe definitions.")

var furnaceDefs = flag.String(
	"furnace", "furnace.json",
	"The JSON file containing furnace fuel and reaction definitions.")

var serverDesc = flag.String(
	"server_desc", "Chunkymonkey Minecraft server",
	"The server description.")

var maintenanceMsg = flag.String(
	"maintenance_msg", "",
	"If set, all logins will be denied and this message will be given as reason.")

var userDefs = flag.String(
	"users", "users.json",
	"The JSON file container user permissions.")

var groupDefs = flag.String(
	"groups", "groups.json",
	"The JSON file containing group permissions.")

// TODO Implement max player count enforcement. Probably would have to be
// implemented atomically at the game level.
var maxPlayerCount = flag.Int(
	"max_player_count", 16,
	"Maximum number of players to allow concurrently. (Does not work yet)")

func usage() {
	os.Stderr.WriteString("usage: " + os.Args[0] + " [flags] <world>\n")
	flag.PrintDefaults()
}

func startHttpServer(addr string) (err error) {
	httpPort, err := net.Listen("tcp", addr)
	if err != nil {
		return
	}
	go http.Serve(httpPort, nil)
	return
}

func main() {
	var err error

	flag.Usage = usage
	flag.Parse()

	if flag.NArg() != 1 {
		flag.Usage()
		os.Exit(1)
	}

	err = gamerules.LoadGameRules(*blockDefs, *itemDefs, *recipeDefs, *furnaceDefs, *userDefs, *groupDefs)
	if err != nil {
		log.Print("Error loading game rules: ", err)
		os.Exit(1)
	}

	worldPath := flag.Arg(0)
	fi, err := os.Stat(worldPath)
	if err != nil {
		log.Printf("Could not load world from directory %v: %v", worldPath, err)
		log.Printf("Creating a new world in directory %v", worldPath)
		err = worldstore.CreateWorld(worldPath)
	}
	if err != nil {
		log.Printf("Error creating new world: %v", err)
	} else {
		fi, err = os.Stat(worldPath)
	}

	if fi == nil || !fi.IsDirectory() {
		log.Printf("Error loading world %v: Not a directory", worldPath)
		os.Exit(1)
	}

	listener, err := net.Listen("tcp", *addr)
	if err != nil {
		log.Fatal(err)
	}

	game, err := chunkymonkey.NewGame(worldPath, listener, *serverDesc, *maintenanceMsg, *maxPlayerCount)
	if err != nil {
		log.Fatal(err)
	}

	err = startHttpServer(*httpAddr)
	if err != nil {
		log.Fatal(err)
	}

	game.Serve()
}
<commit_msg>Now compiles again!<commit_after>package main

import (
	_ "expvar"
	"flag"
	"log"
	"net"
	"net/http"
	_ "net/http/pprof"
	"os"

	"chunkymonkey"
	"chunkymonkey/gamerules"
	"chunkymonkey/worldstore"
)

var addr = flag.String(
	"addr", ":25565",
	"Serves on the given address:port.")

var httpAddr = flag.String(
	"http_addr", ":25566",
	"Serves HTTP diagnostics on the given address:port.")

var blockDefs = flag.String(
	"blocks", "blocks.json",
	"The JSON file containing block type definitions.")

var itemDefs = flag.String(
	"items", "items.json",
	"The JSON file containing item type definitions.")

var recipeDefs = flag.String(
	"recipes", "recipes.json",
	"The JSON file containing recipe definitions.")

var furnaceDefs = flag.String(
	"furnace", "furnace.json",
	"The JSON file containing furnace fuel and reaction definitions.")

var serverDesc = flag.String(
	"server_desc", "Chunkymonkey Minecraft server",
	"The server description.")

var maintenanceMsg = flag.String(
	"maintenance_msg", "",
	"If set, all logins will be denied and this message will be given as reason.")

var userDefs = flag.String(
	"users", "users.json",
	"The JSON file container user permissions.")

var groupDefs = flag.String(
	"groups", "groups.json",
	"The JSON file containing group permissions.")

// TODO Implement max player count enforcement. Probably would have to be
// implemented atomically at the game level.
var maxPlayerCount = flag.Int(
	"max_player_count", 16,
	"Maximum number of players to allow concurrently. (Does not work yet)")

func usage() {
	os.Stderr.WriteString("usage: " + os.Args[0] + " [flags] <world>\n")
	flag.PrintDefaults()
}

func startHttpServer(addr string) (err error) {
	httpPort, err := net.Listen("tcp", addr)
	if err != nil {
		return
	}
	go http.Serve(httpPort, nil)
	return
}

func main() {
	var err error

	flag.Usage = usage
	flag.Parse()

	if flag.NArg() != 1 {
		flag.Usage()
		os.Exit(1)
	}

	err = gamerules.LoadGameRules(*blockDefs, *itemDefs, *recipeDefs, *furnaceDefs, *userDefs, *groupDefs)
	if err != nil {
		log.Print("Error loading game rules: ", err)
		os.Exit(1)
	}

	worldPath := flag.Arg(0)
	fi, err := os.Stat(worldPath)
	if err != nil {
		log.Printf("Could not load world from directory %v: %v", worldPath, err)
		log.Printf("Creating a new world in directory %v", worldPath)
		err = worldstore.CreateWorld(worldPath)
	}
	if err != nil {
		log.Printf("Error creating new world: %v", err)
	} else {
		fi, err = os.Stat(worldPath)
	}

	if fi == nil || !fi.IsDir() {
		log.Printf("Error loading world %v: Not a directory", worldPath)
		os.Exit(1)
	}

	listener, err := net.Listen("tcp", *addr)
	if err != nil {
		log.Fatal(err)
	}

	game, err := chunkymonkey.NewGame(worldPath, listener, *serverDesc, *maintenanceMsg, *maxPlayerCount)
	if err != nil {
		log.Fatal(err)
	}

	err = startHttpServer(*httpAddr)
	if err != nil {
		log.Fatal(err)
	}

	game.Serve()
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"restic"
	"restic/index"

	"github.com/spf13/cobra"
)

var cmdRebuildIndex = &cobra.Command{
	Use:   "rebuild-index [flags]",
	Short: "build a new index file",
	Long: `
The "rebuild-index" command creates a new index based on the pack files in the
repository.
`,
	RunE: func(cmd *cobra.Command, args []string) error {
		return runRebuildIndex(globalOptions)
	},
}

func init() {
	cmdRoot.AddCommand(cmdRebuildIndex)
}

func runRebuildIndex(gopts GlobalOptions) error {
	repo, err := OpenRepository(gopts)
	if err != nil {
		return err
	}

	lock, err := lockRepoExclusive(repo)
	defer unlockRepo(lock)
	if err != nil {
		return err
	}

	ctx, cancel := context.WithCancel(gopts.ctx)
	defer cancel()
	return rebuildIndex(ctx, repo, restic.NewIDSet())
}

func rebuildIndex(ctx context.Context, repo restic.Repository, ignorePacks restic.IDSet) error {
	Verbosef("counting files in repo\n")

	var packs uint64
	for range repo.List(ctx, restic.DataFile) {
		packs++
	}

	bar := newProgressMax(!globalOptions.Quiet, packs, "packs")
	idx, err := index.New(ctx, repo, ignorePacks, bar)
	if err != nil {
		return err
	}

	Verbosef("finding old index files\n")

	var supersedes restic.IDs
	for id := range repo.List(ctx, restic.IndexFile) {
		supersedes = append(supersedes, id)
	}

	id, err := idx.Save(ctx, repo, supersedes)
	if err != nil {
		return err
	}

	Verbosef("saved new index as %v\n", id.Str())

	Verbosef("remove %d old index files\n", len(supersedes))

	for _, id := range supersedes {
		if err := repo.Backend().Remove(ctx, restic.Handle{
			Type: restic.IndexFile,
			Name: id.String(),
		}); err != nil {
			Warnf("error removing old index %v: %v\n", id.Str(), err)
		}
	}

	return nil
}
<commit_msg>prune: Fix progress information<commit_after>package main

import (
	"context"
	"restic"
	"restic/index"

	"github.com/spf13/cobra"
)

var cmdRebuildIndex = &cobra.Command{
	Use:   "rebuild-index [flags]",
	Short: "build a new index file",
	Long: `
The "rebuild-index" command creates a new index based on the pack files in the
repository.
`,
	RunE: func(cmd *cobra.Command, args []string) error {
		return runRebuildIndex(globalOptions)
	},
}

func init() {
	cmdRoot.AddCommand(cmdRebuildIndex)
}

func runRebuildIndex(gopts GlobalOptions) error {
	repo, err := OpenRepository(gopts)
	if err != nil {
		return err
	}

	lock, err := lockRepoExclusive(repo)
	defer unlockRepo(lock)
	if err != nil {
		return err
	}

	ctx, cancel := context.WithCancel(gopts.ctx)
	defer cancel()
	return rebuildIndex(ctx, repo, restic.NewIDSet())
}

func rebuildIndex(ctx context.Context, repo restic.Repository, ignorePacks restic.IDSet) error {
	Verbosef("counting files in repo\n")

	var packs uint64
	for range repo.List(ctx, restic.DataFile) {
		packs++
	}

	bar := newProgressMax(!globalOptions.Quiet, packs-uint64(len(ignorePacks)), "packs")
	idx, err := index.New(ctx, repo, ignorePacks, bar)
	if err != nil {
		return err
	}

	Verbosef("finding old index files\n")

	var supersedes restic.IDs
	for id := range repo.List(ctx, restic.IndexFile) {
		supersedes = append(supersedes, id)
	}

	id, err := idx.Save(ctx, repo, supersedes)
	if err != nil {
		return err
	}

	Verbosef("saved new index as %v\n", id.Str())

	Verbosef("remove %d old index files\n", len(supersedes))

	for _, id := range supersedes {
		if err := repo.Backend().Remove(ctx, restic.Handle{
			Type: restic.IndexFile,
			Name: id.String(),
		}); err != nil {
			Warnf("error removing old index %v: %v\n", id.Str(), err)
		}
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fuse

import (
	"errors"

	"github.com/jacobsa/bazilfuse"
	"golang.org/x/net/context"
)

// A struct representing the status of a mount operation, with methods for
// waiting on the mount to complete, waiting for unmounting, and causing
// unmounting.
type MountedFileSystem struct {
	dir string

	// The result to return from WaitForReady. Not valid until the channel is
	// closed.
	readyStatus          error
	readyStatusAvailable chan struct{}

	// The result to return from Join. Not valid until the channel is closed.
	joinStatus          error
	joinStatusAvailable chan struct{}
}

// Return the directory on which the file system is mounted (or where we
// attempted to mount it.)
func (mfs *MountedFileSystem) Dir() string {
	return mfs.dir
}

// Wait until the mount point is ready to be used. After a successful return
// from this function, the contents of the mounted file system should be
// visible in the directory supplied to NewMountPoint. May be called multiple
// times.
func (mfs *MountedFileSystem) WaitForReady(ctx context.Context) error {
	select {
	case <-mfs.readyStatusAvailable:
		return mfs.readyStatus
	case <-ctx.Done():
		return ctx.Err()
	}
}

// Block until a mounted file system has been unmounted. The return value will
// be non-nil if anything unexpected happened while serving. May be called
// multiple times. Must not be called unless WaitForReady has returned nil.
func (mfs *MountedFileSystem) Join(ctx context.Context) error {
	select {
	case <-mfs.joinStatusAvailable:
		return mfs.joinStatus
	case <-ctx.Done():
		return ctx.Err()
	}
}

// Attempt to unmount the file system. Use Join to wait for it to actually be
// unmounted. You must first call WaitForReady to ensure there is no race with
// mounting.
func (mfs *MountedFileSystem) Unmount() error {
	return bazilfuse.Unmount(mfs.dir)
}

// Runs in the background.
func (mfs *MountedFileSystem) mountAndServe(
	server *server,
	options []bazilfuse.MountOption) {
	logger := getLogger()

	// Open a FUSE connection.
	logger.Println("Opening a FUSE connection.")
	c, err := bazilfuse.Mount(mfs.dir, options...)
	if err != nil {
		mfs.readyStatus = errors.New("bazilfuse.Mount: " + err.Error())
		close(mfs.readyStatusAvailable)
		return
	}

	defer c.Close()

	// Start a goroutine that will notify the MountedFileSystem object when the
	// connection says it is ready (or it fails to become ready).
	go func() {
		logger.Println("Waiting for the FUSE connection to be ready.")
		<-c.Ready
		logger.Println("The FUSE connection is ready.")

		mfs.readyStatus = c.MountError
		close(mfs.readyStatusAvailable)
	}()

	// Serve the connection using the file system object.
	logger.Println("Serving the FUSE connection.")
	if err := server.Serve(c); err != nil {
		mfs.joinStatus = errors.New("Serve: " + err.Error())
		close(mfs.joinStatusAvailable)
		return
	}

	// Signal that everything is okay.
	close(mfs.joinStatusAvailable)
}

// Optional configuration accepted by Mount.
type MountConfig struct {
}

// Convert to mount options to be passed to package bazilfuse.
func (c *MountConfig) bazilfuseOptions() []bazilfuse.MountOption

// Attempt to mount the supplied file system on the given directory.
// mfs.WaitForReady() must be called to find out whether the mount was
// successful.
func Mount(
	dir string,
	fs FileSystem,
	config *MountConfig) (mfs *MountedFileSystem, err error) {
	// Create a server object.
	server, err := newServer(fs)
	if err != nil {
		return
	}

	// Initialize the struct.
	mfs = &MountedFileSystem{
		dir:                  dir,
		readyStatusAvailable: make(chan struct{}),
		joinStatusAvailable:  make(chan struct{}),
	}

	// Mount in the background.
	go mfs.mountAndServe(server, config.bazilfuseOptions())

	return
}
<commit_msg>There are no supported options for now.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fuse

import (
	"errors"

	"github.com/jacobsa/bazilfuse"
	"golang.org/x/net/context"
)

// A struct representing the status of a mount operation, with methods for
// waiting on the mount to complete, waiting for unmounting, and causing
// unmounting.
type MountedFileSystem struct {
	dir string

	// The result to return from WaitForReady. Not valid until the channel is
	// closed.
	readyStatus          error
	readyStatusAvailable chan struct{}

	// The result to return from Join. Not valid until the channel is closed.
	joinStatus          error
	joinStatusAvailable chan struct{}
}

// Return the directory on which the file system is mounted (or where we
// attempted to mount it.)
func (mfs *MountedFileSystem) Dir() string {
	return mfs.dir
}

// Wait until the mount point is ready to be used. After a successful return
// from this function, the contents of the mounted file system should be
// visible in the directory supplied to NewMountPoint. May be called multiple
// times.
func (mfs *MountedFileSystem) WaitForReady(ctx context.Context) error {
	select {
	case <-mfs.readyStatusAvailable:
		return mfs.readyStatus
	case <-ctx.Done():
		return ctx.Err()
	}
}

// Block until a mounted file system has been unmounted. The return value will
// be non-nil if anything unexpected happened while serving. May be called
// multiple times. Must not be called unless WaitForReady has returned nil.
func (mfs *MountedFileSystem) Join(ctx context.Context) error {
	select {
	case <-mfs.joinStatusAvailable:
		return mfs.joinStatus
	case <-ctx.Done():
		return ctx.Err()
	}
}

// Attempt to unmount the file system. Use Join to wait for it to actually be
// unmounted. You must first call WaitForReady to ensure there is no race with
// mounting.
func (mfs *MountedFileSystem) Unmount() error {
	return bazilfuse.Unmount(mfs.dir)
}

// Runs in the background.
func (mfs *MountedFileSystem) mountAndServe(
	server *server,
	options []bazilfuse.MountOption) {
	logger := getLogger()

	// Open a FUSE connection.
	logger.Println("Opening a FUSE connection.")
	c, err := bazilfuse.Mount(mfs.dir, options...)
	if err != nil {
		mfs.readyStatus = errors.New("bazilfuse.Mount: " + err.Error())
		close(mfs.readyStatusAvailable)
		return
	}

	defer c.Close()

	// Start a goroutine that will notify the MountedFileSystem object when the
	// connection says it is ready (or it fails to become ready).
	go func() {
		logger.Println("Waiting for the FUSE connection to be ready.")
		<-c.Ready
		logger.Println("The FUSE connection is ready.")

		mfs.readyStatus = c.MountError
		close(mfs.readyStatusAvailable)
	}()

	// Serve the connection using the file system object.
	logger.Println("Serving the FUSE connection.")
	if err := server.Serve(c); err != nil {
		mfs.joinStatus = errors.New("Serve: " + err.Error())
		close(mfs.joinStatusAvailable)
		return
	}

	// Signal that everything is okay.
	close(mfs.joinStatusAvailable)
}

// Optional configuration accepted by Mount.
type MountConfig struct {
}

// Convert to mount options to be passed to package bazilfuse.
func (c *MountConfig) bazilfuseOptions() (opts []bazilfuse.MountOption) {
	return
}

// Attempt to mount the supplied file system on the given directory.
// mfs.WaitForReady() must be called to find out whether the mount was
// successful.
func Mount(
	dir string,
	fs FileSystem,
	config *MountConfig) (mfs *MountedFileSystem, err error) {
	// Create a server object.
	server, err := newServer(fs)
	if err != nil {
		return
	}

	// Initialize the struct.
	mfs = &MountedFileSystem{
		dir:                  dir,
		readyStatusAvailable: make(chan struct{}),
		joinStatusAvailable:  make(chan struct{}),
	}

	// Mount in the background.
	go mfs.mountAndServe(server, config.bazilfuseOptions())

	return
}
<|endoftext|>