<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/imdario/mergo"
	"github.com/mitchellh/go-homedir"
	"github.com/zackbloom/go-ini"
	"github.com/zackbloom/goamz/aws"
	"github.com/zackbloom/goamz/cloudfront"
	"github.com/zackbloom/goamz/iam"
	"github.com/zackbloom/goamz/route53"
	"github.com/zackbloom/goamz/s3"
	"gopkg.in/yaml.v1"
)

const (
	LIMITED = 60
	FOREVER = 31556926
)

var s3Session *s3.S3
var iamSession *iam.IAM
var r53Session *route53.Route53
var cfSession *cloudfront.CloudFront

func getRegion(region string) aws.Region {
	regionS, ok := aws.Regions[region]
	if !ok {
		panic("Region not found")
	}
	return regionS
}

func openS3(key, secret, region string) *s3.S3 {
	regionS := getRegion(region)

	auth := aws.Auth{
		AccessKey: key,
		SecretKey: secret,
	}
	return s3.New(auth, regionS)
}

func openIAM(key, secret, region string) *iam.IAM {
	regionS := getRegion(region)

	auth := aws.Auth{
		AccessKey: key,
		SecretKey: secret,
	}
	return iam.New(auth, regionS)
}

func openCloudFront(key, secret string) *cloudfront.CloudFront {
	auth := aws.Auth{
		AccessKey: key,
		SecretKey: secret,
	}
	return cloudfront.NewCloudFront(auth)
}

func openRoute53(key, secret string) *route53.Route53 {
	auth := aws.Auth{
		AccessKey: key,
		SecretKey: secret,
	}

	r53, _ := route53.NewRoute53(auth)
	return r53
}

func panicIf(err error) {
	if err != nil {
		panic(err)
	}
}
func must(val interface{}, err error) interface{} {
	if err != nil {
		panic(err)
	}

	return val
}
func mustString(val string, err error) string {
	panicIf(err)
	return val
}
func mustInt(val int, err error) int {
	panicIf(err)
	return val
}

type Options struct {
	Files      string `yaml:"files"`
	Root       string `yaml:"root"`
	Dest       string `yaml:"dest"`
	ConfigFile string `yaml:"-"`
	Env        string `yaml:"-"`
	Bucket     string `yaml:"bucket"`
	AWSKey     string `yaml:"key"`
	AWSSecret  string `yaml:"secret"`
	AWSRegion  string `yaml:"region"`
	NoUser     bool   `yaml:"-"`
}

func parseOptions() (o Options, set *flag.FlagSet) {
	set = flag.NewFlagSet(os.Args[1], flag.ExitOnError)
	//TODO: Set set.Usage

	set.StringVar(&o.Files, "files", "*", "Comma-seperated glob patterns of files to deploy (within root)")
	set.StringVar(&o.Root, "root", "./", "The local directory to deploy")
	set.StringVar(&o.Dest, "dest", "./", "The destination directory to write files to in the S3 bucket")
	set.StringVar(&o.ConfigFile, "config", "", "A yaml file to read configuration from")
	set.StringVar(&o.Env, "env", "", "The env to read from the config file")
	set.StringVar(&o.Bucket, "bucket", "", "The bucket to deploy to")
	set.StringVar(&o.AWSKey, "key", "", "The AWS key to use")
	set.StringVar(&o.AWSSecret, "secret", "", "The AWS secret of the provided key")
	set.StringVar(&o.AWSRegion, "region", "us-east-1", "The AWS region the S3 bucket is in")
	set.BoolVar(&o.NoUser, "no-user", false, "When creating, should we make a user account?")

	set.Parse(os.Args[2:])

	return
}

type ConfigFile map[string]Options

func loadConfigFile(o *Options) {
	isDefault := false
	configPath := o.ConfigFile
	if o.ConfigFile == "" {
		isDefault = true
		configPath = "./deploy.yaml"
	}

	data, err := ioutil.ReadFile(configPath)
	if err != nil {
		if os.IsNotExist(err) && isDefault {
			return
		}

		panic(err)
	}

	var file ConfigFile
	err = yaml.Unmarshal(data, &file)
	panicIf(err)

	var envCfg Options
	if o.Env != "" {
		var ok bool
		envCfg, ok = file[o.Env]
		if !ok {
			panic("Config for specified env not found")
		}
	}

	defCfg, _ := file["default"]

	panicIf(mergo.Merge(o, defCfg))
	panicIf(mergo.Merge(o, envCfg))
}

func addAWSConfig(o *Options) {
	if o.AWSKey == "" && o.AWSSecret == "" {
		o.AWSKey, o.AWSSecret = loadAWSConfig()
	}
}

type AWSConfig struct {
	Default struct {
		AccessKey string `ini:"aws_access_key_id"`
		SecretKey string `ini:"aws_secret_access_key"`
	} `ini:"[default]"`
}

func loadAWSConfig() (access string, secret string) {
	cfg := AWSConfig{}

	path, err := homedir.Expand("~/.aws/config")
	if err != nil {
		return
	}

	content, err := ioutil.ReadFile(path)
	if err != nil {
		return
	}

	ini.Unmarshal(content, &cfg)

	return cfg.Default.AccessKey, cfg.Default.SecretKey
}

func copyFile(bucket *s3.Bucket, from string, to string, contentType string, maxAge int) {
	copyOpts := s3.CopyOptions{
		MetadataDirective: "REPLACE",
		ContentType:       contentType,
		Options: s3.Options{
			CacheControl:    fmt.Sprintf("public, max-age=%d", maxAge),
			ContentEncoding: "gzip",
		},
	}

	_, err := bucket.PutCopy(to, s3.PublicRead, copyOpts, joinPath(bucket.Name, from))
	if err != nil {
		panic(err)
	}
}

var pathRe = regexp.MustCompile("/{2,}")

func joinPath(parts ...string) string {
	// Like filepath.Join, but always uses '/'
	out := filepath.Join(parts...)

	if os.PathSeparator != '/' {
		out = strings.Replace(out, string(os.PathSeparator), "/", -1)
	}

	return out
}
<commit_msg>Also look in AWS Credentials file<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/imdario/mergo"
	"github.com/mitchellh/go-homedir"
	"github.com/zackbloom/go-ini"
	"github.com/zackbloom/goamz/aws"
	"github.com/zackbloom/goamz/cloudfront"
	"github.com/zackbloom/goamz/iam"
	"github.com/zackbloom/goamz/route53"
	"github.com/zackbloom/goamz/s3"
	"gopkg.in/yaml.v1"
)

const (
	LIMITED = 60
	FOREVER = 31556926
)

var s3Session *s3.S3
var iamSession *iam.IAM
var r53Session *route53.Route53
var cfSession *cloudfront.CloudFront

func getRegion(region string) aws.Region {
	regionS, ok := aws.Regions[region]
	if !ok {
		panic("Region not found")
	}
	return regionS
}

func openS3(key, secret, region string) *s3.S3 {
	regionS := getRegion(region)

	auth := aws.Auth{
		AccessKey: key,
		SecretKey: secret,
	}
	return s3.New(auth, regionS)
}

func openIAM(key, secret, region string) *iam.IAM {
	regionS := getRegion(region)

	auth := aws.Auth{
		AccessKey: key,
		SecretKey: secret,
	}
	return iam.New(auth, regionS)
}

func openCloudFront(key, secret string) *cloudfront.CloudFront {
	auth := aws.Auth{
		AccessKey: key,
		SecretKey: secret,
	}
	return cloudfront.NewCloudFront(auth)
}

func openRoute53(key, secret string) *route53.Route53 {
	auth := aws.Auth{
		AccessKey: key,
		SecretKey: secret,
	}

	r53, _ := route53.NewRoute53(auth)
	return r53
}

func panicIf(err error) {
	if err != nil {
		panic(err)
	}
}
func must(val interface{}, err error) interface{} {
	if err != nil {
		panic(err)
	}

	return val
}
func mustString(val string, err error) string {
	panicIf(err)
	return val
}
func mustInt(val int, err error) int {
	panicIf(err)
	return val
}

type Options struct {
	Files      string `yaml:"files"`
	Root       string `yaml:"root"`
	Dest       string `yaml:"dest"`
	ConfigFile string `yaml:"-"`
	Env        string `yaml:"-"`
	Bucket     string `yaml:"bucket"`
	AWSKey     string `yaml:"key"`
	AWSSecret  string `yaml:"secret"`
	AWSRegion  string `yaml:"region"`
	NoUser     bool   `yaml:"-"`
}

func parseOptions() (o Options, set *flag.FlagSet) {
	set = flag.NewFlagSet(os.Args[1], flag.ExitOnError)
	//TODO: Set set.Usage

	set.StringVar(&o.Files, "files", "*", "Comma-seperated glob patterns of files to deploy (within root)")
	set.StringVar(&o.Root, "root", "./", "The local directory to deploy")
	set.StringVar(&o.Dest, "dest", "./", "The destination directory to write files to in the S3 bucket")
	set.StringVar(&o.ConfigFile, "config", "", "A yaml file to read configuration from")
	set.StringVar(&o.Env, "env", "", "The env to read from the config file")
	set.StringVar(&o.Bucket, "bucket", "", "The bucket to deploy to")
	set.StringVar(&o.AWSKey, "key", "", "The AWS key to use")
	set.StringVar(&o.AWSSecret, "secret", "", "The AWS secret of the provided key")
	set.StringVar(&o.AWSRegion, "region", "us-east-1", "The AWS region the S3 bucket is in")
	set.BoolVar(&o.NoUser, "no-user", false, "When creating, should we make a user account?")

	set.Parse(os.Args[2:])

	return
}

type ConfigFile map[string]Options

func loadConfigFile(o *Options) {
	isDefault := false
	configPath := o.ConfigFile
	if o.ConfigFile == "" {
		isDefault = true
		configPath = "./deploy.yaml"
	}

	data, err := ioutil.ReadFile(configPath)
	if err != nil {
		if os.IsNotExist(err) && isDefault {
			return
		}

		panic(err)
	}

	var file ConfigFile
	err = yaml.Unmarshal(data, &file)
	panicIf(err)

	var envCfg Options
	if o.Env != "" {
		var ok bool
		envCfg, ok = file[o.Env]
		if !ok {
			panic("Config for specified env not found")
		}
	}

	defCfg, _ := file["default"]

	panicIf(mergo.Merge(o, defCfg))
	panicIf(mergo.Merge(o, envCfg))
}

func addAWSConfig(o *Options) {
	if o.AWSKey == "" && o.AWSSecret == "" {
		o.AWSKey, o.AWSSecret = loadAWSConfig()
	}
}

type AWSConfig struct {
	Default struct {
		AccessKey string `ini:"aws_access_key_id"`
		SecretKey string `ini:"aws_secret_access_key"`
	} `ini:"[default]"`
}

func loadAWSConfig() (access string, secret string) {
	cfg := AWSConfig{}

	for _, file := range []string{"~/.aws/config", "~/.aws/credentials"} {
		path, err := homedir.Expand(file)
		if err != nil {
			continue
		}

		content, err := ioutil.ReadFile(path)
		if err != nil {
			continue
		}

		ini.Unmarshal(content, &cfg)

		if cfg.Default.AccessKey != "" {
			break
		}
	}

	return cfg.Default.AccessKey, cfg.Default.SecretKey
}

func copyFile(bucket *s3.Bucket, from string, to string, contentType string, maxAge int) {
	copyOpts := s3.CopyOptions{
		MetadataDirective: "REPLACE",
		ContentType:       contentType,
		Options: s3.Options{
			CacheControl:    fmt.Sprintf("public, max-age=%d", maxAge),
			ContentEncoding: "gzip",
		},
	}

	_, err := bucket.PutCopy(to, s3.PublicRead, copyOpts, joinPath(bucket.Name, from))
	if err != nil {
		panic(err)
	}
}

var pathRe = regexp.MustCompile("/{2,}")

func joinPath(parts ...string) string {
	// Like filepath.Join, but always uses '/'
	out := filepath.Join(parts...)

	if os.PathSeparator != '/' {
		out = strings.Replace(out, string(os.PathSeparator), "/", -1)
	}

	return out
}
<|endoftext|>


<commit_before>package app

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/ugorji/go/codec"
	"golang.org/x/net/context"

	"github.com/weaveworks/scope/common/mtime"
	"github.com/weaveworks/scope/report"
)

// Reporter is something that can produce reports on demand. It's a convenient
// interface for parts of the app, and several experimental components.
type Reporter interface {
	Report(context.Context) (report.Report, error)
	WaitOn(context.Context, chan struct{})
	UnWait(context.Context, chan struct{})
}

// Adder is something that can accept reports. It's a convenient interface for
// parts of the app, and several experimental components.
type Adder interface {
	Add(context.Context, report.Report) error
}

// A Collector is a Reporter and an Adder
type Collector interface {
	Reporter
	Adder
}

// Collector receives published reports from multiple producers. It yields a
// single merged report, representing all collected reports.
type collector struct {
	mtx        sync.Mutex
	reports    []report.Report
	timestamps []time.Time
	window     time.Duration
	cached     *report.Report
	merger     Merger
	waitableCondition
}

type waitableCondition struct {
	sync.Mutex
	waiters map[chan struct{}]struct{}
}

func (wc *waitableCondition) WaitOn(_ context.Context, waiter chan struct{}) {
	wc.Lock()
	wc.waiters[waiter] = struct{}{}
	wc.Unlock()
}

func (wc *waitableCondition) UnWait(_ context.Context, waiter chan struct{}) {
	wc.Lock()
	delete(wc.waiters, waiter)
	wc.Unlock()
}

func (wc *waitableCondition) Broadcast() {
	wc.Lock()
	for waiter := range wc.waiters {
		// Non-block write to channel
		select {
		case waiter <- struct{}{}:
		default:
		}
	}
	wc.Unlock()
}

// NewCollector returns a collector ready for use.
func NewCollector(window time.Duration) Collector {
	return &collector{
		window: window,
		waitableCondition: waitableCondition{
			waiters: map[chan struct{}]struct{}{},
		},
		merger: NewSmartMerger(),
	}
}

// Add adds a report to the collector's internal state. It implements Adder.
func (c *collector) Add(_ context.Context, rpt report.Report) error {
	c.mtx.Lock()
	defer c.mtx.Unlock()
	c.reports = append(c.reports, rpt)
	c.timestamps = append(c.timestamps, mtime.Now())

	c.clean()
	c.cached = nil
	if rpt.Shortcut {
		c.Broadcast()
	}
	return nil
}

// Report returns a merged report over all added reports. It implements
// Reporter.
func (c *collector) Report(_ context.Context) (report.Report, error) {
	c.mtx.Lock()
	defer c.mtx.Unlock()

	// If the oldest report is still within range,
	// and there is a cached report, return that.
	if c.cached != nil && len(c.reports) > 0 {
		oldest := mtime.Now().Add(-c.window)
		if c.timestamps[0].After(oldest) {
			return *c.cached, nil
		}
	}

	c.clean()
	return c.merger.Merge(c.reports), nil
}

func (c *collector) clean() {
	var (
		cleanedReports    = make([]report.Report, 0, len(c.reports))
		cleanedTimestamps = make([]time.Time, 0, len(c.timestamps))
		oldest            = mtime.Now().Add(-c.window)
	)
	for i, r := range c.reports {
		if c.timestamps[i].After(oldest) {
			cleanedReports = append(cleanedReports, r)
			cleanedTimestamps = append(cleanedTimestamps, c.timestamps[i])
		}
	}
	c.reports = cleanedReports
	c.timestamps = cleanedTimestamps
}

// StaticCollector always returns the given report.
type StaticCollector report.Report

// Report returns a merged report over all added reports. It implements
// Reporter.
func (c StaticCollector) Report(context.Context) (report.Report, error) { return report.Report(c), nil }

// Add adds a report to the collector's internal state. It implements Adder.
func (c StaticCollector) Add(context.Context, report.Report) error { return nil }

// WaitOn lets other conponents wait on a new report being received. It
// implements Reporter.
func (c StaticCollector) WaitOn(context.Context, chan struct{}) {}

// UnWait lets other conponents stop waiting on a new report being received. It
// implements Reporter.
func (c StaticCollector) UnWait(context.Context, chan struct{}) {}

// NewFileCollector reads and json parses the given path, returning a collector
// which always returns that report.
func NewFileCollector(path string) (Collector, error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	var (
		rpt     report.Report
		handle  codec.Handle
		gzipped bool
	)
	fileType := filepath.Ext(path)
	if fileType == ".gz" {
		gzipped = true
		fileType = filepath.Ext(strings.TrimSuffix(path, fileType))
	}
	switch fileType {
	case ".json":
		handle = &codec.JsonHandle{}
	case ".msgpack":
		handle = &codec.MsgpackHandle{}
	default:
		return nil, fmt.Errorf("Unsupported file extension: %v", fileType)
	}

	if err := rpt.ReadBinary(f, gzipped, handle); err != nil {
		return nil, err
	}

	return StaticCollector(rpt), nil
}
<commit_msg>Review Feedback<commit_after>package app

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/ugorji/go/codec"
	"golang.org/x/net/context"

	"github.com/weaveworks/scope/common/mtime"
	"github.com/weaveworks/scope/report"
)

// Reporter is something that can produce reports on demand. It's a convenient
// interface for parts of the app, and several experimental components.
type Reporter interface {
	Report(context.Context) (report.Report, error)
	WaitOn(context.Context, chan struct{})
	UnWait(context.Context, chan struct{})
}

// Adder is something that can accept reports. It's a convenient interface for
// parts of the app, and several experimental components.
type Adder interface {
	Add(context.Context, report.Report) error
}

// A Collector is a Reporter and an Adder
type Collector interface {
	Reporter
	Adder
}

// Collector receives published reports from multiple producers. It yields a
// single merged report, representing all collected reports.
type collector struct {
	mtx        sync.Mutex
	reports    []report.Report
	timestamps []time.Time
	window     time.Duration
	cached     *report.Report
	merger     Merger
	waitableCondition
}

type waitableCondition struct {
	sync.Mutex
	waiters map[chan struct{}]struct{}
}

func (wc *waitableCondition) WaitOn(_ context.Context, waiter chan struct{}) {
	wc.Lock()
	wc.waiters[waiter] = struct{}{}
	wc.Unlock()
}

func (wc *waitableCondition) UnWait(_ context.Context, waiter chan struct{}) {
	wc.Lock()
	delete(wc.waiters, waiter)
	wc.Unlock()
}

func (wc *waitableCondition) Broadcast() {
	wc.Lock()
	for waiter := range wc.waiters {
		// Non-block write to channel
		select {
		case waiter <- struct{}{}:
		default:
		}
	}
	wc.Unlock()
}

// NewCollector returns a collector ready for use.
func NewCollector(window time.Duration) Collector {
	return &collector{
		window: window,
		waitableCondition: waitableCondition{
			waiters: map[chan struct{}]struct{}{},
		},
		merger: NewSmartMerger(),
	}
}

// Add adds a report to the collector's internal state. It implements Adder.
func (c *collector) Add(_ context.Context, rpt report.Report) error {
	c.mtx.Lock()
	defer c.mtx.Unlock()
	c.reports = append(c.reports, rpt)
	c.timestamps = append(c.timestamps, mtime.Now())

	c.clean()
	c.cached = nil
	if rpt.Shortcut {
		c.Broadcast()
	}
	return nil
}

// Report returns a merged report over all added reports. It implements
// Reporter.
func (c *collector) Report(_ context.Context) (report.Report, error) {
	c.mtx.Lock()
	defer c.mtx.Unlock()

	// If the oldest report is still within range,
	// and there is a cached report, return that.
	if c.cached != nil && len(c.reports) > 0 {
		oldest := mtime.Now().Add(-c.window)
		if c.timestamps[0].After(oldest) {
			return *c.cached, nil
		}
	}

	c.clean()
	return c.merger.Merge(c.reports), nil
}

func (c *collector) clean() {
	var (
		cleanedReports    = make([]report.Report, 0, len(c.reports))
		cleanedTimestamps = make([]time.Time, 0, len(c.timestamps))
		oldest            = mtime.Now().Add(-c.window)
	)
	for i, r := range c.reports {
		if c.timestamps[i].After(oldest) {
			cleanedReports = append(cleanedReports, r)
			cleanedTimestamps = append(cleanedTimestamps, c.timestamps[i])
		}
	}
	c.reports = cleanedReports
	c.timestamps = cleanedTimestamps
}

// StaticCollector always returns the given report.
type StaticCollector report.Report

// Report returns a merged report over all added reports. It implements
// Reporter.
func (c StaticCollector) Report(context.Context) (report.Report, error) { return report.Report(c), nil }

// Add adds a report to the collector's internal state. It implements Adder.
func (c StaticCollector) Add(context.Context, report.Report) error { return nil }

// WaitOn lets other components wait on a new report being received. It
// implements Reporter.
func (c StaticCollector) WaitOn(context.Context, chan struct{}) {}

// UnWait lets other components stop waiting on a new report being received. It
// implements Reporter.
func (c StaticCollector) UnWait(context.Context, chan struct{}) {}

// NewFileCollector reads and parses the given path, returning a collector
// which always returns that report.
func NewFileCollector(path string) (Collector, error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	var (
		rpt     report.Report
		handle  codec.Handle
		gzipped bool
	)
	fileType := filepath.Ext(path)
	if fileType == ".gz" {
		gzipped = true
		fileType = filepath.Ext(strings.TrimSuffix(path, fileType))
	}
	switch fileType {
	case ".json":
		handle = &codec.JsonHandle{}
	case ".msgpack":
		handle = &codec.MsgpackHandle{}
	default:
		return nil, fmt.Errorf("Unsupported file extension: %v", fileType)
	}

	if err := rpt.ReadBinary(f, gzipped, handle); err != nil {
		return nil, err
	}

	return StaticCollector(rpt), nil
}
<|endoftext|>


<commit_before>/**
 * Copyright 2017 Comcast Cable Communications Management, LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */
package main

import (
	"crypto/tls"
	"fmt"
	"net/http"
	_ "net/http/pprof"
	"net/url"
	"os"
	"os/signal"
	"time"

	"github.com/Comcast/webpa-common/concurrent"
	"github.com/Comcast/webpa-common/logging"
	"github.com/Comcast/webpa-common/secure"
	"github.com/Comcast/webpa-common/secure/handler"
	"github.com/Comcast/webpa-common/secure/key"
	"github.com/Comcast/webpa-common/server"
	"github.com/Comcast/webpa-common/webhook"
	"github.com/SermoDigital/jose/jwt"
	"github.com/gorilla/mux"
	"github.com/justinas/alice"
	"github.com/spf13/pflag"
	"github.com/spf13/viper"
)

const (
	applicationName = "caduceus"
	DEFAULT_KEY_ID  = "current"
)

// getValidator returns validator for JWT tokens
func getValidator(v *viper.Viper) (validator secure.Validator, err error) {
	default_validators := make(secure.Validators, 0, 0)
	var jwtVals []JWTValidator

	v.UnmarshalKey("jwtValidators", &jwtVals)

	// make sure there is at least one jwtValidator supplied
	if len(jwtVals) < 1 {
		validator = default_validators
		return
	}

	// if a JWTKeys section was supplied, configure a JWS validator
	// and append it to the chain of validators
	validators := make(secure.Validators, 0, len(jwtVals))

	for _, validatorDescriptor := range jwtVals {
		var keyResolver key.Resolver
		keyResolver, err = validatorDescriptor.Keys.NewResolver()
		if err != nil {
			validator = validators
			return
		}

		validators = append(
			validators,
			secure.JWSValidator{
				DefaultKeyId:  DEFAULT_KEY_ID,
				Resolver:      keyResolver,
				JWTValidators: []*jwt.Validator{validatorDescriptor.Custom.New()},
			},
		)
	}

	// TODO: This should really be part of the unmarshalled validators somehow
	basicAuth := v.GetStringSlice("authHeader")
	for _, authValue := range basicAuth {
		validators = append(
			validators,
			secure.ExactMatchValidator(authValue),
		)
	}

	validator = validators

	return
}

// caduceus is the driver function for Caduceus.  It performs everything main() would do,
// except for obtaining the command-line arguments (which are passed to it).

func caduceus(arguments []string) int {
	beginCaduceus := time.Now()

	var (
		f = pflag.NewFlagSet(applicationName, pflag.ContinueOnError)
		v = viper.New()

		logger, metricsRegistry, webPA, err = server.Initialize(applicationName, arguments, f, v)
	)

	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to initialize Viper environment: %s\n", err)
		return 1
	}

	var (
		infoLog  = logging.Info(logger)
		errorLog = logging.Error(logger)
		debugLog = logging.Debug(logger)
	)

	infoLog.Log("configurationFile", v.ConfigFileUsed())

	caduceusConfig := new(CaduceusConfig)
	err = v.Unmarshal(caduceusConfig)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to unmarshal configuration data into struct: %s\n", err)
		return 1
	}

	workerPool := WorkerPoolFactory{
		NumWorkers: caduceusConfig.NumWorkerThreads,
		QueueSize:  caduceusConfig.JobQueueSize,
	}.New()

	mainCaduceusProfilerFactory := ServerProfilerFactory{
		Frequency: caduceusConfig.ProfilerFrequency,
		Duration:  caduceusConfig.ProfilerDuration,
		QueueSize: caduceusConfig.ProfilerQueueSize,
		Logger:    logger,
	}

	// here we create a profiler specifically for our main server handler
	caduceusHandlerProfiler, err := mainCaduceusProfilerFactory.New("main")
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to profiler for main caduceus handler: %s\n", err)
		return 1
	}

	childCaduceusProfilerFactory := mainCaduceusProfilerFactory
	childCaduceusProfilerFactory.Parent = caduceusHandlerProfiler

	tr := &http.Transport{
		TLSClientConfig:       &tls.Config{InsecureSkipVerify: true},
		MaxIdleConnsPerHost:   caduceusConfig.SenderNumWorkersPerSender,
		ResponseHeaderTimeout: 10 * time.Second, // TODO Make this configurable
	}

	timeout := time.Duration(caduceusConfig.SenderClientTimeout) * time.Second

	// declare a new sender wrapper and pass it a profiler factory so that it can create
	// unique profilers on a per outboundSender basis
	caduceusSenderWrapper, err := SenderWrapperFactory{
		NumWorkersPerSender: caduceusConfig.SenderNumWorkersPerSender,
		QueueSizePerSender:  caduceusConfig.SenderQueueSizePerSender,
		CutOffPeriod:        time.Duration(caduceusConfig.SenderCutOffPeriod) * time.Second,
		Linger:              time.Duration(caduceusConfig.SenderLinger) * time.Second,
		ProfilerFactory:     childCaduceusProfilerFactory,
		Logger:              logger,
		Client:              &http.Client{Transport: tr, Timeout: timeout},
	}.New()

	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to initialize new caduceus sender wrapper: %s\n", err)
		return 1
	}

	serverWrapper := &ServerHandler{
		Logger: logger,
		caduceusHandler: &CaduceusHandler{
			handlerProfiler: caduceusHandlerProfiler,
			senderWrapper:   caduceusSenderWrapper,
			Logger:          logger,
		},
		doJob: workerPool.Send,
	}

	profileWrapper := &ProfileHandler{
		profilerData: caduceusHandlerProfiler,
		Logger:       logger,
	}

	validator, err := getValidator(v)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Validator error: %v\n", err)
		return 1
	}

	authHandler := handler.AuthorizationHandler{
		HeaderName:          "Authorization",
		ForbiddenStatusCode: 403,
		Validator:           validator,
		Logger:              logger,
	}

	caduceusHandler := alice.New(authHandler.Decorate, TrackEmptyRequestBody(metricsRegistry))

	router := mux.NewRouter()

	router = configServerRouter(router, caduceusHandler, serverWrapper)

	router.Handle("/api/v3/profile", caduceusHandler.Then(profileWrapper))

	webhookFactory, err := webhook.NewFactory(v)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error creating new webhook factory: %s\n", err)
		return 1
	}
	webhookRegistry, webhookHandler := webhookFactory.NewRegistryAndHandler()
	webhookFactory.SetExternalUpdate(caduceusSenderWrapper.Update)

	// register webhook end points for api
	router.Handle("/hook", caduceusHandler.ThenFunc(webhookRegistry.UpdateRegistry))
	router.Handle("/hooks", caduceusHandler.ThenFunc(webhookRegistry.GetRegistry))

	selfURL := &url.URL{
		Scheme: "https",
		Host:   v.GetString("fqdn") + v.GetString("primary.address"),
	}

	webhookFactory.Initialize(router, selfURL, webhookHandler, logger, nil)

	caduceusHealth := &CaduceusHealth{}
	var runnable concurrent.Runnable

	caduceusHealth.Monitor, runnable = webPA.Prepare(logger, nil, metricsRegistry, router)
	serverWrapper.caduceusHealth = caduceusHealth

	waitGroup, shutdown, err := concurrent.Execute(runnable)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to start device manager: %s\n", err)
		return 1
	}

	var messageKey = logging.MessageKey()

	debugLog.Log(messageKey, "Calling webhookFactory.PrepareAndStart")
	beginPrepStart := time.Now()
	webhookFactory.PrepareAndStart()
	debugLog.Log(messageKey, "WebhookFactory.PrepareAndStart done.", "elapsedTime", time.Since(beginPrepStart))

	// Attempt to obtain the current listener list from current system without having to wait for listener reregistration.
	debugLog.Log(messageKey, "Attempting to obtain current listener list from source", "source",
		v.GetString("start.apiPath"))
	beginObtainList := time.Now()
	startChan := make(chan webhook.Result, 1)
	webhookFactory.Start.GetCurrentSystemsHooks(startChan)
	var webhookStartResults webhook.Result = <-startChan
	if webhookStartResults.Error != nil {
		errorLog.Log(logging.ErrorKey(), webhookStartResults.Error)
	} else {
		// todo: add message
		webhookFactory.SetList(webhook.NewList(webhookStartResults.Hooks))
		caduceusSenderWrapper.Update(webhookStartResults.Hooks)
	}
	debugLog.Log(messageKey, "Current listener retrieval.", "elapsedTime", time.Since(beginObtainList))

	infoLog.Log(messageKey, "Caduceus is up and running!", "elapsedTime", time.Since(beginCaduceus))

	var (
		signals = make(chan os.Signal, 1)
	)

	signal.Notify(signals)
	<-signals
	close(shutdown)
	waitGroup.Wait()

	// shutdown the sender wrapper gently so that all queued messages get serviced
	caduceusSenderWrapper.Shutdown(true)

	return 0
}

func configServerRouter(router *mux.Router, caduceusHandler alice.Chain, serverWrapper *ServerHandler) *mux.Router {
	var singleContentType = func(r *http.Request, _ *mux.RouteMatch) bool {
		return len(r.Header["Content-Type"]) == 1 //require single specification for Content-Type Header
	}

	router.Handle("/api/v3/notify", caduceusHandler.Then(serverWrapper)).Methods("POST").
		HeadersRegexp("Content-Type", "application/(json|msgpack)").MatcherFunc(singleContentType)

	// Support the old endpoint too.
	router.Handle("/api/v2/notify/{deviceid}/event/{eventtype:.*}", caduceusHandler.Then(serverWrapper)).
		Methods("POST").HeadersRegexp("Content-Type", "application/(json|msgpack)").
		MatcherFunc(singleContentType)

	return router
}

func main() {
	os.Exit(caduceus(os.Args))
}
<commit_msg>Forgot this: add the module metrics<commit_after>/**
 * Copyright 2017 Comcast Cable Communications Management, LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */
package main

import (
	"crypto/tls"
	"fmt"
	"net/http"
	_ "net/http/pprof"
	"net/url"
	"os"
	"os/signal"
	"time"

	"github.com/Comcast/webpa-common/concurrent"
	"github.com/Comcast/webpa-common/logging"
	"github.com/Comcast/webpa-common/secure"
	"github.com/Comcast/webpa-common/secure/handler"
	"github.com/Comcast/webpa-common/secure/key"
	"github.com/Comcast/webpa-common/server"
	"github.com/Comcast/webpa-common/webhook"
	"github.com/SermoDigital/jose/jwt"
	"github.com/gorilla/mux"
	"github.com/justinas/alice"
	"github.com/spf13/pflag"
	"github.com/spf13/viper"
)

const (
	applicationName = "caduceus"
	DEFAULT_KEY_ID  = "current"
)

// getValidator returns validator for JWT tokens
func getValidator(v *viper.Viper) (validator secure.Validator, err error) {
	default_validators := make(secure.Validators, 0, 0)
	var jwtVals []JWTValidator

	v.UnmarshalKey("jwtValidators", &jwtVals)

	// make sure there is at least one jwtValidator supplied
	if len(jwtVals) < 1 {
		validator = default_validators
		return
	}

	// if a JWTKeys section was supplied, configure a JWS validator
	// and append it to the chain of validators
	validators := make(secure.Validators, 0, len(jwtVals))

	for _, validatorDescriptor := range jwtVals {
		var keyResolver key.Resolver
		keyResolver, err = validatorDescriptor.Keys.NewResolver()
		if err != nil {
			validator = validators
			return
		}

		validators = append(
			validators,
			secure.JWSValidator{
				DefaultKeyId:  DEFAULT_KEY_ID,
				Resolver:      keyResolver,
				JWTValidators: []*jwt.Validator{validatorDescriptor.Custom.New()},
			},
		)
	}

	// TODO: This should really be part of the unmarshalled validators somehow
	basicAuth := v.GetStringSlice("authHeader")
	for _, authValue := range basicAuth {
		validators = append(
			validators,
			secure.ExactMatchValidator(authValue),
		)
	}

	validator = validators

	return
}

// caduceus is the driver function for Caduceus.  It performs everything main() would do,
// except for obtaining the command-line arguments (which are passed to it).

func caduceus(arguments []string) int {
	beginCaduceus := time.Now()

	var (
		f = pflag.NewFlagSet(applicationName, pflag.ContinueOnError)
		v = viper.New()

		logger, metricsRegistry, webPA, err = server.Initialize(applicationName, arguments, f, v, Metrics)
	)

	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to initialize Viper environment: %s\n", err)
		return 1
	}

	var (
		infoLog  = logging.Info(logger)
		errorLog = logging.Error(logger)
		debugLog = logging.Debug(logger)
	)

	infoLog.Log("configurationFile", v.ConfigFileUsed())

	caduceusConfig := new(CaduceusConfig)
	err = v.Unmarshal(caduceusConfig)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to unmarshal configuration data into struct: %s\n", err)
		return 1
	}

	workerPool := WorkerPoolFactory{
		NumWorkers: caduceusConfig.NumWorkerThreads,
		QueueSize:  caduceusConfig.JobQueueSize,
	}.New()

	mainCaduceusProfilerFactory := ServerProfilerFactory{
		Frequency: caduceusConfig.ProfilerFrequency,
		Duration:  caduceusConfig.ProfilerDuration,
		QueueSize: caduceusConfig.ProfilerQueueSize,
		Logger:    logger,
	}

	// here we create a profiler specifically for our main server handler
	caduceusHandlerProfiler, err := mainCaduceusProfilerFactory.New("main")
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to profiler for main caduceus handler: %s\n", err)
		return 1
	}

	childCaduceusProfilerFactory := mainCaduceusProfilerFactory
	childCaduceusProfilerFactory.Parent = caduceusHandlerProfiler

	tr := &http.Transport{
		TLSClientConfig:       &tls.Config{InsecureSkipVerify: true},
		MaxIdleConnsPerHost:   caduceusConfig.SenderNumWorkersPerSender,
		ResponseHeaderTimeout: 10 * time.Second, // TODO Make this configurable
	}

	timeout := time.Duration(caduceusConfig.SenderClientTimeout) * time.Second

	// declare a new sender wrapper and pass it a profiler factory so that it can create
	// unique profilers on a per outboundSender basis
	caduceusSenderWrapper, err := SenderWrapperFactory{
		NumWorkersPerSender: caduceusConfig.SenderNumWorkersPerSender,
		QueueSizePerSender:  caduceusConfig.SenderQueueSizePerSender,
		CutOffPeriod:        time.Duration(caduceusConfig.SenderCutOffPeriod) * time.Second,
		Linger:              time.Duration(caduceusConfig.SenderLinger) * time.Second,
		ProfilerFactory:     childCaduceusProfilerFactory,
		Logger:              logger,
		Client:              &http.Client{Transport: tr, Timeout: timeout},
	}.New()

	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to initialize new caduceus sender wrapper: %s\n", err)
		return 1
	}

	serverWrapper := &ServerHandler{
		Logger: logger,
		caduceusHandler: &CaduceusHandler{
			handlerProfiler: caduceusHandlerProfiler,
			senderWrapper:   caduceusSenderWrapper,
			Logger:          logger,
		},
		doJob: workerPool.Send,
	}

	profileWrapper := &ProfileHandler{
		profilerData: caduceusHandlerProfiler,
		Logger:       logger,
	}

	validator, err := getValidator(v)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Validator error: %v\n", err)
		return 1
	}

	authHandler := handler.AuthorizationHandler{
		HeaderName:          "Authorization",
		ForbiddenStatusCode: 403,
		Validator:           validator,
		Logger:              logger,
	}

	caduceusHandler := alice.New(authHandler.Decorate, TrackEmptyRequestBody(metricsRegistry))

	router := mux.NewRouter()

	router = configServerRouter(router, caduceusHandler, serverWrapper)

	router.Handle("/api/v3/profile", caduceusHandler.Then(profileWrapper))

	webhookFactory, err := webhook.NewFactory(v)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error creating new webhook factory: %s\n", err)
		return 1
	}
	webhookRegistry, webhookHandler := webhookFactory.NewRegistryAndHandler()
	webhookFactory.SetExternalUpdate(caduceusSenderWrapper.Update)

	// register webhook end points for api
	router.Handle("/hook", caduceusHandler.ThenFunc(webhookRegistry.UpdateRegistry))
	router.Handle("/hooks", caduceusHandler.ThenFunc(webhookRegistry.GetRegistry))

	selfURL := &url.URL{
		Scheme: "https",
		Host:   v.GetString("fqdn") + v.GetString("primary.address"),
	}

	webhookFactory.Initialize(router, selfURL, webhookHandler, logger, nil)

	caduceusHealth := &CaduceusHealth{}
	var runnable concurrent.Runnable

	caduceusHealth.Monitor, runnable = webPA.Prepare(logger, nil, metricsRegistry, router)
	serverWrapper.caduceusHealth = caduceusHealth

	waitGroup, shutdown, err := concurrent.Execute(runnable)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to start device manager: %s\n", err)
		return 1
	}

	var messageKey = logging.MessageKey()

	debugLog.Log(messageKey, "Calling webhookFactory.PrepareAndStart")
	beginPrepStart := time.Now()
	webhookFactory.PrepareAndStart()
	debugLog.Log(messageKey, "WebhookFactory.PrepareAndStart done.", "elapsedTime", time.Since(beginPrepStart))

	// Attempt to obtain the current listener list from current system without having to wait for listener reregistration.
	debugLog.Log(messageKey, "Attempting to obtain current listener list from source", "source",
		v.GetString("start.apiPath"))
	beginObtainList := time.Now()
	startChan := make(chan webhook.Result, 1)
	webhookFactory.Start.GetCurrentSystemsHooks(startChan)
	var webhookStartResults webhook.Result = <-startChan
	if webhookStartResults.Error != nil {
		errorLog.Log(logging.ErrorKey(), webhookStartResults.Error)
	} else {
		// todo: add message
		webhookFactory.SetList(webhook.NewList(webhookStartResults.Hooks))
		caduceusSenderWrapper.Update(webhookStartResults.Hooks)
	}
	debugLog.Log(messageKey, "Current listener retrieval.", "elapsedTime", time.Since(beginObtainList))

	infoLog.Log(messageKey, "Caduceus is up and running!", "elapsedTime", time.Since(beginCaduceus))

	var (
		signals = make(chan os.Signal, 1)
	)

	signal.Notify(signals)
	<-signals
	close(shutdown)
	waitGroup.Wait()

	// shutdown the sender wrapper gently so that all queued messages get serviced
	caduceusSenderWrapper.Shutdown(true)

	return 0
}

func configServerRouter(router *mux.Router, caduceusHandler alice.Chain, serverWrapper *ServerHandler) *mux.Router {
	var singleContentType = func(r *http.Request, _ *mux.RouteMatch) bool {
		return len(r.Header["Content-Type"]) == 1 //require single specification for Content-Type Header
	}

	router.Handle("/api/v3/notify", caduceusHandler.Then(serverWrapper)).Methods("POST").
		HeadersRegexp("Content-Type", "application/(json|msgpack)").MatcherFunc(singleContentType)

	// Support the old endpoint too.
	router.Handle("/api/v2/notify/{deviceid}/event/{eventtype:.*}", caduceusHandler.Then(serverWrapper)).
		Methods("POST").HeadersRegexp("Content-Type", "application/(json|msgpack)").
		MatcherFunc(singleContentType)

	return router
}

func main() {
	os.Exit(caduceus(os.Args))
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"testing"
)

// Copyright (c) 2015 Uber Technologies, Inc.

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

// This test ensures that the code generator generates valid code that can be built
// in combination with Thrift's autogenerated code.

func TestAllThrift(t *testing.T) {
	files, err := ioutil.ReadDir("test_files")
	if err != nil {
		t.Fatalf("Cannot read test_files directory: %v", err)
	}

	for _, f := range files {
		fname := f.Name()
		if filepath.Ext(fname) != ".thrift" {
			continue
		}

		if err := runTest(t, filepath.Join("test_files", f.Name())); err != nil {
			t.Errorf("Thrift file %v failed: %v", f.Name(), err)
		}
	}
}

func copyFile(src, dst string) error {
	f, err := os.Open(src)
	if err != nil {
		return err
	}
	defer f.Close()

	writeF, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666)
	if err != nil {
		return err
	}
	defer writeF.Close()

	_, err = io.Copy(writeF, f)
	return err
}

func setupDirectory(thriftFile string) (string, string, error) {
	// Create a temporary directory
	tempDir, err := ioutil.TempDir("", "thrift-gen")
	if err != nil {
		return "", "", err
	}

	// Copy the .thrift file to the directory
	outFile := filepath.Join(tempDir, "test.thrift")
	return tempDir, outFile, copyFile(thriftFile, outFile)
}

func runTest(t *testing.T, thriftFile string) error {
	tempDir, thriftFile, err := setupDirectory(thriftFile)
	if err != nil {
		return err
	}

	// Run generate.sh for the given directory
	t.Logf("runTest in %v", tempDir)
	if err := processFile(true /* generateThrift */, thriftFile, ""); err != nil {
		return fmt.Errorf("processFile(%s) failed: %v", thriftFile, err)
	}

	// If the generate is successful, run go build in the directory.
	cmd := exec.Command("go", "build", ".")
	cmd.Dir = filepath.Join(tempDir, "gen-go", "test")
	if output, err := cmd.CombinedOutput(); err != nil {
		return fmt.Errorf("Build failed. Output = \n%v\n", string(output))
	}

	// Only delete the temp directory on success.
	os.RemoveAll(tempDir)
	return nil
}
<commit_msg>minor comment updates<commit_after>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"testing"
)

// Copyright (c) 2015 Uber Technologies, Inc.

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

// This test ensures that the code generator generates valid code that can be built
// in combination with Thrift's autogenerated code.

func TestAllThrift(t *testing.T) {
	files, err := ioutil.ReadDir("test_files")
	if err != nil {
		t.Fatalf("Cannot read test_files directory: %v", err)
	}

	for _, f := range files {
		fname := f.Name()
		if filepath.Ext(fname) != ".thrift" {
			continue
		}

		if err := runTest(t, filepath.Join("test_files", f.Name())); err != nil {
			t.Errorf("Thrift file %v failed: %v", f.Name(), err)
		}
	}
}

func copyFile(src, dst string) error {
	f, err := os.Open(src)
	if err != nil {
		return err
	}
	defer f.Close()

	writeF, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666)
	if err != nil {
		return err
	}
	defer writeF.Close()

	_, err = io.Copy(writeF, f)
	return err
}

// setupDirectory creates a temporary directory and copies the Thrift file into that directory.
func setupDirectory(thriftFile string) (string, string, error) {
	tempDir, err := ioutil.TempDir("", "thrift-gen")
	if err != nil {
		return "", "", err
	}

	outFile := filepath.Join(tempDir, "test.thrift")
	return tempDir, outFile, copyFile(thriftFile, outFile)
}

func runTest(t *testing.T, thriftFile string) error {
	tempDir, thriftFile, err := setupDirectory(thriftFile)
	if err != nil {
		return err
	}

	// Generate code from the Thrift file.
	t.Logf("runTest in %v", tempDir)
	if err := processFile(true /* generateThrift */, thriftFile, ""); err != nil {
		return fmt.Errorf("processFile(%s) failed: %v", thriftFile, err)
	}

	// Run go build to ensure that the generated code builds.
	cmd := exec.Command("go", "build", ".")
	cmd.Dir = filepath.Join(tempDir, "gen-go", "test")
	if output, err := cmd.CombinedOutput(); err != nil {
		return fmt.Errorf("Build failed. Output = \n%v\n", string(output))
	}

	// Only delete the temp directory on success.
	os.RemoveAll(tempDir)
	return nil
}
<|endoftext|>


<commit_before>package path_matcher

import (
	"strings"
)

func NewMultiPathMatcher(PathMatchers ...PathMatcher) PathMatcher {
	if len(PathMatchers) == 0 {
		panic("the multi path matcher cannot be initialized without any matcher")
	}

	return &MultiPathMatcher{PathMatchers: PathMatchers}
}

type MultiPathMatcher struct {
	PathMatchers []PathMatcher
}

func (f *MultiPathMatcher) IsDirOrSubmodulePathMatched(path string) bool {
	return f.IsPathMatched(path) || f.ShouldGoThrough(path)
}

func (m *MultiPathMatcher) IsPathMatched(path string) bool {
	for _, matcher := range m.PathMatchers {
		if !matcher.IsPathMatched(path) {
			return false
		}
	}

	return true
}

func (m *MultiPathMatcher) ShouldGoThrough(path string) bool {
	for _, matcher := range m.PathMatchers {
		if !matcher.ShouldGoThrough(path) {
			return false
		}
	}

	return true
}

func (m *MultiPathMatcher) TrimFileBaseFilepath(path string) string {
	return m.PathMatchers[0].TrimFileBaseFilepath(path)
}

func (m *MultiPathMatcher) BaseFilepath() string {
	return m.PathMatchers[0].BaseFilepath()
}

func (m *MultiPathMatcher) String() string {
	var result []string
	for _, matcher := range m.PathMatchers {
		result = append(result, matcher.String())
	}

	return strings.Join(result, "; ")
}
<commit_msg>[path_matcher] Update multiPathMatcher string format<commit_after>package path_matcher

import (
	"strings"
)

func NewMultiPathMatcher(PathMatchers ...PathMatcher) PathMatcher {
	if len(PathMatchers) == 0 {
		panic("the multi path matcher cannot be initialized without any matcher")
	}

	return &MultiPathMatcher{PathMatchers: PathMatchers}
}

type MultiPathMatcher struct {
	PathMatchers []PathMatcher
}

func (f *MultiPathMatcher) IsDirOrSubmodulePathMatched(path string) bool {
	return f.IsPathMatched(path) || f.ShouldGoThrough(path)
}

func (m *MultiPathMatcher) IsPathMatched(path string) bool {
	for _, matcher := range m.PathMatchers {
		if !matcher.IsPathMatched(path) {
			return false
		}
	}

	return true
}

func (m *MultiPathMatcher) ShouldGoThrough(path string) bool {
	for _, matcher := range m.PathMatchers {
		if !matcher.ShouldGoThrough(path) {
			return false
		}
	}

	return true
}

func (m *MultiPathMatcher) TrimFileBaseFilepath(path string) string {
	return m.PathMatchers[0].TrimFileBaseFilepath(path)
}

func (m *MultiPathMatcher) BaseFilepath() string {
	return m.PathMatchers[0].BaseFilepath()
}

func (m *MultiPathMatcher) String() string {
	var result []string
	for _, matcher := range m.PathMatchers {
		result = append(result, matcher.String())
	}

	return strings.Join(result, " && ")
}
<|endoftext|>


<commit_before>package config

import (
	"time"

	"github.com/hellofresh/logging-go"
	"github.com/kelseyhightower/envconfig"
	log "github.com/sirupsen/logrus"
	"github.com/spf13/viper"
)

// Specification for basic configurations
type Specification struct {
	Port                 int           `envconfig:"PORT"`
	Debug                bool          `envconfig:"DEBUG"`
	GraceTimeOut         int64         `envconfig:"GRACE_TIMEOUT"`
	MaxIdleConnsPerHost  int           `envconfig:"MAX_IDLE_CONNS_PER_HOST"`
	BackendFlushInterval time.Duration `envconfig:"BACKEND_FLUSH_INTERVAL"`
	CloseIdleConnsPeriod time.Duration `envconfig:"CLOSE_IDLE_CONNS_PERIOD"`
	Log                  logging.LogConfig
	Web                  Web
	Database             Database
	Storage              Storage
	Stats                Stats
	Tracing              Tracing
	TLS                  TLS
}

// Web represents the API configurations
type Web struct {
	Port        int  `envconfig:"API_PORT"`
	ReadOnly    bool `envconfig:"API_READONLY"`
	Credentials Credentials
	TLS         TLS
}

// TLS represents the TLS configurations
type TLS struct {
	Port     int    `envconfig:"PORT"`
	CertFile string `envconfig:"CERT_PATH"`
	KeyFile  string `envconfig:"KEY_PATH"`
	Redirect bool   `envconfig:"REDIRECT"`
}

// IsHTTPS checks if you have https enabled
func (s *TLS) IsHTTPS() bool {
	return s.CertFile != "" && s.KeyFile != ""
}

// Storage holds the configuration for a storage
type Storage struct {
	DSN string `envconfig:"STORAGE_DSN"`
}

// Database holds the configuration for a database
type Database struct {
	DSN string `envconfig:"DATABASE_DSN"`
}

// Stats holds the configuration for stats
type Stats struct {
	DSN                   string   `envconfig:"STATS_DSN"`
	Prefix                string   `envconfig:"STATS_PREFIX"`
	IDs                   string   `envconfig:"STATS_IDS"`
	AutoDiscoverThreshold uint     `envconfig:"STATS_AUTO_DISCOVER_THRESHOLD"`
	AutoDiscoverWhiteList []string `envconfig:"STATS_AUTO_DISCOVER_WHITE_LIST"`
	ErrorsSection         string   `envconfig:"STATS_ERRORS_SECTION"`
}

// Credentials represents the credentials that are going to be
// used by admin JWT configuration
type Credentials struct {
	// Algorithm defines admin JWT signing algorithm.
	// Currently the following algorithms are supported: HS256, HS384, HS512.
	Algorithm string `envconfig:"ALGORITHM"`
	Secret    string `envconfig:"SECRET"`
	Username  string `envconfig:"ADMIN_USERNAME"`
	Password  string `envconfig:"ADMIN_PASSWORD"`
}

// GoogleCloudTracing holds the Google Application Default Credentials
type GoogleCloudTracing struct {
	ProjectID    string `envconfig:"TRACING_GC_PROJECT_ID"`
	Email        string `envconfig:"TRACING_GC_EMAIL"`
	PrivateKey   string `envconfig:"TRACING_GC_PRIVATE_KEY"`
	PrivateKeyID string `envconfig:"TRACING_GC_PRIVATE_ID"`
}

// AppdashTracing holds the Appdash tracing configuration
type AppdashTracing struct {
	DSN string `envconfig:"TRACING_APPDASH_DSN"`
	URL string `envconfig:"TRACING_APPDASH_URL"`
}

// Tracing represents the distributed tracing configuration
type Tracing struct {
	GoogleCloudTracing GoogleCloudTracing `mapstructure:"googleCloud"`
	AppdashTracing     AppdashTracing     `mapstructure:"appdash"`
}

// IsGoogleCloudEnabled checks if google cloud is enabled
func (t Tracing) IsGoogleCloudEnabled() bool {
	return len(t.GoogleCloudTracing.Email) > 0 && len(t.GoogleCloudTracing.PrivateKey) > 0 && len(t.GoogleCloudTracing.PrivateKeyID) > 0 && len(t.GoogleCloudTracing.ProjectID) > 0
}

// IsAppdashEnabled checks if appdash is enabled
func (t Tracing) IsAppdashEnabled() bool {
	return len(t.AppdashTracing.DSN) > 0
}

func init() {
	viper.SetDefault("port", "8080")
	viper.SetDefault("tls.port", "8433")
	viper.SetDefault("tls.redirect", true)
	viper.SetDefault("backendFlushInterval", "20ms")
	viper.SetDefault("database.dsn", "file:///etc/janus")
	viper.SetDefault("storage.dsn", "memory://localhost")
	viper.SetDefault("web.port", "8081")
	viper.SetDefault("web.tls.port", "8444")
	viper.SetDefault("web.tls.redisrect", true)
	viper.SetDefault("web.credentials.algorithm", "HS256")
	viper.SetDefault("web.credentials.username", "admin")
	viper.SetDefault("web.credentials.password", "admin")
	viper.SetDefault("stats.dsn", "log://")
	viper.SetDefault("stats.errorsSection", "error-log")

	logging.InitDefaults(viper.GetViper(), "log")
}

//Load configuration variables
func Load(configFile string) (*Specification, error) {
	if configFile != "" {
		viper.SetConfigFile(configFile)
	} else {
		viper.SetConfigName("janus")
		viper.AddConfigPath("/etc/janus")
		viper.AddConfigPath(".")
	}

	if err := viper.ReadInConfig(); err != nil {
		log.WithError(err).Warn("No config file found")
		return LoadEnv()
	}

	var config Specification
	if err := viper.Unmarshal(&config); err != nil {
		return nil, err
	}

	return &config, nil
}

//LoadEnv loads configuration from environment variables
func LoadEnv() (*Specification, error) {
	var config Specification

	if err := viper.Unmarshal(&config); err != nil {
		return nil, err
	}

	err := envconfig.Process("", &config)
	if err != nil {
		return nil, err
	}

	return &config, nil
}
<commit_msg>Added github configuration<commit_after>package config

import (
	"time"

	"github.com/hellofresh/logging-go"
	"github.com/kelseyhightower/envconfig"
	log "github.com/sirupsen/logrus"
	"github.com/spf13/viper"
)

// Specification for basic configurations
type Specification struct {
	Port                 int           `envconfig:"PORT"`
	Debug                bool          `envconfig:"DEBUG"`
	GraceTimeOut         int64         `envconfig:"GRACE_TIMEOUT"`
	MaxIdleConnsPerHost  int           `envconfig:"MAX_IDLE_CONNS_PER_HOST"`
	BackendFlushInterval time.Duration `envconfig:"BACKEND_FLUSH_INTERVAL"`
	CloseIdleConnsPeriod time.Duration `envconfig:"CLOSE_IDLE_CONNS_PERIOD"`
	Log                  logging.LogConfig
	Web                  Web
	Database             Database
	Storage              Storage
	Stats                Stats
	Tracing              Tracing
	TLS                  TLS
}

// Web represents the API configurations
type Web struct {
	Port        int  `envconfig:"API_PORT"`
	ReadOnly    bool `envconfig:"API_READONLY"`
	Credentials Credentials
	TLS         TLS
}

// TLS represents the TLS configurations
type TLS struct {
	Port     int    `envconfig:"PORT"`
	CertFile string `envconfig:"CERT_PATH"`
	KeyFile  string `envconfig:"KEY_PATH"`
	Redirect bool   `envconfig:"REDIRECT"`
}

// IsHTTPS checks if you have https enabled
func (s *TLS) IsHTTPS() bool {
	return s.CertFile != "" && s.KeyFile != ""
}

// Storage holds the configuration for a storage
type Storage struct {
	DSN string `envconfig:"STORAGE_DSN"`
}

// Database holds the configuration for a database
type Database struct {
	DSN string `envconfig:"DATABASE_DSN"`
}

// Stats holds the configuration for stats
type Stats struct {
	DSN                   string   `envconfig:"STATS_DSN"`
	Prefix                string   `envconfig:"STATS_PREFIX"`
	IDs                   string   `envconfig:"STATS_IDS"`
	AutoDiscoverThreshold uint     `envconfig:"STATS_AUTO_DISCOVER_THRESHOLD"`
	AutoDiscoverWhiteList []string `envconfig:"STATS_AUTO_DISCOVER_WHITE_LIST"`
	ErrorsSection         string   `envconfig:"STATS_ERRORS_SECTION"`
}

// Credentials represents the credentials that are going to be
// used by admin JWT configuration
type Credentials struct {
	// Algorithm defines admin JWT signing algorithm.
	// Currently the following algorithms are supported: HS256, HS384, HS512.
	Algorithm string `envconfig:"ALGORITHM"`
	Secret    string `envconfig:"SECRET"`
	Github    Github
}

// Github holds the github configurations
type Github struct {
	Organizations []string           `envconfig:"GITHUB_ORGANIZATIONS"`
	Teams         []GitHubTeamConfig `envconfig:"GITHUB_TEAMS"`
}

// GitHubTeamConfig represents a team configuration
type GitHubTeamConfig struct {
	OrganizationName string `json:"organization_name,omitempty"`
	TeamName         string `json:"team_name,omitempty"`
}

// IsConfigured checks if github is enabled
func (auth *Github) IsConfigured() bool {
	return len(auth.Organizations) > 0 ||
		len(auth.Teams) > 0
}

// GoogleCloudTracing holds the Google Application Default Credentials
type GoogleCloudTracing struct {
	ProjectID    string `envconfig:"TRACING_GC_PROJECT_ID"`
	Email        string `envconfig:"TRACING_GC_EMAIL"`
	PrivateKey   string `envconfig:"TRACING_GC_PRIVATE_KEY"`
	PrivateKeyID string `envconfig:"TRACING_GC_PRIVATE_ID"`
}

// AppdashTracing holds the Appdash tracing configuration
type AppdashTracing struct {
	DSN string `envconfig:"TRACING_APPDASH_DSN"`
	URL string `envconfig:"TRACING_APPDASH_URL"`
}

// Tracing represents the distributed tracing configuration
type Tracing struct {
	GoogleCloudTracing GoogleCloudTracing `mapstructure:"googleCloud"`
	AppdashTracing     AppdashTracing     `mapstructure:"appdash"`
}

// IsGoogleCloudEnabled checks if google cloud is enabled
func (t Tracing) IsGoogleCloudEnabled() bool {
	return len(t.GoogleCloudTracing.Email) > 0 && len(t.GoogleCloudTracing.PrivateKey) > 0 && len(t.GoogleCloudTracing.PrivateKeyID) > 0 && len(t.GoogleCloudTracing.ProjectID) > 0
}

// IsAppdashEnabled checks if appdash is enabled
func (t Tracing) IsAppdashEnabled() bool {
	return len(t.AppdashTracing.DSN) > 0
}

func init() {
	viper.SetDefault("port", "8080")
	viper.SetDefault("tls.port", "8433")
	viper.SetDefault("tls.redirect", true)
	viper.SetDefault("backendFlushInterval", "20ms")
	viper.SetDefault("database.dsn", "file:///etc/janus")
	viper.SetDefault("storage.dsn", "memory://localhost")
	viper.SetDefault("web.port", "8081")
	viper.SetDefault("web.tls.port", "8444")
	viper.SetDefault("web.tls.redisrect", true)
	viper.SetDefault("web.credentials.algorithm", "HS256")
	viper.SetDefault("web.credentials.username", "admin")
	viper.SetDefault("web.credentials.password", "admin")
	viper.SetDefault("stats.dsn", "log://")
	viper.SetDefault("stats.errorsSection", "error-log")

	logging.InitDefaults(viper.GetViper(), "log")
}

//Load configuration variables
func Load(configFile string) (*Specification, error) {
	if configFile != "" {
		viper.SetConfigFile(configFile)
	} else {
		viper.SetConfigName("janus")
		viper.AddConfigPath("/etc/janus")
		viper.AddConfigPath(".")
	}

	if err := viper.ReadInConfig(); err != nil {
		log.WithError(err).Warn("No config file found")
		return LoadEnv()
	}

	var config Specification
	if err := viper.Unmarshal(&config); err != nil {
		return nil, err
	}

	return &config, nil
}

//LoadEnv loads configuration from environment variables
func LoadEnv() (*Specification, error) {
	var config Specification

	if err := viper.Unmarshal(&config); err != nil {
		return nil, err
	}

	err := envconfig.Process("", &config)
	if err != nil {
		return nil, err
	}

	return &config, nil
}
<|endoftext|>


<commit_before>package filesystem

import (
	"github.com/Symantec/Dominator/lib/hash"
	"io"
)

type NumLinksTable map[uint64]int

type GenericInode interface {
	List(w io.Writer, name string, numLinksTable NumLinksTable,
		numLinks int) error
}

type InodeTable map[uint64]GenericInode
type InodeToFilenamesTable map[uint64][]string
type HashToInodesTable map[hash.Hash][]uint64

type FileSystem struct {
	InodeTable            InodeTable
	InodeToFilenamesTable InodeToFilenamesTable
	HashToInodesTable     HashToInodesTable
	NumRegularInodes      uint64
	TotalDataBytes        uint64
	DirectoryCount        uint64
	DirectoryInode
}

func (fs *FileSystem) RebuildInodePointers() {
	fs.rebuildInodePointers()
}

func (fs *FileSystem) BuildInodeToFilenamesTable() {
	fs.buildInodeToFilenamesTable()
}

func (fs *FileSystem) BuildHashToInodesTable() {
	fs.buildHashToInodesTable()
}

func (fs *FileSystem) ComputeTotalDataBytes() {
	fs.computeTotalDataBytes()
}

func (fs *FileSystem) List(w io.Writer) error {
	return fs.list(w)
}

type DirectoryInode struct {
	EntryList     []*DirectoryEntry
	EntriesByName map[string]*DirectoryEntry
	Mode          FileMode
	Uid           uint32
	Gid           uint32
}

func (directory *DirectoryInode) BuildEntryMap() {
	directory.buildEntryMap()
}

func (inode *DirectoryInode) List(w io.Writer, name string,
	numLinksTable NumLinksTable, numLinks int) error {
	return inode.list(w, name, numLinksTable, numLinks)
}

type DirectoryEntry struct {
	Name        string
	InodeNumber uint64
	inode       GenericInode // Keep private to avoid encoding/transmission.
}

func (dirent *DirectoryEntry) Inode() GenericInode {
	return dirent.inode
}

func (dirent *DirectoryEntry) SetInode(inode GenericInode) {
	dirent.inode = inode
}

func (dirent *DirectoryEntry) String() string {
	return dirent.Name
}

type RegularInode struct {
	Mode             FileMode
	Uid              uint32
	Gid              uint32
	MtimeNanoSeconds int32
	MtimeSeconds     int64
	Size             uint64
	Hash             hash.Hash
}

func (inode *RegularInode) List(w io.Writer, name string,
	numLinksTable NumLinksTable, numLinks int) error {
	return inode.list(w, name, numLinksTable, numLinks)
}

type SymlinkInode struct {
	Uid     uint32
	Gid     uint32
	Symlink string
}

func (inode *SymlinkInode) List(w io.Writer, name string,
	numLinksTable NumLinksTable, numLinks int) error {
	return inode.list(w, name, numLinksTable, numLinks)
}

type SpecialInode struct {
	Mode             FileMode
	Uid              uint32
	Gid              uint32
	MtimeNanoSeconds int32
	MtimeSeconds     int64
	Rdev             uint64
}

func (inode *SpecialInode) List(w io.Writer, name string,
	numLinksTable NumLinksTable, numLinks int) error {
	return inode.list(w, name, numLinksTable, numLinks)
}

type FileMode uint32

func (mode FileMode) String() string {
	return mode.string()
}

func CompareFileSystems(left, right *FileSystem, logWriter io.Writer) bool {
	return compareFileSystems(left, right, logWriter)
}

func CompareDirectoryInodes(left, right *DirectoryInode,
	logWriter io.Writer) bool {
	return compareDirectoryInodes(left, right, logWriter)
}

func CompareDirectoriesMetadata(left, right *DirectoryInode,
	logWriter io.Writer) bool {
	return compareDirectoriesMetadata(left, right, logWriter)
}

func CompareDirectoryEntries(left, right *DirectoryEntry,
	logWriter io.Writer) bool {
	return compareDirectoryEntries(left, right, logWriter)
}

func CompareInodes(left, right GenericInode, logWriter io.Writer) (
	sameType, sameMetadata, sameData bool) {
	return compareInodes(left, right, logWriter)
}

func CompareRegularInodes(left, right *RegularInode, logWriter io.Writer) bool {
	return compareRegularInodes(left, right, logWriter)
}

func CompareRegularInodesMetadata(left, right *RegularInode,
	logWriter io.Writer) bool {
	return compareRegularInodesMetadata(left, right, logWriter)
}

func CompareRegularInodesData(left, right *RegularInode,
	logWriter io.Writer) bool {
	return compareRegularInodesData(left, right, logWriter)
}

func CompareSymlinkInodes(left, right *SymlinkInode, logWriter io.Writer) bool {
	return compareSymlinkInodes(left, right, logWriter)
}

func CompareSymlinkInodesMetadata(left, right *SymlinkInode,
	logWriter io.Writer) bool {
	return compareSymlinkInodesMetadata(left, right, logWriter)
}

func CompareSymlinkInodesData(left, right *SymlinkInode,
	logWriter io.Writer) bool {
	return compareSymlinkInodesData(left, right, logWriter)
}

func CompareSpecialInodes(left, right *SpecialInode, logWriter io.Writer) bool {
	return compareSpecialInodes(left, right, logWriter)
}

func CompareSpecialInodesMetadata(left, right *SpecialInode,
	logWriter io.Writer) bool {
	return compareSpecialInodesMetadata(left, right, logWriter)
}

func CompareSpecialInodesData(left, right *SpecialInode,
	logWriter io.Writer) bool {
	return compareSpecialInodesData(left, right, logWriter)
}
<commit_msg>Add GetUid() and GetGid() methods to GenericInode interface.<commit_after>package filesystem

import (
	"github.com/Symantec/Dominator/lib/hash"
	"io"
)

type NumLinksTable map[uint64]int

type GenericInode interface {
	GetUid() uint32
	GetGid() uint32
	List(w io.Writer, name string, numLinksTable NumLinksTable,
		numLinks int) error
}

type InodeTable map[uint64]GenericInode
type InodeToFilenamesTable map[uint64][]string
type HashToInodesTable map[hash.Hash][]uint64

type FileSystem struct {
	InodeTable            InodeTable
	InodeToFilenamesTable InodeToFilenamesTable
	HashToInodesTable     HashToInodesTable
	NumRegularInodes      uint64
	TotalDataBytes        uint64
	DirectoryCount        uint64
	DirectoryInode
}

func (fs *FileSystem) RebuildInodePointers() {
	fs.rebuildInodePointers()
}

func (fs *FileSystem) BuildInodeToFilenamesTable() {
	fs.buildInodeToFilenamesTable()
}

func (fs *FileSystem) BuildHashToInodesTable() {
	fs.buildHashToInodesTable()
}

func (fs *FileSystem) ComputeTotalDataBytes() {
	fs.computeTotalDataBytes()
}

func (fs *FileSystem) List(w io.Writer) error {
	return fs.list(w)
}

type DirectoryInode struct {
	EntryList     []*DirectoryEntry
	EntriesByName map[string]*DirectoryEntry
	Mode          FileMode
	Uid           uint32
	Gid           uint32
}

func (directory *DirectoryInode) BuildEntryMap() {
	directory.buildEntryMap()
}

func (inode *DirectoryInode) GetUid() uint32 {
	return inode.Uid
}

func (inode *DirectoryInode) GetGid() uint32 {
	return inode.Gid
}

func (inode *DirectoryInode) List(w io.Writer, name string,
	numLinksTable NumLinksTable, numLinks int) error {
	return inode.list(w, name, numLinksTable, numLinks)
}

type DirectoryEntry struct {
	Name        string
	InodeNumber uint64
	inode       GenericInode // Keep private to avoid encoding/transmission.
}

func (dirent *DirectoryEntry) Inode() GenericInode {
	return dirent.inode
}

func (dirent *DirectoryEntry) SetInode(inode GenericInode) {
	dirent.inode = inode
}

func (dirent *DirectoryEntry) String() string {
	return dirent.Name
}

type RegularInode struct {
	Mode             FileMode
	Uid              uint32
	Gid              uint32
	MtimeNanoSeconds int32
	MtimeSeconds     int64
	Size             uint64
	Hash             hash.Hash
}

func (inode *RegularInode) GetUid() uint32 {
	return inode.Uid
}

func (inode *RegularInode) GetGid() uint32 {
	return inode.Gid
}

func (inode *RegularInode) List(w io.Writer, name string,
	numLinksTable NumLinksTable, numLinks int) error {
	return inode.list(w, name, numLinksTable, numLinks)
}

type SymlinkInode struct {
	Uid     uint32
	Gid     uint32
	Symlink string
}

func (inode *SymlinkInode) GetUid() uint32 {
	return inode.Uid
}

func (inode *SymlinkInode) GetGid() uint32 {
	return inode.Gid
}

func (inode *SymlinkInode) List(w io.Writer, name string,
	numLinksTable NumLinksTable, numLinks int) error {
	return inode.list(w, name, numLinksTable, numLinks)
}

type SpecialInode struct {
	Mode             FileMode
	Uid              uint32
	Gid              uint32
	MtimeNanoSeconds int32
	MtimeSeconds     int64
	Rdev             uint64
}

func (inode *SpecialInode) GetUid() uint32 {
	return inode.Uid
}

func (inode *SpecialInode) GetGid() uint32 {
	return inode.Gid
}

func (inode *SpecialInode) List(w io.Writer, name string,
	numLinksTable NumLinksTable, numLinks int) error {
	return inode.list(w, name, numLinksTable, numLinks)
}

type FileMode uint32

func (mode FileMode) String() string {
	return mode.string()
}

func CompareFileSystems(left, right *FileSystem, logWriter io.Writer) bool {
	return compareFileSystems(left, right, logWriter)
}

func CompareDirectoryInodes(left, right *DirectoryInode,
	logWriter io.Writer) bool {
	return compareDirectoryInodes(left, right, logWriter)
}

func CompareDirectoriesMetadata(left, right *DirectoryInode,
	logWriter io.Writer) bool {
	return compareDirectoriesMetadata(left, right, logWriter)
}

func CompareDirectoryEntries(left, right *DirectoryEntry,
	logWriter io.Writer) bool {
	return compareDirectoryEntries(left, right, logWriter)
}

func CompareInodes(left, right GenericInode, logWriter io.Writer) (
	sameType, sameMetadata, sameData bool) {
	return compareInodes(left, right, logWriter)
}

func CompareRegularInodes(left, right *RegularInode, logWriter io.Writer) bool {
	return compareRegularInodes(left, right, logWriter)
}

func CompareRegularInodesMetadata(left, right *RegularInode,
	logWriter io.Writer) bool {
	return compareRegularInodesMetadata(left, right, logWriter)
}

func CompareRegularInodesData(left, right *RegularInode,
	logWriter io.Writer) bool {
	return compareRegularInodesData(left, right, logWriter)
}

func CompareSymlinkInodes(left, right *SymlinkInode, logWriter io.Writer) bool {
	return compareSymlinkInodes(left, right, logWriter)
}

func CompareSymlinkInodesMetadata(left, right *SymlinkInode,
	logWriter io.Writer) bool {
	return compareSymlinkInodesMetadata(left, right, logWriter)
}

func CompareSymlinkInodesData(left, right *SymlinkInode,
	logWriter io.Writer) bool {
	return compareSymlinkInodesData(left, right, logWriter)
}

func CompareSpecialInodes(left, right *SpecialInode, logWriter io.Writer) bool {
	return compareSpecialInodes(left, right, logWriter)
}

func CompareSpecialInodesMetadata(left, right *SpecialInode,
	logWriter io.Writer) bool {
	return compareSpecialInodesMetadata(left, right, logWriter)
}

func CompareSpecialInodesData(left, right *SpecialInode,
	logWriter io.Writer) bool {
	return compareSpecialInodesData(left, right, logWriter)
}
<|endoftext|>


<commit_before>package cmd

import (
	"flag"
	"fmt"
	"github.com/driusan/dgit/git"
	"os"
)

func Branch(c *git.Client, args []string) {
	flags := flag.NewFlagSet("branch", flag.ExitOnError)
	flags.SetOutput(flag.CommandLine.Output())
	flags.Usage = func() {
		flag.Usage()
		fmt.Fprintf(flag.CommandLine.Output(), "\n\nOptions:\n")
		flags.PrintDefaults()
	}

	// These flags can be moved out of these lists and below as proper flags as they are implemented
	for _, bf := range []string{"d", "delete", "D", "create-reflog", "f", "force", "m", "move", "M", "c", "copy", "C", "no-color", "i", "ignore-case", "no-column", "r", "remotes", "a", "all", "v", "vv", "verbose", "q", "quiet", "no-abbrev", "no-track", "unset-upstream", "edit-description"} {
		flags.Var(newNotimplBoolValue(), bf, "Not implemented")
	}
	for _, sf := range []string{"color", "abbrev", "column", "sort", "no-merged", "contains", "no-contains", "points-at", "format", "set-upstream-to", "u"} {
		flags.Var(newNotimplStringValue(), sf, "Not implemented")
	}

	flags.Parse(args)

	switch flags.NArg() {
	case 0:
		branches, err := c.GetBranches()
		if err != nil {
			fmt.Fprintln(os.Stderr, "Could not get list of branches.")
			return
		}
		head := c.GetHeadBranch()
		for _, b := range branches {
			if head == b {
				fmt.Print("* ")
			} else {
				fmt.Print("  ")
			}
			fmt.Println(b.BranchName())
		}
	case 1:
		headref, err := git.SymbolicRefGet(c, git.SymbolicRefOptions{}, "HEAD")
		if err != nil {
			return
		}
		b := git.Branch(headref)
		if err := c.CreateBranch(flags.Arg(0), b); err != nil {
			fmt.Fprintf(os.Stderr, "Could not create branch (%v): %v\n", flags.Arg(0), err)
		}
	default:
		flag.Usage()
		os.Exit(2)
	}

}
<commit_msg>Added the ability to create a branch at a startpoint<commit_after>package cmd

import (
	"flag"
	"fmt"
	"github.com/driusan/dgit/git"
	"os"
)

func Branch(c *git.Client, args []string) {
	flags := flag.NewFlagSet("branch", flag.ExitOnError)
	flags.SetOutput(flag.CommandLine.Output())
	flags.Usage = func() {
		flag.Usage()
		fmt.Fprintf(flag.CommandLine.Output(), "\n\nOptions:\n")
		flags.PrintDefaults()
	}

	// These flags can be moved out of these lists and below as proper flags as they are implemented
	for _, bf := range []string{"d", "delete", "D", "create-reflog", "f", "force", "m", "move", "M", "c", "copy", "C", "no-color", "i", "ignore-case", "no-column", "r", "remotes", "a", "all", "v", "vv", "verbose", "q", "quiet", "no-abbrev", "no-track", "unset-upstream", "edit-description"} {
		flags.Var(newNotimplBoolValue(), bf, "Not implemented")
	}
	for _, sf := range []string{"color", "abbrev", "column", "sort", "no-merged", "contains", "no-contains", "points-at", "format", "set-upstream-to", "u"} {
		flags.Var(newNotimplStringValue(), sf, "Not implemented")
	}

	flags.Parse(args)

	switch flags.NArg() {
	case 0:
		branches, err := c.GetBranches()
		if err != nil {
			fmt.Fprintln(os.Stderr, "Could not get list of branches.")
			return
		}
		head := c.GetHeadBranch()
		for _, b := range branches {
			if head == b {
				fmt.Print("* ")
			} else {
				fmt.Print("  ")
			}
			fmt.Println(b.BranchName())
		}
	case 1:
		headref, err := git.SymbolicRefGet(c, git.SymbolicRefOptions{}, "HEAD")
		if err != nil {
			return
		}
		b := git.Branch(headref)
		if err := c.CreateBranch(flags.Arg(0), b); err != nil {
			fmt.Fprintf(os.Stderr, "Could not create branch (%v): %v\n", flags.Arg(0), err)
		}
	case 2:
		startpoint, err := git.RevParseCommitish(c, &git.RevParseOptions{}, flags.Arg(1))
		if err != nil {
			return
		}
		if err := c.CreateBranch(flags.Arg(0), startpoint); err != nil {
			fmt.Fprintf(os.Stderr, "Could not create branch (%v): %v\n", flags.Arg(0), err)
		}
	default:
		flag.Usage()
		os.Exit(2)
	}

}
<|endoftext|>


<commit_before>package asciidocgo

import (
	"fmt"
	"testing"
	. "github.com/smartystreets/goconvey/convey"
)

type testSubstDocumentAble struct {
}

func (tsd *testSubstDocumentAble) Attr(name string, defaultValue interface{}, inherit bool) interface{} {
	return "mathtest"
}
func (tsd *testSubstDocumentAble) Basebackend(base interface{}) bool {
	return true
}

func TestSubstitutor(t *testing.T) {

	Convey("A substitutors can be initialized", t, func() {

		Convey("By default, a substitutors can be created", func() {
			So(&substitutors{}, ShouldNotBeNil)
		})

		Convey("A substitutors has an empty passthroughs array", func() {
			s := substitutors{}
			So(len(s.passthroughs), ShouldEqual, 0)
		})
	})

	Convey("A substitutors has subs type", t, func() {
		So(len(subs[sub.basic]), ShouldEqual, 1)
		So(len(subs[sub.normal]), ShouldEqual, 6)
		So(len(subs[sub.verbatim]), ShouldEqual, 2)
		So(len(subs[sub.title]), ShouldEqual, 6)
		So(len(subs[sub.header]), ShouldEqual, 2)
		So(len(subs[sub.pass]), ShouldEqual, 0)
		So(len(subs[sub.unknown]), ShouldEqual, 0)
	})

	Convey("A substitutors can apply substitutions", t, func() {

		source := "test"
		s := &substitutors{}

		Convey("By default, no substitution or a pass subs will return source unchanged", func() {
			So(s.ApplySubs(source, nil), ShouldEqual, source)
			So(s.ApplySubs(source, subArray{sub.pass}), ShouldResemble, source)
			So(len(s.ApplySubs(source, subArray{sub.unknown})), ShouldEqual, 0)
			So(s.ApplySubs(source, subArray{sub.title}), ShouldEqual, "test")
		})

		Convey("A normal substition will use normal substitution modes", func() {
			testsub = "test_ApplySubs_allsubs"
			So(s.ApplySubs(source, subArray{sub.normal}), ShouldEqual, "[specialcharacters quotes attributes replacements macros post_replacements]")
			So(s.ApplySubs(source, subArray{sub.title}), ShouldEqual, "[title]")
			testsub = ""
		})
		Convey("A macros substition will call extractPassthroughs", func() {
			testsub = "test_ApplySubs_extractPassthroughs"
			So(s.ApplySubs(source, subArray{subValue.macros}), ShouldEqual, "test")
			testsub = ""
		})

	})

	Convey("A substitutors can Extract the passthrough text from the document for reinsertion without processing if escaped", t, func() {
		source := `test \+++for
		a
		passthrough+++ by test2 \$$text
			multiple
			line$$ for
			test3 \pass:quotes[text
			line2
			line3] end test4`
		s := &substitutors{}
		So(s.ApplySubs(source, subArray{subValue.macros}), ShouldEqual, `test +++for
		a
		passthrough+++ by test2 $$text
			multiple
			line$$ for
			test3 pass:quotes[text
			line2
			line3] end test4`)
	})

	Convey("A substitutors can Extract the passthrough text from the document for reinsertion after processing", t, func() {
		source := `test +++for
		a
		passthrough+++ by test2 $$text
			multiple
			line$$ for
			test3 pass:quotes[text
			line2
			line3] end test4`
		s := &substitutors{}

		Convey("If no inline macros substitution detected, return text unchanged", func() {
			So(s.ApplySubs("test ++ nosub", subArray{subValue.macros}), ShouldEqual, "test ++ nosub")
		})

		So(s.ApplySubs(source, subArray{subValue.macros}), ShouldEqual, fmt.Sprintf(`test %s0%s by test2 %s1%s for
			test3 %s2%s end test4`, subPASS_START, subPASS_END, subPASS_START, subPASS_END, subPASS_START, subPASS_END))
	})
	Convey("A substitutors can unescape escaped branckets", t, func() {
		So(unescapeBrackets(""), ShouldEqual, "")
		So(unescapeBrackets(`a\]b]c\]`), ShouldEqual, `a]b]c]`)
	})

	Convey("A substitutors can Extract inline text", t, func() {
		source := "`a few <\\{monospaced\\}> words`" +
			"[input]`A few <\\{monospaced\\}> words`\n" +
			"\\[input]`a few <monospaced> words`\n" +
			"\\[input]\\`a few <monospaced> words`\n" +
			"`a few\n<\\{monospaced\\}> words`" +
			"\\[input]`a few <monospaced> words`\n" +
			"the text `asciimath:[x = y]` should be passed through as `literal` text\n" +
			"`Here`s Johnny!"
		s := &substitutors{}

		So(s.ApplySubs(source, subArray{subValue.macros}), ShouldEqual, fmt.Sprintf(`%s0%s[input]%s1%s
[input]%s2%s
\input`+"`"+`a few <monospaced> words`+"`"+` : \`+"`"+`a few <monospaced> words`+"`"+`
%s3%s[input]%s4%s
the text %s5%s should be passed through as %s6%s text
`+"`"+`Here`+"`"+`s Johnny!`, subPASS_START, subPASS_END, subPASS_START, subPASS_END, subPASS_START, subPASS_END, subPASS_START, subPASS_END, subPASS_START, subPASS_END, subPASS_START, subPASS_END, subPASS_START, subPASS_END))

		Convey("If no literal text substitution detected, return text unchanged", func() {
			So(s.ApplySubs("test`nosub", subArray{subValue.macros}), ShouldEqual, "test`nosub")
		})
	})

	Convey("A substitutors can Extract math inline text", t, func() {
		source := `math:[x != 0]
   \math:[x != 0]
   asciimath:[x != 0]
   latexmath:abc[\sqrt{4} = 2]`
		s := &substitutors{document: &testSubstDocumentAble{}}

		So(s.ApplySubs(source, subArray{subValue.macros}), ShouldEqual, fmt.Sprintf(`%s0%s
   math:[x != 0]
   %s1%s
   %s2%s`, subPASS_START, subPASS_END, subPASS_START, subPASS_END, subPASS_START, subPASS_END))

		Convey("If no math literal substitution detected, return text unchanged", func() {
			So(s.ApplySubs("math:nosub", subArray{subValue.macros}), ShouldEqual, "math:nosub")
		})
	})
}
<commit_msg>Test subsitutors with nil (Subst)Document(able).<commit_after>package asciidocgo

import (
	"fmt"
	"testing"
	. "github.com/smartystreets/goconvey/convey"
)

type testSubstDocumentAble struct {
}

func (tsd *testSubstDocumentAble) Attr(name string, defaultValue interface{}, inherit bool) interface{} {
	return "mathtest"
}
func (tsd *testSubstDocumentAble) Basebackend(base interface{}) bool {
	return true
}

func TestSubstitutor(t *testing.T) {

	Convey("A substitutors can be initialized", t, func() {

		Convey("By default, a substitutors can be created", func() {
			So(&substitutors{}, ShouldNotBeNil)
		})

		Convey("A substitutors has an empty passthroughs array", func() {
			s := substitutors{}
			So(len(s.passthroughs), ShouldEqual, 0)
		})
	})

	Convey("A substitutors has subs type", t, func() {
		So(len(subs[sub.basic]), ShouldEqual, 1)
		So(len(subs[sub.normal]), ShouldEqual, 6)
		So(len(subs[sub.verbatim]), ShouldEqual, 2)
		So(len(subs[sub.title]), ShouldEqual, 6)
		So(len(subs[sub.header]), ShouldEqual, 2)
		So(len(subs[sub.pass]), ShouldEqual, 0)
		So(len(subs[sub.unknown]), ShouldEqual, 0)
	})

	Convey("A substitutors can apply substitutions", t, func() {

		source := "test"
		s := &substitutors{}

		Convey("By default, no substitution or a pass subs will return source unchanged", func() {
			So(s.ApplySubs(source, nil), ShouldEqual, source)
			So(s.ApplySubs(source, subArray{sub.pass}), ShouldResemble, source)
			So(len(s.ApplySubs(source, subArray{sub.unknown})), ShouldEqual, 0)
			So(s.ApplySubs(source, subArray{sub.title}), ShouldEqual, "test")
		})

		Convey("A normal substition will use normal substitution modes", func() {
			testsub = "test_ApplySubs_allsubs"
			So(s.ApplySubs(source, subArray{sub.normal}), ShouldEqual, "[specialcharacters quotes attributes replacements macros post_replacements]")
			So(s.ApplySubs(source, subArray{sub.title}), ShouldEqual, "[title]")
			testsub = ""
		})
		Convey("A macros substition will call extractPassthroughs", func() {
			testsub = "test_ApplySubs_extractPassthroughs"
			So(s.ApplySubs(source, subArray{subValue.macros}), ShouldEqual, "test")
			testsub = ""
		})

	})

	Convey("A substitutors can Extract the passthrough text from the document for reinsertion without processing if escaped", t, func() {
		source := `test \+++for
		a
		passthrough+++ by test2 \$$text
			multiple
			line$$ for
			test3 \pass:quotes[text
			line2
			line3] end test4`
		s := &substitutors{}
		So(s.ApplySubs(source, subArray{subValue.macros}), ShouldEqual, `test +++for
		a
		passthrough+++ by test2 $$text
			multiple
			line$$ for
			test3 pass:quotes[text
			line2
			line3] end test4`)
	})

	Convey("A substitutors can Extract the passthrough text from the document for reinsertion after processing", t, func() {
		source := `test +++for
		a
		passthrough+++ by test2 $$text
			multiple
			line$$ for
			test3 pass:quotes[text
			line2
			line3] end test4`
		s := &substitutors{}

		Convey("If no inline macros substitution detected, return text unchanged", func() {
			So(s.ApplySubs("test ++ nosub", subArray{subValue.macros}), ShouldEqual, "test ++ nosub")
		})

		So(s.ApplySubs(source, subArray{subValue.macros}), ShouldEqual, fmt.Sprintf(`test %s0%s by test2 %s1%s for
			test3 %s2%s end test4`, subPASS_START, subPASS_END, subPASS_START, subPASS_END, subPASS_START, subPASS_END))
	})
	Convey("A substitutors can unescape escaped branckets", t, func() {
		So(unescapeBrackets(""), ShouldEqual, "")
		So(unescapeBrackets(`a\]b]c\]`), ShouldEqual, `a]b]c]`)
	})

	Convey("A substitutors can Extract inline text", t, func() {
		source := "`a few <\\{monospaced\\}> words`" +
			"[input]`A few <\\{monospaced\\}> words`\n" +
			"\\[input]`a few <monospaced> words`\n" +
			"\\[input]\\`a few <monospaced> words`\n" +
			"`a few\n<\\{monospaced\\}> words`" +
			"\\[input]`a few <monospaced> words`\n" +
			"the text `asciimath:[x = y]` should be passed through as `literal` text\n" +
			"`Here`s Johnny!"
		s := &substitutors{}

		So(s.ApplySubs(source, subArray{subValue.macros}), ShouldEqual, fmt.Sprintf(`%s0%s[input]%s1%s
[input]%s2%s
\input`+"`"+`a few <monospaced> words`+"`"+` : \`+"`"+`a few <monospaced> words`+"`"+`
%s3%s[input]%s4%s
the text %s5%s should be passed through as %s6%s text
`+"`"+`Here`+"`"+`s Johnny!`, subPASS_START, subPASS_END, subPASS_START, subPASS_END, subPASS_START, subPASS_END, subPASS_START, subPASS_END, subPASS_START, subPASS_END, subPASS_START, subPASS_END, subPASS_START, subPASS_END))

		Convey("If no literal text substitution detected, return text unchanged", func() {
			So(s.ApplySubs("test`nosub", subArray{subValue.macros}), ShouldEqual, "test`nosub")
		})
	})

	Convey("A substitutors can Extract math inline text", t, func() {
		source := `math:[x != 0]
   \math:[x != 0]
   asciimath:[x != 0]
   latexmath:abc[\sqrt{4} = 2]`
		s := &substitutors{}

		So(s.ApplySubs(source, subArray{subValue.macros}), ShouldEqual, fmt.Sprintf(`%s0%s
   math:[x != 0]
   %s1%s
   %s2%s`, subPASS_START, subPASS_END, subPASS_START, subPASS_END, subPASS_START, subPASS_END))

		s.document = &testSubstDocumentAble{}

		So(s.ApplySubs(source, subArray{subValue.macros}), ShouldEqual, fmt.Sprintf(`%s3%s
   math:[x != 0]
   %s4%s
   %s5%s`, subPASS_START, subPASS_END, subPASS_START, subPASS_END, subPASS_START, subPASS_END))

		Convey("If no math literal substitution detected, return text unchanged", func() {
			So(s.ApplySubs("math:nosub", subArray{subValue.macros}), ShouldEqual, "math:nosub")
		})
	})
}
<|endoftext|>


<commit_before>package triggerbuild

import (
	"fmt"
	"net/http"

	"github.com/concourse/atc/builder"
	"github.com/concourse/atc/db"
	"github.com/pivotal-golang/lager"
	"github.com/tedsuo/rata"

	"github.com/concourse/atc/config"
	"github.com/concourse/atc/server/routes"
)

type handler struct {
	logger lager.Logger

	jobs config.Jobs

	db      db.DB
	builder builder.Builder
}

func NewHandler(
	logger lager.Logger,
	jobs config.Jobs,
	db db.DB,
	builder builder.Builder,
) http.Handler {
	return &handler{
		logger: logger,

		jobs: jobs,

		db:      db,
		builder: builder,
	}
}

func (handler *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	job, found := handler.jobs.Lookup(r.FormValue(":job"))
	if !found {
		w.WriteHeader(http.StatusNotFound)
		return
	}

	log := handler.logger.Session("trigger-build", lager.Data{
		"job": job.Name,
	})

	log.Debug("triggering")

	build, err := handler.db.CreateBuild(job.Name)
	if err != nil {
		log.Error("failed-to-create-build", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	err = handler.builder.Build(build, job, nil)
	if err != nil {
		log.Error("triggering-failed", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	redirectPath, err := routes.Routes.CreatePathForRoute(routes.GetBuild, rata.Params{
		"job":   job.Name,
		"build": fmt.Sprintf("%d", build.ID),
	})
	if err != nil {
		log.Fatal("failed-to-construct-redirect-uri", err, lager.Data{
			"build": build.ID,
		})
	}

	http.Redirect(w, r, redirectPath, 302)
}
<commit_msg>when triggering, don't sidestep passed: config<commit_after>package triggerbuild

import (
	"fmt"
	"net/http"

	"github.com/concourse/atc/builder"
	"github.com/concourse/atc/builds"
	"github.com/concourse/atc/db"
	"github.com/pivotal-golang/lager"
	"github.com/tedsuo/rata"

	"github.com/concourse/atc/config"
	"github.com/concourse/atc/server/routes"
)

type handler struct {
	logger lager.Logger

	jobs config.Jobs

	db      db.DB
	builder builder.Builder
}

func NewHandler(
	logger lager.Logger,
	jobs config.Jobs,
	db db.DB,
	builder builder.Builder,
) http.Handler {
	return &handler{
		logger: logger,

		jobs: jobs,

		db:      db,
		builder: builder,
	}
}

func (handler *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	job, found := handler.jobs.Lookup(r.FormValue(":job"))
	if !found {
		w.WriteHeader(http.StatusNotFound)
		return
	}

	log := handler.logger.Session("trigger-build", lager.Data{
		"job": job.Name,
	})

	log.Debug("triggering")

	build, err := handler.db.CreateBuild(job.Name)
	if err != nil {
		log.Error("failed-to-create-build", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	passedInputs := []config.Input{}
	for _, input := range job.Inputs {
		if len(input.Passed) == 0 {
			continue
		}

		passedInputs = append(passedInputs, input)
	}

	var inputs builds.VersionedResources

	if len(passedInputs) > 0 {
		inputs, err = handler.db.GetLatestInputVersions(passedInputs)
		if err != nil {
			log.Error("failed-to-get-build-inputs", err)
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
	}

	err = handler.builder.Build(build, job, inputs)
	if err != nil {
		log.Error("triggering-failed", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	redirectPath, err := routes.Routes.CreatePathForRoute(routes.GetBuild, rata.Params{
		"job":   job.Name,
		"build": fmt.Sprintf("%d", build.ID),
	})
	if err != nil {
		log.Fatal("failed-to-construct-redirect-uri", err, lager.Data{
			"build": build.ID,
		})
	}

	http.Redirect(w, r, redirectPath, 302)
}
<|endoftext|>


<commit_before>package cache

import (
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"sync"
)

var blobNameSHA256 = regexp.MustCompile("^/?(ac/|cas/)?([a-f0-9]{64})$")

// HTTPCache ...
type HTTPCache interface {
	Serve()
}

type httpCache struct {
	addr              string
	cache             Cache
	ensureSpacer      EnsureSpacer
	ongoingUploads    map[string]*sync.Mutex
	ongoingUploadsMux *sync.Mutex
}

// NewHTTPCache ...
func NewHTTPCache(listenAddr string, cacheDir string, maxBytes int64, ensureSpacer EnsureSpacer) HTTPCache {
	ensureCacheDir(cacheDir)
	cache := NewCache(cacheDir, maxBytes)
	loadFilesIntoCache(cache)
	return &httpCache{listenAddr, cache, ensureSpacer, make(map[string]*sync.Mutex), &sync.Mutex{}}
}

// Serve ...
func (h *httpCache) Serve() {
	s := &http.Server{
		Addr:    h.addr,
		Handler: h,
	}
	log.Fatal(s.ListenAndServe())
}

func ensureCacheDir(path string) {
	d, err := os.Open(path)
	if err != nil {
		err := os.MkdirAll(path, os.FileMode(0644))
		if err != nil {
			log.Fatal(err)
		}
	}
	d.Close()
}

func loadFilesIntoCache(cache Cache) {
	filepath.Walk(cache.Dir(), func(name string, info os.FileInfo, err error) error {
		if !info.IsDir() {
			cache.AddFile(filepath.Base(name), info.Size())
		}
		return nil
	})
}

func (h *httpCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	parts, err := parseURL(r.URL.Path)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	var hash string
	var verifyHash bool
	if len(parts) == 1 {
		// For backwards compatibiliy with older Bazel version's that don't
		// support {cas,actioncache} prefixes.
		verifyHash = false
		hash = parts[0]
	} else {
		verifyHash = parts[0] == "cas/"
		hash = parts[1]
	}

	switch m := r.Method; m {
	case http.MethodGet:
		if !h.cache.ContainsFile(hash) {
			w.WriteHeader(http.StatusNotFound)
			return
		}
		http.ServeFile(w, r, h.filePath(hash))
	case http.MethodPut:
		if h.cache.ContainsFile(hash) {
			h.discardUpload(w, r.Body)
			return
		}
		uploadMux := h.startUpload(hash)
		uploadMux.Lock()
		defer h.stopUpload(hash)
		defer uploadMux.Unlock()
		if h.cache.ContainsFile(hash) {
			h.discardUpload(w, r.Body)
			return
		}
		if !h.ensureSpacer.EnsureSpace(h.cache, r.ContentLength) {
			http.Error(w, "The disk is full. File could not be uploaded.",
				http.StatusInsufficientStorage)
			return
		}
		written, err := h.saveToDisk(r.Body, hash, verifyHash)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		h.cache.AddFile(hash, written)
		w.WriteHeader(http.StatusOK)
	case http.MethodHead:
		if !h.cache.ContainsFile(hash) {
			http.Error(w, err.Error(), http.StatusNotFound)
		}
		w.WriteHeader(http.StatusOK)
	default:
		msg := fmt.Sprintf("Method '%s' not supported.", m)
		http.Error(w, msg, http.StatusMethodNotAllowed)
	}
}

func (h *httpCache) startUpload(hash string) *sync.Mutex {
	h.ongoingUploadsMux.Lock()
	defer h.ongoingUploadsMux.Unlock()
	mux, ok := h.ongoingUploads[hash]
	if !ok {
		mux = &sync.Mutex{}
		h.ongoingUploads[hash] = mux
		return mux
	}
	return mux
}

func (h *httpCache) stopUpload(hash string) {
	h.ongoingUploadsMux.Lock()
	defer h.ongoingUploadsMux.Unlock()
	delete(h.ongoingUploads, hash)
}

func (h *httpCache) discardUpload(w http.ResponseWriter, r io.Reader) {
	io.Copy(ioutil.Discard, r)
	w.WriteHeader(http.StatusOK)
}

func parseURL(url string) ([]string, error) {
	m := blobNameSHA256.FindStringSubmatch(url)
	if m == nil {
		msg := fmt.Sprintf("Resource name must be a SHA256 hash in hex. "+
			"Got '%s'.", url)
		return nil, errors.New(msg)
	}
	return m[1:], nil
}

func (h *httpCache) saveToDisk(content io.Reader, hash string, verifyHash bool) (written int64, err error) {
	f, err := ioutil.TempFile(h.cache.Dir(), "upload")
	if err != nil {
		return 0, err
	}
	tmpName := f.Name()
	if verifyHash {
		hasher := sha256.New()
		written, err = io.Copy(io.MultiWriter(f, hasher), content)
		actualHash := hex.EncodeToString(hasher.Sum(nil))
		if hash != actualHash {
			os.Remove(tmpName)
			msg := fmt.Sprintf("Hashes don't match. Provided '%s', Actual '%s'.",
				hash, actualHash)
			return 0, errors.New(msg)
		}
	} else {
		written, err = io.Copy(f, content)
	}
	if err != nil {
		return 0, err
	}
	err = f.Sync()
	if err != nil {
		log.Fatal(err)
	}
	f.Close()
	err2 := os.Rename(tmpName, h.filePath(hash))
	if err2 != nil {
		return 0, err2
	}
	return written, nil
}

func (h httpCache) filePath(hash string) string {
	return fmt.Sprintf("%s%c%s", h.cache.Dir(), os.PathSeparator, hash)
}
<commit_msg>cache/http: fix potential XSS vulnerability<commit_after>package cache

import (
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"fmt"
	"html"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"sync"
)

var blobNameSHA256 = regexp.MustCompile("^/?(ac/|cas/)?([a-f0-9]{64})$")

// HTTPCache ...
type HTTPCache interface {
	Serve()
}

type httpCache struct {
	addr              string
	cache             Cache
	ensureSpacer      EnsureSpacer
	ongoingUploads    map[string]*sync.Mutex
	ongoingUploadsMux *sync.Mutex
}

// NewHTTPCache ...
func NewHTTPCache(listenAddr string, cacheDir string, maxBytes int64, ensureSpacer EnsureSpacer) HTTPCache {
	ensureCacheDir(cacheDir)
	cache := NewCache(cacheDir, maxBytes)
	loadFilesIntoCache(cache)
	return &httpCache{listenAddr, cache, ensureSpacer, make(map[string]*sync.Mutex), &sync.Mutex{}}
}

// Serve ...
func (h *httpCache) Serve() {
	s := &http.Server{
		Addr:    h.addr,
		Handler: h,
	}
	log.Fatal(s.ListenAndServe())
}

func ensureCacheDir(path string) {
	d, err := os.Open(path)
	if err != nil {
		err := os.MkdirAll(path, os.FileMode(0644))
		if err != nil {
			log.Fatal(err)
		}
	}
	d.Close()
}

func loadFilesIntoCache(cache Cache) {
	filepath.Walk(cache.Dir(), func(name string, info os.FileInfo, err error) error {
		if !info.IsDir() {
			cache.AddFile(filepath.Base(name), info.Size())
		}
		return nil
	})
}

func (h *httpCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	parts, err := parseURL(r.URL.Path)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	var hash string
	var verifyHash bool
	if len(parts) == 1 {
		// For backwards compatibiliy with older Bazel version's that don't
		// support {cas,actioncache} prefixes.
		verifyHash = false
		hash = parts[0]
	} else {
		verifyHash = parts[0] == "cas/"
		hash = parts[1]
	}

	switch m := r.Method; m {
	case http.MethodGet:
		if !h.cache.ContainsFile(hash) {
			w.WriteHeader(http.StatusNotFound)
			return
		}
		http.ServeFile(w, r, h.filePath(hash))
	case http.MethodPut:
		if h.cache.ContainsFile(hash) {
			h.discardUpload(w, r.Body)
			return
		}
		uploadMux := h.startUpload(hash)
		uploadMux.Lock()
		defer h.stopUpload(hash)
		defer uploadMux.Unlock()
		if h.cache.ContainsFile(hash) {
			h.discardUpload(w, r.Body)
			return
		}
		if !h.ensureSpacer.EnsureSpace(h.cache, r.ContentLength) {
			http.Error(w, "The disk is full. File could not be uploaded.",
				http.StatusInsufficientStorage)
			return
		}
		written, err := h.saveToDisk(r.Body, hash, verifyHash)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		h.cache.AddFile(hash, written)
		w.WriteHeader(http.StatusOK)
	case http.MethodHead:
		if !h.cache.ContainsFile(hash) {
			http.Error(w, err.Error(), http.StatusNotFound)
		}
		w.WriteHeader(http.StatusOK)
	default:
		msg := fmt.Sprintf("Method '%s' not supported.", html.EscapeString(m))
		http.Error(w, msg, http.StatusMethodNotAllowed)
	}
}

func (h *httpCache) startUpload(hash string) *sync.Mutex {
	h.ongoingUploadsMux.Lock()
	defer h.ongoingUploadsMux.Unlock()
	mux, ok := h.ongoingUploads[hash]
	if !ok {
		mux = &sync.Mutex{}
		h.ongoingUploads[hash] = mux
		return mux
	}
	return mux
}

func (h *httpCache) stopUpload(hash string) {
	h.ongoingUploadsMux.Lock()
	defer h.ongoingUploadsMux.Unlock()
	delete(h.ongoingUploads, hash)
}

func (h *httpCache) discardUpload(w http.ResponseWriter, r io.Reader) {
	io.Copy(ioutil.Discard, r)
	w.WriteHeader(http.StatusOK)
}

func parseURL(url string) ([]string, error) {
	m := blobNameSHA256.FindStringSubmatch(url)
	if m == nil {
		msg := fmt.Sprintf("Resource name must be a SHA256 hash in hex. "+
			"Got '%s'.", html.EscapeString(url))
		return nil, errors.New(msg)
	}
	return m[1:], nil
}

func (h *httpCache) saveToDisk(content io.Reader, hash string, verifyHash bool) (written int64, err error) {
	f, err := ioutil.TempFile(h.cache.Dir(), "upload")
	if err != nil {
		return 0, err
	}
	tmpName := f.Name()
	if verifyHash {
		hasher := sha256.New()
		written, err = io.Copy(io.MultiWriter(f, hasher), content)
		actualHash := hex.EncodeToString(hasher.Sum(nil))
		if hash != actualHash {
			os.Remove(tmpName)
			msg := fmt.Sprintf("Hashes don't match. Provided '%s', Actual '%s'.",
				hash, html.EscapeString(actualHash))
			return 0, errors.New(msg)
		}
	} else {
		written, err = io.Copy(f, content)
	}
	if err != nil {
		return 0, err
	}
	err = f.Sync()
	if err != nil {
		log.Fatal(err)
	}
	f.Close()
	err2 := os.Rename(tmpName, h.filePath(hash))
	if err2 != nil {
		return 0, err2
	}
	return written, nil
}

func (h httpCache) filePath(hash string) string {
	return fmt.Sprintf("%s%c%s", h.cache.Dir(), os.PathSeparator, hash)
}
<|endoftext|>