<commit_before>package main

import (
	"fmt"
	"os"
	"os/signal"
	"path"
	"strings"
	"syscall"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/dustin/go-humanize"
	"github.com/pkg/errors"
	"github.com/rcrowley/go-metrics"

	"github.intel.com/hpdd/logging/alert"
	"github.intel.com/hpdd/logging/audit"
	"github.intel.com/hpdd/logging/debug"
	"github.intel.com/hpdd/policy/pdm/dmplugin"
)

type (
	archiveConfig struct {
		Name   string `hcl:",key"`
		ID     int
		Region string
		Bucket string
		Prefix string
	}

	archiveSet []*archiveConfig

	s3Config struct {
		NumThreads     int        `hcl:"num_threads"`
		Region         string     `hcl:"region"`
		AWSAccessKeyID string     `hcl:"aws_access_key_id"`
		AWSSecretKey   string     `hcl:"aws_secret_key"`
		Archives       archiveSet `hcl:"archive"`
	}
)

// Should this be configurable?
const updateInterval = 10 * time.Second

var rate metrics.Meter

func (c *s3Config) String() string {
	return dmplugin.DisplayConfig(c)
}

func (a *archiveConfig) String() string {
	return fmt.Sprintf("%d:%s:%s/%s", a.ID, a.Region, a.Bucket, a.Prefix)
}

func (a *archiveConfig) checkValid() error {
	var errors []string

	if a.Bucket == "" {
		errors = append(errors, fmt.Sprintf("Archive %s: bucket not set", a.Name))
	}

	if a.ID < 1 {
		errors = append(errors, fmt.Sprintf("Archive %s: archive id not set", a.Name))

	}

	if len(errors) > 0 {
		return fmt.Errorf("Errors: %s", strings.Join(errors, ", "))
	}

	return nil
}

func (c *s3Config) Merge(other *s3Config) *s3Config {
	result := new(s3Config)

	result.NumThreads = c.NumThreads
	if other.NumThreads > 0 {
		result.NumThreads = other.NumThreads
	}

	result.Region = c.Region
	if other.Region != "" {
		result.Region = other.Region
	}

	result.Archives = c.Archives
	if len(other.Archives) > 0 {
		result.Archives = other.Archives
	}

	return result
}

func init() {
	rate = metrics.NewMeter()

	// if debug.Enabled() {
	go func() {
		var lastCount int64
		for {
			if lastCount != rate.Count() {
				audit.Logf("total %s (1 min/5 min/15 min/inst): %s/%s/%s/%s msg/sec\n",
					humanize.Comma(rate.Count()),
					humanize.Comma(int64(rate.Rate1())),
					humanize.Comma(int64(rate.Rate5())),
					humanize.Comma(int64(rate.Rate15())),
					humanize.Comma(int64(rate.RateMean())),
				)
				lastCount = rate.Count()
			}
			time.Sleep(10 * time.Second)
		}
	}()
	// }
}

func s3Svc(region string) *s3.S3 {
	// TODO: Allow more per-archive configuration options?
	return s3.New(session.New(aws.NewConfig().WithRegion(region)))
}

func getMergedConfig(plugin dmplugin.Plugin) (*s3Config, error) {
	baseCfg := &s3Config{
		Region: "us-east-1",
	}

	var cfg s3Config
	err := dmplugin.LoadConfig(plugin.ConfigFile(), &cfg)

	if err != nil {
		return nil, fmt.Errorf("Failed to load config: %s", err)
	}

	return baseCfg.Merge(&cfg), nil
}

func main() {
	plugin, err := dmplugin.New(path.Base(os.Args[0]))
	if err != nil {
		alert.Abort(errors.Wrap(err, "failed to initialize plugin"))
	}
	defer plugin.Close()

	cfg, err := getMergedConfig(plugin)
	if err != nil {
		alert.Abort(errors.Wrap(err, "Unable to determine plugin configuration"))
	}

	debug.Printf("S3Mover configuration:\n%v", cfg)

	if len(cfg.Archives) == 0 {
		alert.Abort(errors.New("Invalid configuration: No archives defined"))
	}

	for _, archive := range cfg.Archives {
		debug.Print(archive)
		if err := archive.checkValid(); err != nil {
			alert.Abort(errors.Wrap(err, "Invalid configuration"))
		}
	}

	// Set the configured AWS credentials in the environment for use
	// by the SDK.
	if cfg.AWSAccessKeyID != "" {
		os.Setenv("AWS_ACCESS_KEY_ID", cfg.AWSAccessKeyID)
	}
	if cfg.AWSSecretKey != "" {
		os.Setenv("AWS_SECRET_KEY", cfg.AWSSecretKey)
	}

	// TODO: Check to make sure that the SDK can find some credentials
	// before continuing; otherwise there will be long timeouts.

	// All base filesystem operations will be relative to current directory
	err = os.Chdir(plugin.Base())
	if err != nil {
		alert.Abort(errors.Wrap(err, "chdir failed"))
	}

	interruptHandler(func() {
		plugin.Stop()
	})

	for _, a := range cfg.Archives {
		// Allow each archive to set its own region, but default
		// to the region set for the plugin.
		region := cfg.Region
		if a.Region != "" {
			region = a.Region
		}
		s3Svc := s3Svc(region)
		plugin.AddMover(&dmplugin.Config{
			Mover:      S3Mover(s3Svc, uint32(a.ID), a.Bucket, a.Prefix),
			NumThreads: cfg.NumThreads,
			ArchiveID:  uint32(a.ID),
		})
	}

	plugin.Run()
}

func interruptHandler(once func()) {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, syscall.SIGQUIT, syscall.SIGTERM)

	go func() {
		stopping := false
		for sig := range c {
			debug.Printf("signal received: %s", sig)
			if !stopping {
				stopping = true
				once()
			}
		}
	}()
}
<commit_msg>Sanity-check S3 config before starting mover<commit_after>package main

import (
	"fmt"
	"os"
	"os/signal"
	"path"
	"strings"
	"syscall"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/defaults"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/dustin/go-humanize"
	"github.com/pkg/errors"
	"github.com/rcrowley/go-metrics"

	"github.intel.com/hpdd/logging/alert"
	"github.intel.com/hpdd/logging/audit"
	"github.intel.com/hpdd/logging/debug"
	"github.intel.com/hpdd/policy/pdm/dmplugin"
)

type (
	archiveConfig struct {
		Name   string `hcl:",key"`
		ID     int
		Region string
		Bucket string
		Prefix string
	}

	archiveSet []*archiveConfig

	s3Config struct {
		NumThreads     int        `hcl:"num_threads"`
		Region         string     `hcl:"region"`
		AWSAccessKeyID string     `hcl:"aws_access_key_id"`
		AWSSecretKey   string     `hcl:"aws_secret_key"`
		Archives       archiveSet `hcl:"archive"`
	}
)

// Should this be configurable?
const updateInterval = 10 * time.Second

var rate metrics.Meter

func (c *s3Config) String() string {
	return dmplugin.DisplayConfig(c)
}

func (a *archiveConfig) String() string {
	return fmt.Sprintf("%d:%s:%s/%s", a.ID, a.Region, a.Bucket, a.Prefix)
}

func (a *archiveConfig) checkValid() error {
	var errors []string

	if a.Bucket == "" {
		errors = append(errors, fmt.Sprintf("Archive %s: bucket not set", a.Name))
	}

	if a.ID < 1 {
		errors = append(errors, fmt.Sprintf("Archive %s: archive id not set", a.Name))

	}

	if len(errors) > 0 {
		return fmt.Errorf("Errors: %s", strings.Join(errors, ", "))
	}

	return nil
}

func (c *s3Config) Merge(other *s3Config) *s3Config {
	result := new(s3Config)

	result.NumThreads = c.NumThreads
	if other.NumThreads > 0 {
		result.NumThreads = other.NumThreads
	}

	result.Region = c.Region
	if other.Region != "" {
		result.Region = other.Region
	}

	result.Archives = c.Archives
	if len(other.Archives) > 0 {
		result.Archives = other.Archives
	}

	return result
}

func init() {
	rate = metrics.NewMeter()

	// if debug.Enabled() {
	go func() {
		var lastCount int64
		for {
			if lastCount != rate.Count() {
				audit.Logf("total %s (1 min/5 min/15 min/inst): %s/%s/%s/%s msg/sec\n",
					humanize.Comma(rate.Count()),
					humanize.Comma(int64(rate.Rate1())),
					humanize.Comma(int64(rate.Rate5())),
					humanize.Comma(int64(rate.Rate15())),
					humanize.Comma(int64(rate.RateMean())),
				)
				lastCount = rate.Count()
			}
			time.Sleep(10 * time.Second)
		}
	}()
	// }
}

func s3Svc(region string) *s3.S3 {
	// TODO: Allow more per-archive configuration options?
	return s3.New(session.New(aws.NewConfig().WithRegion(region)))
}

func getMergedConfig(plugin dmplugin.Plugin) (*s3Config, error) {
	baseCfg := &s3Config{
		Region: "us-east-1",
	}

	var cfg s3Config
	err := dmplugin.LoadConfig(plugin.ConfigFile(), &cfg)

	if err != nil {
		return nil, fmt.Errorf("Failed to load config: %s", err)
	}

	return baseCfg.Merge(&cfg), nil
}

func checkS3Configuration(cfg *s3Config) error {
	// Check to make sure that the SDK can find some credentials
	// before continuing; otherwise there will be long timeouts and
	// mysterious failures on HSM actions.
	// Hopefully this will work for non-AWS S3 implementations as well.
	s3Creds := defaults.CredChain(aws.NewConfig().WithRegion(cfg.Region), defaults.Handlers())
	if _, err := s3Creds.Get(); err != nil {
		return errors.Wrap(err, "No S3 credentials found; cannot initialize data mover")
	}

	svc := s3Svc(cfg.Region)
	if _, err := svc.ListBuckets(&s3.ListBucketsInput{}); err != nil {
		return errors.Wrap(err, "Unable to list S3 buckets")
	}

	return nil
}

func main() {
	plugin, err := dmplugin.New(path.Base(os.Args[0]))
	if err != nil {
		alert.Abort(errors.Wrap(err, "failed to initialize plugin"))
	}
	defer plugin.Close()

	cfg, err := getMergedConfig(plugin)
	if err != nil {
		alert.Abort(errors.Wrap(err, "Unable to determine plugin configuration"))
	}

	debug.Printf("S3Mover configuration:\n%v", cfg)

	if len(cfg.Archives) == 0 {
		alert.Abort(errors.New("Invalid configuration: No archives defined"))
	}

	for _, archive := range cfg.Archives {
		debug.Print(archive)
		if err := archive.checkValid(); err != nil {
			alert.Abort(errors.Wrap(err, "Invalid configuration"))
		}
	}

	// Set the configured AWS credentials in the environment for use
	// by the SDK. If there are no explicitly configured credentials,
	// then the SDK will look for them in other ways
	// (e.g. ~/.aws/credentials, ec2 role, etc.).
	if cfg.AWSAccessKeyID != "" {
		os.Setenv("AWS_ACCESS_KEY_ID", cfg.AWSAccessKeyID)
	}
	if cfg.AWSSecretKey != "" {
		os.Setenv("AWS_SECRET_KEY", cfg.AWSSecretKey)
	}

	if err := checkS3Configuration(cfg); err != nil {
		alert.Abort(errors.Wrap(err, "S3 config check failed"))
	}

	// All base filesystem operations will be relative to current directory
	err = os.Chdir(plugin.Base())
	if err != nil {
		alert.Abort(errors.Wrap(err, "chdir failed"))
	}

	interruptHandler(func() {
		plugin.Stop()
	})

	for _, a := range cfg.Archives {
		// Allow each archive to set its own region, but default
		// to the region set for the plugin.
		region := cfg.Region
		if a.Region != "" {
			region = a.Region
		}
		s3Svc := s3Svc(region)
		plugin.AddMover(&dmplugin.Config{
			Mover:      S3Mover(s3Svc, uint32(a.ID), a.Bucket, a.Prefix),
			NumThreads: cfg.NumThreads,
			ArchiveID:  uint32(a.ID),
		})
	}

	plugin.Run()
}

func interruptHandler(once func()) {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, syscall.SIGQUIT, syscall.SIGTERM)

	go func() {
		stopping := false
		for sig := range c {
			debug.Printf("signal received: %s", sig)
			if !stopping {
				stopping = true
				once()
			}
		}
	}()
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"math"
	"os"

	"github.com/grafana/metrictank/api"
	"github.com/grafana/metrictank/mdata/chunk"
	"github.com/grafana/metrictank/store/cassandra"
	"github.com/raintank/schema"
)

// printPoints prints points in the store corresponding to the given requirements
func printPoints(ctx context.Context, store *cassandra.CassandraStore, tables []cassandra.Table, metrics []Metric, fromUnix, toUnix, fix uint32) {
	for _, metric := range metrics {
		fmt.Println("## Metric", metric)
		for _, table := range tables {
			fmt.Println("### Table", table.Name)
			if fix != 0 {
				points := getSeries(ctx, store, table, metric.AMKey, fromUnix, toUnix, fix)
				printPointsNormal(points, fromUnix, toUnix)
			} else {
				igens, err := store.SearchTable(ctx, metric.AMKey, table, fromUnix, toUnix)
				if err != nil {
					panic(err)
				}
				printNormal(igens, fromUnix, toUnix)
			}
		}
	}
}

// printPointSummary prints a summarized view of the points in the store corresponding to the given requirements
func printPointSummary(ctx context.Context, store *cassandra.CassandraStore, tables []cassandra.Table, metrics []Metric, fromUnix, toUnix, fix uint32) {
	for _, metric := range metrics {
		fmt.Println("## Metric", metric)
		for _, table := range tables {
			fmt.Println("### Table", table.Name)
			if fix != 0 {
				points := getSeries(ctx, store, table, metric.AMKey, fromUnix, toUnix, fix)
				printPointsSummary(points, fromUnix, toUnix)
			} else {
				igens, err := store.SearchTable(ctx, metric.AMKey, table, fromUnix, toUnix)
				if err != nil {
					panic(err)
				}
				printSummary(igens, fromUnix, toUnix)
			}
		}
	}
}

func getSeries(ctx context.Context, store *cassandra.CassandraStore, table cassandra.Table, amkey schema.AMKey, fromUnix, toUnix, interval uint32) []schema.Point {
	var points []schema.Point
	itgens, err := store.SearchTable(ctx, amkey, table, fromUnix, toUnix)
	if err != nil {
		panic(err)
	}

	for i, itgen := range itgens {
		iter, err := itgen.Get()
		if err != nil {
			fmt.Fprintf(os.Stderr, "chunk %d itergen.Get: %s", i, err)
			continue
		}
		for iter.Next() {
			ts, val := iter.Values()
			if ts >= fromUnix && ts < toUnix {
				points = append(points, schema.Point{Val: val, Ts: ts})
			}
		}
	}
	return api.Fix(points, fromUnix, toUnix, interval)
}

func printNormal(igens []chunk.IterGen, from, to uint32) {
	fmt.Println("number of chunks:", len(igens))
	for i, ig := range igens {
		fmt.Printf("#### chunk %d (t0:%s, span:%d, format:%s)\n", i, printTime(ig.T0), ig.Span(), ig.Format())
		iter, err := ig.Get()
		if err != nil {
			fmt.Fprintf(os.Stderr, "chunk %d itergen.Get: %s", i, err)
			continue
		}
		for iter.Next() {
			ts, val := iter.Values()
			printRecord(ts, val, ts >= from && ts < to, math.IsNaN(val))
		}
	}
}

func printPointsNormal(points []schema.Point, from, to uint32) {
	fmt.Println("number of points:", len(points))
	for _, p := range points {
		printRecord(p.Ts, p.Val, p.Ts >= from && p.Ts < to, math.IsNaN(p.Val))
	}
}

func printRecord(ts uint32, val float64, in, nan bool) {
	prefix := "- "
	if in {
		prefix = "> "
	}
	if nan {
		fmt.Println(prefix, printTime(ts), "NAN")
	} else {
		fmt.Println(prefix, printTime(ts), val)
	}
}

func printSummary(igens []chunk.IterGen, from, to uint32) {

	var count int
	first := true
	var prevIn, prevNaN bool
	var ts uint32
	var val float64

	var followup = func(count int, in, nan bool) {
		fmt.Printf("... and %d more of in_range=%t nan=%t ...\n", count, in, nan)
	}

	for i, ig := range igens {
		iter, err := ig.Get()
		if err != nil {
			fmt.Fprintf(os.Stderr, "chunk %d itergen.Get: %s", i, err)
			continue
		}
		for iter.Next() {
			ts, val = iter.Values()

			nan := math.IsNaN(val)
			in := (ts >= from && ts < to)

			if first {
				printRecord(ts, val, in, nan)
			} else if nan == prevNaN && in == prevIn {
				count++
			} else {
				followup(count, prevIn, prevNaN)
				printRecord(ts, val, in, nan)
				count = 0
			}

			prevNaN = nan
			prevIn = in
			first = false
		}
	}
	if count > 0 {
		followup(count, prevIn, prevNaN)
		fmt.Println("last value was:")
		printRecord(ts, val, prevIn, prevNaN)
	}
}

func printPointsSummary(points []schema.Point, from, to uint32) {

	var count int
	first := true
	var prevIn, prevNaN bool
	var ts uint32
	var val float64

	var followup = func(count int, in, nan bool) {
		fmt.Printf("... and %d more of in_range=%t nan=%t ...\n", count, in, nan)
	}

	for _, p := range points {
		ts, val = p.Ts, p.Val

		nan := math.IsNaN(val)
		in := (ts >= from && ts < to)

		if first {
			printRecord(ts, val, in, nan)
		} else if nan == prevNaN && in == prevIn {
			count++
		} else {
			followup(count, prevIn, prevNaN)
			printRecord(ts, val, in, nan)
			count = 0
		}

		prevNaN = nan
		prevIn = in
		first = false
	}
	if count > 0 {
		followup(count, prevIn, prevNaN)
		fmt.Println("last value was:")
		printRecord(ts, val, prevIn, prevNaN)
	}
}
<commit_msg>print size also<commit_after>package main

import (
	"context"
	"fmt"
	"math"
	"os"

	"github.com/grafana/metrictank/api"
	"github.com/grafana/metrictank/mdata/chunk"
	"github.com/grafana/metrictank/store/cassandra"
	"github.com/raintank/schema"
)

// printPoints prints points in the store corresponding to the given requirements
func printPoints(ctx context.Context, store *cassandra.CassandraStore, tables []cassandra.Table, metrics []Metric, fromUnix, toUnix, fix uint32) {
	for _, metric := range metrics {
		fmt.Println("## Metric", metric)
		for _, table := range tables {
			fmt.Println("### Table", table.Name)
			if fix != 0 {
				points := getSeries(ctx, store, table, metric.AMKey, fromUnix, toUnix, fix)
				printPointsNormal(points, fromUnix, toUnix)
			} else {
				igens, err := store.SearchTable(ctx, metric.AMKey, table, fromUnix, toUnix)
				if err != nil {
					panic(err)
				}
				printNormal(igens, fromUnix, toUnix)
			}
		}
	}
}

// printPointSummary prints a summarized view of the points in the store corresponding to the given requirements
func printPointSummary(ctx context.Context, store *cassandra.CassandraStore, tables []cassandra.Table, metrics []Metric, fromUnix, toUnix, fix uint32) {
	for _, metric := range metrics {
		fmt.Println("## Metric", metric)
		for _, table := range tables {
			fmt.Println("### Table", table.Name)
			if fix != 0 {
				points := getSeries(ctx, store, table, metric.AMKey, fromUnix, toUnix, fix)
				printPointsSummary(points, fromUnix, toUnix)
			} else {
				igens, err := store.SearchTable(ctx, metric.AMKey, table, fromUnix, toUnix)
				if err != nil {
					panic(err)
				}
				printSummary(igens, fromUnix, toUnix)
			}
		}
	}
}

func getSeries(ctx context.Context, store *cassandra.CassandraStore, table cassandra.Table, amkey schema.AMKey, fromUnix, toUnix, interval uint32) []schema.Point {
	var points []schema.Point
	itgens, err := store.SearchTable(ctx, amkey, table, fromUnix, toUnix)
	if err != nil {
		panic(err)
	}

	for i, itgen := range itgens {
		iter, err := itgen.Get()
		if err != nil {
			fmt.Fprintf(os.Stderr, "chunk %d itergen.Get: %s", i, err)
			continue
		}
		for iter.Next() {
			ts, val := iter.Values()
			if ts >= fromUnix && ts < toUnix {
				points = append(points, schema.Point{Val: val, Ts: ts})
			}
		}
	}
	return api.Fix(points, fromUnix, toUnix, interval)
}

func printNormal(igens []chunk.IterGen, from, to uint32) {
	fmt.Println("number of chunks:", len(igens))
	for i, ig := range igens {
		fmt.Printf("#### chunk %d (t0:%s, span:%d, format:%s, size:%d)\n", i, printTime(ig.T0), ig.Span(), ig.Format(), ig.Size())
		iter, err := ig.Get()
		if err != nil {
			fmt.Fprintf(os.Stderr, "chunk %d itergen.Get: %s", i, err)
			continue
		}
		for iter.Next() {
			ts, val := iter.Values()
			printRecord(ts, val, ts >= from && ts < to, math.IsNaN(val))
		}
	}
}

func printPointsNormal(points []schema.Point, from, to uint32) {
	fmt.Println("number of points:", len(points))
	for _, p := range points {
		printRecord(p.Ts, p.Val, p.Ts >= from && p.Ts < to, math.IsNaN(p.Val))
	}
}

func printRecord(ts uint32, val float64, in, nan bool) {
	prefix := "- "
	if in {
		prefix = "> "
	}
	if nan {
		fmt.Println(prefix, printTime(ts), "NAN")
	} else {
		fmt.Println(prefix, printTime(ts), val)
	}
}

func printSummary(igens []chunk.IterGen, from, to uint32) {

	var count int
	first := true
	var prevIn, prevNaN bool
	var ts uint32
	var val float64

	var followup = func(count int, in, nan bool) {
		fmt.Printf("... and %d more of in_range=%t nan=%t ...\n", count, in, nan)
	}

	for i, ig := range igens {
		iter, err := ig.Get()
		if err != nil {
			fmt.Fprintf(os.Stderr, "chunk %d itergen.Get: %s", i, err)
			continue
		}
		for iter.Next() {
			ts, val = iter.Values()

			nan := math.IsNaN(val)
			in := (ts >= from && ts < to)

			if first {
				printRecord(ts, val, in, nan)
			} else if nan == prevNaN && in == prevIn {
				count++
			} else {
				followup(count, prevIn, prevNaN)
				printRecord(ts, val, in, nan)
				count = 0
			}

			prevNaN = nan
			prevIn = in
			first = false
		}
	}
	if count > 0 {
		followup(count, prevIn, prevNaN)
		fmt.Println("last value was:")
		printRecord(ts, val, prevIn, prevNaN)
	}
}

func printPointsSummary(points []schema.Point, from, to uint32) {

	var count int
	first := true
	var prevIn, prevNaN bool
	var ts uint32
	var val float64

	var followup = func(count int, in, nan bool) {
		fmt.Printf("... and %d more of in_range=%t nan=%t ...\n", count, in, nan)
	}

	for _, p := range points {
		ts, val = p.Ts, p.Val

		nan := math.IsNaN(val)
		in := (ts >= from && ts < to)

		if first {
			printRecord(ts, val, in, nan)
		} else if nan == prevNaN && in == prevIn {
			count++
		} else {
			followup(count, prevIn, prevNaN)
			printRecord(ts, val, in, nan)
			count = 0
		}

		prevNaN = nan
		prevIn = in
		first = false
	}
	if count > 0 {
		followup(count, prevIn, prevNaN)
		fmt.Println("last value was:")
		printRecord(ts, val, prevIn, prevNaN)
	}
}
<|endoftext|>


<commit_before>package main_test

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"path"
	"strconv"
	"strings"
	"testing"
	"time"

	"code.cloudfoundry.org/bbs"
	bbsconfig "code.cloudfoundry.org/bbs/cmd/bbs/config"
	bbstestrunner "code.cloudfoundry.org/bbs/cmd/bbs/testrunner"
	"code.cloudfoundry.org/bbs/encryption"
	"code.cloudfoundry.org/bbs/test_helpers"
	"code.cloudfoundry.org/bbs/test_helpers/sqlrunner"
	"code.cloudfoundry.org/consuladapter/consulrunner"
	"code.cloudfoundry.org/diego-logging-client/testhelpers"
	"code.cloudfoundry.org/durationjson"
	"code.cloudfoundry.org/go-loggregator/rpc/loggregator_v2"
	"code.cloudfoundry.org/inigo/helpers/portauthority"
	"code.cloudfoundry.org/lager/lagerflags"
	"code.cloudfoundry.org/locket"
	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/config"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
	"github.com/onsi/gomega/ghttp"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"
	"google.golang.org/grpc/grpclog"
)

var (
	cellID              string
	representativePath  string
	serverPort          uint16
	serverPortSecurable uint16
	consulRunner        *consulrunner.ClusterRunner

	bbsConfig        bbsconfig.BBSConfig
	bbsBinPath       string
	bbsURL           *url.URL
	bbsRunner        *ginkgomon.Runner
	bbsProcess       ifrit.Process
	bbsClient        bbs.InternalClient
	auctioneerServer *ghttp.Server
	locketBinPath    string
	node             int

	testIngressServer *testhelpers.TestIngressServer

	testMetricsChan   chan *loggregator_v2.Envelope
	signalMetricsChan chan struct{}

	sqlProcess    ifrit.Process
	sqlRunner     sqlrunner.SQLRunner
	portAllocator portauthority.PortAllocator

	fixturesPath = path.Join(os.Getenv("GOPATH"), "src/code.cloudfoundry.org/rep/cmd/rep/fixtures")
)

func TestRep(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "Rep Integration Suite")
}

var _ = SynchronizedBeforeSuite(func() []byte {
	bbsConfig, err := gexec.Build("code.cloudfoundry.org/bbs/cmd/bbs", "-race")
	Expect(err).NotTo(HaveOccurred())

	locketPath, err := gexec.Build("code.cloudfoundry.org/locket/cmd/locket", "-race")
	Expect(err).NotTo(HaveOccurred())

	representative, err := gexec.Build("code.cloudfoundry.org/rep/cmd/rep", "-race")
	Expect(err).NotTo(HaveOccurred())

	return []byte(strings.Join([]string{representative, locketPath, bbsConfig}, ","))
}, func(pathsByte []byte) {

	node = GinkgoParallelNode()
	startPort := 1050 * node
	portRange := 1000
	endPort := startPort + portRange

	var err error
	portAllocator, err = portauthority.New(startPort, endPort)
	Expect(err).NotTo(HaveOccurred())

	grpclog.SetLogger(log.New(ioutil.Discard, "", 0))

	// tests here are fairly Eventually driven which tends to flake out under
	// load (for insignificant reasons); bump the default a bit higher than the
	// default (1 second)
	SetDefaultEventuallyTimeout(5 * time.Second)

	strPath := string(pathsByte)
	representativePath = strings.Split(strPath, ",")[0]
	locketBinPath = strings.Split(strPath, ",")[1]
	bbsBinPath = strings.Split(strPath, ",")[2]

	cellID = "the_rep_id-" + strconv.Itoa(GinkgoParallelNode())

	serverPort, err = portAllocator.ClaimPorts(1)
	Expect(err).NotTo(HaveOccurred())
	serverPortSecurable, err = portAllocator.ClaimPorts(1)
	Expect(err).NotTo(HaveOccurred())

	dbName := fmt.Sprintf("diego_%d", GinkgoParallelNode())

	sqlRunner = test_helpers.NewSQLRunner(dbName)
	sqlProcess = ginkgomon.Invoke(sqlRunner)

	consulRunner = consulrunner.NewClusterRunner(
		consulrunner.ClusterRunnerConfig{
			StartingPort: 9001 + config.GinkgoConfig.ParallelNode*consulrunner.PortOffsetLength,
			NumNodes:     1,
			Scheme:       "http",
		},
	)

	consulRunner.Start()

	bbsPort, err := portAllocator.ClaimPorts(2)
	Expect(err).NotTo(HaveOccurred())
	healthPort := bbsPort + 1
	bbsAddress := fmt.Sprintf("127.0.0.1:%d", bbsPort)
	healthAddress := fmt.Sprintf("127.0.0.1:%d", healthPort)

	bbsURL = &url.URL{
		Scheme: "https",
		Host:   bbsAddress,
	}

	fixturesPath := path.Join(os.Getenv("GOPATH"), "src/code.cloudfoundry.org/rep/cmd/rep/fixtures")

	auctioneerServer = ghttp.NewServer()
	auctioneerServer.UnhandledRequestStatusCode = http.StatusAccepted
	auctioneerServer.AllowUnhandledRequests = true

	bbsConfig = bbsconfig.BBSConfig{
		ListenAddress:                  bbsAddress,
		AdvertiseURL:                   bbsURL.String(),
		AuctioneerAddress:              auctioneerServer.URL(),
		DatabaseDriver:                 sqlRunner.DriverName(),
		DatabaseConnectionString:       sqlRunner.ConnectionString(),
		DetectConsulCellRegistrations:  true,
		ConsulCluster:                  consulRunner.ConsulCluster(),
		HealthAddress:                  healthAddress,
		LocksLocketEnabled:             false,
		CellRegistrationsLocketEnabled: false,
		EncryptionConfig: encryption.EncryptionConfig{
			EncryptionKeys: map[string]string{"label": "key"},
			ActiveKeyLabel: "label",
		},

		CaFile:   path.Join(fixturesPath, "green-certs", "server-ca.crt"),
		CertFile: path.Join(fixturesPath, "green-certs", "server.crt"),
		KeyFile:  path.Join(fixturesPath, "green-certs", "server.key"),

		SessionName:                     "bbs",
		CommunicationTimeout:            durationjson.Duration(10 * time.Second),
		RequireSSL:                      true,
		DesiredLRPCreationTimeout:       durationjson.Duration(1 * time.Minute),
		ExpireCompletedTaskDuration:     durationjson.Duration(2 * time.Minute),
		ExpirePendingTaskDuration:       durationjson.Duration(30 * time.Minute),
		EnableConsulServiceRegistration: false,
		ConvergeRepeatInterval:          durationjson.Duration(30 * time.Second),
		KickTaskDuration:                durationjson.Duration(30 * time.Second),
		LockTTL:                         durationjson.Duration(locket.DefaultSessionTTL),
		LockRetryInterval:               durationjson.Duration(locket.RetryInterval),
		ReportInterval:                  durationjson.Duration(1 * time.Minute),
		ConvergenceWorkers:              20,
		UpdateWorkers:                   1000,
		TaskCallbackWorkers:             1000,
		MaxOpenDatabaseConnections:      200,
		MaxIdleDatabaseConnections:      200,
		AuctioneerRequireTLS:            false,
		RepClientSessionCacheSize:       0,
		RepRequireTLS:                   false,
		LagerConfig: lagerflags.LagerConfig{
			LogLevel: "info",
		},
	}
})

var _ = BeforeEach(func() {
	consulRunner.WaitUntilReady()
	consulRunner.Reset()

	bbsRunner = bbstestrunner.New(bbsBinPath, bbsConfig)
	bbsProcess = ginkgomon.Invoke(bbsRunner)

	metronCAFile := path.Join(fixturesPath, "metron", "CA.crt")
	metronServerCertFile := path.Join(fixturesPath, "metron", "metron.crt")
	metronServerKeyFile := path.Join(fixturesPath, "metron", "metron.key")

	var err error
	testIngressServer, err = testhelpers.NewTestIngressServer(metronServerCertFile, metronServerKeyFile, metronCAFile)
	Expect(err).NotTo(HaveOccurred())
	Expect(testIngressServer.Start()).To(Succeed())

	receiversChan := testIngressServer.Receivers()

	testMetricsChan, signalMetricsChan = testhelpers.TestMetricChan(receiversChan)
})

var _ = AfterEach(func() {
	sqlRunner.Reset()

	ginkgomon.Kill(bbsProcess)
})

var _ = SynchronizedAfterSuite(func() {
	ginkgomon.Kill(sqlProcess)
	if consulRunner != nil {
		consulRunner.Stop()
	}
	if runner != nil {
		runner.KillWithFire()
	}
	if auctioneerServer != nil {
		auctioneerServer.Close()
	}
}, func() {
	gexec.CleanupBuildArtifacts()
})
<commit_msg>Allow sqlProcess to take longer to exit<commit_after>package main_test

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"path"
	"strconv"
	"strings"
	"testing"
	"time"

	"code.cloudfoundry.org/bbs"
	bbsconfig "code.cloudfoundry.org/bbs/cmd/bbs/config"
	bbstestrunner "code.cloudfoundry.org/bbs/cmd/bbs/testrunner"
	"code.cloudfoundry.org/bbs/encryption"
	"code.cloudfoundry.org/bbs/test_helpers"
	"code.cloudfoundry.org/bbs/test_helpers/sqlrunner"
	"code.cloudfoundry.org/consuladapter/consulrunner"
	"code.cloudfoundry.org/diego-logging-client/testhelpers"
	"code.cloudfoundry.org/durationjson"
	"code.cloudfoundry.org/go-loggregator/rpc/loggregator_v2"
	"code.cloudfoundry.org/inigo/helpers/portauthority"
	"code.cloudfoundry.org/lager/lagerflags"
	"code.cloudfoundry.org/locket"
	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/config"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
	"github.com/onsi/gomega/ghttp"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"
	"google.golang.org/grpc/grpclog"
)

var (
	cellID              string
	representativePath  string
	serverPort          uint16
	serverPortSecurable uint16
	consulRunner        *consulrunner.ClusterRunner

	bbsConfig        bbsconfig.BBSConfig
	bbsBinPath       string
	bbsURL           *url.URL
	bbsRunner        *ginkgomon.Runner
	bbsProcess       ifrit.Process
	bbsClient        bbs.InternalClient
	auctioneerServer *ghttp.Server
	locketBinPath    string
	node             int

	testIngressServer *testhelpers.TestIngressServer

	testMetricsChan   chan *loggregator_v2.Envelope
	signalMetricsChan chan struct{}

	sqlProcess    ifrit.Process
	sqlRunner     sqlrunner.SQLRunner
	portAllocator portauthority.PortAllocator

	fixturesPath = path.Join(os.Getenv("GOPATH"), "src/code.cloudfoundry.org/rep/cmd/rep/fixtures")
)

func TestRep(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "Rep Integration Suite")
}

var _ = SynchronizedBeforeSuite(func() []byte {
	bbsConfig, err := gexec.Build("code.cloudfoundry.org/bbs/cmd/bbs", "-race")
	Expect(err).NotTo(HaveOccurred())

	locketPath, err := gexec.Build("code.cloudfoundry.org/locket/cmd/locket", "-race")
	Expect(err).NotTo(HaveOccurred())

	representative, err := gexec.Build("code.cloudfoundry.org/rep/cmd/rep", "-race")
	Expect(err).NotTo(HaveOccurred())

	return []byte(strings.Join([]string{representative, locketPath, bbsConfig}, ","))
}, func(pathsByte []byte) {

	node = GinkgoParallelNode()
	startPort := 1050 * node
	portRange := 1000
	endPort := startPort + portRange

	var err error
	portAllocator, err = portauthority.New(startPort, endPort)
	Expect(err).NotTo(HaveOccurred())

	grpclog.SetLogger(log.New(ioutil.Discard, "", 0))

	// tests here are fairly Eventually driven which tends to flake out under
	// load (for insignificant reasons); bump the default a bit higher than the
	// default (1 second)
	SetDefaultEventuallyTimeout(5 * time.Second)

	strPath := string(pathsByte)
	representativePath = strings.Split(strPath, ",")[0]
	locketBinPath = strings.Split(strPath, ",")[1]
	bbsBinPath = strings.Split(strPath, ",")[2]

	cellID = "the_rep_id-" + strconv.Itoa(GinkgoParallelNode())

	serverPort, err = portAllocator.ClaimPorts(1)
	Expect(err).NotTo(HaveOccurred())
	serverPortSecurable, err = portAllocator.ClaimPorts(1)
	Expect(err).NotTo(HaveOccurred())

	dbName := fmt.Sprintf("diego_%d", GinkgoParallelNode())

	sqlRunner = test_helpers.NewSQLRunner(dbName)
	sqlProcess = ginkgomon.Invoke(sqlRunner)

	consulRunner = consulrunner.NewClusterRunner(
		consulrunner.ClusterRunnerConfig{
			StartingPort: 9001 + config.GinkgoConfig.ParallelNode*consulrunner.PortOffsetLength,
			NumNodes:     1,
			Scheme:       "http",
		},
	)

	consulRunner.Start()

	bbsPort, err := portAllocator.ClaimPorts(2)
	Expect(err).NotTo(HaveOccurred())
	healthPort := bbsPort + 1
	bbsAddress := fmt.Sprintf("127.0.0.1:%d", bbsPort)
	healthAddress := fmt.Sprintf("127.0.0.1:%d", healthPort)

	bbsURL = &url.URL{
		Scheme: "https",
		Host:   bbsAddress,
	}

	fixturesPath := path.Join(os.Getenv("GOPATH"), "src/code.cloudfoundry.org/rep/cmd/rep/fixtures")

	auctioneerServer = ghttp.NewServer()
	auctioneerServer.UnhandledRequestStatusCode = http.StatusAccepted
	auctioneerServer.AllowUnhandledRequests = true

	bbsConfig = bbsconfig.BBSConfig{
		ListenAddress:                  bbsAddress,
		AdvertiseURL:                   bbsURL.String(),
		AuctioneerAddress:              auctioneerServer.URL(),
		DatabaseDriver:                 sqlRunner.DriverName(),
		DatabaseConnectionString:       sqlRunner.ConnectionString(),
		DetectConsulCellRegistrations:  true,
		ConsulCluster:                  consulRunner.ConsulCluster(),
		HealthAddress:                  healthAddress,
		LocksLocketEnabled:             false,
		CellRegistrationsLocketEnabled: false,
		EncryptionConfig: encryption.EncryptionConfig{
			EncryptionKeys: map[string]string{"label": "key"},
			ActiveKeyLabel: "label",
		},

		CaFile:   path.Join(fixturesPath, "green-certs", "server-ca.crt"),
		CertFile: path.Join(fixturesPath, "green-certs", "server.crt"),
		KeyFile:  path.Join(fixturesPath, "green-certs", "server.key"),

		SessionName:                     "bbs",
		CommunicationTimeout:            durationjson.Duration(10 * time.Second),
		RequireSSL:                      true,
		DesiredLRPCreationTimeout:       durationjson.Duration(1 * time.Minute),
		ExpireCompletedTaskDuration:     durationjson.Duration(2 * time.Minute),
		ExpirePendingTaskDuration:       durationjson.Duration(30 * time.Minute),
		EnableConsulServiceRegistration: false,
		ConvergeRepeatInterval:          durationjson.Duration(30 * time.Second),
		KickTaskDuration:                durationjson.Duration(30 * time.Second),
		LockTTL:                         durationjson.Duration(locket.DefaultSessionTTL),
		LockRetryInterval:               durationjson.Duration(locket.RetryInterval),
		ReportInterval:                  durationjson.Duration(1 * time.Minute),
		ConvergenceWorkers:              20,
		UpdateWorkers:                   1000,
		TaskCallbackWorkers:             1000,
		MaxOpenDatabaseConnections:      200,
		MaxIdleDatabaseConnections:      200,
		AuctioneerRequireTLS:            false,
		RepClientSessionCacheSize:       0,
		RepRequireTLS:                   false,
		LagerConfig: lagerflags.LagerConfig{
			LogLevel: "info",
		},
	}
})

var _ = BeforeEach(func() {
	consulRunner.WaitUntilReady()
	consulRunner.Reset()

	bbsRunner = bbstestrunner.New(bbsBinPath, bbsConfig)
	bbsProcess = ginkgomon.Invoke(bbsRunner)

	metronCAFile := path.Join(fixturesPath, "metron", "CA.crt")
	metronServerCertFile := path.Join(fixturesPath, "metron", "metron.crt")
	metronServerKeyFile := path.Join(fixturesPath, "metron", "metron.key")

	var err error
	testIngressServer, err = testhelpers.NewTestIngressServer(metronServerCertFile, metronServerKeyFile, metronCAFile)
	Expect(err).NotTo(HaveOccurred())
	Expect(testIngressServer.Start()).To(Succeed())

	receiversChan := testIngressServer.Receivers()

	testMetricsChan, signalMetricsChan = testhelpers.TestMetricChan(receiversChan)
})

var _ = AfterEach(func() {
	sqlRunner.Reset()

	ginkgomon.Kill(bbsProcess)
})

var _ = SynchronizedAfterSuite(func() {
	ginkgomon.Kill(sqlProcess, "10s")
	if consulRunner != nil {
		consulRunner.Stop()
	}
	if runner != nil {
		runner.KillWithFire()
	}
	if auctioneerServer != nil {
		auctioneerServer.Close()
	}
}, func() {
	gexec.CleanupBuildArtifacts()
})
<|endoftext|>


<commit_before>package ctrl

import (
	. "github.com/mickael-kerjean/filestash/server/common"
	"fmt"
	"io"
	"text/template"
	"net/http"
	"os"
	"path/filepath"
	"strings"
)

func StaticHandler(_path string) func(App, http.ResponseWriter, *http.Request) {
	return func(ctx App, res http.ResponseWriter, req *http.Request) {
		var base string = GetAbsolutePath(_path)
		var srcPath string
		if srcPath = JoinPath(base, req.URL.Path); srcPath == base {
			http.NotFound(res, req)
			return
		}
		ServeFile(res, req, srcPath)
	}
}

func IndexHandler(_path string) func(App, http.ResponseWriter, *http.Request) {
	return func(ctx App, res http.ResponseWriter, req *http.Request) {
		if req.URL.String() != URL_SETUP && Config.Get("auth.admin").String() == "" {
			http.Redirect(res, req, URL_SETUP, http.StatusTemporaryRedirect)
			return
		}
		if ua := req.Header.Get("User-Agent"); strings.Contains(ua, "MSIE ") {
			res.WriteHeader(http.StatusBadRequest)
			res.Write([]byte(`<!DOCTYPE html>

    <meta charset="utf-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no" name="viewport">

Internet explorer is not yet supported

      To provide the best possible experience for everyone else, we don't support IE at this time.

      Use either Chromium, Firefox or Chrome

`))
			return
		}
		srcPath := GetAbsolutePath(_path)
		ServeFile(res, req, srcPath)
	}
}

func AboutHandler(ctx App, res http.ResponseWriter, req *http.Request) {
	page := `<!DOCTYPE html>

    <meta charset="utf-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no" name="viewport">

 {{index .App 0}}
({{index .App 1}} - {{index .App 2}})

`
	t, _ := template.New("about").Parse(page)
	t.Execute(res, struct {
		App     []string
	}{ []string{
		"Filestash " + APP_VERSION + "." + BUILD_NUMBER,
		hashFile(filepath.Join(GetCurrentDir(), "/filestash"), 6),
		hashFile(filepath.Join(GetCurrentDir(), CONFIG_PATH, "config.json"), 6),
	}})
}

func hashFile (path string, n int) string {
	f, err := os.OpenFile(path, os.O_RDONLY, os.ModePerm)
	if err != nil {
		return ""
	}
	defer f.Close()

	stat, err := f.Stat()
	if err != nil {
		return ""
	}
	return QuickHash(fmt.Sprintf("%s %d %d %s", path, stat.Size(), stat.Mode(), stat.ModTime()), n)
}

func ServeFile(res http.ResponseWriter, req *http.Request, filePath string) {
	zFilePath := filePath + ".gz"
	etagNormal := hashFile(filePath, 10)
	etagGzip := hashFile(zFilePath, 10)

	if req.Header.Get("If-None-Match") != "" {
		browserTag := req.Header.Get("If-None-Match")
		if browserTag == etagNormal {
			res.WriteHeader(http.StatusNotModified)
			return
		} else if browserTag == etagGzip {
			res.WriteHeader(http.StatusNotModified)
			return
		}
	}
	head := res.Header()
	if strings.Contains(req.Header.Get("Accept-Encoding"), "gzip") {
		if file, err := os.OpenFile(zFilePath, os.O_RDONLY, os.ModePerm); err == nil {
			head.Set("Content-Encoding", "gzip")
			head.Set("Etag", etagGzip)
			io.Copy(res, file)
			file.Close()
			return
		}
	}

	file, err := os.OpenFile(filePath, os.O_RDONLY, os.ModePerm)
	if err != nil {
		http.NotFound(res, req)
		return
	}
	head.Set("Etag", etagNormal)
	io.Copy(res, file)
	file.Close()
}
<commit_msg>feature (pages): create some error pages<commit_after>package ctrl

import (
	. "github.com/mickael-kerjean/filestash/server/common"
	"fmt"
	"io"
	"text/template"
	"net/http"
	URL "net/url"
	"os"
	"path/filepath"
	"strings"
)

func StaticHandler(_path string) func(App, http.ResponseWriter, *http.Request) {
	return func(ctx App, res http.ResponseWriter, req *http.Request) {
		var base string = GetAbsolutePath(_path)
		var srcPath string
		if srcPath = JoinPath(base, req.URL.Path); srcPath == base {
			http.NotFound(res, req)
			return
		}
		ServeFile(res, req, srcPath)
	}
}

func IndexHandler(_path string) func(App, http.ResponseWriter, *http.Request) {
	return func(ctx App, res http.ResponseWriter, req *http.Request) {
		urlObj, err := URL.Parse(req.URL.String())
		if err != nil {
			res.WriteHeader(http.StatusInternalServerError)
			res.Write([]byte(dumbPage("
404 - Not Found
")))
			return
		}
		url := urlObj.Path

		if url != URL_SETUP && Config.Get("auth.admin").String() == "" {
			http.Redirect(res, req, URL_SETUP, http.StatusTemporaryRedirect)
			return
		} else if url != "/" && strings.HasPrefix(url, "/s/") == false &&
			strings.HasPrefix(url, "/view/") == false && strings.HasPrefix(url, "/files/") == false &&
			url != "/login" && url != "/logout" && strings.HasPrefix(url, "/admin") == false {
			res.WriteHeader(http.StatusNotFound)
			res.Write([]byte(dumbPage("
404 - Not Found
")))
			return
		} else if ua := req.Header.Get("User-Agent"); strings.Contains(ua, "MSIE ") {
			res.WriteHeader(http.StatusBadRequest)
			res.Write([]byte(
				dumbPage(`

Internet explorer is not yet supported

                    To provide the best possible experience for everyone else, we don't support IE at this time.

                    Use either Chromium, Firefox or Chrome

                `)))
			return
		}
		srcPath := GetAbsolutePath(_path)
		ServeFile(res, req, srcPath)
	}
}

func AboutHandler(ctx App, res http.ResponseWriter, req *http.Request) {
	t, _ := template.New("about").Parse(dumbPage(`

 {{index .App 0}}

        ({{index .App 1}} - {{index .App 2}})

	`))
	t.Execute(res, struct {
		App     []string
	}{ []string{
		"Filestash " + APP_VERSION + "." + BUILD_NUMBER,
		hashFile(filepath.Join(GetCurrentDir(), "/filestash"), 6),
		hashFile(filepath.Join(GetCurrentDir(), CONFIG_PATH, "config.json"), 6),
	}})
}

func ServeFile(res http.ResponseWriter, req *http.Request, filePath string) {
	zFilePath := filePath + ".gz"
	etagNormal := hashFile(filePath, 10)
	etagGzip := hashFile(zFilePath, 10)

	if req.Header.Get("If-None-Match") != "" {
		browserTag := req.Header.Get("If-None-Match")
		if browserTag == etagNormal {
			res.WriteHeader(http.StatusNotModified)
			return
		} else if browserTag == etagGzip {
			res.WriteHeader(http.StatusNotModified)
			return
		}
	}
	head := res.Header()
	if strings.Contains(req.Header.Get("Accept-Encoding"), "gzip") {
		if file, err := os.OpenFile(zFilePath, os.O_RDONLY, os.ModePerm); err == nil {
			head.Set("Content-Encoding", "gzip")
			head.Set("Etag", etagGzip)
			io.Copy(res, file)
			file.Close()
			return
		}
	}

	file, err := os.OpenFile(filePath, os.O_RDONLY, os.ModePerm)
	if err != nil {
		http.NotFound(res, req)
		return
	}
	head.Set("Etag", etagNormal)
	io.Copy(res, file)
	file.Close()
}

func hashFile (path string, n int) string {
	f, err := os.OpenFile(path, os.O_RDONLY, os.ModePerm)
	if err != nil {
		return ""
	}
	defer f.Close()

	stat, err := f.Stat()
	if err != nil {
		return ""
	}
	return QuickHash(fmt.Sprintf("%s %d %d %s", path, stat.Size(), stat.Mode(), stat.ModTime()), n)
}

func dumbPage (stuff string) string {
	return `<!DOCTYPE html>

    <meta charset="utf-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no" name="viewport">

    ` + stuff + `

`
}
<|endoftext|>


<commit_before>package convox

import (
	"fmt"
	"log"
	"strings"
	"time"

	"github.com/convox/rack/client"
	"github.com/hashicorp/errwrap"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceConvoxApp() *schema.Resource {
	return &schema.Resource{
		Schema: map[string]*schema.Schema{
			"rack": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"status": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
			"environment": &schema.Schema{
				Type:     schema.TypeMap,
				Optional: true,
			},
			"params": &schema.Schema{
				Type:     schema.TypeMap,
				Optional: true,
				Computed: true,
			},
			"formation": &schema.Schema{
				Type:     schema.TypeMap,
				Computed: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"name": &schema.Schema{
							Type:     schema.TypeString,
							Computed: true,
						},
						"balancer": &schema.Schema{
							Type:     schema.TypeString,
							Computed: true,
						},
						"cpu": &schema.Schema{
							Type:     schema.TypeInt,
							Computed: true,
						},
						"count": &schema.Schema{
							Type:     schema.TypeInt,
							Computed: true,
						},
						"memory": &schema.Schema{
							Type:     schema.TypeInt,
							Computed: true,
						},
						"ports": &schema.Schema{
							Type:     schema.TypeList,
							Computed: true,
							Elem:     &schema.Schema{Type: schema.TypeInt},
						},
					},
				},
			},
		},
		Create: resourceConvoxAppCreate,
		Read:   resourceConvoxAppRead,
		Update: resourceConvoxAppUpdate,
		Delete: resourceConvoxAppDelete,
	}
}

func resourceConvoxAppCreate(d *schema.ResourceData, meta interface{}) error {
	c := RackClient(d, meta)
	if c == nil {
		return fmt.Errorf("Error rack client is nil: %#v", meta)
	}

	name := d.Get("name").(string)
	app, err := c.CreateApp(name)
	if err != nil {
		return fmt.Errorf(
			"Error creating app (%s): %s", name, err)
	}

	d.SetId(app.Name)
	stateConf := &resource.StateChangeConf{
		Pending: []string{"creating"},
		Target:  []string{"running"},
		Refresh: appRefreshFunc(c, app.Name),
		Timeout: 10 * time.Minute,
		Delay:   25 * time.Second,
	}

	if _, err = stateConf.WaitForState(); err != nil {
		return fmt.Errorf(
			"Error waiting for app (%s) to be created: %s", app.Name, err)
	}
	return resourceConvoxAppUpdate(d, meta)
}

func resourceConvoxAppRead(d *schema.ResourceData, meta interface{}) error {
	c := RackClient(d, meta)
	app, err := c.GetApp(d.Get("name").(string))
	if err != nil {
		return err
	}
	d.SetId(app.Name)
	d.Set("release", app.Release)
	d.Set("status", app.Status)

	params, err := c.ListParameters(app.Name)
	if err != nil {
		return err
	}
	d.Set("params", params)

	env, err := c.GetEnvironment(app.Name)
	if err != nil {
		return err
	}
	d.Set("environment", env)

	formation, err := c.ListFormation(app.Name)
	if err != nil {
		return errwrap.Wrapf("Error while reading formation from Convox API: {{err}}", err)
	}
	return readFormation(d, formation)
}

func resourceConvoxAppUpdate(d *schema.ResourceData, meta interface{}) error {
	d.Partial(true)
	c := RackClient(d, meta)
	if err := setParams(c, d); err != nil {
		return err
	}
	d.SetPartial("params")

	if err := setEnv(c, d); err != nil {
		return err
	}
	d.SetPartial("environment")

	d.Partial(false)
	return resourceConvoxAppRead(d, meta)
}

func resourceConvoxAppDelete(d *schema.ResourceData, meta interface{}) error {
	c := RackClient(d, meta)
	_, err := c.DeleteApp(d.Id())
	return err
}

func readFormation(d *schema.ResourceData, v client.Formation) error {
	//formation := map[string]map[string]interface{}{}
	//// endpoints := []map[string]interface{}{}
	//for _, f := range v {
	//entry := map[string]interface{}{
	//"name":     f.Name,
	//"balancer": f.Balancer,
	//"cpu":      f.CPU,
	//"count":    f.Count,
	//"memory":   f.Memory,
	//"ports":    f.Ports,
	//}
	//formation[f.Name] = entry
	//// for _, port := range f.Ports {
	//// 	endpoints = append(endpoints, fmt.Sprintf("%s:%d (%s)", f.Balancer, port, f.Name))
	//// }
	//}

	//if err := d.Set("formation", formation); err != nil {
	//return errwrap.Wrapf("Unable to store formation: {{err}}", err)
	//}

	return nil
}

func setParams(c *client.Client, d *schema.ResourceData) error {
	if !d.HasChange("params") {
		return nil
	}

	raw := d.Get("params").(map[string]interface{})
	params := make(client.Parameters)
	for key := range raw {
		params[key] = raw[key].(string)
	}

	log.Printf("[DEBUG] Setting params: (%#v) for %s", params, d.Id())
	if err := c.SetParameters(d.Id(), params); err != nil {
		return fmt.Errorf("Error setting params (%#v) for %s: %s", params, d.Id(), err)
	}

	return nil
}

func setEnv(c *client.Client, d *schema.ResourceData) error {
	if !d.HasChange("environment") {
		return nil
	}

	env := d.Get("environment").(map[string]interface{})
	log.Printf("[DEBUG] Setting environment to (%#v) for %s", env, d.Id())
	data := ""
	for key, value := range env {
		data += fmt.Sprintf("%s=%s\n", key, value)
	}
	_, rel, err := c.SetEnvironment(d.Id(), strings.NewReader(data))
	if err != nil {
		return fmt.Errorf("Error setting vars (%#v) for %s: %s", env, d.Id(), err)
	}
	log.Printf("[INFO] Release (%s) created on: %s", rel, d.Id())

	return nil
}

func appRefreshFunc(c *client.Client, app string) resource.StateRefreshFunc {
	return func() (interface{}, string, error) {
		app, err := c.GetApp(app)
		if err != nil {
			return app, "", err
		}
		return app, app.Status, err
	}
}
<commit_msg>put the formation back<commit_after>package convox

import (
	"fmt"
	"log"
	"strings"
	"time"

	"github.com/convox/rack/client"
	"github.com/hashicorp/errwrap"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceConvoxApp() *schema.Resource {
	return &schema.Resource{
		Schema: map[string]*schema.Schema{
			"rack": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"status": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
			"environment": &schema.Schema{
				Type:     schema.TypeMap,
				Optional: true,
			},
			"params": &schema.Schema{
				Type:     schema.TypeMap,
				Optional: true,
				Computed: true,
			},
			"formation": &schema.Schema{
				Type:     schema.TypeMap,
				Computed: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"name": &schema.Schema{
							Type:     schema.TypeString,
							Computed: true,
						},
						"balancer": &schema.Schema{
							Type:     schema.TypeString,
							Computed: true,
						},
						"cpu": &schema.Schema{
							Type:     schema.TypeInt,
							Computed: true,
						},
						"count": &schema.Schema{
							Type:     schema.TypeInt,
							Computed: true,
						},
						"memory": &schema.Schema{
							Type:     schema.TypeInt,
							Computed: true,
						},
						"ports": &schema.Schema{
							Type:     schema.TypeList,
							Computed: true,
							Elem:     &schema.Schema{Type: schema.TypeInt},
						},
					},
				},
			},
		},
		Create: resourceConvoxAppCreate,
		Read:   resourceConvoxAppRead,
		Update: resourceConvoxAppUpdate,
		Delete: resourceConvoxAppDelete,
	}
}

func resourceConvoxAppCreate(d *schema.ResourceData, meta interface{}) error {
	c := RackClient(d, meta)
	if c == nil {
		return fmt.Errorf("Error rack client is nil: %#v", meta)
	}

	name := d.Get("name").(string)
	app, err := c.CreateApp(name)
	if err != nil {
		return fmt.Errorf(
			"Error creating app (%s): %s", name, err)
	}

	d.SetId(app.Name)
	stateConf := &resource.StateChangeConf{
		Pending: []string{"creating"},
		Target:  []string{"running"},
		Refresh: appRefreshFunc(c, app.Name),
		Timeout: 10 * time.Minute,
		Delay:   25 * time.Second,
	}

	if _, err = stateConf.WaitForState(); err != nil {
		return fmt.Errorf(
			"Error waiting for app (%s) to be created: %s", app.Name, err)
	}
	return resourceConvoxAppUpdate(d, meta)
}

func resourceConvoxAppRead(d *schema.ResourceData, meta interface{}) error {
	c := RackClient(d, meta)
	app, err := c.GetApp(d.Get("name").(string))
	if err != nil {
		return err
	}
	d.SetId(app.Name)
	d.Set("release", app.Release)
	d.Set("status", app.Status)

	params, err := c.ListParameters(app.Name)
	if err != nil {
		return err
	}
	d.Set("params", params)

	env, err := c.GetEnvironment(app.Name)
	if err != nil {
		return err
	}
	d.Set("environment", env)

	formation, err := c.ListFormation(app.Name)
	if err != nil {
		return errwrap.Wrapf("Error while reading formation from Convox API: {{err}}", err)
	}
	return readFormation(d, formation)
}

func resourceConvoxAppUpdate(d *schema.ResourceData, meta interface{}) error {
	d.Partial(true)
	c := RackClient(d, meta)
	if err := setParams(c, d); err != nil {
		return err
	}
	d.SetPartial("params")

	if err := setEnv(c, d); err != nil {
		return err
	}
	d.SetPartial("environment")

	d.Partial(false)
	return resourceConvoxAppRead(d, meta)
}

func resourceConvoxAppDelete(d *schema.ResourceData, meta interface{}) error {
	c := RackClient(d, meta)
	_, err := c.DeleteApp(d.Id())
	return err
}

func readFormation(d *schema.ResourceData, v client.Formation) error {
	formation := map[string]map[string]interface{}{}
	// endpoints := []map[string]interface{}{}
	for _, f := range v {
		entry := map[string]interface{}{
			"name":     f.Name,
			"balancer": f.Balancer,
			"cpu":      f.CPU,
			"count":    f.Count,
			"memory":   f.Memory,
			"ports":    f.Ports,
		}
		formation[f.Name] = entry

		// for _, port := range f.Ports {
		// 	endpoints = append(endpoints, fmt.Sprintf("%s:%d (%s)", f.Balancer, port, f.Name))
		// }
	}

	if err := d.Set("formation", formation); err != nil {
		return errwrap.Wrapf("Unable to store formation: {{err}}", err)
	}

	return nil
}

func setParams(c *client.Client, d *schema.ResourceData) error {
	if !d.HasChange("params") {
		return nil
	}

	raw := d.Get("params").(map[string]interface{})
	params := make(client.Parameters)
	for key := range raw {
		params[key] = raw[key].(string)
	}

	log.Printf("[DEBUG] Setting params: (%#v) for %s", params, d.Id())
	if err := c.SetParameters(d.Id(), params); err != nil {
		return fmt.Errorf("Error setting params (%#v) for %s: %s", params, d.Id(), err)
	}

	return nil
}

func setEnv(c *client.Client, d *schema.ResourceData) error {
	if !d.HasChange("environment") {
		return nil
	}

	env := d.Get("environment").(map[string]interface{})
	log.Printf("[DEBUG] Setting environment to (%#v) for %s", env, d.Id())
	data := ""
	for key, value := range env {
		data += fmt.Sprintf("%s=%s\n", key, value)
	}
	_, rel, err := c.SetEnvironment(d.Id(), strings.NewReader(data))
	if err != nil {
		return fmt.Errorf("Error setting vars (%#v) for %s: %s", env, d.Id(), err)
	}
	log.Printf("[INFO] Release (%s) created on: %s", rel, d.Id())

	return nil
}

func appRefreshFunc(c *client.Client, app string) resource.StateRefreshFunc {
	return func() (interface{}, string, error) {
		app, err := c.GetApp(app)
		if err != nil {
			return app, "", err
		}
		return app, app.Status, err
	}
}
<|endoftext|>


<commit_before>package account

import (
	"bytes"
	"testing"

	"golang.org/x/net/context"

	"chain/core/signers"
	"chain/cos/txscript"
	"chain/database/pg"
	"chain/database/pg/pgtest"
	"chain/testutil"
)

var dummyXPub = testutil.TestXPub.String()

func resetSeqs(ctx context.Context, t testing.TB) {
	acpIndexNext, acpIndexCap = 1, 100
	pgtest.Exec(ctx, t, `ALTER SEQUENCE account_control_program_seq RESTART`)
	pgtest.Exec(ctx, t, `ALTER SEQUENCE signers_key_index_seq RESTART`)
}

func TestCreateControlProgram(t *testing.T) {
	ctx := pg.NewContext(context.Background(), pgtest.NewTx(t))
	resetSeqs(ctx, t)

	account, err := Create(ctx, []string{dummyXPub}, 1, nil, nil)
	if err != nil {
		testutil.FatalErr(t, err)
	}

	got, err := CreateControlProgram(ctx, account.ID)
	if err != nil {
		testutil.FatalErr(t, err)
	}

	want, err := txscript.ParseScriptString("OP_DUP OP_SHA3 OP_DATA_32 0x4a3663784142075fc63c53bcbbe4745e82de6a1b6142337f7673c7eb8e8a18d9 OP_EQUALVERIFY 0 OP_CHECKPREDICATE")
	if err != nil {
		testutil.FatalErr(t, err)
	}

	if !bytes.Equal(got, want) {
		t.Errorf("got control program = %x want %x", got, want)
	}
}

func createTestAccount(ctx context.Context, t testing.TB) *signers.Signer {
	account, err := Create(ctx, []string{dummyXPub}, 1, nil, nil)
	if err != nil {
		testutil.FatalErr(t, err)
	}

	return account
}

func createTestControlProgram(ctx context.Context, t testing.TB, accountID string) []byte {
	if accountID == "" {
		account := createTestAccount(ctx, t)
		accountID = account.ID
	}

	acp, err := CreateControlProgram(ctx, accountID)
	if err != nil {
		testutil.FatalErr(t, err)
	}
	return acp
}
<commit_msg>crypto/ed25519/hd25519: Little-Endian -> Big-Endian<commit_after>package account

import (
	"bytes"
	"testing"

	"golang.org/x/net/context"

	"chain/core/signers"
	"chain/cos/txscript"
	"chain/database/pg"
	"chain/database/pg/pgtest"
	"chain/testutil"
)

var dummyXPub = testutil.TestXPub.String()

func resetSeqs(ctx context.Context, t testing.TB) {
	acpIndexNext, acpIndexCap = 1, 100
	pgtest.Exec(ctx, t, `ALTER SEQUENCE account_control_program_seq RESTART`)
	pgtest.Exec(ctx, t, `ALTER SEQUENCE signers_key_index_seq RESTART`)
}

func TestCreateControlProgram(t *testing.T) {
	ctx := pg.NewContext(context.Background(), pgtest.NewTx(t))
	resetSeqs(ctx, t)

	account, err := Create(ctx, []string{dummyXPub}, 1, nil, nil)
	if err != nil {
		testutil.FatalErr(t, err)
	}

	got, err := CreateControlProgram(ctx, account.ID)
	if err != nil {
		testutil.FatalErr(t, err)
	}

	want, err := txscript.ParseScriptString("OP_DUP OP_SHA3 OP_DATA_32 0xffba1b8515ea01b4eb0026e65721a884c8cf4a15be58adc3dce24306a81026fe OP_EQUALVERIFY 0 OP_CHECKPREDICATE")
	if err != nil {
		testutil.FatalErr(t, err)
	}

	if !bytes.Equal(got, want) {
		t.Errorf("got control program = %x want %x", got, want)
	}
}

func createTestAccount(ctx context.Context, t testing.TB) *signers.Signer {
	account, err := Create(ctx, []string{dummyXPub}, 1, nil, nil)
	if err != nil {
		testutil.FatalErr(t, err)
	}

	return account
}

func createTestControlProgram(ctx context.Context, t testing.TB, accountID string) []byte {
	if accountID == "" {
		account := createTestAccount(ctx, t)
		accountID = account.ID
	}

	acp, err := CreateControlProgram(ctx, accountID)
	if err != nil {
		testutil.FatalErr(t, err)
	}
	return acp
}
<|endoftext|>


<commit_before>package operatingsystem // import "github.com/docker/docker/pkg/parsers/operatingsystem"

import (
	"fmt"

	"github.com/docker/docker/pkg/system"
	"golang.org/x/sys/windows/registry"
)

// GetOperatingSystem gets the name of the current operating system.
func GetOperatingSystem() (string, error) {
	os, err := withCurrentVersionRegistryKey(func(key registry.Key) (os string, err error) {
		if os, _, err = key.GetStringValue("ProductName"); err != nil {
			return "", err
		}

		releaseId, _, err := key.GetStringValue("ReleaseId")
		if err != nil {
			return
		}
		os = fmt.Sprintf("%s Version %s", os, releaseId)

		buildNumber, _, err := key.GetStringValue("CurrentBuildNumber")
		if err != nil {
			return
		}
		ubr, _, err := key.GetIntegerValue("UBR")
		if err != nil {
			return
		}
		os = fmt.Sprintf("%s (OS Build %s.%d)", os, buildNumber, ubr)

		return
	})

	if os == "" {
		// Default return value
		os = "Unknown Operating System"
	}

	return os, err
}

func withCurrentVersionRegistryKey(f func(registry.Key) (string, error)) (string, error) {
	key, err := registry.OpenKey(registry.LOCAL_MACHINE, `SOFTWARE\Microsoft\Windows NT\CurrentVersion`, registry.QUERY_VALUE)
	if err != nil {
		return "", err
	}
	defer key.Close()
	return f(key)
}

// GetOperatingSystemVersion gets the version of the current operating system, as a string.
func GetOperatingSystemVersion() (string, error) {
	version := system.GetOSVersion()
	return fmt.Sprintf("%d.%d.%d", version.MajorVersion, version.MinorVersion, version.Build), nil
}

// IsContainerized returns true if we are running inside a container.
// No-op on Windows, always returns false.
func IsContainerized() (bool, error) {
	return false, nil
}
<commit_msg>pkg/parsers: remove use of deprecated system.GetOSVersion()<commit_after>package operatingsystem // import "github.com/docker/docker/pkg/parsers/operatingsystem"

import (
	"fmt"

	"github.com/Microsoft/hcsshim/osversion"
	"golang.org/x/sys/windows/registry"
)

// GetOperatingSystem gets the name of the current operating system.
func GetOperatingSystem() (string, error) {
	os, err := withCurrentVersionRegistryKey(func(key registry.Key) (os string, err error) {
		if os, _, err = key.GetStringValue("ProductName"); err != nil {
			return "", err
		}

		releaseId, _, err := key.GetStringValue("ReleaseId")
		if err != nil {
			return
		}
		os = fmt.Sprintf("%s Version %s", os, releaseId)

		buildNumber, _, err := key.GetStringValue("CurrentBuildNumber")
		if err != nil {
			return
		}
		ubr, _, err := key.GetIntegerValue("UBR")
		if err != nil {
			return
		}
		os = fmt.Sprintf("%s (OS Build %s.%d)", os, buildNumber, ubr)

		return
	})

	if os == "" {
		// Default return value
		os = "Unknown Operating System"
	}

	return os, err
}

func withCurrentVersionRegistryKey(f func(registry.Key) (string, error)) (string, error) {
	key, err := registry.OpenKey(registry.LOCAL_MACHINE, `SOFTWARE\Microsoft\Windows NT\CurrentVersion`, registry.QUERY_VALUE)
	if err != nil {
		return "", err
	}
	defer key.Close()
	return f(key)
}

// GetOperatingSystemVersion gets the version of the current operating system, as a string.
func GetOperatingSystemVersion() (string, error) {
	version := osversion.Get()
	return fmt.Sprintf("%d.%d.%d", version.MajorVersion, version.MinorVersion, version.Build), nil
}

// IsContainerized returns true if we are running inside a container.
// No-op on Windows, always returns false.
func IsContainerized() (bool, error) {
	return false, nil
}
<|endoftext|>


<commit_before>package iface

import (
	"context"

	"github.com/ipfs/go-ipfs/core/coreapi/interface/options"

	peer "gx/ipfs/QmQsErDt8Qgw1XrsXf2BpEzDgGWtB1YLsTAARBup5b6B9W/go-libp2p-peer"
	pstore "gx/ipfs/Qmda4cPRvSRyox3SqgJN6DfSZGU5TtHufPTp9uXjFj71X6/go-libp2p-peerstore"
)

// DhtAPI specifies the interface to the DHT
type DhtAPI interface {
	// FindPeer queries the DHT for all of the multiaddresses associated with a
	// Peer ID
	FindPeer(context.Context, peer.ID) (pstore.PeerInfo, error)

	// FindProviders finds peers in the DHT who can provide a specific value
	// given a key.
	FindProviders(context.Context, Path, ...options.DhtFindProvidersOption) (<-chan pstore.PeerInfo, error)

	// Provide announces to the network that you are providing given values
	Provide(context.Context, Path, ...options.DhtProvideOption) error
}
<commit_msg>coreapi dht: add a note on name change<commit_after>package iface

import (
	"context"

	"github.com/ipfs/go-ipfs/core/coreapi/interface/options"

	peer "gx/ipfs/QmQsErDt8Qgw1XrsXf2BpEzDgGWtB1YLsTAARBup5b6B9W/go-libp2p-peer"
	pstore "gx/ipfs/Qmda4cPRvSRyox3SqgJN6DfSZGU5TtHufPTp9uXjFj71X6/go-libp2p-peerstore"
)

// DhtAPI specifies the interface to the DHT
// Note: This API will likely get renamed in near future, see
// https://github.com/ipfs/interface-ipfs-core/issues/249 for more context.
type DhtAPI interface {
	// FindPeer queries the DHT for all of the multiaddresses associated with a
	// Peer ID
	FindPeer(context.Context, peer.ID) (pstore.PeerInfo, error)

	// FindProviders finds peers in the DHT who can provide a specific value
	// given a key.
	FindProviders(context.Context, Path, ...options.DhtFindProvidersOption) (<-chan pstore.PeerInfo, error)

	// Provide announces to the network that you are providing given values
	Provide(context.Context, Path, ...options.DhtProvideOption) error
}
<|endoftext|>


<commit_before>package router

import (
	"errors"

	pb_broker "github.com/TheThingsNetwork/ttn/api/broker"
	pb "github.com/TheThingsNetwork/ttn/api/router"
	"github.com/TheThingsNetwork/ttn/core/types"
	"github.com/brocaar/lorawan"
)

func (r *router) HandleUplink(gatewayEUI types.GatewayEUI, uplink *pb.UplinkMessage) error {
	gateway := r.getGateway(gatewayEUI)

	gateway.Utilization.AddRx(uplink)

	downlinkOptions := r.buildDownlinkOptions(uplink, false, gateway)

	// LoRaWAN: Unmarshal
	var phyPayload lorawan.PHYPayload
	err := phyPayload.UnmarshalBinary(uplink.Payload)
	if err != nil {
		return err
	}

	switch phyPayload.MHDR.MType {
	case lorawan.JoinRequest:
		joinRequestPayload, ok := phyPayload.MACPayload.(*lorawan.JoinRequestPayload)
		if !ok {
			return errors.New("Join Request message does not contain a join payload.")
		}
		devEUI := types.DevEUI(joinRequestPayload.DevEUI)
		appEUI := types.AppEUI(joinRequestPayload.AppEUI)
		_, err := r.HandleActivation(gatewayEUI, &pb.DeviceActivationRequest{
			Payload:          uplink.Payload,
			DevEui:           &devEUI,
			AppEui:           &appEUI,
			ProtocolMetadata: uplink.ProtocolMetadata,
			GatewayMetadata:  uplink.GatewayMetadata,
		})
		return err
	case lorawan.UnconfirmedDataUp, lorawan.ConfirmedDataUp:
		// Just continue handling uplink
	default:
		return errors.New("ttn/router: Unhandled message type")
	}

	macPayload, ok := phyPayload.MACPayload.(*lorawan.MACPayload)
	if !ok {
		return errors.New("Uplink message does not contain a MAC payload.")
	}
	devAddr := types.DevAddr(macPayload.FHDR.DevAddr)

	// Find Broker
	brokers, err := r.brokerDiscovery.Discover(devAddr)
	if err != nil {
		return err
	}

	// Forward to all brokers
	for _, broker := range brokers {
		broker, err := r.getBroker(broker)
		if err != nil {
			continue
		}
		broker.association.Send(&pb_broker.UplinkMessage{
			Payload:          uplink.Payload,
			ProtocolMetadata: uplink.ProtocolMetadata,
			GatewayMetadata:  uplink.GatewayMetadata,
			DownlinkOptions:  downlinkOptions,
		})
	}

	return nil
}
<commit_msg>Sync gateway schedule on uplink<commit_after>package router

import (
	"errors"

	pb_broker "github.com/TheThingsNetwork/ttn/api/broker"
	pb "github.com/TheThingsNetwork/ttn/api/router"
	"github.com/TheThingsNetwork/ttn/core/types"
	"github.com/brocaar/lorawan"
)

func (r *router) HandleUplink(gatewayEUI types.GatewayEUI, uplink *pb.UplinkMessage) error {

	gateway := r.getGateway(gatewayEUI)
	gateway.Schedule.Sync(uplink.GatewayMetadata.Timestamp)
	gateway.Utilization.AddRx(uplink)

	downlinkOptions := r.buildDownlinkOptions(uplink, false, gateway)

	// LoRaWAN: Unmarshal
	var phyPayload lorawan.PHYPayload
	err := phyPayload.UnmarshalBinary(uplink.Payload)
	if err != nil {
		return err
	}

	switch phyPayload.MHDR.MType {
	case lorawan.JoinRequest:
		joinRequestPayload, ok := phyPayload.MACPayload.(*lorawan.JoinRequestPayload)
		if !ok {
			return errors.New("Join Request message does not contain a join payload.")
		}
		devEUI := types.DevEUI(joinRequestPayload.DevEUI)
		appEUI := types.AppEUI(joinRequestPayload.AppEUI)
		_, err := r.HandleActivation(gatewayEUI, &pb.DeviceActivationRequest{
			Payload:          uplink.Payload,
			DevEui:           &devEUI,
			AppEui:           &appEUI,
			ProtocolMetadata: uplink.ProtocolMetadata,
			GatewayMetadata:  uplink.GatewayMetadata,
		})
		return err
	case lorawan.UnconfirmedDataUp, lorawan.ConfirmedDataUp:
		// Just continue handling uplink
	default:
		return errors.New("ttn/router: Unhandled message type")
	}

	macPayload, ok := phyPayload.MACPayload.(*lorawan.MACPayload)
	if !ok {
		return errors.New("Uplink message does not contain a MAC payload.")
	}
	devAddr := types.DevAddr(macPayload.FHDR.DevAddr)

	// Find Broker
	brokers, err := r.brokerDiscovery.Discover(devAddr)
	if err != nil {
		return err
	}

	// Forward to all brokers
	for _, broker := range brokers {
		broker, err := r.getBroker(broker)
		if err != nil {
			continue
		}
		broker.association.Send(&pb_broker.UplinkMessage{
			Payload:          uplink.Payload,
			ProtocolMetadata: uplink.ProtocolMetadata,
			GatewayMetadata:  uplink.GatewayMetadata,
			DownlinkOptions:  downlinkOptions,
		})
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Time-related runtime and pieces of package time.

package runtime

import (
	"runtime/internal/sys"
	"unsafe"
)

// Package time knows the layout of this structure.
// If this struct changes, adjust ../time/sleep.go:/runtimeTimer.
// For GOOS=nacl, package syscall knows the layout of this structure.
// If this struct changes, adjust ../syscall/net_nacl.go:/runtimeTimer.
type timer struct {
	tb *timersBucket // the bucket the timer lives in
	i  int           // heap index

	// Timer wakes up at when, and then at when+period, ... (period > 0 only)
	// each time calling f(arg, now) in the timer goroutine, so f must be
	// a well-behaved function and not block.
	when   int64
	period int64
	f      func(interface{}, uintptr)
	arg    interface{}
	seq    uintptr
}

// timersLen is the length of timers array.
//
// Ideally, this would be set to GOMAXPROCS, but that would require
// dynamic reallocation
//
// The current value is a compromise between memory usage and performance
// that should cover the majority of GOMAXPROCS values used in the wild.
const timersLen = 64

// timers contains "per-P" timer heaps.
//
// Timers are queued into timersBucket associated with the current P,
// so each P may work with its own timers independently of other P instances.
//
// Each timersBucket may be associated with multiple P
// if GOMAXPROCS > timersLen.
var timers [timersLen]struct {
	timersBucket

	// The padding should eliminate false sharing
	// between timersBucket values.
	pad [sys.CacheLineSize - unsafe.Sizeof(timersBucket{})%sys.CacheLineSize]byte
}

func (t *timer) assignBucket() *timersBucket {
	id := uint8(getg().m.p.ptr().id) % timersLen
	t.tb = &timers[id].timersBucket
	return t.tb
}

type timersBucket struct {
	lock         mutex
	gp           *g
	created      bool
	sleeping     bool
	rescheduling bool
	sleepUntil   int64
	waitnote     note
	t            []*timer
}

// nacl fake time support - time in nanoseconds since 1970
var faketime int64

// Package time APIs.
// Godoc uses the comments in package time, not these.

// time.now is implemented in assembly.

// timeSleep puts the current goroutine to sleep for at least ns nanoseconds.
//go:linkname timeSleep time.Sleep
func timeSleep(ns int64) {
	if ns <= 0 {
		return
	}

	gp := getg()
	t := gp.timer
	if t == nil {
		t = new(timer)
		gp.timer = t
	}
	*t = timer{}
	t.when = nanotime() + ns
	t.f = goroutineReady
	t.arg = gp
	tb := t.assignBucket()
	lock(&tb.lock)
	tb.addtimerLocked(t)
	goparkunlock(&tb.lock, "sleep", traceEvGoSleep, 2)
}

// startTimer adds t to the timer heap.
//go:linkname startTimer time.startTimer
func startTimer(t *timer) {
	if raceenabled {
		racerelease(unsafe.Pointer(t))
	}
	addtimer(t)
}

// stopTimer removes t from the timer heap if it is there.
// It returns true if t was removed, false if t wasn't even there.
//go:linkname stopTimer time.stopTimer
func stopTimer(t *timer) bool {
	return deltimer(t)
}

// Go runtime.

// Ready the goroutine arg.
func goroutineReady(arg interface{}, seq uintptr) {
	goready(arg.(*g), 0)
}

func addtimer(t *timer) {
	tb := t.assignBucket()
	lock(&tb.lock)
	tb.addtimerLocked(t)
	unlock(&tb.lock)
}

// Add a timer to the heap and start or kick timerproc if the new timer is
// earlier than any of the others.
// Timers are locked.
func (tb *timersBucket) addtimerLocked(t *timer) {
	// when must never be negative; otherwise timerproc will overflow
	// during its delta calculation and never expire other runtime timers.
	if t.when < 0 {
		t.when = 1<<63 - 1
	}
	t.i = len(tb.t)
	tb.t = append(tb.t, t)
	tb.siftupTimer(t.i)
	if t.i == 0 {
		// siftup moved to top: new earliest deadline.
		if tb.sleeping {
			tb.sleeping = false
			notewakeup(&tb.waitnote)
		}
		if tb.rescheduling {
			tb.rescheduling = false
			goready(tb.gp, 0)
		}
	}
	if !tb.created {
		tb.created = true
		go timerproc(tb)
	}
}

// Delete timer t from the heap.
// Do not need to update the timerproc: if it wakes up early, no big deal.
func deltimer(t *timer) bool {
	tb := t.tb

	lock(&tb.lock)
	// t may not be registered anymore and may have
	// a bogus i (typically 0, if generated by Go).
	// Verify it before proceeding.
	i := t.i
	last := len(tb.t) - 1
	if i < 0 || i > last || tb.t[i] != t {
		unlock(&tb.lock)
		return false
	}
	if i != last {
		tb.t[i] = tb.t[last]
		tb.t[i].i = i
	}
	tb.t[last] = nil
	tb.t = tb.t[:last]
	if i != last {
		tb.siftupTimer(i)
		tb.siftdownTimer(i)
	}
	unlock(&tb.lock)
	return true
}

// Timerproc runs the time-driven events.
// It sleeps until the next event in the tb heap.
// If addtimer inserts a new earlier event, it wakes timerproc early.
func timerproc(tb *timersBucket) {
	tb.gp = getg()
	for {
		lock(&tb.lock)
		tb.sleeping = false
		now := nanotime()
		delta := int64(-1)
		for {
			if len(tb.t) == 0 {
				delta = -1
				break
			}
			t := tb.t[0]
			delta = t.when - now
			if delta > 0 {
				break
			}
			if t.period > 0 {
				// leave in heap but adjust next time to fire
				t.when += t.period * (1 + -delta/t.period)
				tb.siftdownTimer(0)
			} else {
				// remove from heap
				last := len(tb.t) - 1
				if last > 0 {
					tb.t[0] = tb.t[last]
					tb.t[0].i = 0
				}
				tb.t[last] = nil
				tb.t = tb.t[:last]
				if last > 0 {
					tb.siftdownTimer(0)
				}
				t.i = -1 // mark as removed
			}
			f := t.f
			arg := t.arg
			seq := t.seq
			unlock(&tb.lock)
			if raceenabled {
				raceacquire(unsafe.Pointer(t))
			}
			f(arg, seq)
			lock(&tb.lock)
		}
		if delta < 0 || faketime > 0 {
			// No timers left - put goroutine to sleep.
			tb.rescheduling = true
			goparkunlock(&tb.lock, "timer goroutine (idle)", traceEvGoBlock, 1)
			continue
		}
		// At least one timer pending. Sleep until then.
		tb.sleeping = true
		tb.sleepUntil = now + delta
		noteclear(&tb.waitnote)
		unlock(&tb.lock)
		notetsleepg(&tb.waitnote, delta)
	}
}

func timejump() *g {
	if faketime == 0 {
		return nil
	}

	for i := range timers {
		lock(&timers[i].lock)
	}
	gp := timejumpLocked()
	for i := range timers {
		unlock(&timers[i].lock)
	}

	return gp
}

func timejumpLocked() *g {
	// Determine a timer bucket with minimum when.
	var minT *timer
	for i := range timers {
		tb := &timers[i]
		if !tb.created || len(tb.t) == 0 {
			continue
		}
		t := tb.t[0]
		if minT == nil || t.when < minT.when {
			minT = t
		}
	}
	if minT == nil || minT.when <= faketime {
		return nil
	}

	faketime = minT.when
	tb := minT.tb
	if !tb.rescheduling {
		return nil
	}
	tb.rescheduling = false
	return tb.gp
}

func timeSleepUntil() int64 {
	next := int64(1<<63 - 1)

	// Determine minimum sleepUntil across all the timer buckets.
	//
	// The function can not return a precise answer,
	// as another timer may pop in as soon as timers have been unlocked.
	// So lock the timers one by one instead of all at once.
	for i := range timers {
		tb := &timers[i]

		lock(&tb.lock)
		if tb.sleeping && tb.sleepUntil < next {
			next = tb.sleepUntil
		}
		unlock(&tb.lock)
	}

	return next
}

// Heap maintenance algorithms.

func (tb *timersBucket) siftupTimer(i int) {
	t := tb.t
	when := t[i].when
	tmp := t[i]
	for i > 0 {
		p := (i - 1) / 4 // parent
		if when >= t[p].when {
			break
		}
		t[i] = t[p]
		t[i].i = i
		t[p] = tmp
		t[p].i = p
		i = p
	}
}

func (tb *timersBucket) siftdownTimer(i int) {
	t := tb.t
	n := len(t)
	when := t[i].when
	tmp := t[i]
	for {
		c := i*4 + 1 // left child
		c3 := c + 2  // mid child
		if c >= n {
			break
		}
		w := t[c].when
		if c+1 < n && t[c+1].when < w {
			w = t[c+1].when
			c++
		}
		if c3 < n {
			w3 := t[c3].when
			if c3+1 < n && t[c3+1].when < w3 {
				w3 = t[c3+1].when
				c3++
			}
			if w3 < w {
				w = w3
				c = c3
			}
		}
		if w >= when {
			break
		}
		t[i] = t[c]
		t[i].i = i
		t[c] = tmp
		t[c].i = c
		i = c
	}
}

// Entry points for net, time to call nanotime.

//go:linkname poll_runtimeNano internal/poll.runtimeNano
func poll_runtimeNano() int64 {
	return nanotime()
}

//go:linkname time_runtimeNano time.runtimeNano
func time_runtimeNano() int64 {
	return nanotime()
}

var startNano int64 = nanotime()
<commit_msg>runtime: optimize siftupTimer and siftdownTimer a bit<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Time-related runtime and pieces of package time.

package runtime

import (
	"runtime/internal/sys"
	"unsafe"
)

// Package time knows the layout of this structure.
// If this struct changes, adjust ../time/sleep.go:/runtimeTimer.
// For GOOS=nacl, package syscall knows the layout of this structure.
// If this struct changes, adjust ../syscall/net_nacl.go:/runtimeTimer.
type timer struct {
	tb *timersBucket // the bucket the timer lives in
	i  int           // heap index

	// Timer wakes up at when, and then at when+period, ... (period > 0 only)
	// each time calling f(arg, now) in the timer goroutine, so f must be
	// a well-behaved function and not block.
	when   int64
	period int64
	f      func(interface{}, uintptr)
	arg    interface{}
	seq    uintptr
}

// timersLen is the length of timers array.
//
// Ideally, this would be set to GOMAXPROCS, but that would require
// dynamic reallocation
//
// The current value is a compromise between memory usage and performance
// that should cover the majority of GOMAXPROCS values used in the wild.
const timersLen = 64

// timers contains "per-P" timer heaps.
//
// Timers are queued into timersBucket associated with the current P,
// so each P may work with its own timers independently of other P instances.
//
// Each timersBucket may be associated with multiple P
// if GOMAXPROCS > timersLen.
var timers [timersLen]struct {
	timersBucket

	// The padding should eliminate false sharing
	// between timersBucket values.
	pad [sys.CacheLineSize - unsafe.Sizeof(timersBucket{})%sys.CacheLineSize]byte
}

func (t *timer) assignBucket() *timersBucket {
	id := uint8(getg().m.p.ptr().id) % timersLen
	t.tb = &timers[id].timersBucket
	return t.tb
}

type timersBucket struct {
	lock         mutex
	gp           *g
	created      bool
	sleeping     bool
	rescheduling bool
	sleepUntil   int64
	waitnote     note
	t            []*timer
}

// nacl fake time support - time in nanoseconds since 1970
var faketime int64

// Package time APIs.
// Godoc uses the comments in package time, not these.

// time.now is implemented in assembly.

// timeSleep puts the current goroutine to sleep for at least ns nanoseconds.
//go:linkname timeSleep time.Sleep
func timeSleep(ns int64) {
	if ns <= 0 {
		return
	}

	gp := getg()
	t := gp.timer
	if t == nil {
		t = new(timer)
		gp.timer = t
	}
	*t = timer{}
	t.when = nanotime() + ns
	t.f = goroutineReady
	t.arg = gp
	tb := t.assignBucket()
	lock(&tb.lock)
	tb.addtimerLocked(t)
	goparkunlock(&tb.lock, "sleep", traceEvGoSleep, 2)
}

// startTimer adds t to the timer heap.
//go:linkname startTimer time.startTimer
func startTimer(t *timer) {
	if raceenabled {
		racerelease(unsafe.Pointer(t))
	}
	addtimer(t)
}

// stopTimer removes t from the timer heap if it is there.
// It returns true if t was removed, false if t wasn't even there.
//go:linkname stopTimer time.stopTimer
func stopTimer(t *timer) bool {
	return deltimer(t)
}

// Go runtime.

// Ready the goroutine arg.
func goroutineReady(arg interface{}, seq uintptr) {
	goready(arg.(*g), 0)
}

func addtimer(t *timer) {
	tb := t.assignBucket()
	lock(&tb.lock)
	tb.addtimerLocked(t)
	unlock(&tb.lock)
}

// Add a timer to the heap and start or kick timerproc if the new timer is
// earlier than any of the others.
// Timers are locked.
func (tb *timersBucket) addtimerLocked(t *timer) {
	// when must never be negative; otherwise timerproc will overflow
	// during its delta calculation and never expire other runtime timers.
	if t.when < 0 {
		t.when = 1<<63 - 1
	}
	t.i = len(tb.t)
	tb.t = append(tb.t, t)
	siftupTimer(tb.t, t.i)
	if t.i == 0 {
		// siftup moved to top: new earliest deadline.
		if tb.sleeping {
			tb.sleeping = false
			notewakeup(&tb.waitnote)
		}
		if tb.rescheduling {
			tb.rescheduling = false
			goready(tb.gp, 0)
		}
	}
	if !tb.created {
		tb.created = true
		go timerproc(tb)
	}
}

// Delete timer t from the heap.
// Do not need to update the timerproc: if it wakes up early, no big deal.
func deltimer(t *timer) bool {
	tb := t.tb

	lock(&tb.lock)
	// t may not be registered anymore and may have
	// a bogus i (typically 0, if generated by Go).
	// Verify it before proceeding.
	i := t.i
	last := len(tb.t) - 1
	if i < 0 || i > last || tb.t[i] != t {
		unlock(&tb.lock)
		return false
	}
	if i != last {
		tb.t[i] = tb.t[last]
		tb.t[i].i = i
	}
	tb.t[last] = nil
	tb.t = tb.t[:last]
	if i != last {
		siftupTimer(tb.t, i)
		siftdownTimer(tb.t, i)
	}
	unlock(&tb.lock)
	return true
}

// Timerproc runs the time-driven events.
// It sleeps until the next event in the tb heap.
// If addtimer inserts a new earlier event, it wakes timerproc early.
func timerproc(tb *timersBucket) {
	tb.gp = getg()
	for {
		lock(&tb.lock)
		tb.sleeping = false
		now := nanotime()
		delta := int64(-1)
		for {
			if len(tb.t) == 0 {
				delta = -1
				break
			}
			t := tb.t[0]
			delta = t.when - now
			if delta > 0 {
				break
			}
			if t.period > 0 {
				// leave in heap but adjust next time to fire
				t.when += t.period * (1 + -delta/t.period)
				siftdownTimer(tb.t, 0)
			} else {
				// remove from heap
				last := len(tb.t) - 1
				if last > 0 {
					tb.t[0] = tb.t[last]
					tb.t[0].i = 0
				}
				tb.t[last] = nil
				tb.t = tb.t[:last]
				if last > 0 {
					siftdownTimer(tb.t, 0)
				}
				t.i = -1 // mark as removed
			}
			f := t.f
			arg := t.arg
			seq := t.seq
			unlock(&tb.lock)
			if raceenabled {
				raceacquire(unsafe.Pointer(t))
			}
			f(arg, seq)
			lock(&tb.lock)
		}
		if delta < 0 || faketime > 0 {
			// No timers left - put goroutine to sleep.
			tb.rescheduling = true
			goparkunlock(&tb.lock, "timer goroutine (idle)", traceEvGoBlock, 1)
			continue
		}
		// At least one timer pending. Sleep until then.
		tb.sleeping = true
		tb.sleepUntil = now + delta
		noteclear(&tb.waitnote)
		unlock(&tb.lock)
		notetsleepg(&tb.waitnote, delta)
	}
}

func timejump() *g {
	if faketime == 0 {
		return nil
	}

	for i := range timers {
		lock(&timers[i].lock)
	}
	gp := timejumpLocked()
	for i := range timers {
		unlock(&timers[i].lock)
	}

	return gp
}

func timejumpLocked() *g {
	// Determine a timer bucket with minimum when.
	var minT *timer
	for i := range timers {
		tb := &timers[i]
		if !tb.created || len(tb.t) == 0 {
			continue
		}
		t := tb.t[0]
		if minT == nil || t.when < minT.when {
			minT = t
		}
	}
	if minT == nil || minT.when <= faketime {
		return nil
	}

	faketime = minT.when
	tb := minT.tb
	if !tb.rescheduling {
		return nil
	}
	tb.rescheduling = false
	return tb.gp
}

func timeSleepUntil() int64 {
	next := int64(1<<63 - 1)

	// Determine minimum sleepUntil across all the timer buckets.
	//
	// The function can not return a precise answer,
	// as another timer may pop in as soon as timers have been unlocked.
	// So lock the timers one by one instead of all at once.
	for i := range timers {
		tb := &timers[i]

		lock(&tb.lock)
		if tb.sleeping && tb.sleepUntil < next {
			next = tb.sleepUntil
		}
		unlock(&tb.lock)
	}

	return next
}

// Heap maintenance algorithms.

func siftupTimer(t []*timer, i int) {
	when := t[i].when
	tmp := t[i]
	for i > 0 {
		p := (i - 1) / 4 // parent
		if when >= t[p].when {
			break
		}
		t[i] = t[p]
		t[i].i = i
		i = p
	}
	if tmp != t[i] {
		t[i] = tmp
		t[i].i = i
	}
}

func siftdownTimer(t []*timer, i int) {
	n := len(t)
	when := t[i].when
	tmp := t[i]
	for {
		c := i*4 + 1 // left child
		c3 := c + 2  // mid child
		if c >= n {
			break
		}
		w := t[c].when
		if c+1 < n && t[c+1].when < w {
			w = t[c+1].when
			c++
		}
		if c3 < n {
			w3 := t[c3].when
			if c3+1 < n && t[c3+1].when < w3 {
				w3 = t[c3+1].when
				c3++
			}
			if w3 < w {
				w = w3
				c = c3
			}
		}
		if w >= when {
			break
		}
		t[i] = t[c]
		t[i].i = i
		i = c
	}
	if tmp != t[i] {
		t[i] = tmp
		t[i].i = i
	}
}

// Entry points for net, time to call nanotime.

//go:linkname poll_runtimeNano internal/poll.runtimeNano
func poll_runtimeNano() int64 {
	return nanotime()
}

//go:linkname time_runtimeNano time.runtimeNano
func time_runtimeNano() int64 {
	return nanotime()
}

var startNano int64 = nanotime()
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"compress/gzip"
	"crypto/sha1"
	"flag"
	"fmt"
	"go/build"
	"hash"
	"io"
	"os"
	"path/filepath"
	"unicode"
	"unicode/utf8"
)

func usage() {
	app := filepath.Base(os.Args[0])
	fmt.Fprintf(os.Stderr, "Usage:\n  %s [OPTIONS] FILE...\n\n", app)
	fmt.Fprintf(os.Stderr, "By default %s reads the input files and writes their\n", app)
	fmt.Fprintf(os.Stderr, "contents as embedded data in one Go file per input\n")
	fmt.Fprintf(os.Stderr, "file, by appending .go to the filename. Specifying -o\n")
	fmt.Fprintf(os.Stderr, "overrides this by writing all files to a single output\n")
	fmt.Fprintf(os.Stderr, "with the given name. %s attempts to detect the package\n", app)
	fmt.Fprintf(os.Stderr, "name but it can be specified with -package.\n\n")
	flag.PrintDefaults()
	os.Exit(2)
}

var (
	pkg      = flag.String("package", "", "Package name in output file(s)")
	output   = flag.String("o", "", "Output all data to this file")
	compress = flag.Bool("gzip", false, "Compress data with gzip before embedding")
	sha      = flag.Bool("sha1", false, "Also embed SHA1 hash of data")
)

func main() {
	flag.Parse()
	args := flag.Args()
	if len(args) == 0 {
		usage()
	}

	// Package name

	if *pkg != "" {
		*pkg = sanitise(*pkg)
	} else {
		dir := "."
		if *output != "" {
			dir = filepath.Dir(*output)
		}

		p, err := build.ImportDir(dir, 0)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Failed to determine package name: %v\n", err)
			os.Exit(1)
		}

		if p.Name == "" || p.Name == "." {
			fmt.Fprintf(os.Stderr, "Failed to determine package name\n")
			os.Exit(1)
		}

		*pkg = p.Name
	}

	// Output

	var (
		dst *os.File
		err error
	)

	if *output != "" {
		dst, err = os.Create(*output)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}

		if err = WritePackage(dst); err != nil {
			fmt.Fprintf(os.Stderr, "Failed to write package statement: %v\n", err)
			dst.Close()
			os.Exit(1)
		}
	}

	// Inputs

	for _, name := range args {
		src, err := os.Open(name)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			continue
		}

		if dst == nil {
			dst, err = os.Create(filepath.Base(name) + ".go")
			if err != nil {
				fmt.Fprintln(os.Stderr, err)
				src.Close()
				continue
			}

			if err = WritePackage(dst); err != nil {
				fmt.Fprintf(os.Stderr, "Failed to write package statement: %v\n", err)
				dst.Close()
				src.Close()
				os.Exit(1)
			}
		}

		if err = Embed(dst, src, name); err != nil {
			fmt.Fprintf(os.Stderr, "Failed to embed data: %v\n", err)
			dst.Close()
			src.Close()
			os.Exit(1)
		}

		if err = src.Close(); err != nil {
			fmt.Fprintf(os.Stderr, "Failed to close source: %v\n", err)
			dst.Close()
			os.Exit(1)
		}

		if *output == "" {
			if err = dst.Close(); err != nil {
				fmt.Fprintf(os.Stderr, "Failed to close output: %v\n", err)
				os.Exit(1)
			}

			dst = nil
		}
	}
}

func WritePackage(dst io.Writer) error {
	_, err := fmt.Fprintf(dst, "// MACHINE GENERATED - DO NOT EDIT //\n\npackage %s\n", *pkg)
	return err
}

func Embed(dst io.Writer, src io.Reader, name string) (err error) {
	if *compress {
		var wc io.WriteCloser
		wc, err = gzip.NewWriterLevel(dst, gzip.BestCompression)
		if err != nil {
			return err
		}

		defer func() {
			e := wc.Close()
			if err == nil && e != nil {
				err = e
			}
		}()

		dst = wc
	}

	var hasher hash.Hash
	if *sha {
		hasher = sha1.New()
		dst = io.MultiWriter(dst, hasher)
	}

	const BUF_SIZE = 12

	var n int
	var sanitised = sanitise(name)
	var buf [BUF_SIZE]byte

	_, err = fmt.Fprintf(dst, "\n// %s\nvar %s = []byte{\n", name, sanitised)
	if err != nil {
		return err
	}

	for {
		n, err = io.ReadFull(src, buf[:])
		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
			return err
		}

		if n == 0 {
			break
		}

		var w bytes.Buffer
		for i := 0; i < n; i++ {
			fmt.Fprintf(&w, "0x%02x, ", buf[i])
		}

		data := w.String()
		_, err = fmt.Fprintf(dst, "\t%s\n", data[:len(data)-1])
		if err != nil {
			return err
		}

		if n != BUF_SIZE {
			break
		}
	}

	_, err = fmt.Fprintf(dst, "}\n")
	if err != nil {
		return err
	}

	if hasher != nil {
		sum := hasher.Sum(nil)
		_, err = fmt.Fprintf(dst, "\n// SHA1 hash of %s\nvar %s_SHA1 = []byte{\n", name, sanitised)
		if err != nil {
			return err
		}

		for len(sum) > 0 {
			n := len(sum)
			if n > BUF_SIZE {
				n = BUF_SIZE
			}

			var w bytes.Buffer
			for i := 0; i < n; i++ {
				fmt.Fprintf(&w, "0x%02x, ", sum[i])
			}

			data := w.String()
			_, err = fmt.Fprintf(dst, "\t%s\n", data[:len(data)-1])
			if err != nil {
				return err
			}

			sum = sum[n:]
		}

		_, err = fmt.Fprintf(dst, "}\n")
		if err != nil {
			return err
		}
	}

	return err
}

func sanitise(name string) string {
	var buf bytes.Buffer
	var first = true

	name = filepath.Base(name)
	for len(name) > 0 {
		r, n := utf8.DecodeRuneInString(name)
		if unicode.IsLetter(r) || (!first && unicode.IsNumber(r)) {
			first = false
			buf.WriteRune(r)
		} else {
			buf.WriteByte('_')
		}

		name = name[n:]
	}

	if buf.Len() == 0 {
		buf.WriteByte('_')
	}

	return buf.String()
}
<commit_msg>Added package doc<commit_after>// Command embed stores files as resources in Go source.
//
// Embed reads the input files and writes their contents as embedded data
// in one Go file per input file, by appending .go to the filename.
// Specifying -o overrides this by writing all files to a single output
// with the given name. Embed attempts to detect the package name but
// it can be specified with -package.
//
// 	$ embed -o content.go -gzip -sha1 content/index.html content/style.css
//
package main

import (
	"bytes"
	"compress/gzip"
	"crypto/sha1"
	"flag"
	"fmt"
	"go/build"
	"hash"
	"io"
	"os"
	"path/filepath"
	"unicode"
	"unicode/utf8"
)

func usage() {
	app := filepath.Base(os.Args[0])
	fmt.Fprintf(os.Stderr, "Usage:\n  %s [OPTIONS] FILE...\n\n", app)
	fmt.Fprintf(os.Stderr, "By default %s reads the input files and writes their\n", app)
	fmt.Fprintf(os.Stderr, "contents as embedded data in one Go file per input\n")
	fmt.Fprintf(os.Stderr, "file, by appending .go to the filename. Specifying -o\n")
	fmt.Fprintf(os.Stderr, "overrides this by writing all files to a single output\n")
	fmt.Fprintf(os.Stderr, "with the given name. %s attempts to detect the package\n", app)
	fmt.Fprintf(os.Stderr, "name but it can be specified with -package.\n\n")
	flag.PrintDefaults()
	os.Exit(2)
}

var (
	pkg      = flag.String("package", "", "Package name in output file(s)")
	output   = flag.String("o", "", "Output all data to this file")
	compress = flag.Bool("gzip", false, "Compress data with gzip before embedding")
	sha      = flag.Bool("sha1", false, "Also embed SHA1 hash of data")
)

func main() {
	flag.Parse()
	args := flag.Args()
	if len(args) == 0 {
		usage()
	}

	// Package name

	if *pkg != "" {
		*pkg = sanitise(*pkg)
	} else {
		dir := "."
		if *output != "" {
			dir = filepath.Dir(*output)
		}

		p, err := build.ImportDir(dir, 0)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Failed to determine package name: %v\n", err)
			os.Exit(1)
		}

		if p.Name == "" || p.Name == "." {
			fmt.Fprintf(os.Stderr, "Failed to determine package name\n")
			os.Exit(1)
		}

		*pkg = p.Name
	}

	// Output

	var (
		dst *os.File
		err error
	)

	if *output != "" {
		dst, err = os.Create(*output)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}

		if err = WritePackage(dst); err != nil {
			fmt.Fprintf(os.Stderr, "Failed to write package statement: %v\n", err)
			dst.Close()
			os.Exit(1)
		}
	}

	// Inputs

	for _, name := range args {
		src, err := os.Open(name)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			continue
		}

		if dst == nil {
			dst, err = os.Create(filepath.Base(name) + ".go")
			if err != nil {
				fmt.Fprintln(os.Stderr, err)
				src.Close()
				continue
			}

			if err = WritePackage(dst); err != nil {
				fmt.Fprintf(os.Stderr, "Failed to write package statement: %v\n", err)
				dst.Close()
				src.Close()
				os.Exit(1)
			}
		}

		if err = Embed(dst, src, name); err != nil {
			fmt.Fprintf(os.Stderr, "Failed to embed data: %v\n", err)
			dst.Close()
			src.Close()
			os.Exit(1)
		}

		if err = src.Close(); err != nil {
			fmt.Fprintf(os.Stderr, "Failed to close source: %v\n", err)
			dst.Close()
			os.Exit(1)
		}

		if *output == "" {
			if err = dst.Close(); err != nil {
				fmt.Fprintf(os.Stderr, "Failed to close output: %v\n", err)
				os.Exit(1)
			}

			dst = nil
		}
	}
}

func WritePackage(dst io.Writer) error {
	_, err := fmt.Fprintf(dst, "// MACHINE GENERATED - DO NOT EDIT //\n\npackage %s\n", *pkg)
	return err
}

func Embed(dst io.Writer, src io.Reader, name string) (err error) {
	if *compress {
		var wc io.WriteCloser
		wc, err = gzip.NewWriterLevel(dst, gzip.BestCompression)
		if err != nil {
			return err
		}

		defer func() {
			e := wc.Close()
			if err == nil && e != nil {
				err = e
			}
		}()

		dst = wc
	}

	var hasher hash.Hash
	if *sha {
		hasher = sha1.New()
		dst = io.MultiWriter(dst, hasher)
	}

	const BUF_SIZE = 12

	var n int
	var sanitised = sanitise(name)
	var buf [BUF_SIZE]byte

	_, err = fmt.Fprintf(dst, "\n// %s\nvar %s = []byte{\n", name, sanitised)
	if err != nil {
		return err
	}

	for {
		n, err = io.ReadFull(src, buf[:])
		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
			return err
		}

		if n == 0 {
			break
		}

		var w bytes.Buffer
		for i := 0; i < n; i++ {
			fmt.Fprintf(&w, "0x%02x, ", buf[i])
		}

		data := w.String()
		_, err = fmt.Fprintf(dst, "\t%s\n", data[:len(data)-1])
		if err != nil {
			return err
		}

		if n != BUF_SIZE {
			break
		}
	}

	_, err = fmt.Fprintf(dst, "}\n")
	if err != nil {
		return err
	}

	if hasher != nil {
		sum := hasher.Sum(nil)
		_, err = fmt.Fprintf(dst, "\n// SHA1 hash of %s\nvar %s_SHA1 = []byte{\n", name, sanitised)
		if err != nil {
			return err
		}

		for len(sum) > 0 {
			n := len(sum)
			if n > BUF_SIZE {
				n = BUF_SIZE
			}

			var w bytes.Buffer
			for i := 0; i < n; i++ {
				fmt.Fprintf(&w, "0x%02x, ", sum[i])
			}

			data := w.String()
			_, err = fmt.Fprintf(dst, "\t%s\n", data[:len(data)-1])
			if err != nil {
				return err
			}

			sum = sum[n:]
		}

		_, err = fmt.Fprintf(dst, "}\n")
		if err != nil {
			return err
		}
	}

	return err
}

func sanitise(name string) string {
	var buf bytes.Buffer
	var first = true

	name = filepath.Base(name)
	for len(name) > 0 {
		r, n := utf8.DecodeRuneInString(name)
		if unicode.IsLetter(r) || (!first && unicode.IsNumber(r)) {
			first = false
			buf.WriteRune(r)
		} else {
			buf.WriteByte('_')
		}

		name = name[n:]
	}

	if buf.Len() == 0 {
		buf.WriteByte('_')
	}

	return buf.String()
}
<|endoftext|>