<commit_before>package main

import (
	"io/ioutil"
	"os"

	alsa "github.com/Narsil/alsa-go"
	"gopkg.in/gin-gonic/gin.v1"
)

func aplay(filename string) error {
	rate := 44100
	channels := 1

	handle := alsa.New()
	err := handle.Open("default", alsa.StreamTypePlayback, alsa.ModeBlock)
	if err != nil {
		return err
	}
	handle.SampleFormat = alsa.SampleFormatU8
	handle.SampleRate = rate
	handle.Channels = channels
	err = handle.ApplyHwParams()
	if err != nil {
		return err
	}

	buf, err := ioutil.ReadFile(filename)
	if err != nil {
		return err
	}

	_, err := handle.Write(buf)
	if err != nil {
		return err
	}
}

func main() {
	r := gin.Default()

	r.Get("/play", func(c *gin.Context) {
		err := aplay("/usr/local/share/bell.wav")
		if err != nil {
			c.JSON(http.StatusInternalServerError, err)
			return
		}
		c.JSON(http.StatusOk, gin.H{})
	})

	r.Run()
}
<commit_msg>Fix error<commit_after>package main

import (
	"io/ioutil"
	"net/http"

	alsa "github.com/Narsil/alsa-go"
	"gopkg.in/gin-gonic/gin.v1"
)

func aplay(filename string) error {
	rate := 44100
	channels := 1

	handle := alsa.New()
	err := handle.Open("default", alsa.StreamTypePlayback, alsa.ModeBlock)
	if err != nil {
		return err
	}
	handle.SampleFormat = alsa.SampleFormatU8
	handle.SampleRate = rate
	handle.Channels = channels
	err = handle.ApplyHwParams()
	if err != nil {
		return err
	}

	buf, err := ioutil.ReadFile(filename)
	if err != nil {
		return err
	}

	_, err = handle.Write(buf)
	return err
}

func main() {
	r := gin.Default()

	r.GET("/play", func(c *gin.Context) {
		err := aplay("/usr/local/share/bell.wav")
		if err != nil {
			c.JSON(http.StatusInternalServerError, err)
			return
		}
		c.JSON(http.StatusOK, gin.H{})
	})

	r.Run()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"time"

	log "github.com/sirupsen/logrus"

	"github.com/araddon/dateparse"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"gopkg.in/urfave/cli.v1"
)

func setVersion(svc *s3.S3, bucket string, key string, versionId string) error {
	log.Info("Setting version: ", bucket, " ", key, " ", versionId)
	if versionId == "DELETE" {
		input := &s3.DeleteObjectInput{
			Bucket: &bucket,
			Key:    &key,
		}
		_, err := svc.DeleteObject(input)
		if err != nil {
			return err
		}
	} else {
		src := "/" + bucket + "/" + key + "?versionId=" + versionId
		input := &s3.CopyObjectInput{
			CopySource: &src,
			Bucket:     &bucket,
			Key:        &key,
		}

		_, err := svc.CopyObject(input)
		if err != nil {
			return err
		}
	}
	return nil
}

func processVersion(final *map[string]s3.ObjectVersion, target time.Time, obj *s3.ObjectVersion) {
	log.Print("Processing: ", obj)

	key := *obj.Key
	curObj, exists := (*final)[key]
	log.Info("Key ", key, " exists ", exists)

	if !exists {
		log.Info("Adding key that doesn't exist", key)
		// insert a delete marker for things that didn't exist
		if obj.LastModified.After(target) {
			fakeVersionID := "DELETE"
			deleteObj := s3.ObjectVersion{
				Key:          obj.Key,
				LastModified: obj.LastModified,
				VersionId:    &fakeVersionID,
			}
			(*final)[key] = deleteObj
			return
		}
		(*final)[key] = *obj
		return
	}
	if obj.LastModified.After(target) {
		return
	}
	// This is our object if
	// If it is newer than what is currently in place
	// b) curObj is currently a future delete marker
	if obj.LastModified.After(*(*final)[key].LastModified) || curObj.LastModified.After(target) {
		(*final)[key] = *obj
	}
}

func processDeleteMarker(final *map[string]s3.ObjectVersion, target time.Time, obj *s3.DeleteMarkerEntry) {
	if obj.LastModified.After(target) {
		return
	}
	fakeVersionID := "DELETE"
	key := *obj.Key
	curObj, exists := (*final)[key]
	deleteObj := s3.ObjectVersion{
		Key:          obj.Key,
		LastModified: obj.LastModified,
		VersionId:    &fakeVersionID,
	}
	if obj.LastModified.After(target) {
		return
	}

	if !exists {
		(*final)[key] = deleteObj
		return
	}

	if obj.LastModified.After(*(*final)[key].LastModified) || curObj.LastModified.After(target) {
		(*final)[key] = deleteObj
	}
}

func buildVersionDictionary(svc *s3.S3, bucket string, target time.Time) (final map[string]s3.ObjectVersion, err error) {
	final = make(map[string]s3.ObjectVersion)
	log.Print("Getting versions...")
	err = svc.ListObjectVersionsPages(&s3.ListObjectVersionsInput{
		Bucket: &bucket,
	}, func(p *s3.ListObjectVersionsOutput, last bool) (shouldContinue bool) {
		for _, obj := range p.Versions {
			processVersion(&final, target, obj)
		}
		for _, obj := range p.DeleteMarkers {
			processDeleteMarker(&final, target, obj)
		}
		return true
	})
	if err != nil {
		fmt.Println("failed to list objects", err)
		return
	}
	return
}

func processDictionary(svc *s3.S3, bucket string, final map[string]s3.ObjectVersion) error {
	log.Info("Processing final map", final)
	for key, value := range final {
		err := setVersion(svc, bucket, key, *value.VersionId)
		if err != nil {
			return err
		}
	}
	return nil
}

func process(c *cli.Context) error {
	fmt.Println("Beginning processing")
	sess := session.Must(session.NewSession())
	svc := s3.New(sess)
	targetTime, err := dateparse.ParseAny(c.GlobalString("time"))
	log.Print("Parsed time ", err)
	if err != nil {
		fmt.Println("Invalid time string", err)
		return err
	}
	objects, err := buildVersionDictionary(svc, c.GlobalString("bucket"), targetTime)
	if err != nil {
		fmt.Println("Failed building dictionary", err)
		return err
	}
	fmt.Println("Found ", len(objects), " keys")
	err = processDictionary(svc, c.GlobalString("bucket"), objects)
	if err != nil {
		fmt.Println("Failed processing dictionary", err)
	}
	fmt.Println("Done")
	return err
}
func init() {
	log.SetOutput(os.Stderr)

	// Only log the warning severity or above.
	log.SetLevel(log.ErrorLevel)
}

func main() {
	app := cli.NewApp()
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "bucket",
			Usage: "s3 bucket to process",
		},
		cli.StringFlag{
			Name:  "time",
			Usage: "time to restore to. Use format: 2006-01-02T15:04:05.999999999Z07:00",
		},
		cli.BoolFlag{
			Name:  "debug",
			Usage: "Use to enable debug logging",
		},
	}

	app.Commands = []cli.Command{

		{
			Name:   "restore",
			Usage:  "restore the s3 bucket to the time specified",
			Action: process,
		},
	}
	log.Print("Running App..")

	app.Before = func(c *cli.Context) error {
		if c.Bool("debug") {
			log.SetLevel(log.DebugLevel)
		}
		return nil
	}

	app.Run(os.Args)
}
<commit_msg>Add support for endpoint-url<commit_after>package main

import (
	"fmt"
	"os"
	"time"

	log "github.com/sirupsen/logrus"

	"github.com/araddon/dateparse"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"gopkg.in/urfave/cli.v1"
)

func setVersion(svc *s3.S3, bucket string, key string, versionId string) error {
	log.Info("Setting version: ", bucket, " ", key, " ", versionId)
	if versionId == "DELETE" {
		input := &s3.DeleteObjectInput{
			Bucket: &bucket,
			Key:    &key,
		}
		_, err := svc.DeleteObject(input)
		if err != nil {
			return err
		}
	} else {
		src := "/" + bucket + "/" + key + "?versionId=" + versionId
		input := &s3.CopyObjectInput{
			CopySource: &src,
			Bucket:     &bucket,
			Key:        &key,
		}

		_, err := svc.CopyObject(input)
		if err != nil {
			return err
		}
	}
	return nil
}

func makeDeleteMarker(obj *s3.ObjectVersion) s3.ObjectVersion {
	fakeVersionID := "DELETE"
	return s3.ObjectVersion{
		Key:          obj.Key,
		LastModified: obj.LastModified,
		VersionId:    &fakeVersionID,
	}
}

func makeDeleteMarker2(obj *s3.DeleteMarkerEntry) s3.ObjectVersion {
	fakeVersionID := "DELETE"
	return s3.ObjectVersion{
		Key:          obj.Key,
		LastModified: obj.LastModified,
		VersionId:    &fakeVersionID,
	}
}
func processVersion(final *map[string]s3.ObjectVersion, target time.Time, obj *s3.ObjectVersion) {
	log.Print("Processing: ", obj)

	key := *obj.Key
	curObj, exists := (*final)[key]
	log.Info("Key ", key, " exists ", exists)

	if !exists {
		log.Info("Adding key that doesn't exist", key)
		// insert a delete marker for things that didn't exist
		if obj.LastModified.After(target) {
			(*final)[key] = makeDeleteMarker(obj)
			return
		}
		(*final)[key] = *obj
		return
	}
	if obj.LastModified.After(target) {
		return
	}
	// This is our object if
	// If it is newer than what is currently in place
	// b) curObj is currently a future delete marker
	if obj.LastModified.After(*(*final)[key].LastModified) || curObj.LastModified.After(target) {
		(*final)[key] = *obj
	}
}

func processDeleteMarker(final *map[string]s3.ObjectVersion, target time.Time, obj *s3.DeleteMarkerEntry) {
	if obj.LastModified.After(target) {
		return
	}
	key := *obj.Key
	curObj, exists := (*final)[key]
	if obj.LastModified.After(target) {
		return
	}

	if !exists || obj.LastModified.After(*(*final)[key].LastModified) || curObj.LastModified.After(target) {
		(*final)[key] = makeDeleteMarker2(obj)
	}
}

func buildVersionDictionary(svc *s3.S3, bucket string, target time.Time) (final map[string]s3.ObjectVersion, err error) {
	final = make(map[string]s3.ObjectVersion)
	log.Print("Getting versions...")
	err = svc.ListObjectVersionsPages(&s3.ListObjectVersionsInput{
		Bucket: &bucket,
	}, func(p *s3.ListObjectVersionsOutput, last bool) (shouldContinue bool) {
		for _, obj := range p.Versions {
			processVersion(&final, target, obj)
		}
		for _, obj := range p.DeleteMarkers {
			processDeleteMarker(&final, target, obj)
		}
		return true
	})
	if err != nil {
		fmt.Println("failed to list objects", err)
		return
	}
	return
}

func processDictionary(svc *s3.S3, bucket string, final map[string]s3.ObjectVersion) error {
	log.Info("Processing final map", final)
	for key, value := range final {
		err := setVersion(svc, bucket, key, *value.VersionId)
		if err != nil {
			return err
		}
	}
	return nil
}

func process(c *cli.Context) error {
	fmt.Println("Beginning processing")
	endpointurl := c.GlobalString("endpoint-url")
	awsConfig := &aws.Config{
		Endpoint: &endpointurl,
	}

	sess := session.Must(session.NewSession(awsConfig))
	if sess == nil {
		fmt.Println("Failed to create aws session")
	}
	svc := s3.New(sess, awsConfig)
	if c.GlobalString("bucket") == "" {
		fmt.Println("No bucket specified")
		return nil
	}
	targetTime, err := dateparse.ParseAny(c.GlobalString("time"))
	log.Print("Parsed time ", targetTime)
	if err != nil {
		fmt.Println("Invalid time string", err)
		return err
	}
	objects, err := buildVersionDictionary(svc, c.GlobalString("bucket"), targetTime)
	if err != nil {
		fmt.Println("Failed building dictionary", err)
		return err
	}
	fmt.Println("Found ", len(objects), " keys")
	err = processDictionary(svc, c.GlobalString("bucket"), objects)
	if err != nil {
		fmt.Println("Failed processing dictionary", err)
	}
	fmt.Println("Done")
	return err
}
func init() {
	log.SetOutput(os.Stderr)

	// Only log the warning severity or above.
	log.SetLevel(log.ErrorLevel)
}

func main() {
	app := cli.NewApp()
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "bucket",
			Usage: "required: s3 bucket to process",
		},
		cli.StringFlag{
			Name:  "time",
			Usage: "required: time to restore to. Use format: 2006-01-02T15:04:05.999999999Z07:00",
		},
		cli.StringFlag{
			Name:  "endpoint-url",
			Usage: "S3 compatible endpoint url",
		},
		cli.BoolFlag{
			Name:  "debug",
			Usage: "Use to enable debug logging",
		},
	}

	app.Commands = []cli.Command{

		{
			Name:   "restore",
			Usage:  "restore the s3 bucket to the time specified",
			Action: process,
		},
	}
	log.Print("Running App..")

	app.Before = func(c *cli.Context) error {
		if c.Bool("debug") {
			log.SetLevel(log.DebugLevel)
		}
		return nil
	}

	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"strings"

	"github.com/foursquare/gohfile"
)

type SettingDefs struct {
	port  int
	debug bool
	mlock bool

	configJsonUrl string

	cachePath      string
	remotePrefixes RemotePrefix

	graphite       string
	graphitePrefix string
}

var Settings SettingDefs

type RemotePrefix struct {
	prefixes map[string]string
}

func (r *RemotePrefix) String() string {
	return "remote prefix"
}

func (r *RemotePrefix) Set(s string) error {
	parts := strings.SplitN(s, "=", 2)
	if len(parts) != 2 {
		return fmt.Errorf("remote prefix must be of form /prefix/path=http://example/foo%%s")
	}
	r.prefixes[parts[0]] = parts[1]
	return nil
}

func readSettings() []string {
	s := SettingDefs{}
	s.remotePrefixes.prefixes = make(map[string]string)
	flag.IntVar(&s.port, "port", 9999, "listen port")
	flag.BoolVar(&s.debug, "debug", false, "print debug output")

	flag.BoolVar(&s.mlock, "mem", false, "mlock ALL mapped files to keep them im memory (ignores = vs @ specs).")

	flag.StringVar(&s.configJsonUrl, "config-json", "", "URL of collection configuration json")

	flag.StringVar(&s.cachePath, "cache", os.TempDir(), "local path to write files fetched (*not* cleaned up automatically)")

	flag.StringVar(&s.graphite, "graphite", "", "graphite server to report to")
	flag.StringVar(&s.graphitePrefix, "graphite-prefix", "", "prefix to prepend to reported metrics")

	flag.Var(&s.remotePrefixes, "remote", "/prefix/path=<url-format-string>")

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr,
			`
Usage: %s [options] col1=path1 col2=path2 ...

	By default, collections are mmaped and locked into memory.
	Use 'col@path' to serve directly off disk.

	You may need to set 'ulimit -Hl' and 'ulimit -Sl' to allow locking.

`, os.Args[0])
		flag.PrintDefaults()
	}

	flag.Parse()
	Settings = s

	if len(flag.Args()) < 1 && Settings.configJsonUrl == "" {
		log.Println("bah!", Settings)
		flag.Usage()
		os.Exit(-1)
	}

	return flag.Args()
}

func getCollectionConfig(args []string) []hfile.CollectionConfig {
	var configs []hfile.CollectionConfig
	var err error

	if Settings.configJsonUrl != "" {
		if len(args) > 0 {
			log.Fatalln("Only one of command-line collection specs or json config may be used.")
		}
		configs, err = LoadFromUrl(Settings.configJsonUrl)
		if err != nil {
			log.Fatalln(err)
		}
	} else {
		configs := make([]hfile.CollectionConfig, len(args))
		for i, pair := range flag.Args() {
			parts := strings.SplitN(pair, "=", 2)
			mlock := true
			if len(parts) != 2 {
				mlock = false
				parts = strings.SplitN(pair, "@", 2)
			}
			if len(parts) != 2 {
				log.Fatal("collections must be specified in the form 'name=path' or 'name@path'")
			}
			configs[i] = hfile.CollectionConfig{parts[0], parts[1], mlock}
		}
	}
	if Settings.mlock {
		for i, _ := range configs {
			if Settings.debug && !configs[i].Mlock {
				log.Printf("[getCollectionConfig] Forcing %s to be in-memory", configs[i].Name)
			}
			configs[i].Mlock = true
		}
	}
	return configs
}

func main() {
	args := readSettings()

	SetupStats(false, Settings.graphite, Settings.graphitePrefix)

	configs := getCollectionConfig(args)

	log.Println("Loading collections...")
	cs, err := hfile.LoadCollections(configs, Settings.debug)
	if err != nil {
		log.Fatal(err)
	}

	name, err := os.Hostname()
	if err != nil {
		name = "localhost"
	}
	log.Printf("Serving on http://%s:%d/ \n", name, Settings.port)

	http.Handle("/rpc/HFileService", NewHttpRpcHandler(cs))
	http.Handle("/", &DebugHandler{cs})

	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", Settings.port), nil))
}
<commit_msg>fix shadowed collection list when passing mappings<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"strings"

	"github.com/foursquare/gohfile"
)

type SettingDefs struct {
	port  int
	debug bool
	mlock bool

	configJsonUrl string

	cachePath      string
	remotePrefixes RemotePrefix

	graphite       string
	graphitePrefix string
}

var Settings SettingDefs

type RemotePrefix struct {
	prefixes map[string]string
}

func (r *RemotePrefix) String() string {
	return "remote prefix"
}

func (r *RemotePrefix) Set(s string) error {
	parts := strings.SplitN(s, "=", 2)
	if len(parts) != 2 {
		return fmt.Errorf("remote prefix must be of form /prefix/path=http://example/foo%%s")
	}
	r.prefixes[parts[0]] = parts[1]
	return nil
}

func readSettings() []string {
	s := SettingDefs{}
	s.remotePrefixes.prefixes = make(map[string]string)
	flag.IntVar(&s.port, "port", 9999, "listen port")
	flag.BoolVar(&s.debug, "debug", false, "print debug output")

	flag.BoolVar(&s.mlock, "mem", false, "mlock ALL mapped files to keep them im memory (ignores = vs @ specs).")

	flag.StringVar(&s.configJsonUrl, "config-json", "", "URL of collection configuration json")

	flag.StringVar(&s.cachePath, "cache", os.TempDir(), "local path to write files fetched (*not* cleaned up automatically)")

	flag.StringVar(&s.graphite, "graphite", "", "graphite server to report to")
	flag.StringVar(&s.graphitePrefix, "graphite-prefix", "", "prefix to prepend to reported metrics")

	flag.Var(&s.remotePrefixes, "remote", "/prefix/path=<url-format-string>")

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr,
			`
Usage: %s [options] col1=path1 col2=path2 ...

	By default, collections are mmaped and locked into memory.
	Use 'col@path' to serve directly off disk.

	You may need to set 'ulimit -Hl' and 'ulimit -Sl' to allow locking.

`, os.Args[0])
		flag.PrintDefaults()
	}

	flag.Parse()
	Settings = s

	if len(flag.Args()) < 1 && Settings.configJsonUrl == "" {
		log.Println("bah!", Settings)
		flag.Usage()
		os.Exit(-1)
	}

	return flag.Args()
}

func getCollectionConfig(args []string) []hfile.CollectionConfig {
	var configs []hfile.CollectionConfig
	var err error

	if Settings.configJsonUrl != "" {
		if len(args) > 0 {
			log.Fatalln("Only one of command-line collection specs or json config may be used.")
		}
		configs, err = LoadFromUrl(Settings.configJsonUrl)
		if err != nil {
			log.Fatalln(err)
		}
	} else {
		configs = make([]hfile.CollectionConfig, len(args))
		for i, pair := range args {
			parts := strings.SplitN(pair, "=", 2)
			mlock := true
			if len(parts) != 2 {
				mlock = false
				parts = strings.SplitN(pair, "@", 2)
			}
			if len(parts) != 2 {
				log.Fatal("collections must be specified in the form 'name=path' or 'name@path'")
			}
			configs[i] = hfile.CollectionConfig{parts[0], parts[1], mlock}
		}
	}
	if Settings.mlock {
		for i, _ := range configs {
			if Settings.debug && !configs[i].Mlock {
				log.Printf("[getCollectionConfig] Forcing %s to be in-memory", configs[i].Name)
			}
			configs[i].Mlock = true
		}
	}
	return configs
}

func main() {
	args := readSettings()

	SetupStats(false, Settings.graphite, Settings.graphitePrefix)

	configs := getCollectionConfig(args)

	log.Printf("Loading collections (debug %v)...\n", Settings.debug)
	cs, err := hfile.LoadCollections(configs, Settings.debug)
	if err != nil {
		log.Fatal(err)
	}

	name, err := os.Hostname()
	if err != nil {
		name = "localhost"
	}
	log.Printf("Serving on http://%s:%d/ \n", name, Settings.port)

	http.Handle("/rpc/HFileService", NewHttpRpcHandler(cs))
	http.Handle("/", &DebugHandler{cs})

	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", Settings.port), nil))
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"runtime"
	"strings"
	"text/tabwriter"
	"time"

	"gopkg.in/alecthomas/kingpin.v2"

	"github.com/rackerlabs/carina/version"
	"github.com/rackerlabs/libcarina"
)

// Application is, our, well, application
type Application struct {
	*Context
	*kingpin.Application
}

// Command is a command needing a ClusterClient
type Command struct {
	*Context
	*kingpin.CmdClause
}

// Context context for the  App
type Context struct {
	ClusterClient *libcarina.ClusterClient
	TabWriter     *tabwriter.Writer
	Username      string
	APIKey        string
	Endpoint      string
}

// ClusterCommand is a Command with a ClusterName set
type ClusterCommand struct {
	*Command
	ClusterName string
}

// CredentialsCommand keeps context about the download command
type CredentialsCommand struct {
	*ClusterCommand
	Path string
}

// CreateCommand keeps context about the create command
type CreateCommand struct {
	*ClusterCommand

	// Options passed along to Carina's API
	Nodes     int
	AutoScale bool

	// Whether to wait until the cluster is created (or errored)
	Wait bool

	// TODO: See if setting flavor or image makes sense, even if the API takes it
	// Flavor    string
	// Image     string
}

// GrowCommand keeps context about the number of nodes to scale by
type GrowCommand struct {
	*ClusterCommand
	Nodes int
}

// New creates a new Application
func New() *Application {

	app := kingpin.New("carina", "command line interface to launch and work with Docker Swarm clusters")

	cap := new(Application)
	ctx := new(Context)

	cap.Application = app

	cap.Version(version.Version)
	cap.Context = ctx

	cap.Flag("username", "Rackspace username - can also set env var RACKSPACE_USERNAME").OverrideDefaultFromEnvar("RACKSPACE_USERNAME").StringVar(&ctx.Username)
	cap.Flag("api-key", "Rackspace API Key - can also set env var RACKSPACE_APIKEY").OverrideDefaultFromEnvar("RACKSPACE_APIKEY").PlaceHolder("RACKSPACE_APIKEY").StringVar(&ctx.APIKey)
	cap.Flag("endpoint", "Carina API endpoint").Default(libcarina.BetaEndpoint).StringVar(&ctx.Endpoint)

	writer := new(tabwriter.Writer)
	writer.Init(os.Stdout, 0, 8, 1, '\t', 0)

	ctx.TabWriter = writer

	listCommand := cap.NewCommand(ctx, "list", "list swarm clusters")
	listCommand.Action(listCommand.List)

	getCommand := cap.NewClusterCommand(ctx, "get", "get information about a swarm cluster")
	getCommand.Action(getCommand.Get)

	deleteCommand := cap.NewClusterCommand(ctx, "delete", "delete a swarm cluster")
	deleteCommand.Action(deleteCommand.Delete)

	createCommand := new(CreateCommand)
	createCommand.ClusterCommand = cap.NewClusterCommand(ctx, "create", "create a swarm cluster")
	createCommand.Flag("wait", "wait for swarm cluster completion").BoolVar(&createCommand.Wait)
	createCommand.Flag("nodes", "number of nodes for the initial cluster").Default("1").IntVar(&createCommand.Nodes)
	createCommand.Flag("autoscale", "whether autoscale is on or off").BoolVar(&createCommand.AutoScale)
	createCommand.Action(createCommand.Create)

	credentialsCommand := new(CredentialsCommand)
	credentialsCommand.ClusterCommand = cap.NewClusterCommand(ctx, "credentials", "download credentials")
	credentialsCommand.Flag("path", "path to write credentials out to").StringVar(&credentialsCommand.Path)
	credentialsCommand.Action(credentialsCommand.Download)

	growCommand := new(GrowCommand)
	growCommand.ClusterCommand = cap.NewClusterCommand(ctx, "grow", "Grow a cluster by the requested number of nodes")
	growCommand.Flag("nodes", "number of nodes to increase the cluster by").Required().IntVar(&growCommand.Nodes)
	growCommand.Action(growCommand.Grow)

	return cap
}

// NewCommand creates a command wrapped with carina.Context
func (app *Application) NewCommand(ctx *Context, name, help string) *Command {
	carina := new(Command)
	carina.Context = ctx
	carina.CmdClause = app.Command(name, help)
	carina.PreAction(carina.Auth)
	return carina
}

// NewClusterCommand is a command that uses a cluster name
func (app *Application) NewClusterCommand(ctx *Context, name, help string) *ClusterCommand {
	cc := new(ClusterCommand)
	cc.Command = app.NewCommand(ctx, name, help)
	cc.Arg("cluster-name", "name of the cluster").Required().StringVar(&cc.ClusterName)
	return cc
}

// Auth does the authentication
func (carina *Command) Auth(pc *kingpin.ParseContext) (err error) {
	carina.ClusterClient, err = libcarina.NewClusterClient(carina.Endpoint, carina.Username, carina.APIKey)
	return err
}

// List the current swarm clusters
func (carina *Command) List(pc *kingpin.ParseContext) (err error) {
	clusterList, err := carina.ClusterClient.List()
	if err != nil {
		return err
	}

	headerFields := []string{
		"ClusterName",
		"Username",
		"Flavor",
		"Image",
		"Nodes",
		"Status",
	}
	s := strings.Join(headerFields, "\t")

	_, err = carina.TabWriter.Write([]byte(s + "\n"))
	if err != nil {
		return err
	}

	for _, cluster := range clusterList {
		err = writeCluster(carina.TabWriter, &cluster)
		if err != nil {
			return err
		}
	}
	err = carina.TabWriter.Flush()
	return err
}

type clusterOp func(clusterName string) (*libcarina.Cluster, error)

// Does an func against a cluster then returns the new cluster representation
func (carina *ClusterCommand) clusterApply(op clusterOp) (err error) {
	cluster, err := op(carina.ClusterName)
	if err != nil {
		return err
	}

	err = writeCluster(carina.TabWriter, cluster)
	if err != nil {
		return err
	}
	return carina.TabWriter.Flush()
}

// Get an individual cluster
func (carina *ClusterCommand) Get(pc *kingpin.ParseContext) (err error) {
	return carina.clusterApply(carina.ClusterClient.Get)
}

// Delete a cluster
func (carina *ClusterCommand) Delete(pc *kingpin.ParseContext) (err error) {
	return carina.clusterApply(carina.ClusterClient.Delete)
}

// Grow increases the size of the given cluster
func (carina *GrowCommand) Grow(pc *kingpin.ParseContext) (err error) {
	return carina.clusterApply(func(clusterName string) (*libcarina.Cluster, error) {
		return carina.ClusterClient.Grow(clusterName, carina.Nodes)
	})
}

// Create a cluster
func (carina *CreateCommand) Create(pc *kingpin.ParseContext) (err error) {
	if carina.Nodes < 1 {
		return errors.New("nodes must be >= 1")
	}

	nodes := libcarina.Number(carina.Nodes)

	c := libcarina.Cluster{
		ClusterName: carina.ClusterName,
		Nodes:       nodes,
		AutoScale:   carina.AutoScale,
	}

	cluster, err := carina.ClusterClient.Create(c)

	// Transitions past point of "new" or "building" are assumed to be states we
	// can stop on.
	if carina.Wait {
		for cluster.Status == "new" || cluster.Status == "building" {
			time.Sleep(13 * time.Second)
			cluster, err = carina.ClusterClient.Get(carina.ClusterName)
			if err != nil {
				break
			}
		}
	}

	if err != nil {
		return err
	}

	err = writeCluster(carina.TabWriter, cluster)

	if err != nil {
		return err
	}
	return carina.TabWriter.Flush()
}

// Download credentials for a cluster
func (carina *CredentialsCommand) Download(pc *kingpin.ParseContext) (err error) {
	credentials, err := carina.ClusterClient.GetCredentials(carina.ClusterName)
	if err != nil {
		return err
	}

	p := path.Clean(carina.Path)

	if p != "." {
		err = os.MkdirAll(p, 0777)
	}

	if err != nil {
		return err
	}

	err = writeCredentials(carina.TabWriter, credentials, p)
	if err != nil {
		return err
	}

	if runtime.GOOS == "windows" {
		fmt.Fprintf(os.Stdout, "\"%v\"\n", path.Join(p, "docker.cmd"))
		fmt.Fprintf(os.Stdout, "# Run the above to set your docker environment\n")
	} else {
		fmt.Fprintf(os.Stdout, "source \"%v\"\n", path.Join(p, "docker.env"))
		fmt.Fprintf(os.Stdout, "# Run the above or eval a subshell with your arguments to %v\n", os.Args[0])
		fmt.Fprintf(os.Stdout, "# eval \"$( %v command... )\" \n", os.Args[0])
	}

	err = carina.TabWriter.Flush()
	return err
}

func writeCredentials(w *tabwriter.Writer, creds *libcarina.Credentials, pth string) (err error) {
	// TODO: Prompt when file already exists?
	for fname, b := range creds.Files {
		p := path.Join(pth, fname)
		err = ioutil.WriteFile(p, b, 0600)
		if err != nil {
			return err
		}
	}

	return nil
}

func writeCluster(w *tabwriter.Writer, cluster *libcarina.Cluster) (err error) {
	s := strings.Join([]string{cluster.ClusterName,
		cluster.Username,
		cluster.Flavor,
		cluster.Image,
		fmt.Sprintf("%v", cluster.Nodes),
		cluster.Status}, "\t")
	_, err = w.Write([]byte(s + "\n"))
	return
}

func main() {
	app := New()
	kingpin.MustParse(app.Parse(os.Args[1:]))
}
<commit_msg>Include the version and commit in --version<commit_after>package main

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"runtime"
	"strings"
	"text/tabwriter"
	"time"

	"gopkg.in/alecthomas/kingpin.v2"

	"github.com/rackerlabs/carina/version"
	"github.com/rackerlabs/libcarina"
)

// Application is, our, well, application
type Application struct {
	*Context
	*kingpin.Application
}

// Command is a command needing a ClusterClient
type Command struct {
	*Context
	*kingpin.CmdClause
}

// Context context for the  App
type Context struct {
	ClusterClient *libcarina.ClusterClient
	TabWriter     *tabwriter.Writer
	Username      string
	APIKey        string
	Endpoint      string
}

// ClusterCommand is a Command with a ClusterName set
type ClusterCommand struct {
	*Command
	ClusterName string
}

// CredentialsCommand keeps context about the download command
type CredentialsCommand struct {
	*ClusterCommand
	Path string
}

// CreateCommand keeps context about the create command
type CreateCommand struct {
	*ClusterCommand

	// Options passed along to Carina's API
	Nodes     int
	AutoScale bool

	// Whether to wait until the cluster is created (or errored)
	Wait bool

	// TODO: See if setting flavor or image makes sense, even if the API takes it
	// Flavor    string
	// Image     string
}

// GrowCommand keeps context about the number of nodes to scale by
type GrowCommand struct {
	*ClusterCommand
	Nodes int
}

// New creates a new Application
func New() *Application {

	app := kingpin.New("carina", "command line interface to launch and work with Docker Swarm clusters")
	app.Version(VersionString())

	cap := new(Application)
	ctx := new(Context)

	cap.Application = app

	cap.Context = ctx

	cap.Flag("username", "Rackspace username - can also set env var RACKSPACE_USERNAME").OverrideDefaultFromEnvar("RACKSPACE_USERNAME").StringVar(&ctx.Username)
	cap.Flag("api-key", "Rackspace API Key - can also set env var RACKSPACE_APIKEY").OverrideDefaultFromEnvar("RACKSPACE_APIKEY").PlaceHolder("RACKSPACE_APIKEY").StringVar(&ctx.APIKey)
	cap.Flag("endpoint", "Carina API endpoint").Default(libcarina.BetaEndpoint).StringVar(&ctx.Endpoint)

	writer := new(tabwriter.Writer)
	writer.Init(os.Stdout, 0, 8, 1, '\t', 0)

	ctx.TabWriter = writer

	listCommand := cap.NewCommand(ctx, "list", "list swarm clusters")
	listCommand.Action(listCommand.List)

	getCommand := cap.NewClusterCommand(ctx, "get", "get information about a swarm cluster")
	getCommand.Action(getCommand.Get)

	deleteCommand := cap.NewClusterCommand(ctx, "delete", "delete a swarm cluster")
	deleteCommand.Action(deleteCommand.Delete)

	createCommand := new(CreateCommand)
	createCommand.ClusterCommand = cap.NewClusterCommand(ctx, "create", "create a swarm cluster")
	createCommand.Flag("wait", "wait for swarm cluster completion").BoolVar(&createCommand.Wait)
	createCommand.Flag("nodes", "number of nodes for the initial cluster").Default("1").IntVar(&createCommand.Nodes)
	createCommand.Flag("autoscale", "whether autoscale is on or off").BoolVar(&createCommand.AutoScale)
	createCommand.Action(createCommand.Create)

	credentialsCommand := new(CredentialsCommand)
	credentialsCommand.ClusterCommand = cap.NewClusterCommand(ctx, "credentials", "download credentials")
	credentialsCommand.Flag("path", "path to write credentials out to").StringVar(&credentialsCommand.Path)
	credentialsCommand.Action(credentialsCommand.Download)

	growCommand := new(GrowCommand)
	growCommand.ClusterCommand = cap.NewClusterCommand(ctx, "grow", "Grow a cluster by the requested number of nodes")
	growCommand.Flag("nodes", "number of nodes to increase the cluster by").Required().IntVar(&growCommand.Nodes)
	growCommand.Action(growCommand.Grow)

	return cap
}

// VersionString returns the current version and commit of this binary (if set)
func VersionString() string {
	s := ""
	s += fmt.Sprintf("Version: %s\n", version.Version)
	s += fmt.Sprintf("Commit:  %s", version.Commit)
	return s
}

// NewCommand creates a command wrapped with carina.Context
func (app *Application) NewCommand(ctx *Context, name, help string) *Command {
	carina := new(Command)
	carina.Context = ctx
	carina.CmdClause = app.Command(name, help)
	carina.PreAction(carina.Auth)
	return carina
}

// NewClusterCommand is a command that uses a cluster name
func (app *Application) NewClusterCommand(ctx *Context, name, help string) *ClusterCommand {
	cc := new(ClusterCommand)
	cc.Command = app.NewCommand(ctx, name, help)
	cc.Arg("cluster-name", "name of the cluster").Required().StringVar(&cc.ClusterName)
	return cc
}

// Auth does the authentication
func (carina *Command) Auth(pc *kingpin.ParseContext) (err error) {
	carina.ClusterClient, err = libcarina.NewClusterClient(carina.Endpoint, carina.Username, carina.APIKey)
	return err
}

// List the current swarm clusters
func (carina *Command) List(pc *kingpin.ParseContext) (err error) {
	clusterList, err := carina.ClusterClient.List()
	if err != nil {
		return err
	}

	headerFields := []string{
		"ClusterName",
		"Username",
		"Flavor",
		"Image",
		"Nodes",
		"Status",
	}
	s := strings.Join(headerFields, "\t")

	_, err = carina.TabWriter.Write([]byte(s + "\n"))
	if err != nil {
		return err
	}

	for _, cluster := range clusterList {
		err = writeCluster(carina.TabWriter, &cluster)
		if err != nil {
			return err
		}
	}
	err = carina.TabWriter.Flush()
	return err
}

type clusterOp func(clusterName string) (*libcarina.Cluster, error)

// Does an func against a cluster then returns the new cluster representation
func (carina *ClusterCommand) clusterApply(op clusterOp) (err error) {
	cluster, err := op(carina.ClusterName)
	if err != nil {
		return err
	}

	err = writeCluster(carina.TabWriter, cluster)
	if err != nil {
		return err
	}
	return carina.TabWriter.Flush()
}

// Get an individual cluster
func (carina *ClusterCommand) Get(pc *kingpin.ParseContext) (err error) {
	return carina.clusterApply(carina.ClusterClient.Get)
}

// Delete a cluster
func (carina *ClusterCommand) Delete(pc *kingpin.ParseContext) (err error) {
	return carina.clusterApply(carina.ClusterClient.Delete)
}

// Grow increases the size of the given cluster
func (carina *GrowCommand) Grow(pc *kingpin.ParseContext) (err error) {
	return carina.clusterApply(func(clusterName string) (*libcarina.Cluster, error) {
		return carina.ClusterClient.Grow(clusterName, carina.Nodes)
	})
}

// Create a cluster
func (carina *CreateCommand) Create(pc *kingpin.ParseContext) (err error) {
	if carina.Nodes < 1 {
		return errors.New("nodes must be >= 1")
	}

	nodes := libcarina.Number(carina.Nodes)

	c := libcarina.Cluster{
		ClusterName: carina.ClusterName,
		Nodes:       nodes,
		AutoScale:   carina.AutoScale,
	}

	cluster, err := carina.ClusterClient.Create(c)

	// Transitions past point of "new" or "building" are assumed to be states we
	// can stop on.
	if carina.Wait {
		for cluster.Status == "new" || cluster.Status == "building" {
			time.Sleep(13 * time.Second)
			cluster, err = carina.ClusterClient.Get(carina.ClusterName)
			if err != nil {
				break
			}
		}
	}

	if err != nil {
		return err
	}

	err = writeCluster(carina.TabWriter, cluster)

	if err != nil {
		return err
	}
	return carina.TabWriter.Flush()
}

// Download credentials for a cluster
func (carina *CredentialsCommand) Download(pc *kingpin.ParseContext) (err error) {
	credentials, err := carina.ClusterClient.GetCredentials(carina.ClusterName)
	if err != nil {
		return err
	}

	p := path.Clean(carina.Path)

	if p != "." {
		err = os.MkdirAll(p, 0777)
	}

	if err != nil {
		return err
	}

	err = writeCredentials(carina.TabWriter, credentials, p)
	if err != nil {
		return err
	}

	if runtime.GOOS == "windows" {
		fmt.Fprintf(os.Stdout, "\"%v\"\n", path.Join(p, "docker.cmd"))
		fmt.Fprintf(os.Stdout, "# Run the above to set your docker environment\n")
	} else {
		fmt.Fprintf(os.Stdout, "source \"%v\"\n", path.Join(p, "docker.env"))
		fmt.Fprintf(os.Stdout, "# Run the above or eval a subshell with your arguments to %v\n", os.Args[0])
		fmt.Fprintf(os.Stdout, "# eval \"$( %v command... )\" \n", os.Args[0])
	}

	err = carina.TabWriter.Flush()
	return err
}

func writeCredentials(w *tabwriter.Writer, creds *libcarina.Credentials, pth string) (err error) {
	// TODO: Prompt when file already exists?
	for fname, b := range creds.Files {
		p := path.Join(pth, fname)
		err = ioutil.WriteFile(p, b, 0600)
		if err != nil {
			return err
		}
	}

	return nil
}

func writeCluster(w *tabwriter.Writer, cluster *libcarina.Cluster) (err error) {
	s := strings.Join([]string{cluster.ClusterName,
		cluster.Username,
		cluster.Flavor,
		cluster.Image,
		fmt.Sprintf("%v", cluster.Nodes),
		cluster.Status}, "\t")
	_, err = w.Write([]byte(s + "\n"))
	return
}

func main() {
	app := New()
	kingpin.MustParse(app.Parse(os.Args[1:]))
}
<|endoftext|>


<commit_before>// Copyright 2013 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"fmt"
	_ "net/http/pprof" // Comment this line to disable pprof endpoint.
	"os"
	"os/signal"
	"sort"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/prometheus/log"

	clientmodel "github.com/prometheus/client_golang/model"
	registry "github.com/prometheus/client_golang/prometheus"

	"github.com/prometheus/prometheus/config"
	"github.com/prometheus/prometheus/notification"
	"github.com/prometheus/prometheus/promql"
	"github.com/prometheus/prometheus/retrieval"
	"github.com/prometheus/prometheus/rules"
	"github.com/prometheus/prometheus/storage"
	"github.com/prometheus/prometheus/storage/local"
	"github.com/prometheus/prometheus/storage/remote"
	"github.com/prometheus/prometheus/storage/remote/influxdb"
	"github.com/prometheus/prometheus/storage/remote/opentsdb"
	"github.com/prometheus/prometheus/web"
	"github.com/prometheus/prometheus/web/api"
)

const deletionBatchSize = 100

// Commandline flags.
var (
	configFile = flag.String("config.file", "prometheus.yml", "Prometheus configuration file name.")

	alertmanagerURL           = flag.String("alertmanager.url", "", "The URL of the alert manager to send notifications to.")
	notificationQueueCapacity = flag.Int("alertmanager.notification-queue-capacity", 100, "The capacity of the queue for pending alert manager notifications.")

	persistenceStoragePath = flag.String("storage.local.path", "/tmp/metrics", "Base path for metrics storage.")

	opentsdbURL          = flag.String("storage.remote.opentsdb-url", "", "The URL of the remote OpenTSDB server to send samples to. None, if empty.")
	influxdbURL          = flag.String("storage.remote.influxdb-url", "", "The URL of the remote InfluxDB server to send samples to. None, if empty.")
	remoteStorageTimeout = flag.Duration("storage.remote.timeout", 30*time.Second, "The timeout to use when sending samples to the remote storage.")

	numMemoryChunks = flag.Int("storage.local.memory-chunks", 1024*1024, "How many chunks to keep in memory. While the size of a chunk is 1kiB, the total memory usage will be significantly higher than this value * 1kiB. Furthermore, for various reasons, more chunks might have to be kept in memory temporarily.")

	persistenceRetentionPeriod = flag.Duration("storage.local.retention", 15*24*time.Hour, "How long to retain samples in the local storage.")
	maxChunksToPersist         = flag.Int("storage.local.max-chunks-to-persist", 1024*1024, "How many chunks can be waiting for persistence before sample ingestion will stop. Many chunks waiting to be persisted will increase the checkpoint size.")

	checkpointInterval         = flag.Duration("storage.local.checkpoint-interval", 5*time.Minute, "The period at which the in-memory metrics and the chunks not yet persisted to series files are checkpointed.")
	checkpointDirtySeriesLimit = flag.Int("storage.local.checkpoint-dirty-series-limit", 5000, "If approx. that many time series are in a state that would require a recovery operation after a crash, a checkpoint is triggered, even if the checkpoint interval hasn't passed yet. A recovery operation requires a disk seek. The default limit intends to keep the recovery time below 1min even on spinning disks. With SSD, recovery is much faster, so you might want to increase this value in that case to avoid overly frequent checkpoints.")
	seriesSyncStrategy         = flag.String("storage.local.series-sync-strategy", "adaptive", "When to sync series files after modification. Possible values: 'never', 'always', 'adaptive'. Sync'ing slows down storage performance but reduces the risk of data loss in case of an OS crash. With the 'adaptive' strategy, series files are sync'd for as long as the storage is not too much behind on chunk persistence.")

	storageDirty          = flag.Bool("storage.local.dirty", false, "If set, the local storage layer will perform crash recovery even if the last shutdown appears to be clean.")
	storagePedanticChecks = flag.Bool("storage.local.pedantic-checks", false, "If set, a crash recovery will perform checks on each series file. This might take a very long time.")

	pathPrefix = flag.String("web.path-prefix", "", "Prefix for all web paths.")

	printVersion = flag.Bool("version", false, "Print version information.")
)

type prometheus struct {
	queryEngine         *promql.Engine
	ruleManager         *rules.Manager
	targetManager       *retrieval.TargetManager
	notificationHandler *notification.NotificationHandler
	storage             local.Storage
	remoteStorageQueues []*remote.StorageQueueManager

	webService *web.WebService

	closeOnce sync.Once
}

// NewPrometheus creates a new prometheus object based on flag values.
// Call Serve() to start serving and Close() for clean shutdown.
func NewPrometheus() *prometheus {
	notificationHandler := notification.NewNotificationHandler(*alertmanagerURL, *notificationQueueCapacity)

	var syncStrategy local.SyncStrategy
	switch *seriesSyncStrategy {
	case "never":
		syncStrategy = local.Never
	case "always":
		syncStrategy = local.Always
	case "adaptive":
		syncStrategy = local.Adaptive
	default:
		log.Errorf("Invalid flag value for 'storage.local.series-sync-strategy': %s\n", *seriesSyncStrategy)
		os.Exit(2)
	}

	o := &local.MemorySeriesStorageOptions{
		MemoryChunks:               *numMemoryChunks,
		MaxChunksToPersist:         *maxChunksToPersist,
		PersistenceStoragePath:     *persistenceStoragePath,
		PersistenceRetentionPeriod: *persistenceRetentionPeriod,
		CheckpointInterval:         *checkpointInterval,
		CheckpointDirtySeriesLimit: *checkpointDirtySeriesLimit,
		Dirty:          *storageDirty,
		PedanticChecks: *storagePedanticChecks,
		SyncStrategy:   syncStrategy,
	}
	memStorage := local.NewMemorySeriesStorage(o)

	var sampleAppender storage.SampleAppender
	var remoteStorageQueues []*remote.StorageQueueManager
	if *opentsdbURL == "" && *influxdbURL == "" {
		log.Warnf("No remote storage URLs provided; not sending any samples to long-term storage")
		sampleAppender = memStorage
	} else {
		fanout := storage.Fanout{memStorage}

		addRemoteStorage := func(c remote.StorageClient) {
			qm := remote.NewStorageQueueManager(c, 100*1024)
			fanout = append(fanout, qm)
			remoteStorageQueues = append(remoteStorageQueues, qm)
		}

		if *opentsdbURL != "" {
			addRemoteStorage(opentsdb.NewClient(*opentsdbURL, *remoteStorageTimeout))
		}
		if *influxdbURL != "" {
			addRemoteStorage(influxdb.NewClient(*influxdbURL, *remoteStorageTimeout))
		}

		sampleAppender = fanout
	}

	targetManager := retrieval.NewTargetManager(sampleAppender)

	queryEngine := promql.NewEngine(memStorage)

	ruleManager := rules.NewManager(&rules.ManagerOptions{
		SampleAppender:      sampleAppender,
		NotificationHandler: notificationHandler,
		QueryEngine:         queryEngine,
		PrometheusURL:       web.MustBuildServerURL(*pathPrefix),
		PathPrefix:          *pathPrefix,
	})

	flags := map[string]string{}
	flag.VisitAll(func(f *flag.Flag) {
		flags[f.Name] = f.Value.String()
	})
	prometheusStatus := &web.PrometheusStatusHandler{
		BuildInfo:   BuildInfo,
		RuleManager: ruleManager,
		TargetPools: targetManager.Pools,
		Flags:       flags,
		Birth:       time.Now(),
		PathPrefix:  *pathPrefix,
	}

	alertsHandler := &web.AlertsHandler{
		RuleManager: ruleManager,
		PathPrefix:  *pathPrefix,
	}

	consolesHandler := &web.ConsolesHandler{
		QueryEngine: queryEngine,
		PathPrefix:  *pathPrefix,
	}

	graphsHandler := &web.GraphsHandler{
		PathPrefix: *pathPrefix,
	}

	metricsService := &api.MetricsService{
		Now:         clientmodel.Now,
		Storage:     memStorage,
		QueryEngine: queryEngine,
	}

	webService := &web.WebService{
		StatusHandler:   prometheusStatus,
		MetricsHandler:  metricsService,
		ConsolesHandler: consolesHandler,
		AlertsHandler:   alertsHandler,
		GraphsHandler:   graphsHandler,
	}

	p := &prometheus{
		queryEngine:         queryEngine,
		ruleManager:         ruleManager,
		targetManager:       targetManager,
		notificationHandler: notificationHandler,
		storage:             memStorage,
		remoteStorageQueues: remoteStorageQueues,

		webService: webService,
	}
	webService.QuitChan = make(chan struct{})

	if !p.reloadConfig() {
		os.Exit(1)
	}

	return p
}

func (p *prometheus) reloadConfig() bool {
	log.Infof("Loading configuration file %s", *configFile)

	conf, err := config.LoadFromFile(*configFile)
	if err != nil {
		log.Errorf("Couldn't load configuration (-config.file=%s): %v", *configFile, err)
		log.Errorf("Note: The configuration format has changed with version 0.14, please check the documentation.")
		return false
	}

	p.webService.StatusHandler.ApplyConfig(conf)
	p.targetManager.ApplyConfig(conf)
	p.ruleManager.ApplyConfig(conf)

	return true
}

// Serve starts the Prometheus server. It returns after the server has been shut
// down. The method installs an interrupt handler, allowing to trigger a
// shutdown by sending SIGTERM to the process.
func (p *prometheus) Serve() {
	// Start all components.
	if err := p.storage.Start(); err != nil {
		log.Errorln("Error opening memory series storage:", err)
		os.Exit(1)
	}
	defer func() {
		if err := p.storage.Stop(); err != nil {
			log.Errorln("Error stopping storage:", err)
		}
	}()

	// The storage has to be fully initialized before registering Prometheus.
	registry.MustRegister(p)

	for _, q := range p.remoteStorageQueues {
		go q.Run()
		defer q.Stop()
	}

	go p.ruleManager.Run()
	defer p.ruleManager.Stop()

	go p.notificationHandler.Run()
	defer p.notificationHandler.Stop()

	go p.targetManager.Run()
	defer p.targetManager.Stop()

	defer p.queryEngine.Stop()

	go p.webService.ServeForever(*pathPrefix)

	// Wait for reload or termination signals.
	hup := make(chan os.Signal)
	signal.Notify(hup, syscall.SIGHUP)
	go func() {
		for range hup {
			p.reloadConfig()
		}
	}()

	term := make(chan os.Signal)
	signal.Notify(term, os.Interrupt, syscall.SIGTERM)
	select {
	case <-term:
		log.Warn("Received SIGTERM, exiting gracefully...")
	case <-p.webService.QuitChan:
		log.Warn("Received termination request via web service, exiting gracefully...")
	}

	close(hup)

	log.Info("See you next time!")
}

// Describe implements registry.Collector.
func (p *prometheus) Describe(ch chan<- *registry.Desc) {
	p.notificationHandler.Describe(ch)
	p.storage.Describe(ch)
	for _, q := range p.remoteStorageQueues {
		q.Describe(ch)
	}
}

// Collect implements registry.Collector.
func (p *prometheus) Collect(ch chan<- registry.Metric) {
	p.notificationHandler.Collect(ch)
	p.storage.Collect(ch)
	for _, q := range p.remoteStorageQueues {
		q.Collect(ch)
	}
}

func usage() {
	groups := make(map[string][]*flag.Flag)
	// Set a default group for ungrouped flags.
	groups["."] = make([]*flag.Flag, 0)

	// Bucket flags into groups based on the first of their dot-separated levels.
	flag.VisitAll(func(fl *flag.Flag) {
		parts := strings.SplitN(fl.Name, ".", 2)
		if len(parts) == 1 {
			groups["."] = append(groups["."], fl)
		} else {
			name := parts[0]
			groups[name] = append(groups[name], fl)
		}
	})

	groupsOrdered := make(sort.StringSlice, 0, len(groups))
	for groupName := range groups {
		groupsOrdered = append(groupsOrdered, groupName)
	}
	sort.Sort(groupsOrdered)

	fmt.Fprintf(os.Stderr, "Usage: %s [options ...]:\n\n", os.Args[0])

	const (
		maxLineLength = 80
		lineSep       = "\n      "
	)
	for _, groupName := range groupsOrdered {
		if groupName != "." {
			fmt.Fprintf(os.Stderr, "\n%s:\n", strings.Title(groupName))
		}

		for _, fl := range groups[groupName] {
			format := "  -%s=%s"
			if strings.Contains(fl.DefValue, " ") || fl.DefValue == "" {
				format = "  -%s=%q"
			}
			flagUsage := fmt.Sprintf(format+lineSep, fl.Name, fl.DefValue)

			// Format the usage text to not exceed maxLineLength characters per line.
			words := strings.SplitAfter(fl.Usage, " ")
			lineLength := len(lineSep) - 1
			for _, w := range words {
				if lineLength+len(w) > maxLineLength {
					flagUsage += lineSep
					lineLength = len(lineSep) - 1
				}
				flagUsage += w
				lineLength += len(w)
			}
			fmt.Fprintf(os.Stderr, "%s\n", flagUsage)
		}
	}
}

func main() {
	flag.CommandLine.Init(os.Args[0], flag.ContinueOnError)
	flag.CommandLine.Usage = usage

	if err := flag.CommandLine.Parse(os.Args[1:]); err != nil {
		if err != flag.ErrHelp {
			log.Errorf("Invalid command line arguments. Help: %s -h", os.Args[0])
		}
		os.Exit(2)
	}

	*pathPrefix = strings.TrimRight(*pathPrefix, "/")
	if *pathPrefix != "" && !strings.HasPrefix(*pathPrefix, "/") {
		*pathPrefix = "/" + *pathPrefix
	}

	versionInfoTmpl.Execute(os.Stdout, BuildInfo)

	if *printVersion {
		os.Exit(0)
	}

	p := NewPrometheus()
	p.Serve()
}
<commit_msg>Add links to configuration change notice.<commit_after>// Copyright 2013 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"fmt"
	_ "net/http/pprof" // Comment this line to disable pprof endpoint.
	"os"
	"os/signal"
	"sort"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/prometheus/log"

	clientmodel "github.com/prometheus/client_golang/model"
	registry "github.com/prometheus/client_golang/prometheus"

	"github.com/prometheus/prometheus/config"
	"github.com/prometheus/prometheus/notification"
	"github.com/prometheus/prometheus/promql"
	"github.com/prometheus/prometheus/retrieval"
	"github.com/prometheus/prometheus/rules"
	"github.com/prometheus/prometheus/storage"
	"github.com/prometheus/prometheus/storage/local"
	"github.com/prometheus/prometheus/storage/remote"
	"github.com/prometheus/prometheus/storage/remote/influxdb"
	"github.com/prometheus/prometheus/storage/remote/opentsdb"
	"github.com/prometheus/prometheus/web"
	"github.com/prometheus/prometheus/web/api"
)

const deletionBatchSize = 100

// Commandline flags.
var (
	configFile = flag.String("config.file", "prometheus.yml", "Prometheus configuration file name.")

	alertmanagerURL           = flag.String("alertmanager.url", "", "The URL of the alert manager to send notifications to.")
	notificationQueueCapacity = flag.Int("alertmanager.notification-queue-capacity", 100, "The capacity of the queue for pending alert manager notifications.")

	persistenceStoragePath = flag.String("storage.local.path", "/tmp/metrics", "Base path for metrics storage.")

	opentsdbURL          = flag.String("storage.remote.opentsdb-url", "", "The URL of the remote OpenTSDB server to send samples to. None, if empty.")
	influxdbURL          = flag.String("storage.remote.influxdb-url", "", "The URL of the remote InfluxDB server to send samples to. None, if empty.")
	remoteStorageTimeout = flag.Duration("storage.remote.timeout", 30*time.Second, "The timeout to use when sending samples to the remote storage.")

	numMemoryChunks = flag.Int("storage.local.memory-chunks", 1024*1024, "How many chunks to keep in memory. While the size of a chunk is 1kiB, the total memory usage will be significantly higher than this value * 1kiB. Furthermore, for various reasons, more chunks might have to be kept in memory temporarily.")

	persistenceRetentionPeriod = flag.Duration("storage.local.retention", 15*24*time.Hour, "How long to retain samples in the local storage.")
	maxChunksToPersist         = flag.Int("storage.local.max-chunks-to-persist", 1024*1024, "How many chunks can be waiting for persistence before sample ingestion will stop. Many chunks waiting to be persisted will increase the checkpoint size.")

	checkpointInterval         = flag.Duration("storage.local.checkpoint-interval", 5*time.Minute, "The period at which the in-memory metrics and the chunks not yet persisted to series files are checkpointed.")
	checkpointDirtySeriesLimit = flag.Int("storage.local.checkpoint-dirty-series-limit", 5000, "If approx. that many time series are in a state that would require a recovery operation after a crash, a checkpoint is triggered, even if the checkpoint interval hasn't passed yet. A recovery operation requires a disk seek. The default limit intends to keep the recovery time below 1min even on spinning disks. With SSD, recovery is much faster, so you might want to increase this value in that case to avoid overly frequent checkpoints.")
	seriesSyncStrategy         = flag.String("storage.local.series-sync-strategy", "adaptive", "When to sync series files after modification. Possible values: 'never', 'always', 'adaptive'. Sync'ing slows down storage performance but reduces the risk of data loss in case of an OS crash. With the 'adaptive' strategy, series files are sync'd for as long as the storage is not too much behind on chunk persistence.")

	storageDirty          = flag.Bool("storage.local.dirty", false, "If set, the local storage layer will perform crash recovery even if the last shutdown appears to be clean.")
	storagePedanticChecks = flag.Bool("storage.local.pedantic-checks", false, "If set, a crash recovery will perform checks on each series file. This might take a very long time.")

	pathPrefix = flag.String("web.path-prefix", "", "Prefix for all web paths.")

	printVersion = flag.Bool("version", false, "Print version information.")
)

type prometheus struct {
	queryEngine         *promql.Engine
	ruleManager         *rules.Manager
	targetManager       *retrieval.TargetManager
	notificationHandler *notification.NotificationHandler
	storage             local.Storage
	remoteStorageQueues []*remote.StorageQueueManager

	webService *web.WebService

	closeOnce sync.Once
}

// NewPrometheus creates a new prometheus object based on flag values.
// Call Serve() to start serving and Close() for clean shutdown.
func NewPrometheus() *prometheus {
	notificationHandler := notification.NewNotificationHandler(*alertmanagerURL, *notificationQueueCapacity)

	var syncStrategy local.SyncStrategy
	switch *seriesSyncStrategy {
	case "never":
		syncStrategy = local.Never
	case "always":
		syncStrategy = local.Always
	case "adaptive":
		syncStrategy = local.Adaptive
	default:
		log.Errorf("Invalid flag value for 'storage.local.series-sync-strategy': %s\n", *seriesSyncStrategy)
		os.Exit(2)
	}

	o := &local.MemorySeriesStorageOptions{
		MemoryChunks:               *numMemoryChunks,
		MaxChunksToPersist:         *maxChunksToPersist,
		PersistenceStoragePath:     *persistenceStoragePath,
		PersistenceRetentionPeriod: *persistenceRetentionPeriod,
		CheckpointInterval:         *checkpointInterval,
		CheckpointDirtySeriesLimit: *checkpointDirtySeriesLimit,
		Dirty:          *storageDirty,
		PedanticChecks: *storagePedanticChecks,
		SyncStrategy:   syncStrategy,
	}
	memStorage := local.NewMemorySeriesStorage(o)

	var sampleAppender storage.SampleAppender
	var remoteStorageQueues []*remote.StorageQueueManager
	if *opentsdbURL == "" && *influxdbURL == "" {
		log.Warnf("No remote storage URLs provided; not sending any samples to long-term storage")
		sampleAppender = memStorage
	} else {
		fanout := storage.Fanout{memStorage}

		addRemoteStorage := func(c remote.StorageClient) {
			qm := remote.NewStorageQueueManager(c, 100*1024)
			fanout = append(fanout, qm)
			remoteStorageQueues = append(remoteStorageQueues, qm)
		}

		if *opentsdbURL != "" {
			addRemoteStorage(opentsdb.NewClient(*opentsdbURL, *remoteStorageTimeout))
		}
		if *influxdbURL != "" {
			addRemoteStorage(influxdb.NewClient(*influxdbURL, *remoteStorageTimeout))
		}

		sampleAppender = fanout
	}

	targetManager := retrieval.NewTargetManager(sampleAppender)

	queryEngine := promql.NewEngine(memStorage)

	ruleManager := rules.NewManager(&rules.ManagerOptions{
		SampleAppender:      sampleAppender,
		NotificationHandler: notificationHandler,
		QueryEngine:         queryEngine,
		PrometheusURL:       web.MustBuildServerURL(*pathPrefix),
		PathPrefix:          *pathPrefix,
	})

	flags := map[string]string{}
	flag.VisitAll(func(f *flag.Flag) {
		flags[f.Name] = f.Value.String()
	})
	prometheusStatus := &web.PrometheusStatusHandler{
		BuildInfo:   BuildInfo,
		RuleManager: ruleManager,
		TargetPools: targetManager.Pools,
		Flags:       flags,
		Birth:       time.Now(),
		PathPrefix:  *pathPrefix,
	}

	alertsHandler := &web.AlertsHandler{
		RuleManager: ruleManager,
		PathPrefix:  *pathPrefix,
	}

	consolesHandler := &web.ConsolesHandler{
		QueryEngine: queryEngine,
		PathPrefix:  *pathPrefix,
	}

	graphsHandler := &web.GraphsHandler{
		PathPrefix: *pathPrefix,
	}

	metricsService := &api.MetricsService{
		Now:         clientmodel.Now,
		Storage:     memStorage,
		QueryEngine: queryEngine,
	}

	webService := &web.WebService{
		StatusHandler:   prometheusStatus,
		MetricsHandler:  metricsService,
		ConsolesHandler: consolesHandler,
		AlertsHandler:   alertsHandler,
		GraphsHandler:   graphsHandler,
	}

	p := &prometheus{
		queryEngine:         queryEngine,
		ruleManager:         ruleManager,
		targetManager:       targetManager,
		notificationHandler: notificationHandler,
		storage:             memStorage,
		remoteStorageQueues: remoteStorageQueues,

		webService: webService,
	}
	webService.QuitChan = make(chan struct{})

	if !p.reloadConfig() {
		os.Exit(1)
	}

	return p
}

func (p *prometheus) reloadConfig() bool {
	log.Infof("Loading configuration file %s", *configFile)

	conf, err := config.LoadFromFile(*configFile)
	if err != nil {
		log.Errorf("Couldn't load configuration (-config.file=%s): %v", *configFile, err)
		log.Errorf("Note: The configuration format has changed with version 0.14. Please see the documentation (http://prometheus.io/docs/operating/configuration/) and the provided configuration migration tool (https://github.com/prometheus/migrate).")
		return false
	}

	p.webService.StatusHandler.ApplyConfig(conf)
	p.targetManager.ApplyConfig(conf)
	p.ruleManager.ApplyConfig(conf)

	return true
}

// Serve starts the Prometheus server. It returns after the server has been shut
// down. The method installs an interrupt handler, allowing to trigger a
// shutdown by sending SIGTERM to the process.
func (p *prometheus) Serve() {
	// Start all components.
	if err := p.storage.Start(); err != nil {
		log.Errorln("Error opening memory series storage:", err)
		os.Exit(1)
	}
	defer func() {
		if err := p.storage.Stop(); err != nil {
			log.Errorln("Error stopping storage:", err)
		}
	}()

	// The storage has to be fully initialized before registering Prometheus.
	registry.MustRegister(p)

	for _, q := range p.remoteStorageQueues {
		go q.Run()
		defer q.Stop()
	}

	go p.ruleManager.Run()
	defer p.ruleManager.Stop()

	go p.notificationHandler.Run()
	defer p.notificationHandler.Stop()

	go p.targetManager.Run()
	defer p.targetManager.Stop()

	defer p.queryEngine.Stop()

	go p.webService.ServeForever(*pathPrefix)

	// Wait for reload or termination signals.
	hup := make(chan os.Signal)
	signal.Notify(hup, syscall.SIGHUP)
	go func() {
		for range hup {
			p.reloadConfig()
		}
	}()

	term := make(chan os.Signal)
	signal.Notify(term, os.Interrupt, syscall.SIGTERM)
	select {
	case <-term:
		log.Warn("Received SIGTERM, exiting gracefully...")
	case <-p.webService.QuitChan:
		log.Warn("Received termination request via web service, exiting gracefully...")
	}

	close(hup)

	log.Info("See you next time!")
}

// Describe implements registry.Collector.
func (p *prometheus) Describe(ch chan<- *registry.Desc) {
	p.notificationHandler.Describe(ch)
	p.storage.Describe(ch)
	for _, q := range p.remoteStorageQueues {
		q.Describe(ch)
	}
}

// Collect implements registry.Collector.
func (p *prometheus) Collect(ch chan<- registry.Metric) {
	p.notificationHandler.Collect(ch)
	p.storage.Collect(ch)
	for _, q := range p.remoteStorageQueues {
		q.Collect(ch)
	}
}

func usage() {
	groups := make(map[string][]*flag.Flag)
	// Set a default group for ungrouped flags.
	groups["."] = make([]*flag.Flag, 0)

	// Bucket flags into groups based on the first of their dot-separated levels.
	flag.VisitAll(func(fl *flag.Flag) {
		parts := strings.SplitN(fl.Name, ".", 2)
		if len(parts) == 1 {
			groups["."] = append(groups["."], fl)
		} else {
			name := parts[0]
			groups[name] = append(groups[name], fl)
		}
	})

	groupsOrdered := make(sort.StringSlice, 0, len(groups))
	for groupName := range groups {
		groupsOrdered = append(groupsOrdered, groupName)
	}
	sort.Sort(groupsOrdered)

	fmt.Fprintf(os.Stderr, "Usage: %s [options ...]:\n\n", os.Args[0])

	const (
		maxLineLength = 80
		lineSep       = "\n      "
	)
	for _, groupName := range groupsOrdered {
		if groupName != "." {
			fmt.Fprintf(os.Stderr, "\n%s:\n", strings.Title(groupName))
		}

		for _, fl := range groups[groupName] {
			format := "  -%s=%s"
			if strings.Contains(fl.DefValue, " ") || fl.DefValue == "" {
				format = "  -%s=%q"
			}
			flagUsage := fmt.Sprintf(format+lineSep, fl.Name, fl.DefValue)

			// Format the usage text to not exceed maxLineLength characters per line.
			words := strings.SplitAfter(fl.Usage, " ")
			lineLength := len(lineSep) - 1
			for _, w := range words {
				if lineLength+len(w) > maxLineLength {
					flagUsage += lineSep
					lineLength = len(lineSep) - 1
				}
				flagUsage += w
				lineLength += len(w)
			}
			fmt.Fprintf(os.Stderr, "%s\n", flagUsage)
		}
	}
}

func main() {
	flag.CommandLine.Init(os.Args[0], flag.ContinueOnError)
	flag.CommandLine.Usage = usage

	if err := flag.CommandLine.Parse(os.Args[1:]); err != nil {
		if err != flag.ErrHelp {
			log.Errorf("Invalid command line arguments. Help: %s -h", os.Args[0])
		}
		os.Exit(2)
	}

	*pathPrefix = strings.TrimRight(*pathPrefix, "/")
	if *pathPrefix != "" && !strings.HasPrefix(*pathPrefix, "/") {
		*pathPrefix = "/" + *pathPrefix
	}

	versionInfoTmpl.Execute(os.Stdout, BuildInfo)

	if *printVersion {
		os.Exit(0)
	}

	p := NewPrometheus()
	p.Serve()
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/jawher/mow.cli"
	"io"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"strings"
	"sync"
	"time"
)

func main() {

	app := cli.App("up-restutil", "A RESTful resource utility")

	app.Command("put-resources", "read json resources from stdin and PUT them to an endpoint", func(cmd *cli.Cmd) {
		idProp := cmd.StringArg("IDPROP", "", "property name of identity property")
		baseUrl := cmd.StringArg("BASEURL", "", "base URL to PUT resources to")
		cmd.Action = func() {
			if err := putAllRest(*baseUrl, *idProp, 32); err != nil {
				log.Fatal(err)
			}
		}

	})

	app.Run(os.Args)
}

func putAllRest(baseurl string, idProperty string, conns int) error {

	dec := json.NewDecoder(os.Stdin)

	docs := make(chan resource)

	httpClient := &http.Client{
		Transport: &http.Transport{
			MaxIdleConnsPerHost: conns,
			Dial: (&net.Dialer{
				Timeout:   30 * time.Second,
				KeepAlive: 30 * time.Second,
			}).Dial,
		},
	}

	rp := &resourcePutter{baseurl, idProperty, httpClient}

	errs := make(chan error, 1)

	wg := sync.WaitGroup{}
	for i := 0; i < conns; i++ {
		wg.Add(1)
		go func() {
			if err := rp.putAll(docs); err != nil {
				select {
				case errs <- err:
				default:
				}
			}
			wg.Done()
		}()
	}

	for {
		var doc map[string]interface{}
		if err := dec.Decode(&doc); err != nil {
			if err == io.EOF {
				break
			}
			return err
		}
		select {
		case docs <- doc:
		case err := <-errs:
			return err
		}
	}

	close(docs)

	wg.Wait()

	select {
	case err := <-errs:
		return err
	default:
		return nil
	}

}

func (rp *resourcePutter) putAll(resources <-chan resource) error {
	for r := range resources {
		id := r[rp.idProperty]
		idStr, ok := id.(string)
		if !ok {
			log.Printf("unable to extract id property from resource, skipping")
		}

		msg, err := json.Marshal(r)
		if err != nil {
			return err
		}
		b := rp.baseUrl
		if !strings.HasSuffix(b, "/") {
			b = b + "/"
		}
		u, err := url.Parse(b)
		if err != nil {
			return err
		}
		u, err = u.Parse(idStr)
		if err != nil {
			return err
		}
		req, err := http.NewRequest("PUT", u.String(), bytes.NewReader(msg))
		if err != nil {
			return err
		}
		resp, err := rp.client.Do(req)
		if err != nil {
			return err
		}
		resp.Body.Close()
		if resp.StatusCode != 200 && resp.StatusCode != 202 {
			return fmt.Errorf("http fail: %v for request %v", resp.Status, req)
		}
	}
	return nil
}

type resource map[string]interface{}

type resourcePutter struct {
	baseUrl    string
	idProperty string
	client     *http.Client
}
<commit_msg>better failure message<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/jawher/mow.cli"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"strings"
	"sync"
	"time"
)

func main() {

	app := cli.App("up-restutil", "A RESTful resource utility")

	app.Command("put-resources", "read json resources from stdin and PUT them to an endpoint", func(cmd *cli.Cmd) {
		idProp := cmd.StringArg("IDPROP", "", "property name of identity property")
		baseUrl := cmd.StringArg("BASEURL", "", "base URL to PUT resources to")
		cmd.Action = func() {
			if err := putAllRest(*baseUrl, *idProp, 32); err != nil {
				log.Fatal(err)
			}
		}

	})

	app.Run(os.Args)
}

func putAllRest(baseurl string, idProperty string, conns int) error {

	dec := json.NewDecoder(os.Stdin)

	docs := make(chan resource)

	httpClient := &http.Client{
		Transport: &http.Transport{
			MaxIdleConnsPerHost: conns,
			Dial: (&net.Dialer{
				Timeout:   30 * time.Second,
				KeepAlive: 30 * time.Second,
			}).Dial,
		},
	}

	rp := &resourcePutter{baseurl, idProperty, httpClient}

	errs := make(chan error, 1)

	wg := sync.WaitGroup{}
	for i := 0; i < conns; i++ {
		wg.Add(1)
		go func() {
			if err := rp.putAll(docs); err != nil {
				select {
				case errs <- err:
				default:
				}
			}
			wg.Done()
		}()
	}

	for {
		var doc map[string]interface{}
		if err := dec.Decode(&doc); err != nil {
			if err == io.EOF {
				break
			}
			return err
		}
		select {
		case docs <- doc:
		case err := <-errs:
			return err
		}
	}

	close(docs)

	wg.Wait()

	select {
	case err := <-errs:
		return err
	default:
		return nil
	}

}

func (rp *resourcePutter) putAll(resources <-chan resource) error {
	for r := range resources {
		id := r[rp.idProperty]
		idStr, ok := id.(string)
		if !ok {
			log.Printf("unable to extract id property from resource, skipping")
		}

		msg, err := json.Marshal(r)
		if err != nil {
			return err
		}
		b := rp.baseUrl
		if !strings.HasSuffix(b, "/") {
			b = b + "/"
		}
		u, err := url.Parse(b)
		if err != nil {
			return err
		}
		u, err = u.Parse(idStr)
		if err != nil {
			return err
		}
		req, err := http.NewRequest("PUT", u.String(), bytes.NewReader(msg))
		if err != nil {
			return err
		}
		resp, err := rp.client.Do(req)
		if err != nil {
			return err
		}
		contents, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return err
		}
		resp.Body.Close()
		if resp.StatusCode != 200 && resp.StatusCode != 202 {
			return fmt.Errorf("http fail: %v :\n%s\n", resp.Status, contents)
		}

	}
	return nil
}

type resource map[string]interface{}

type resourcePutter struct {
	baseUrl    string
	idProperty string
	client     *http.Client
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
)

type PostCache struct {
	Url   string
	Items map[string]*item
}

var postCache = PostCache{Items: map[string]*item{}}

const DEFAULT_FOLDER = "<from title>"
const HOST_HACKER_NEWS = "news.ycombinator.com"

var Verbose *bool
var Quiet *bool

func main() {
	cacheFile := flag.String("readfrom", "", "Read posts from a json file instead of querying firebase (mainly used for testing)")
	dump := flag.Bool("dump", false, "Fetch the given url only and dump the response. Do not fetch its children")
	folder := flag.String("o", DEFAULT_FOLDER, "Save mails into the given directory instead of the default one")
	Verbose = flag.Bool("v", false, "Verbose")
	Quiet = flag.Bool("q", false, "Be quiet")
	flag.Parse()
	if *Quiet {
		*Verbose = false
	}

	var arg string
	if *cacheFile != "" {
		if fileExists(*cacheFile) {
			readCacheFile(*cacheFile)
		} else {
			bail("readfrom: file %s does not exist", *cacheFile)
		}
		arg = postCache.Url
	} else {
		if flag.NArg() != 1 {
			printError("Usage: fetchpost <url|dir>")
			os.Exit(1)
		}
		arg = flag.Arg(0)
	}

	var url string
	if strings.HasPrefix(arg, "https://") {
		url = arg
		postCache.Url = url
	} else {
		url = readUrlFile(arg)
		if strings.HasPrefix(url, "https://") {
			postCache.Url = url
		}
	}

	/**
	1. Get post
	2. If mail for post doesn't exist:
		2.1 write mail in maildir
	3. else if post body or title changed:
		3.1 Update subject or body and mark mail as new
	4. Repeat from step 1 for each kid in post.
	*/
	post := mustGetPost(url)
	if *dump {
		buf, err := json.Marshal(post)
		if err != nil {
			bail("dump: %v", err)
		}
		fmt.Printf("%s\n", string(buf))
		os.Exit(0)
	}

	added, updated, deleted := saveComments(post, arg, url, *folder)
	if !*Quiet {
		fmt.Printf("Comments: %d added, %d updated, %d deleted\n",
			added, updated, deleted)
	}

	postDir := getPostDir(post)
	fmt.Println(postDir)
	postTestFile := filepath.Join(postDir, postDir+".json")
	buf, err := json.Marshal(postCache)
	if err != nil {
		printError("Marshal postCache: %v", err)
	}
	err = ioutil.WriteFile(postTestFile, append(buf, '\n'), 0644)
	if err != nil {
		printError("Write postCache: %v", err)
	}
}

func saveComments(post Post, arg, url string, folder string) (added, updated, deleted int) {
	dir := getPostDir(post)
	if folder == DEFAULT_FOLDER && arg != url && arg != dir {
		renamePostDir(arg, dir)
	}

	mails, err := NewMails(dir)
	if err != nil {
		bail("saveComments(): error: %v", err)
	}
	maybeSavePostUrl(dir, url)
	maybeReportScoreChange(mails, post)
	return mails.SavePost(nil, post, 0)
}

func maybeReportScoreChange(mails *Mails, post Post) {
	mid := post.MessageId()
	key, exist := mails.keyForMessageId[mid]
	if !exist {
		return
	}

	msg := mails.mustGetMessage(key)
	score, exist := msg.Header["X-Score"]
	if !exist || len(score[0]) == 0 {
		printError("X-Score header missing in %s", key)
		return
	}
	oldScore, err := strconv.Atoi(score[0])
	if err != nil {
		printError("X-Score: %v", err)
		return
	}
	newScore := post.Points()

	if oldScore != newScore {
		fmt.Printf("Points changed from %d to %d\n", oldScore, newScore)
	}
}

func renamePostDir(olddir, newdir string) {
	if fileExists(newdir) {
		bail("renamePostDir(): post title changed but directory for new title"+
			" already exist\n\told title: %s\n\tnew title: %s",
			olddir, newdir)
	}
	err := os.Rename(olddir, newdir)
	if err != nil {
		bail("renamePostDir(): Rename: %v", err)
	}
	printNotice("post title has changed:\n\told: %s\n\tnew: %s", olddir, newdir)

}

func readCacheFile(f string) {
	buf, err := ioutil.ReadFile(f)
	if err != nil {
		bail("Read %s: %v", f, err)
	}
	err = json.Unmarshal(buf, &postCache)
	if err != nil {
		bail("Unmarshal %s: %v", f, err)
	}
}

func fileExists(path string) bool {
	_, err := os.Stat(path)
	if err != nil {
		if os.IsNotExist(err) {
			return false
		} else {
			bail("fileExists: %s: %v", path, err)
		}
	}
	return true
}

func (mails *Mails) SavePost(parent, child Post, threadLevel int) (added, updated, deleted int) {
	mid := child.MessageId()
	key, exist := mails.keyForMessageId[mid]
	if exist {
		msg, err := mails.md.Message(key)
		if err != nil {
			bail("SavePost(): Error getting mail for key %s: %v", key, err)
		}

		path, err := mails.md.Filename(key)
		if err != nil {
			bail("SavePost(): could not get Filename for key %s", key)
		}
		contentChanged := updateMail(parent, child, path, msg)
		if contentChanged {
			p := strings.Replace(path, "/cur/", "/new/", 1)
			if path != p {
				err = os.Rename(path, p)
				if err != nil {
					bail("SavePost(): Rename(%s, %s): %v", path, p, err)
				}
				verbose("Renamed %s to %s", path, p)
			}
			if child.Removed() {
				deleted++
			} else {
				updated++
			}
		}
		showThreadLevel(threadLevel, "*")
	} else {
		mail := NewMail(parent, child)
		err := mails.SaveMail(mail)
		if err != nil {
			bail("SavePost(): %v", err)
		}
		showThreadLevel(threadLevel, "+")
		added++
	}

	n, u, d := mails.SavePostChildren(child, threadLevel+1)
	added += n
	updated += u
	deleted += d
	return
}

func (mails *Mails) SavePostChildren(p Post, threadLevel int) (added, updated, deleted int) {
	for _, idStr := range p.Children() {
		child, err := fetchHNPost(idStr)
		if err != nil {
			printError("error: %v", err)
			continue
		}
		a, u, d := mails.SavePost(p, child, threadLevel)
		added += a
		updated += u
		deleted += d
	}
	return
}

func showThreadLevel(level int, symbol string) {
	for i := 0; i < level; i++ {
		fmt.Print(" ")
	}
	fmt.Println(symbol)
}

func maybeSavePostUrl(dir, url string) {
	if !fileExists(dir) {
		os.Mkdir(dir, 0755)
	}
	f := filepath.Join(dir, "url.txt")
	if fileExists(f) {
		return
	}
	err := ioutil.WriteFile(f, []byte(url), 0644)
	if err != nil {
		bail("%s: %v", f, err)
	}
}

func mustGetPost(href string) Post {
	url, err := url.Parse(href)
	if err != nil {
		bail("bad url: %s", err)
	}

	switch strings.ToLower(url.Host) {
	case HOST_HACKER_NEWS:
		return GetHNPost(url)
	default:
		bail("Unsupported url: %s", url.Host)
	}
	return nil
}

func bail(format string, args ...interface{}) {
	printError(format, args...)
	os.Exit(1)
}

func readUrlFile(arg string) string {
	urlFile := filepath.Join(arg, "url.txt")
	buf, err := ioutil.ReadFile(urlFile)
	if err != nil {
		bail("readUrlFile: error: %v", err)
	}
	return string(buf)
}

func verbose(format string, args ...interface{}) {
	if !*Verbose {
		return
	}
	fmt.Fprintf(os.Stderr, format+"\n", args...)
}

func printError(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, format+"\n", args...)
}

func printNotice(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, "notice:"+format+"\n", args...)
}
<commit_msg>Support /path/to/post_dir argument<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
)

type PostCache struct {
	Url   string
	Items map[string]*item
}

var postCache = PostCache{Items: map[string]*item{}}

const DEFAULT_FOLDER = "<from title>"
const HOST_HACKER_NEWS = "news.ycombinator.com"

var Verbose *bool
var Quiet *bool

func main() {
	cacheFile := flag.String("readfrom", "", "Read posts from a json file instead of querying firebase (mainly used for testing)")
	dump := flag.Bool("dump", false, "Fetch the given url only and dump the response. Do not fetch its children")
	folder := flag.String("o", DEFAULT_FOLDER, "Save mails into the given directory instead of the default one")
	Verbose = flag.Bool("v", false, "Verbose")
	Quiet = flag.Bool("q", false, "Be quiet")
	flag.Parse()
	if *Quiet {
		*Verbose = false
	}

	var arg string
	if *cacheFile != "" {
		if fileExists(*cacheFile) {
			readCacheFile(*cacheFile)
		} else {
			bail("readfrom: file %s does not exist", *cacheFile)
		}
		arg = postCache.Url
	} else {
		if flag.NArg() != 1 {
			printError("Usage: fetchpost <url|dir>")
			os.Exit(1)
		}
		arg = flag.Arg(0)
	}

	var url string
	if strings.HasPrefix(arg, "https://") {
		url = arg
		postCache.Url = url
	} else {
		dir, file := filepath.Split(filepath.Clean(arg))
		if dir != "" {
			err := os.Chdir(dir)
			if err != nil {
				bail(err.Error())
			}
		}
		arg = file
		url = readUrlFile(arg)
		if strings.HasPrefix(url, "https://") {
			postCache.Url = url
		}
	}

	/**
	1. Get post
	2. If mail for post doesn't exist:
		2.1 write mail in maildir
	3. else if post body or title changed:
		3.1 Update subject or body and mark mail as new
	4. Repeat from step 1 for each kid in post.
	*/
	post := mustGetPost(url)
	if *dump {
		buf, err := json.Marshal(post)
		if err != nil {
			bail("dump: %v", err)
		}
		fmt.Printf("%s\n", string(buf))
		os.Exit(0)
	}

	added, updated, deleted := saveComments(post, arg, url, *folder)
	if !*Quiet {
		fmt.Printf("Comments: %d added, %d updated, %d deleted\n",
			added, updated, deleted)
	}

	postDir := getPostDir(post)
	fmt.Println(postDir)
	postTestFile := filepath.Join(postDir, postDir+".json")
	buf, err := json.Marshal(postCache)
	if err != nil {
		printError("Marshal postCache: %v", err)
	}
	err = ioutil.WriteFile(postTestFile, append(buf, '\n'), 0644)
	if err != nil {
		printError("Write postCache: %v", err)
	}
}

func saveComments(post Post, arg, url string, folder string) (added, updated, deleted int) {
	dir := getPostDir(post)
	if folder == DEFAULT_FOLDER && arg != url && arg != dir {
		renamePostDir(arg, dir)
	}

	mails, err := NewMails(dir)
	if err != nil {
		bail("saveComments(): error: %v", err)
	}
	maybeSavePostUrl(dir, url)
	maybeReportScoreChange(mails, post)
	return mails.SavePost(nil, post, 0)
}

func maybeReportScoreChange(mails *Mails, post Post) {
	mid := post.MessageId()
	key, exist := mails.keyForMessageId[mid]
	if !exist {
		return
	}

	msg := mails.mustGetMessage(key)
	score, exist := msg.Header["X-Score"]
	if !exist || len(score[0]) == 0 {
		printError("X-Score header missing in %s", key)
		return
	}
	oldScore, err := strconv.Atoi(score[0])
	if err != nil {
		printError("X-Score: %v", err)
		return
	}
	newScore := post.Points()

	if oldScore != newScore {
		fmt.Printf("Points changed from %d to %d\n", oldScore, newScore)
	}
}

func renamePostDir(olddir, newdir string) {
	if fileExists(newdir) {
		bail("renamePostDir(): post title changed but directory for new title"+
			" already exist\n\told title: %s\n\tnew title: %s",
			olddir, newdir)
	}
	err := os.Rename(olddir, newdir)
	if err != nil {
		bail("renamePostDir(): Rename: %v", err)
	}
	printNotice("post title has changed:\n\told: %s\n\tnew: %s", olddir, newdir)

}

func readCacheFile(f string) {
	buf, err := ioutil.ReadFile(f)
	if err != nil {
		bail("Read %s: %v", f, err)
	}
	err = json.Unmarshal(buf, &postCache)
	if err != nil {
		bail("Unmarshal %s: %v", f, err)
	}
}

func fileExists(path string) bool {
	_, err := os.Stat(path)
	if err != nil {
		if os.IsNotExist(err) {
			return false
		} else {
			bail("fileExists: %s: %v", path, err)
		}
	}
	return true
}

func (mails *Mails) SavePost(parent, child Post, threadLevel int) (added, updated, deleted int) {
	mid := child.MessageId()
	key, exist := mails.keyForMessageId[mid]
	if exist {
		msg, err := mails.md.Message(key)
		if err != nil {
			bail("SavePost(): Error getting mail for key %s: %v", key, err)
		}

		path, err := mails.md.Filename(key)
		if err != nil {
			bail("SavePost(): could not get Filename for key %s", key)
		}
		contentChanged := updateMail(parent, child, path, msg)
		if contentChanged {
			p := strings.Replace(path, "/cur/", "/new/", 1)
			if path != p {
				err = os.Rename(path, p)
				if err != nil {
					bail("SavePost(): Rename(%s, %s): %v", path, p, err)
				}
				verbose("Renamed %s to %s", path, p)
			}
			if child.Removed() {
				deleted++
			} else {
				updated++
			}
		}
		showThreadLevel(threadLevel, "*")
	} else {
		mail := NewMail(parent, child)
		err := mails.SaveMail(mail)
		if err != nil {
			bail("SavePost(): %v", err)
		}
		showThreadLevel(threadLevel, "+")
		added++
	}

	n, u, d := mails.SavePostChildren(child, threadLevel+1)
	added += n
	updated += u
	deleted += d
	return
}

func (mails *Mails) SavePostChildren(p Post, threadLevel int) (added, updated, deleted int) {
	for _, idStr := range p.Children() {
		child, err := fetchHNPost(idStr)
		if err != nil {
			printError("error: %v", err)
			continue
		}
		a, u, d := mails.SavePost(p, child, threadLevel)
		added += a
		updated += u
		deleted += d
	}
	return
}

func showThreadLevel(level int, symbol string) {
	for i := 0; i < level; i++ {
		fmt.Print(" ")
	}
	fmt.Println(symbol)
}

func maybeSavePostUrl(dir, url string) {
	if !fileExists(dir) {
		os.Mkdir(dir, 0755)
	}
	f := filepath.Join(dir, "url.txt")
	if fileExists(f) {
		return
	}
	err := ioutil.WriteFile(f, []byte(url), 0644)
	if err != nil {
		bail("%s: %v", f, err)
	}
}

func mustGetPost(href string) Post {
	url, err := url.Parse(href)
	if err != nil {
		bail("bad url: %s", err)
	}

	switch strings.ToLower(url.Host) {
	case HOST_HACKER_NEWS:
		return GetHNPost(url)
	default:
		bail("Unsupported url: %s", url.Host)
	}
	return nil
}

func bail(format string, args ...interface{}) {
	printError(format, args...)
	os.Exit(1)
}

func readUrlFile(arg string) string {
	urlFile := filepath.Join(arg, "url.txt")
	buf, err := ioutil.ReadFile(urlFile)
	if err != nil {
		bail("readUrlFile: error: %v", err)
	}
	return string(buf)
}

func verbose(format string, args ...interface{}) {
	if !*Verbose {
		return
	}
	fmt.Fprintf(os.Stderr, format+"\n", args...)
}

func printError(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, format+"\n", args...)
}

func printNotice(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, "notice:"+format+"\n", args...)
}
<|endoftext|>