<commit_before>package main

import (
	golog "log"
	"net"
	"net/http"
	"os"

	"github.com/NYTimes/gziphandler"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3iface"
	"github.com/gorilla/handlers"
	"github.com/gorilla/mux"
	"github.com/namsral/flag"
	"github.com/oxtoacart/bpool"

	"github.com/tilezen/tapalcatl/pkg/buffer"
	"github.com/tilezen/tapalcatl/pkg/config"
	"github.com/tilezen/tapalcatl/pkg/handler"
	"github.com/tilezen/tapalcatl/pkg/log"
	"github.com/tilezen/tapalcatl/pkg/metrics"
	"github.com/tilezen/tapalcatl/pkg/storage"
	"github.com/tilezen/tapalcatl/pkg/tile"
)

func main() {
	var listen, healthcheck string
	var poolNumEntries, poolEntrySize int
	var metricsStatsdAddr, metricsStatsdPrefix string

	hc := config.HandlerConfig{}

	systemLogger := golog.New(os.Stdout, "", golog.LstdFlags|golog.LUTC|golog.Lmicroseconds)
	hostname, err := os.Hostname()
	if err != nil {
		// NOTE: if there are legitimate cases when this can fail, we
		// can leave off the hostname in the logger.
		// But for now we prefer to get notified of it.
		systemLogger.Fatalf("ERROR: Cannot find hostname to use for logger")
	}
	// use this logger everywhere.
	logger := log.NewJsonLogger(systemLogger, hostname)

	f := flag.NewFlagSetWithEnvPrefix(os.Args[0], "TAPALCATL", 0)
	f.Var(&hc, "handler",
		`JSON object defining how request patterns will be handled.
	 Aws { Object present when Aws-wide configuration is needed, eg session config.
     Region string Name of aws region
   }
   Storage { key -> storage definition mapping
     storage name string -> {
        Type string storage type, can be "s3" or "file
        MetatileSize int      Number of 256px tiles in each dimension of the metatile.
        TileSize int        Size of tile in 256px tile units.

       (s3 storage)
        Layer      string   Name of layer to use in this bucket. Only relevant for s3.
        Bucket     string   Name of S3 bucket to fetch from.
        KeyPattern string   Pattern to fill with variables from the main pattern to make the S3 key.
        Healthcheck string Name of S3 key to use when querying health of S3 system.

       (file storage)
        BaseDir    string   Base directory to look for files under.
        Healthcheck string  Path to a file (inside BaseDir) when querying health of system.
     }
   }
   Pattern { request pattern -> storage configuration mapping
     request pattern string -> {
       storage string Name of storage defintion to use
       list of optional storage configuration to use:
       prefix is required for s3, others are optional overrides of relevant definition
     	 Prefix string  Prefix to use in this bucket.
     }
   }
   Mime { extension -> content-type used in http response
   }
`)
	f.StringVar(&listen, "listen", ":8080", "interface and port to listen on")
	f.String("config", "", "Config file to read values from.")
	f.StringVar(&healthcheck, "healthcheck", "", "A URL path for healthcheck. Intended for use by load balancer health checks.")

	f.IntVar(&poolNumEntries, "poolnumentries", 0, "Number of buffers to pool.")
	f.IntVar(&poolEntrySize, "poolentrysize", 0, "Size of each buffer in pool.")

	f.StringVar(&metricsStatsdAddr, "metrics-statsd-addr", "", "host:port to use to send data to statsd")
	f.StringVar(&metricsStatsdPrefix, "metrics-statsd-prefix", "", "prefix to prepend to metrics")

	err = f.Parse(os.Args[1:])
	if err == flag.ErrHelp {
		return
	} else if err != nil {
		logFatalCfgErr(logger, "Unable to parse input command line, environment or config: %s", err.Error())
	}

	if len(hc.Pattern) == 0 {
		logFatalCfgErr(logger, "You must provide at least one pattern.")
	}
	if len(hc.Storage) == 0 {
		logFatalCfgErr(logger, "You must provide at least one storage.")
	}

	r := mux.NewRouter()

	// buffer manager shared by all handlers
	var bufferManager buffer.BufferManager

	if poolNumEntries > 0 && poolEntrySize > 0 {
		bufferManager = bpool.NewSizedBufferPool(poolNumEntries, poolEntrySize)
	} else {
		bufferManager = &buffer.OnDemandBufferManager{}
	}

	// metrics writer configuration
	var mw metrics.MetricsWriter
	if metricsStatsdAddr != "" {
		udpAddr, err := net.ResolveUDPAddr("udp4", metricsStatsdAddr)
		if err != nil {
			logFatalCfgErr(logger, "Invalid metricsstatsdaddr %s: %s", metricsStatsdAddr, err)
		}
		mw = metrics.NewStatsdMetricsWriter(udpAddr, metricsStatsdPrefix, logger)
	} else {
		mw = &metrics.NilMetricsWriter{}
	}

	// set if we have s3 storage configured, and shared across all s3 sessions
	var awsSession *session.Session

	for sName, sd := range hc.Storage {
		t := sd.Type
		switch t {
		case "s3":
		case "file":
		default:
			logFatalCfgErr(logger, "Unknown storage type for storage %s: %s", sName, t)
		}
	}

	// keep track of the storages so we can healthcheck them
	// we only need to check unique type/healthcheck configurations
	healthCheckStorages := make(map[config.HealthCheckConfig]storage.Storage)

	// create the storage implementations and handler routes for patterns
	var stg storage.Storage
	for reqPattern, rhc := range hc.Pattern {

		storageDefinitionName := rhc.Storage
		sd, ok := hc.Storage[storageDefinitionName]
		if !ok {
			logFatalCfgErr(logger, "Unknown storage definition: %s", storageDefinitionName)
		}
		metatileSize := sd.MetatileSize
		if rhc.MetatileSize != nil {
			metatileSize = *rhc.MetatileSize
		}
		if !tile.IsPowerOfTwo(metatileSize) {
			logFatalCfgErr(logger, "Metatile size must be power of two, but %d is not", metatileSize)
		}
		tileSize := 1
		if sd.TileSize != nil {
			tileSize = *sd.TileSize
		}
		if rhc.TileSize != nil {
			tileSize = *rhc.TileSize
		}
		if !tile.IsPowerOfTwo(tileSize) {
			logFatalCfgErr(logger, "Tile size must be power of two, but %d is not", tileSize)
		}
		layer := sd.Layer
		if rhc.Layer != nil {
			layer = *rhc.Layer
		}

		var healthcheck string

		switch sd.Type {
		case "s3":
			if rhc.Prefix == nil {
				logFatalCfgErr(logger, "S3 configuration requires prefix")
			}
			prefix := *rhc.Prefix

			if awsSession == nil {
				if hc.Aws != nil && hc.Aws.Region != nil {
					awsSession, err = session.NewSessionWithOptions(session.Options{
						Config: aws.Config{Region: hc.Aws.Region},
					})
				} else {
					awsSession, err = session.NewSession()
				}
			}
			if err != nil {
				logFatalCfgErr(logger, "Unable to set up AWS session: %s", err.Error())
			}

			var s3Client s3iface.S3API
			if hc.Aws.Role != nil {
				creds := stscreds.NewCredentials(awsSession, *hc.Aws.Role)
				s3Client = s3.New(awsSession, &aws.Config{Credentials: creds})
			} else {
				s3Client = s3.New(awsSession)
			}

			keyPattern := sd.KeyPattern
			if rhc.KeyPattern != nil {
				keyPattern = *rhc.KeyPattern
			}

			if sd.Bucket == "" {
				logFatalCfgErr(logger, "S3 storage missing bucket configuration")
			}
			if keyPattern == "" {
				logFatalCfgErr(logger, "S3 storage missing key pattern")
			}

			if sd.Healthcheck == "" {
				logger.Warning(log.LogCategory_ConfigError, "Missing healthcheck for storage s3")
			}

			healthcheck = sd.Healthcheck
			stg = storage.NewS3Storage(s3Client, sd.Bucket, keyPattern, prefix, layer, healthcheck)

		case "file":
			if sd.BaseDir == "" {
				logFatalCfgErr(logger, "File storage missing base dir")
			}

			if sd.Healthcheck == "" {
				logger.Warning(log.LogCategory_ConfigError, "Missing healthcheck for storage file")
			}

			healthcheck = sd.Healthcheck
			stg = storage.NewFileStorage(sd.BaseDir, layer, healthcheck)

		default:
			logFatalCfgErr(logger, "Unknown storage type: %s", sd.Type)
		}

		if healthcheck != "" {
			storageErr := stg.HealthCheck()
			if storageErr != nil {
				logger.Warning(log.LogCategory_ConfigError, "Healthcheck failed on storage: %s", storageErr)
			}

			hcc := config.HealthCheckConfig{
				Type:        sd.Type,
				Healthcheck: healthcheck,
			}

			if _, ok := healthCheckStorages[hcc]; !ok {
				healthCheckStorages[hcc] = stg
			}
		}

		if rhc.Type == nil || *rhc.Type == "metatile" {
			parser := &handler.MetatileMuxParser{
				MimeMap: hc.Mime,
			}

			h := handler.MetatileHandler(parser, metatileSize, tileSize, hc.Mime, stg, bufferManager, mw, logger)
			gzipped := gziphandler.GzipHandler(h)

			r.Handle(reqPattern, gzipped).Methods("GET")

		} else if rhc.Type != nil && *rhc.Type == "tilejson" {
			parser := &handler.TileJsonParser{}
			h := handler.TileJsonHandler(parser, stg, mw, logger)
			gzipped := gziphandler.GzipHandler(h)
			r.Handle(reqPattern, gzipped).Methods("GET")
		} else {
			systemLogger.Fatalf("ERROR: Invalid route handler type: %s\n", *rhc.Type)
		}

	}

	if len(healthcheck) > 0 {
		storagesToCheck := make([]storage.Storage, len(healthCheckStorages))
		i := 0
		for _, s := range healthCheckStorages {
			storagesToCheck[i] = s
			i++
		}
		hc := handler.HealthCheckHandler(storagesToCheck, logger)
		r.Handle(healthcheck, hc).Methods("GET")
	}

	corsHandler := handlers.CORS()(r)
	loggingHandler := log.LoggingMiddleware(logger)(corsHandler)

	logger.Info("Server started and listening on %s", listen)

	err = http.ListenAndServe(listen, loggingHandler)

	systemLogger.Fatalf("Failed to listen: %+v", err)
}

func logFatalCfgErr(logger log.JsonLogger, msg string, xs ...interface{}) {
	logger.Error(log.LogCategory_ConfigError, msg, xs...)
	os.Exit(1)
}
<commit_msg>Add h2c support + graceful shutdown<commit_after>package main

import (
	"context"
	golog "log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"sync/atomic"
	"syscall"
	"time"

	"github.com/NYTimes/gziphandler"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3iface"
	"github.com/gorilla/handlers"
	"github.com/gorilla/mux"
	"github.com/namsral/flag"
	"github.com/oxtoacart/bpool"
	"golang.org/x/net/http2"
	"golang.org/x/net/http2/h2c"

	"github.com/tilezen/tapalcatl/pkg/buffer"
	"github.com/tilezen/tapalcatl/pkg/config"
	"github.com/tilezen/tapalcatl/pkg/handler"
	"github.com/tilezen/tapalcatl/pkg/log"
	"github.com/tilezen/tapalcatl/pkg/metrics"
	"github.com/tilezen/tapalcatl/pkg/storage"
	"github.com/tilezen/tapalcatl/pkg/tile"
)

const (
	// The time to wait after responding /ready with non-200 before starting to shut down the HTTP server
	gracefulShutdownSleep = 20 * time.Second
	// The time to wait for the in-flight HTTP requests to complete before exiting
	gracefulShutdownTimeout = 5 * time.Second
)

func main() {
	var listen, healthcheck, readyCheck string
	var poolNumEntries, poolEntrySize int
	var metricsStatsdAddr, metricsStatsdPrefix string

	hc := config.HandlerConfig{}

	systemLogger := golog.New(os.Stdout, "", golog.LstdFlags|golog.LUTC|golog.Lmicroseconds)
	hostname, err := os.Hostname()
	if err != nil {
		// NOTE: if there are legitimate cases when this can fail, we
		// can leave off the hostname in the logger.
		// But for now we prefer to get notified of it.
		systemLogger.Fatalf("ERROR: Cannot find hostname to use for logger")
	}
	// use this logger everywhere.
	logger := log.NewJsonLogger(systemLogger, hostname)

	f := flag.NewFlagSetWithEnvPrefix(os.Args[0], "TAPALCATL", 0)
	f.Var(&hc, "handler",
		`JSON object defining how request patterns will be handled.
	 Aws { Object present when Aws-wide configuration is needed, eg session config.
     Region string Name of aws region
   }
   Storage { key -> storage definition mapping
     storage name string -> {
        Type string storage type, can be "s3" or "file
        MetatileSize int      Number of 256px tiles in each dimension of the metatile.
        TileSize int        Size of tile in 256px tile units.

       (s3 storage)
        Layer      string   Name of layer to use in this bucket. Only relevant for s3.
        Bucket     string   Name of S3 bucket to fetch from.
        KeyPattern string   Pattern to fill with variables from the main pattern to make the S3 key.
        Healthcheck string Name of S3 key to use when querying health of S3 system.

       (file storage)
        BaseDir    string   Base directory to look for files under.
        Healthcheck string  Path to a file (inside BaseDir) when querying health of system.
     }
   }
   Pattern { request pattern -> storage configuration mapping
     request pattern string -> {
       storage string Name of storage defintion to use
       list of optional storage configuration to use:
       prefix is required for s3, others are optional overrides of relevant definition
     	 Prefix string  Prefix to use in this bucket.
     }
   }
   Mime { extension -> content-type used in http response
   }
`)
	f.StringVar(&listen, "listen", ":8080", "interface and port to listen on")
	f.String("config", "", "Config file to read values from.")
	f.StringVar(&healthcheck, "healthcheck", "", "A URL path for healthcheck. Intended for use by load balancer health checks.")
	f.StringVar(&readyCheck, "readycheck", "", "A URL path for readiness check. Intended for use by Kubernetes readinessProbe.")

	f.IntVar(&poolNumEntries, "poolnumentries", 0, "Number of buffers to pool.")
	f.IntVar(&poolEntrySize, "poolentrysize", 0, "Size of each buffer in pool.")

	f.StringVar(&metricsStatsdAddr, "metrics-statsd-addr", "", "host:port to use to send data to statsd")
	f.StringVar(&metricsStatsdPrefix, "metrics-statsd-prefix", "", "prefix to prepend to metrics")

	err = f.Parse(os.Args[1:])
	if err == flag.ErrHelp {
		return
	} else if err != nil {
		logFatalCfgErr(logger, "Unable to parse input command line, environment or config: %s", err.Error())
	}

	if len(hc.Pattern) == 0 {
		logFatalCfgErr(logger, "You must provide at least one pattern.")
	}
	if len(hc.Storage) == 0 {
		logFatalCfgErr(logger, "You must provide at least one storage.")
	}

	r := mux.NewRouter()

	// buffer manager shared by all handlers
	var bufferManager buffer.BufferManager

	if poolNumEntries > 0 && poolEntrySize > 0 {
		bufferManager = bpool.NewSizedBufferPool(poolNumEntries, poolEntrySize)
	} else {
		bufferManager = &buffer.OnDemandBufferManager{}
	}

	// metrics writer configuration
	var mw metrics.MetricsWriter
	if metricsStatsdAddr != "" {
		udpAddr, err := net.ResolveUDPAddr("udp4", metricsStatsdAddr)
		if err != nil {
			logFatalCfgErr(logger, "Invalid metricsstatsdaddr %s: %s", metricsStatsdAddr, err)
		}
		mw = metrics.NewStatsdMetricsWriter(udpAddr, metricsStatsdPrefix, logger)
	} else {
		mw = &metrics.NilMetricsWriter{}
	}

	// set if we have s3 storage configured, and shared across all s3 sessions
	var awsSession *session.Session

	for sName, sd := range hc.Storage {
		t := sd.Type
		switch t {
		case "s3":
		case "file":
		default:
			logFatalCfgErr(logger, "Unknown storage type for storage %s: %s", sName, t)
		}
	}

	// keep track of the storages so we can healthcheck them
	// we only need to check unique type/healthcheck configurations
	healthCheckStorages := make(map[config.HealthCheckConfig]storage.Storage)

	// create the storage implementations and handler routes for patterns
	var stg storage.Storage
	for reqPattern, rhc := range hc.Pattern {

		storageDefinitionName := rhc.Storage
		sd, ok := hc.Storage[storageDefinitionName]
		if !ok {
			logFatalCfgErr(logger, "Unknown storage definition: %s", storageDefinitionName)
		}
		metatileSize := sd.MetatileSize
		if rhc.MetatileSize != nil {
			metatileSize = *rhc.MetatileSize
		}
		if !tile.IsPowerOfTwo(metatileSize) {
			logFatalCfgErr(logger, "Metatile size must be power of two, but %d is not", metatileSize)
		}
		tileSize := 1
		if sd.TileSize != nil {
			tileSize = *sd.TileSize
		}
		if rhc.TileSize != nil {
			tileSize = *rhc.TileSize
		}
		if !tile.IsPowerOfTwo(tileSize) {
			logFatalCfgErr(logger, "Tile size must be power of two, but %d is not", tileSize)
		}
		layer := sd.Layer
		if rhc.Layer != nil {
			layer = *rhc.Layer
		}

		var healthcheck string

		switch sd.Type {
		case "s3":
			if rhc.Prefix == nil {
				logFatalCfgErr(logger, "S3 configuration requires prefix")
			}
			prefix := *rhc.Prefix

			if awsSession == nil {
				if hc.Aws != nil && hc.Aws.Region != nil {
					awsSession, err = session.NewSessionWithOptions(session.Options{
						Config: aws.Config{Region: hc.Aws.Region},
					})
				} else {
					awsSession, err = session.NewSession()
				}
			}
			if err != nil {
				logFatalCfgErr(logger, "Unable to set up AWS session: %s", err.Error())
			}

			var s3Client s3iface.S3API
			if hc.Aws.Role != nil {
				creds := stscreds.NewCredentials(awsSession, *hc.Aws.Role)
				s3Client = s3.New(awsSession, &aws.Config{Credentials: creds})
			} else {
				s3Client = s3.New(awsSession)
			}

			keyPattern := sd.KeyPattern
			if rhc.KeyPattern != nil {
				keyPattern = *rhc.KeyPattern
			}

			if sd.Bucket == "" {
				logFatalCfgErr(logger, "S3 storage missing bucket configuration")
			}
			if keyPattern == "" {
				logFatalCfgErr(logger, "S3 storage missing key pattern")
			}

			if sd.Healthcheck == "" {
				logger.Warning(log.LogCategory_ConfigError, "Missing healthcheck for storage s3")
			}

			healthcheck = sd.Healthcheck
			stg = storage.NewS3Storage(s3Client, sd.Bucket, keyPattern, prefix, layer, healthcheck)

		case "file":
			if sd.BaseDir == "" {
				logFatalCfgErr(logger, "File storage missing base dir")
			}

			if sd.Healthcheck == "" {
				logger.Warning(log.LogCategory_ConfigError, "Missing healthcheck for storage file")
			}

			healthcheck = sd.Healthcheck
			stg = storage.NewFileStorage(sd.BaseDir, layer, healthcheck)

		default:
			logFatalCfgErr(logger, "Unknown storage type: %s", sd.Type)
		}

		if healthcheck != "" {
			storageErr := stg.HealthCheck()
			if storageErr != nil {
				logger.Warning(log.LogCategory_ConfigError, "Healthcheck failed on storage: %s", storageErr)
			}

			hcc := config.HealthCheckConfig{
				Type:        sd.Type,
				Healthcheck: healthcheck,
			}

			if _, ok := healthCheckStorages[hcc]; !ok {
				healthCheckStorages[hcc] = stg
			}
		}

		if rhc.Type == nil || *rhc.Type == "metatile" {
			parser := &handler.MetatileMuxParser{
				MimeMap: hc.Mime,
			}

			h := handler.MetatileHandler(parser, metatileSize, tileSize, hc.Mime, stg, bufferManager, mw, logger)
			gzipped := gziphandler.GzipHandler(h)

			r.Handle(reqPattern, gzipped).Methods("GET")

		} else if rhc.Type != nil && *rhc.Type == "tilejson" {
			parser := &handler.TileJsonParser{}
			h := handler.TileJsonHandler(parser, stg, mw, logger)
			gzipped := gziphandler.GzipHandler(h)
			r.Handle(reqPattern, gzipped).Methods("GET")
		} else {
			systemLogger.Fatalf("ERROR: Invalid route handler type: %s\n", *rhc.Type)
		}

	}

	if len(healthcheck) > 0 {
		storagesToCheck := make([]storage.Storage, len(healthCheckStorages))
		i := 0
		for _, s := range healthCheckStorages {
			storagesToCheck[i] = s
			i++
		}
		hc := handler.HealthCheckHandler(storagesToCheck, logger)
		r.Handle(healthcheck, hc).Methods("GET")
	}

	// Readiness probe for graceful shutdown support
	readinessResponseCode := uint32(http.StatusOK)
	if len(readyCheck) > 0 {
		r.HandleFunc(readyCheck, func(w http.ResponseWriter, r *http.Request) {
			w.WriteHeader(int(atomic.LoadUint32(&readinessResponseCode)))
		})
	}

	corsHandler := handlers.CORS()(r)
	loggingHandler := log.LoggingMiddleware(logger)(corsHandler)

	logger.Info("Server started and listening on %s", listen)

	// Support for upgrading an http/1.1 connection to http/2
	// See https://github.com/thrawn01/h2c-golang-example
	http2Server := &http2.Server{}
	server := &http.Server{
		Addr:    listen,
		Handler: h2c.NewHandler(loggingHandler, http2Server),
	}

	// Code to handle shutdown gracefully
	shutdownChan := make(chan struct{})
	go func() {
		defer close(shutdownChan)

		// Wait for SIGTERM to come in
		signals := make(chan os.Signal, 1)
		signal.Notify(signals, syscall.SIGTERM)
		<-signals

		logger.Info("SIGTERM received. Starting graceful shutdown.")

		// Start failing readiness probes
		atomic.StoreUint32(&readinessResponseCode, http.StatusInternalServerError)
		// Wait for upstream clients
		time.Sleep(gracefulShutdownSleep)
		// Begin shutdown of in-flight requests
		shutdownCtx, shutdownCtxCancel := context.WithTimeout(context.Background(), gracefulShutdownTimeout)
		if err := server.Shutdown(shutdownCtx); err != nil {
			logger.Info("Error waiting for server shutdown: %+v", err)
		}
		shutdownCtxCancel()
	}()

	logger.Info("Service started")
	if err := server.ListenAndServe(); err != nil {
		logger.Info("Couldn't start HTTP server: %+v", err)
	}
	<-shutdownChan
}

func logFatalCfgErr(logger log.JsonLogger, msg string, xs ...interface{}) {
	logger.Error(log.LogCategory_ConfigError, msg, xs...)
	os.Exit(1)
}
<|endoftext|>


<commit_before>// Copyright 2013 gopm authors.
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package cmd

import (
	"encoding/json"
	"fmt"
	"net/http"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"runtime"

	"github.com/Unknwon/com"
	"github.com/codegangsta/cli"

	"github.com/gpmgo/gopm/doc"
	"github.com/gpmgo/gopm/log"
)

var CmdUpdate = cli.Command{
	Name:  "update",
	Usage: "check and update gopm resources including itself",
	Description: `Command update checks updates of resources and gopm itself.

gopm update

Resources will be updated automatically after executed this command,
but you have to confirm before updaing gopm itself.`,
	Action: runUpdate,
	Flags: []cli.Flag{
		cli.BoolFlag{"verbose, v", "show process details"},
	},
}

func exePath() string {
	file, err := exec.LookPath(os.Args[0])
	if err != nil {
		log.Error("Update", "Fail to execute exec.LookPath")
		log.Fatal("", err.Error())
	}
	path, err := filepath.Abs(file)
	if err != nil {
		log.Error("Update", "Fail to get absolutely path")
		log.Fatal("", err.Error())
	}
	return path
}

type version struct {
	Gopm            int
	PackageNameList int `json:"package_name_list"`
}

func runUpdate(ctx *cli.Context) {
	setup(ctx)

	isAnythingUpdated := false
	// Load local version info.
	localVerInfo := loadLocalVerInfo()

	// Get remote version info.
	var remoteVerInfo version
	if err := com.HttpGetJSON(http.DefaultClient, "http://gopm.io/VERSION.json", &remoteVerInfo); err != nil {
		log.Error("Update", "Fail to fetch VERSION.json")
		log.Fatal("", err.Error())
	}

	// Package name list.
	if remoteVerInfo.PackageNameList > localVerInfo.PackageNameList {
		log.Log("Updating pkgname.list...%v > %v",
			localVerInfo.PackageNameList, remoteVerInfo.PackageNameList)
		data, err := com.HttpGetBytes(http.DefaultClient, "https://raw2.github.com/gpmgo/docs/master/pkgname.list", nil)
		if err != nil {
			log.Error("Update", "Fail to update pkgname.list")
			log.Fatal("", err.Error())
		}
		_, err = com.SaveFile(path.Join(doc.HomeDir, doc.PKG_NAME_LIST_PATH), data)
		if err != nil {
			log.Error("Update", "Fail to save pkgname.list")
			log.Fatal("", err.Error())
		}
		log.Log("Update pkgname.list to %v succeed!", remoteVerInfo.PackageNameList)
		isAnythingUpdated = true
	}

	// Gopm.
	if remoteVerInfo.Gopm > localVerInfo.Gopm {
		log.Log("Updating gopm...%v > %v",
			localVerInfo.Gopm, remoteVerInfo.Gopm)
		installRepoPath = doc.HomeDir + "/repos"

		tmpDirPath := path.Join(doc.HomeDir, "temp")
		tmpBinPath := path.Join(tmpDirPath, "gopm")
		if runtime.GOOS == "windows" {
			tmpBinPath += ".exe"
		}

		os.MkdirAll(tmpDirPath, os.ModePerm)
		os.Remove(tmpBinPath)
		// Fetch code.
		stdout, stderr, err := com.ExecCmd("gopm", "bin", "-u", "-d",
			"github.com/gpmgo/gopm", tmpDirPath)
		if err != nil {
			log.Error("Update", "Fail to execute 'gopm bin -u -d github.com/gpmgo/gopm "+tmpDirPath+"'")
			log.Fatal("", err.Error())
		}
		if len(stderr) > 0 {
			fmt.Print(stderr)
		}
		if len(stdout) > 0 {
			fmt.Print(stdout)
		}

		// Check if previous steps were successful.
		if !com.IsExist(tmpBinPath) {
			log.Error("Update", "Fail to continue command")
			log.Fatal("", "Previous steps weren't successful, no binary produced")
		}

		movePath := exePath()
		log.Log("New binary will be replaced for %s", movePath)
		// Move binary to given directory.
		if runtime.GOOS != "windows" {
			err := os.Rename(tmpBinPath, movePath)
			if err != nil {
				log.Error("Update", "Fail to move binary")
				log.Fatal("", err.Error())
			}
			os.Chmod(movePath+"/"+path.Base(tmpBinPath), os.ModePerm)
		} else {
			batPath := filepath.Join(workDir, "a.bat")
			f, err := os.Create(batPath)
			if err != nil {
				log.Error("Update", "Fail to generate bat file")
				log.Fatal("", err.Error())
			}
			//f.WriteString("@echo off\r\n")
			f.WriteString(fmt.Sprintf("ping -n 1 127.0.0.1>nul\r\ncopy \"%v\" \"%v\"\r\ndel \"%v\"\r\ndel \"%v\"\r\n",
				tmpBinPath, movePath, tmpBinPath, batPath))
			f.Close()

			attr := &os.ProcAttr{
				Dir:   workDir,
				Env:   os.Environ(),
				Files: []*os.File{os.Stdin, os.Stdout, os.Stderr},
			}

			_, err = os.StartProcess(batPath, []string{"a.bat"}, attr)
			if err != nil {
				log.Error("Update", "Fail to start bat process")
				log.Fatal("", err.Error())
			}
		}

		log.Success("SUCC", "Update", "Command execute successfully!")
		isAnythingUpdated = true
	}

	// Save JSON.
	f, err := os.Create(path.Join(doc.HomeDir, doc.VER_PATH))
	if err != nil {
		log.Error("Update", "Fail to create VERSION.json")
		log.Fatal("", err.Error())
	}
	if err := json.NewEncoder(f).Encode(&remoteVerInfo); err != nil {
		log.Error("Update", "Fail to encode VERSION.json")
		log.Fatal("", err.Error())
	}

	if !isAnythingUpdated {
		log.Log("Nothing need to be updated")
	}
}

func loadLocalVerInfo() (ver version) {
	verPath := path.Join(doc.HomeDir, doc.VER_PATH)

	// First time run should not exist.
	if !com.IsExist(verPath) {
		return ver
	}

	f, err := os.Open(verPath)
	if err != nil {
		log.Error("Update", "Fail to open VERSION.json")
		log.Fatal("", err.Error())
	}

	if err := json.NewDecoder(f).Decode(&ver); err != nil {
		log.Error("Update", "Fail to decode VERSION.json")
		log.Fatal("", err.Error())
	}
	return ver
}
<commit_msg>update bug fixed on windows<commit_after>// Copyright 2013 gopm authors.
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package cmd

import (
	"encoding/json"
	"fmt"
	"net/http"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"runtime"

	"github.com/Unknwon/com"
	"github.com/codegangsta/cli"

	"github.com/gpmgo/gopm/doc"
	"github.com/gpmgo/gopm/log"
)

var CmdUpdate = cli.Command{
	Name:  "update",
	Usage: "check and update gopm resources including itself",
	Description: `Command update checks updates of resources and gopm itself.

gopm update

Resources will be updated automatically after executed this command,
but you have to confirm before updaing gopm itself.`,
	Action: runUpdate,
	Flags: []cli.Flag{
		cli.BoolFlag{"verbose, v", "show process details"},
	},
}

func exePath() string {
	file, err := exec.LookPath(os.Args[0])
	if err != nil {
		log.Error("Update", "Fail to execute exec.LookPath")
		log.Fatal("", err.Error())
	}
	path, err := filepath.Abs(file)
	if err != nil {
		log.Error("Update", "Fail to get absolutely path")
		log.Fatal("", err.Error())
	}
	return path
}

type version struct {
	Gopm            int
	PackageNameList int `json:"package_name_list"`
}

func runUpdate(ctx *cli.Context) {
	setup(ctx)

	isAnythingUpdated := false
	// Load local version info.
	localVerInfo := loadLocalVerInfo()

	// Get remote version info.
	var remoteVerInfo version
	if err := com.HttpGetJSON(http.DefaultClient, "http://gopm.io/VERSION.json", &remoteVerInfo); err != nil {
		log.Error("Update", "Fail to fetch VERSION.json")
		log.Fatal("", err.Error())
	}

	// Package name list.
	if remoteVerInfo.PackageNameList > localVerInfo.PackageNameList {
		log.Log("Updating pkgname.list...%v > %v",
			localVerInfo.PackageNameList, remoteVerInfo.PackageNameList)
		data, err := com.HttpGetBytes(http.DefaultClient, "https://raw2.github.com/gpmgo/docs/master/pkgname.list", nil)
		if err != nil {
			log.Error("Update", "Fail to update pkgname.list")
			log.Fatal("", err.Error())
		}
		_, err = com.SaveFile(path.Join(doc.HomeDir, doc.PKG_NAME_LIST_PATH), data)
		if err != nil {
			log.Error("Update", "Fail to save pkgname.list")
			log.Fatal("", err.Error())
		}
		log.Log("Update pkgname.list to %v succeed!", remoteVerInfo.PackageNameList)
		isAnythingUpdated = true
	}

	// Gopm.
	if remoteVerInfo.Gopm > localVerInfo.Gopm {
		log.Log("Updating gopm...%v > %v",
			localVerInfo.Gopm, remoteVerInfo.Gopm)
		installRepoPath = doc.HomeDir + "/repos"

		tmpDirPath := filepath.Join(doc.HomeDir, "temp")
		tmpBinPath := filepath.Join(tmpDirPath, "gopm")
		if runtime.GOOS == "windows" {
			tmpBinPath += ".exe"
		}

		os.MkdirAll(tmpDirPath, os.ModePerm)
		os.Remove(tmpBinPath)
		// Fetch code.
		stdout, stderr, err := com.ExecCmd("gopm", "bin", "-u", "-d",
			"github.com/gpmgo/gopm", tmpDirPath)
		if err != nil {
			log.Error("Update", "Fail to execute 'gopm bin -u -d github.com/gpmgo/gopm "+tmpDirPath+"'")
			log.Fatal("", err.Error())
		}
		if len(stderr) > 0 {
			fmt.Print(stderr)
		}
		if len(stdout) > 0 {
			fmt.Print(stdout)
		}

		// Check if previous steps were successful.
		if !com.IsExist(tmpBinPath) {
			log.Error("Update", "Fail to continue command")
			log.Fatal("", "Previous steps weren't successful, no binary produced")
		}

		movePath := exePath()
		log.Log("New binary will be replaced for %s", movePath)
		// Move binary to given directory.
		if runtime.GOOS != "windows" {
			err := os.Rename(tmpBinPath, movePath)
			if err != nil {
				log.Error("Update", "Fail to move binary")
				log.Fatal("", err.Error())
			}
			os.Chmod(movePath+"/"+path.Base(tmpBinPath), os.ModePerm)
		} else {
			batPath := filepath.Join(tmpDirPath, "update.bat")
			f, err := os.Create(batPath)
			if err != nil {
				log.Error("Update", "Fail to generate bat file")
				log.Fatal("", err.Error())
			}
			f.WriteString("@echo off\r\n")
			f.WriteString(fmt.Sprintf("ping -n 1 127.0.0.1>nul\r\ncopy \"%v\" \"%v\" >nul\r\ndel \"%v\" >nul\r\n\r\n",
				tmpBinPath, movePath, tmpBinPath))
			//f.WriteString(fmt.Sprintf("del \"%v\"\r\n", batPath))
			f.Close()

			attr := &os.ProcAttr{
				Dir:   workDir,
				Env:   os.Environ(),
				Files: []*os.File{os.Stdin, os.Stdout, os.Stderr},
			}

			_, err = os.StartProcess(batPath, []string{batPath}, attr)
			if err != nil {
				log.Error("Update", "Fail to start bat process")
				log.Fatal("", err.Error())
			}
		}

		log.Success("SUCC", "Update", "Command execute successfully!")
		isAnythingUpdated = true
	}

	// Save JSON.
	f, err := os.Create(path.Join(doc.HomeDir, doc.VER_PATH))
	if err != nil {
		log.Error("Update", "Fail to create VERSION.json")
		log.Fatal("", err.Error())
	}
	if err := json.NewEncoder(f).Encode(&remoteVerInfo); err != nil {
		log.Error("Update", "Fail to encode VERSION.json")
		log.Fatal("", err.Error())
	}

	if !isAnythingUpdated {
		log.Log("Nothing need to be updated")
	}
	log.Log("Exit old gopm")
}

func loadLocalVerInfo() (ver version) {
	verPath := path.Join(doc.HomeDir, doc.VER_PATH)

	// First time run should not exist.
	if !com.IsExist(verPath) {
		return ver
	}

	f, err := os.Open(verPath)
	if err != nil {
		log.Error("Update", "Fail to open VERSION.json")
		log.Fatal("", err.Error())
	}

	if err := json.NewDecoder(f).Decode(&ver); err != nil {
		log.Error("Update", "Fail to decode VERSION.json")
		log.Fatal("", err.Error())
	}
	return ver
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
)

// `direnv stdlib`
var CmdStdlib = &Cmd{
	Name: "stdlib",
	Desc: "Displays the stdlib available in the .envrc execution context",
	Fn: func(env Env, args []string) (err error) {
		var config *Config
		if config, err = LoadConfig(env); err != nil {
			return
		}

		fmt.Printf(STDLIB, config.SelfPath)
		return
	},
}

const STDLIB = `# These are the commands available in an .envrc context
set -e
DIRENV_PATH="%s"

# Usage: has <command>
#
# Returns 0 if the <command> is available. Returns 1 otherwise. It can be a
# binary in the PATH or a shell function.
#
# Example:
#
#    if has curl; then
#      echo "Yes we do"
#    fi
#
has() {
	type "$1" &>/dev/null
}

# Usage: expand_path <rel_path> [<relative_to>]
#
# Outputs the absolute path of <rel_path> relaitve to <relative_to> or the
# current directory.
#
# Example:
#
#    cd /usr/local/games
#    expand_path ../foo
#    # output: /usr/local/foo
#
expand_path() {
	"$DIRENV_PATH" expand_path "$@"
}

# Usage: dotenv [<dotenv>]
#
# Loads a ".env" file into the current environment
#
dotenv() {
	eval "$("$DIRENV_PATH" dotenv bash "$@")"
}

# Usage: user_rel_path <abs_path>
#
# Transforms an absolute path <abs_path> into a user-relative path if
# possible.
#
# Example:
#
#    echo $HOME
#    # output: /home/user
#    user_rel_path /home/user/my/project
#    # output: ~/my/project
#    user_rel_path /usr/local/lib
#    # output: /usr/local/lib
#
user_rel_path() {
	local path="${1#-}"

	if [ -z "$path" ]; then return; fi

	if [ -n "$HOME" ]; then
		local rel_path="${path#$HOME}"
		if [ "$rel_path" != "$path" ]; then
			path="~${rel_path}"
		fi
	fi

	echo "$path"
}

# Usage: find_up <filename>
#
# Outputs the path of <filename> when searched from the current directory up to
# /. Returns 1 if the file has not been found.
#
# Example:
#
#    cd /usr/local/my
#    mkdir -p project/foo
#    touch bar
#    cd project/foo
#    find_up bar
#    # output: /usr/local/my/bar
#
find_up() {
	(
		cd "$(pwd -P 2>/dev/null)"
		while true; do
			if [ -f "$1" ]; then
				echo "$PWD/$1"
				return 0
			fi
			if [ "$PWD" = "/" ] || [ "$PWD" = "//" ]; then
				return 1
			fi
			cd ..
		done
	)
}

# Usage: source_env <file_or_dir_path>
#
# Loads another ".envrc" either by specifying its path or filename.
source_env() {
	local rcfile="$1"
	local rcpath="${1/#\~/$HOME}"
	if ! [ -f "$rcpath" ]; then
		rcfile="$rcfile/.envrc"
		rcpath="$rcpath/.envrc"
	fi
	echo "direnv: loading $rcfile"
	pushd "$(dirname "$rcpath")" > /dev/null
	. "./$(basename "$rcpath")"
	popd > /dev/null
}

# Usage: source_up [<filename>]
#
# Loads another ".envrc" if found with the find_up command.
#
source_up() {
	local file="$1"
	if [ -z "$file" ]; then
		file=".envrc"
	fi
	local path="$(cd .. && find_up "$file")"
	if [ -n "$path" ]; then
		source_env "$(user_rel_path "$path")"
	fi
}

# Usage: PATH_add <path>
#
# Prepends the expanded <path> to the PATH environment variable. It prevents a
# common mistake where PATH is replaced by only the new <path>.
#
# Example:
#
#    pwd
#    # output: /home/user/my/project
#    PATH_add bin
#    echo $PATH
#    # output: /home/user/my/project/bin:/usr/bin:/bin
#
PATH_add() {
	export PATH="$(expand_path "$1"):$PATH"
}

# Usage: path_add <varname> <path>
#
# Works like PATH_add except that it's for an arbitrary <varname>.
path_add() {
	local old_paths="${!1}"
	local path="$(expand_path "$2")"

	if [ -z "$old_paths" ]; then
		old_paths="$path"
	else
		old_paths="$path:$old_paths"
	fi

	export $1="$old_paths"
}

# Usage: load_prefix <prefix_path>
#
# Expands some common path variables for the given <prefix_path> prefix. This is
# useful if you installed something in the <prefix_path> using
# $(./configure --prefix=<prefix_path> && make install) and want to use it in
# the project.
#
# Variables set:
#
#    CPATH
#    LD_LIBRARY_PATH
#    LIBRARY_PATH
#    MANPATH
#    PATH
#    PKG_CONFIG_PATH
#
# Example:
#
#    ./configure --prefix=$HOME/rubies/ruby-1.9.3
#    make && make install
#    # Then in the .envrc
#    load_prefix ~/rubies/ruby-1.9.3
#
load_prefix() {
	local path="$(expand_path "$1")"
	path_add CPATH "$path/include"
	path_add LD_LIBRARY_PATH "$path/lib"
	path_add LIBRARY_PATH "$path/lib"
	path_add MANPATH "$path/man"
	path_add MANPATH "$path/share/man"
	path_add PATH "$path/bin"
	path_add PKG_CONFIG_PATH "$path/lib/pkgconfig"
}

# Usage: layout <type>
#
# A semantic dispatch used to describe common project layouts.
#
layout() {
	eval "layout_$1"
}

# Usage: layout ruby
#
# Sets the GEM_HOME environment variable to "$PWD/.direnv/ruby/RUBY_VERSION".
# This forces the installation of any gems into the project's sub-folder.
# If you're using bundler it will create wrapper programs that can be invoked
# directly instead of using the $(bundle exec) prefix.
#
layout_ruby() {
	local ruby_version="$(ruby -e"puts (defined?(RUBY_ENGINE) ? RUBY_ENGINE : 'ruby') + '-' + RUBY_VERSION")"

	export GEM_HOME="$PWD/.direnv/${ruby_version}"
	export BUNDLE_BIN="$PWD/.direnv/bin"

	PATH_add ".direnv/${ruby_version}/bin"
	PATH_add ".direnv/bin"
}

# Usage: layout python
#
# Creates and loads a virtualenv environment under "$PWD/.direnv/virtualenv".
# This forces the installation of any egg into the project's sub-folder.
#
layout_python() {
	if ! [ -d .direnv/virtualenv ]; then
		virtualenv --no-site-packages --distribute .direnv/virtualenv
		virtualenv --relocatable .direnv/virtualenv
	fi
	source .direnv/virtualenv/bin/activate
}

# Usage: layout node
#
# Adds "$PWD/node_modules/.bin" to the PATH environment variable.
layout_node() {
	PATH_add node_modules/.bin
}

# Usage: layout go
#
# Sets the GOPATH environment variable to the current directory.
#
layout_go() {
	path_add GOPATH "$PWD"
}

# Usage: use <program_name> [<version>]
#
# A semantic command dispatch intended for loading external dependencies into
# the environment.
#
# Example:
#
#    use_ruby() {
#      echo "Ruby $1"
#    }
#    use ruby 1.9.3
#    # output: Ruby 1.9.3
#
use() {
	local cmd="$1"
	echo "Using $@"
	shift
	use_$cmd "$@"
}

# Usage: use rbenv
#
# Loads rbenv which add the ruby wrappers available on the PATH.
#
use_rbenv() {
	eval "$(rbenv init -)"
}

# Usage: rvm [...]
#
# Should work just like in the shell if you have rvm installed.#
#
rvm() {
	unset rvm
	if [ -n "${rvm_scripts_path:-}" ]; then
		source "${rvm_scripts_path}/rvm"
	elif [ -n "${rvm_path:-}" ]; then
		source "${rvm_path}/scripts/rvm"
	else
		source "$HOME/.rvm/scripts/rvm"
	fi
	rvm "$@"
}

## Load the global ~/.direnvrc if present
if [ -f "$HOME/.direnvrc" ]; then
	source_env "~/.direnvrc" >&2
fi
`
<commit_msg>Renames DIRENV_PATH to direnv in the stdlib. It's not exported.<commit_after>package main

import (
	"fmt"
)

// `direnv stdlib`
var CmdStdlib = &Cmd{
	Name: "stdlib",
	Desc: "Displays the stdlib available in the .envrc execution context",
	Fn: func(env Env, args []string) (err error) {
		var config *Config
		if config, err = LoadConfig(env); err != nil {
			return
		}

		fmt.Printf(STDLIB, config.SelfPath)
		return
	},
}

const STDLIB = `# These are the commands available in an .envrc context
set -e
direnv="%s"

# Usage: has <command>
#
# Returns 0 if the <command> is available. Returns 1 otherwise. It can be a
# binary in the PATH or a shell function.
#
# Example:
#
#    if has curl; then
#      echo "Yes we do"
#    fi
#
has() {
	type "$1" &>/dev/null
}

# Usage: expand_path <rel_path> [<relative_to>]
#
# Outputs the absolute path of <rel_path> relaitve to <relative_to> or the
# current directory.
#
# Example:
#
#    cd /usr/local/games
#    expand_path ../foo
#    # output: /usr/local/foo
#
expand_path() {
	"$direnv" expand_path "$@"
}

# Usage: dotenv [<dotenv>]
#
# Loads a ".env" file into the current environment
#
dotenv() {
	eval "$("$direnv" dotenv bash "$@")"
}

# Usage: user_rel_path <abs_path>
#
# Transforms an absolute path <abs_path> into a user-relative path if
# possible.
#
# Example:
#
#    echo $HOME
#    # output: /home/user
#    user_rel_path /home/user/my/project
#    # output: ~/my/project
#    user_rel_path /usr/local/lib
#    # output: /usr/local/lib
#
user_rel_path() {
	local path="${1#-}"

	if [ -z "$path" ]; then return; fi

	if [ -n "$HOME" ]; then
		local rel_path="${path#$HOME}"
		if [ "$rel_path" != "$path" ]; then
			path="~${rel_path}"
		fi
	fi

	echo "$path"
}

# Usage: find_up <filename>
#
# Outputs the path of <filename> when searched from the current directory up to
# /. Returns 1 if the file has not been found.
#
# Example:
#
#    cd /usr/local/my
#    mkdir -p project/foo
#    touch bar
#    cd project/foo
#    find_up bar
#    # output: /usr/local/my/bar
#
find_up() {
	(
		cd "$(pwd -P 2>/dev/null)"
		while true; do
			if [ -f "$1" ]; then
				echo "$PWD/$1"
				return 0
			fi
			if [ "$PWD" = "/" ] || [ "$PWD" = "//" ]; then
				return 1
			fi
			cd ..
		done
	)
}

# Usage: source_env <file_or_dir_path>
#
# Loads another ".envrc" either by specifying its path or filename.
source_env() {
	local rcfile="$1"
	local rcpath="${1/#\~/$HOME}"
	if ! [ -f "$rcpath" ]; then
		rcfile="$rcfile/.envrc"
		rcpath="$rcpath/.envrc"
	fi
	echo "direnv: loading $rcfile"
	pushd "$(dirname "$rcpath")" > /dev/null
	. "./$(basename "$rcpath")"
	popd > /dev/null
}

# Usage: source_up [<filename>]
#
# Loads another ".envrc" if found with the find_up command.
#
source_up() {
	local file="$1"
	if [ -z "$file" ]; then
		file=".envrc"
	fi
	local path="$(cd .. && find_up "$file")"
	if [ -n "$path" ]; then
		source_env "$(user_rel_path "$path")"
	fi
}

# Usage: PATH_add <path>
#
# Prepends the expanded <path> to the PATH environment variable. It prevents a
# common mistake where PATH is replaced by only the new <path>.
#
# Example:
#
#    pwd
#    # output: /home/user/my/project
#    PATH_add bin
#    echo $PATH
#    # output: /home/user/my/project/bin:/usr/bin:/bin
#
PATH_add() {
	export PATH="$(expand_path "$1"):$PATH"
}

# Usage: path_add <varname> <path>
#
# Works like PATH_add except that it's for an arbitrary <varname>.
path_add() {
	local old_paths="${!1}"
	local path="$(expand_path "$2")"

	if [ -z "$old_paths" ]; then
		old_paths="$path"
	else
		old_paths="$path:$old_paths"
	fi

	export $1="$old_paths"
}

# Usage: load_prefix <prefix_path>
#
# Expands some common path variables for the given <prefix_path> prefix. This is
# useful if you installed something in the <prefix_path> using
# $(./configure --prefix=<prefix_path> && make install) and want to use it in
# the project.
#
# Variables set:
#
#    CPATH
#    LD_LIBRARY_PATH
#    LIBRARY_PATH
#    MANPATH
#    PATH
#    PKG_CONFIG_PATH
#
# Example:
#
#    ./configure --prefix=$HOME/rubies/ruby-1.9.3
#    make && make install
#    # Then in the .envrc
#    load_prefix ~/rubies/ruby-1.9.3
#
load_prefix() {
	local path="$(expand_path "$1")"
	path_add CPATH "$path/include"
	path_add LD_LIBRARY_PATH "$path/lib"
	path_add LIBRARY_PATH "$path/lib"
	path_add MANPATH "$path/man"
	path_add MANPATH "$path/share/man"
	path_add PATH "$path/bin"
	path_add PKG_CONFIG_PATH "$path/lib/pkgconfig"
}

# Usage: layout <type>
#
# A semantic dispatch used to describe common project layouts.
#
layout() {
	eval "layout_$1"
}

# Usage: layout ruby
#
# Sets the GEM_HOME environment variable to "$PWD/.direnv/ruby/RUBY_VERSION".
# This forces the installation of any gems into the project's sub-folder.
# If you're using bundler it will create wrapper programs that can be invoked
# directly instead of using the $(bundle exec) prefix.
#
layout_ruby() {
	local ruby_version="$(ruby -e"puts (defined?(RUBY_ENGINE) ? RUBY_ENGINE : 'ruby') + '-' + RUBY_VERSION")"

	export GEM_HOME="$PWD/.direnv/${ruby_version}"
	export BUNDLE_BIN="$PWD/.direnv/bin"

	PATH_add ".direnv/${ruby_version}/bin"
	PATH_add ".direnv/bin"
}

# Usage: layout python
#
# Creates and loads a virtualenv environment under "$PWD/.direnv/virtualenv".
# This forces the installation of any egg into the project's sub-folder.
#
layout_python() {
	if ! [ -d .direnv/virtualenv ]; then
		virtualenv --no-site-packages --distribute .direnv/virtualenv
		virtualenv --relocatable .direnv/virtualenv
	fi
	source .direnv/virtualenv/bin/activate
}

# Usage: layout node
#
# Adds "$PWD/node_modules/.bin" to the PATH environment variable.
layout_node() {
	PATH_add node_modules/.bin
}

# Usage: layout go
#
# Sets the GOPATH environment variable to the current directory.
#
layout_go() {
	path_add GOPATH "$PWD"
}

# Usage: use <program_name> [<version>]
#
# A semantic command dispatch intended for loading external dependencies into
# the environment.
#
# Example:
#
#    use_ruby() {
#      echo "Ruby $1"
#    }
#    use ruby 1.9.3
#    # output: Ruby 1.9.3
#
use() {
	local cmd="$1"
	echo "Using $@"
	shift
	use_$cmd "$@"
}

# Usage: use rbenv
#
# Loads rbenv which add the ruby wrappers available on the PATH.
#
use_rbenv() {
	eval "$(rbenv init -)"
}

# Usage: rvm [...]
#
# Should work just like in the shell if you have rvm installed.#
#
rvm() {
	unset rvm
	if [ -n "${rvm_scripts_path:-}" ]; then
		source "${rvm_scripts_path}/rvm"
	elif [ -n "${rvm_path:-}" ]; then
		source "${rvm_path}/scripts/rvm"
	else
		source "$HOME/.rvm/scripts/rvm"
	fi
	rvm "$@"
}

## Load the global ~/.direnvrc if present
if [ -f "$HOME/.direnvrc" ]; then
	source_env "~/.direnvrc" >&2
fi
`
<|endoftext|>


<commit_before>// Copyright 2014 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.
//
// Author: Spencer Kimball (spencer.kimball@gmail.com)

package util

import (
	"sync"
	"sync/atomic"
)

// Closer is an interface for objects to attach to the stopper to
// be closed once the stopper completes.
type Closer interface {
	Close()
}

// A Stopper provides a channel-based mechanism to stop an arbitrary
// array of workers. Each worker is registered with the stopper via
// the AddWorker() method. The system further tracks each task which
// is outstanding by calling StartTask() when a task is started and
// FinishTask() when completed.
//
// Stopping occurs in two phases: the first is the request to stop,
// which moves the stopper into a draining phase. While draining,
// calls to StartTask() return false, meaning the system is draining
// and new tasks should not be accepted. When all outstanding tasks
// have been completed via calls to FinishTask(), the stopper closes
// its stopper channel, which signals all live workers that it's safe
// to shut down. Once shutdown, each worker invokes SetStopped(). When
// all workers have shutdown, the stopper is complete.
//
// An arbitrary list of objects implementing the Closer interface may
// be added to the stopper via AddCloser(), to be closed after the
// stopper has stopped.
type Stopper struct {
	stopper  chan struct{}
	draining int32 // Uses atomic operations instead of mu.
	drain    sync.WaitGroup
	stop     sync.WaitGroup
	mu       sync.Mutex // Protects the slice of Closers
	closers  []Closer
}

// NewStopper returns an instance of Stopper.
func NewStopper() *Stopper {
	return &Stopper{
		stopper: make(chan struct{}),
	}
}

// RunWorker runs the supplied function as a "worker" to be stopped
// by the stopper. The function <f> is run in a goroutine.
func (s *Stopper) RunWorker(f func()) {
	s.AddWorker()
	go func() {
		defer s.SetStopped()
		f()
	}()
}

// AddWorker worker to the stopper.
func (s *Stopper) AddWorker() {
	s.stop.Add(1)
}

// AddCloser adds an object to close after the stopper has been stopped.
func (s *Stopper) AddCloser(c Closer) {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.closers = append(s.closers, c)
}

// StartTask adds one to the count of tasks left to drain in the
// system. Any worker which is a "first mover" when starting tasks
// must call this method before starting work on a new task and must
// subsequently invoke FinishTask() when the task is complete.
// First movers include goroutines launched to do periodic work and
// the kv/db.go gateway which accepts external client
// requests.
//
// Returns true if the task can be launched or false to indicate the
// system is currently draining and the task should be refused.
func (s *Stopper) StartTask() bool {
	if atomic.LoadInt32(&s.draining) == 0 {
		s.drain.Add(1)
		return true
	}
	return false
}

// FinishTask removes one from the count of tasks left to drain in the
// system. This function must be invoked for every call to StartTask().
func (s *Stopper) FinishTask() {
	s.drain.Done()
}

// Stop signals all live workers to stop and then waits for each to
// confirm it has stopped (workers do this by calling SetStopped()).
func (s *Stopper) Stop() {
	atomic.StoreInt32(&s.draining, 1)
	s.drain.Wait()
	close(s.stopper)
	s.stop.Wait()
	s.mu.Lock()
	defer s.mu.Unlock()
	for _, c := range s.closers {
		c.Close()
	}
}

// ShouldStop returns a channel which will be closed when Stop() has
// been invoked. SetStopped() should be called to confirm.
func (s *Stopper) ShouldStop() <-chan struct{} {
	if s == nil {
		// A nil stopper will never signal ShouldStop, but will also never panic.
		return nil
	}
	return s.stopper
}

// SetStopped should be called after the ShouldStop() channel has
// been closed to confirm the worker has stopped.
func (s *Stopper) SetStopped() {
	if s != nil {
		s.stop.Done()
	}
}

// Quiesce moves the stopper to state draining, waits until all tasks
// complete, then moves back to non-draining state. This is used from
// unittests.
func (s *Stopper) Quiesce() {
	atomic.StoreInt32(&s.draining, 1)
	defer atomic.StoreInt32(&s.draining, 0)
	s.drain.Wait()
}
<commit_msg>Small typo in comments of Stopper.AddWorker()<commit_after>// Copyright 2014 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.
//
// Author: Spencer Kimball (spencer.kimball@gmail.com)

package util

import (
	"sync"
	"sync/atomic"
)

// Closer is an interface for objects to attach to the stopper to
// be closed once the stopper completes.
type Closer interface {
	Close()
}

// A Stopper provides a channel-based mechanism to stop an arbitrary
// array of workers. Each worker is registered with the stopper via
// the AddWorker() method. The system further tracks each task which
// is outstanding by calling StartTask() when a task is started and
// FinishTask() when completed.
//
// Stopping occurs in two phases: the first is the request to stop,
// which moves the stopper into a draining phase. While draining,
// calls to StartTask() return false, meaning the system is draining
// and new tasks should not be accepted. When all outstanding tasks
// have been completed via calls to FinishTask(), the stopper closes
// its stopper channel, which signals all live workers that it's safe
// to shut down. Once shutdown, each worker invokes SetStopped(). When
// all workers have shutdown, the stopper is complete.
//
// An arbitrary list of objects implementing the Closer interface may
// be added to the stopper via AddCloser(), to be closed after the
// stopper has stopped.
type Stopper struct {
	stopper  chan struct{}
	draining int32 // Uses atomic operations instead of mu.
	drain    sync.WaitGroup
	stop     sync.WaitGroup
	mu       sync.Mutex // Protects the slice of Closers
	closers  []Closer
}

// NewStopper returns an instance of Stopper.
func NewStopper() *Stopper {
	return &Stopper{
		stopper: make(chan struct{}),
	}
}

// RunWorker runs the supplied function as a "worker" to be stopped
// by the stopper. The function <f> is run in a goroutine.
func (s *Stopper) RunWorker(f func()) {
	s.AddWorker()
	go func() {
		defer s.SetStopped()
		f()
	}()
}

// AddWorker adds a worker to the stopper.
func (s *Stopper) AddWorker() {
	s.stop.Add(1)
}

// AddCloser adds an object to close after the stopper has been stopped.
func (s *Stopper) AddCloser(c Closer) {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.closers = append(s.closers, c)
}

// StartTask adds one to the count of tasks left to drain in the
// system. Any worker which is a "first mover" when starting tasks
// must call this method before starting work on a new task and must
// subsequently invoke FinishTask() when the task is complete.
// First movers include goroutines launched to do periodic work and
// the kv/db.go gateway which accepts external client
// requests.
//
// Returns true if the task can be launched or false to indicate the
// system is currently draining and the task should be refused.
func (s *Stopper) StartTask() bool {
	if atomic.LoadInt32(&s.draining) == 0 {
		s.drain.Add(1)
		return true
	}
	return false
}

// FinishTask removes one from the count of tasks left to drain in the
// system. This function must be invoked for every call to StartTask().
func (s *Stopper) FinishTask() {
	s.drain.Done()
}

// Stop signals all live workers to stop and then waits for each to
// confirm it has stopped (workers do this by calling SetStopped()).
func (s *Stopper) Stop() {
	atomic.StoreInt32(&s.draining, 1)
	s.drain.Wait()
	close(s.stopper)
	s.stop.Wait()
	s.mu.Lock()
	defer s.mu.Unlock()
	for _, c := range s.closers {
		c.Close()
	}
}

// ShouldStop returns a channel which will be closed when Stop() has
// been invoked. SetStopped() should be called to confirm.
func (s *Stopper) ShouldStop() <-chan struct{} {
	if s == nil {
		// A nil stopper will never signal ShouldStop, but will also never panic.
		return nil
	}
	return s.stopper
}

// SetStopped should be called after the ShouldStop() channel has
// been closed to confirm the worker has stopped.
func (s *Stopper) SetStopped() {
	if s != nil {
		s.stop.Done()
	}
}

// Quiesce moves the stopper to state draining, waits until all tasks
// complete, then moves back to non-draining state. This is used from
// unittests.
func (s *Stopper) Quiesce() {
	atomic.StoreInt32(&s.draining, 1)
	defer atomic.StoreInt32(&s.draining, 0)
	s.drain.Wait()
}
<|endoftext|>


<commit_before>// Copyright 2016 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package https provides a helper for starting an HTTPS server.
package https

import (
	"crypto/tls"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"path/filepath"

	"golang.org/x/crypto/acme/autocert"
	gContext "golang.org/x/net/context"
	"google.golang.org/api/option"

	"cloud.google.com/go/compute/metadata"
	"cloud.google.com/go/storage"

	"upspin.io/access"
	"upspin.io/errors"
	"upspin.io/flags"
	"upspin.io/log"
	"upspin.io/serverutil"
)

// Options permits the configuration of TLS certificates for servers running
// outside GCE. The default is the self-signed certificate in
// upspin.io/rpc/testdata.
type Options struct {
	// LetsEncryptCache specifies the cache file for Let's Encrypt.
	// If non-empty, enables Let's Encrypt certificates for this server.
	LetsEncryptCache string

	// LetsEncryptHosts specifies the list of hosts for which we should
	// obtain TLS certificates through Let's Encrypt. If LetsEncryptCache
	// is specified this should be specified also.
	LetsEncryptHosts []string

	// CertFile and KeyFile specifies the TLS certificates to use.
	// It has no effect if LetsEncryptCache is non-empty.
	CertFile string
	KeyFile  string
}

var defaultOptions = &Options{
	CertFile: filepath.Join(os.Getenv("GOPATH"), "/src/upspin.io/rpc/testdata/cert.pem"),
	KeyFile:  filepath.Join(os.Getenv("GOPATH"), "/src/upspin.io/rpc/testdata/key.pem"),
}

func (opt *Options) applyDefaults() {
	if opt.CertFile == "" {
		opt.CertFile = defaultOptions.CertFile
	}
	if opt.KeyFile == "" {
		opt.KeyFile = defaultOptions.KeyFile
	}
}

// ListenAndServe serves the http.DefaultServeMux by HTTPS (and HTTP,
// redirecting to HTTPS), storing SSL credentials in the Google Cloud Storage
// buckets letsencrypt*.
//
// If the server is running outside GCE, instead an HTTPS server is started on
// the address specified by addr using the certificate details specified by opt.
//
// The given channel, if any, is closed when the TCP listener has succeeded.
// It may be used to signal that the server is ready to start serving requests.
//
// ListenAndServe does not return. It exits the program when the server is
// shut down (via SIGTERM or due to an error) and calls serverutil.Shutdown.
func ListenAndServe(ready chan<- struct{}, serverName, addr string, opt *Options) {
	if opt == nil {
		opt = defaultOptions
	} else {
		opt.applyDefaults()
	}

	var m autocert.Manager
	m.Prompt = autocert.AcceptTOS
	if h := opt.LetsEncryptHosts; len(h) > 0 {
		m.HostPolicy = autocert.HostWhitelist(h...)
	}

	var config *tls.Config
	if file := opt.LetsEncryptCache; file != "" {
		log.Info.Printf("https: serving HTTPS on %q using Let's Encrypt certificates", addr)
		m.Cache = autocert.DirCache(file)
		config = &tls.Config{GetCertificate: m.GetCertificate}
	} else if metadata.OnGCE() {
		addr = ":443"
		log.Info.Printf("https: serving HTTPS on GCE %q using Let's Encrypt certificates", addr)
		const key = "letsencrypt-bucket"
		bucket, err := metadata.InstanceAttributeValue(key)
		if err != nil {
			log.Fatalf("https: couldn't read %q metadata value: %v", key, err)
		}
		cache, err := newAutocertCache(bucket, serverName)
		if err != nil {
			log.Fatalf("https: couldn't set up letsencrypt cache: %v", err)
		}
		m.Cache = cache
		config = &tls.Config{GetCertificate: m.GetCertificate}
	} else {
		log.Info.Printf("https: not on GCE; serving HTTPS on %q using provided certificates", addr)
		if opt.CertFile == defaultOptions.CertFile || opt.KeyFile == defaultOptions.KeyFile {
			log.Error.Print("https: WARNING: using self-signed test certificates.")
		}
		var err error
		config, err = newDefaultTLSConfig(opt.CertFile, opt.KeyFile)
		if err != nil {
			log.Fatalf("https: setting up TLS config: %v", err)
		}
	}
	server := &http.Server{
		// ReadTimeout:  15 * time.Second,
		// WriteTimeout: 15 * time.Second,
		// IdleTimeout:  60 * time.Second,
		TLSConfig: config,
	}
	// TODO(adg): enable HTTP/2 once it's fast enough
	//err := http2.ConfigureServer(server, nil)
	//if err != nil {
	//	log.Fatalf("https: %v", err)
	//}

	ln, err := net.Listen("tcp", addr)
	if err != nil {
		log.Fatalf("https: %v", err)
	}
	if ready != nil {
		close(ready)
	}
	serverutil.RegisterShutdown(func() {
		// Stop accepting connections and forces the server to stop
		// its serving loop.
		ln.Close()
	})
	err = server.Serve(tls.NewListener(ln, config))
	if err != nil {
		log.Printf("https: %v", err)
	}
	// Ensure we terminate cleanly.
	serverutil.Shutdown()
}

// ListenAndServeFromFlags is the same as ListenAndServe, but it determines the
// listen address and Options from command-line flags in the flags package.
func ListenAndServeFromFlags(ready chan<- struct{}, serverName string) {
	ListenAndServe(ready, serverName, flags.HTTPSAddr, &Options{
		LetsEncryptCache: flags.LetsEncryptCache,
		CertFile:         flags.TLSCertFile,
		KeyFile:          flags.TLSKeyFile,
	})
}

// newDefaultTLSConfig creates a new TLS config based on the certificate files given.
func newDefaultTLSConfig(certFile string, certKeyFile string) (*tls.Config, error) {
	const op = "cloud/https.newDefaultTLSConfig"
	certReadable, err := isReadableFile(certFile)
	if err != nil {
		return nil, errors.E(op, errors.Invalid, errors.Errorf("SSL certificate in %q: %q", certFile, err))
	}
	if !certReadable {
		return nil, errors.E(op, errors.Invalid, errors.Errorf("certificate file %q not readable", certFile))
	}
	keyReadable, err := isReadableFile(certKeyFile)
	if err != nil {
		return nil, errors.E(op, errors.Invalid, errors.Errorf("SSL key in %q: %v", certKeyFile, err))
	}
	if !keyReadable {
		return nil, errors.E(op, errors.Invalid, errors.Errorf("certificate key file %q not readable", certKeyFile))
	}

	cert, err := tls.LoadX509KeyPair(certFile, certKeyFile)
	if err != nil {
		return nil, errors.E(op, err)
	}

	tlsConfig := &tls.Config{
		CipherSuites: []uint16{
			tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
			tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
			tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
			tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
		},
		MinVersion:               tls.VersionTLS12,
		PreferServerCipherSuites: true, // Use our choice, not the client's choice
		CurvePreferences:         []tls.CurveID{tls.CurveP521, tls.CurveP384, tls.CurveP256, tls.X25519},
		Certificates:             []tls.Certificate{cert},
	}
	tlsConfig.BuildNameToCertificate()
	return tlsConfig, nil
}

// isReadableFile reports whether the file exists and is readable.
// If the error is non-nil, it means there might be a file or directory
// with that name but we cannot read it.
func isReadableFile(path string) (bool, error) {
	// Is it stattable and is it a plain file?
	info, err := os.Stat(path)
	if err != nil {
		if os.IsNotExist(err) {
			return false, nil // Item does not exist.
		}
		return false, err // Item is problematic.
	}
	if info.IsDir() {
		return false, errors.Str("is directory")
	}
	// Is it readable?
	fd, err := os.Open(path)
	if err != nil {
		return false, access.ErrPermissionDenied
	}
	fd.Close()
	return true, nil // Item exists and is readable.
}

// autocertCache implements autocert.Cache.
type autocertCache struct {
	b      *storage.BucketHandle
	server string
}

func newAutocertCache(bucket, prefix string) (cache autocertCache, err error) {
	ctx := gContext.Background()
	client, err := storage.NewClient(ctx, option.WithScopes(storage.ScopeFullControl))
	if err != nil {
		return
	}
	cache.b = client.Bucket(bucket)
	cache.server = prefix + "-"
	return
}

func (cache autocertCache) Get(ctx gContext.Context, name string) ([]byte, error) {
	r, err := cache.b.Object(cache.server + name).NewReader(ctx)
	if err == storage.ErrObjectNotExist {
		return nil, autocert.ErrCacheMiss
	}
	if err != nil {
		return nil, err
	}
	defer r.Close()
	return ioutil.ReadAll(r)
}

func (cache autocertCache) Put(ctx gContext.Context, name string, data []byte) error {
	// TODO(ehg) Do we need to add contentType="text/plain; charset=utf-8"?
	w := cache.b.Object(cache.server + name).NewWriter(ctx)
	_, err := w.Write(data)
	if err != nil {
		log.Printf("https: writing letsencrypt cache: %s %v", name, err)
	}
	if err := w.Close(); err != nil {
		log.Printf("https: writing letsencrypt cache: %s %v", name, err)
	}
	return err
}

func (cache autocertCache) Delete(ctx gContext.Context, name string) error {
	return cache.b.Object(cache.server + name).Delete(ctx)
}
<commit_msg>cloud/https: add timeouts for abuse prevention<commit_after>// Copyright 2016 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package https provides a helper for starting an HTTPS server.
package https

import (
	"crypto/tls"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"time"

	"golang.org/x/crypto/acme/autocert"
	gContext "golang.org/x/net/context"
	"google.golang.org/api/option"

	"cloud.google.com/go/compute/metadata"
	"cloud.google.com/go/storage"

	"upspin.io/access"
	"upspin.io/errors"
	"upspin.io/flags"
	"upspin.io/log"
	"upspin.io/serverutil"
)

// Options permits the configuration of TLS certificates for servers running
// outside GCE. The default is the self-signed certificate in
// upspin.io/rpc/testdata.
type Options struct {
	// LetsEncryptCache specifies the cache file for Let's Encrypt.
	// If non-empty, enables Let's Encrypt certificates for this server.
	LetsEncryptCache string

	// LetsEncryptHosts specifies the list of hosts for which we should
	// obtain TLS certificates through Let's Encrypt. If LetsEncryptCache
	// is specified this should be specified also.
	LetsEncryptHosts []string

	// CertFile and KeyFile specifies the TLS certificates to use.
	// It has no effect if LetsEncryptCache is non-empty.
	CertFile string
	KeyFile  string
}

var defaultOptions = &Options{
	CertFile: filepath.Join(os.Getenv("GOPATH"), "/src/upspin.io/rpc/testdata/cert.pem"),
	KeyFile:  filepath.Join(os.Getenv("GOPATH"), "/src/upspin.io/rpc/testdata/key.pem"),
}

func (opt *Options) applyDefaults() {
	if opt.CertFile == "" {
		opt.CertFile = defaultOptions.CertFile
	}
	if opt.KeyFile == "" {
		opt.KeyFile = defaultOptions.KeyFile
	}
}

// ListenAndServe serves the http.DefaultServeMux by HTTPS (and HTTP,
// redirecting to HTTPS), storing SSL credentials in the Google Cloud Storage
// buckets letsencrypt*.
//
// If the server is running outside GCE, instead an HTTPS server is started on
// the address specified by addr using the certificate details specified by opt.
//
// The given channel, if any, is closed when the TCP listener has succeeded.
// It may be used to signal that the server is ready to start serving requests.
//
// ListenAndServe does not return. It exits the program when the server is
// shut down (via SIGTERM or due to an error) and calls serverutil.Shutdown.
func ListenAndServe(ready chan<- struct{}, serverName, addr string, opt *Options) {
	if opt == nil {
		opt = defaultOptions
	} else {
		opt.applyDefaults()
	}

	var m autocert.Manager
	m.Prompt = autocert.AcceptTOS
	if h := opt.LetsEncryptHosts; len(h) > 0 {
		m.HostPolicy = autocert.HostWhitelist(h...)
	}

	var config *tls.Config
	if file := opt.LetsEncryptCache; file != "" {
		log.Info.Printf("https: serving HTTPS on %q using Let's Encrypt certificates", addr)
		m.Cache = autocert.DirCache(file)
		config = &tls.Config{GetCertificate: m.GetCertificate}
	} else if metadata.OnGCE() {
		addr = ":443"
		log.Info.Printf("https: serving HTTPS on GCE %q using Let's Encrypt certificates", addr)
		const key = "letsencrypt-bucket"
		bucket, err := metadata.InstanceAttributeValue(key)
		if err != nil {
			log.Fatalf("https: couldn't read %q metadata value: %v", key, err)
		}
		cache, err := newAutocertCache(bucket, serverName)
		if err != nil {
			log.Fatalf("https: couldn't set up letsencrypt cache: %v", err)
		}
		m.Cache = cache
		config = &tls.Config{GetCertificate: m.GetCertificate}
	} else {
		log.Info.Printf("https: not on GCE; serving HTTPS on %q using provided certificates", addr)
		if opt.CertFile == defaultOptions.CertFile || opt.KeyFile == defaultOptions.KeyFile {
			log.Error.Print("https: WARNING: using self-signed test certificates.")
		}
		var err error
		config, err = newDefaultTLSConfig(opt.CertFile, opt.KeyFile)
		if err != nil {
			log.Fatalf("https: setting up TLS config: %v", err)
		}
	}
	server := &http.Server{
		ReadHeaderTimeout: 5 * time.Second,
		ReadTimeout:       15 * time.Second,
		WriteTimeout:      15 * time.Second,
		IdleTimeout:       60 * time.Second,
		TLSConfig:         config,
	}
	// TODO(adg): enable HTTP/2 once it's fast enough
	//err := http2.ConfigureServer(server, nil)
	//if err != nil {
	//	log.Fatalf("https: %v", err)
	//}

	ln, err := net.Listen("tcp", addr)
	if err != nil {
		log.Fatalf("https: %v", err)
	}
	if ready != nil {
		close(ready)
	}
	serverutil.RegisterShutdown(func() {
		// Stop accepting connections and forces the server to stop
		// its serving loop.
		ln.Close()
	})
	err = server.Serve(tls.NewListener(ln, config))
	if err != nil {
		log.Printf("https: %v", err)
	}
	// Ensure we terminate cleanly.
	serverutil.Shutdown()
}

// ListenAndServeFromFlags is the same as ListenAndServe, but it determines the
// listen address and Options from command-line flags in the flags package.
func ListenAndServeFromFlags(ready chan<- struct{}, serverName string) {
	ListenAndServe(ready, serverName, flags.HTTPSAddr, &Options{
		LetsEncryptCache: flags.LetsEncryptCache,
		CertFile:         flags.TLSCertFile,
		KeyFile:          flags.TLSKeyFile,
	})
}

// newDefaultTLSConfig creates a new TLS config based on the certificate files given.
func newDefaultTLSConfig(certFile string, certKeyFile string) (*tls.Config, error) {
	const op = "cloud/https.newDefaultTLSConfig"
	certReadable, err := isReadableFile(certFile)
	if err != nil {
		return nil, errors.E(op, errors.Invalid, errors.Errorf("SSL certificate in %q: %q", certFile, err))
	}
	if !certReadable {
		return nil, errors.E(op, errors.Invalid, errors.Errorf("certificate file %q not readable", certFile))
	}
	keyReadable, err := isReadableFile(certKeyFile)
	if err != nil {
		return nil, errors.E(op, errors.Invalid, errors.Errorf("SSL key in %q: %v", certKeyFile, err))
	}
	if !keyReadable {
		return nil, errors.E(op, errors.Invalid, errors.Errorf("certificate key file %q not readable", certKeyFile))
	}

	cert, err := tls.LoadX509KeyPair(certFile, certKeyFile)
	if err != nil {
		return nil, errors.E(op, err)
	}

	tlsConfig := &tls.Config{
		CipherSuites: []uint16{
			tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
			tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
			tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
			tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
		},
		MinVersion:               tls.VersionTLS12,
		PreferServerCipherSuites: true, // Use our choice, not the client's choice
		CurvePreferences:         []tls.CurveID{tls.CurveP521, tls.CurveP384, tls.CurveP256, tls.X25519},
		Certificates:             []tls.Certificate{cert},
	}
	tlsConfig.BuildNameToCertificate()
	return tlsConfig, nil
}

// isReadableFile reports whether the file exists and is readable.
// If the error is non-nil, it means there might be a file or directory
// with that name but we cannot read it.
func isReadableFile(path string) (bool, error) {
	// Is it stattable and is it a plain file?
	info, err := os.Stat(path)
	if err != nil {
		if os.IsNotExist(err) {
			return false, nil // Item does not exist.
		}
		return false, err // Item is problematic.
	}
	if info.IsDir() {
		return false, errors.Str("is directory")
	}
	// Is it readable?
	fd, err := os.Open(path)
	if err != nil {
		return false, access.ErrPermissionDenied
	}
	fd.Close()
	return true, nil // Item exists and is readable.
}

// autocertCache implements autocert.Cache.
type autocertCache struct {
	b      *storage.BucketHandle
	server string
}

func newAutocertCache(bucket, prefix string) (cache autocertCache, err error) {
	ctx := gContext.Background()
	client, err := storage.NewClient(ctx, option.WithScopes(storage.ScopeFullControl))
	if err != nil {
		return
	}
	cache.b = client.Bucket(bucket)
	cache.server = prefix + "-"
	return
}

func (cache autocertCache) Get(ctx gContext.Context, name string) ([]byte, error) {
	r, err := cache.b.Object(cache.server + name).NewReader(ctx)
	if err == storage.ErrObjectNotExist {
		return nil, autocert.ErrCacheMiss
	}
	if err != nil {
		return nil, err
	}
	defer r.Close()
	return ioutil.ReadAll(r)
}

func (cache autocertCache) Put(ctx gContext.Context, name string, data []byte) error {
	// TODO(ehg) Do we need to add contentType="text/plain; charset=utf-8"?
	w := cache.b.Object(cache.server + name).NewWriter(ctx)
	_, err := w.Write(data)
	if err != nil {
		log.Printf("https: writing letsencrypt cache: %s %v", name, err)
	}
	if err := w.Close(); err != nil {
		log.Printf("https: writing letsencrypt cache: %s %v", name, err)
	}
	return err
}

func (cache autocertCache) Delete(ctx gContext.Context, name string) error {
	return cache.b.Object(cache.server + name).Delete(ctx)
}
<|endoftext|>


<commit_before>package vault

import (
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"path"
	"strings"
)

func NewRecipient(recipient string) (*VaultRecipient, error) {
	args := strings.Split(recipient, ":")

	if len(args) != 2 {
		return nil, fmt.Errorf("\nInvalid format \"%s\"\nRecipient should be in the form of fingerprint:name\n", recipient)
	}

	if len(args[0]) == 0 || len(args[1]) == 0 {
		return nil, fmt.Errorf("\nInvalid format \"%s\"\nRecipient should be in the form of fingerprint:name\n", recipient)
	}

	recipientFingerprint := strings.Split(recipient, ":")[0]

	if hexFingerprint, err := hex.DecodeString(recipientFingerprint); err != nil {
		return nil, fmt.Errorf("\nSupplied fingerprint \"%s\" does not have the correct format\n", hexFingerprint)
	} else {

		if len(hexFingerprint) != 16 && len(hexFingerprint) != 20 {
			return nil, fmt.Errorf("\nSupplied fingerprint \"%s\" does not have the correct size\n", hexFingerprint)
		}

	}

	recipientName := strings.Split(recipient, ":")[1]

	return &VaultRecipient{Fingerprint: recipientFingerprint, Name: recipientName}, nil
}

func LoadVaultfile() (*Vaultfile, error) {
	v := &Vaultfile{}
	content, err := ioutil.ReadFile(path.Join(GetHomeDir(), "Vaultfile"))
	if err != nil {
		return v, nil
	}
	if err := json.Unmarshal(content, v); err != nil {
		return nil, fmt.Errorf("\nCouldn't read Vaultfile.\n")
	} else {
		return v, nil
	}
}

type VaultRecipient struct {
	Name        string
	Fingerprint string
}

func (v VaultRecipient) ToString() string {
	return v.Fingerprint + ":" + v.Name
}

type Vaultfile struct {
	Recipients []VaultRecipient
}

func (v Vaultfile) Save() error {
	js, err := json.Marshal(v)
	if err != nil {
		return err
	}

	err2 := ioutil.WriteFile(path.Join(GetHomeDir(), "Vaultfile"), js, 0644)
	if err2 != nil {
		return err2
	}
	return nil
}
<commit_msg>Fix vault add message<commit_after>package vault

import (
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"path"
	"strings"
)

func NewRecipient(recipient string) (*VaultRecipient, error) {
	args := strings.Split(recipient, ":")

	if len(args) != 2 {
		return nil, fmt.Errorf("\nInvalid format \"%s\"\nRecipient should be in the form of fingerprint:name\n", recipient)
	}

	if len(args[0]) == 0 || len(args[1]) == 0 {
		return nil, fmt.Errorf("\nInvalid format \"%s\"\nRecipient should be in the form of fingerprint:name\n", recipient)
	}

	recipientFingerprint := strings.Split(recipient, ":")[0]

	if hexFingerprint, err := hex.DecodeString(recipientFingerprint); err != nil {
		return nil, fmt.Errorf("\nSupplied fingerprint \"%s\" does not have the correct format\n", recipientFingerprint)
	} else {

		if len(hexFingerprint) != 16 && len(hexFingerprint) != 20 {
			return nil, fmt.Errorf("\nSupplied fingerprint \"%s\" does not have the correct size\n", hexFingerprint)
		}

	}

	recipientName := strings.Split(recipient, ":")[1]

	return &VaultRecipient{Fingerprint: recipientFingerprint, Name: recipientName}, nil
}

func LoadVaultfile() (*Vaultfile, error) {
	v := &Vaultfile{}
	content, err := ioutil.ReadFile(path.Join(GetHomeDir(), "Vaultfile"))
	if err != nil {
		return v, nil
	}
	if err := json.Unmarshal(content, v); err != nil {
		return nil, fmt.Errorf("\nCouldn't read Vaultfile.\n")
	} else {
		return v, nil
	}
}

type VaultRecipient struct {
	Name        string
	Fingerprint string
}

func (v VaultRecipient) ToString() string {
	return v.Fingerprint + ":" + v.Name
}

type Vaultfile struct {
	Recipients []VaultRecipient
}

func (v Vaultfile) Save() error {
	js, err := json.Marshal(v)
	if err != nil {
		return err
	}

	err2 := ioutil.WriteFile(path.Join(GetHomeDir(), "Vaultfile"), js, 0644)
	if err2 != nil {
		return err2
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2013 docker-cluster authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cluster

import (
	"github.com/dotcloud/docker"
	dcli "github.com/fsouza/go-dockerclient"
	"sync"
	"sync/atomic"
)

// Scheduler represents a scheduling strategy, that will be used when creating
// new containers.
type Scheduler interface {
	// Schedule creates a new container, returning the ID of the node where
	// the container is running, and the container, or an error.
	Schedule(config *docker.Config) (string, *docker.Container, error)

	// Nodes returns a slice of nodes in the scheduler.
	Nodes() ([]Node, error)
}

// Registrable represents a scheduler that can get new nodes via the Register
// method.
type Registrable interface {
	// Register adds new nodes to the scheduler.
	Register(nodes ...Node) error
}

type node struct {
	*dcli.Client
	id  string
	edp string
}

type roundRobin struct {
	nodes    []node
	lastUsed int64
	mut      sync.RWMutex
}

func (s *roundRobin) Schedule(config *docker.Config) (string, *docker.Container, error) {
	node := s.next()
	container, err := node.CreateContainer(config)
	return node.id, container, err
}

func (s *roundRobin) next() node {
	s.mut.RLock()
	defer s.mut.RUnlock()
	if len(s.nodes) == 0 {
		panic("No nodes available")
	}
	index := atomic.AddInt64(&s.lastUsed, 1) % int64(len(s.nodes))
	return s.nodes[index]
}

func (s *roundRobin) Register(nodes ...Node) error {
	s.mut.Lock()
	defer s.mut.Unlock()
	if len(s.nodes) == 0 {
		s.lastUsed = -1
		s.nodes = make([]node, 0, len(nodes))
	}
	for _, n := range nodes {
		client, err := dcli.NewClient(n.Address)
		if err != nil {
			return err
		}
		s.nodes = append(s.nodes, node{Client: client, edp: n.Address, id: n.ID})
	}
	return nil
}

func (s *roundRobin) Nodes() ([]Node, error) {
	s.mut.RLock()
	defer s.mut.RUnlock()
	nodes := make([]Node, len(s.nodes))
	for i, node := range s.nodes {
		nodes[i] = Node{ID: node.id, Address: node.edp}
	}
	return nodes, nil
}
<commit_msg>cluster: fix call to CreateContainer<commit_after>// Copyright 2013 docker-cluster authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cluster

import (
	"github.com/dotcloud/docker"
	dcli "github.com/fsouza/go-dockerclient"
	"sync"
	"sync/atomic"
)

// Scheduler represents a scheduling strategy, that will be used when creating
// new containers.
type Scheduler interface {
	// Schedule creates a new container, returning the ID of the node where
	// the container is running, and the container, or an error.
	Schedule(config *docker.Config) (string, *docker.Container, error)

	// Nodes returns a slice of nodes in the scheduler.
	Nodes() ([]Node, error)
}

// Registrable represents a scheduler that can get new nodes via the Register
// method.
type Registrable interface {
	// Register adds new nodes to the scheduler.
	Register(nodes ...Node) error
}

type node struct {
	*dcli.Client
	id  string
	edp string
}

type roundRobin struct {
	nodes    []node
	lastUsed int64
	mut      sync.RWMutex
}

func (s *roundRobin) Schedule(config *docker.Config) (string, *docker.Container, error) {
	node := s.next()
	container, err := node.CreateContainer(dcli.CreateContainerOptions{}, config)
	return node.id, container, err
}

func (s *roundRobin) next() node {
	s.mut.RLock()
	defer s.mut.RUnlock()
	if len(s.nodes) == 0 {
		panic("No nodes available")
	}
	index := atomic.AddInt64(&s.lastUsed, 1) % int64(len(s.nodes))
	return s.nodes[index]
}

func (s *roundRobin) Register(nodes ...Node) error {
	s.mut.Lock()
	defer s.mut.Unlock()
	if len(s.nodes) == 0 {
		s.lastUsed = -1
		s.nodes = make([]node, 0, len(nodes))
	}
	for _, n := range nodes {
		client, err := dcli.NewClient(n.Address)
		if err != nil {
			return err
		}
		s.nodes = append(s.nodes, node{Client: client, edp: n.Address, id: n.ID})
	}
	return nil
}

func (s *roundRobin) Nodes() ([]Node, error) {
	s.mut.RLock()
	defer s.mut.RUnlock()
	nodes := make([]Node, len(s.nodes))
	for i, node := range s.nodes {
		nodes[i] = Node{ID: node.id, Address: node.edp}
	}
	return nodes, nil
}
<|endoftext|>


<commit_before><commit_msg>Handle errors in manifest set-up and already replicated items in DPN copier<commit_after><|endoftext|>