<commit_before>// Copyright 2016 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"fmt"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/common/log"
	"github.com/prometheus/common/route"
	"github.com/prometheus/prometheus/promql"
	"github.com/prometheus/prometheus/web/api/v1"
	"github.com/weaveworks/scope/common/middleware"
	"golang.org/x/net/context"

	"github.com/weaveworks/cortex"
	"github.com/weaveworks/cortex/chunk"
	"github.com/weaveworks/cortex/ingester"
	"github.com/weaveworks/cortex/ring"
	"github.com/weaveworks/cortex/ui"
	"github.com/weaveworks/cortex/user"
)

const (
	modeDistributor = "distributor"
	modeIngester    = "ingester"

	infName          = "eth0"
	userIDHeaderName = "X-Scope-OrgID"
)

var (
	requestDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Namespace: "cortex",
		Name:      "request_duration_seconds",
		Help:      "Time (in seconds) spent serving HTTP requests.",
		Buckets:   prometheus.DefBuckets,
	}, []string{"method", "route", "status_code", "ws"})
)

func init() {
	prometheus.MustRegister(requestDuration)
}

type cfg struct {
	mode                 string
	listenPort           int
	consulHost           string
	consulPrefix         string
	s3URL                string
	dynamodbURL          string
	dynamodbCreateTables bool
	memcachedHostname    string
	memcachedTimeout     time.Duration
	memcachedExpiration  time.Duration
	memcachedService     string
	remoteTimeout        time.Duration
	numTokens            int
	logSuccess           bool

	ingesterConfig    ingester.Config
	distributorConfig cortex.DistributorConfig
}

func main() {
	var cfg cfg
	flag.StringVar(&cfg.mode, "mode", modeDistributor, "Mode (distributor, ingester).")
	flag.IntVar(&cfg.listenPort, "web.listen-port", 9094, "HTTP server listen port.")
	flag.StringVar(&cfg.consulHost, "consul.hostname", "localhost:8500", "Hostname and port of Consul.")
	flag.StringVar(&cfg.consulPrefix, "consul.prefix", "collectors/", "Prefix for keys in Consul.")
	flag.StringVar(&cfg.s3URL, "s3.url", "localhost:4569", "S3 endpoint URL.")
	flag.StringVar(&cfg.dynamodbURL, "dynamodb.url", "localhost:8000", "DynamoDB endpoint URL.")
	flag.BoolVar(&cfg.dynamodbCreateTables, "dynamodb.create-tables", false, "Create required DynamoDB tables on startup.")
	flag.StringVar(&cfg.memcachedHostname, "memcached.hostname", "", "Hostname for memcached service to use when caching chunks. If empty, no memcached will be used.")
	flag.DurationVar(&cfg.memcachedTimeout, "memcached.timeout", 100*time.Millisecond, "Maximum time to wait before giving up on memcached requests.")
	flag.DurationVar(&cfg.memcachedExpiration, "memcached.expiration", 0, "How long chunks stay in the memcache.")
	flag.StringVar(&cfg.memcachedService, "memcached.service", "memcached", "SRV service used to discover memcache servers.")
	flag.DurationVar(&cfg.remoteTimeout, "remote.timeout", 5*time.Second, "Timeout for downstream ingesters.")
	flag.DurationVar(&cfg.ingesterConfig.FlushCheckPeriod, "ingester.flush-period", 1*time.Minute, "Period with which to attempt to flush chunks.")
	flag.DurationVar(&cfg.ingesterConfig.RateUpdatePeriod, "ingester.rate-update-period", 15*time.Second, "Period with which to update the per-user ingestion rates.")
	flag.DurationVar(&cfg.ingesterConfig.MaxChunkAge, "ingester.max-chunk-age", 10*time.Minute, "Maximum chunk age before flushing.")
	flag.IntVar(&cfg.numTokens, "ingester.num-tokens", 128, "Number of tokens for each ingester.")
	flag.IntVar(&cfg.distributorConfig.ReplicationFactor, "distributor.replication-factor", 3, "The number of ingesters to write to and read from.")
	flag.IntVar(&cfg.distributorConfig.MinReadSuccesses, "distributor.min-read-successes", 2, "The minimum number of ingesters from which a read must succeed.")
	flag.IntVar(&cfg.distributorConfig.MinWriteSuccesses, "distributor.min-write-successes", 2, "The minimum number of ingesters to which a write must succeed.")
	flag.DurationVar(&cfg.distributorConfig.HeartbeatTimeout, "distributor.heartbeat-timeout", time.Minute, "The heartbeat timeout after which ingesters are skipped for reads/writes.")
	flag.BoolVar(&cfg.logSuccess, "log.success", false, "Log successful requests")
	flag.Parse()

	chunkStore, err := setupChunkStore(cfg)
	if err != nil {
		log.Fatalf("Error initializing chunk store: %v", err)
	}

	consul, err := ring.NewConsulClient(cfg.consulHost)
	if err != nil {
		log.Fatalf("Error initializing Consul client: %v", err)
	}
	consul = ring.PrefixClient(consul, cfg.consulPrefix)

	switch cfg.mode {
	case modeDistributor:
		ring := ring.New(consul)
		cfg.distributorConfig.Ring = ring
		cfg.distributorConfig.ClientFactory = func(address string) (*cortex.IngesterClient, error) {
			return cortex.NewIngesterClient(address, cfg.remoteTimeout)
		}
		defer ring.Stop()
		setupDistributor(cfg.distributorConfig, chunkStore, cfg.logSuccess)
		if err != nil {
			log.Fatalf("Error initializing distributor: %v", err)
		}
	case modeIngester:
		registration, err := ring.RegisterIngester(consul, cfg.listenPort, cfg.numTokens)
		prometheus.MustRegister(registration)
		if err != nil {
			// This only happens for errors in configuration & set-up, not for
			// network errors.
			log.Fatalf("Could not register ingester: %v", err)
		}
		defer registration.Unregister()
		ing := setupIngester(chunkStore, cfg.ingesterConfig, cfg.logSuccess)
		defer ing.Stop()
	default:
		log.Fatalf("Mode %s not supported!", cfg.mode)
	}

	http.Handle("/metrics", prometheus.Handler())
	go http.ListenAndServe(fmt.Sprintf(":%d", cfg.listenPort), nil)

	term := make(chan os.Signal)
	signal.Notify(term, os.Interrupt, syscall.SIGTERM)
	<-term
	log.Warn("Received SIGTERM, exiting gracefully...")
}

func setupChunkStore(cfg cfg) (chunk.Store, error) {
	var chunkCache *chunk.Cache
	if cfg.memcachedHostname != "" {
		chunkCache = &chunk.Cache{
			Memcache: chunk.NewMemcacheClient(chunk.MemcacheConfig{
				Host:           cfg.memcachedHostname,
				Service:        cfg.memcachedService,
				Timeout:        cfg.memcachedTimeout,
				UpdateInterval: 1 * time.Minute,
			}),
			Expiration: cfg.memcachedExpiration,
		}
	}
	chunkStore, err := chunk.NewAWSStore(chunk.StoreConfig{
		S3URL:       cfg.s3URL,
		DynamoDBURL: cfg.dynamodbURL,
		ChunkCache:  chunkCache,
	})
	if err != nil {
		return nil, err
	}
	if cfg.dynamodbCreateTables {
		if err = chunkStore.CreateTables(); err != nil {
			return nil, err
		}
	}
	return chunkStore, err
}

func setupDistributor(
	cfg cortex.DistributorConfig,
	chunkStore chunk.Store,
	logSuccess bool,
) {
	distributor, err := cortex.NewDistributor(cfg)
	if err != nil {
		log.Fatal(err)
	}
	prometheus.MustRegister(distributor)

	prefix := "/api/prom"
	http.Handle(prefix+"/push", instrument(logSuccess, cortex.AppenderHandler(distributor)))

	// TODO: Move querier to separate binary.
	setupQuerier(distributor, chunkStore, prefix, logSuccess)
}

// setupQuerier sets up a complete querying pipeline:
//
// PromQL -> MergeQuerier -> Distributor -> IngesterQuerier -> Ingester
//              |
//              `----------> ChunkQuerier -> DynamoDB/S3
func setupQuerier(
	distributor *cortex.Distributor,
	chunkStore chunk.Store,
	prefix string,
	logSuccess bool,
) {
	querier := cortex.MergeQuerier{
		Queriers: []cortex.Querier{
			distributor,
			&cortex.ChunkQuerier{
				Store: chunkStore,
			},
		},
	}
	engine := promql.NewEngine(querier, nil)

	api := v1.NewAPI(engine, querier)
	router := route.New(func(r *http.Request) (context.Context, error) {
		userID := r.Header.Get(userIDHeaderName)
		if r.Method != "OPTIONS" && userID == "" {
			// For now, getting the user ID from basic auth allows for easy testing
			// with Grafana.
			// TODO: Remove basic auth support.
			userID, _, _ = r.BasicAuth()
			if userID == "" {
				return nil, fmt.Errorf("missing user ID")
			}
		}
		return user.WithID(context.Background(), userID), nil
	})
	api.Register(router.WithPrefix(prefix + "/api/v1"))
	http.Handle("/", router)

	http.Handle(prefix+"/user_stats", instrument(logSuccess, cortex.DistributorUserStatsHandler(distributor.UserStats)))

	http.Handle(prefix+"/graph", instrument(logSuccess, ui.GraphHandler()))
	http.Handle(prefix+"/static/", instrument(logSuccess, ui.StaticAssetsHandler(prefix+"/static/")))
}

func setupIngester(
	chunkStore chunk.Store,
	cfg ingester.Config,
	logSuccess bool,
) *ingester.Ingester {
	ingester, err := ingester.New(cfg, chunkStore)
	if err != nil {
		log.Fatal(err)
	}
	prometheus.MustRegister(ingester)

	http.Handle("/push", instrument(logSuccess, cortex.AppenderHandler(ingester)))
	http.Handle("/query", instrument(logSuccess, cortex.QueryHandler(ingester)))
	http.Handle("/label_values", instrument(logSuccess, cortex.LabelValuesHandler(ingester)))
	http.Handle("/user_stats", instrument(logSuccess, cortex.IngesterUserStatsHandler(ingester.UserStats)))
	return ingester
}

// instrument instruments a handler.
func instrument(logSuccess bool, handler http.Handler) http.Handler {
	return middleware.Merge(
		middleware.Log{
			LogSuccess: logSuccess,
		},
		middleware.Instrument{
			Duration: requestDuration,
		},
	).Wrap(handler)
}
<commit_msg>Juggle around order of operations to rudec scope<commit_after>// Copyright 2016 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"fmt"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/common/log"
	"github.com/prometheus/common/route"
	"github.com/prometheus/prometheus/promql"
	"github.com/prometheus/prometheus/web/api/v1"
	"github.com/weaveworks/scope/common/middleware"
	"golang.org/x/net/context"

	"github.com/weaveworks/cortex"
	"github.com/weaveworks/cortex/chunk"
	"github.com/weaveworks/cortex/ingester"
	"github.com/weaveworks/cortex/ring"
	"github.com/weaveworks/cortex/ui"
	"github.com/weaveworks/cortex/user"
)

const (
	modeDistributor = "distributor"
	modeIngester    = "ingester"

	infName          = "eth0"
	userIDHeaderName = "X-Scope-OrgID"
)

var (
	requestDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Namespace: "cortex",
		Name:      "request_duration_seconds",
		Help:      "Time (in seconds) spent serving HTTP requests.",
		Buckets:   prometheus.DefBuckets,
	}, []string{"method", "route", "status_code", "ws"})
)

func init() {
	prometheus.MustRegister(requestDuration)
}

type cfg struct {
	mode                 string
	listenPort           int
	consulHost           string
	consulPrefix         string
	s3URL                string
	dynamodbURL          string
	dynamodbCreateTables bool
	memcachedHostname    string
	memcachedTimeout     time.Duration
	memcachedExpiration  time.Duration
	memcachedService     string
	remoteTimeout        time.Duration
	numTokens            int
	logSuccess           bool

	ingesterConfig    ingester.Config
	distributorConfig cortex.DistributorConfig
}

func main() {
	var cfg cfg
	flag.StringVar(&cfg.mode, "mode", modeDistributor, "Mode (distributor, ingester).")
	flag.IntVar(&cfg.listenPort, "web.listen-port", 9094, "HTTP server listen port.")
	flag.StringVar(&cfg.consulHost, "consul.hostname", "localhost:8500", "Hostname and port of Consul.")
	flag.StringVar(&cfg.consulPrefix, "consul.prefix", "collectors/", "Prefix for keys in Consul.")
	flag.StringVar(&cfg.s3URL, "s3.url", "localhost:4569", "S3 endpoint URL.")
	flag.StringVar(&cfg.dynamodbURL, "dynamodb.url", "localhost:8000", "DynamoDB endpoint URL.")
	flag.BoolVar(&cfg.dynamodbCreateTables, "dynamodb.create-tables", false, "Create required DynamoDB tables on startup.")
	flag.StringVar(&cfg.memcachedHostname, "memcached.hostname", "", "Hostname for memcached service to use when caching chunks. If empty, no memcached will be used.")
	flag.DurationVar(&cfg.memcachedTimeout, "memcached.timeout", 100*time.Millisecond, "Maximum time to wait before giving up on memcached requests.")
	flag.DurationVar(&cfg.memcachedExpiration, "memcached.expiration", 0, "How long chunks stay in the memcache.")
	flag.StringVar(&cfg.memcachedService, "memcached.service", "memcached", "SRV service used to discover memcache servers.")
	flag.DurationVar(&cfg.remoteTimeout, "remote.timeout", 5*time.Second, "Timeout for downstream ingesters.")
	flag.DurationVar(&cfg.ingesterConfig.FlushCheckPeriod, "ingester.flush-period", 1*time.Minute, "Period with which to attempt to flush chunks.")
	flag.DurationVar(&cfg.ingesterConfig.RateUpdatePeriod, "ingester.rate-update-period", 15*time.Second, "Period with which to update the per-user ingestion rates.")
	flag.DurationVar(&cfg.ingesterConfig.MaxChunkAge, "ingester.max-chunk-age", 10*time.Minute, "Maximum chunk age before flushing.")
	flag.IntVar(&cfg.numTokens, "ingester.num-tokens", 128, "Number of tokens for each ingester.")
	flag.IntVar(&cfg.distributorConfig.ReplicationFactor, "distributor.replication-factor", 3, "The number of ingesters to write to and read from.")
	flag.IntVar(&cfg.distributorConfig.MinReadSuccesses, "distributor.min-read-successes", 2, "The minimum number of ingesters from which a read must succeed.")
	flag.IntVar(&cfg.distributorConfig.MinWriteSuccesses, "distributor.min-write-successes", 2, "The minimum number of ingesters to which a write must succeed.")
	flag.DurationVar(&cfg.distributorConfig.HeartbeatTimeout, "distributor.heartbeat-timeout", time.Minute, "The heartbeat timeout after which ingesters are skipped for reads/writes.")
	flag.BoolVar(&cfg.logSuccess, "log.success", false, "Log successful requests")
	flag.Parse()

	chunkStore, err := setupChunkStore(cfg)
	if err != nil {
		log.Fatalf("Error initializing chunk store: %v", err)
	}

	consul, err := ring.NewConsulClient(cfg.consulHost)
	if err != nil {
		log.Fatalf("Error initializing Consul client: %v", err)
	}
	consul = ring.PrefixClient(consul, cfg.consulPrefix)

	switch cfg.mode {
	case modeDistributor:
		ring := ring.New(consul)
		cfg.distributorConfig.Ring = ring
		cfg.distributorConfig.ClientFactory = func(address string) (*cortex.IngesterClient, error) {
			return cortex.NewIngesterClient(address, cfg.remoteTimeout)
		}
		defer ring.Stop()
		setupDistributor(cfg.distributorConfig, chunkStore, cfg.logSuccess)
	case modeIngester:
		registration, err := ring.RegisterIngester(consul, cfg.listenPort, cfg.numTokens)
		if err != nil {
			// This only happens for errors in configuration & set-up, not for
			// network errors.
			log.Fatalf("Could not register ingester: %v", err)
		}
		defer registration.Unregister()
		prometheus.MustRegister(registration)
		ing := setupIngester(chunkStore, cfg.ingesterConfig, cfg.logSuccess)
		defer ing.Stop()
	default:
		log.Fatalf("Mode %s not supported!", cfg.mode)
	}

	http.Handle("/metrics", prometheus.Handler())
	go http.ListenAndServe(fmt.Sprintf(":%d", cfg.listenPort), nil)

	term := make(chan os.Signal)
	signal.Notify(term, os.Interrupt, syscall.SIGTERM)
	<-term
	log.Warn("Received SIGTERM, exiting gracefully...")
}

func setupChunkStore(cfg cfg) (chunk.Store, error) {
	var chunkCache *chunk.Cache
	if cfg.memcachedHostname != "" {
		chunkCache = &chunk.Cache{
			Memcache: chunk.NewMemcacheClient(chunk.MemcacheConfig{
				Host:           cfg.memcachedHostname,
				Service:        cfg.memcachedService,
				Timeout:        cfg.memcachedTimeout,
				UpdateInterval: 1 * time.Minute,
			}),
			Expiration: cfg.memcachedExpiration,
		}
	}
	chunkStore, err := chunk.NewAWSStore(chunk.StoreConfig{
		S3URL:       cfg.s3URL,
		DynamoDBURL: cfg.dynamodbURL,
		ChunkCache:  chunkCache,
	})
	if err != nil {
		return nil, err
	}
	if cfg.dynamodbCreateTables {
		if err = chunkStore.CreateTables(); err != nil {
			return nil, err
		}
	}
	return chunkStore, err
}

func setupDistributor(
	cfg cortex.DistributorConfig,
	chunkStore chunk.Store,
	logSuccess bool,
) {
	distributor, err := cortex.NewDistributor(cfg)
	if err != nil {
		log.Fatal(err)
	}
	prometheus.MustRegister(distributor)

	prefix := "/api/prom"
	http.Handle(prefix+"/push", instrument(logSuccess, cortex.AppenderHandler(distributor)))

	// TODO: Move querier to separate binary.
	setupQuerier(distributor, chunkStore, prefix, logSuccess)
}

// setupQuerier sets up a complete querying pipeline:
//
// PromQL -> MergeQuerier -> Distributor -> IngesterQuerier -> Ingester
//              |
//              `----------> ChunkQuerier -> DynamoDB/S3
func setupQuerier(
	distributor *cortex.Distributor,
	chunkStore chunk.Store,
	prefix string,
	logSuccess bool,
) {
	querier := cortex.MergeQuerier{
		Queriers: []cortex.Querier{
			distributor,
			&cortex.ChunkQuerier{
				Store: chunkStore,
			},
		},
	}
	engine := promql.NewEngine(querier, nil)

	api := v1.NewAPI(engine, querier)
	router := route.New(func(r *http.Request) (context.Context, error) {
		userID := r.Header.Get(userIDHeaderName)
		if r.Method != "OPTIONS" && userID == "" {
			// For now, getting the user ID from basic auth allows for easy testing
			// with Grafana.
			// TODO: Remove basic auth support.
			userID, _, _ = r.BasicAuth()
			if userID == "" {
				return nil, fmt.Errorf("missing user ID")
			}
		}
		return user.WithID(context.Background(), userID), nil
	})
	api.Register(router.WithPrefix(prefix + "/api/v1"))
	http.Handle("/", router)

	http.Handle(prefix+"/user_stats", instrument(logSuccess, cortex.DistributorUserStatsHandler(distributor.UserStats)))

	http.Handle(prefix+"/graph", instrument(logSuccess, ui.GraphHandler()))
	http.Handle(prefix+"/static/", instrument(logSuccess, ui.StaticAssetsHandler(prefix+"/static/")))
}

func setupIngester(
	chunkStore chunk.Store,
	cfg ingester.Config,
	logSuccess bool,
) *ingester.Ingester {
	ingester, err := ingester.New(cfg, chunkStore)
	if err != nil {
		log.Fatal(err)
	}
	prometheus.MustRegister(ingester)

	http.Handle("/push", instrument(logSuccess, cortex.AppenderHandler(ingester)))
	http.Handle("/query", instrument(logSuccess, cortex.QueryHandler(ingester)))
	http.Handle("/label_values", instrument(logSuccess, cortex.LabelValuesHandler(ingester)))
	http.Handle("/user_stats", instrument(logSuccess, cortex.IngesterUserStatsHandler(ingester.UserStats)))
	return ingester
}

// instrument instruments a handler.
func instrument(logSuccess bool, handler http.Handler) http.Handler {
	return middleware.Merge(
		middleware.Log{
			LogSuccess: logSuccess,
		},
		middleware.Instrument{
			Duration: requestDuration,
		},
	).Wrap(handler)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/google/uuid"
	"github.com/musl/gofr/lib/gofr"
	"github.com/nfnt/resize"
	"image"
	"image/png"
	"io"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"strconv"
	"sync"
	"time"
)

const Version = "0.1.1"

var render_jobs = make(map[string]RenderJob)
var render_jobs_mutex = &sync.Mutex{}

type RenderJob struct {
	Parameters gofr.Parameters
	Cancel     chan bool
	Threads    int
}

func (self *RenderJob) Render() (image.Image, error) {
	img := image.NewNRGBA64(image.Rect(0, 0, self.Parameters.ImageWidth, self.Parameters.ImageHeight))
	contexts := gofr.MakeContexts(img, self.Threads, &self.Parameters)

	err := gofr.Render(self.Threads, contexts, self.Cancel)
	if err != nil {
		return nil, err
	}

	image := resize.Resize(self.Parameters.Width, self.Parameters.Height, image.Image(img), resize.Lanczos3)
	return image, nil
}

type LogResponseWriter struct {
	http.ResponseWriter
	Status int
	Start  time.Time
	End    time.Time
}

func NewLogResponseWriter(w http.ResponseWriter) *LogResponseWriter {
	return &LogResponseWriter{w, 0, time.Now(), time.Now()}
}

func (self *LogResponseWriter) WriteHeader(code int) {
	self.Status = code
	self.ResponseWriter.WriteHeader(code)
}

func (self LogResponseWriter) Log(message string) {
	self.End = time.Now()
	log.Printf("%s %v\n", message, self.End.Sub(self.Start))
}

func wrapHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		lrw := NewLogResponseWriter(w)

		lrw.Log(fmt.Sprintf("%s -> %s %s", r.Method, r.URL.Path, r.RemoteAddr))
		h.ServeHTTP(lrw, r)
		lrw.Log(fmt.Sprintf("%d <- %s %s %s", lrw.Status, r.Method, r.URL.Path, r.RemoteAddr))
	})
}

func wrapHandlerFunc(h http.HandlerFunc) http.Handler {
	return wrapHandler(http.Handler(h))
}

func finish(w http.ResponseWriter, status int, message string) {
	w.WriteHeader(status)
	io.WriteString(w, message)
}

func make_spa_route(docroot, index string) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		req_path := filepath.Clean(r.URL.Path)
		req_path = path.Join(docroot, req_path)

		_, err := os.Stat(req_path)
		if r.URL.Path == "/" || err != nil {
			req_path = path.Join(docroot, index)
		}

		f, err := os.Open(req_path)
		if err != nil {
			finish(w, 500, fmt.Sprintf("Unable to open file: %s", req_path))
			return
		}

		i, err := os.Stat(req_path)
		if err != nil {
			finish(w, 500, fmt.Sprintf("Unable to stat file: %s", req_path))
			return
		}

		http.ServeContent(w, r, req_path, i.ModTime(), f)
		return
	}
}

func route_png(w http.ResponseWriter, r *http.Request) {
	id := uuid.New()

	// TODO either check a table for currently rendering IDs and return
	// HTTP 429 (one request per id in flight), or cancel the existing,
	// running request and continue this one.

	if r.Method != "GET" {
		finish(w, http.StatusMethodNotAllowed, "Method not allowed.")
		return
	}

	q := r.URL.Query()

	s, err := strconv.Atoi(q.Get("s"))
	if err != nil {
		s = 1
	}

	e, err := strconv.Atoi(q.Get("p"))
	if err != nil {
		e = 1
	}

	width, err := strconv.Atoi(q.Get("w"))
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid width")
		return
	}

	height, err := strconv.Atoi(q.Get("h"))
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid height")
		return
	}

	iterations, err := strconv.Atoi(q.Get("i"))
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid i")
		return
	}

	er, err := strconv.ParseFloat(q.Get("e"), 64)
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid e")
		return
	}

	rmin, err := strconv.ParseFloat(q.Get("rmin"), 64)
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid rmin")
		return
	}

	imin, err := strconv.ParseFloat(q.Get("imin"), 64)
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid rmin")
		return
	}

	rmax, err := strconv.ParseFloat(q.Get("rmax"), 64)
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid rmax")
		return
	}

	imax, err := strconv.ParseFloat(q.Get("imax"), 64)
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid rmin")
		return
	}

	j := RenderJob{
		Parameters: gofr.Parameters{
			Width:        uint(width),
			Height:       uint(height),
			ImageWidth:   width * s,
			ImageHeight:  height * s,
			MaxI:         iterations,
			EscapeRadius: er,
			Min:          complex(rmin, imin),
			Max:          complex(rmax, imax),
			RenderFunc:   q.Get("r"),
			ColorFunc:    q.Get("c"),
			MemberColor:  q.Get("m"),
			Power:        e,
		},
		Threads: runtime.NumCPU(),
		Cancel:  make(chan bool),
	}

	render_id := q.Get("render_id")
	if render_id == "" {
		finish(w, http.StatusUnprocessableEntity, "Missing render_id")
		return
	}

	defer func() {
		render_jobs_mutex.Lock()
		delete(render_jobs, render_id)
		render_jobs_mutex.Unlock()
	}()

	render_jobs_mutex.Lock()
	if _, exists := render_jobs[render_id]; exists {
		close(render_jobs[render_id].Cancel)
		delete(render_jobs, render_id)
	}
	render_jobs[render_id] = j
	render_jobs_mutex.Unlock()

	image, err := j.Render()
	if err != nil {
		finish(w, http.StatusTooManyRequests, err.Error())
		return
	}

	w.Header().Set("Content-Type", "image/png")
	w.Header().Set("X-Render-Job-ID", id.String())
	w.WriteHeader(http.StatusOK)
	png.Encode(w, image)
}

func route_status(w http.ResponseWriter, r *http.Request) {
	finish(w, http.StatusOK, "OK")
}

func main() {
	var value string

	log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds | log.LUTC)
	log.Printf("gofrd v%s", Version)
	log.Printf("libgofrd v%s", gofr.Version)

	static_dir := "./build"
	if value = os.Getenv("GOFR_STATIC_DIR"); value != "" {
		static_dir = value
	}
	static_dir, err := filepath.Abs(static_dir)
	if err != nil {
		panic(err)
	}
	log.Printf("Serving from: %s\n", static_dir)

	bind_addr := "0.0.0.0:8000"
	if value = os.Getenv("GOFR_BIND_ADDR"); value != "" {
		bind_addr = value
	}
	log.Printf("Listening on: %s\n", bind_addr)

	//http.Handle("/", wrapHandler(http.FileServer(http.Dir(static_dir))))
	http.Handle("/", wrapHandlerFunc(make_spa_route(static_dir, "index.html")))
	http.Handle("/png", wrapHandlerFunc(route_png))
	http.Handle("/status", wrapHandlerFunc(route_status))

	/* Run the thing. */
	log.Fatal(http.ListenAndServe(bind_addr, nil))
}
<commit_msg>Make SPA handling more reliable<commit_after>package main

import (
	"fmt"
	"github.com/google/uuid"
	"github.com/musl/gofr/lib/gofr"
	"github.com/nfnt/resize"
	"image"
	"image/png"
	"io"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"sync"
	"time"
)

const Version = "0.1.1"

var render_jobs = make(map[string]RenderJob)
var render_jobs_mutex = &sync.Mutex{}

type RenderJob struct {
	Parameters gofr.Parameters
	Cancel     chan bool
	Threads    int
}

func (self *RenderJob) Render() (image.Image, error) {
	img := image.NewNRGBA64(image.Rect(0, 0, self.Parameters.ImageWidth, self.Parameters.ImageHeight))
	contexts := gofr.MakeContexts(img, self.Threads, &self.Parameters)

	err := gofr.Render(self.Threads, contexts, self.Cancel)
	if err != nil {
		return nil, err
	}

	image := resize.Resize(self.Parameters.Width, self.Parameters.Height, image.Image(img), resize.Lanczos3)
	return image, nil
}

type LogResponseWriter struct {
	http.ResponseWriter
	Status int
	Start  time.Time
	End    time.Time
}

func NewLogResponseWriter(w http.ResponseWriter) *LogResponseWriter {
	return &LogResponseWriter{w, 0, time.Now(), time.Now()}
}

func (self *LogResponseWriter) WriteHeader(code int) {
	self.Status = code
	self.ResponseWriter.WriteHeader(code)
}

func (self LogResponseWriter) Log(message string) {
	self.End = time.Now()
	log.Printf("%s %v\n", message, self.End.Sub(self.Start))
}

func wrapHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		lrw := NewLogResponseWriter(w)

		lrw.Log(fmt.Sprintf("%s -> %s %s", r.Method, r.URL.Path, r.RemoteAddr))
		h.ServeHTTP(lrw, r)
		lrw.Log(fmt.Sprintf("%d <- %s %s %s", lrw.Status, r.Method, r.URL.Path, r.RemoteAddr))
	})
}

func wrapHandlerFunc(h http.HandlerFunc) http.Handler {
	return wrapHandler(http.Handler(h))
}

func finish(w http.ResponseWriter, status int, message string) {
	w.WriteHeader(status)
	io.WriteString(w, message)
}

func make_spa_route(docroot, index string) http.HandlerFunc {

	// Rule: routes must end in '/', files must not. This means that
	// requests for directories will always be served with the spa index
	// file.
	pattern := regexp.MustCompile(`/$`)

	return func(w http.ResponseWriter, r *http.Request) {
		var req_path string

		if r.URL.Path == "/" || pattern.MatchString(r.URL.Path) {
			req_path = path.Join(docroot, index)
		} else {
			req_path = filepath.Clean(r.URL.Path)
			req_path = path.Join(docroot, req_path)
		}

		_, err := os.Stat(req_path)
		if err != nil {
			finish(w, 404, "File Not Found")
			return
		}

		f, err := os.Open(req_path)
		if err != nil {
			finish(w, 500, fmt.Sprintf("Unable to open file: %s", req_path))
			return
		}

		i, err := os.Stat(req_path)
		if err != nil {
			finish(w, 500, fmt.Sprintf("Unable to stat file: %s", req_path))
			return
		}

		http.ServeContent(w, r, req_path, i.ModTime(), f)
		return
	}
}

func route_png(w http.ResponseWriter, r *http.Request) {
	id := uuid.New()

	// TODO either check a table for currently rendering IDs and return
	// HTTP 429 (one request per id in flight), or cancel the existing,
	// running request and continue this one.

	if r.Method != "GET" {
		finish(w, http.StatusMethodNotAllowed, "Method not allowed.")
		return
	}

	q := r.URL.Query()

	s, err := strconv.Atoi(q.Get("s"))
	if err != nil {
		s = 1
	}

	e, err := strconv.Atoi(q.Get("p"))
	if err != nil {
		e = 1
	}

	width, err := strconv.Atoi(q.Get("w"))
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid width")
		return
	}

	height, err := strconv.Atoi(q.Get("h"))
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid height")
		return
	}

	iterations, err := strconv.Atoi(q.Get("i"))
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid i")
		return
	}

	er, err := strconv.ParseFloat(q.Get("e"), 64)
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid e")
		return
	}

	rmin, err := strconv.ParseFloat(q.Get("rmin"), 64)
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid rmin")
		return
	}

	imin, err := strconv.ParseFloat(q.Get("imin"), 64)
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid rmin")
		return
	}

	rmax, err := strconv.ParseFloat(q.Get("rmax"), 64)
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid rmax")
		return
	}

	imax, err := strconv.ParseFloat(q.Get("imax"), 64)
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid rmin")
		return
	}

	j := RenderJob{
		Parameters: gofr.Parameters{
			Width:        uint(width),
			Height:       uint(height),
			ImageWidth:   width * s,
			ImageHeight:  height * s,
			MaxI:         iterations,
			EscapeRadius: er,
			Min:          complex(rmin, imin),
			Max:          complex(rmax, imax),
			RenderFunc:   q.Get("r"),
			ColorFunc:    q.Get("c"),
			MemberColor:  q.Get("m"),
			Power:        e,
		},
		Threads: runtime.NumCPU(),
		Cancel:  make(chan bool),
	}

	render_id := q.Get("render_id")
	if render_id == "" {
		finish(w, http.StatusUnprocessableEntity, "Missing render_id")
		return
	}

	defer func() {
		render_jobs_mutex.Lock()
		delete(render_jobs, render_id)
		render_jobs_mutex.Unlock()
	}()

	render_jobs_mutex.Lock()
	if _, exists := render_jobs[render_id]; exists {
		close(render_jobs[render_id].Cancel)
		delete(render_jobs, render_id)
	}
	render_jobs[render_id] = j
	render_jobs_mutex.Unlock()

	image, err := j.Render()
	if err != nil {
		finish(w, http.StatusTooManyRequests, err.Error())
		return
	}

	w.Header().Set("Content-Type", "image/png")
	w.Header().Set("X-Render-Job-ID", id.String())
	w.WriteHeader(http.StatusOK)
	png.Encode(w, image)
}

func route_status(w http.ResponseWriter, r *http.Request) {
	finish(w, http.StatusOK, "OK")
}

func main() {
	var value string

	log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds | log.LUTC)
	log.Printf("gofrd v%s", Version)
	log.Printf("libgofrd v%s", gofr.Version)

	static_dir := "./build"
	if value = os.Getenv("GOFR_STATIC_DIR"); value != "" {
		static_dir = value
	}
	static_dir, err := filepath.Abs(static_dir)
	if err != nil {
		panic(err)
	}
	log.Printf("Serving from: %s\n", static_dir)

	bind_addr := "0.0.0.0:8000"
	if value = os.Getenv("GOFR_BIND_ADDR"); value != "" {
		bind_addr = value
	}
	log.Printf("Listening on: %s\n", bind_addr)

	//http.Handle("/", wrapHandler(http.FileServer(http.Dir(static_dir))))
	http.Handle("/", wrapHandlerFunc(make_spa_route(static_dir, "index.html")))
	http.Handle("/png", wrapHandlerFunc(route_png))
	http.Handle("/status", wrapHandlerFunc(route_status))

	/* Run the thing. */
	log.Fatal(http.ListenAndServe(bind_addr, nil))
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"strings"

	ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"
	cmds "github.com/jbenet/go-ipfs/commands"
	"github.com/jbenet/go-ipfs/core"
	commands "github.com/jbenet/go-ipfs/core/commands"
	corehttp "github.com/jbenet/go-ipfs/core/corehttp"
	fsrepo "github.com/jbenet/go-ipfs/repo/fsrepo"
	util "github.com/jbenet/go-ipfs/util"
	"github.com/jbenet/go-ipfs/util/debugerror"
)

const (
	initOptionKwd = "init"
	mountKwd      = "mount"
	writableKwd   = "writable"
	ipfsMountKwd  = "mount-ipfs"
	ipnsMountKwd  = "mount-ipns"
	// apiAddrKwd    = "address-api"
	// swarmAddrKwd  = "address-swarm"
)

var daemonCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Run a network-connected IPFS node",
		ShortDescription: `
'ipfs daemon' runs a persistent IPFS daemon that can serve commands
over the network. Most applications that use IPFS will do so by
communicating with a daemon over the HTTP API. While the daemon is
running, calls to 'ipfs' commands will be sent over the network to
the daemon.
`,
	},

	Options: []cmds.Option{
		cmds.BoolOption(initOptionKwd, "Initialize IPFS with default settings if not already initialized"),
		cmds.BoolOption(mountKwd, "Mounts IPFS to the filesystem"),
		cmds.BoolOption(writableKwd, "Enable writing objects (with POST, PUT and DELETE)"),
		cmds.StringOption(ipfsMountKwd, "Path to the mountpoint for IPFS (if using --mount)"),
		cmds.StringOption(ipnsMountKwd, "Path to the mountpoint for IPNS (if using --mount)"),

		// TODO: add way to override addresses. tricky part: updating the config if also --init.
		// cmds.StringOption(apiAddrKwd, "Address for the daemon rpc API (overrides config)"),
		// cmds.StringOption(swarmAddrKwd, "Address for the swarm socket (overrides config)"),
	},
	Subcommands: map[string]*cmds.Command{},
	Run:         daemonFunc,
}

func daemonFunc(req cmds.Request, res cmds.Response) {
	// let the user know we're going.
	fmt.Printf("Initializing daemon...\n")

	// first, whether user has provided the initialization flag. we may be
	// running in an uninitialized state.
	initialize, _, err := req.Option(initOptionKwd).Bool()
	if err != nil {
		res.SetError(err, cmds.ErrNormal)
		return
	}

	if initialize {

		// now, FileExists is our best method of detecting whether IPFS is
		// configured. Consider moving this into a config helper method
		// `IsInitialized` where the quality of the signal can be improved over
		// time, and many call-sites can benefit.
		if !util.FileExists(req.Context().ConfigRoot) {
			err := initWithDefaults(os.Stdout, req.Context().ConfigRoot)
			if err != nil {
				res.SetError(debugerror.Wrap(err), cmds.ErrNormal)
				return
			}
		}
	}

	// To ensure that IPFS has been initialized, fetch the config. Do this
	// _before_ acquiring the daemon lock so the user gets an appropriate error
	// message.
	// NB: It's safe to read the config without the daemon lock, but not safe
	// to write.
	ctx := req.Context()
	cfg, err := ctx.GetConfig()
	if err != nil {
		res.SetError(err, cmds.ErrNormal)
		return
	}

	// acquire the repo lock _before_ constructing a node. we need to make
	// sure we are permitted to access the resources (datastore, etc.)
	repo := fsrepo.At(req.Context().ConfigRoot)
	if err := repo.Open(); err != nil {
		res.SetError(debugerror.Errorf("Couldn't obtain lock. Is another daemon already running?"), cmds.ErrNormal)
		return
	}

	// OK!!! Now we're ready to construct the node.
	// make sure we construct an online node.
	node, err := core.NewIPFSNode(ctx.Context, core.Online(repo))
	if err != nil {
		res.SetError(err, cmds.ErrNormal)
		return
	}
	defer node.Close()
	req.Context().ConstructNode = func() (*core.IpfsNode, error) {
		return node, nil
	}

	// verify api address is valid multiaddr
	apiMaddr, err := ma.NewMultiaddr(cfg.Addresses.API)
	if err != nil {
		res.SetError(err, cmds.ErrNormal)
		return
	}

	var gatewayMaddr ma.Multiaddr
	if len(cfg.Addresses.Gateway) > 0 {
		// ignore error for gateway address
		// if there is an error (invalid address), then don't run the gateway
		gatewayMaddr, _ = ma.NewMultiaddr(cfg.Addresses.Gateway)
		if gatewayMaddr == nil {
			log.Errorf("Invalid gateway address: %s", cfg.Addresses.Gateway)
		}
	}

	// mount if the user provided the --mount flag
	mount, _, err := req.Option(mountKwd).Bool()
	if err != nil {
		res.SetError(err, cmds.ErrNormal)
		return
	}
	if mount {
		fsdir, found, err := req.Option(ipfsMountKwd).String()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		if !found {
			fsdir = cfg.Mounts.IPFS
		}

		nsdir, found, err := req.Option(ipnsMountKwd).String()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		if !found {
			nsdir = cfg.Mounts.IPNS
		}

		err = commands.Mount(node, fsdir, nsdir)
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		fmt.Printf("IPFS mounted at: %s\n", fsdir)
		fmt.Printf("IPNS mounted at: %s\n", nsdir)
	}

	var rootRedirect corehttp.ServeOption
	if len(cfg.Gateway.RootRedirect) > 0 {
		rootRedirect = corehttp.RedirectOption("", cfg.Gateway.RootRedirect)
	}

	writable, writableOptionFound, err := req.Option(writableKwd).Bool()
	if err != nil {
		res.SetError(err, cmds.ErrNormal)
		return
	}
	if !writableOptionFound {
		writable = cfg.Gateway.Writable
	}

	if gatewayMaddr != nil {
		go func() {
			var opts = []corehttp.ServeOption{corehttp.GatewayOption(writable)}
			if rootRedirect != nil {
				opts = append(opts, rootRedirect)
			}
			fmt.Printf("Gateway server listening on %s\n", gatewayMaddr)
			if writable {
				fmt.Printf("Gateway server is writable\n")
			}
			err := corehttp.ListenAndServe(node, gatewayMaddr.String(), opts...)
			if err != nil {
				log.Error(err)
			}
		}()
	}

	blocklist := &corehttp.BlockList{}
	blocklist.SetDecider(func(s string) bool {
		// for now, only allow paths in the WebUI path
		for _, webuipath := range corehttp.WebUIPaths {
			if strings.HasPrefix(s, webuipath) {
				return true
			}
		}
		return false
	})
	gateway := corehttp.NewGateway(corehttp.GatewayConfig{
		Writable:  true,
		BlockList: blocklist,
	})
	var opts = []corehttp.ServeOption{
		corehttp.CommandsOption(*req.Context()),
		corehttp.WebUIOption,
		gateway.ServeOption(),
	}
	if rootRedirect != nil {
		opts = append(opts, rootRedirect)
	}
	fmt.Printf("API server listening on %s\n", apiMaddr)
	if err := corehttp.ListenAndServe(node, apiMaddr.String(), opts...); err != nil {
		res.SetError(err, cmds.ErrNormal)
		return
	}
}
<commit_msg>refactor(cmd/daemon) more compact representation!<commit_after>package main

import (
	"fmt"
	"os"
	"strings"

	ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"
	cmds "github.com/jbenet/go-ipfs/commands"
	"github.com/jbenet/go-ipfs/core"
	commands "github.com/jbenet/go-ipfs/core/commands"
	corehttp "github.com/jbenet/go-ipfs/core/corehttp"
	fsrepo "github.com/jbenet/go-ipfs/repo/fsrepo"
	util "github.com/jbenet/go-ipfs/util"
	"github.com/jbenet/go-ipfs/util/debugerror"
)

const (
	initOptionKwd = "init"
	mountKwd      = "mount"
	writableKwd   = "writable"
	ipfsMountKwd  = "mount-ipfs"
	ipnsMountKwd  = "mount-ipns"
	// apiAddrKwd    = "address-api"
	// swarmAddrKwd  = "address-swarm"
)

var daemonCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Run a network-connected IPFS node",
		ShortDescription: `
'ipfs daemon' runs a persistent IPFS daemon that can serve commands
over the network. Most applications that use IPFS will do so by
communicating with a daemon over the HTTP API. While the daemon is
running, calls to 'ipfs' commands will be sent over the network to
the daemon.
`,
	},

	Options: []cmds.Option{
		cmds.BoolOption(initOptionKwd, "Initialize IPFS with default settings if not already initialized"),
		cmds.BoolOption(mountKwd, "Mounts IPFS to the filesystem"),
		cmds.BoolOption(writableKwd, "Enable writing objects (with POST, PUT and DELETE)"),
		cmds.StringOption(ipfsMountKwd, "Path to the mountpoint for IPFS (if using --mount)"),
		cmds.StringOption(ipnsMountKwd, "Path to the mountpoint for IPNS (if using --mount)"),

		// TODO: add way to override addresses. tricky part: updating the config if also --init.
		// cmds.StringOption(apiAddrKwd, "Address for the daemon rpc API (overrides config)"),
		// cmds.StringOption(swarmAddrKwd, "Address for the swarm socket (overrides config)"),
	},
	Subcommands: map[string]*cmds.Command{},
	Run:         daemonFunc,
}

func daemonFunc(req cmds.Request, res cmds.Response) {
	// let the user know we're going.
	fmt.Printf("Initializing daemon...\n")

	// first, whether user has provided the initialization flag. we may be
	// running in an uninitialized state.
	initialize, _, err := req.Option(initOptionKwd).Bool()
	if err != nil {
		res.SetError(err, cmds.ErrNormal)
		return
	}

	if initialize {

		// now, FileExists is our best method of detecting whether IPFS is
		// configured. Consider moving this into a config helper method
		// `IsInitialized` where the quality of the signal can be improved over
		// time, and many call-sites can benefit.
		if !util.FileExists(req.Context().ConfigRoot) {
			err := initWithDefaults(os.Stdout, req.Context().ConfigRoot)
			if err != nil {
				res.SetError(debugerror.Wrap(err), cmds.ErrNormal)
				return
			}
		}
	}

	// To ensure that IPFS has been initialized, fetch the config. Do this
	// _before_ acquiring the daemon lock so the user gets an appropriate error
	// message.
	// NB: It's safe to read the config without the daemon lock, but not safe
	// to write.
	ctx := req.Context()
	cfg, err := ctx.GetConfig()
	if err != nil {
		res.SetError(err, cmds.ErrNormal)
		return
	}

	// acquire the repo lock _before_ constructing a node. we need to make
	// sure we are permitted to access the resources (datastore, etc.)
	repo := fsrepo.At(req.Context().ConfigRoot)
	if err := repo.Open(); err != nil {
		res.SetError(debugerror.Errorf("Couldn't obtain lock. Is another daemon already running?"), cmds.ErrNormal)
		return
	}

	// OK!!! Now we're ready to construct the node.
	// make sure we construct an online node.
	node, err := core.NewIPFSNode(ctx.Context, core.Online(repo))
	if err != nil {
		res.SetError(err, cmds.ErrNormal)
		return
	}
	defer node.Close()
	req.Context().ConstructNode = func() (*core.IpfsNode, error) {
		return node, nil
	}

	// verify api address is valid multiaddr
	apiMaddr, err := ma.NewMultiaddr(cfg.Addresses.API)
	if err != nil {
		res.SetError(err, cmds.ErrNormal)
		return
	}

	var gatewayMaddr ma.Multiaddr
	if len(cfg.Addresses.Gateway) > 0 {
		// ignore error for gateway address
		// if there is an error (invalid address), then don't run the gateway
		gatewayMaddr, _ = ma.NewMultiaddr(cfg.Addresses.Gateway)
		if gatewayMaddr == nil {
			log.Errorf("Invalid gateway address: %s", cfg.Addresses.Gateway)
		}
	}

	// mount if the user provided the --mount flag
	mount, _, err := req.Option(mountKwd).Bool()
	if err != nil {
		res.SetError(err, cmds.ErrNormal)
		return
	}
	if mount {
		fsdir, found, err := req.Option(ipfsMountKwd).String()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		if !found {
			fsdir = cfg.Mounts.IPFS
		}

		nsdir, found, err := req.Option(ipnsMountKwd).String()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		if !found {
			nsdir = cfg.Mounts.IPNS
		}

		err = commands.Mount(node, fsdir, nsdir)
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		fmt.Printf("IPFS mounted at: %s\n", fsdir)
		fmt.Printf("IPNS mounted at: %s\n", nsdir)
	}

	var rootRedirect corehttp.ServeOption
	if len(cfg.Gateway.RootRedirect) > 0 {
		rootRedirect = corehttp.RedirectOption("", cfg.Gateway.RootRedirect)
	}

	writable, writableOptionFound, err := req.Option(writableKwd).Bool()
	if err != nil {
		res.SetError(err, cmds.ErrNormal)
		return
	}
	if !writableOptionFound {
		writable = cfg.Gateway.Writable
	}

	if gatewayMaddr != nil {
		go func() {
			var opts = []corehttp.ServeOption{corehttp.GatewayOption(writable)}
			if rootRedirect != nil {
				opts = append(opts, rootRedirect)
			}
			fmt.Printf("Gateway server listening on %s\n", gatewayMaddr)
			if writable {
				fmt.Printf("Gateway server is writable\n")
			}
			err := corehttp.ListenAndServe(node, gatewayMaddr.String(), opts...)
			if err != nil {
				log.Error(err)
			}
		}()
	}

	gateway := corehttp.NewGateway(corehttp.GatewayConfig{
		Writable: true,
		BlockList: &corehttp.BlockList{
			Decider: func(s string) bool {
				// for now, only allow paths in the WebUI path
				for _, webuipath := range corehttp.WebUIPaths {
					if strings.HasPrefix(s, webuipath) {
						return true
					}
				}
				return false
			},
		},
	})
	var opts = []corehttp.ServeOption{
		corehttp.CommandsOption(*req.Context()),
		corehttp.WebUIOption,
		gateway.ServeOption(),
	}
	if rootRedirect != nil {
		opts = append(opts, rootRedirect)
	}
	fmt.Printf("API server listening on %s\n", apiMaddr)
	if err := corehttp.ListenAndServe(node, apiMaddr.String(), opts...); err != nil {
		res.SetError(err, cmds.ErrNormal)
		return
	}
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"io/ioutil"
	"launchpad.net/gnuflag"
	"launchpad.net/juju-core/charm"
	"launchpad.net/juju-core/cmd"
	"launchpad.net/juju-core/constraints"
	"launchpad.net/juju-core/juju"
	"launchpad.net/juju-core/state"
	"os"
)

type DeployCommand struct {
	EnvCommandBase
	CharmName    string
	ServiceName  string
	Config       cmd.FileVar
	Constraints  constraints.Value
	NumUnits     int // defaults to 1
	BumpRevision bool
	RepoPath     string // defaults to JUJU_REPOSITORY
	MachineId    string
}

const deployDoc = `
<charm name> can be a charm URL, or an unambiguously condensed form of it;
assuming a current default series of "precise", the following forms will be
accepted.

For cs:precise/mysql
  mysql
  precise/mysql

For cs:~user/precise/mysql
  cs:~user/mysql

For local:precise/mysql
  local:mysql

In all cases, a versioned charm URL will be expanded as expected (for example,
mysql-33 becomes cs:precise/mysql-33).

<service name>, if omitted, will be derived from <charm name>.
`

func (c *DeployCommand) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "deploy",
		Args:    "<charm name> [<service name>]",
		Purpose: "deploy a new service",
		Doc:     deployDoc,
	}
}

func (c *DeployCommand) SetFlags(f *gnuflag.FlagSet) {
	c.EnvCommandBase.SetFlags(f)
	f.IntVar(&c.NumUnits, "n", 1, "number of service units to deploy for principal charms")
	f.IntVar(&c.NumUnits, "num-units", 1, "")
	f.StringVar(&c.MachineId, "force-machine", "", "Machine to deploy initial unit, bypasses constraints")
	f.BoolVar(&c.BumpRevision, "u", false, "increment local charm directory revision")
	f.BoolVar(&c.BumpRevision, "upgrade", false, "")
	f.Var(&c.Config, "config", "path to yaml-formatted service config")
	f.Var(constraints.ConstraintsValue{&c.Constraints}, "constraints", "set service constraints")
	f.StringVar(&c.RepoPath, "repository", os.Getenv("JUJU_REPOSITORY"), "local charm repository")
}

func (c *DeployCommand) Init(args []string) error {
	// TODO --constraints
	switch len(args) {
	case 2:
		if !state.IsServiceName(args[1]) {
			return fmt.Errorf("invalid service name %q", args[1])
		}
		c.ServiceName = args[1]
		fallthrough
	case 1:
		if _, err := charm.InferURL(args[0], "fake"); err != nil {
			return fmt.Errorf("invalid charm name %q", args[0])
		}
		c.CharmName = args[0]
	case 0:
		return errors.New("no charm specified")
	default:
		return cmd.CheckEmpty(args[2:])
	}
	if c.NumUnits < 1 {
		// TODO improve/remove: this is misleading when deploying subordinates.
		return errors.New("must deploy at least one unit")
	}

	if c.MachineId != "" {
		if !state.IsMachineId(c.MachineId) {
			return fmt.Errorf("Invalid machine id %q", c.MachineId)
		}
	}
	return nil
}

func (c *DeployCommand) Run(ctx *cmd.Context) error {
	conn, err := juju.NewConnFromName(c.EnvName)
	if err != nil {
		return err
	}
	defer conn.Close()
	conf, err := conn.State.EnvironConfig()
	if err != nil {
		return err
	}
	curl, err := charm.InferURL(c.CharmName, conf.DefaultSeries())
	if err != nil {
		return err
	}
	repo, err := charm.InferRepository(curl, ctx.AbsPath(c.RepoPath))
	if err != nil {
		return err
	}
	var configYAML []byte
	if c.Config.Path != "" {
		configYAML, err = ioutil.ReadFile(c.Config.Path)
		if err != nil {
			return err
		}
	}
	charm, err := conn.PutCharm(curl, repo, c.BumpRevision)
	if err != nil {
		return err
	}
	if charm.Meta().Subordinate {
		empty := constraints.Value{}
		if c.Constraints != empty {
			return state.ErrSubordinateConstraints
		}
	}
	serviceName := c.ServiceName
	if serviceName == "" {
		serviceName = curl.Name
	}
	args := juju.DeployServiceParams{
		Charm:       charm,
		ServiceName: serviceName,
		NumUnits:    c.NumUnits,
		// BUG(lp:1162122): --config has no tests.
		ConfigYAML:  string(configYAML),
		Constraints: c.Constraints,
		ForceMachineId:   c.MachineId,
	}
	_, err = conn.DeployService(args)
	return err
}
<commit_msg>make gofmt happy<commit_after>package main

import (
	"errors"
	"fmt"
	"io/ioutil"
	"launchpad.net/gnuflag"
	"launchpad.net/juju-core/charm"
	"launchpad.net/juju-core/cmd"
	"launchpad.net/juju-core/constraints"
	"launchpad.net/juju-core/juju"
	"launchpad.net/juju-core/state"
	"os"
)

type DeployCommand struct {
	EnvCommandBase
	CharmName    string
	ServiceName  string
	Config       cmd.FileVar
	Constraints  constraints.Value
	NumUnits     int // defaults to 1
	BumpRevision bool
	RepoPath     string // defaults to JUJU_REPOSITORY
	MachineId    string
}

const deployDoc = `
<charm name> can be a charm URL, or an unambiguously condensed form of it;
assuming a current default series of "precise", the following forms will be
accepted.

For cs:precise/mysql
  mysql
  precise/mysql

For cs:~user/precise/mysql
  cs:~user/mysql

For local:precise/mysql
  local:mysql

In all cases, a versioned charm URL will be expanded as expected (for example,
mysql-33 becomes cs:precise/mysql-33).

<service name>, if omitted, will be derived from <charm name>.
`

func (c *DeployCommand) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "deploy",
		Args:    "<charm name> [<service name>]",
		Purpose: "deploy a new service",
		Doc:     deployDoc,
	}
}

func (c *DeployCommand) SetFlags(f *gnuflag.FlagSet) {
	c.EnvCommandBase.SetFlags(f)
	f.IntVar(&c.NumUnits, "n", 1, "number of service units to deploy for principal charms")
	f.IntVar(&c.NumUnits, "num-units", 1, "")
	f.StringVar(&c.MachineId, "force-machine", "", "Machine to deploy initial unit, bypasses constraints")
	f.BoolVar(&c.BumpRevision, "u", false, "increment local charm directory revision")
	f.BoolVar(&c.BumpRevision, "upgrade", false, "")
	f.Var(&c.Config, "config", "path to yaml-formatted service config")
	f.Var(constraints.ConstraintsValue{&c.Constraints}, "constraints", "set service constraints")
	f.StringVar(&c.RepoPath, "repository", os.Getenv("JUJU_REPOSITORY"), "local charm repository")
}

func (c *DeployCommand) Init(args []string) error {
	// TODO --constraints
	switch len(args) {
	case 2:
		if !state.IsServiceName(args[1]) {
			return fmt.Errorf("invalid service name %q", args[1])
		}
		c.ServiceName = args[1]
		fallthrough
	case 1:
		if _, err := charm.InferURL(args[0], "fake"); err != nil {
			return fmt.Errorf("invalid charm name %q", args[0])
		}
		c.CharmName = args[0]
	case 0:
		return errors.New("no charm specified")
	default:
		return cmd.CheckEmpty(args[2:])
	}
	if c.NumUnits < 1 {
		// TODO improve/remove: this is misleading when deploying subordinates.
		return errors.New("must deploy at least one unit")
	}

	if c.MachineId != "" {
		if !state.IsMachineId(c.MachineId) {
			return fmt.Errorf("Invalid machine id %q", c.MachineId)
		}
	}
	return nil
}

func (c *DeployCommand) Run(ctx *cmd.Context) error {
	conn, err := juju.NewConnFromName(c.EnvName)
	if err != nil {
		return err
	}
	defer conn.Close()
	conf, err := conn.State.EnvironConfig()
	if err != nil {
		return err
	}
	curl, err := charm.InferURL(c.CharmName, conf.DefaultSeries())
	if err != nil {
		return err
	}
	repo, err := charm.InferRepository(curl, ctx.AbsPath(c.RepoPath))
	if err != nil {
		return err
	}
	var configYAML []byte
	if c.Config.Path != "" {
		configYAML, err = ioutil.ReadFile(c.Config.Path)
		if err != nil {
			return err
		}
	}
	charm, err := conn.PutCharm(curl, repo, c.BumpRevision)
	if err != nil {
		return err
	}
	if charm.Meta().Subordinate {
		empty := constraints.Value{}
		if c.Constraints != empty {
			return state.ErrSubordinateConstraints
		}
	}
	serviceName := c.ServiceName
	if serviceName == "" {
		serviceName = curl.Name
	}
	args := juju.DeployServiceParams{
		Charm:       charm,
		ServiceName: serviceName,
		NumUnits:    c.NumUnits,
		// BUG(lp:1162122): --config has no tests.
		ConfigYAML:     string(configYAML),
		Constraints:    c.Constraints,
		ForceMachineId: c.MachineId,
	}
	_, err = conn.DeployService(args)
	return err
}
<|endoftext|>


<commit_before>package main

//go:generate goembed static

import (
	"flag"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"time"

	"github.com/jeanfric/embedfs"
	"github.com/jeanfric/liftca"
	"github.com/jeanfric/liftca/cmd/liftca/handlers"
	"github.com/jeanfric/liftca/ht"
)

func main() {
	quit := make(chan bool)

	var addressArg string
	var storeFileArg string
	var serveDir string

	flag.StringVar(&addressArg, "a", ":8080", "listen address")
	flag.StringVar(&storeFileArg, "s", "store.gob", "path to state storage file")
	flag.StringVar(&serveDir, "d", "", "if set, directory to serve static assets from; else use embedded assets")
	flag.Parse()

	storeFile := filepath.Clean(storeFileArg)
	backingFile, err := os.OpenFile(storeFile, os.O_CREATE|os.O_RDWR, 0666)
	if err != nil {
		log.Fatal(err)
	}
	defer backingFile.Close()
	store := liftca.LoadStore(backingFile)
	storeChanged := make(chan struct{})
	store.Updates(storeChanged)
	go func(c <-chan struct{}) {
		for {
			<-c
			store.DumpStore(backingFile)
		}
	}(storeChanged)

	var fileServer http.Handler

	if serveDir != "" {
		log.Printf("Serving from '%s'", serveDir)
		fileServer = ht.NewLoggingFileServer(http.Dir(serveDir))
	} else {
		assets, err := loadAssets() // method generated by goembed
		if err != nil {
			log.Fatal(err)
		}
		fileServer = ht.NewLoggingFileServer(embedfs.New(assets))
	}

	r := ht.NewRouter()

	r.Handle("GET", "/ca", ht.NewHandler(store, handlers.GetCAs))
	r.Handle("POST", "/ca", ht.NewHandler(store, handlers.PostCA))
	r.Handle("GET", "/ca/{ca_id}-certificate.cer", ht.NewHandler(store, handlers.GetCACertificateCER))
	r.Handle("GET", "/ca/{ca_id}-crl.crl", ht.NewHandler(store, handlers.GetCACRLCER))
	r.Handle("GET", "/ca/{ca_id}-private-key.cer", ht.NewHandler(store, handlers.GetCAPrivateKeyCER))
	r.Handle("GET", "/ca/{ca_id}-certificate.pem", ht.NewHandler(store, handlers.GetCACertificatePEM))
	r.Handle("GET", "/ca/{ca_id}-certificate.pem.txt", ht.NewHandler(store, handlers.GetCACertificatePEMTXT))
	r.Handle("GET", "/ca/{ca_id}-private-key.pem", ht.NewHandler(store, handlers.GetCAPrivateKeyPEM))
	r.Handle("GET", "/ca/{ca_id}-private-key.pem.txt", ht.NewHandler(store, handlers.GetCAPrivateKeyPEMTXT))
	r.Handle("GET", "/ca/{ca_id}-crl.pem", ht.NewHandler(store, handlers.GetCACRLPEM))
	r.Handle("GET", "/ca/{ca_id}-crl.pem.txt", ht.NewHandler(store, handlers.GetCACRLPEMTXT))
	r.Handle("GET", "/ca/{ca_id}", ht.NewHandler(store, handlers.GetCA))
	r.Handle("GET", "/ca/{ca_id}/cert", ht.NewHandler(store, handlers.GetCerts))
	r.Handle("POST", "/ca/{ca_id}/cert", ht.NewHandler(store, handlers.PostCert))
	r.Handle("GET", "/ca/{ca_id}/cert/{cert_id}-certificate.pem", ht.NewHandler(store, handlers.GetCertificatePEM))
	r.Handle("GET", "/ca/{ca_id}/cert/{cert_id}-certificate.pem.txt", ht.NewHandler(store, handlers.GetCertificatePEMTXT))
	r.Handle("GET", "/ca/{ca_id}/cert/{cert_id}-private-key.pem", ht.NewHandler(store, handlers.GetCertificatePrivateKeyPEM))
	r.Handle("GET", "/ca/{ca_id}/cert/{cert_id}-private-key.pem.txt", ht.NewHandler(store, handlers.GetCertificatePrivateKeyPEMTXT))
	r.Handle("GET", "/ca/{ca_id}/cert/{cert_id}-private-key.cer", ht.NewHandler(store, handlers.GetCertificatePrivateKeyCER))
	r.Handle("GET", "/ca/{ca_id}/cert/{cert_id}-certificate.cer", ht.NewHandler(store, handlers.GetCertificateCER))
	r.Handle("GET", "/ca/{ca_id}/cert/{cert_id}", ht.NewHandler(store, handlers.GetCert))
	r.Handle("POST", "/ca/{ca_id}/crl", ht.NewHandler(store, handlers.PostCRL))
	r.Handle("GET", "/ca/{ca_id}/crl", ht.NewHandler(store, handlers.GetCRL))
	r.Handle("DELETE", "/ca/{ca_id}/crl/{cert_id}", ht.NewHandler(store, handlers.DeleteCRL))
	r.Handle("GET", "/", fileServer)
	r.Handle("GET", "/{f}", fileServer)
	r.Handle("GET", "/js/{f}", fileServer)
	r.Handle("GET", "/fonts/{f}", fileServer)
	r.Handle("GET", "/css/{f}", fileServer)
	r.Handle("GET", "/img/{f}", fileServer)
	r.Handle("GET", "/partials/{f}", fileServer)

	s := &http.Server{
		Addr:           addressArg,
		ReadTimeout:    10 * time.Second,
		WriteTimeout:   10 * time.Second,
		MaxHeaderBytes: 1 << 20,
		Handler:        r,
	}
	go s.ListenAndServe()

	log.Printf("liftCA engaged at '%v', data file '%v'", addressArg, storeFile)
	<-quit
}
<commit_msg>Fix issue in persistent storage<commit_after>package main

//go:generate goembed static

import (
	"flag"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"time"

	"github.com/jeanfric/embedfs"
	"github.com/jeanfric/liftca"
	"github.com/jeanfric/liftca/cmd/liftca/handlers"
	"github.com/jeanfric/liftca/ht"
)

func main() {
	quit := make(chan bool)

	var addressArg string
	var storeFileArg string
	var serveDir string

	flag.StringVar(&addressArg, "a", ":8080", "listen address")
	flag.StringVar(&storeFileArg, "s", "store.gob", "path to state storage file")
	flag.StringVar(&serveDir, "d", "", "if set, directory to serve static assets from; else use embedded assets")
	flag.Parse()

	storeFile := filepath.Clean(storeFileArg)
	backingFile, err := os.OpenFile(storeFile, os.O_CREATE|os.O_RDWR, 0666)
	if err != nil {
		log.Fatal(err)
	}
	store := liftca.LoadStore(backingFile)
	backingFile.Close()
	storeChanged := make(chan struct{})
	store.Updates(storeChanged)
	go func(path string, c <-chan struct{}) {
		for {
			<-c
			err := os.Remove(path + ".backup")
			if err != nil {
				log.Fatal(err)
			}
			err = os.Rename(path, path+".backup")
			if err != nil {
				log.Fatal(err)
			}
			backingFile, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0666)
			if err != nil {
				log.Fatal(err)
			}
			store.DumpStore(backingFile)
			backingFile.Close()
		}
	}(storeFileArg, storeChanged)

	var fileServer http.Handler

	if serveDir != "" {
		log.Printf("Serving from '%s'", serveDir)
		fileServer = ht.NewLoggingFileServer(http.Dir(serveDir))
	} else {
		assets, err := loadAssets() // method generated by goembed
		if err != nil {
			log.Fatal(err)
		}
		fileServer = ht.NewLoggingFileServer(embedfs.New(assets))
	}

	r := ht.NewRouter()

	r.Handle("GET", "/ca", ht.NewHandler(store, handlers.GetCAs))
	r.Handle("POST", "/ca", ht.NewHandler(store, handlers.PostCA))
	r.Handle("GET", "/ca/{ca_id}-certificate.cer", ht.NewHandler(store, handlers.GetCACertificateCER))
	r.Handle("GET", "/ca/{ca_id}-crl.crl", ht.NewHandler(store, handlers.GetCACRLCER))
	r.Handle("GET", "/ca/{ca_id}-private-key.cer", ht.NewHandler(store, handlers.GetCAPrivateKeyCER))
	r.Handle("GET", "/ca/{ca_id}-certificate.pem", ht.NewHandler(store, handlers.GetCACertificatePEM))
	r.Handle("GET", "/ca/{ca_id}-certificate.pem.txt", ht.NewHandler(store, handlers.GetCACertificatePEMTXT))
	r.Handle("GET", "/ca/{ca_id}-private-key.pem", ht.NewHandler(store, handlers.GetCAPrivateKeyPEM))
	r.Handle("GET", "/ca/{ca_id}-private-key.pem.txt", ht.NewHandler(store, handlers.GetCAPrivateKeyPEMTXT))
	r.Handle("GET", "/ca/{ca_id}-crl.pem", ht.NewHandler(store, handlers.GetCACRLPEM))
	r.Handle("GET", "/ca/{ca_id}-crl.pem.txt", ht.NewHandler(store, handlers.GetCACRLPEMTXT))
	r.Handle("GET", "/ca/{ca_id}", ht.NewHandler(store, handlers.GetCA))
	r.Handle("GET", "/ca/{ca_id}/cert", ht.NewHandler(store, handlers.GetCerts))
	r.Handle("POST", "/ca/{ca_id}/cert", ht.NewHandler(store, handlers.PostCert))
	r.Handle("GET", "/ca/{ca_id}/cert/{cert_id}-certificate.pem", ht.NewHandler(store, handlers.GetCertificatePEM))
	r.Handle("GET", "/ca/{ca_id}/cert/{cert_id}-certificate.pem.txt", ht.NewHandler(store, handlers.GetCertificatePEMTXT))
	r.Handle("GET", "/ca/{ca_id}/cert/{cert_id}-private-key.pem", ht.NewHandler(store, handlers.GetCertificatePrivateKeyPEM))
	r.Handle("GET", "/ca/{ca_id}/cert/{cert_id}-private-key.pem.txt", ht.NewHandler(store, handlers.GetCertificatePrivateKeyPEMTXT))
	r.Handle("GET", "/ca/{ca_id}/cert/{cert_id}-private-key.cer", ht.NewHandler(store, handlers.GetCertificatePrivateKeyCER))
	r.Handle("GET", "/ca/{ca_id}/cert/{cert_id}-certificate.cer", ht.NewHandler(store, handlers.GetCertificateCER))
	r.Handle("GET", "/ca/{ca_id}/cert/{cert_id}", ht.NewHandler(store, handlers.GetCert))
	r.Handle("POST", "/ca/{ca_id}/crl", ht.NewHandler(store, handlers.PostCRL))
	r.Handle("GET", "/ca/{ca_id}/crl", ht.NewHandler(store, handlers.GetCRL))
	r.Handle("DELETE", "/ca/{ca_id}/crl/{cert_id}", ht.NewHandler(store, handlers.DeleteCRL))
	r.Handle("GET", "/", fileServer)
	r.Handle("GET", "/{f}", fileServer)
	r.Handle("GET", "/js/{f}", fileServer)
	r.Handle("GET", "/fonts/{f}", fileServer)
	r.Handle("GET", "/css/{f}", fileServer)
	r.Handle("GET", "/img/{f}", fileServer)
	r.Handle("GET", "/partials/{f}", fileServer)

	s := &http.Server{
		Addr:           addressArg,
		ReadTimeout:    10 * time.Second,
		WriteTimeout:   10 * time.Second,
		MaxHeaderBytes: 1 << 20,
		Handler:        r,
	}
	go s.ListenAndServe()

	log.Printf("liftCA engaged at '%v', data file '%v'", addressArg, storeFile)
	<-quit
}
<|endoftext|>


<commit_before>package es

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"strings"
	"time"
)

const (
	OK  = '2'
	AND = "AND"
	OR  = "OR"
)

type Term map[string]interface{}

type Terms map[string][]interface{}

type Filter struct {
	And   []*Filter         `json:"and,omitempty`
	Term  *Term             `json:"term,omitempty"`
	Terms *Terms            `json:"term,omitempty"`
	Range map[string]*Range `json:"range,omitempty"`
}

type Range struct {
	From interface{}
	To   interface{}
}

type Filtered struct {
	Filter *Filter
}

type QueryString struct {
	Query           string `json:"query,omitempty"`
	DefaultOperator string `json:"default_operator,omitempty"`
}

type Query struct {
	Filtered    *Filtered    `json:"filtered,omitempty"`
	QueryString *QueryString `json:"query_string,omitempty"`
}

var query = Query{
	Filtered: &Filtered{
		Filter: &Filter{
			And: []*Filter{
				{
					Term: &Term{
						"Device": "Anrdoi",
					},
				},
				{
					Terms: &Terms{
						"Action": []interface{}{
							"api/v1/my/photos#create",
							"api/v1/photos#create",
						},
					},
					Range: map[string]*Range{
						"Time": {
							From: "",
						},
					},
				},
			},
		},
	},
}

type BulkIndexJob struct {
	Key    string
	Record interface{}
}

type Index struct {
	Host          string
	Port          int
	Index         string
	Type          string
	bulkIndexJobs []*BulkIndexJob
	BatchSize     int
	Debug         bool
}

func (index *Index) EnqueueBulkIndex(key string, record interface{}) (bool, error) {
	if index.BatchSize == 0 {
		index.BatchSize = 100
	}
	if cap(index.bulkIndexJobs) == 0 {
		index.ResetQueue()
	}
	index.bulkIndexJobs = append(index.bulkIndexJobs, &BulkIndexJob{
		Key: key, Record: record,
	})
	if len(index.bulkIndexJobs) >= index.BatchSize {
		return true, index.RunBatchIndex()
	}
	return false, nil
}

func (index *Index) DeleteIndex() error {
	req, e := http.NewRequest("DELETE", index.TypeUrl(), nil)
	if e != nil {
		return e
	}
	rsp, e := http.DefaultClient.Do(req)
	if e != nil {
		return e
	}
	defer rsp.Body.Close()
	if rsp.Status[0] != '2' {
		return fmt.Errorf("Error delting index at %s: %s", index.TypeUrl(), rsp.Status)
	}
	return nil
}

func (index *Index) Refresh() error {
	rsp, e := index.request("POST", index.IndexUrl()+"/_refresh", nil)
	if e != nil {
		return e
	}
	if rsp.Status[0] != '2' {
		return fmt.Errorf("Error refreshing index: %s", rsp.Status)
	}
	return nil
}

func (index *Index) RunBatchIndex() error {
	if len(index.bulkIndexJobs) == 0 {
		return nil
	}
	started := time.Now()
	buf := &bytes.Buffer{}
	enc := json.NewEncoder(buf)
	for _, r := range index.bulkIndexJobs {
		enc.Encode(map[string]map[string]string{
			"index": map[string]string{
				"_index": index.Index,
				"_type":  index.Type,
				"_id":    r.Key,
			},
		})
		enc.Encode(r.Record)
	}
	rsp, e := http.Post(index.BaseUrl()+"/_bulk", "application/json", buf)
	if e != nil {
		return e
	}
	defer rsp.Body.Close()
	b, _ := ioutil.ReadAll(rsp.Body)
	if rsp.Status[0] != OK {
		return fmt.Errorf("Error sending bulk request: %s %s", rsp.Status, string(b))
	}
	perSecond := float64(len(index.bulkIndexJobs)) / time.Now().Sub(started).Seconds()
	if index.Debug {
		fmt.Printf("indexed %d, %.1f/second\n", len(index.bulkIndexJobs), perSecond)
	}
	index.ResetQueue()
	return nil
}

func (index *Index) ResetQueue() {
	index.bulkIndexJobs = make([]*BulkIndexJob, 0, index.BatchSize)
}

type IndexStatus struct {
	Index  string `json:"_index"`
	Type   string `json:"_type"`
	Id     string `json:"_id"`
	Exists bool   `json:"exists"`
}

func (index *Index) Status() (status *IndexStatus, e error) {
	rsp, e := http.Get(index.BaseUrl() + "/_status")
	if e != nil {
		return nil, e
	}
	defer rsp.Body.Close()
	b, e := ioutil.ReadAll(rsp.Body)
	if e != nil {
		return nil, e
	}
	status = &IndexStatus{}
	e = json.Unmarshal(b, status)
	return status, e
}

func (index *Index) GlobalMapping() (m *Mapping, e error) {
	m = &Mapping{}
	u := index.BaseUrl() + "/_mapping"
	log.Printf("checking for url %s", u)
	rsp, e := index.request("GET", u, m)
	if rsp != nil && rsp.StatusCode == 404 {
		return nil, nil
	} else if e != nil {
		return nil, e
	}
	e = json.Unmarshal(rsp.Body, m)
	if e != nil {
		return nil, e
	}
	return m, nil
}

func (index *Index) Mapping() (i interface{}, e error) {
	u := index.IndexUrl() + "/_mapping"
	log.Printf("checking for url %s", u)
	rsp, e := index.request("GET", u, i)
	if rsp != nil && rsp.StatusCode == 404 {
		return nil, nil
	} else if e != nil {
		return nil, e
	}
	e = json.Unmarshal(rsp.Body, &i)
	if e != nil {
		return nil, e
	}
	return i, nil
}

func (index *Index) PutMapping(mapping interface{}) (rsp *HttpResponse, e error) {
	return index.request("PUT", index.IndexUrl()+"/", mapping)
}

func (index *Index) BaseUrl() string {
	if index.Port == 0 {
		index.Port = 9200
	}
	return fmt.Sprintf("http://%s:%d", index.Host, index.Port)
}

func (index *Index) IndexUrl() string {
	if index.Index != "" {
		return index.BaseUrl() + "/" + index.Index
	}
	return ""
}

func (index *Index) TypeUrl() string {
	if index.Type == "" {
		return index.IndexUrl()
	} else if idx := index.IndexUrl(); idx != "" {
		return idx + "/" + index.Type
	} else {
		return ""
	}
}

func (index *Index) DeleteByQuery(query string) error {
	q := &url.Values{}
	q.Add("q", query)
	rsp, e := index.request("DELETE", index.TypeUrl()+"/_query?"+q.Encode(), nil)
	if e != nil {
		return e
	}
	if rsp.Status[0] != '2' {
		return fmt.Errorf("error deleting with query: %s", rsp.Status)
	}
	return nil
}

func (index *Index) Search(req *Request) (rsp *Response, e error) {
	writer := &bytes.Buffer{}
	js := json.NewEncoder(writer)
	e = js.Encode(req)
	if e != nil {
		return nil, e
	}
	u := index.TypeUrl()
	if !strings.HasSuffix(u, "/") {
		u += "/"
	}
	u += "/_search"
	httpRequest, e := http.NewRequest("POST", u, writer)
	if e != nil {
		return nil, e
	}
	httpResponse, e := http.DefaultClient.Do(httpRequest)
	if e != nil {
		return nil, e
	}
	defer httpResponse.Body.Close()
	dec := json.NewDecoder(httpResponse.Body)
	rsp = &Response{}
	e = dec.Decode(rsp)
	if e != nil {
		return nil, e
	}
	return rsp, nil
}

func (index *Index) Post(u string, i interface{}) (*HttpResponse, error) {
	return index.request("POST", u, i)
}

func (index *Index) PutObject(id string, i interface{}) (*HttpResponse, error) {
	return index.request("PUT", index.TypeUrl()+"/"+id, i)
}

func (index *Index) Put(u string, i interface{}) (*HttpResponse, error) {
	return index.request("PUT", u, i)
}

type HttpResponse struct {
	*http.Response
	Body []byte
}

func (index *Index) request(method string, u string, i interface{}) (httpResponse *HttpResponse, e error) {
	var req *http.Request
	if i != nil {
		buf := &bytes.Buffer{}
		encoder := json.NewEncoder(buf)
		if e := encoder.Encode(i); e != nil {
			return nil, e
		}
		req, e = http.NewRequest(method, u, buf)
	} else {
		req, e = http.NewRequest(method, u, nil)
	}
	if e != nil {
		return nil, e
	}
	rsp, e := http.DefaultClient.Do(req)
	if e != nil {
		return nil, e
	}
	defer rsp.Body.Close()
	b, e := ioutil.ReadAll(rsp.Body)
	if e != nil {
		return nil, e
	}
	httpResponse = &HttpResponse{
		Response: rsp,
		Body:     b,
	}
	if e != nil {
		return httpResponse, e
	}
	if rsp.Status[0] != OK {
		return httpResponse, fmt.Errorf("error indexing: %s %s", rsp.Status, string(b))
	}
	return httpResponse, nil
}
<commit_msg>add method to create ES entries by posting<commit_after>package es

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"strings"
	"time"
)

const (
	OK  = '2'
	AND = "AND"
	OR  = "OR"
)

type Term map[string]interface{}

type Terms map[string][]interface{}

type Filter struct {
	And   []*Filter         `json:"and,omitempty`
	Term  *Term             `json:"term,omitempty"`
	Terms *Terms            `json:"term,omitempty"`
	Range map[string]*Range `json:"range,omitempty"`
}

type Range struct {
	From interface{}
	To   interface{}
}

type Filtered struct {
	Filter *Filter
}

type QueryString struct {
	Query           string `json:"query,omitempty"`
	DefaultOperator string `json:"default_operator,omitempty"`
}

type Query struct {
	Filtered    *Filtered    `json:"filtered,omitempty"`
	QueryString *QueryString `json:"query_string,omitempty"`
}

var query = Query{
	Filtered: &Filtered{
		Filter: &Filter{
			And: []*Filter{
				{
					Term: &Term{
						"Device": "Anrdoi",
					},
				},
				{
					Terms: &Terms{
						"Action": []interface{}{
							"api/v1/my/photos#create",
							"api/v1/photos#create",
						},
					},
					Range: map[string]*Range{
						"Time": {
							From: "",
						},
					},
				},
			},
		},
	},
}

type BulkIndexJob struct {
	Key    string
	Record interface{}
}

type Index struct {
	Host          string
	Port          int
	Index         string
	Type          string
	bulkIndexJobs []*BulkIndexJob
	BatchSize     int
	Debug         bool
}

func (index *Index) EnqueueBulkIndex(key string, record interface{}) (bool, error) {
	if index.BatchSize == 0 {
		index.BatchSize = 100
	}
	if cap(index.bulkIndexJobs) == 0 {
		index.ResetQueue()
	}
	index.bulkIndexJobs = append(index.bulkIndexJobs, &BulkIndexJob{
		Key: key, Record: record,
	})
	if len(index.bulkIndexJobs) >= index.BatchSize {
		return true, index.RunBatchIndex()
	}
	return false, nil
}

func (index *Index) DeleteIndex() error {
	req, e := http.NewRequest("DELETE", index.TypeUrl(), nil)
	if e != nil {
		return e
	}
	rsp, e := http.DefaultClient.Do(req)
	if e != nil {
		return e
	}
	defer rsp.Body.Close()
	if rsp.Status[0] != '2' {
		return fmt.Errorf("Error delting index at %s: %s", index.TypeUrl(), rsp.Status)
	}
	return nil
}

func (index *Index) Refresh() error {
	rsp, e := index.request("POST", index.IndexUrl()+"/_refresh", nil)
	if e != nil {
		return e
	}
	if rsp.Status[0] != '2' {
		return fmt.Errorf("Error refreshing index: %s", rsp.Status)
	}
	return nil
}

func (index *Index) RunBatchIndex() error {
	if len(index.bulkIndexJobs) == 0 {
		return nil
	}
	started := time.Now()
	buf := &bytes.Buffer{}
	enc := json.NewEncoder(buf)
	for _, r := range index.bulkIndexJobs {
		enc.Encode(map[string]map[string]string{
			"index": map[string]string{
				"_index": index.Index,
				"_type":  index.Type,
				"_id":    r.Key,
			},
		})
		enc.Encode(r.Record)
	}
	rsp, e := http.Post(index.BaseUrl()+"/_bulk", "application/json", buf)
	if e != nil {
		return e
	}
	defer rsp.Body.Close()
	b, _ := ioutil.ReadAll(rsp.Body)
	if rsp.Status[0] != OK {
		return fmt.Errorf("Error sending bulk request: %s %s", rsp.Status, string(b))
	}
	perSecond := float64(len(index.bulkIndexJobs)) / time.Now().Sub(started).Seconds()
	if index.Debug {
		fmt.Printf("indexed %d, %.1f/second\n", len(index.bulkIndexJobs), perSecond)
	}
	index.ResetQueue()
	return nil
}

func (index *Index) ResetQueue() {
	index.bulkIndexJobs = make([]*BulkIndexJob, 0, index.BatchSize)
}

type IndexStatus struct {
	Index  string `json:"_index"`
	Type   string `json:"_type"`
	Id     string `json:"_id"`
	Exists bool   `json:"exists"`
}

func (index *Index) Status() (status *IndexStatus, e error) {
	rsp, e := http.Get(index.BaseUrl() + "/_status")
	if e != nil {
		return nil, e
	}
	defer rsp.Body.Close()
	b, e := ioutil.ReadAll(rsp.Body)
	if e != nil {
		return nil, e
	}
	status = &IndexStatus{}
	e = json.Unmarshal(b, status)
	return status, e
}

func (index *Index) GlobalMapping() (m *Mapping, e error) {
	m = &Mapping{}
	u := index.BaseUrl() + "/_mapping"
	log.Printf("checking for url %s", u)
	rsp, e := index.request("GET", u, m)
	if rsp != nil && rsp.StatusCode == 404 {
		return nil, nil
	} else if e != nil {
		return nil, e
	}
	e = json.Unmarshal(rsp.Body, m)
	if e != nil {
		return nil, e
	}
	return m, nil
}

func (index *Index) Mapping() (i interface{}, e error) {
	u := index.IndexUrl() + "/_mapping"
	log.Printf("checking for url %s", u)
	rsp, e := index.request("GET", u, i)
	if rsp != nil && rsp.StatusCode == 404 {
		return nil, nil
	} else if e != nil {
		return nil, e
	}
	e = json.Unmarshal(rsp.Body, &i)
	if e != nil {
		return nil, e
	}
	return i, nil
}

func (index *Index) PutMapping(mapping interface{}) (rsp *HttpResponse, e error) {
	return index.request("PUT", index.IndexUrl()+"/", mapping)
}

func (index *Index) BaseUrl() string {
	if index.Port == 0 {
		index.Port = 9200
	}
	return fmt.Sprintf("http://%s:%d", index.Host, index.Port)
}

func (index *Index) IndexUrl() string {
	if index.Index != "" {
		return index.BaseUrl() + "/" + index.Index
	}
	return ""
}

func (index *Index) TypeUrl() string {
	if index.Type == "" {
		return index.IndexUrl()
	} else if idx := index.IndexUrl(); idx != "" {
		return idx + "/" + index.Type
	} else {
		return ""
	}
}

func (index *Index) DeleteByQuery(query string) error {
	q := &url.Values{}
	q.Add("q", query)
	rsp, e := index.request("DELETE", index.TypeUrl()+"/_query?"+q.Encode(), nil)
	if e != nil {
		return e
	}
	if rsp.Status[0] != '2' {
		return fmt.Errorf("error deleting with query: %s", rsp.Status)
	}
	return nil
}

func (index *Index) Search(req *Request) (rsp *Response, e error) {
	writer := &bytes.Buffer{}
	js := json.NewEncoder(writer)
	e = js.Encode(req)
	if e != nil {
		return nil, e
	}
	u := index.TypeUrl()
	if !strings.HasSuffix(u, "/") {
		u += "/"
	}
	u += "/_search"
	httpRequest, e := http.NewRequest("POST", u, writer)
	if e != nil {
		return nil, e
	}
	httpResponse, e := http.DefaultClient.Do(httpRequest)
	if e != nil {
		return nil, e
	}
	defer httpResponse.Body.Close()
	dec := json.NewDecoder(httpResponse.Body)
	rsp = &Response{}
	e = dec.Decode(rsp)
	if e != nil {
		return nil, e
	}
	return rsp, nil
}

func (index *Index) Post(u string, i interface{}) (*HttpResponse, error) {
	return index.request("POST", u, i)
}

func (index *Index) PostObject(i interface{}) (*HttpResponse, error) {
	return index.request("POST", index.TypeUrl(), i)
}

func (index *Index) PutObject(id string, i interface{}) (*HttpResponse, error) {
	return index.request("PUT", index.TypeUrl()+"/"+id, i)
}

func (index *Index) Put(u string, i interface{}) (*HttpResponse, error) {
	return index.request("PUT", u, i)
}

type HttpResponse struct {
	*http.Response
	Body []byte
}

func (index *Index) request(method string, u string, i interface{}) (httpResponse *HttpResponse, e error) {
	var req *http.Request
	if i != nil {
		buf := &bytes.Buffer{}
		encoder := json.NewEncoder(buf)
		if e := encoder.Encode(i); e != nil {
			return nil, e
		}
		req, e = http.NewRequest(method, u, buf)
	} else {
		req, e = http.NewRequest(method, u, nil)
	}
	if e != nil {
		return nil, e
	}
	rsp, e := http.DefaultClient.Do(req)
	if e != nil {
		return nil, e
	}
	defer rsp.Body.Close()
	b, e := ioutil.ReadAll(rsp.Body)
	if e != nil {
		return nil, e
	}
	httpResponse = &HttpResponse{
		Response: rsp,
		Body:     b,
	}
	if e != nil {
		return httpResponse, e
	}
	if rsp.Status[0] != OK {
		return httpResponse, fmt.Errorf("error indexing: %s %s", rsp.Status, string(b))
	}
	return httpResponse, nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/base64"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"

	"github.com/drosseau/degob"
)

var (
	outFile     = flag.String("ofile", "", "Output file (defaults to stdout)")
	inFile      = flag.String("ifile", "", "Input file (defaults to stdin)")
	truncateOut = flag.Bool("trunc", false, "Truncate output file")
	base64d     = flag.Bool("b64", false, "base64 input")
	base64urld  = flag.Bool("b64url", false, "base64url input")
	noComments  = flag.Bool("nc", false, "don't print additional comments")
	noTypes     = flag.Bool("nt", false, "don't print type information")
	json        = flag.Bool("json", false, "show value as json")
	pkgName     = flag.String("pkg", "", "include a package definition in the output with the given name")
)

func errorf(s string, v ...interface{}) {
	_, _ = fmt.Fprintf(os.Stderr, s, v...)
	os.Exit(1)
}

func getWriter() io.WriteCloser {
	if *outFile != "" {
		opts := os.O_WRONLY | os.O_CREATE
		if *truncateOut {
			opts |= os.O_TRUNC
		} else {
			opts |= os.O_APPEND
		}
		f, err := os.OpenFile(*outFile, opts, 0644)
		if err != nil {
			errorf("failed to open `%s` for writing: %v\n", *outFile, err)
		}
		return f
	}
	return os.Stdout
}

func getReader() io.ReadCloser {
	if *inFile != "" {
		f, err := os.Open(*outFile)
		if err != nil {
			errorf("failed to open `%s` for reading: %v\n", *inFile, err)
		}
		return f
	}
	return ioutil.NopCloser(os.Stdin)
}

type writer struct {
	w   io.Writer
	err error
}

func (w writer) Write(b []byte) (int, error) {
	if w.err != nil {
		errorf("error writing output: %v\n", w.err)
	}
	var n int
	n, w.err = w.w.Write(b)
	return n, w.err
}

func (w writer) writeComment(s string, v ...interface{}) {
	if *noComments {
		return
	}
	w.writeStr(s, v...)
}

func (w writer) writeStr(s string, v ...interface{}) {
	if w.err != nil {
		errorf("error writing output: %v\n")
	}
	_, w.err = fmt.Fprintf(w.w, s, v...)
}

func main() {
	flag.Parse()
	out := getWriter()
	defer out.Close()
	in := getReader()
	defer in.Close()

	if *base64d {
		in = ioutil.NopCloser(base64.NewDecoder(base64.StdEncoding, in))
	} else if *base64urld {
		in = ioutil.NopCloser(base64.NewDecoder(base64.URLEncoding, in))
	}

	w := writer{w: out}

	dec := degob.NewDecoder(in)
	gobs, err := dec.Decode()
	if err != nil {
		errorf("failed to decode gob: %s\n", err)
	}
	if *pkgName != "" {
		w.writeStr("package %s\n\n", *pkgName)
	}
	for i, g := range gobs {
		w.writeComment("// Decoded gob %d\n\n//Types\n", i+1)
		if !*noTypes {
			err = g.WriteTypes(w)
			if err != nil {
				errorf("error writing types: %v\n", err)
			}
		}
		w.writeComment("// Values:")
		if *json {
			err = g.WriteValue(w, degob.JSON)
		} else {
			err = g.WriteValue(w, degob.SingleLine)
		}
		if err != nil {
			errorf("error writing values: %v\n", err)
		}
		w.writeComment("\n// End gob %d\n\n", i+1)
	}
}
<commit_msg>updated cmd<commit_after>package main

import (
	"encoding/base64"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"

	"github.com/drosseau/degob"
)

var (
	outFile     = flag.String("ofile", "", "Output file (defaults to stdout)")
	inFile      = flag.String("ifile", "", "Input file (defaults to stdin)")
	truncateOut = flag.Bool("trunc", false, "Truncate output file")
	base64d     = flag.Bool("b64", false, "base64 input")
	base64urld  = flag.Bool("b64url", false, "base64url input")
	noComments  = flag.Bool("nc", false, "don't print additional comments")
	noTypes     = flag.Bool("nt", false, "don't print type information")
	json        = flag.Bool("json", false, "show value as json")
	pkgName     = flag.String("pkg", "", "include a package definition in the output with the given name")
)

func errorf(s string, v ...interface{}) {
	_, _ = fmt.Fprintf(os.Stderr, s, v...)
	os.Exit(1)
}

func getWriter() io.WriteCloser {
	if *outFile != "" {
		opts := os.O_WRONLY | os.O_CREATE
		if *truncateOut {
			opts |= os.O_TRUNC
		} else {
			opts |= os.O_APPEND
		}
		f, err := os.OpenFile(*outFile, opts, 0644)
		if err != nil {
			errorf("failed to open `%s` for writing: %v\n", *outFile, err)
		}
		return f
	}
	return os.Stdout
}

func getReader() io.ReadCloser {
	if *inFile != "" {
		f, err := os.Open(*outFile)
		if err != nil {
			errorf("failed to open `%s` for reading: %v\n", *inFile, err)
		}
		return f
	}
	return ioutil.NopCloser(os.Stdin)
}

type writer struct {
	w   io.Writer
	err error
}

func (w writer) Write(b []byte) (int, error) {
	if w.err != nil {
		errorf("error writing output: %v\n", w.err)
	}
	var n int
	n, w.err = w.w.Write(b)
	return n, w.err
}

func (w writer) writeComment(s string, v ...interface{}) {
	if *noComments {
		return
	}
	w.writeStr(s, v...)
}

func (w writer) writeStr(s string, v ...interface{}) {
	if w.err != nil {
		errorf("error writing output: %v\n")
	}
	_, w.err = fmt.Fprintf(w.w, s, v...)
}

func main() {
	flag.Parse()
	out := getWriter()
	defer out.Close()
	in := getReader()
	defer in.Close()

	if *base64d {
		in = ioutil.NopCloser(base64.NewDecoder(base64.StdEncoding, in))
	} else if *base64urld {
		in = ioutil.NopCloser(base64.NewDecoder(base64.URLEncoding, in))
	}

	w := writer{w: out}

	dec := degob.NewDecoder(in)
	gobs, err := dec.Decode()
	if err != nil {
		errorf("failed to decode gob: %s\n", err)
	}
	if *pkgName != "" {
		w.writeStr("package %s\n\n", *pkgName)
	}
	for i, g := range gobs {
		w.writeComment("// Decoded gob %d\n\n", i+1)
		if !*noTypes && g.Types != nil {
			w.writeComment("//Types\n")
			err = g.WriteTypes(w)
			if err != nil {
				errorf("error writing types: %v\n", err)
			}
		}
		w.writeComment("// Value: ")
		if *json {
			err = g.WriteValue(w, degob.JSON)
		} else {
			err = g.WriteValue(w, degob.SingleLine)
		}
		if err != nil {
			errorf("error writing values: %v\n", err)
		}
		w.writeComment("\n// End gob %d\n\n", i+1)
	}
}
<|endoftext|>


<commit_before>// Package iprof provides methods for global instrumented profiling.
package iprof

import (
	"math"
	"sort"
	"time"
)

const defaultWsize uint = 5000

type reading struct {
	duration time.Duration
	end      time.Time
}

type nreading struct {
	reading
	section string
}

var profs chan nreading
var stats map[string][]reading
var count map[string]uint
var wsize map[string]uint

func init() {
	stats = make(map[string][]reading)
	wsize = make(map[string]uint)
	count = make(map[string]uint)
	profs = make(chan nreading)
	go func() {
		for r := range profs {
			l := uint(len(stats[r.section]))
			w, ok := wsize[r.section]
			if !ok {
				w = defaultWsize
			}

			if l == w {
				stats[r.section] = append(stats[r.section][1:], r.reading)
			} else {
				stats[r.section] = append(stats[r.section], r.reading)
			}
			count[r.section]++
		}
	}()
}

// Start indicates the start of a new timed section.
// The returned function should be called when the section ends.
// Start may be called by multiple concurrent goroutines.
func Start(section string) func() time.Duration {
	start := time.Now()
	return func() time.Duration {
		end := time.Now()
		d := end.Sub(start)
		go func() {
			profs <- nreading{reading{d, end}, section}
		}()
		return d
	}
}

// Log allows the direct recording of timing information for a section.
// Under most circumstances, Start should be used instead.
// Log may be called by multiple concurrent goroutines.
func Log(section string, duration time.Duration, end time.Time) {
	profs <- nreading{reading{duration, end}, section}
}

// SetWindow sets the window sampling size for a section.
// The default window size is 5000 samples, after which the oldest sample will
// be expired when a new sample is recorded.
// SetWindow should be called before Start or Log.
func SetWindow(section string, window uint) {
	wsize[section] = window
}

type durationSlice []float64

func (s durationSlice) Len() int {
	return len(s)
}
func (s durationSlice) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}
func (s durationSlice) Less(i, j int) bool {
	return s[i] < s[j]
}

// Stat returns aggregated timing information about a section.
// It returns the average time spent in the section in milliseconds, as well as
// a function for computing the Nth percentile of the section's samples.
func Stat(section string) (num uint, average float64, percentile func(float64) float64) {
	total := float64(0)
	vals := make(durationSlice, len(stats[section]))
	for i, r := range stats[section] {
		v := r.duration.Seconds() * 1000
		vals[i] = v
		total += v
	}

	num = count[section]
	length := uint(len(vals))
	average = total / float64(length)

	sort.Sort(vals)
	percentile = func(perc float64) float64 {
		n := (perc / 100.0) * float64(length+1)
		k := uint(math.Floor(n))
		if k == 0 {
			return vals[0]
		}
		if k >= length-1 {
			return vals[length-1]
		}

		d := n - float64(k)
		return vals[k] + d*(vals[k+1]-vals[k])
	}

	return
}

type Profile struct {
	Count      uint
	Average    float64
	Percentile func(float64) float64
}

func Stats() map[string]Profile {
	ret := make(map[string]Profile)
	for section := range stats {
		c, a, p := Stat(section)
		ret[section] = Profile{c, a, p}
	}
	return ret
}
<commit_msg>Avoid potential racy index out of range<commit_after>// Package iprof provides methods for global instrumented profiling.
package iprof

import (
	"math"
	"sort"
	"time"
)

const defaultWsize uint = 5000

type reading struct {
	duration time.Duration
	end      time.Time
}

type nreading struct {
	reading
	section string
}

var profs chan nreading
var stats map[string][]reading
var count map[string]uint
var wsize map[string]uint

func init() {
	stats = make(map[string][]reading)
	wsize = make(map[string]uint)
	count = make(map[string]uint)
	profs = make(chan nreading)
	go func() {
		for r := range profs {
			l := uint(len(stats[r.section]))
			w, ok := wsize[r.section]
			if !ok {
				w = defaultWsize
			}

			if l == w {
				stats[r.section] = append(stats[r.section][1:], r.reading)
			} else {
				stats[r.section] = append(stats[r.section], r.reading)
			}
			count[r.section]++
		}
	}()
}

// Start indicates the start of a new timed section.
// The returned function should be called when the section ends.
// Start may be called by multiple concurrent goroutines.
func Start(section string) func() time.Duration {
	start := time.Now()
	return func() time.Duration {
		end := time.Now()
		d := end.Sub(start)
		go func() {
			profs <- nreading{reading{d, end}, section}
		}()
		return d
	}
}

// Log allows the direct recording of timing information for a section.
// Under most circumstances, Start should be used instead.
// Log may be called by multiple concurrent goroutines.
func Log(section string, duration time.Duration, end time.Time) {
	profs <- nreading{reading{duration, end}, section}
}

// SetWindow sets the window sampling size for a section.
// The default window size is 5000 samples, after which the oldest sample will
// be expired when a new sample is recorded.
// SetWindow should be called before Start or Log.
func SetWindow(section string, window uint) {
	wsize[section] = window
}

type durationSlice []float64

func (s durationSlice) Len() int {
	return len(s)
}
func (s durationSlice) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}
func (s durationSlice) Less(i, j int) bool {
	return s[i] < s[j]
}

// Stat returns aggregated timing information about a section.
// It returns the average time spent in the section in milliseconds, as well as
// a function for computing the Nth percentile of the section's samples.
func Stat(section string) (num uint, average float64, percentile func(float64) float64) {
	total := float64(0)
	vals := make(durationSlice, 0, len(stats[section]))
	for _, r := range stats[section] {
		v := r.duration.Seconds() * 1000
		vals = append(vals, v)
		total += v
	}

	num = count[section]
	length := uint(len(vals))
	average = total / float64(length)

	sort.Sort(vals)
	percentile = func(perc float64) float64 {
		n := (perc / 100.0) * float64(length+1)
		k := uint(math.Floor(n))
		if k == 0 {
			return vals[0]
		}
		if k >= length-1 {
			return vals[length-1]
		}

		d := n - float64(k)
		return vals[k] + d*(vals[k+1]-vals[k])
	}

	return
}

type Profile struct {
	Count      uint
	Average    float64
	Percentile func(float64) float64
}

func Stats() map[string]Profile {
	ret := make(map[string]Profile)
	for section := range stats {
		c, a, p := Stat(section)
		ret[section] = Profile{c, a, p}
	}
	return ret
}
<|endoftext|>