<commit_before>package main

import (
	"context"
	"database/sql"
	"errors"
	"flag"
	"fmt"
	"os"
	"time"

	"github.com/honeycombio/beeline-go"
	"github.com/jmhodges/clock"
	"github.com/prometheus/client_golang/prometheus"

	capb "github.com/letsencrypt/boulder/ca/proto"
	"github.com/letsencrypt/boulder/cmd"
	"github.com/letsencrypt/boulder/core"
	"github.com/letsencrypt/boulder/db"
	"github.com/letsencrypt/boulder/features"
	bgrpc "github.com/letsencrypt/boulder/grpc"
	blog "github.com/letsencrypt/boulder/log"
	"github.com/letsencrypt/boulder/sa"
)

// ocspDB and ocspReadOnlyDB are interfaces collecting the gorp.DbMap functions that
// the various parts of OCSPUpdater rely on. Using this adapter shim allows tests to
// swap out the dbMap implementation.

// ocspReadOnlyDB provides only read-only portions of the gorp.DbMap interface
type ocspReadOnlyDB interface {
	Select(i interface{}, query string, args ...interface{}) ([]interface{}, error)
}

// ocspDB provides read-write portions of the gorp.DbMap interface
type ocspDB interface {
	ocspReadOnlyDB
	Exec(query string, args ...interface{}) (sql.Result, error)
}

// OCSPUpdater contains the useful objects for the Updater
type OCSPUpdater struct {
	log blog.Logger
	clk clock.Clock

	dbMap         ocspDB
	readOnlyDbMap ocspReadOnlyDB

	ogc capb.OCSPGeneratorClient

	tickWindow    time.Duration
	batchSize     int
	tickHistogram *prometheus.HistogramVec

	maxBackoff    time.Duration
	backoffFactor float64
	tickFailures  int

	// Used to calculate how far back stale OCSP responses should be looked for
	ocspMinTimeToExpiry time.Duration
	// Maximum number of individual OCSP updates to attempt in parallel. Making
	// these requests in parallel allows us to get higher total throughput.
	parallelGenerateOCSPRequests int

	stalenessHistogram prometheus.Histogram
	genStoreHistogram  prometheus.Histogram
	generatedCounter   *prometheus.CounterVec
	storedCounter      *prometheus.CounterVec
}

func newUpdater(
	stats prometheus.Registerer,
	clk clock.Clock,
	dbMap ocspDB,
	readOnlyDbMap ocspReadOnlyDB,
	ogc capb.OCSPGeneratorClient,
	config OCSPUpdaterConfig,
	log blog.Logger,
) (*OCSPUpdater, error) {
	if config.OldOCSPBatchSize == 0 {
		return nil, fmt.Errorf("Loop batch sizes must be non-zero")
	}
	if config.OldOCSPWindow.Duration == 0 {
		return nil, fmt.Errorf("Loop window sizes must be non-zero")
	}
	if config.ParallelGenerateOCSPRequests == 0 {
		// Default to 1
		config.ParallelGenerateOCSPRequests = 1
	}

	genStoreHistogram := prometheus.NewHistogram(prometheus.HistogramOpts{
		Name: "ocsp_updater_generate_and_store",
		Help: "A histogram of latencies of OCSP generation and storage latencies",
	})
	stats.MustRegister(genStoreHistogram)
	generatedCounter := prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "ocsp_updater_generated",
		Help: "A counter of OCSP response generation calls labelled by result",
	}, []string{"result"})
	stats.MustRegister(generatedCounter)
	storedCounter := prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "ocsp_updater_stored",
		Help: "A counter of OCSP response storage calls labelled by result",
	}, []string{"result"})
	stats.MustRegister(storedCounter)
	tickHistogram := prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Name: "ocsp_updater_ticks",
		Help: "A histogram of ocsp-updater tick latencies labelled by result and whether the tick was considered longer than expected",
	}, []string{"result", "long"})
	stats.MustRegister(tickHistogram)
	stalenessHistogram := prometheus.NewHistogram(prometheus.HistogramOpts{
		Name:    "ocsp_status_staleness",
		Help:    "How long past the refresh time a status is when we try to refresh it. Will always be > 0, but must stay well below 12 hours.",
		Buckets: []float64{10, 100, 1000, 10000, 21600, 32400, 36000, 39600, 43200},
	})
	stats.MustRegister(stalenessHistogram)

	updater := OCSPUpdater{
		clk:                          clk,
		dbMap:                        dbMap,
		readOnlyDbMap:                readOnlyDbMap,
		ogc:                          ogc,
		log:                          log,
		ocspMinTimeToExpiry:          config.OCSPMinTimeToExpiry.Duration,
		parallelGenerateOCSPRequests: config.ParallelGenerateOCSPRequests,
		genStoreHistogram:            genStoreHistogram,
		generatedCounter:             generatedCounter,
		storedCounter:                storedCounter,
		stalenessHistogram:           stalenessHistogram,
		tickHistogram:                tickHistogram,
		tickWindow:                   config.OldOCSPWindow.Duration,
		batchSize:                    config.OldOCSPBatchSize,
		maxBackoff:                   config.SignFailureBackoffMax.Duration,
		backoffFactor:                config.SignFailureBackoffFactor,
	}

	return &updater, nil
}

func (updater *OCSPUpdater) findStaleOCSPResponses(oldestLastUpdatedTime time.Time, batchSize int) ([]core.CertificateStatus, error) {
	statuses, err := sa.SelectCertificateStatuses(
		updater.readOnlyDbMap,
		`WHERE ocspLastUpdated < :lastUpdate
		 AND NOT isExpired
		 ORDER BY ocspLastUpdated ASC
		 LIMIT :limit`,
		map[string]interface{}{
			"lastUpdate": oldestLastUpdatedTime,
			"limit":      batchSize,
		},
	)
	if db.IsNoRows(err) {
		return nil, nil
	}

	for _, status := range statuses {
		staleness := oldestLastUpdatedTime.Sub(status.OCSPLastUpdated).Seconds()
		updater.stalenessHistogram.Observe(staleness)
	}

	return statuses, err
}

func (updater *OCSPUpdater) generateResponse(ctx context.Context, status core.CertificateStatus) (*core.CertificateStatus, error) {
	if status.IssuerID == 0 {
		return nil, errors.New("cert status has 0 IssuerID")
	}
	ocspReq := capb.GenerateOCSPRequest{
		Serial:    status.Serial,
		IssuerID:  status.IssuerID,
		Status:    string(status.Status),
		Reason:    int32(status.RevokedReason),
		RevokedAt: status.RevokedDate.UnixNano(),
	}

	ocspResponse, err := updater.ogc.GenerateOCSP(ctx, &ocspReq)
	if err != nil {
		return nil, err
	}

	status.OCSPLastUpdated = updater.clk.Now()
	status.OCSPResponse = ocspResponse.Response

	return &status, nil
}

func (updater *OCSPUpdater) storeResponse(status *core.CertificateStatus) error {
	// Update the certificateStatus table with the new OCSP response, the status
	// WHERE is used make sure we don't overwrite a revoked response with a one
	// containing a 'good' status.
	_, err := updater.dbMap.Exec(
		`UPDATE certificateStatus
		 SET ocspResponse=?,ocspLastUpdated=?
		 WHERE serial=?
		 AND status=?`,
		status.OCSPResponse,
		status.OCSPLastUpdated,
		status.Serial,
		string(status.Status),
	)
	return err
}

// markExpired updates a given CertificateStatus to have `isExpired` set.
func (updater *OCSPUpdater) markExpired(status core.CertificateStatus) error {
	_, err := updater.dbMap.Exec(
		`UPDATE certificateStatus
 		SET isExpired = TRUE
 		WHERE serial = ?`,
		status.Serial,
	)
	return err
}

func (updater *OCSPUpdater) generateOCSPResponses(ctx context.Context, statuses []core.CertificateStatus) error {
	// Use the semaphore pattern from
	// https://github.com/golang/go/wiki/BoundingResourceUse to send a number of
	// GenerateOCSP / storeResponse requests in parallel, while limiting the total number of
	// outstanding requests. The number of outstanding requests equals the
	// capacity of the channel.
	sem := make(chan int, updater.parallelGenerateOCSPRequests)
	wait := func() {
		sem <- 1 // Block until there's capacity.
	}
	done := func(start time.Time) {
		<-sem // Indicate there's more capacity.
		updater.genStoreHistogram.Observe(time.Since(start).Seconds())
	}

	work := func(status core.CertificateStatus) {
		defer done(updater.clk.Now())
		meta, err := updater.generateResponse(ctx, status)
		if err != nil {
			updater.log.AuditErrf("Failed to generate OCSP response: %s", err)
			updater.generatedCounter.WithLabelValues("failed").Inc()
			return
		}
		updater.generatedCounter.WithLabelValues("success").Inc()
		err = updater.storeResponse(meta)
		if err != nil {
			updater.log.AuditErrf("Failed to store OCSP response: %s", err)
			updater.storedCounter.WithLabelValues("failed").Inc()
			return
		}
		updater.storedCounter.WithLabelValues("success").Inc()
	}

	for _, status := range statuses {
		wait()
		go work(status)
	}
	// Block until the channel reaches its full capacity again, indicating each
	// goroutine has completed.
	for i := 0; i < updater.parallelGenerateOCSPRequests; i++ {
		wait()
	}
	return nil
}

// updateOCSPResponses looks for certificates with stale OCSP responses and
// generates/stores new ones
func (updater *OCSPUpdater) updateOCSPResponses(ctx context.Context, batchSize int) error {
	tickStart := updater.clk.Now()
	statuses, err := updater.findStaleOCSPResponses(tickStart.Add(-updater.ocspMinTimeToExpiry), batchSize)
	if err != nil {
		updater.log.AuditErrf("Failed to find stale OCSP responses: %s", err)
		return err
	}

	for _, s := range statuses {
		if !s.IsExpired && tickStart.After(s.NotAfter) {
			err := updater.markExpired(s)
			if err != nil {
				return err
			}
		}
	}

	return updater.generateOCSPResponses(ctx, statuses)
}

type config struct {
	OCSPUpdater OCSPUpdaterConfig

	Syslog  cmd.SyslogConfig
	Beeline cmd.BeelineConfig
}

// OCSPUpdaterConfig provides the various window tick times and batch sizes needed
// for the OCSP (and SCT) updater
type OCSPUpdaterConfig struct {
	cmd.ServiceConfig
	DB         cmd.DBConfig
	ReadOnlyDB cmd.DBConfig

	OldOCSPWindow    cmd.ConfigDuration
	OldOCSPBatchSize int

	OCSPMinTimeToExpiry          cmd.ConfigDuration
	ParallelGenerateOCSPRequests int

	SignFailureBackoffFactor float64
	SignFailureBackoffMax    cmd.ConfigDuration

	OCSPGeneratorService *cmd.GRPCClientConfig

	Features map[string]bool
}

func (updater *OCSPUpdater) tick() {
	start := updater.clk.Now()
	err := updater.updateOCSPResponses(context.Background(), updater.batchSize)
	end := updater.clk.Now()
	took := end.Sub(start)
	long, state := "false", "success"
	if took > updater.tickWindow {
		long = "true"
	}
	sleepDur := start.Add(updater.tickWindow).Sub(end)
	if err != nil {
		state = "failed"
		updater.tickFailures++
		sleepDur = core.RetryBackoff(
			updater.tickFailures,
			updater.tickWindow,
			updater.maxBackoff,
			updater.backoffFactor,
		)
	} else if updater.tickFailures > 0 {
		updater.tickFailures = 0
	}
	updater.tickHistogram.WithLabelValues(state, long).Observe(took.Seconds())
	updater.clk.Sleep(sleepDur)
}

func main() {
	configFile := flag.String("config", "", "File path to the configuration file for this service")
	flag.Parse()
	if *configFile == "" {
		flag.Usage()
		os.Exit(1)
	}

	var c config
	err := cmd.ReadConfigFile(*configFile, &c)
	cmd.FailOnError(err, "Reading JSON config file into config structure")

	conf := c.OCSPUpdater
	err = features.Set(conf.Features)
	cmd.FailOnError(err, "Failed to set feature flags")

	bc, err := c.Beeline.Load()
	cmd.FailOnError(err, "Failed to load Beeline config")
	beeline.Init(bc)
	defer beeline.Close()

	stats, logger := cmd.StatsAndLogging(c.Syslog, conf.DebugAddr)
	defer logger.AuditPanic()
	logger.Info(cmd.VersionString())

	// Configure DB
	configureDb := func(scope prometheus.Registerer, databaseConfig cmd.DBConfig) *db.WrappedMap {
		dbSettings := sa.DbSettings{
			MaxOpenConns:    databaseConfig.MaxOpenConns,
			MaxIdleConns:    databaseConfig.MaxIdleConns,
			ConnMaxLifetime: databaseConfig.ConnMaxLifetime.Duration,
			ConnMaxIdleTime: databaseConfig.ConnMaxIdleTime.Duration,
		}

		dbDSN, err := databaseConfig.URL()
		cmd.FailOnError(err, "Couldn't load DB URL")

		dbMap, err := sa.NewDbMap(dbDSN, dbSettings)
		cmd.FailOnError(err, "Couldn't connect to SA database")

		dbAddr, dbUser, err := databaseConfig.DSNAddressAndUser()
		cmd.FailOnError(err, "Could not determine address or user of DB DSN")

		// Collect and periodically report DB metrics using the DBMap and prometheus scope.
		sa.InitDBMetrics(dbMap, scope, dbSettings, dbAddr, dbUser)

		return dbMap
	}

	dbMap := configureDb(stats, conf.DB)

	dbReadOnlyURL, err := conf.ReadOnlyDB.URL()
	cmd.FailOnError(err, "Couldn't load read-only DB URL")

	var dbReadOnlyMap *db.WrappedMap
	if dbReadOnlyURL == "" {
		dbReadOnlyMap = dbMap
	} else {
		dbReadOnlyMap = configureDb(stats, conf.ReadOnlyDB)
	}

	clk := cmd.Clock()

	tlsConfig, err := c.OCSPUpdater.TLS.Load()
	cmd.FailOnError(err, "TLS config")
	clientMetrics := bgrpc.NewClientMetrics(stats)
	caConn, err := bgrpc.ClientSetup(c.OCSPUpdater.OCSPGeneratorService, tlsConfig, clientMetrics, clk)
	cmd.FailOnError(err, "Failed to load credentials and create gRPC connection to CA")
	ogc := capb.NewOCSPGeneratorClient(caConn)

	updater, err := newUpdater(
		stats,
		clk,
		dbMap,
		dbReadOnlyMap,
		ogc,
		// Necessary evil for now
		conf,
		logger,
	)
	cmd.FailOnError(err, "Failed to create updater")

	go cmd.CatchSignals(logger, nil)

	for {
		updater.tick()
	}
}
<commit_msg>Export ocsp_status_staleness for 15, 18, 21, and 24hr buckets (#5627)<commit_after>package main

import (
	"context"
	"database/sql"
	"errors"
	"flag"
	"fmt"
	"os"
	"time"

	"github.com/honeycombio/beeline-go"
	"github.com/jmhodges/clock"
	"github.com/prometheus/client_golang/prometheus"

	capb "github.com/letsencrypt/boulder/ca/proto"
	"github.com/letsencrypt/boulder/cmd"
	"github.com/letsencrypt/boulder/core"
	"github.com/letsencrypt/boulder/db"
	"github.com/letsencrypt/boulder/features"
	bgrpc "github.com/letsencrypt/boulder/grpc"
	blog "github.com/letsencrypt/boulder/log"
	"github.com/letsencrypt/boulder/sa"
)

// ocspDB and ocspReadOnlyDB are interfaces collecting the gorp.DbMap functions that
// the various parts of OCSPUpdater rely on. Using this adapter shim allows tests to
// swap out the dbMap implementation.

// ocspReadOnlyDB provides only read-only portions of the gorp.DbMap interface
type ocspReadOnlyDB interface {
	Select(i interface{}, query string, args ...interface{}) ([]interface{}, error)
}

// ocspDB provides read-write portions of the gorp.DbMap interface
type ocspDB interface {
	ocspReadOnlyDB
	Exec(query string, args ...interface{}) (sql.Result, error)
}

// OCSPUpdater contains the useful objects for the Updater
type OCSPUpdater struct {
	log blog.Logger
	clk clock.Clock

	dbMap         ocspDB
	readOnlyDbMap ocspReadOnlyDB

	ogc capb.OCSPGeneratorClient

	tickWindow    time.Duration
	batchSize     int
	tickHistogram *prometheus.HistogramVec

	maxBackoff    time.Duration
	backoffFactor float64
	tickFailures  int

	// Used to calculate how far back stale OCSP responses should be looked for
	ocspMinTimeToExpiry time.Duration
	// Maximum number of individual OCSP updates to attempt in parallel. Making
	// these requests in parallel allows us to get higher total throughput.
	parallelGenerateOCSPRequests int

	stalenessHistogram prometheus.Histogram
	genStoreHistogram  prometheus.Histogram
	generatedCounter   *prometheus.CounterVec
	storedCounter      *prometheus.CounterVec
}

func newUpdater(
	stats prometheus.Registerer,
	clk clock.Clock,
	dbMap ocspDB,
	readOnlyDbMap ocspReadOnlyDB,
	ogc capb.OCSPGeneratorClient,
	config OCSPUpdaterConfig,
	log blog.Logger,
) (*OCSPUpdater, error) {
	if config.OldOCSPBatchSize == 0 {
		return nil, fmt.Errorf("Loop batch sizes must be non-zero")
	}
	if config.OldOCSPWindow.Duration == 0 {
		return nil, fmt.Errorf("Loop window sizes must be non-zero")
	}
	if config.ParallelGenerateOCSPRequests == 0 {
		// Default to 1
		config.ParallelGenerateOCSPRequests = 1
	}

	genStoreHistogram := prometheus.NewHistogram(prometheus.HistogramOpts{
		Name: "ocsp_updater_generate_and_store",
		Help: "A histogram of latencies of OCSP generation and storage latencies",
	})
	stats.MustRegister(genStoreHistogram)
	generatedCounter := prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "ocsp_updater_generated",
		Help: "A counter of OCSP response generation calls labelled by result",
	}, []string{"result"})
	stats.MustRegister(generatedCounter)
	storedCounter := prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "ocsp_updater_stored",
		Help: "A counter of OCSP response storage calls labelled by result",
	}, []string{"result"})
	stats.MustRegister(storedCounter)
	tickHistogram := prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Name: "ocsp_updater_ticks",
		Help: "A histogram of ocsp-updater tick latencies labelled by result and whether the tick was considered longer than expected",
	}, []string{"result", "long"})
	stats.MustRegister(tickHistogram)
	stalenessHistogram := prometheus.NewHistogram(prometheus.HistogramOpts{
		Name:    "ocsp_status_staleness",
		Help:    "How long past the refresh time a status is when we try to refresh it. Will always be > 0, but must stay well below 12 hours.",
		Buckets: []float64{10, 100, 1000, 10000, 21600, 32400, 36000, 39600, 43200, 54000, 64800, 75600, 86400},
	})
	stats.MustRegister(stalenessHistogram)

	updater := OCSPUpdater{
		clk:                          clk,
		dbMap:                        dbMap,
		readOnlyDbMap:                readOnlyDbMap,
		ogc:                          ogc,
		log:                          log,
		ocspMinTimeToExpiry:          config.OCSPMinTimeToExpiry.Duration,
		parallelGenerateOCSPRequests: config.ParallelGenerateOCSPRequests,
		genStoreHistogram:            genStoreHistogram,
		generatedCounter:             generatedCounter,
		storedCounter:                storedCounter,
		stalenessHistogram:           stalenessHistogram,
		tickHistogram:                tickHistogram,
		tickWindow:                   config.OldOCSPWindow.Duration,
		batchSize:                    config.OldOCSPBatchSize,
		maxBackoff:                   config.SignFailureBackoffMax.Duration,
		backoffFactor:                config.SignFailureBackoffFactor,
	}

	return &updater, nil
}

func (updater *OCSPUpdater) findStaleOCSPResponses(oldestLastUpdatedTime time.Time, batchSize int) ([]core.CertificateStatus, error) {
	statuses, err := sa.SelectCertificateStatuses(
		updater.readOnlyDbMap,
		`WHERE ocspLastUpdated < :lastUpdate
		 AND NOT isExpired
		 ORDER BY ocspLastUpdated ASC
		 LIMIT :limit`,
		map[string]interface{}{
			"lastUpdate": oldestLastUpdatedTime,
			"limit":      batchSize,
		},
	)
	if db.IsNoRows(err) {
		return nil, nil
	}

	for _, status := range statuses {
		staleness := oldestLastUpdatedTime.Sub(status.OCSPLastUpdated).Seconds()
		updater.stalenessHistogram.Observe(staleness)
	}

	return statuses, err
}

func (updater *OCSPUpdater) generateResponse(ctx context.Context, status core.CertificateStatus) (*core.CertificateStatus, error) {
	if status.IssuerID == 0 {
		return nil, errors.New("cert status has 0 IssuerID")
	}
	ocspReq := capb.GenerateOCSPRequest{
		Serial:    status.Serial,
		IssuerID:  status.IssuerID,
		Status:    string(status.Status),
		Reason:    int32(status.RevokedReason),
		RevokedAt: status.RevokedDate.UnixNano(),
	}

	ocspResponse, err := updater.ogc.GenerateOCSP(ctx, &ocspReq)
	if err != nil {
		return nil, err
	}

	status.OCSPLastUpdated = updater.clk.Now()
	status.OCSPResponse = ocspResponse.Response

	return &status, nil
}

func (updater *OCSPUpdater) storeResponse(status *core.CertificateStatus) error {
	// Update the certificateStatus table with the new OCSP response, the status
	// WHERE is used make sure we don't overwrite a revoked response with a one
	// containing a 'good' status.
	_, err := updater.dbMap.Exec(
		`UPDATE certificateStatus
		 SET ocspResponse=?,ocspLastUpdated=?
		 WHERE serial=?
		 AND status=?`,
		status.OCSPResponse,
		status.OCSPLastUpdated,
		status.Serial,
		string(status.Status),
	)
	return err
}

// markExpired updates a given CertificateStatus to have `isExpired` set.
func (updater *OCSPUpdater) markExpired(status core.CertificateStatus) error {
	_, err := updater.dbMap.Exec(
		`UPDATE certificateStatus
 		SET isExpired = TRUE
 		WHERE serial = ?`,
		status.Serial,
	)
	return err
}

func (updater *OCSPUpdater) generateOCSPResponses(ctx context.Context, statuses []core.CertificateStatus) error {
	// Use the semaphore pattern from
	// https://github.com/golang/go/wiki/BoundingResourceUse to send a number of
	// GenerateOCSP / storeResponse requests in parallel, while limiting the total number of
	// outstanding requests. The number of outstanding requests equals the
	// capacity of the channel.
	sem := make(chan int, updater.parallelGenerateOCSPRequests)
	wait := func() {
		sem <- 1 // Block until there's capacity.
	}
	done := func(start time.Time) {
		<-sem // Indicate there's more capacity.
		updater.genStoreHistogram.Observe(time.Since(start).Seconds())
	}

	work := func(status core.CertificateStatus) {
		defer done(updater.clk.Now())
		meta, err := updater.generateResponse(ctx, status)
		if err != nil {
			updater.log.AuditErrf("Failed to generate OCSP response: %s", err)
			updater.generatedCounter.WithLabelValues("failed").Inc()
			return
		}
		updater.generatedCounter.WithLabelValues("success").Inc()
		err = updater.storeResponse(meta)
		if err != nil {
			updater.log.AuditErrf("Failed to store OCSP response: %s", err)
			updater.storedCounter.WithLabelValues("failed").Inc()
			return
		}
		updater.storedCounter.WithLabelValues("success").Inc()
	}

	for _, status := range statuses {
		wait()
		go work(status)
	}
	// Block until the channel reaches its full capacity again, indicating each
	// goroutine has completed.
	for i := 0; i < updater.parallelGenerateOCSPRequests; i++ {
		wait()
	}
	return nil
}

// updateOCSPResponses looks for certificates with stale OCSP responses and
// generates/stores new ones
func (updater *OCSPUpdater) updateOCSPResponses(ctx context.Context, batchSize int) error {
	tickStart := updater.clk.Now()
	statuses, err := updater.findStaleOCSPResponses(tickStart.Add(-updater.ocspMinTimeToExpiry), batchSize)
	if err != nil {
		updater.log.AuditErrf("Failed to find stale OCSP responses: %s", err)
		return err
	}

	for _, s := range statuses {
		if !s.IsExpired && tickStart.After(s.NotAfter) {
			err := updater.markExpired(s)
			if err != nil {
				return err
			}
		}
	}

	return updater.generateOCSPResponses(ctx, statuses)
}

type config struct {
	OCSPUpdater OCSPUpdaterConfig

	Syslog  cmd.SyslogConfig
	Beeline cmd.BeelineConfig
}

// OCSPUpdaterConfig provides the various window tick times and batch sizes needed
// for the OCSP (and SCT) updater
type OCSPUpdaterConfig struct {
	cmd.ServiceConfig
	DB         cmd.DBConfig
	ReadOnlyDB cmd.DBConfig

	OldOCSPWindow    cmd.ConfigDuration
	OldOCSPBatchSize int

	OCSPMinTimeToExpiry          cmd.ConfigDuration
	ParallelGenerateOCSPRequests int

	SignFailureBackoffFactor float64
	SignFailureBackoffMax    cmd.ConfigDuration

	OCSPGeneratorService *cmd.GRPCClientConfig

	Features map[string]bool
}

func (updater *OCSPUpdater) tick() {
	start := updater.clk.Now()
	err := updater.updateOCSPResponses(context.Background(), updater.batchSize)
	end := updater.clk.Now()
	took := end.Sub(start)
	long, state := "false", "success"
	if took > updater.tickWindow {
		long = "true"
	}
	sleepDur := start.Add(updater.tickWindow).Sub(end)
	if err != nil {
		state = "failed"
		updater.tickFailures++
		sleepDur = core.RetryBackoff(
			updater.tickFailures,
			updater.tickWindow,
			updater.maxBackoff,
			updater.backoffFactor,
		)
	} else if updater.tickFailures > 0 {
		updater.tickFailures = 0
	}
	updater.tickHistogram.WithLabelValues(state, long).Observe(took.Seconds())
	updater.clk.Sleep(sleepDur)
}

func main() {
	configFile := flag.String("config", "", "File path to the configuration file for this service")
	flag.Parse()
	if *configFile == "" {
		flag.Usage()
		os.Exit(1)
	}

	var c config
	err := cmd.ReadConfigFile(*configFile, &c)
	cmd.FailOnError(err, "Reading JSON config file into config structure")

	conf := c.OCSPUpdater
	err = features.Set(conf.Features)
	cmd.FailOnError(err, "Failed to set feature flags")

	bc, err := c.Beeline.Load()
	cmd.FailOnError(err, "Failed to load Beeline config")
	beeline.Init(bc)
	defer beeline.Close()

	stats, logger := cmd.StatsAndLogging(c.Syslog, conf.DebugAddr)
	defer logger.AuditPanic()
	logger.Info(cmd.VersionString())

	// Configure DB
	configureDb := func(scope prometheus.Registerer, databaseConfig cmd.DBConfig) *db.WrappedMap {
		dbSettings := sa.DbSettings{
			MaxOpenConns:    databaseConfig.MaxOpenConns,
			MaxIdleConns:    databaseConfig.MaxIdleConns,
			ConnMaxLifetime: databaseConfig.ConnMaxLifetime.Duration,
			ConnMaxIdleTime: databaseConfig.ConnMaxIdleTime.Duration,
		}

		dbDSN, err := databaseConfig.URL()
		cmd.FailOnError(err, "Couldn't load DB URL")

		dbMap, err := sa.NewDbMap(dbDSN, dbSettings)
		cmd.FailOnError(err, "Couldn't connect to SA database")

		dbAddr, dbUser, err := databaseConfig.DSNAddressAndUser()
		cmd.FailOnError(err, "Could not determine address or user of DB DSN")

		// Collect and periodically report DB metrics using the DBMap and prometheus scope.
		sa.InitDBMetrics(dbMap, scope, dbSettings, dbAddr, dbUser)

		return dbMap
	}

	dbMap := configureDb(stats, conf.DB)

	dbReadOnlyURL, err := conf.ReadOnlyDB.URL()
	cmd.FailOnError(err, "Couldn't load read-only DB URL")

	var dbReadOnlyMap *db.WrappedMap
	if dbReadOnlyURL == "" {
		dbReadOnlyMap = dbMap
	} else {
		dbReadOnlyMap = configureDb(stats, conf.ReadOnlyDB)
	}

	clk := cmd.Clock()

	tlsConfig, err := c.OCSPUpdater.TLS.Load()
	cmd.FailOnError(err, "TLS config")
	clientMetrics := bgrpc.NewClientMetrics(stats)
	caConn, err := bgrpc.ClientSetup(c.OCSPUpdater.OCSPGeneratorService, tlsConfig, clientMetrics, clk)
	cmd.FailOnError(err, "Failed to load credentials and create gRPC connection to CA")
	ogc := capb.NewOCSPGeneratorClient(caConn)

	updater, err := newUpdater(
		stats,
		clk,
		dbMap,
		dbReadOnlyMap,
		ogc,
		// Necessary evil for now
		conf,
		logger,
	)
	cmd.FailOnError(err, "Failed to create updater")

	go cmd.CatchSignals(logger, nil)

	for {
		updater.tick()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"sync"

	"github.com/akhenakh/hunspellgo"
	"github.com/trustmaster/go-aspell"
)

type Speller interface {
	Check(w string) bool
	Suggest(w string) []string
	Close()
}

type aspeller struct {
	sp aspell.Speller
	mu sync.Mutex
}

func NewASpeller() (Speller, error) {
	opts := map[string]string{
		"lang":           "en",
		"filter":         "url",
		"mode":           "url",
		"encoding":       "ascii",
		"guess":          "true",
		"ignore":         "0",
		"ignore-case":    "false",
		"ignore-accents": "false",
	}

	sp, err := aspell.NewSpeller(opts)
	if err != nil {
		return nil, err
	}
	return &aspeller{sp: sp}, nil
}

func (s *aspeller) Check(w string) bool {
	s.mu.Lock()
	defer s.mu.Unlock()
	return s.sp.Check(w)
}

func (s *aspeller) Suggest(w string) []string {
	s.mu.Lock()
	defer s.mu.Unlock()
	return s.sp.Suggest(w)
}

func (s *aspeller) Close() { s.sp.Delete() }

type hunspeller struct {
	sp *hunspellgo.Hunhandle
}

// hunspellPaths has a list of paths where dictionaries might be
// in future, pass path through command line argument
var hunspellPaths = []string{
	"/usr/share/myspell",
	"/usr/share/hunspell",
}

func NewHunSpeller() Speller {
	for _, p := range hunspellPaths {
		if sp := hunspellgo.Hunspell(p+"/en_US.aff", p+"/en_US.dic"); sp != nil {
			return &hunspeller{sp}
		}
	}
	return nil
}

func (s *hunspeller) Check(w string) bool {
	return s.sp.Spell(w)
}

func (s *hunspeller) Suggest(w string) []string {
	return s.sp.Suggest(w)
}

func (s *hunspeller) Close() { s.sp = nil }

type multispeller struct {
	sp []Speller
}

func NewMultiSpeller(sp ...Speller) Speller {
	m := &multispeller{}
	for _, s := range sp {
		m.sp = append(m.sp, s)
	}
	return m
}

func (s *multispeller) Check(w string) bool {
	for _, sp := range s.sp {
		if sp.Check(w) {
			return true
		}
	}
	return false
}

func (s *multispeller) Suggest(w string) (ret []string) {
	for _, sp := range s.sp {
		if ret = sp.Suggest(w); len(ret) != 0 {
			break
		}
	}
	return ret
}

func (s *multispeller) Close() {
	for _, sp := range s.sp {
		sp.Close()
	}
	s.sp = nil
}
<commit_msg>speller: check if files exist before creating hunspell instance<commit_after>package main

import (
	"os"
	"sync"

	"github.com/akhenakh/hunspellgo"
	"github.com/trustmaster/go-aspell"
)

type Speller interface {
	Check(w string) bool
	Suggest(w string) []string
	Close()
}

type aspeller struct {
	sp aspell.Speller
	mu sync.Mutex
}

func NewASpeller() (Speller, error) {
	opts := map[string]string{
		"lang":           "en",
		"filter":         "url",
		"mode":           "url",
		"encoding":       "ascii",
		"guess":          "true",
		"ignore":         "0",
		"ignore-case":    "false",
		"ignore-accents": "false",
	}

	sp, err := aspell.NewSpeller(opts)
	if err != nil {
		return nil, err
	}
	return &aspeller{sp: sp}, nil
}

func (s *aspeller) Check(w string) bool {
	s.mu.Lock()
	defer s.mu.Unlock()
	return s.sp.Check(w)
}

func (s *aspeller) Suggest(w string) []string {
	s.mu.Lock()
	defer s.mu.Unlock()
	return s.sp.Suggest(w)
}

func (s *aspeller) Close() { s.sp.Delete() }

type hunspeller struct {
	sp *hunspellgo.Hunhandle
}

// hunspellPaths has a list of paths where dictionaries might be
// in future, pass path through command line argument
var hunspellPaths = []string{
	"/usr/share/myspell",
	"/usr/share/hunspell",
}

func NewHunSpeller() Speller {
	for _, p := range hunspellPaths {
		aff, dic := p+"/en_US.aff", p+"/en_US.dic"
		if _, err := os.Stat(aff); err != nil {
			continue
		}
		if _, err := os.Stat(dic); err != nil {
			continue
		}
		if sp := hunspellgo.Hunspell(aff, dic); sp != nil {
			return &hunspeller{sp}
		}
	}
	return nil
}

func (s *hunspeller) Check(w string) bool {
	return s.sp.Spell(w)
}

func (s *hunspeller) Suggest(w string) []string {
	return s.sp.Suggest(w)
}

func (s *hunspeller) Close() { s.sp = nil }

type multispeller struct {
	sp []Speller
}

func NewMultiSpeller(sp ...Speller) Speller {
	m := &multispeller{}
	for _, s := range sp {
		m.sp = append(m.sp, s)
	}
	return m
}

func (s *multispeller) Check(w string) bool {
	for _, sp := range s.sp {
		if sp.Check(w) {
			return true
		}
	}
	return false
}

func (s *multispeller) Suggest(w string) (ret []string) {
	for _, sp := range s.sp {
		if ret = sp.Suggest(w); len(ret) != 0 {
			break
		}
	}
	return ret
}

func (s *multispeller) Close() {
	for _, sp := range s.sp {
		sp.Close()
	}
	s.sp = nil
}
<|endoftext|>


<commit_before>package termite

import (
	"fmt"
	"github.com/hanwen/go-fuse/fuse"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"
)

var _ = fmt.Println

type FsServer struct {
	contentServer *ContentServer
	contentCache  *ContentCache
	Root          string
	excluded      map[string]bool

	multiplyPaths func(string)[]string

	hashCacheMutex sync.RWMutex
	// TODO - should use string (immutable) throughout for storing MD5 signatures.
	hashCache map[string][]byte

	attrCacheMutex sync.RWMutex
	attrCache map[string]FileAttr
}

func NewFsServer(root string, cache *ContentCache, excluded []string) *FsServer {
	fs := &FsServer{
		contentCache:  cache,
		contentServer: &ContentServer{Cache: cache},
		Root:          root,
		hashCache:     make(map[string][]byte),
		attrCache:     make(map[string]FileAttr),
	}

	fs.excluded = make(map[string]bool)
	for _, e := range excluded {
		fs.excluded[e] = true
	}
	return fs
}

type AttrRequest struct {
	Name string
}

type FileAttr struct {
	Path string
	*os.FileInfo
	fuse.Status
	Hash    []byte
	Link    string
	Content []byte // optional.
}

type AttrResponse struct {
	Attrs []FileAttr
}

func (me FileAttr) String() string {
	id := ""
	if me.Hash != nil {
		id = fmt.Sprintf(" sz %d", me.FileInfo.Size)
	}
	if me.Link != "" {
		id = fmt.Sprintf(" -> %s", me.Link)
	}
	if me.Deletion() {
		id = " (del)"
	}
	return fmt.Sprintf("%s%s", me.Path, id)
}

func (me FileAttr) Deletion() bool {
	return me.Status == fuse.ENOENT
}

type DirRequest struct {
	Name string
}

type DirResponse struct {
	NameModeMap map[string]uint32
}

func (me *FsServer) path(n string) string {
	if me.Root == "" {
		return n
	}
	return filepath.Join(me.Root, strings.TrimLeft(n, "/"))
}

func (me *FsServer) FileContent(req *ContentRequest, rep *ContentResponse) os.Error {
	return me.contentServer.FileContent(req, rep)
}

func (me *FsServer) ReadDir(req *DirRequest, r *DirResponse) os.Error {
	d, e := ioutil.ReadDir(me.path(req.Name))
	log.Println("ReadDir", req)
	r.NameModeMap = make(map[string]uint32)
	for _, v := range d {
		r.NameModeMap[v.Name] = v.Mode
	}
	return e
}

func (me *FsServer) GetAttr(req *AttrRequest, rep *AttrResponse) os.Error {
	log.Println("GetAttr req", req.Name)
	names := []string{}
	if me.multiplyPaths != nil {
		names = me.multiplyPaths(req.Name)
	} else {
		names = append(names, req.Name)
	}
	for _, n := range names {
		a := FileAttr{}
		err := me.oneGetAttr(n, &a)
		if err != nil {
			return err
		}
		if a.Hash != nil {
			log.Printf("GetAttr %s %v %x", n, a, a.Hash)
		}
		rep.Attrs = append(rep.Attrs, a)
	}
	return nil
}

func (me *FsServer) oneGetAttr(name string, rep *FileAttr) os.Error {
	rep.Path = name
	// TODO - this is not a good security measure, as we are not
	// checking the prefix; someone might directly ask for
	// /forbidden/subdir/
	if me.excluded[name] {
		rep.Status = fuse.ENOENT
		return nil
	}

	me.attrCacheMutex.RLock()
	attr, ok := me.attrCache[name]
	me.attrCacheMutex.RUnlock()

	if ok {
		*rep = attr
		return nil
	}
	me.attrCacheMutex.Lock()
	defer me.attrCacheMutex.Unlock()
	attr, ok = me.attrCache[name]
	if ok {
		*rep = attr
		return nil
	}

	fi, err := os.Lstat(me.path(name))
	rep.FileInfo = fi
	rep.Status = fuse.OsErrorToErrno(err)
	rep.Path = name
	if fi != nil {
		me.fillContent(rep)
	}

	me.attrCache[name] = *rep
	return nil
}

func (me *FsServer) fillContent(rep *FileAttr) {
	if rep.FileInfo.IsSymlink() {
		rep.Link, _ = os.Readlink(rep.Path)
	}
	if rep.FileInfo.IsRegular() {
		// TODO - saving the content easily overflows memory
		// on 32-bit.
		rep.Hash, _ = me.getHash(rep.Path)
	}
}

func (me *FsServer) updateFiles(infos []FileAttr) {
	me.updateHashes(infos)
	me.updateAttrs(infos)
}

func (me *FsServer) updateAttrs(infos []FileAttr) {
	me.attrCacheMutex.Lock()
	defer me.attrCacheMutex.Unlock()

	for _, r := range infos {
		name := r.Path
		me.attrCache[name] = r
	}
}

func (me *FsServer) updateHashes(infos []FileAttr) {
	me.hashCacheMutex.Lock()
	defer me.hashCacheMutex.Unlock()

	for _, r := range infos {
		name := r.Path
		if !r.Status.Ok() || r.Link != "" {
			me.hashCache[name] = nil, false
		}
		if r.Hash != nil {
			me.hashCache[name] = r.Hash
		}
	}
}

func (me *FsServer) getHash(name string) (hash []byte, content []byte) {
	fullPath := me.path(name)

	me.hashCacheMutex.RLock()
	hash = me.hashCache[name]
	me.hashCacheMutex.RUnlock()

	if hash != nil {
		return []byte(hash), nil
	}

	me.hashCacheMutex.Lock()
	defer me.hashCacheMutex.Unlock()
	hash = me.hashCache[name]
	if hash != nil {
		return []byte(hash), nil
	}

	// TODO - would it be better to not stop other hash lookups
	// from succeeding?

	// TODO - /usr should be configurable.
	if strings.HasPrefix(fullPath, "/usr") {
		hash, content = me.contentCache.SaveImmutablePath(fullPath)
	} else {
		hash, content = me.contentCache.SavePath(fullPath)
	}

	me.hashCache[name] = hash
	return hash, content
}

// TODO - decide between []FileAttr and []*FileAttr.
func (me *FsServer) refreshAttributeCache(prefix string) []FileAttr {
	me.attrCacheMutex.Lock()
	defer me.attrCacheMutex.Unlock()

	updated := []FileAttr{}
	for key, attr := range me.attrCache {
		// TODO -should just do everything?
		if !HasDirPrefix(key, prefix) {
			continue;
		}

		fi, _ := os.Lstat(me.path(key))
		if fi == nil && attr.Status.Ok() {
			del := FileAttr{
				Path: key,
				Status: fuse.ENOENT,
			}
			updated = append(updated, del)
		}
		if fi != nil && attr.FileInfo != nil && EncodeFileInfo(*attr.FileInfo) != EncodeFileInfo(*fi) {
			newEnt := FileAttr{
			Path: key,
			Status: fuse.OK,
			FileInfo: fi,
			}
			me.fillContent(&newEnt)
			updated = append(updated, newEnt)
		}
	}

	for _, u := range updated {
		me.attrCache[u.Path] = u
	}
	return updated
}

func (me *FsServer) copyCache() []FileAttr {
	me.attrCacheMutex.RLock()
	defer me.attrCacheMutex.RUnlock()

	dump := []FileAttr{}
	for _, attr := range me.attrCache {
		dump = append(dump, attr)
	}

	return dump
}
<commit_msg>Don't use SaveImmutablePath() on /usr/local.<commit_after>package termite

import (
	"fmt"
	"github.com/hanwen/go-fuse/fuse"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"
)

var _ = fmt.Println

type FsServer struct {
	contentServer *ContentServer
	contentCache  *ContentCache
	Root          string
	excluded      map[string]bool

	multiplyPaths func(string)[]string

	hashCacheMutex sync.RWMutex
	// TODO - should use string (immutable) throughout for storing MD5 signatures.
	hashCache map[string][]byte

	attrCacheMutex sync.RWMutex
	attrCache map[string]FileAttr
}

func NewFsServer(root string, cache *ContentCache, excluded []string) *FsServer {
	fs := &FsServer{
		contentCache:  cache,
		contentServer: &ContentServer{Cache: cache},
		Root:          root,
		hashCache:     make(map[string][]byte),
		attrCache:     make(map[string]FileAttr),
	}

	fs.excluded = make(map[string]bool)
	for _, e := range excluded {
		fs.excluded[e] = true
	}
	return fs
}

type AttrRequest struct {
	Name string
}

type FileAttr struct {
	Path string
	*os.FileInfo
	fuse.Status
	Hash    []byte
	Link    string
	Content []byte // optional.
}

type AttrResponse struct {
	Attrs []FileAttr
}

func (me FileAttr) String() string {
	id := ""
	if me.Hash != nil {
		id = fmt.Sprintf(" sz %d", me.FileInfo.Size)
	}
	if me.Link != "" {
		id = fmt.Sprintf(" -> %s", me.Link)
	}
	if me.Deletion() {
		id = " (del)"
	}
	return fmt.Sprintf("%s%s", me.Path, id)
}

func (me FileAttr) Deletion() bool {
	return me.Status == fuse.ENOENT
}

type DirRequest struct {
	Name string
}

type DirResponse struct {
	NameModeMap map[string]uint32
}

func (me *FsServer) path(n string) string {
	if me.Root == "" {
		return n
	}
	return filepath.Join(me.Root, strings.TrimLeft(n, "/"))
}

func (me *FsServer) FileContent(req *ContentRequest, rep *ContentResponse) os.Error {
	return me.contentServer.FileContent(req, rep)
}

func (me *FsServer) ReadDir(req *DirRequest, r *DirResponse) os.Error {
	d, e := ioutil.ReadDir(me.path(req.Name))
	log.Println("ReadDir", req)
	r.NameModeMap = make(map[string]uint32)
	for _, v := range d {
		r.NameModeMap[v.Name] = v.Mode
	}
	return e
}

func (me *FsServer) GetAttr(req *AttrRequest, rep *AttrResponse) os.Error {
	log.Println("GetAttr req", req.Name)
	names := []string{}
	if me.multiplyPaths != nil {
		names = me.multiplyPaths(req.Name)
	} else {
		names = append(names, req.Name)
	}
	for _, n := range names {
		a := FileAttr{}
		err := me.oneGetAttr(n, &a)
		if err != nil {
			return err
		}
		if a.Hash != nil {
			log.Printf("GetAttr %s %v %x", n, a, a.Hash)
		}
		rep.Attrs = append(rep.Attrs, a)
	}
	return nil
}

func (me *FsServer) oneGetAttr(name string, rep *FileAttr) os.Error {
	rep.Path = name
	// TODO - this is not a good security measure, as we are not
	// checking the prefix; someone might directly ask for
	// /forbidden/subdir/
	if me.excluded[name] {
		rep.Status = fuse.ENOENT
		return nil
	}

	me.attrCacheMutex.RLock()
	attr, ok := me.attrCache[name]
	me.attrCacheMutex.RUnlock()

	if ok {
		*rep = attr
		return nil
	}
	me.attrCacheMutex.Lock()
	defer me.attrCacheMutex.Unlock()
	attr, ok = me.attrCache[name]
	if ok {
		*rep = attr
		return nil
	}

	fi, err := os.Lstat(me.path(name))
	rep.FileInfo = fi
	rep.Status = fuse.OsErrorToErrno(err)
	rep.Path = name
	if fi != nil {
		me.fillContent(rep)
	}

	me.attrCache[name] = *rep
	return nil
}

func (me *FsServer) fillContent(rep *FileAttr) {
	if rep.FileInfo.IsSymlink() {
		rep.Link, _ = os.Readlink(rep.Path)
	}
	if rep.FileInfo.IsRegular() {
		// TODO - saving the content easily overflows memory
		// on 32-bit.
		rep.Hash, _ = me.getHash(rep.Path)
	}
}

func (me *FsServer) updateFiles(infos []FileAttr) {
	me.updateHashes(infos)
	me.updateAttrs(infos)
}

func (me *FsServer) updateAttrs(infos []FileAttr) {
	me.attrCacheMutex.Lock()
	defer me.attrCacheMutex.Unlock()

	for _, r := range infos {
		name := r.Path
		me.attrCache[name] = r
	}
}

func (me *FsServer) updateHashes(infos []FileAttr) {
	me.hashCacheMutex.Lock()
	defer me.hashCacheMutex.Unlock()

	for _, r := range infos {
		name := r.Path
		if !r.Status.Ok() || r.Link != "" {
			me.hashCache[name] = nil, false
		}
		if r.Hash != nil {
			me.hashCache[name] = r.Hash
		}
	}
}

func (me *FsServer) getHash(name string) (hash []byte, content []byte) {
	fullPath := me.path(name)

	me.hashCacheMutex.RLock()
	hash = me.hashCache[name]
	me.hashCacheMutex.RUnlock()

	if hash != nil {
		return []byte(hash), nil
	}

	me.hashCacheMutex.Lock()
	defer me.hashCacheMutex.Unlock()
	hash = me.hashCache[name]
	if hash != nil {
		return []byte(hash), nil
	}

	// TODO - would it be better to not stop other hash lookups
	// from succeeding?

	// TODO - /usr should be configurable.
	if HasDirPrefix(fullPath, "/usr") && !HasDirPrefix(fullPath, "/usr/local") {
		hash, content = me.contentCache.SaveImmutablePath(fullPath)
	} else {
		hash, content = me.contentCache.SavePath(fullPath)
	}

	me.hashCache[name] = hash
	return hash, content
}

// TODO - decide between []FileAttr and []*FileAttr.
func (me *FsServer) refreshAttributeCache(prefix string) []FileAttr {
	me.attrCacheMutex.Lock()
	defer me.attrCacheMutex.Unlock()

	updated := []FileAttr{}
	for key, attr := range me.attrCache {
		// TODO -should just do everything?
		if !HasDirPrefix(key, prefix) {
			continue;
		}

		fi, _ := os.Lstat(me.path(key))
		if fi == nil && attr.Status.Ok() {
			del := FileAttr{
				Path: key,
				Status: fuse.ENOENT,
			}
			updated = append(updated, del)
		}
		if fi != nil && attr.FileInfo != nil && EncodeFileInfo(*attr.FileInfo) != EncodeFileInfo(*fi) {
			newEnt := FileAttr{
			Path: key,
			Status: fuse.OK,
			FileInfo: fi,
			}
			me.fillContent(&newEnt)
			updated = append(updated, newEnt)
		}
	}

	for _, u := range updated {
		me.attrCache[u.Path] = u
	}
	return updated
}

func (me *FsServer) copyCache() []FileAttr {
	me.attrCacheMutex.RLock()
	defer me.attrCacheMutex.RUnlock()

	dump := []FileAttr{}
	for _, attr := range me.attrCache {
		dump = append(dump, attr)
	}

	return dump
}
<|endoftext|>


<commit_before>package terse

import (
	"strings"
	"testing"

	"github.com/acsellers/multitemplate"
	"github.com/acsellers/multitemplate/helpers"
)

func TestErrors(t *testing.T) {
	helpers.LoadHelpers("all")
	for _, test := range errorTests {
		tmpl := multitemplate.New("terse")
		var e error
		tmpl, e = tmpl.Parse("parse", test.Source, "terse")
		if e == nil {
			t.Error("No Error:", test.Name)
			continue
		} else if test.Contains != "" {
			if !strings.Contains(e.Error(), test.Contains) {
				t.Error("Incorrect error for", test.Name, "Expected:", test.Contains, "Was:", e)
			}
		}
	}
}

var errorTests = []errorTest{
	errorTest{
		Name:     "Unclosed attribute parentheses",
		Source:   "html name=(link_to",
		Contains: "Unclosed parentheses",
	},
	errorTest{
		Name:     "Empty attribute parentheses",
		Source:   "html name=()",
		Contains: "Empty parentheses",
	},
	errorTest{
		Name:     "Unclosed attribute single quotes ",
		Source:   "html name='link_to",
		Contains: "Unclosed single quotes",
	},
	errorTest{
		Name:     "Empty attribute single quotes",
		Source:   "html name=''",
		Contains: "Empty single quotes",
	},
	errorTest{
		Name:     "Unclosed attribute double quotes ",
		Source:   "html name=\"link_to",
		Contains: "Unclosed double quotes",
	},
	errorTest{
		Name:     "Empty attribute double quotes",
		Source:   "html name=\"\"",
		Contains: "Empty double quotes",
	},
	errorTest{
		Name:     "Blank attribute variable",
		Source:   "html name=",
		Contains: "Blank attribute",
	},
	errorTest{
		Name:     "Headless Totem Pole",
		Source:   "html > head >",
		Contains: "Missing tag",
	},
	errorTest{
		Name:   "Malformed code test",
		Source: "= link_to (",
	},
	errorTest{
		Name:     "Multiple errors test",
		Source:   "head >\n= link_to (",
		Contains: "Missing tag",
	},
	errorTest{
		Name:   "Bad function name for if",
		Source: "?asdfasdfsafd\n  no",
	},
	errorTest{
		Name:   "Bad function name for if",
		Source: "&asdfasdfsafd\n  no",
	},
}

type errorTest struct {
	Name     string
	Source   string
	Contains string
}
<commit_msg>More error tests<commit_after>package terse

import (
	"strings"
	"testing"

	"github.com/acsellers/multitemplate"
	"github.com/acsellers/multitemplate/helpers"
)

func TestErrors(t *testing.T) {
	helpers.LoadHelpers("all")
	for _, test := range errorTests {
		tmpl := multitemplate.New("terse")
		var e error
		tmpl, e = tmpl.Parse("parse", test.Source, "terse")
		if e == nil {
			t.Error("No Error:", test.Name)
			continue
		} else if test.Contains != "" {
			if !strings.Contains(e.Error(), test.Contains) {
				t.Error("Incorrect error for", test.Name, "Expected:", test.Contains, "Was:", e)
			}
		}
	}
}

var errorTests = []errorTest{
	errorTest{
		Name:     "Unclosed attribute parentheses",
		Source:   "html name=(link_to",
		Contains: "Unclosed parentheses",
	},
	errorTest{
		Name:     "Empty attribute parentheses",
		Source:   "html name=()",
		Contains: "Empty parentheses",
	},
	errorTest{
		Name:     "Unclosed attribute single quotes ",
		Source:   "html name='link_to",
		Contains: "Unclosed single quotes",
	},
	errorTest{
		Name:     "Empty attribute single quotes",
		Source:   "html name=''",
		Contains: "Empty single quotes",
	},
	errorTest{
		Name:     "Unclosed attribute double quotes ",
		Source:   "html name=\"link_to",
		Contains: "Unclosed double quotes",
	},
	errorTest{
		Name:     "Empty attribute double quotes",
		Source:   "html name=\"\"",
		Contains: "Empty double quotes",
	},
	errorTest{
		Name:     "Blank attribute variable",
		Source:   "html name=",
		Contains: "Blank attribute",
	},
	errorTest{
		Name:     "Headless Totem Pole",
		Source:   "html > head >",
		Contains: "Missing tag",
	},
	errorTest{
		Name:   "Malformed code test",
		Source: "= link_to (",
	},
	errorTest{
		Name:     "Multiple errors test",
		Source:   "head >\n= link_to (",
		Contains: "Missing tag",
	},
	errorTest{
		Name:   "Bad function name for if",
		Source: "?asdfasdfsafd\n  no",
	},
	errorTest{
		Name:   "Bad function name for if",
		Source: "&asdfasdfsafd\n  no",
	},
	errorTest{
		Name:   "Bad interpolated code 1",
		Source: "asdf {{ asdfasdf }}",
	},
	errorTest{
		Name:   "Bad interpolated code 2",
		Source: "asdf\n  {{ asdfasdf }}",
	},
	errorTest{
		Name:   "Bad interpolated code 3",
		Source: "asdf\n  {{ {}{}} }}",
	},
	errorTest{
		Name:   "Bad interpolated code 3",
		Source: "asdf}} {{  }}",
	},
}

type errorTest struct {
	Name     string
	Source   string
	Contains string
}
<|endoftext|>


<commit_before>package service

import (
	"fmt"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/filters"
	swarmtypes "github.com/docker/docker/api/types/swarm"
	"github.com/docker/docker/client"
	"golang.org/x/net/context"
)

// parseSecrets retrieves the secrets from the requested names and converts
// them to secret references to use with the spec
func parseSecrets(client client.APIClient, requestedSecrets []*types.SecretRequestOptions) ([]*swarmtypes.SecretReference, error) {
	secretRefs := make(map[string]*swarmtypes.SecretReference)
	ctx := context.Background()

	for _, secret := range requestedSecrets {
		secretRef := &swarmtypes.SecretReference{
			SecretName: secret.Source,
			Target: &swarmtypes.SecretReferenceFileTarget{
				Name: secret.Target,
				UID:  secret.UID,
				GID:  secret.GID,
				Mode: secret.Mode,
			},
		}

		if _, exists := secretRefs[secret.Target]; exists {
			return nil, fmt.Errorf("duplicate secret target for %s not allowed", secret.Source)
		}
		secretRefs[secret.Target] = secretRef
	}

	args := filters.NewArgs()
	for _, s := range secretRefs {
		args.Add("names", s.SecretName)
	}

	secrets, err := client.SecretList(ctx, types.SecretListOptions{
		Filters: args,
	})
	if err != nil {
		return nil, err
	}

	foundSecrets := make(map[string]string)
	for _, secret := range secrets {
		foundSecrets[secret.Spec.Annotations.Name] = secret.ID
	}

	addedSecrets := []*swarmtypes.SecretReference{}

	for _, ref := range secretRefs {
		id, ok := foundSecrets[ref.SecretName]
		if !ok {
			return nil, fmt.Errorf("secret not found: %s", ref.SecretName)
		}

		// set the id for the ref to properly assign in swarm
		// since swarm needs the ID instead of the name
		ref.SecretID = id
		addedSecrets = append(addedSecrets, ref)
	}

	return addedSecrets, nil
}
<commit_msg>SecretRequestOptions -> SecretRequestOption<commit_after>package service

import (
	"fmt"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/filters"
	swarmtypes "github.com/docker/docker/api/types/swarm"
	"github.com/docker/docker/client"
	"golang.org/x/net/context"
)

// parseSecrets retrieves the secrets from the requested names and converts
// them to secret references to use with the spec
func parseSecrets(client client.APIClient, requestedSecrets []*types.SecretRequestOption) ([]*swarmtypes.SecretReference, error) {
	secretRefs := make(map[string]*swarmtypes.SecretReference)
	ctx := context.Background()

	for _, secret := range requestedSecrets {
		secretRef := &swarmtypes.SecretReference{
			SecretName: secret.Source,
			Target: &swarmtypes.SecretReferenceFileTarget{
				Name: secret.Target,
				UID:  secret.UID,
				GID:  secret.GID,
				Mode: secret.Mode,
			},
		}

		if _, exists := secretRefs[secret.Target]; exists {
			return nil, fmt.Errorf("duplicate secret target for %s not allowed", secret.Source)
		}
		secretRefs[secret.Target] = secretRef
	}

	args := filters.NewArgs()
	for _, s := range secretRefs {
		args.Add("names", s.SecretName)
	}

	secrets, err := client.SecretList(ctx, types.SecretListOptions{
		Filters: args,
	})
	if err != nil {
		return nil, err
	}

	foundSecrets := make(map[string]string)
	for _, secret := range secrets {
		foundSecrets[secret.Spec.Annotations.Name] = secret.ID
	}

	addedSecrets := []*swarmtypes.SecretReference{}

	for _, ref := range secretRefs {
		id, ok := foundSecrets[ref.SecretName]
		if !ok {
			return nil, fmt.Errorf("secret not found: %s", ref.SecretName)
		}

		// set the id for the ref to properly assign in swarm
		// since swarm needs the ID instead of the name
		ref.SecretID = id
		addedSecrets = append(addedSecrets, ref)
	}

	return addedSecrets, nil
}
<|endoftext|>


<commit_before>package commands

import (
	"github.com/stretchr/testify/assert"
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"
	"time"
	"github.com/EMSSConsulting/Thargo"
)

const UntrackedFileName = "some_fancy_untracked_file"

var currentDir, _ = os.Getwd()

func randomTempFile(t *testing.T) string {
	file, err := ioutil.TempFile("", "archive_")
	assert.NoError(t, err)
	file.Close()
	os.Remove(file.Name())

	return file.Name()
}

func createArchiveCommand(t *testing.T) *ArchiveCommand {
	err := os.Chdir(filepath.Join(currentDir, ".."))
	assert.NoError(t, err)

	return &ArchiveCommand{
		Output: randomTempFile(t),
		Silent: true,
	}
}

func filesInFolder(path string) []string {
	matches, _ := filepath.Glob(path)
	return matches
}

func readArchiveContent(t *testing.T, c *ArchiveCommand) (resultMap map[string]bool) {
	resultMap = make(map[string]bool)

	archive, err := thargo.NewArchiveFile(c.Output, nil)
	assert.NoError(t, err)

	assert.NoError(t, archive.Extract(func(entry thargo.SaveableEntry) error {
		header, err := entry.Header()
		assert.NoError(t, err)

		resultMap[header.Name] = true

		return nil
	}))

	return resultMap
}

func verifyArchiveContent(t *testing.T, c *ArchiveCommand, files ...string) {
	resultMap := readArchiveContent(t, c)
	for _, file := range files {
		assert.True(t, resultMap[file], "File should exist %q", file)
		delete(resultMap, file)
	}
	assert.Len(t, resultMap, 0, "No extra file should exist")
}

func TestArchiveNotCreatingArchive(t *testing.T) {
	cmd := createArchiveCommand(t)
	defer os.Remove(cmd.Output)
	cmd.Execute(nil)
	_, err := os.Stat(cmd.Output)
	assert.True(t, os.IsNotExist(err), "File should not exist", cmd.Output, err)
}

func TestArchiveAddingSomeLocalFiles(t *testing.T) {
	cmd := createArchiveCommand(t)
	defer os.Remove(cmd.Output)
	cmd.Paths = []string{
		"commands/*",
	}
	cmd.Execute(nil)
	verifyArchiveContent(t, cmd, filesInFolder("commands/*")...)
}

func TestArchiveNotAddingDuplicateFiles(t *testing.T) {
	cmd := createArchiveCommand(t)
	defer os.Remove(cmd.Output)
	cmd.Paths = []string{
		"commands/*",
		"commands/archive.go",
	}
	cmd.Execute(nil)
	verifyArchiveContent(t, cmd, filesInFolder("commands/*")...)
}

func TestArchiveAddingUntrackedFiles(t *testing.T) {
	cmd := createArchiveCommand(t)
	defer os.Remove(cmd.Output)
	err := ioutil.WriteFile(UntrackedFileName, []byte{}, 0700)
	assert.NoError(t, err)
	cmd.Untracked = true
	cmd.Execute(nil)
	files := readArchiveContent(t, cmd)
	assert.NotEmpty(t, files)
	assert.True(t, files[UntrackedFileName])
}

func TestArchiveUpdating(t *testing.T) {
	tempFile := randomTempFile(t)
	defer os.Remove(tempFile)

	cmd := createArchiveCommand(t)
	defer os.Remove(cmd.Output)
	cmd.Paths = []string{
		"commands/*",
		tempFile,
	}

	cmd.Execute(nil)
	archive1, err := os.Stat(cmd.Output)
	assert.NoError(t, err, "Archive is created")

	cmd.Execute(nil)
	archive2, err := os.Stat(cmd.Output)
	assert.NoError(t, err, "Archive is created")
	assert.Equal(t, archive1.ModTime(), archive2.ModTime(), "Archive should not be modified")

	time.Sleep(time.Second)
	err = ioutil.WriteFile(tempFile, []byte{}, 0700)
	assert.NoError(t, err, "File is created")

	cmd.Execute(nil)
	archive3, err := os.Stat(cmd.Output)
	assert.NoError(t, err, "Archive is created")
	assert.NotEqual(t, archive2.ModTime(), archive3.ModTime(), "File is added to archive")

	time.Sleep(time.Second)
	err = ioutil.WriteFile(tempFile, []byte{}, 0700)
	assert.NoError(t, err, "File is updated")

	cmd.Execute(nil)
	archive4, err := os.Stat(cmd.Output)
	assert.NoError(t, err, "Archive is created")
	assert.NotEqual(t, archive3.ModTime(), archive4.ModTime(), "File is updated in archive")
}
<commit_msg>test: Ensure that archive is properly closed<commit_after>package commands

import (
	"github.com/stretchr/testify/assert"
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"
	"time"
	"github.com/EMSSConsulting/Thargo"
)

const UntrackedFileName = "some_fancy_untracked_file"

var currentDir, _ = os.Getwd()

func randomTempFile(t *testing.T) string {
	file, err := ioutil.TempFile("", "archive_")
	assert.NoError(t, err)
	file.Close()
	os.Remove(file.Name())

	return file.Name()
}

func createArchiveCommand(t *testing.T) *ArchiveCommand {
	err := os.Chdir(filepath.Join(currentDir, ".."))
	assert.NoError(t, err)

	return &ArchiveCommand{
		Output: randomTempFile(t),
		Silent: true,
	}
}

func filesInFolder(path string) []string {
	matches, _ := filepath.Glob(path)
	return matches
}

func readArchiveContent(t *testing.T, c *ArchiveCommand) (resultMap map[string]bool) {
	resultMap = make(map[string]bool)

	archive, err := thargo.NewArchiveFile(c.Output, nil)
	assert.NoError(t, err)
	defer archive.Close()

	assert.NoError(t, archive.Extract(func(entry thargo.SaveableEntry) error {
		header, err := entry.Header()
		assert.NoError(t, err)

		resultMap[header.Name] = true

		return nil
	}))

	return resultMap
}

func verifyArchiveContent(t *testing.T, c *ArchiveCommand, files ...string) {
	resultMap := readArchiveContent(t, c)
	for _, file := range files {
		assert.True(t, resultMap[file], "File should exist %q", file)
		delete(resultMap, file)
	}
	assert.Len(t, resultMap, 0, "No extra file should exist")
}

func TestArchiveNotCreatingArchive(t *testing.T) {
	cmd := createArchiveCommand(t)
	defer os.Remove(cmd.Output)
	cmd.Execute(nil)
	_, err := os.Stat(cmd.Output)
	assert.True(t, os.IsNotExist(err), "File should not exist", cmd.Output, err)
}

func TestArchiveAddingSomeLocalFiles(t *testing.T) {
	cmd := createArchiveCommand(t)
	defer os.Remove(cmd.Output)
	cmd.Paths = []string{
		"commands/*",
	}
	cmd.Execute(nil)
	verifyArchiveContent(t, cmd, filesInFolder("commands/*")...)
}

func TestArchiveNotAddingDuplicateFiles(t *testing.T) {
	cmd := createArchiveCommand(t)
	defer os.Remove(cmd.Output)
	cmd.Paths = []string{
		"commands/*",
		"commands/archive.go",
	}
	cmd.Execute(nil)
	verifyArchiveContent(t, cmd, filesInFolder("commands/*")...)
}

func TestArchiveAddingUntrackedFiles(t *testing.T) {
	cmd := createArchiveCommand(t)
	defer os.Remove(cmd.Output)
	err := ioutil.WriteFile(UntrackedFileName, []byte{}, 0700)
	assert.NoError(t, err)
	cmd.Untracked = true
	cmd.Execute(nil)
	files := readArchiveContent(t, cmd)
	assert.NotEmpty(t, files)
	assert.True(t, files[UntrackedFileName])
}

func TestArchiveUpdating(t *testing.T) {
	tempFile := randomTempFile(t)
	defer os.Remove(tempFile)

	cmd := createArchiveCommand(t)
	defer os.Remove(cmd.Output)
	cmd.Paths = []string{
		"commands/*",
		tempFile,
	}

	cmd.Execute(nil)
	archive1, err := os.Stat(cmd.Output)
	assert.NoError(t, err, "Archive is created")

	cmd.Execute(nil)
	archive2, err := os.Stat(cmd.Output)
	assert.NoError(t, err, "Archive is created")
	assert.Equal(t, archive1.ModTime(), archive2.ModTime(), "Archive should not be modified")

	time.Sleep(time.Second)
	err = ioutil.WriteFile(tempFile, []byte{}, 0700)
	assert.NoError(t, err, "File is created")

	cmd.Execute(nil)
	archive3, err := os.Stat(cmd.Output)
	assert.NoError(t, err, "Archive is created")
	assert.NotEqual(t, archive2.ModTime(), archive3.ModTime(), "File is added to archive")

	time.Sleep(time.Second)
	err = ioutil.WriteFile(tempFile, []byte{}, 0700)
	assert.NoError(t, err, "File is updated")

	cmd.Execute(nil)
	archive4, err := os.Stat(cmd.Output)
	assert.NoError(t, err, "Archive is created")
	assert.NotEqual(t, archive3.ModTime(), archive4.ModTime(), "File is updated in archive")
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"
	"sync"
	"time"

	"github.com/git-lfs/git-lfs/filepathfilter"
	"github.com/git-lfs/git-lfs/git"
	"github.com/git-lfs/git-lfs/lfs"
	"github.com/git-lfs/git-lfs/progress"
	"github.com/rubyist/tracerx"
	"github.com/spf13/cobra"
)

func pullCommand(cmd *cobra.Command, args []string) {
	requireGitVersion()
	requireInRepo()

	if len(args) > 0 {
		// Remote is first arg
		if err := git.ValidateRemote(args[0]); err != nil {
			Panic(err, fmt.Sprintf("Invalid remote name '%v'", args[0]))
		}
		cfg.CurrentRemote = args[0]
	} else {
		// Actively find the default remote, don't just assume origin
		defaultRemote, err := git.DefaultRemote()
		if err != nil {
			Panic(err, "No default remote")
		}
		cfg.CurrentRemote = defaultRemote
	}

	includeArg, excludeArg := getIncludeExcludeArgs(cmd)
	filter := buildFilepathFilter(cfg, includeArg, excludeArg)
	pull(filter)
}

func pull(filter *filepathfilter.Filter) {
	ref, err := git.CurrentRef()
	if err != nil {
		Panic(err, "Could not pull")
	}

	pointers := newPointerMap()
	meter := progress.NewMeter(progress.WithOSEnv(cfg.Os))
	singleCheckout := newSingleCheckout()
	q := lfs.NewDownloadQueue(lfs.WithProgress(meter))
	gitscanner := lfs.NewGitScanner(func(p *lfs.WrappedPointer, err error) {
		if err != nil {
			LoggedError(err, "Scanner error")
			return
		}

		if pointers.Seen(p) {
			return
		}

		// no need to download objects that exist locally already
		lfs.LinkOrCopyFromReference(p.Oid, p.Size)
		if lfs.ObjectExistsOfSize(p.Oid, p.Size) {
			singleCheckout.Run(p)
			return
		}

		meter.Add(p.Size)
		meter.StartTransfer(p.Name)
		tracerx.Printf("fetch %v [%v]", p.Name, p.Oid)
		pointers.Add(p)
		q.Add(lfs.NewDownloadable(p))
	})

	gitscanner.Filter = filter

	dlwatch := q.Watch()
	var wg sync.WaitGroup
	wg.Add(1)

	go func() {
		for oid := range dlwatch {
			for _, p := range pointers.All(oid) {
				singleCheckout.Run(p)
			}
		}
		wg.Done()
	}()

	processQueue := time.Now()
	if err := gitscanner.ScanTree(ref.Sha); err != nil {
		ExitWithError(err)
	}

	meter.Start()
	gitscanner.Close()
	q.Wait()
	wg.Wait()
	tracerx.PerformanceSince("process queue", processQueue)

	singleCheckout.Close()

	for _, err := range q.Errors() {
		FullError(err)
	}
}

// tracks LFS objects being downloaded, according to their unique OIDs.
type pointerMap struct {
	pointers map[string][]*lfs.WrappedPointer
	mu       sync.Mutex
}

func newPointerMap() *pointerMap {
	return &pointerMap{pointers: make(map[string][]*lfs.WrappedPointer)}
}

func (m *pointerMap) Seen(p *lfs.WrappedPointer) bool {
	m.mu.Lock()
	existing, ok := m.pointers[p.Oid]
	if ok {
		m.pointers[p.Oid] = append(existing, p)
	}
	m.mu.Unlock()
	return ok
}

func (m *pointerMap) Add(p *lfs.WrappedPointer) {
	m.mu.Lock()
	defer m.mu.Unlock()
	m.pointers[p.Oid] = append(m.pointers[p.Oid], p)
}

func (m *pointerMap) All(oid string) []*lfs.WrappedPointer {
	m.mu.Lock()
	pointers := m.pointers[oid]
	delete(m.pointers, oid)
	m.mu.Unlock()

	return pointers
}

func init() {
	RegisterCommand("pull", pullCommand, func(cmd *cobra.Command) {
		cmd.Flags().StringVarP(&includeArg, "include", "I", "", "Include a list of paths")
		cmd.Flags().StringVarP(&excludeArg, "exclude", "X", "", "Exclude a list of paths")
	})
}
<commit_msg>defer those unlocks<commit_after>package commands

import (
	"fmt"
	"sync"
	"time"

	"github.com/git-lfs/git-lfs/filepathfilter"
	"github.com/git-lfs/git-lfs/git"
	"github.com/git-lfs/git-lfs/lfs"
	"github.com/git-lfs/git-lfs/progress"
	"github.com/rubyist/tracerx"
	"github.com/spf13/cobra"
)

func pullCommand(cmd *cobra.Command, args []string) {
	requireGitVersion()
	requireInRepo()

	if len(args) > 0 {
		// Remote is first arg
		if err := git.ValidateRemote(args[0]); err != nil {
			Panic(err, fmt.Sprintf("Invalid remote name '%v'", args[0]))
		}
		cfg.CurrentRemote = args[0]
	} else {
		// Actively find the default remote, don't just assume origin
		defaultRemote, err := git.DefaultRemote()
		if err != nil {
			Panic(err, "No default remote")
		}
		cfg.CurrentRemote = defaultRemote
	}

	includeArg, excludeArg := getIncludeExcludeArgs(cmd)
	filter := buildFilepathFilter(cfg, includeArg, excludeArg)
	pull(filter)
}

func pull(filter *filepathfilter.Filter) {
	ref, err := git.CurrentRef()
	if err != nil {
		Panic(err, "Could not pull")
	}

	pointers := newPointerMap()
	meter := progress.NewMeter(progress.WithOSEnv(cfg.Os))
	singleCheckout := newSingleCheckout()
	q := lfs.NewDownloadQueue(lfs.WithProgress(meter))
	gitscanner := lfs.NewGitScanner(func(p *lfs.WrappedPointer, err error) {
		if err != nil {
			LoggedError(err, "Scanner error")
			return
		}

		if pointers.Seen(p) {
			return
		}

		// no need to download objects that exist locally already
		lfs.LinkOrCopyFromReference(p.Oid, p.Size)
		if lfs.ObjectExistsOfSize(p.Oid, p.Size) {
			singleCheckout.Run(p)
			return
		}

		meter.Add(p.Size)
		meter.StartTransfer(p.Name)
		tracerx.Printf("fetch %v [%v]", p.Name, p.Oid)
		pointers.Add(p)
		q.Add(lfs.NewDownloadable(p))
	})

	gitscanner.Filter = filter

	dlwatch := q.Watch()
	var wg sync.WaitGroup
	wg.Add(1)

	go func() {
		for oid := range dlwatch {
			for _, p := range pointers.All(oid) {
				singleCheckout.Run(p)
			}
		}
		wg.Done()
	}()

	processQueue := time.Now()
	if err := gitscanner.ScanTree(ref.Sha); err != nil {
		ExitWithError(err)
	}

	meter.Start()
	gitscanner.Close()
	q.Wait()
	wg.Wait()
	tracerx.PerformanceSince("process queue", processQueue)

	singleCheckout.Close()

	for _, err := range q.Errors() {
		FullError(err)
	}
}

// tracks LFS objects being downloaded, according to their unique OIDs.
type pointerMap struct {
	pointers map[string][]*lfs.WrappedPointer
	mu       sync.Mutex
}

func newPointerMap() *pointerMap {
	return &pointerMap{pointers: make(map[string][]*lfs.WrappedPointer)}
}

func (m *pointerMap) Seen(p *lfs.WrappedPointer) bool {
	m.mu.Lock()
	defer m.mu.Unlock()
	if existing, ok := m.pointers[p.Oid]; ok {
		m.pointers[p.Oid] = append(existing, p)
		return true
	}
	return false
}

func (m *pointerMap) Add(p *lfs.WrappedPointer) {
	m.mu.Lock()
	defer m.mu.Unlock()
	m.pointers[p.Oid] = append(m.pointers[p.Oid], p)
}

func (m *pointerMap) All(oid string) []*lfs.WrappedPointer {
	m.mu.Lock()
	defer m.mu.Unlock()
	pointers := m.pointers[oid]
	delete(m.pointers, oid)
	return pointers
}

func init() {
	RegisterCommand("pull", pullCommand, func(cmd *cobra.Command) {
		cmd.Flags().StringVarP(&includeArg, "include", "I", "", "Include a list of paths")
		cmd.Flags().StringVarP(&excludeArg, "exclude", "X", "", "Exclude a list of paths")
	})
}
<|endoftext|>


<commit_before>package http

import (
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"runtime/debug"
	"strconv"
	"strings"
	"sync"

	cors "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/rs/cors"
	"github.com/ipfs/go-ipfs/repo/config"
	context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"

	cmds "github.com/ipfs/go-ipfs/commands"
	logging "gx/ipfs/Qmazh5oNUVsDZTs2g59rq8aYQqwpss8tcUWQzor5sCCEuH/go-log"
)

var log = logging.Logger("commands/http")

// the internal handler for the API
type internalHandler struct {
	ctx  cmds.Context
	root *cmds.Command
	cfg  *ServerConfig
}

// The Handler struct is funny because we want to wrap our internal handler
// with CORS while keeping our fields.
type Handler struct {
	internalHandler
	corsHandler http.Handler
}

var (
	ErrNotFound           = errors.New("404 page not found")
	errApiVersionMismatch = errors.New("api version mismatch")
)

const (
	StreamErrHeader        = "X-Stream-Error"
	streamHeader           = "X-Stream-Output"
	channelHeader          = "X-Chunked-Output"
	uaHeader               = "User-Agent"
	contentTypeHeader      = "Content-Type"
	contentLengthHeader    = "Content-Length"
	contentDispHeader      = "Content-Disposition"
	transferEncodingHeader = "Transfer-Encoding"
	applicationJson        = "application/json"
	applicationOctetStream = "application/octet-stream"
	plainText              = "text/plain"
	originHeader           = "origin"
)

const (
	ACAOrigin      = "Access-Control-Allow-Origin"
	ACAMethods     = "Access-Control-Allow-Methods"
	ACACredentials = "Access-Control-Allow-Credentials"
)

var mimeTypes = map[string]string{
	cmds.JSON: "application/json",
	cmds.XML:  "application/xml",
	cmds.Text: "text/plain",
}

type ServerConfig struct {
	// Headers is an optional map of headers that is written out.
	Headers map[string][]string

	// cORSOpts is a set of options for CORS headers.
	cORSOpts *cors.Options

	// cORSOptsRWMutex is a RWMutex for read/write CORSOpts
	cORSOptsRWMutex sync.RWMutex
}

func skipAPIHeader(h string) bool {
	switch h {
	case "Access-Control-Allow-Origin":
		return true
	case "Access-Control-Allow-Methods":
		return true
	case "Access-Control-Allow-Credentials":
		return true
	default:
		return false
	}
}

func NewHandler(ctx cmds.Context, root *cmds.Command, cfg *ServerConfig) http.Handler {
	if cfg == nil {
		panic("must provide a valid ServerConfig")
	}

	// setup request logger
	ctx.ReqLog = new(cmds.ReqLog)

	// Wrap the internal handler with CORS handling-middleware.
	// Create a handler for the API.
	internal := internalHandler{
		ctx:  ctx,
		root: root,
		cfg:  cfg,
	}
	c := cors.New(*cfg.cORSOpts)
	return &Handler{internal, c.Handler(internal)}
}

func (i Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	// Call the CORS handler which wraps the internal handler.
	i.corsHandler.ServeHTTP(w, r)
}

func (i internalHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	log.Debug("Incoming API request: ", r.URL)

	defer func() {
		if r := recover(); r != nil {
			log.Error("A panic has occurred in the commands handler!")
			log.Error(r)

			debug.PrintStack()
		}
	}()

	// get the node's context to pass into the commands.
	node, err := i.ctx.GetNode()
	if err != nil {
		s := fmt.Sprintf("cmds/http: couldn't GetNode(): %s", err)
		http.Error(w, s, http.StatusInternalServerError)
		return
	}

	ctx, cancel := context.WithCancel(node.Context())
	defer cancel()
	if cn, ok := w.(http.CloseNotifier); ok {
		go func() {
			select {
			case <-cn.CloseNotify():
			case <-ctx.Done():
			}
			cancel()
		}()
	}

	if !allowOrigin(r, i.cfg) || !allowReferer(r, i.cfg) {
		w.WriteHeader(http.StatusForbidden)
		w.Write([]byte("403 - Forbidden"))
		log.Warningf("API blocked request to %s. (possible CSRF)", r.URL)
		return
	}

	req, err := Parse(r, i.root)
	if err != nil {
		if err == ErrNotFound {
			w.WriteHeader(http.StatusNotFound)
		} else {
			w.WriteHeader(http.StatusBadRequest)
		}
		w.Write([]byte(err.Error()))
		return
	}

	rlog := i.ctx.ReqLog.Add(req)
	defer rlog.Finish()

	//ps: take note of the name clash - commands.Context != context.Context
	req.SetInvocContext(i.ctx)

	err = req.SetRootContext(ctx)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	// call the command
	res := i.root.Call(req)

	// set user's headers first.
	for k, v := range i.cfg.Headers {
		if !skipAPIHeader(k) {
			w.Header()[k] = v
		}
	}

	// now handle responding to the client properly
	sendResponse(w, r, res, req)
}

func guessMimeType(res cmds.Response) (string, error) {
	// Try to guess mimeType from the encoding option
	enc, found, err := res.Request().Option(cmds.EncShort).String()
	if err != nil {
		return "", err
	}
	if !found {
		return "", errors.New("no encoding option set")
	}

	if m, ok := mimeTypes[enc]; ok {
		return m, nil
	}

	return mimeTypes[cmds.JSON], nil
}

func sendResponse(w http.ResponseWriter, r *http.Request, res cmds.Response, req cmds.Request) {
	h := w.Header()
	// Expose our agent to allow identification
	h.Set("Server", "go-ipfs/"+config.CurrentVersionNumber)

	mime, err := guessMimeType(res)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	status := http.StatusOK
	// if response contains an error, write an HTTP error status code
	if e := res.Error(); e != nil {
		if e.Code == cmds.ErrClient {
			status = http.StatusBadRequest
		} else {
			status = http.StatusInternalServerError
		}
		// NOTE: The error will actually be written out by the reader below
	}

	out, err := res.Reader()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	// Set up our potential trailer
	h.Set("Trailer", StreamErrHeader)

	if res.Length() > 0 {
		h.Set(contentLengthHeader, strconv.FormatUint(res.Length(), 10))
	}

	if _, ok := res.Output().(io.Reader); ok {
		// set streams output type to text to avoid issues with browsers rendering
		// html pages on priveleged api ports
		mime = "text/plain"
		h.Set(streamHeader, "1")
	}

	// if output is a channel and user requested streaming channels,
	// use chunk copier for the output
	_, isChan := res.Output().(chan interface{})
	if !isChan {
		_, isChan = res.Output().(<-chan interface{})
	}

	if isChan {
		h.Set(channelHeader, "1")
	}

	// catch-all, set to text as default
	if mime == "" {
		mime = "text/plain"
	}

	h.Set(contentTypeHeader, mime)
	h.Set(transferEncodingHeader, "chunked")

	// set 'allowed' headers
	h.Set("Access-Control-Allow-Headers", "X-Stream-Output, X-Chunked-Output")
	// expose those headers
	h.Set("Access-Control-Expose-Headers", "X-Stream-Output, X-Chunked-Output")

	if r.Method == "HEAD" { // after all the headers.
		return
	}

	w.WriteHeader(status)
	err = flushCopy(w, out)
	if err != nil {
		log.Error("err: ", err)
		w.Header().Set(StreamErrHeader, sanitizedErrStr(err))
	}
}

func flushCopy(w io.Writer, r io.Reader) error {
	buf := make([]byte, 4096)
	f, ok := w.(http.Flusher)
	if !ok {
		_, err := io.Copy(w, r)
		return err
	}
	for {
		n, err := r.Read(buf)
		switch err {
		case io.EOF:
			if n <= 0 {
				return nil
			}
			// if data was returned alongside the EOF, pretend we didnt
			// get an EOF. The next read call should also EOF.
		case nil:
			// continue
		default:
			return err
		}

		nw, err := w.Write(buf[:n])
		if err != nil {
			return err
		}

		if nw != n {
			return fmt.Errorf("http write failed to write full amount: %d != %d", nw, n)
		}

		f.Flush()
	}
	return nil
}

func sanitizedErrStr(err error) string {
	s := err.Error()
	s = strings.Split(s, "\n")[0]
	s = strings.Split(s, "\r")[0]
	return s
}

func NewServerConfig() *ServerConfig {
	cfg := new(ServerConfig)
	cfg.cORSOpts = new(cors.Options)
	return cfg
}

func (cfg ServerConfig) AllowedOrigins() []string {
	cfg.cORSOptsRWMutex.RLock()
	defer cfg.cORSOptsRWMutex.RUnlock()
	return cfg.cORSOpts.AllowedOrigins
}

func (cfg *ServerConfig) SetAllowedOrigins(origins ...string) {
	cfg.cORSOptsRWMutex.Lock()
	defer cfg.cORSOptsRWMutex.Unlock()
	o := make([]string, len(origins))
	copy(o, origins)
	cfg.cORSOpts.AllowedOrigins = o
}

func (cfg *ServerConfig) AppendAllowedOrigins(origins ...string) {
	cfg.cORSOptsRWMutex.Lock()
	defer cfg.cORSOptsRWMutex.Unlock()
	cfg.cORSOpts.AllowedOrigins = append(cfg.cORSOpts.AllowedOrigins, origins...)
}

func (cfg ServerConfig) AllowedMethods() []string {
	cfg.cORSOptsRWMutex.RLock()
	defer cfg.cORSOptsRWMutex.RUnlock()
	return []string(cfg.cORSOpts.AllowedMethods)
}

func (cfg *ServerConfig) SetAllowedMethods(methods ...string) {
	cfg.cORSOptsRWMutex.Lock()
	defer cfg.cORSOptsRWMutex.Unlock()
	if cfg.cORSOpts == nil {
		cfg.cORSOpts = new(cors.Options)
	}
	cfg.cORSOpts.AllowedMethods = methods
}

func (cfg *ServerConfig) SetAllowCredentials(flag bool) {
	cfg.cORSOptsRWMutex.Lock()
	defer cfg.cORSOptsRWMutex.Unlock()
	cfg.cORSOpts.AllowCredentials = flag
}

// allowOrigin just stops the request if the origin is not allowed.
// the CORS middleware apparently does not do this for us...
func allowOrigin(r *http.Request, cfg *ServerConfig) bool {
	origin := r.Header.Get("Origin")

	// curl, or ipfs shell, typing it in manually, or clicking link
	// NOT in a browser. this opens up a hole. we should close it,
	// but right now it would break things. TODO
	if origin == "" {
		return true
	}
	origins := cfg.AllowedOrigins()
	for _, o := range origins {
		if o == "*" { // ok! you asked for it!
			return true
		}

		if o == origin { // allowed explicitly
			return true
		}
	}

	return false
}

// allowReferer this is here to prevent some CSRF attacks that
// the API would be vulnerable to. We check that the Referer
// is allowed by CORS Origin (origins and referrers here will
// work similarly in the normla uses of the API).
// See discussion at https://github.com/ipfs/go-ipfs/issues/1532
func allowReferer(r *http.Request, cfg *ServerConfig) bool {
	referer := r.Referer()

	// curl, or ipfs shell, typing it in manually, or clicking link
	// NOT in a browser. this opens up a hole. we should close it,
	// but right now it would break things. TODO
	if referer == "" {
		return true
	}

	u, err := url.Parse(referer)
	if err != nil {
		// bad referer. but there _is_ something, so bail.
		log.Debug("failed to parse referer: ", referer)
		// debug because referer comes straight from the client. dont want to
		// let people DOS by putting a huge referer that gets stored in log files.
		return false
	}
	origin := u.Scheme + "://" + u.Host

	// check CORS ACAOs and pretend Referer works like an origin.
	// this is valid for many (most?) sane uses of the API in
	// other applications, and will have the desired effect.
	origins := cfg.AllowedOrigins()
	for _, o := range origins {
		if o == "*" { // ok! you asked for it!
			return true
		}

		// referer is allowed explicitly
		if o == origin {
			return true
		}
	}

	return false
}

// apiVersionMatches checks whether the api client is running the
// same version of go-ipfs. for now, only the exact same version of
// client + server work. In the future, we should use semver for
// proper API versioning! \o/
func apiVersionMatches(r *http.Request) error {
	clientVersion := r.UserAgent()
	// skips check if client is not go-ipfs
	if clientVersion == "" || !strings.Contains(clientVersion, "/go-ipfs/") {
		return nil
	}

	daemonVersion := config.ApiVersion
	if daemonVersion != clientVersion {
		return fmt.Errorf("%s (%s != %s)", errApiVersionMismatch, daemonVersion, clientVersion)
	}
	return nil
}
<commit_msg>fix panic where closenotify was called from wrong goroutine<commit_after>package http

import (
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"runtime/debug"
	"strconv"
	"strings"
	"sync"

	cors "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/rs/cors"
	"github.com/ipfs/go-ipfs/repo/config"
	context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"

	cmds "github.com/ipfs/go-ipfs/commands"
	logging "gx/ipfs/Qmazh5oNUVsDZTs2g59rq8aYQqwpss8tcUWQzor5sCCEuH/go-log"
)

var log = logging.Logger("commands/http")

// the internal handler for the API
type internalHandler struct {
	ctx  cmds.Context
	root *cmds.Command
	cfg  *ServerConfig
}

// The Handler struct is funny because we want to wrap our internal handler
// with CORS while keeping our fields.
type Handler struct {
	internalHandler
	corsHandler http.Handler
}

var (
	ErrNotFound           = errors.New("404 page not found")
	errApiVersionMismatch = errors.New("api version mismatch")
)

const (
	StreamErrHeader        = "X-Stream-Error"
	streamHeader           = "X-Stream-Output"
	channelHeader          = "X-Chunked-Output"
	uaHeader               = "User-Agent"
	contentTypeHeader      = "Content-Type"
	contentLengthHeader    = "Content-Length"
	contentDispHeader      = "Content-Disposition"
	transferEncodingHeader = "Transfer-Encoding"
	applicationJson        = "application/json"
	applicationOctetStream = "application/octet-stream"
	plainText              = "text/plain"
	originHeader           = "origin"
)

const (
	ACAOrigin      = "Access-Control-Allow-Origin"
	ACAMethods     = "Access-Control-Allow-Methods"
	ACACredentials = "Access-Control-Allow-Credentials"
)

var mimeTypes = map[string]string{
	cmds.JSON: "application/json",
	cmds.XML:  "application/xml",
	cmds.Text: "text/plain",
}

type ServerConfig struct {
	// Headers is an optional map of headers that is written out.
	Headers map[string][]string

	// cORSOpts is a set of options for CORS headers.
	cORSOpts *cors.Options

	// cORSOptsRWMutex is a RWMutex for read/write CORSOpts
	cORSOptsRWMutex sync.RWMutex
}

func skipAPIHeader(h string) bool {
	switch h {
	case "Access-Control-Allow-Origin":
		return true
	case "Access-Control-Allow-Methods":
		return true
	case "Access-Control-Allow-Credentials":
		return true
	default:
		return false
	}
}

func NewHandler(ctx cmds.Context, root *cmds.Command, cfg *ServerConfig) http.Handler {
	if cfg == nil {
		panic("must provide a valid ServerConfig")
	}

	// setup request logger
	ctx.ReqLog = new(cmds.ReqLog)

	// Wrap the internal handler with CORS handling-middleware.
	// Create a handler for the API.
	internal := internalHandler{
		ctx:  ctx,
		root: root,
		cfg:  cfg,
	}
	c := cors.New(*cfg.cORSOpts)
	return &Handler{internal, c.Handler(internal)}
}

func (i Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	// Call the CORS handler which wraps the internal handler.
	i.corsHandler.ServeHTTP(w, r)
}

func (i internalHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	log.Debug("Incoming API request: ", r.URL)

	defer func() {
		if r := recover(); r != nil {
			log.Error("A panic has occurred in the commands handler!")
			log.Error(r)

			debug.PrintStack()
		}
	}()

	// get the node's context to pass into the commands.
	node, err := i.ctx.GetNode()
	if err != nil {
		s := fmt.Sprintf("cmds/http: couldn't GetNode(): %s", err)
		http.Error(w, s, http.StatusInternalServerError)
		return
	}

	ctx, cancel := context.WithCancel(node.Context())
	defer cancel()
	if cn, ok := w.(http.CloseNotifier); ok {
		clientGone := cn.CloseNotify()
		go func() {
			select {
			case <-clientGone:
			case <-ctx.Done():
			}
			cancel()
		}()
	}

	if !allowOrigin(r, i.cfg) || !allowReferer(r, i.cfg) {
		w.WriteHeader(http.StatusForbidden)
		w.Write([]byte("403 - Forbidden"))
		log.Warningf("API blocked request to %s. (possible CSRF)", r.URL)
		return
	}

	req, err := Parse(r, i.root)
	if err != nil {
		if err == ErrNotFound {
			w.WriteHeader(http.StatusNotFound)
		} else {
			w.WriteHeader(http.StatusBadRequest)
		}
		w.Write([]byte(err.Error()))
		return
	}

	rlog := i.ctx.ReqLog.Add(req)
	defer rlog.Finish()

	//ps: take note of the name clash - commands.Context != context.Context
	req.SetInvocContext(i.ctx)

	err = req.SetRootContext(ctx)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	// call the command
	res := i.root.Call(req)

	// set user's headers first.
	for k, v := range i.cfg.Headers {
		if !skipAPIHeader(k) {
			w.Header()[k] = v
		}
	}

	// now handle responding to the client properly
	sendResponse(w, r, res, req)
}

func guessMimeType(res cmds.Response) (string, error) {
	// Try to guess mimeType from the encoding option
	enc, found, err := res.Request().Option(cmds.EncShort).String()
	if err != nil {
		return "", err
	}
	if !found {
		return "", errors.New("no encoding option set")
	}

	if m, ok := mimeTypes[enc]; ok {
		return m, nil
	}

	return mimeTypes[cmds.JSON], nil
}

func sendResponse(w http.ResponseWriter, r *http.Request, res cmds.Response, req cmds.Request) {
	h := w.Header()
	// Expose our agent to allow identification
	h.Set("Server", "go-ipfs/"+config.CurrentVersionNumber)

	mime, err := guessMimeType(res)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	status := http.StatusOK
	// if response contains an error, write an HTTP error status code
	if e := res.Error(); e != nil {
		if e.Code == cmds.ErrClient {
			status = http.StatusBadRequest
		} else {
			status = http.StatusInternalServerError
		}
		// NOTE: The error will actually be written out by the reader below
	}

	out, err := res.Reader()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	// Set up our potential trailer
	h.Set("Trailer", StreamErrHeader)

	if res.Length() > 0 {
		h.Set(contentLengthHeader, strconv.FormatUint(res.Length(), 10))
	}

	if _, ok := res.Output().(io.Reader); ok {
		// set streams output type to text to avoid issues with browsers rendering
		// html pages on priveleged api ports
		mime = "text/plain"
		h.Set(streamHeader, "1")
	}

	// if output is a channel and user requested streaming channels,
	// use chunk copier for the output
	_, isChan := res.Output().(chan interface{})
	if !isChan {
		_, isChan = res.Output().(<-chan interface{})
	}

	if isChan {
		h.Set(channelHeader, "1")
	}

	// catch-all, set to text as default
	if mime == "" {
		mime = "text/plain"
	}

	h.Set(contentTypeHeader, mime)
	h.Set(transferEncodingHeader, "chunked")

	// set 'allowed' headers
	h.Set("Access-Control-Allow-Headers", "X-Stream-Output, X-Chunked-Output")
	// expose those headers
	h.Set("Access-Control-Expose-Headers", "X-Stream-Output, X-Chunked-Output")

	if r.Method == "HEAD" { // after all the headers.
		return
	}

	w.WriteHeader(status)
	err = flushCopy(w, out)
	if err != nil {
		log.Error("err: ", err)
		w.Header().Set(StreamErrHeader, sanitizedErrStr(err))
	}
}

func flushCopy(w io.Writer, r io.Reader) error {
	buf := make([]byte, 4096)
	f, ok := w.(http.Flusher)
	if !ok {
		_, err := io.Copy(w, r)
		return err
	}
	for {
		n, err := r.Read(buf)
		switch err {
		case io.EOF:
			if n <= 0 {
				return nil
			}
			// if data was returned alongside the EOF, pretend we didnt
			// get an EOF. The next read call should also EOF.
		case nil:
			// continue
		default:
			return err
		}

		nw, err := w.Write(buf[:n])
		if err != nil {
			return err
		}

		if nw != n {
			return fmt.Errorf("http write failed to write full amount: %d != %d", nw, n)
		}

		f.Flush()
	}
	return nil
}

func sanitizedErrStr(err error) string {
	s := err.Error()
	s = strings.Split(s, "\n")[0]
	s = strings.Split(s, "\r")[0]
	return s
}

func NewServerConfig() *ServerConfig {
	cfg := new(ServerConfig)
	cfg.cORSOpts = new(cors.Options)
	return cfg
}

func (cfg ServerConfig) AllowedOrigins() []string {
	cfg.cORSOptsRWMutex.RLock()
	defer cfg.cORSOptsRWMutex.RUnlock()
	return cfg.cORSOpts.AllowedOrigins
}

func (cfg *ServerConfig) SetAllowedOrigins(origins ...string) {
	cfg.cORSOptsRWMutex.Lock()
	defer cfg.cORSOptsRWMutex.Unlock()
	o := make([]string, len(origins))
	copy(o, origins)
	cfg.cORSOpts.AllowedOrigins = o
}

func (cfg *ServerConfig) AppendAllowedOrigins(origins ...string) {
	cfg.cORSOptsRWMutex.Lock()
	defer cfg.cORSOptsRWMutex.Unlock()
	cfg.cORSOpts.AllowedOrigins = append(cfg.cORSOpts.AllowedOrigins, origins...)
}

func (cfg ServerConfig) AllowedMethods() []string {
	cfg.cORSOptsRWMutex.RLock()
	defer cfg.cORSOptsRWMutex.RUnlock()
	return []string(cfg.cORSOpts.AllowedMethods)
}

func (cfg *ServerConfig) SetAllowedMethods(methods ...string) {
	cfg.cORSOptsRWMutex.Lock()
	defer cfg.cORSOptsRWMutex.Unlock()
	if cfg.cORSOpts == nil {
		cfg.cORSOpts = new(cors.Options)
	}
	cfg.cORSOpts.AllowedMethods = methods
}

func (cfg *ServerConfig) SetAllowCredentials(flag bool) {
	cfg.cORSOptsRWMutex.Lock()
	defer cfg.cORSOptsRWMutex.Unlock()
	cfg.cORSOpts.AllowCredentials = flag
}

// allowOrigin just stops the request if the origin is not allowed.
// the CORS middleware apparently does not do this for us...
func allowOrigin(r *http.Request, cfg *ServerConfig) bool {
	origin := r.Header.Get("Origin")

	// curl, or ipfs shell, typing it in manually, or clicking link
	// NOT in a browser. this opens up a hole. we should close it,
	// but right now it would break things. TODO
	if origin == "" {
		return true
	}
	origins := cfg.AllowedOrigins()
	for _, o := range origins {
		if o == "*" { // ok! you asked for it!
			return true
		}

		if o == origin { // allowed explicitly
			return true
		}
	}

	return false
}

// allowReferer this is here to prevent some CSRF attacks that
// the API would be vulnerable to. We check that the Referer
// is allowed by CORS Origin (origins and referrers here will
// work similarly in the normla uses of the API).
// See discussion at https://github.com/ipfs/go-ipfs/issues/1532
func allowReferer(r *http.Request, cfg *ServerConfig) bool {
	referer := r.Referer()

	// curl, or ipfs shell, typing it in manually, or clicking link
	// NOT in a browser. this opens up a hole. we should close it,
	// but right now it would break things. TODO
	if referer == "" {
		return true
	}

	u, err := url.Parse(referer)
	if err != nil {
		// bad referer. but there _is_ something, so bail.
		log.Debug("failed to parse referer: ", referer)
		// debug because referer comes straight from the client. dont want to
		// let people DOS by putting a huge referer that gets stored in log files.
		return false
	}
	origin := u.Scheme + "://" + u.Host

	// check CORS ACAOs and pretend Referer works like an origin.
	// this is valid for many (most?) sane uses of the API in
	// other applications, and will have the desired effect.
	origins := cfg.AllowedOrigins()
	for _, o := range origins {
		if o == "*" { // ok! you asked for it!
			return true
		}

		// referer is allowed explicitly
		if o == origin {
			return true
		}
	}

	return false
}

// apiVersionMatches checks whether the api client is running the
// same version of go-ipfs. for now, only the exact same version of
// client + server work. In the future, we should use semver for
// proper API versioning! \o/
func apiVersionMatches(r *http.Request) error {
	clientVersion := r.UserAgent()
	// skips check if client is not go-ipfs
	if clientVersion == "" || !strings.Contains(clientVersion, "/go-ipfs/") {
		return nil
	}

	daemonVersion := config.ApiVersion
	if daemonVersion != clientVersion {
		return fmt.Errorf("%s (%s != %s)", errApiVersionMismatch, daemonVersion, clientVersion)
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2013 Kelsey Hightower. All rights reserved.
// Use of this source code is governed by the MIT License that can be found in
// the LICENSE file.

package envconfig

import (
	"errors"
	"fmt"
	"os"
	"reflect"
	"strconv"
	"strings"
)

// ErrInvalidSpecification indicates that a specification is of the wrong type.
var ErrInvalidSpecification = errors.New("invalid specification must be a struct")

// A ParseError occurs when an environment variable cannot be converted to
// the type required by a struct field during assignment.
type ParseError struct {
	KeyName   string
	FieldName string
	TypeName  string
	Value     string
}

func (e *ParseError) Error() string {
	return fmt.Sprintf("envconfig.Process: assigning %[1]s to %[2]s: converting '%[3]s' to type %[4]s", e.KeyName, e.FieldName, e.Value, e.TypeName)
}

func Process(prefix string, spec interface{}) error {
	s := reflect.ValueOf(spec).Elem()
	if s.Kind() != reflect.Struct {
		return ErrInvalidSpecification
	}
	typeOfSpec := s.Type()
	for i := 0; i < s.NumField(); i++ {
		f := s.Field(i)
		if f.CanSet() {
			if typeOfSpec.Field(i).Anonymous && f.Kind() == reflect.Struct {
				embeddedPtr := reflect.New(f.Type()).Interface()
				if err := Process(prefix, embeddedPtr); err != nil {
					return err
				}
				f.Set(reflect.ValueOf(embeddedPtr).Elem())
			}

			alt := typeOfSpec.Field(i).Tag.Get("envconfig")
			fieldName := typeOfSpec.Field(i).Name
			if alt != "" {
				fieldName = alt
			}
			key := strings.ToUpper(fmt.Sprintf("%s_%s", prefix, fieldName))
			value := os.Getenv(key)
			if value == "" && alt != "" {
				key := strings.ToUpper(fieldName)
				value = os.Getenv(key)
			}

			def := typeOfSpec.Field(i).Tag.Get("default")
			if def != "" && value == "" {
				value = def
			}

			req := typeOfSpec.Field(i).Tag.Get("required")
			if value == "" {
				if req == "true" {
					return fmt.Errorf("required key %s missing value", key)
				}
				continue
			}

			switch f.Kind() {
			case reflect.String:
				f.SetString(value)
			case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
				intValue, err := strconv.ParseInt(value, 0, f.Type().Bits())
				if err != nil {
					return &ParseError{
						KeyName:   key,
						FieldName: fieldName,
						TypeName:  f.Type().String(),
						Value:     value,
					}
				}
				f.SetInt(intValue)
			case reflect.Bool:
				boolValue, err := strconv.ParseBool(value)
				if err != nil {
					return &ParseError{
						KeyName:   key,
						FieldName: fieldName,
						TypeName:  f.Type().String(),
						Value:     value,
					}
				}
				f.SetBool(boolValue)
			case reflect.Float32, reflect.Float64:
				floatValue, err := strconv.ParseFloat(value, f.Type().Bits())
				if err != nil {
					return &ParseError{
						KeyName:   key,
						FieldName: fieldName,
						TypeName:  f.Type().String(),
						Value:     value,
					}
				}
				f.SetFloat(floatValue)
			}
		}
	}
	return nil
}

func MustProcess(prefix string, spec interface{}) {
	if err := Process(prefix, spec); err != nil {
		panic(err)
	}
}
<commit_msg>Simplify grabbing a pointer to process an embedded struct<commit_after>// Copyright (c) 2013 Kelsey Hightower. All rights reserved.
// Use of this source code is governed by the MIT License that can be found in
// the LICENSE file.

package envconfig

import (
	"errors"
	"fmt"
	"os"
	"reflect"
	"strconv"
	"strings"
)

// ErrInvalidSpecification indicates that a specification is of the wrong type.
var ErrInvalidSpecification = errors.New("invalid specification must be a struct")

// A ParseError occurs when an environment variable cannot be converted to
// the type required by a struct field during assignment.
type ParseError struct {
	KeyName   string
	FieldName string
	TypeName  string
	Value     string
}

func (e *ParseError) Error() string {
	return fmt.Sprintf("envconfig.Process: assigning %[1]s to %[2]s: converting '%[3]s' to type %[4]s", e.KeyName, e.FieldName, e.Value, e.TypeName)
}

func Process(prefix string, spec interface{}) error {
	s := reflect.ValueOf(spec).Elem()
	if s.Kind() != reflect.Struct {
		return ErrInvalidSpecification
	}
	typeOfSpec := s.Type()
	for i := 0; i < s.NumField(); i++ {
		f := s.Field(i)
		if f.CanSet() {
			if typeOfSpec.Field(i).Anonymous && f.Kind() == reflect.Struct {
				embeddedPtr := f.Addr().Interface()
				if err := Process(prefix, embeddedPtr); err != nil {
					return err
				}
				f.Set(reflect.ValueOf(embeddedPtr).Elem())
			}

			alt := typeOfSpec.Field(i).Tag.Get("envconfig")
			fieldName := typeOfSpec.Field(i).Name
			if alt != "" {
				fieldName = alt
			}
			key := strings.ToUpper(fmt.Sprintf("%s_%s", prefix, fieldName))
			value := os.Getenv(key)
			if value == "" && alt != "" {
				key := strings.ToUpper(fieldName)
				value = os.Getenv(key)
			}

			def := typeOfSpec.Field(i).Tag.Get("default")
			if def != "" && value == "" {
				value = def
			}

			req := typeOfSpec.Field(i).Tag.Get("required")
			if value == "" {
				if req == "true" {
					return fmt.Errorf("required key %s missing value", key)
				}
				continue
			}

			switch f.Kind() {
			case reflect.String:
				f.SetString(value)
			case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
				intValue, err := strconv.ParseInt(value, 0, f.Type().Bits())
				if err != nil {
					return &ParseError{
						KeyName:   key,
						FieldName: fieldName,
						TypeName:  f.Type().String(),
						Value:     value,
					}
				}
				f.SetInt(intValue)
			case reflect.Bool:
				boolValue, err := strconv.ParseBool(value)
				if err != nil {
					return &ParseError{
						KeyName:   key,
						FieldName: fieldName,
						TypeName:  f.Type().String(),
						Value:     value,
					}
				}
				f.SetBool(boolValue)
			case reflect.Float32, reflect.Float64:
				floatValue, err := strconv.ParseFloat(value, f.Type().Bits())
				if err != nil {
					return &ParseError{
						KeyName:   key,
						FieldName: fieldName,
						TypeName:  f.Type().String(),
						Value:     value,
					}
				}
				f.SetFloat(floatValue)
			}
		}
	}
	return nil
}

func MustProcess(prefix string, spec interface{}) {
	if err := Process(prefix, spec); err != nil {
		panic(err)
	}
}
<|endoftext|>