<commit_before>package storage

import (
	"github.com/docker/distribution"
	"github.com/docker/distribution/context"
	"github.com/docker/distribution/reference"
	"github.com/docker/distribution/registry/storage/cache"
	storagedriver "github.com/docker/distribution/registry/storage/driver"
)

// registry is the top-level implementation of Registry for use in the storage
// package. All instances should descend from this object.
type registry struct {
	blobStore                   *blobStore
	blobServer                  *blobServer
	statter                     *blobStatter // global statter service.
	blobDescriptorCacheProvider cache.BlobDescriptorCacheProvider
	deleteEnabled               bool
	resumableDigestEnabled      bool
}

// RegistryOption is the type used for functional options for NewRegistry.
type RegistryOption func(*registry) error

// EnableRedirect is a functional option for NewRegistry. It causes the backend
// blob server to attempt using (StorageDriver).URLFor to serve all blobs.
func EnableRedirect(registry *registry) error {
	registry.blobServer.redirect = true
	return nil
}

// EnableDelete is a functional option for NewRegistry. It enables deletion on
// the registry.
func EnableDelete(registry *registry) error {
	registry.deleteEnabled = true
	return nil
}

// DisableDigestResumption is a functional option for NewRegistry. It should be
// used if the registry is acting as a caching proxy.
func DisableDigestResumption(registry *registry) error {
	registry.resumableDigestEnabled = false
	return nil
}

// BlobDescriptorCacheProvider returns a functional option for
// NewRegistry. It creates a cached blob statter for use by the
// registry.
func BlobDescriptorCacheProvider(blobDescriptorCacheProvider cache.BlobDescriptorCacheProvider) RegistryOption {
	// TODO(aaronl): The duplication of statter across several objects is
	// ugly, and prevents us from using interface types in the registry
	// struct. Ideally, blobStore and blobServer should be lazily
	// initialized, and use the current value of
	// blobDescriptorCacheProvider.
	return func(registry *registry) error {
		if blobDescriptorCacheProvider != nil {
			statter := cache.NewCachedBlobStatter(blobDescriptorCacheProvider, registry.statter)
			registry.blobStore.statter = statter
			registry.blobServer.statter = statter
			registry.blobDescriptorCacheProvider = blobDescriptorCacheProvider
		}
		return nil
	}
}

// NewRegistry creates a new registry instance from the provided driver. The
// resulting registry may be shared by multiple goroutines but is cheap to
// allocate. If the Redirect option is specified, the backend blob server will
// attempt to use (StorageDriver).URLFor to serve all blobs.
func NewRegistry(ctx context.Context, driver storagedriver.StorageDriver, options ...RegistryOption) (distribution.Namespace, error) {
	// create global statter
	statter := &blobStatter{
		driver: driver,
	}

	bs := &blobStore{
		driver:  driver,
		statter: statter,
	}

	registry := &registry{
		blobStore: bs,
		blobServer: &blobServer{
			driver:  driver,
			statter: statter,
			pathFn:  bs.path,
		},
		statter:                statter,
		resumableDigestEnabled: true,
	}

	for _, option := range options {
		if err := option(registry); err != nil {
			return nil, err
		}
	}

	return registry, nil
}

// Scope returns the namespace scope for a registry. The registry
// will only serve repositories contained within this scope.
func (reg *registry) Scope() distribution.Scope {
	return distribution.GlobalScope
}

// Repository returns an instance of the repository tied to the registry.
// Instances should not be shared between goroutines but are cheap to
// allocate. In general, they should be request scoped.
func (reg *registry) Repository(ctx context.Context, canonicalName string) (distribution.Repository, error) {
	if _, err := reference.ParseNamed(canonicalName); err != nil {
		return nil, distribution.ErrRepositoryNameInvalid{
			Name:   canonicalName,
			Reason: err,
		}
	}

	var descriptorCache distribution.BlobDescriptorService
	if reg.blobDescriptorCacheProvider != nil {
		var err error
		descriptorCache, err = reg.blobDescriptorCacheProvider.RepositoryScoped(canonicalName)
		if err != nil {
			return nil, err
		}
	}

	return &repository{
		ctx:             ctx,
		registry:        reg,
		name:            canonicalName,
		descriptorCache: descriptorCache,
	}, nil
}

// repository provides name-scoped access to various services.
type repository struct {
	*registry
	ctx             context.Context
	name            string
	descriptorCache distribution.BlobDescriptorService
}

// Name returns the name of the repository.
func (repo *repository) Name() string {
	return repo.name
}

// Manifests returns an instance of ManifestService. Instantiation is cheap and
// may be context sensitive in the future. The instance should be used similar
// to a request local.
func (repo *repository) Manifests(ctx context.Context, options ...distribution.ManifestServiceOption) (distribution.ManifestService, error) {
	manifestLinkPathFns := []linkPathFunc{
		// NOTE(stevvooe): Need to search through multiple locations since
		// 2.1.0 unintentionally linked into  _layers.
		manifestRevisionLinkPath,
		blobLinkPath,
	}

	ms := &manifestStore{
		ctx:        ctx,
		repository: repo,
		revisionStore: &revisionStore{
			ctx:        ctx,
			repository: repo,
			blobStore: &linkedBlobStore{
				ctx:           ctx,
				blobStore:     repo.blobStore,
				repository:    repo,
				deleteEnabled: repo.registry.deleteEnabled,
				blobAccessController: &linkedBlobStatter{
					blobStore:   repo.blobStore,
					repository:  repo,
					linkPathFns: manifestLinkPathFns,
				},

				// TODO(stevvooe): linkPath limits this blob store to only
				// manifests. This instance cannot be used for blob checks.
				linkPathFns: manifestLinkPathFns,
			},
		},
		tagStore: &tagStore{
			ctx:        ctx,
			repository: repo,
			blobStore:  repo.registry.blobStore,
		},
	}

	// Apply options
	for _, option := range options {
		err := option(ms)
		if err != nil {
			return nil, err
		}
	}

	return ms, nil
}

// Blobs returns an instance of the BlobStore. Instantiation is cheap and
// may be context sensitive in the future. The instance should be used similar
// to a request local.
func (repo *repository) Blobs(ctx context.Context) distribution.BlobStore {
	var statter distribution.BlobDescriptorService = &linkedBlobStatter{
		blobStore:   repo.blobStore,
		repository:  repo,
		linkPathFns: []linkPathFunc{blobLinkPath},
	}

	if repo.descriptorCache != nil {
		statter = cache.NewCachedBlobStatter(repo.descriptorCache, statter)
	}

	return &linkedBlobStore{
		blobStore:            repo.blobStore,
		blobServer:           repo.blobServer,
		blobAccessController: statter,
		repository:           repo,
		ctx:                  ctx,

		// TODO(stevvooe): linkPath limits this blob store to only layers.
		// This instance cannot be used for manifest checks.
		linkPathFns:   []linkPathFunc{blobLinkPath},
		deleteEnabled: repo.registry.deleteEnabled,
	}
}

func (repo *repository) Signatures() distribution.SignatureService {
	return &signatureStore{
		repository: repo,
		blobStore:  repo.blobStore,
		ctx:        repo.ctx,
	}
}
<commit_msg>Redundant digest verification in validateBlob<commit_after>package storage

import (
	"github.com/docker/distribution"
	"github.com/docker/distribution/context"
	"github.com/docker/distribution/reference"
	"github.com/docker/distribution/registry/storage/cache"
	storagedriver "github.com/docker/distribution/registry/storage/driver"
)

// registry is the top-level implementation of Registry for use in the storage
// package. All instances should descend from this object.
type registry struct {
	blobStore                   *blobStore
	blobServer                  *blobServer
	statter                     *blobStatter // global statter service.
	blobDescriptorCacheProvider cache.BlobDescriptorCacheProvider
	deleteEnabled               bool
	resumableDigestEnabled      bool
}

// RegistryOption is the type used for functional options for NewRegistry.
type RegistryOption func(*registry) error

// EnableRedirect is a functional option for NewRegistry. It causes the backend
// blob server to attempt using (StorageDriver).URLFor to serve all blobs.
func EnableRedirect(registry *registry) error {
	registry.blobServer.redirect = true
	return nil
}

// EnableDelete is a functional option for NewRegistry. It enables deletion on
// the registry.
func EnableDelete(registry *registry) error {
	registry.deleteEnabled = true
	return nil
}

// DisableDigestResumption is a functional option for NewRegistry. It should be
// used if the registry is acting as a caching proxy.
func DisableDigestResumption(registry *registry) error {
	registry.resumableDigestEnabled = false
	return nil
}

// BlobDescriptorCacheProvider returns a functional option for
// NewRegistry. It creates a cached blob statter for use by the
// registry.
func BlobDescriptorCacheProvider(blobDescriptorCacheProvider cache.BlobDescriptorCacheProvider) RegistryOption {
	// TODO(aaronl): The duplication of statter across several objects is
	// ugly, and prevents us from using interface types in the registry
	// struct. Ideally, blobStore and blobServer should be lazily
	// initialized, and use the current value of
	// blobDescriptorCacheProvider.
	return func(registry *registry) error {
		if blobDescriptorCacheProvider != nil {
			statter := cache.NewCachedBlobStatter(blobDescriptorCacheProvider, registry.statter)
			registry.blobStore.statter = statter
			registry.blobServer.statter = statter
			registry.blobDescriptorCacheProvider = blobDescriptorCacheProvider
		}
		return nil
	}
}

// NewRegistry creates a new registry instance from the provided driver. The
// resulting registry may be shared by multiple goroutines but is cheap to
// allocate. If the Redirect option is specified, the backend blob server will
// attempt to use (StorageDriver).URLFor to serve all blobs.
func NewRegistry(ctx context.Context, driver storagedriver.StorageDriver, options ...RegistryOption) (distribution.Namespace, error) {
	// create global statter
	statter := &blobStatter{
		driver: driver,
	}

	bs := &blobStore{
		driver:  driver,
		statter: statter,
	}

	registry := &registry{
		blobStore: bs,
		blobServer: &blobServer{
			driver:  driver,
			statter: statter,
			pathFn:  bs.path,
		},
		statter:                statter,
		resumableDigestEnabled: true,
	}

	for _, option := range options {
		if err := option(registry); err != nil {
			return nil, err
		}
	}

	return registry, nil
}

// Scope returns the namespace scope for a registry. The registry
// will only serve repositories contained within this scope.
func (reg *registry) Scope() distribution.Scope {
	return distribution.GlobalScope
}

// Repository returns an instance of the repository tied to the registry.
// Instances should not be shared between goroutines but are cheap to
// allocate. In general, they should be request scoped.
func (reg *registry) Repository(ctx context.Context, canonicalName string) (distribution.Repository, error) {
	if _, err := reference.ParseNamed(canonicalName); err != nil {
		return nil, distribution.ErrRepositoryNameInvalid{
			Name:   canonicalName,
			Reason: err,
		}
	}

	var descriptorCache distribution.BlobDescriptorService
	if reg.blobDescriptorCacheProvider != nil {
		var err error
		descriptorCache, err = reg.blobDescriptorCacheProvider.RepositoryScoped(canonicalName)
		if err != nil {
			return nil, err
		}
	}

	return &repository{
		ctx:             ctx,
		registry:        reg,
		name:            canonicalName,
		descriptorCache: descriptorCache,
	}, nil
}

// repository provides name-scoped access to various services.
type repository struct {
	*registry
	ctx             context.Context
	name            string
	descriptorCache distribution.BlobDescriptorService
}

// Name returns the name of the repository.
func (repo *repository) Name() string {
	return repo.name
}

// Manifests returns an instance of ManifestService. Instantiation is cheap and
// may be context sensitive in the future. The instance should be used similar
// to a request local.
func (repo *repository) Manifests(ctx context.Context, options ...distribution.ManifestServiceOption) (distribution.ManifestService, error) {
	manifestLinkPathFns := []linkPathFunc{
		// NOTE(stevvooe): Need to search through multiple locations since
		// 2.1.0 unintentionally linked into  _layers.
		manifestRevisionLinkPath,
		blobLinkPath,
	}

	ms := &manifestStore{
		ctx:        ctx,
		repository: repo,
		revisionStore: &revisionStore{
			ctx:        ctx,
			repository: repo,
			blobStore: &linkedBlobStore{
				ctx:           ctx,
				blobStore:     repo.blobStore,
				repository:    repo,
				deleteEnabled: repo.registry.deleteEnabled,
				blobAccessController: &linkedBlobStatter{
					blobStore:   repo.blobStore,
					repository:  repo,
					linkPathFns: manifestLinkPathFns,
				},

				// TODO(stevvooe): linkPath limits this blob store to only
				// manifests. This instance cannot be used for blob checks.
				linkPathFns:            manifestLinkPathFns,
				resumableDigestEnabled: repo.resumableDigestEnabled,
			},
		},
		tagStore: &tagStore{
			ctx:        ctx,
			repository: repo,
			blobStore:  repo.registry.blobStore,
		},
	}

	// Apply options
	for _, option := range options {
		err := option(ms)
		if err != nil {
			return nil, err
		}
	}

	return ms, nil
}

// Blobs returns an instance of the BlobStore. Instantiation is cheap and
// may be context sensitive in the future. The instance should be used similar
// to a request local.
func (repo *repository) Blobs(ctx context.Context) distribution.BlobStore {
	var statter distribution.BlobDescriptorService = &linkedBlobStatter{
		blobStore:   repo.blobStore,
		repository:  repo,
		linkPathFns: []linkPathFunc{blobLinkPath},
	}

	if repo.descriptorCache != nil {
		statter = cache.NewCachedBlobStatter(repo.descriptorCache, statter)
	}

	return &linkedBlobStore{
		blobStore:            repo.blobStore,
		blobServer:           repo.blobServer,
		blobAccessController: statter,
		repository:           repo,
		ctx:                  ctx,

		// TODO(stevvooe): linkPath limits this blob store to only layers.
		// This instance cannot be used for manifest checks.
		linkPathFns:            []linkPathFunc{blobLinkPath},
		deleteEnabled:          repo.registry.deleteEnabled,
		resumableDigestEnabled: repo.resumableDigestEnabled,
	}
}

func (repo *repository) Signatures() distribution.SignatureService {
	return &signatureStore{
		repository: repo,
		blobStore:  repo.blobStore,
		ctx:        repo.ctx,
	}
}
<|endoftext|>


<commit_before>package fixity

import (
	"fmt"
	"sync"

	"github.com/leeola/fixity/config"
)

var (
	blobstoreRegistry   map[string]BlobstoreConstructor
	blobstoreRegistryMu sync.Mutex
	indexRegistry       map[string]IndexConstructor
	indexRegistryMu     sync.Mutex
	storeRegistry       map[string]StoreConstructor
	storeRegistryMu     sync.Mutex
)

func init() {
	blobstoreRegistry = map[string]BlobstoreConstructor{}
	indexRegistry = map[string]IndexConstructor{}
	storeRegistry = map[string]StoreConstructor{}
}

type BlobstoreConstructor interface {
	New(name string, c config.Config) (Blobstore, error)
}

type IndexConstructor interface {
	New(name string, c config.Config) (Index, error)
}

type StoreConstructor interface {
	New(name string, c config.Config) (Store, error)
}

type BlobstoreConstructorFunc func(string, config.Config) (Blobstore, error)

type IndexConstructorFunc func(string, config.Config) (Index, error)

type StoreConstructorFunc func(string, config.Config) (Store, error)

func RegisterBlobstore(key string, c BlobstoreConstructor) {
	if key == "" {
		panic(fmt.Sprintf("key cannot be empty"))
	}

	blobstoreRegistryMu.Lock()
	defer blobstoreRegistryMu.Unlock()

	if _, ok := blobstoreRegistry[key]; ok {
		panic(fmt.Sprintf("already registered blobstore: %s", key))
	}

	blobstoreRegistry[key] = c
}

func RegisterIndex(key string, c IndexConstructor) {
	if key == "" {
		panic(fmt.Sprintf("key cannot be empty"))
	}

	indexRegistryMu.Lock()
	defer indexRegistryMu.Unlock()

	if _, ok := indexRegistry[key]; ok {
		panic(fmt.Sprintf("already registered index: %s", key))
	}

	indexRegistry[key] = c
}

func RegisterStore(key string, c StoreConstructor) {
	if key == "" {
		panic(fmt.Sprintf("key cannot be empty"))
	}

	storeRegistryMu.Lock()
	defer storeRegistryMu.Unlock()

	if _, ok := storeRegistry[key]; ok {
		panic(fmt.Sprintf("already registered store: %s", key))
	}

	storeRegistry[key] = c
}

func (f BlobstoreConstructorFunc) New(name string, c config.Config) (Blobstore, error) {
	return f(name, c)
}

func (f IndexConstructorFunc) New(name string, c config.Config) (Index, error) {
	return f(name, c)
}

func (f StoreConstructorFunc) New(name string, c config.Config) (Store, error) {
	return f(name, c)
}
<commit_msg>refactor: key -> xType<commit_after>package fixity

import (
	"fmt"
	"sync"

	"github.com/leeola/fixity/config"
)

var (
	blobstoreRegistry   map[string]BlobstoreConstructor
	blobstoreRegistryMu sync.Mutex
	indexRegistry       map[string]IndexConstructor
	indexRegistryMu     sync.Mutex
	storeRegistry       map[string]StoreConstructor
	storeRegistryMu     sync.Mutex
)

func init() {
	blobstoreRegistry = map[string]BlobstoreConstructor{}
	indexRegistry = map[string]IndexConstructor{}
	storeRegistry = map[string]StoreConstructor{}
}

type BlobstoreConstructor interface {
	New(name string, c config.Config) (Blobstore, error)
}

type IndexConstructor interface {
	New(name string, c config.Config) (Index, error)
}

type StoreConstructor interface {
	New(name string, c config.Config) (Store, error)
}

type BlobstoreConstructorFunc func(string, config.Config) (Blobstore, error)

type IndexConstructorFunc func(string, config.Config) (Index, error)

type StoreConstructorFunc func(string, config.Config) (Store, error)

func RegisterBlobstore(blobstoreType string, c BlobstoreConstructor) {
	if blobstoreType == "" {
		panic(fmt.Sprintf("blobstoreType cannot be empty"))
	}

	blobstoreRegistryMu.Lock()
	defer blobstoreRegistryMu.Unlock()

	if _, ok := blobstoreRegistry[blobstoreType]; ok {
		panic(fmt.Sprintf("already registered blobstore: %s", blobstoreType))
	}

	blobstoreRegistry[blobstoreType] = c
}

func RegisterIndex(indexType string, c IndexConstructor) {
	if indexType == "" {
		panic(fmt.Sprintf("indexType cannot be empty"))
	}

	indexRegistryMu.Lock()
	defer indexRegistryMu.Unlock()

	if _, ok := indexRegistry[indexType]; ok {
		panic(fmt.Sprintf("already registered index: %s", indexType))
	}

	indexRegistry[indexType] = c
}

func RegisterStore(storeType string, c StoreConstructor) {
	if storeType == "" {
		panic(fmt.Sprintf("storeType cannot be empty"))
	}

	storeRegistryMu.Lock()
	defer storeRegistryMu.Unlock()

	if _, ok := storeRegistry[storeType]; ok {
		panic(fmt.Sprintf("already registered store: %s", storeType))
	}

	storeRegistry[storeType] = c
}

func (f BlobstoreConstructorFunc) New(n string, c config.Config) (Blobstore, error) {
	return f(n, c)
}

func (f IndexConstructorFunc) New(n string, c config.Config) (Index, error) {
	return f(n, c)
}

func (f StoreConstructorFunc) New(n string, c config.Config) (Store, error) {
	return f(n, c)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"github.com/elazarl/goproxy"
	"github.com/jackc/pgx"
	"log"
	"net/http"
	"os"
	"os/exec"
	"strings"
)

func checkError(err error) {
	if err != nil {
		log.Fatalf("Error: %s", err)
	}
}

func UrlHasPrefix(prefix string) goproxy.ReqConditionFunc {
	return func(req *http.Request, ctx *goproxy.ProxyCtx) bool {
		return req.Method == http.MethodGet && strings.HasPrefix(req.URL.Path, prefix)
	}
}

var pool *pgx.ConnPool

func afterConnect(conn *pgx.Conn) (err error) {
	_, err = conn.Prepare("getPackage", `
		SELECT name, url FROM packages WHERE name = $1
  `)
	if err != nil {
		return
	}

	return
}

func main() {

	var err error
	pgxcfg, err := pgx.ParseURI(os.Getenv("DATABASE_URL"))
	if err != nil {
		log.Fatalf("Parse URI error: %s", err)
	}
	pool, err = pgx.NewConnPool(pgx.ConnPoolConfig{
		ConnConfig:     pgxcfg,
		MaxConnections: 10,
		AfterConnect:   afterConnect,
	})
	if err != nil {
		log.Fatalf("Connection error: %s", err)
	}
	defer pool.Close()

	port := os.Getenv("PORT")

	if port == "" {
		port = "3000"
	}

	binary, err := exec.LookPath("node")
	checkError(err)
	cmd := exec.Command(binary, "--expose_gc", "index.js")
	env := os.Environ()
	env = append([]string{"PORT=3001"}, env...)
	cmd.Env = env
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	err = cmd.Start()
	checkError(err)
	defer func() {
		err = cmd.Wait()
		checkError(err)
	}()

	proxy := goproxy.NewProxyHttpServer()
	proxy.Verbose = true

	proxy.NonproxyHandler = http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		req.Host = "bower.herokuapp.com"
		req.URL.Scheme = "http"
		req.URL.Host = "localhost:3001"
		proxy.ServeHTTP(w, req)
	})

	proxy.OnRequest(UrlHasPrefix("/packages/")).DoFunc(
		func(r *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
			var name string
			var url string
			elements := strings.Split(r.URL.Path, "/")
			packageName := elements[len(elements)-1]
			err := pool.QueryRow("getPackage", packageName).Scan(&name, &url)
			switch err {
			case nil:
				result := map[string]string{"name": name, "url": url}
				resultByteArray, _ := json.Marshal(result)
				return r, goproxy.NewResponse(r, "application/json", http.StatusOK, string(resultByteArray))
			case pgx.ErrNoRows:
				return r, goproxy.NewResponse(r, "text/html", http.StatusNotFound, "Package not found")
			default:
				return r, goproxy.NewResponse(r, "text/html", http.StatusInternalServerError, "Internal server error")
			}
		})

	fmt.Println("Starting web server at port", port)

	log.Fatal(http.ListenAndServe(":"+port, proxy))
}
<commit_msg>Use ffjson instead of json<commit_after>package main

import (
	"fmt"
	"github.com/elazarl/goproxy"
	"github.com/jackc/pgx"
	"github.com/pquerna/ffjson/ffjson"
	"log"
	"net/http"
	"os"
	"os/exec"
	"strings"
)

func checkError(err error) {
	if err != nil {
		log.Fatalf("Error: %s", err)
	}
}

func UrlHasPrefix(prefix string) goproxy.ReqConditionFunc {
	return func(req *http.Request, ctx *goproxy.ProxyCtx) bool {
		return req.Method == http.MethodGet && strings.HasPrefix(req.URL.Path, prefix)
	}
}

var pool *pgx.ConnPool

func afterConnect(conn *pgx.Conn) (err error) {
	_, err = conn.Prepare("getPackage", `
		SELECT name, url FROM packages WHERE name = $1
  `)
	if err != nil {
		return
	}

	return
}

func main() {

	var err error
	pgxcfg, err := pgx.ParseURI(os.Getenv("DATABASE_URL"))
	if err != nil {
		log.Fatalf("Parse URI error: %s", err)
	}
	pool, err = pgx.NewConnPool(pgx.ConnPoolConfig{
		ConnConfig:     pgxcfg,
		MaxConnections: 10,
		AfterConnect:   afterConnect,
	})
	if err != nil {
		log.Fatalf("Connection error: %s", err)
	}
	defer pool.Close()

	port := os.Getenv("PORT")

	if port == "" {
		port = "3000"
	}

	binary, err := exec.LookPath("node")
	checkError(err)
	cmd := exec.Command(binary, "--expose_gc", "index.js")
	env := os.Environ()
	env = append([]string{"PORT=3001"}, env...)
	cmd.Env = env
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	err = cmd.Start()
	checkError(err)
	defer func() {
		err = cmd.Wait()
		checkError(err)
	}()

	proxy := goproxy.NewProxyHttpServer()
	proxy.Verbose = true

	proxy.NonproxyHandler = http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		req.Host = "bower.herokuapp.com"
		req.URL.Scheme = "http"
		req.URL.Host = "localhost:3001"
		proxy.ServeHTTP(w, req)
	})

	proxy.OnRequest(UrlHasPrefix("/packages/")).DoFunc(
		func(r *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
			var name string
			var url string
			elements := strings.Split(r.URL.Path, "/")
			packageName := elements[len(elements)-1]
			err := pool.QueryRow("getPackage", packageName).Scan(&name, &url)
			switch err {
			case nil:
				result := map[string]string{"name": name, "url": url}
				resultByteArray, _ := ffjson.Marshal(result)
				return r, goproxy.NewResponse(r, "application/json", http.StatusOK, string(resultByteArray))
			case pgx.ErrNoRows:
				return r, goproxy.NewResponse(r, "text/html", http.StatusNotFound, "Package not found")
			default:
				return r, goproxy.NewResponse(r, "text/html", http.StatusInternalServerError, "Internal server error")
			}
		})

	fmt.Println("Starting web server at port", port)

	log.Fatal(http.ListenAndServe(":"+port, proxy))
}
<|endoftext|>


<commit_before>package qpid

import (
	"math"
	"time"
)

// Location of where something is
type Location int

// TODO: Need other location constants, think more here
const (
	_ Location = iota
	Food
	Top
	Bottom
	Outside
	Inside
)

// Temp is a temperature in Celcius
type Temp int

// C returns the temperature in Celcius
func (t *Temp) C() int {
	return t
}

// F returns the temperature in Fahrenheit
func (t *Temp) F() int {
	return math.Floor(t*9/5) + 32
}

// A Grill represents the cooking chamber of a BBQ cooker.
// It can contain one or more Sensors which report the current
// temperature.
type Grill struct {
	GrillSensors  []Sensor
	MeatSensors   []Sensor
	FoodMonitors  []Monitor
	GrillMonitors []Monitor
	Reporter      GrillReporter
}

// A Probe measures a temperature at a unique location.
type Probe struct {
	ID          int // Unique ID
	Location    Location
	Description string
	Temperature Temp // Current Temperature
}

// GrillStatus reports the temperatures of the Grill and Food probes at
// a point in time
type GrillStatus struct {
	Time              time.Time
	GrillTemperatures []Probe
	FoodTemperatures  []Probe
}

// Sensor is the interface to retrieve the current temperature
// of a probe
type Sensor interface {
	TemperatureF() (int, error)
	TemperatureC() (int, error)
	Location() Location
	Description() string
}

// Sourcer provides a string representing the source of an Alert
type Sourcer interface {
	Source() string
}

// An Alert is triggered when a Threshold is reached.
type Alert struct {
	Time    time.Time
	Message string
	Source  Sourcer
}

// A Thresholder watches a probe for high and low values,
// firing an Alert on the channel returned by Alerts()
type Thresholder interface {
	HighThreshold(Temp) error
	LowThreshold(Temp) error
	Alerts() chan<- Alert
}

// A Targeter sets the desired temperature for a device.
type Targeter interface {
	// Target sets the desired temperature, returns current and/or error
	Target(int) (int, error)
	// Setpoint() returns the currently set desired temperature
	Setpoint() (int, error)
}

type Monitor interface {
	Targeter
	Thresholder
}

// A CookController manages the cook.  A new cook is started by
// calling Run()
type CookController interface {
	FoodMonitors() []Monitor
	GrillMonitor() Monitor
	Run() error
}

// GrillReporter outputs metrics from a Grill
type GrillReporter interface {
	Status() (GrillStatus, error)
}
<commit_msg>missed that temp<commit_after>package qpid

import (
	"math"
	"time"
)

// Location of where something is
type Location int

// TODO: Need other location constants, think more here
const (
	_ Location = iota
	Food
	Top
	Bottom
	Outside
	Inside
)

// Temp is a temperature in Celcius
type Temp int

// C returns the temperature in Celcius
func (t *Temp) C() int {
	return t
}

// F returns the temperature in Fahrenheit
func (t *Temp) F() int {
	return math.Floor(t*9/5) + 32
}

// A Grill represents the cooking chamber of a BBQ cooker.
// It can contain one or more Sensors which report the current
// temperature.
type Grill struct {
	GrillSensors  []Sensor
	MeatSensors   []Sensor
	FoodMonitors  []Monitor
	GrillMonitors []Monitor
	Reporter      GrillReporter
}

// A Probe measures a temperature at a unique location.
type Probe struct {
	ID          int // Unique ID
	Location    Location
	Description string
	Temperature Temp // Current Temperature
}

// GrillStatus reports the temperatures of the Grill and Food probes at
// a point in time
type GrillStatus struct {
	Time              time.Time
	GrillTemperatures []Probe
	FoodTemperatures  []Probe
}

// Sensor is the interface to retrieve the current temperature
// of a probe
type Sensor interface {
	Temperature() (Temp, error)
	Location() Location
	Description() string
}

// Sourcer provides a string representing the source of an Alert
type Sourcer interface {
	Source() string
}

// An Alert is triggered when a Threshold is reached.
type Alert struct {
	Time    time.Time
	Message string
	Source  Sourcer
}

// A Thresholder watches a probe for high and low values,
// firing an Alert on the channel returned by Alerts()
type Thresholder interface {
	HighThreshold(Temp) error
	LowThreshold(Temp) error
	Alerts() chan<- Alert
}

// A Targeter sets the desired temperature for a device.
type Targeter interface {
	// Target sets the desired temperature, returns current and/or error
	Target(int) (int, error)
	// Setpoint() returns the currently set desired temperature
	Setpoint() (int, error)
}

type Monitor interface {
	Targeter
	Thresholder
}

// A CookController manages the cook.  A new cook is started by
// calling Run()
type CookController interface {
	FoodMonitors() []Monitor
	GrillMonitor() Monitor
	Run() error
}

// GrillReporter outputs metrics from a Grill
type GrillReporter interface {
	Status() (GrillStatus, error)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"strings"
	"time"

	"github.com/aktau/github-release/github"
	"github.com/dustin/go-humanize"
)

const (
	RELEASE_LIST_URI    = "/repos/%s/%s/releases"
	RELEASE_BY_TAG_URI  = "/repos/%s/%s/releases/tags/%s" // /repos/:owner/:repo/releases/tags/:tag
	RELEASE_LATEST_URI  = "/repos/%s/%s/releases/latest"
	RELEASE_DATE_FORMAT = "02/01/2006 at 15:04"
)

type Release struct {
	Url         string     `json:"url"`
	PageUrl     string     `json:"html_url"`
	UploadUrl   string     `json:"upload_url"`
	Id          int        `json:"id"`
	Name        string     `json:"name"`
	Description string     `json:"body"`
	TagName     string     `json:"tag_name"`
	Draft       bool       `json:"draft"`
	Prerelease  bool       `json:"prerelease"`
	Created     *time.Time `json:"created_at"`
	Published   *time.Time `json:"published_at"`
	Assets      []Asset    `json:"assets"`
}

func (r *Release) CleanUploadUrl() string {
	bracket := strings.Index(r.UploadUrl, "{")

	if bracket == -1 {
		return r.UploadUrl
	}

	return r.UploadUrl[0:bracket]
}

func (r *Release) String() string {
	str := make([]string, len(r.Assets)+1)
	str[0] = fmt.Sprintf(
		"%s, name: '%s', description: '%s', id: %d, tagged: %s, published: %s, draft: %v, prerelease: %v",
		r.TagName, r.Name, r.Description, r.Id,
		timeFmtOr(r.Created, RELEASE_DATE_FORMAT, ""),
		timeFmtOr(r.Published, RELEASE_DATE_FORMAT, ""),
		Mark(r.Draft), Mark(r.Prerelease))

	for idx, asset := range r.Assets {
		str[idx+1] = fmt.Sprintf("  - artifact: %s, downloads: %d, state: %s, type: %s, size: %s, id: %d",
			asset.Name, asset.Downloads, asset.State, asset.ContentType,
			humanize.Bytes(asset.Size), asset.Id)
	}

	return strings.Join(str, "\n")
}

type ReleaseCreate struct {
	TagName         string `json:"tag_name"`
	TargetCommitish string `json:"target_commitish,omitempty"`
	Name            string `json:"name"`
	Body            string `json:"body"`
	Draft           bool   `json:"draft"`
	Prerelease      bool   `json:"prerelease"`
}

func Releases(user, repo, token string) ([]Release, error) {
	var releases []Release
	err := github.Client{Token: token, BaseURL: EnvApiEndpoint}.Get(fmt.Sprintf(RELEASE_LIST_URI, user, repo), &releases)
	if err != nil {
		return nil, err
	}

	return releases, nil
}

func latestReleaseApi(user, repo, token string) (*Release, error) {
	var release Release
	return &release, github.Client{Token: token, BaseURL: EnvApiEndpoint}.Get(fmt.Sprintf(RELEASE_LATEST_URI, user, repo), &release)
}

func LatestRelease(user, repo, token string) (*Release, error) {
	// If latestReleaseApi DOESN'T give an error, return the release.
	if latestRelease, err := latestReleaseApi(user, repo, token); err == nil {
		return latestRelease, nil
	}

	// The enterprise api doesnt support the latest release endpoint. Get
	// all releases and compare the published date to get the latest.
	releases, err := Releases(user, repo, token)
	if err != nil {
		return nil, err
	}

	var latestRelIndex = -1
	maxDate := time.Time{}
	for i, release := range releases {
		if relDate := *release.Published; relDate.After(maxDate) {
			maxDate = relDate
			latestRelIndex = i
		}
	}
	if latestRelIndex == -1 {
		return nil, fmt.Errorf("could not find the latest release")
	}

	vprintln("Scanning ", len(releases), "releases, latest release is", releases[latestRelIndex])
	return &releases[latestRelIndex], nil
}

// ReleaseOfTag returns the Github release corresponding to a git tag, if it
// exists.
func ReleaseOfTag(user, repo, tag, token string) (*Release, error) {
	var release Release
	return &release, github.Client{Token: token, BaseURL: EnvApiEndpoint}.Get(fmt.Sprintf(RELEASE_BY_TAG_URI, user, repo, tag), &release)
}

/* find the release-id of the specified tag */
func IdOfTag(user, repo, tag, token string) (int, error) {
	release, err := ReleaseOfTag(user, repo, tag, token)
	if err != nil {
		return 0, err
	}

	return release.Id, nil
}
<commit_msg>Revert "releases: use specific release-of-tag endpoint"<commit_after>package main

import (
	"fmt"
	"strings"
	"time"

	"github.com/aktau/github-release/github"
	"github.com/dustin/go-humanize"
)

const (
	RELEASE_LIST_URI    = "/repos/%s/%s/releases"
	RELEASE_LATEST_URI  = "/repos/%s/%s/releases/latest"
	RELEASE_DATE_FORMAT = "02/01/2006 at 15:04"
)

type Release struct {
	Url         string     `json:"url"`
	PageUrl     string     `json:"html_url"`
	UploadUrl   string     `json:"upload_url"`
	Id          int        `json:"id"`
	Name        string     `json:"name"`
	Description string     `json:"body"`
	TagName     string     `json:"tag_name"`
	Draft       bool       `json:"draft"`
	Prerelease  bool       `json:"prerelease"`
	Created     *time.Time `json:"created_at"`
	Published   *time.Time `json:"published_at"`
	Assets      []Asset    `json:"assets"`
}

func (r *Release) CleanUploadUrl() string {
	bracket := strings.Index(r.UploadUrl, "{")

	if bracket == -1 {
		return r.UploadUrl
	}

	return r.UploadUrl[0:bracket]
}

func (r *Release) String() string {
	str := make([]string, len(r.Assets)+1)
	str[0] = fmt.Sprintf(
		"%s, name: '%s', description: '%s', id: %d, tagged: %s, published: %s, draft: %v, prerelease: %v",
		r.TagName, r.Name, r.Description, r.Id,
		timeFmtOr(r.Created, RELEASE_DATE_FORMAT, ""),
		timeFmtOr(r.Published, RELEASE_DATE_FORMAT, ""),
		Mark(r.Draft), Mark(r.Prerelease))

	for idx, asset := range r.Assets {
		str[idx+1] = fmt.Sprintf("  - artifact: %s, downloads: %d, state: %s, type: %s, size: %s, id: %d",
			asset.Name, asset.Downloads, asset.State, asset.ContentType,
			humanize.Bytes(asset.Size), asset.Id)
	}

	return strings.Join(str, "\n")
}

type ReleaseCreate struct {
	TagName         string `json:"tag_name"`
	TargetCommitish string `json:"target_commitish,omitempty"`
	Name            string `json:"name"`
	Body            string `json:"body"`
	Draft           bool   `json:"draft"`
	Prerelease      bool   `json:"prerelease"`
}

func Releases(user, repo, token string) ([]Release, error) {
	var releases []Release
	err := github.Client{Token: token, BaseURL: EnvApiEndpoint}.Get(fmt.Sprintf(RELEASE_LIST_URI, user, repo), &releases)
	if err != nil {
		return nil, err
	}

	return releases, nil
}

func latestReleaseApi(user, repo, token string) (*Release, error) {
	var release Release
	return &release, github.Client{Token: token, BaseURL: EnvApiEndpoint}.Get(fmt.Sprintf(RELEASE_LATEST_URI, user, repo), &release)
}

func LatestRelease(user, repo, token string) (*Release, error) {
	// If latestReleaseApi DOESN'T give an error, return the release.
	if latestRelease, err := latestReleaseApi(user, repo, token); err == nil {
		return latestRelease, nil
	}

	// The enterprise api doesnt support the latest release endpoint. Get
	// all releases and compare the published date to get the latest.
	releases, err := Releases(user, repo, token)
	if err != nil {
		return nil, err
	}

	var latestRelIndex = -1
	maxDate := time.Time{}
	for i, release := range releases {
		if relDate := *release.Published; relDate.After(maxDate) {
			maxDate = relDate
			latestRelIndex = i
		}
	}
	if latestRelIndex == -1 {
		return nil, fmt.Errorf("could not find the latest release")
	}

	vprintln("Scanning ", len(releases), "releases, latest release is", releases[latestRelIndex])
	return &releases[latestRelIndex], nil
}

func ReleaseOfTag(user, repo, tag, token string) (*Release, error) {
	releases, err := Releases(user, repo, token)
	if err != nil {
		return nil, err
	}

	for _, release := range releases {
		if release.TagName == tag {
			return &release, nil
		}
	}

	return nil, fmt.Errorf("could not find the release corresponding to tag %s", tag)
}

/* find the release-id of the specified tag */
func IdOfTag(user, repo, tag, token string) (int, error) {
	release, err := ReleaseOfTag(user, repo, tag, token)
	if err != nil {
		return 0, err
	}

	return release.Id, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Matthew Holt and The Caddy Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package caddy

import (
	"fmt"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"time"
)

// NewReplacer returns a new Replacer.
func NewReplacer() *Replacer {
	rep := &Replacer{
		static: make(map[string]interface{}),
	}
	rep.providers = []ReplacerFunc{
		globalDefaultReplacements,
		rep.fromStatic,
	}
	return rep
}

// NewEmptyReplacer returns a new Replacer,
// without the global default replacements.
func NewEmptyReplacer() *Replacer {
	rep := &Replacer{
		static: make(map[string]interface{}),
	}
	rep.providers = []ReplacerFunc{
		rep.fromStatic,
	}
	return rep
}

// Replacer can replace values in strings.
// A default/empty Replacer is not valid;
// use NewReplacer to make one.
type Replacer struct {
	providers []ReplacerFunc
	static    map[string]interface{}
}

// Map adds mapFunc to the list of value providers.
// mapFunc will be executed only at replace-time.
func (r *Replacer) Map(mapFunc ReplacerFunc) {
	r.providers = append(r.providers, mapFunc)
}

// Set sets a custom variable to a static value.
func (r *Replacer) Set(variable string, value interface{}) {
	r.static[variable] = value
}

// Get gets a value from the replacer. It returns
// the value and whether the variable was known.
func (r *Replacer) Get(variable string) (interface{}, bool) {
	for _, mapFunc := range r.providers {
		if val, ok := mapFunc(variable); ok {
			return val, true
		}
	}
	return nil, false
}

// GetString  is the same as Get, but coerces the value to a
// string representation.
func (r *Replacer) GetString(variable string) (string, bool) {
	s, found := r.Get(variable)
	return toString(s), found
}

// Delete removes a variable with a static value
// that was created using Set.
func (r *Replacer) Delete(variable string) {
	delete(r.static, variable)
}

// fromStatic provides values from r.static.
func (r *Replacer) fromStatic(key string) (interface{}, bool) {
	val, ok := r.static[key]
	return val, ok
}

// ReplaceOrErr is like ReplaceAll, but any placeholders
// that are empty or not recognized will cause an error to
// be returned.
func (r *Replacer) ReplaceOrErr(input string, errOnEmpty, errOnUnknown bool) (string, error) {
	return r.replace(input, "", false, errOnEmpty, errOnUnknown, nil)
}

// ReplaceKnown is like ReplaceAll but only replaces
// placeholders that are known (recognized). Unrecognized
// placeholders will remain in the output.
func (r *Replacer) ReplaceKnown(input, empty string) string {
	out, _ := r.replace(input, empty, false, false, false, nil)
	return out
}

// ReplaceAll efficiently replaces placeholders in input with
// their values. All placeholders are replaced in the output
// whether they are recognized or not. Values that are empty
// string will be substituted with empty.
func (r *Replacer) ReplaceAll(input, empty string) string {
	out, _ := r.replace(input, empty, true, false, false, nil)
	return out
}

// ReplaceFunc is the same as ReplaceAll, but calls f for every
// replacement to be made, in case f wants to change or inspect
// the replacement.
func (r *Replacer) ReplaceFunc(input string, f ReplacementFunc) (string, error) {
	return r.replace(input, "", true, false, false, f)
}

func (r *Replacer) replace(input, empty string,
	treatUnknownAsEmpty, errOnEmpty, errOnUnknown bool,
	f ReplacementFunc) (string, error) {
	if !strings.Contains(input, string(phOpen)) {
		return input, nil
	}

	var sb strings.Builder

	// it is reasonable to assume that the output
	// will be approximately as long as the input
	sb.Grow(len(input))

	// iterate the input to find each placeholder
	var lastWriteCursor int

scan:
	for i := 0; i < len(input); i++ {

		// check for escaped braces
		if i > 0 && input[i-1] == phEscape && (input[i] == phClose || input[i] == phOpen) {
			sb.WriteString(input[lastWriteCursor : i-1])
			lastWriteCursor = i
			continue
		}

		if input[i] != phOpen {
			continue
		}

		// find the end of the placeholder
		end := strings.Index(input[i:], string(phClose)) + i
		if end < i {
			continue
		}

		// if necessary look for the first closing brace that is not escaped
		for end > 0 && end < len(input)-1 && input[end-1] == phEscape {
			nextEnd := strings.Index(input[end+1:], string(phClose))
			if nextEnd < 0 {
				continue scan
			}
			end += nextEnd + 1
		}

		// write the substring from the last cursor to this point
		sb.WriteString(input[lastWriteCursor:i])

		// trim opening bracket
		key := input[i+1 : end]

		// try to get a value for this key, handle empty values accordingly
		val, found := r.Get(key)
		if !found {
			// placeholder is unknown (unrecognized); handle accordingly
			if errOnUnknown {
				return "", fmt.Errorf("unrecognized placeholder %s%s%s",
					string(phOpen), key, string(phClose))
			} else if !treatUnknownAsEmpty {
				// if treatUnknownAsEmpty is true, we'll handle an empty
				// val later; so only continue otherwise
				lastWriteCursor = i
				continue
			}
		}

		// apply any transformations
		if f != nil {
			var err error
			val, err = f(key, val)
			if err != nil {
				return "", err
			}
		}

		// convert val to a string as efficiently as possible
		valStr := toString(val)

		// write the value; if it's empty, either return
		// an error or write a default value
		if valStr == "" {
			if errOnEmpty {
				return "", fmt.Errorf("evaluated placeholder %s%s%s is empty",
					string(phOpen), key, string(phClose))
			} else if empty != "" {
				sb.WriteString(empty)
			}
		} else {
			sb.WriteString(valStr)
		}

		// advance cursor to end of placeholder
		i = end
		lastWriteCursor = i + 1
	}

	// flush any unwritten remainder
	sb.WriteString(input[lastWriteCursor:])

	return sb.String(), nil
}

func toString(val interface{}) string {
	switch v := val.(type) {
	case nil:
		return ""
	case string:
		return v
	case fmt.Stringer:
		return v.String()
	case byte:
		return string(v)
	case []byte:
		return string(v)
	case []rune:
		return string(v)
	case int:
		return strconv.Itoa(v)
	case int32:
		return strconv.Itoa(int(v))
	case int64:
		return strconv.Itoa(int(v))
	case uint:
		return strconv.Itoa(int(v))
	case uint32:
		return strconv.Itoa(int(v))
	case uint64:
		return strconv.Itoa(int(v))
	case float32:
		return strconv.FormatFloat(float64(v), 'f', -1, 32)
	case float64:
		return strconv.FormatFloat(v, 'f', -1, 64)
	case bool:
		if v {
			return "true"
		}
		return "false"
	default:
		return fmt.Sprintf("%+v", v)
	}
}

// ReplacerFunc is a function that returns a replacement
// for the given key along with true if the function is able
// to service that key (even if the value is blank). If the
// function does not recognize the key, false should be
// returned.
type ReplacerFunc func(key string) (interface{}, bool)

func globalDefaultReplacements(key string) (interface{}, bool) {
	// check environment variable
	const envPrefix = "env."
	if strings.HasPrefix(key, envPrefix) {
		return os.Getenv(key[len(envPrefix):]), true
	}

	switch key {
	case "system.hostname":
		// OK if there is an error; just return empty string
		name, _ := os.Hostname()
		return name, true
	case "system.slash":
		return string(filepath.Separator), true
	case "system.os":
		return runtime.GOOS, true
	case "system.arch":
		return runtime.GOARCH, true
	case "time.now":
		return nowFunc(), true
	case "time.now.common_log":
		return nowFunc().Format("02/Jan/2006:15:04:05 -0700"), true
	case "time.now.year":
		return strconv.Itoa(nowFunc().Year()), true
	}

	return nil, false
}

// ReplacementFunc is a function that is called when a
// replacement is being performed. It receives the
// variable (i.e. placeholder name) and the value that
// will be the replacement, and returns the value that
// will actually be the replacement, or an error. Note
// that errors are sometimes ignored by replacers.
type ReplacementFunc func(variable string, val interface{}) (interface{}, error)

// nowFunc is a variable so tests can change it
// in order to obtain a deterministic time.
var nowFunc = time.Now

// ReplacerCtxKey is the context key for a replacer.
const ReplacerCtxKey CtxKey = "replacer"

const phOpen, phClose, phEscape = '{', '}', '\\'
<commit_msg>core: Unix ns and Unix ms time placeholders (#4280)<commit_after>// Copyright 2015 Matthew Holt and The Caddy Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package caddy

import (
	"fmt"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"time"
)

// NewReplacer returns a new Replacer.
func NewReplacer() *Replacer {
	rep := &Replacer{
		static: make(map[string]interface{}),
	}
	rep.providers = []ReplacerFunc{
		globalDefaultReplacements,
		rep.fromStatic,
	}
	return rep
}

// NewEmptyReplacer returns a new Replacer,
// without the global default replacements.
func NewEmptyReplacer() *Replacer {
	rep := &Replacer{
		static: make(map[string]interface{}),
	}
	rep.providers = []ReplacerFunc{
		rep.fromStatic,
	}
	return rep
}

// Replacer can replace values in strings.
// A default/empty Replacer is not valid;
// use NewReplacer to make one.
type Replacer struct {
	providers []ReplacerFunc
	static    map[string]interface{}
}

// Map adds mapFunc to the list of value providers.
// mapFunc will be executed only at replace-time.
func (r *Replacer) Map(mapFunc ReplacerFunc) {
	r.providers = append(r.providers, mapFunc)
}

// Set sets a custom variable to a static value.
func (r *Replacer) Set(variable string, value interface{}) {
	r.static[variable] = value
}

// Get gets a value from the replacer. It returns
// the value and whether the variable was known.
func (r *Replacer) Get(variable string) (interface{}, bool) {
	for _, mapFunc := range r.providers {
		if val, ok := mapFunc(variable); ok {
			return val, true
		}
	}
	return nil, false
}

// GetString  is the same as Get, but coerces the value to a
// string representation.
func (r *Replacer) GetString(variable string) (string, bool) {
	s, found := r.Get(variable)
	return toString(s), found
}

// Delete removes a variable with a static value
// that was created using Set.
func (r *Replacer) Delete(variable string) {
	delete(r.static, variable)
}

// fromStatic provides values from r.static.
func (r *Replacer) fromStatic(key string) (interface{}, bool) {
	val, ok := r.static[key]
	return val, ok
}

// ReplaceOrErr is like ReplaceAll, but any placeholders
// that are empty or not recognized will cause an error to
// be returned.
func (r *Replacer) ReplaceOrErr(input string, errOnEmpty, errOnUnknown bool) (string, error) {
	return r.replace(input, "", false, errOnEmpty, errOnUnknown, nil)
}

// ReplaceKnown is like ReplaceAll but only replaces
// placeholders that are known (recognized). Unrecognized
// placeholders will remain in the output.
func (r *Replacer) ReplaceKnown(input, empty string) string {
	out, _ := r.replace(input, empty, false, false, false, nil)
	return out
}

// ReplaceAll efficiently replaces placeholders in input with
// their values. All placeholders are replaced in the output
// whether they are recognized or not. Values that are empty
// string will be substituted with empty.
func (r *Replacer) ReplaceAll(input, empty string) string {
	out, _ := r.replace(input, empty, true, false, false, nil)
	return out
}

// ReplaceFunc is the same as ReplaceAll, but calls f for every
// replacement to be made, in case f wants to change or inspect
// the replacement.
func (r *Replacer) ReplaceFunc(input string, f ReplacementFunc) (string, error) {
	return r.replace(input, "", true, false, false, f)
}

func (r *Replacer) replace(input, empty string,
	treatUnknownAsEmpty, errOnEmpty, errOnUnknown bool,
	f ReplacementFunc) (string, error) {
	if !strings.Contains(input, string(phOpen)) {
		return input, nil
	}

	var sb strings.Builder

	// it is reasonable to assume that the output
	// will be approximately as long as the input
	sb.Grow(len(input))

	// iterate the input to find each placeholder
	var lastWriteCursor int

scan:
	for i := 0; i < len(input); i++ {

		// check for escaped braces
		if i > 0 && input[i-1] == phEscape && (input[i] == phClose || input[i] == phOpen) {
			sb.WriteString(input[lastWriteCursor : i-1])
			lastWriteCursor = i
			continue
		}

		if input[i] != phOpen {
			continue
		}

		// find the end of the placeholder
		end := strings.Index(input[i:], string(phClose)) + i
		if end < i {
			continue
		}

		// if necessary look for the first closing brace that is not escaped
		for end > 0 && end < len(input)-1 && input[end-1] == phEscape {
			nextEnd := strings.Index(input[end+1:], string(phClose))
			if nextEnd < 0 {
				continue scan
			}
			end += nextEnd + 1
		}

		// write the substring from the last cursor to this point
		sb.WriteString(input[lastWriteCursor:i])

		// trim opening bracket
		key := input[i+1 : end]

		// try to get a value for this key, handle empty values accordingly
		val, found := r.Get(key)
		if !found {
			// placeholder is unknown (unrecognized); handle accordingly
			if errOnUnknown {
				return "", fmt.Errorf("unrecognized placeholder %s%s%s",
					string(phOpen), key, string(phClose))
			} else if !treatUnknownAsEmpty {
				// if treatUnknownAsEmpty is true, we'll handle an empty
				// val later; so only continue otherwise
				lastWriteCursor = i
				continue
			}
		}

		// apply any transformations
		if f != nil {
			var err error
			val, err = f(key, val)
			if err != nil {
				return "", err
			}
		}

		// convert val to a string as efficiently as possible
		valStr := toString(val)

		// write the value; if it's empty, either return
		// an error or write a default value
		if valStr == "" {
			if errOnEmpty {
				return "", fmt.Errorf("evaluated placeholder %s%s%s is empty",
					string(phOpen), key, string(phClose))
			} else if empty != "" {
				sb.WriteString(empty)
			}
		} else {
			sb.WriteString(valStr)
		}

		// advance cursor to end of placeholder
		i = end
		lastWriteCursor = i + 1
	}

	// flush any unwritten remainder
	sb.WriteString(input[lastWriteCursor:])

	return sb.String(), nil
}

func toString(val interface{}) string {
	switch v := val.(type) {
	case nil:
		return ""
	case string:
		return v
	case fmt.Stringer:
		return v.String()
	case byte:
		return string(v)
	case []byte:
		return string(v)
	case []rune:
		return string(v)
	case int:
		return strconv.Itoa(v)
	case int32:
		return strconv.Itoa(int(v))
	case int64:
		return strconv.Itoa(int(v))
	case uint:
		return strconv.Itoa(int(v))
	case uint32:
		return strconv.Itoa(int(v))
	case uint64:
		return strconv.Itoa(int(v))
	case float32:
		return strconv.FormatFloat(float64(v), 'f', -1, 32)
	case float64:
		return strconv.FormatFloat(v, 'f', -1, 64)
	case bool:
		if v {
			return "true"
		}
		return "false"
	default:
		return fmt.Sprintf("%+v", v)
	}
}

// ReplacerFunc is a function that returns a replacement
// for the given key along with true if the function is able
// to service that key (even if the value is blank). If the
// function does not recognize the key, false should be
// returned.
type ReplacerFunc func(key string) (interface{}, bool)

func globalDefaultReplacements(key string) (interface{}, bool) {
	// check environment variable
	const envPrefix = "env."
	if strings.HasPrefix(key, envPrefix) {
		return os.Getenv(key[len(envPrefix):]), true
	}

	switch key {
	case "system.hostname":
		// OK if there is an error; just return empty string
		name, _ := os.Hostname()
		return name, true
	case "system.slash":
		return string(filepath.Separator), true
	case "system.os":
		return runtime.GOOS, true
	case "system.arch":
		return runtime.GOARCH, true
	case "time.now":
		return nowFunc(), true
	case "time.now.common_log":
		return nowFunc().Format("02/Jan/2006:15:04:05 -0700"), true
	case "time.now.year":
		return strconv.Itoa(nowFunc().Year()), true
	case "time.now.unix":
		return strconv.FormatInt(nowFunc().Unix(), 10), true
	case "time.now.unix_ms":
		return strconv.FormatInt(nowFunc().UnixNano()/int64(time.Millisecond), 10), true
	}

	return nil, false
}

// ReplacementFunc is a function that is called when a
// replacement is being performed. It receives the
// variable (i.e. placeholder name) and the value that
// will be the replacement, and returns the value that
// will actually be the replacement, or an error. Note
// that errors are sometimes ignored by replacers.
type ReplacementFunc func(variable string, val interface{}) (interface{}, error)

// nowFunc is a variable so tests can change it
// in order to obtain a deterministic time.
var nowFunc = time.Now

// ReplacerCtxKey is the context key for a replacer.
const ReplacerCtxKey CtxKey = "replacer"

const phOpen, phClose, phEscape = '{', '}', '\\'
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"log"
	"net/url"
	"os"
	"os/signal"
	"regexp"
	"strings"
	"sync"
	"syscall"

	"gopkg.in/gorp.v1"

	"github.com/ChimeraCoder/anaconda"
	"github.com/robfig/cron"
)

const (
	SearchWord = "VoteSanrio"
)

var (
	pattern       = regexp.MustCompile(`サンリオキャラクター大賞で(.+?)に投票したよ`)
	spaceReplacer = strings.NewReplacer("　", " ")
)

type Crawler struct{}

func (c *Crawler) Help() string {
	return `sanrio-character-ranking-viewer crawler`
}

func (c *Crawler) Run(args []string) int {
	context, err := newContext()
	if err != nil {
		log.Fatal(err)
	}
	defer context.Close()

	anaconda.SetConsumerKey(os.Getenv("TWITTER_CONSUMER_KEY"))
	anaconda.SetConsumerSecret(os.Getenv("TWITTER_CONSUMER_SECRET"))

	api := anaconda.NewTwitterApi(os.Getenv("TWITTER_OAUTH_TOKEN"), os.Getenv("TWITTER_OAUTH_TOKEN_SECRET"))

	stream := api.PublicStreamFilter(url.Values{
		"track": []string{SearchWord},
	})

	waitGroup := &sync.WaitGroup{}

	statusChan := make(chan anaconda.Tweet)

	processorQuitChan := make(chan struct{})
	streamingQuitChan := make(chan struct{})
	quitChans := []chan struct{}{
		processorQuitChan,
		streamingQuitChan,
	}

	waitGroup.Add(1)
	go func() {
		defer waitGroup.Done()

		for {
			select {
			case status := <-statusChan:
				if err := processStatus(context.dbMap, status); err != nil {
					log.Print(err)
				}
				break
			case <-processorQuitChan:
				return
			}
		}
	}()

	waitGroup.Add(1)
	go func() {
		defer waitGroup.Done()

		for {
			select {
			case item := <-stream.C:
				switch status := item.(type) {
				case anaconda.Tweet:
					statusChan <- status
					break
				}
			case <-streamingQuitChan:
				return
			}
		}
	}()

	cron := cron.New()
	cron.AddFunc("0 */5 * * * *", func() {
		waitGroup.Add(1)
		defer waitGroup.Done()

		v := url.Values{}
		v.Set("count", "200")
		searchResult, err := api.GetSearch(SearchWord, v)
		if err != nil {
			log.Print(err)
			return
		}
		for _, status := range searchResult.Statuses {
			statusChan <- status
		}
	})
	cron.Start()

	signalChan := make(chan os.Signal)
	signal.Notify(signalChan, syscall.SIGTERM, syscall.SIGINT, syscall.SIGHUP, syscall.SIGQUIT)

	waitGroup.Add(1)
	go func() {
		defer waitGroup.Done()

		for {
			s := <-signalChan
			log.Print(s)
			for _, quitChan := range quitChans {
				quitChan <- struct{}{}
			}
			break
		}
	}()

	waitGroup.Wait()
	return 0
}

func (c *Crawler) Synopsis() string {
	return `Start crawler`
}

func processStatus(dbMap *gorp.DbMap, status anaconda.Tweet) error {
	createdAt, err := status.CreatedAtTime()
	if err != nil {
		return err
	}

	count, err := dbMap.SelectInt(`SELECT COUNT(*) FROM status WHERE id = $1`, status.Id)
	if err != nil {
		return err
	}
	if count != 0 {
		return nil
	}

	statusJson, err := json.Marshal(&status)
	if err != nil {
		return err
	}
	statusObject := Status{
		Id:        status.Id,
		CreatedAt: createdAt,
		Source:    string(statusJson),
	}
	if err := dbMap.Insert(&statusObject); err != nil {
		return err
	}

	submatches := pattern.FindStringSubmatch(status.Text)
	if submatches == nil {
		return nil
	}

	name := spaceReplacer.Replace(submatches[1])

	entry, err := FindEntryByName(dbMap, 2, name)
	if err != nil {
		return err
	}

	vote := Vote{
		RankingId:   entry.RankingId,
		CharacterId: entry.CharacterId,
		StatusId:    status.Id,
	}
	if err := dbMap.Insert(&vote); err != nil {
		return err
	}
	return nil
}
<commit_msg>Change match pattern for 31st ranking<commit_after>package main

import (
	"encoding/json"
	"log"
	"net/url"
	"os"
	"os/signal"
	"regexp"
	"strings"
	"sync"
	"syscall"

	"gopkg.in/gorp.v1"

	"github.com/ChimeraCoder/anaconda"
	"github.com/robfig/cron"
)

const (
	SearchWord = "サンリオキャラクター大賞"
)

var (
	pattern       = regexp.MustCompile(`(.+?)を、「(?:.+?)」なでました！みんなもなでてみよう！`)
	spaceReplacer = strings.NewReplacer("　", " ")
)

type Crawler struct{}

func (c *Crawler) Help() string {
	return `sanrio-character-ranking-viewer crawler`
}

func (c *Crawler) Run(args []string) int {
	context, err := newContext()
	if err != nil {
		log.Fatal(err)
	}
	defer context.Close()

	anaconda.SetConsumerKey(os.Getenv("TWITTER_CONSUMER_KEY"))
	anaconda.SetConsumerSecret(os.Getenv("TWITTER_CONSUMER_SECRET"))

	api := anaconda.NewTwitterApi(os.Getenv("TWITTER_OAUTH_TOKEN"), os.Getenv("TWITTER_OAUTH_TOKEN_SECRET"))

	stream := api.PublicStreamFilter(url.Values{
		"track": []string{SearchWord},
	})

	waitGroup := &sync.WaitGroup{}

	statusChan := make(chan anaconda.Tweet)

	processorQuitChan := make(chan struct{})
	streamingQuitChan := make(chan struct{})
	quitChans := []chan struct{}{
		processorQuitChan,
		streamingQuitChan,
	}

	waitGroup.Add(1)
	go func() {
		defer waitGroup.Done()

		for {
			select {
			case status := <-statusChan:
				if err := processStatus(context.dbMap, status); err != nil {
					log.Print(err)
				}
				break
			case <-processorQuitChan:
				return
			}
		}
	}()

	waitGroup.Add(1)
	go func() {
		defer waitGroup.Done()

		for {
			select {
			case item := <-stream.C:
				switch status := item.(type) {
				case anaconda.Tweet:
					statusChan <- status
					break
				}
			case <-streamingQuitChan:
				return
			}
		}
	}()

	cron := cron.New()
	cron.AddFunc("0 */5 * * * *", func() {
		waitGroup.Add(1)
		defer waitGroup.Done()

		v := url.Values{}
		v.Set("count", "200")
		searchResult, err := api.GetSearch(SearchWord, v)
		if err != nil {
			log.Print(err)
			return
		}
		for _, status := range searchResult.Statuses {
			statusChan <- status
		}
	})
	cron.Start()

	signalChan := make(chan os.Signal)
	signal.Notify(signalChan, syscall.SIGTERM, syscall.SIGINT, syscall.SIGHUP, syscall.SIGQUIT)

	waitGroup.Add(1)
	go func() {
		defer waitGroup.Done()

		for {
			s := <-signalChan
			log.Print(s)
			for _, quitChan := range quitChans {
				quitChan <- struct{}{}
			}
			break
		}
	}()

	waitGroup.Wait()
	return 0
}

func (c *Crawler) Synopsis() string {
	return `Start crawler`
}

func processStatus(dbMap *gorp.DbMap, status anaconda.Tweet) error {
	createdAt, err := status.CreatedAtTime()
	if err != nil {
		return err
	}

	count, err := dbMap.SelectInt(`SELECT COUNT(*) FROM status WHERE id = $1`, status.Id)
	if err != nil {
		return err
	}
	if count != 0 {
		return nil
	}

	statusJson, err := json.Marshal(&status)
	if err != nil {
		return err
	}
	statusObject := Status{
		Id:        status.Id,
		CreatedAt: createdAt,
		Source:    string(statusJson),
	}
	if err := dbMap.Insert(&statusObject); err != nil {
		return err
	}

	submatches := pattern.FindStringSubmatch(status.Text)
	if submatches == nil {
		return nil
	}

	name := spaceReplacer.Replace(submatches[1])

	entry, err := FindEntryByName(dbMap, 2, name)
	if err != nil {
		return err
	}

	vote := Vote{
		RankingId:   entry.RankingId,
		CharacterId: entry.CharacterId,
		StatusId:    status.Id,
	}
	if err := dbMap.Insert(&vote); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package runner

import (
	"bytes"
	"flag"
	"io"
	"log"
	"mime/multipart"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"time"
)

var (
	maxPendingReports = 64
	numPublishRetries = 8
	backoffTimeMs     = 1000
)

type ReportPayload struct {
	path     string
	data     map[string]string
	filename string
}

type Reporter struct {
	publishUri      string
	publishChannel  chan ReportPayload
	shutdownChannel chan bool
}

func httpPost(uri string, params map[string]string, file string) (resp *http.Response, err error) {
	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)

	for key, val := range params {
		err = writer.WriteField(key, val)
		if err != nil {
			log.Printf("[reporter] Couldn't write field %s", key)
			return nil, err
		}
	}

	if len(file) > 0 {
		handle, err := os.Open(file)
		if err != nil {
			return nil, err
		}

		err = writer.WriteField("name", filepath.Base(file))
		if err != nil {
			return nil, err
		}

		fileField, err := writer.CreateFormFile("file", filepath.Base(file))
		if err != nil {
			return nil, err
		}

		_, err = io.Copy(fileField, handle)
		if err != nil {
			return nil, err
		}
	}

	_ = writer.Close()
	return http.Post(uri, writer.FormDataContentType(), body)
}

func transportSend(r *Reporter) {
	for req := range r.publishChannel {
		path := r.publishUri + req.path
		if req.data == nil {
			req.data = make(map[string]string)
		}

		req.data["date"] = time.Now().UTC().Format("2006-01-02T15:04:05.0Z")
		for tryCnt := 1; tryCnt <= numPublishRetries; tryCnt++ {
			log.Printf("[reporter] POST %s try: %d", path, tryCnt)
			resp, err := httpPost(path, req.data, req.filename)

			if err != nil {
				defer resp.Body.Close()
			}

			status := "-1"
			if resp != nil {
				status = resp.Status
			}

			if resp != nil && resp.StatusCode/100 == 2 {
				break
			}

			if resp != nil && resp.StatusCode == http.StatusGone {
				// TODO(dcramer): this shouldn't really be a panic, but
				// we want to exit at this point
				panic("Unknown error occurred with publish endpoint")
			}

			log.Printf("[reporter] POST %s failed, try: %d, resp: %s, err: %s",
				path, tryCnt, status, err)

			/* We are unable to publish to the endpoint.
			 * Fail fast and let the above layers handle the outage */
			if tryCnt == numPublishRetries {
				panic("Couldn't to connect to publish endpoint")
			}
			log.Printf("[reporter] Sleep for %d ms", backoffTimeMs)
			time.Sleep(time.Duration(backoffTimeMs) * time.Millisecond)
		}
	}
	r.shutdownChannel <- true
}

func NewReporter(publishUri string) *Reporter {
	log.Printf("[reporter] Construct reporter with publish uri: %s", publishUri)
	r := &Reporter{}
	r.publishUri = publishUri
	r.publishChannel = make(chan ReportPayload, maxPendingReports)
	r.shutdownChannel = make(chan bool)

	go transportSend(r)
	return r
}

func (r *Reporter) PushJobStatus(jobID string, status string, result string) {
	form := make(map[string]string)
	form["status"] = status
	if len(result) > 0 {
		form["result"] = result
	}
	r.publishChannel <- ReportPayload{"/jobsteps/" + jobID + "/", form, ""}
}

func (r *Reporter) PushStatus(cId string, status string, retCode int) {
	form := make(map[string]string)
	form["status"] = status
	if retCode >= 0 {
		form["return_code"] = strconv.Itoa(retCode)
	}
	r.publishChannel <- ReportPayload{"/commands/" + cId + "/", form, ""}
}

func (r *Reporter) PushLogChunk(ID string, source string, offset int, payload []byte) {
	form := make(map[string]string)
	form["source"] = source
	form["offset"] = strconv.Itoa(offset)
	form["text"] = string(payload)
	r.publishChannel <- ReportPayload{"/jobsteps/" + ID + "/logappend/", form, ""}
}

func (r *Reporter) PushArtifacts(ID string, artifacts []string) {
	for _, artifact := range artifacts {
		r.publishChannel <- ReportPayload{"/jobsteps/" + ID + "/artifacts/", nil, artifact}
	}
}

func (r *Reporter) Shutdown() {
	close(r.publishChannel)
	<-r.shutdownChannel
	close(r.shutdownChannel)
	log.Print("[reporter] Shutdown complete")
}

func init() {
	flag.IntVar(&maxPendingReports, "max_pending_reports", 64, "Backlog size")
	flag.IntVar(&numPublishRetries, "num_publish_retries", 8,
		"Number of times to retry")
	flag.IntVar(&maxPendingReports, "backoff_time_ms", 1000,
		"Time to wait between two consecutive retries")
}
<commit_msg>Minor cleanup of http handling<commit_after>package runner

import (
	"bytes"
	"flag"
	"io"
	"log"
	"mime/multipart"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"time"
)

var (
	maxPendingReports = 64
	numPublishRetries = 8
	backoffTimeMs     = 1000
)

type ReportPayload struct {
	path     string
	data     map[string]string
	filename string
}

type Reporter struct {
	publishUri      string
	publishChannel  chan ReportPayload
	shutdownChannel chan bool
}

func httpPost(uri string, params map[string]string, file string) (resp *http.Response, err error) {
	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)

	for key, val := range params {
		err = writer.WriteField(key, val)
		if err != nil {
			log.Printf("[reporter] Couldn't write field %s", key)
			return nil, err
		}
	}

	if len(file) > 0 {
		handle, err := os.Open(file)
		if err != nil {
			return nil, err
		}

		err = writer.WriteField("name", filepath.Base(file))
		if err != nil {
			return nil, err
		}

		fileField, err := writer.CreateFormFile("file", filepath.Base(file))
		if err != nil {
			return nil, err
		}

		_, err = io.Copy(fileField, handle)
		if err != nil {
			return nil, err
		}
	}

	_ = writer.Close()

	resp, err = http.Post(uri, writer.FormDataContentType(), body)

	if err != nil {
		// Close the Body channel immediately as we don't use it
		// and this loop can stay open for an extremely long period
		// of time
		resp.Body.Close()
	}

	return resp, err
}

func transportSend(r *Reporter) {
	var resp *http.Response
	var err error
	var status string

	for req := range r.publishChannel {
		path := r.publishUri + req.path
		if req.data == nil {
			req.data = make(map[string]string)
		}

		req.data["date"] = time.Now().UTC().Format("2006-01-02T15:04:05.0Z")
		for tryCnt := 1; tryCnt <= numPublishRetries; tryCnt++ {
			log.Printf("[reporter] POST %s try: %d", path, tryCnt)
			resp, err = httpPost(path, req.data, req.filename)

			if resp != nil {
				status = resp.Status
			} else {
				status = "-1"
			}

			if resp != nil && resp.StatusCode/100 == 2 {
				break
			}

			if resp != nil && resp.StatusCode == http.StatusGone {
				// TODO(dcramer): this shouldn't really be a panic, but
				// we want to exit at this point
				panic("Unknown error occurred with publish endpoint")
			}

			log.Printf("[reporter] POST %s failed, try: %d, resp: %s, err: %s",
				path, tryCnt, status, err)

			/* We are unable to publish to the endpoint.
			 * Fail fast and let the above layers handle the outage */
			if tryCnt == numPublishRetries {
				panic("Couldn't to connect to publish endpoint")
			}
			log.Printf("[reporter] Sleep for %d ms", backoffTimeMs)
			time.Sleep(time.Duration(backoffTimeMs) * time.Millisecond)
		}
	}
	r.shutdownChannel <- true
}

func NewReporter(publishUri string) *Reporter {
	log.Printf("[reporter] Construct reporter with publish uri: %s", publishUri)
	r := &Reporter{}
	r.publishUri = publishUri
	r.publishChannel = make(chan ReportPayload, maxPendingReports)
	r.shutdownChannel = make(chan bool)

	go transportSend(r)
	return r
}

func (r *Reporter) PushJobStatus(jobID string, status string, result string) {
	form := make(map[string]string)
	form["status"] = status
	if len(result) > 0 {
		form["result"] = result
	}
	r.publishChannel <- ReportPayload{"/jobsteps/" + jobID + "/", form, ""}
}

func (r *Reporter) PushStatus(cId string, status string, retCode int) {
	form := make(map[string]string)
	form["status"] = status
	if retCode >= 0 {
		form["return_code"] = strconv.Itoa(retCode)
	}
	r.publishChannel <- ReportPayload{"/commands/" + cId + "/", form, ""}
}

func (r *Reporter) PushLogChunk(ID string, source string, offset int, payload []byte) {
	form := make(map[string]string)
	form["source"] = source
	form["offset"] = strconv.Itoa(offset)
	form["text"] = string(payload)
	r.publishChannel <- ReportPayload{"/jobsteps/" + ID + "/logappend/", form, ""}
}

func (r *Reporter) PushArtifacts(ID string, artifacts []string) {
	for _, artifact := range artifacts {
		r.publishChannel <- ReportPayload{"/jobsteps/" + ID + "/artifacts/", nil, artifact}
	}
}

func (r *Reporter) Shutdown() {
	close(r.publishChannel)
	<-r.shutdownChannel
	close(r.shutdownChannel)
	log.Print("[reporter] Shutdown complete")
}

func init() {
	flag.IntVar(&maxPendingReports, "max_pending_reports", 64, "Backlog size")
	flag.IntVar(&numPublishRetries, "num_publish_retries", 8,
		"Number of times to retry")
	flag.IntVar(&maxPendingReports, "backoff_time_ms", 1000,
		"Time to wait between two consecutive retries")
}
<|endoftext|>


<commit_before>// Copyright 2014-2015 The DevMine authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package repotool is able to fetch information from a source code repository.
// Typically, it can get all commits, their authors and commiters and so on
// and return this information in a JSON object. Alternatively, it is able
// to populate the information into a PostgreSQL database.
// Currently, on the Git VCS is supported.
package main

import (
	"bufio"
	"bytes"
	"database/sql"
	"encoding/hex"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	_ "github.com/lib/pq"
	mmh3 "github.com/spaolacci/murmur3"

	"github.com/DevMine/srcanlzr/src"

	"github.com/DevMine/repotool/config"
	"github.com/DevMine/repotool/model"
	"github.com/DevMine/repotool/repo"
)

const version = "0.1.0"

// database fields per tables
var (
	diffDeltaFields = []string{
		"commit_id",
		"file_status",
		"is_file_binary",
		"similarity",
		"old_file_path",
		"new_file_path"}

	commitFields = []string{
		"repository_id",
		"author_id",
		"committer_id",
		"hash",
		"vcs_id",
		"message",
		"author_date",
		"commit_date",
		"file_changed_count",
		"insertions_count",
		"deletions_count"}
)

func main() {
	var err error

	flag.Usage = func() {
		fmt.Printf("usage: %s [OPTION(S)] [REPOSITORY PATH]\n", os.Args[0])
		flag.PrintDefaults()
		os.Exit(0)
	}

	configPath := flag.String("c", "", "configuration file")
	vflag := flag.Bool("v", false, "print version.")
	jsonflag := flag.Bool("json", true, "json output")
	dbflag := flag.Bool("db", false, "import data into the database")
	srctoolflag := flag.String("srctool", "", "read json file produced by srctool (give stdin to read from stdin)")
	flag.Parse()

	if *vflag {
		fmt.Printf("%s - %s\n", filepath.Base(os.Args[0]), version)
		os.Exit(0)
	}

	if len(flag.Args()) != 1 {
		fmt.Fprintln(os.Stderr, "invalid # of arguments")
		flag.Usage()
	}

	if *dbflag && len(*configPath) == 0 {
		fatal(errors.New("a configuration file must be specified when using db option"))
	}

	if !*jsonflag && (len(*srctoolflag) > 0) {
		fatal(errors.New("srctool flag may be used only in conjonction with json flag"))
	}

	var cfg *config.Config
	cfg, err = config.ReadConfig(*configPath)
	if err != nil {
		fatal(err)
	}

	repoPath := flag.Arg(0)
	var repository repo.Repo
	repository, err = repo.New(*cfg, repoPath)
	if err != nil {
		fatal(err)
	}
	defer func() {
		repository.Cleanup()
		if err != nil {
			fatal(err)
		}
	}()

	fmt.Fprintln(os.Stderr, "fetching repository commits...")
	tic := time.Now()
	err = repository.FetchCommits()
	if err != nil {
		return
	}
	toc := time.Now()
	fmt.Fprintln(os.Stderr, "done in ", toc.Sub(tic))

	if *jsonflag && (len(*srctoolflag) == 0) {
		var bs []byte
		bs, err = json.Marshal(repository)
		if err != nil {
			return
		}
		fmt.Println(string(bs))
	}

	if *jsonflag && (len(*srctoolflag)) > 0 {
		var r *bufio.Reader
		if *srctoolflag == strings.ToLower("stdin") {
			// read from stdin
			r = bufio.NewReader(os.Stdin)
		} else {
			// read from srctool json file
			var f *os.File
			if f, err = os.Open(*srctoolflag); err != nil {
				return
			}
			r = bufio.NewReader(f)
		}

		buf := new(bytes.Buffer)
		if _, err = io.Copy(buf, r); err != nil {
			fail(err)
			return
		}

		bs := buf.Bytes()
		var p *src.Project
		p, err = src.Unmarshal(bs)
		if err != nil {
			return
		}

		p.Repo = repository.GetRepository()
		bs, err = src.Marshal(p)
		if err != nil {
			return
		}

		fmt.Println(string(bs))
	}

	if *dbflag {
		var db *sql.DB
		db, err = openDBSession(cfg.Database)
		if err != nil {
			return
		}
		defer db.Close()

		fmt.Fprintf(os.Stderr,
			"inserting %d commits from %s repository into the database...\n",
			len(repository.GetCommits()), repository.GetName())
		tic := time.Now()
		err = insertRepoData(db, repository)
		toc := time.Now()
		if err != nil {
			return
		}
		fmt.Fprintln(os.Stderr, "done in ", toc.Sub(tic))
	}
}

// fatal prints an error on standard error stream and exits.
func fatal(a ...interface{}) {
	fmt.Fprintln(os.Stderr, a...)
	os.Exit(1)
}

// fail prints an error on standard error stream.
func fail(a ...interface{}) {
	fmt.Fprintln(os.Stderr, a...)
}

// openDBSession creates a session to the database.
func openDBSession(cfg config.DatabaseConfig) (*sql.DB, error) {
	dbURL := fmt.Sprintf(
		"user='%s' password='%s' host='%s' port=%d dbname='%s' sslmode='%s'",
		cfg.UserName, cfg.Password, cfg.HostName, cfg.Port, cfg.DBName, cfg.SSLMode)

	return sql.Open("postgres", dbURL)
}

// insertRepoData inserts repository data into the database, or updates it
// if it is already there.
func insertRepoData(db *sql.DB, r repo.Repo) error {
	if db == nil {
		return errors.New("nil database given")
	}

	repoID, err := getRepoID(db, r)
	if err != nil {
		return err
	}
	if repoID == nil {
		return errors.New("cannot find corresponding repository in database")
	}

	userIDs, err := getAllUsers(db)
	if err != nil {
		return err
	}

	tx, err := db.Begin()
	if err != nil {
		return err
	}
	defer tx.Rollback()

	commitStmt, err := tx.Prepare(genInsQuery("commits", commitFields...) + " RETURNING id")
	if err != nil {
		return err
	}

	deltaStmt, err := tx.Prepare(genInsQuery("commit_diff_deltas", diffDeltaFields...))
	if err != nil {
		return err
	}

	for _, c := range r.GetCommits() {
		if err := insertCommit(userIDs, *repoID, c, tx, commitStmt, deltaStmt); err != nil {
			return err
		}
	}

	if err := commitStmt.Close(); err != nil {
		return err
	}
	if err := deltaStmt.Close(); err != nil {
		return err
	}
	if err := tx.Commit(); err != nil {
		return err
	}

	return nil
}

// insertCommit inserts a commit into the database
func insertCommit(userIDs map[string]uint64, repoID uint64, c model.Commit, tx *sql.Tx, commitStmt, deltaStmt *sql.Stmt) error {
	authorID := userIDs[c.Author.Email]
	committerID := userIDs[c.Committer.Email]
	hash := genCommitHash(c)

	var commitID uint64
	err := commitStmt.QueryRow(
		repoID, authorID, committerID, hash,
		c.VCSID, c.Message, c.AuthorDate, c.CommitDate,
		c.FileChangedCount, c.InsertionsCount, c.DeletionsCount).Scan(&commitID)
	if err != nil {
		return err
	}

	for _, d := range c.DiffDelta {
		if err := insertDiffDelta(commitID, d, deltaStmt); err != nil {
			return err
		}
	}

	return nil
}

// insertDiffDelta inserts a commit diff delta into the database.
func insertDiffDelta(commitID uint64, d model.DiffDelta, stmt *sql.Stmt) error {
	_, err := stmt.Exec(commitID, d.Status, d.Binary, d.Similarity, d.OldFilePath, d.NewFilePath)
	if err != nil {
		return err
	}
	return nil
}

// getAllUsers returns a map of all users IDs with their email address as keys.
func getAllUsers(db *sql.DB) (map[string]uint64, error) {
	rows, err := db.Query("SELECT id, email FROM users WHERE email IS NOT NULL AND email != ''")
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	userIDs := map[string]uint64{}
	for rows.Next() {
		var email string
		var id uint64
		if err := rows.Scan(&id, &email); err != nil {
			return nil, err
		}
		userIDs[email] = id
	}

	return userIDs, nil
}

// genCommitHash generates a hash (mmh3) from commit fields.
// This hash can then be used to uniquely identify a commit.
// Typically, we want to make sure not to insert twice the same commit into the
// database after an eventual second repotool run on the same repository.
func genCommitHash(c model.Commit) string {
	h := mmh3.New128()

	io.WriteString(h, c.VCSID)
	io.WriteString(h, c.Message)
	io.WriteString(h, c.Author.Name)
	io.WriteString(h, c.Author.Email)
	io.WriteString(h, c.Committer.Name)
	io.WriteString(h, c.Committer.Email)
	io.WriteString(h, c.AuthorDate.String())
	io.WriteString(h, c.CommitDate.String())
	io.WriteString(h, strconv.FormatInt(int64(c.FileChangedCount), 10))
	io.WriteString(h, strconv.FormatInt(int64(c.InsertionsCount), 10))
	io.WriteString(h, strconv.FormatInt(int64(c.DeletionsCount), 10))

	return hex.EncodeToString(h.Sum(nil))
}

// getRepoID returns the repository id of a repo in repositories table.
// If repo is not in the table, then 0 is returned.
func getRepoID(db *sql.DB, r repo.Repo) (*uint64, error) {
	if db == nil {
		return nil, errors.New("nil database given")
	}

	var id *uint64
	// Clone URL is unique
	err := db.QueryRow("SELECT id FROM repositories WHERE clone_url=$1", r.GetCloneURL()).Scan(&id)
	switch {
	case err == sql.ErrNoRows:
		return nil, nil
	case err != nil:
		return nil, err
	}
	return id, nil
}

// genInsQuery generates a query string for an insertion in the database.
func genInsQuery(tableName string, fields ...string) string {
	var buf bytes.Buffer

	buf.WriteString(fmt.Sprintf("INSERT INTO %s(%s)\n",
		tableName, strings.Join(fields, ",")))
	buf.WriteString("VALUES(")

	for ind := range fields {
		if ind > 0 {
			buf.WriteString(",")
		}

		buf.WriteString(fmt.Sprintf("$%d", ind+1))
	}

	buf.WriteString(")\n")

	return buf.String()
}
<commit_msg>repotool: move flags to var<commit_after>// Copyright 2014-2015 The DevMine authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package repotool is able to fetch information from a source code repository.
// Typically, it can get all commits, their authors and commiters and so on
// and return this information in a JSON object. Alternatively, it is able
// to populate the information into a PostgreSQL database.
// Currently, on the Git VCS is supported.
package main

import (
	"bufio"
	"bytes"
	"database/sql"
	"encoding/hex"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	_ "github.com/lib/pq"
	mmh3 "github.com/spaolacci/murmur3"

	"github.com/DevMine/srcanlzr/src"

	"github.com/DevMine/repotool/config"
	"github.com/DevMine/repotool/model"
	"github.com/DevMine/repotool/repo"
)

const version = "0.1.0"

// database fields per tables
var (
	diffDeltaFields = []string{
		"commit_id",
		"file_status",
		"is_file_binary",
		"similarity",
		"old_file_path",
		"new_file_path"}

	commitFields = []string{
		"repository_id",
		"author_id",
		"committer_id",
		"hash",
		"vcs_id",
		"message",
		"author_date",
		"commit_date",
		"file_changed_count",
		"insertions_count",
		"deletions_count"}
)

// program flags
var (
	configPath  = flag.String("c", "", "configuration file")
	vflag       = flag.Bool("v", false, "print version.")
	jsonflag    = flag.Bool("json", true, "json output")
	dbflag      = flag.Bool("db", false, "import data into the database")
	srctoolflag = flag.String("srctool", "", "read json file produced by srctool (give stdin to read from stdin)")
)

func main() {
	var err error

	flag.Usage = func() {
		fmt.Printf("usage: %s [OPTION(S)] [REPOSITORY PATH]\n", os.Args[0])
		flag.PrintDefaults()
		os.Exit(0)
	}
	flag.Parse()

	if *vflag {
		fmt.Printf("%s - %s\n", filepath.Base(os.Args[0]), version)
		os.Exit(0)
	}

	if len(flag.Args()) != 1 {
		fmt.Fprintln(os.Stderr, "invalid # of arguments")
		flag.Usage()
	}

	if *dbflag && len(*configPath) == 0 {
		fatal(errors.New("a configuration file must be specified when using db option"))
	}

	if !*jsonflag && (len(*srctoolflag) > 0) {
		fatal(errors.New("srctool flag may be used only in conjonction with json flag"))
	}

	var cfg *config.Config
	cfg, err = config.ReadConfig(*configPath)
	if err != nil {
		fatal(err)
	}

	repoPath := flag.Arg(0)
	var repository repo.Repo
	repository, err = repo.New(*cfg, repoPath)
	if err != nil {
		fatal(err)
	}
	defer func() {
		repository.Cleanup()
		if err != nil {
			fatal(err)
		}
	}()

	fmt.Fprintln(os.Stderr, "fetching repository commits...")
	tic := time.Now()
	err = repository.FetchCommits()
	if err != nil {
		return
	}
	toc := time.Now()
	fmt.Fprintln(os.Stderr, "done in ", toc.Sub(tic))

	if *jsonflag && (len(*srctoolflag) == 0) {
		var bs []byte
		bs, err = json.Marshal(repository)
		if err != nil {
			return
		}
		fmt.Println(string(bs))
	}

	if *jsonflag && (len(*srctoolflag)) > 0 {
		var r *bufio.Reader
		if *srctoolflag == strings.ToLower("stdin") {
			// read from stdin
			r = bufio.NewReader(os.Stdin)
		} else {
			// read from srctool json file
			var f *os.File
			if f, err = os.Open(*srctoolflag); err != nil {
				return
			}
			r = bufio.NewReader(f)
		}

		buf := new(bytes.Buffer)
		if _, err = io.Copy(buf, r); err != nil {
			fail(err)
			return
		}

		bs := buf.Bytes()
		var p *src.Project
		p, err = src.Unmarshal(bs)
		if err != nil {
			return
		}

		p.Repo = repository.GetRepository()
		bs, err = src.Marshal(p)
		if err != nil {
			return
		}

		fmt.Println(string(bs))
	}

	if *dbflag {
		var db *sql.DB
		db, err = openDBSession(cfg.Database)
		if err != nil {
			return
		}
		defer db.Close()

		fmt.Fprintf(os.Stderr,
			"inserting %d commits from %s repository into the database...\n",
			len(repository.GetCommits()), repository.GetName())
		tic := time.Now()
		err = insertRepoData(db, repository)
		toc := time.Now()
		if err != nil {
			return
		}
		fmt.Fprintln(os.Stderr, "done in ", toc.Sub(tic))
	}
}

// fatal prints an error on standard error stream and exits.
func fatal(a ...interface{}) {
	fmt.Fprintln(os.Stderr, a...)
	os.Exit(1)
}

// fail prints an error on standard error stream.
func fail(a ...interface{}) {
	fmt.Fprintln(os.Stderr, a...)
}

// openDBSession creates a session to the database.
func openDBSession(cfg config.DatabaseConfig) (*sql.DB, error) {
	dbURL := fmt.Sprintf(
		"user='%s' password='%s' host='%s' port=%d dbname='%s' sslmode='%s'",
		cfg.UserName, cfg.Password, cfg.HostName, cfg.Port, cfg.DBName, cfg.SSLMode)

	return sql.Open("postgres", dbURL)
}

// insertRepoData inserts repository data into the database, or updates it
// if it is already there.
func insertRepoData(db *sql.DB, r repo.Repo) error {
	if db == nil {
		return errors.New("nil database given")
	}

	repoID, err := getRepoID(db, r)
	if err != nil {
		return err
	}
	if repoID == nil {
		return errors.New("cannot find corresponding repository in database")
	}

	userIDs, err := getAllUsers(db)
	if err != nil {
		return err
	}

	tx, err := db.Begin()
	if err != nil {
		return err
	}
	defer tx.Rollback()

	commitStmt, err := tx.Prepare(genInsQuery("commits", commitFields...) + " RETURNING id")
	if err != nil {
		return err
	}

	deltaStmt, err := tx.Prepare(genInsQuery("commit_diff_deltas", diffDeltaFields...))
	if err != nil {
		return err
	}

	for _, c := range r.GetCommits() {
		if err := insertCommit(userIDs, *repoID, c, tx, commitStmt, deltaStmt); err != nil {
			return err
		}
	}

	if err := commitStmt.Close(); err != nil {
		return err
	}
	if err := deltaStmt.Close(); err != nil {
		return err
	}
	if err := tx.Commit(); err != nil {
		return err
	}

	return nil
}

// insertCommit inserts a commit into the database
func insertCommit(userIDs map[string]uint64, repoID uint64, c model.Commit, tx *sql.Tx, commitStmt, deltaStmt *sql.Stmt) error {
	authorID := userIDs[c.Author.Email]
	committerID := userIDs[c.Committer.Email]
	hash := genCommitHash(c)

	var commitID uint64
	err := commitStmt.QueryRow(
		repoID, authorID, committerID, hash,
		c.VCSID, c.Message, c.AuthorDate, c.CommitDate,
		c.FileChangedCount, c.InsertionsCount, c.DeletionsCount).Scan(&commitID)
	if err != nil {
		return err
	}

	for _, d := range c.DiffDelta {
		if err := insertDiffDelta(commitID, d, deltaStmt); err != nil {
			return err
		}
	}

	return nil
}

// insertDiffDelta inserts a commit diff delta into the database.
func insertDiffDelta(commitID uint64, d model.DiffDelta, stmt *sql.Stmt) error {
	_, err := stmt.Exec(commitID, d.Status, d.Binary, d.Similarity, d.OldFilePath, d.NewFilePath)
	if err != nil {
		return err
	}
	return nil
}

// getAllUsers returns a map of all users IDs with their email address as keys.
func getAllUsers(db *sql.DB) (map[string]uint64, error) {
	rows, err := db.Query("SELECT id, email FROM users WHERE email IS NOT NULL AND email != ''")
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	userIDs := map[string]uint64{}
	for rows.Next() {
		var email string
		var id uint64
		if err := rows.Scan(&id, &email); err != nil {
			return nil, err
		}
		userIDs[email] = id
	}

	return userIDs, nil
}

// genCommitHash generates a hash (mmh3) from commit fields.
// This hash can then be used to uniquely identify a commit.
// Typically, we want to make sure not to insert twice the same commit into the
// database after an eventual second repotool run on the same repository.
func genCommitHash(c model.Commit) string {
	h := mmh3.New128()

	io.WriteString(h, c.VCSID)
	io.WriteString(h, c.Message)
	io.WriteString(h, c.Author.Name)
	io.WriteString(h, c.Author.Email)
	io.WriteString(h, c.Committer.Name)
	io.WriteString(h, c.Committer.Email)
	io.WriteString(h, c.AuthorDate.String())
	io.WriteString(h, c.CommitDate.String())
	io.WriteString(h, strconv.FormatInt(int64(c.FileChangedCount), 10))
	io.WriteString(h, strconv.FormatInt(int64(c.InsertionsCount), 10))
	io.WriteString(h, strconv.FormatInt(int64(c.DeletionsCount), 10))

	return hex.EncodeToString(h.Sum(nil))
}

// getRepoID returns the repository id of a repo in repositories table.
// If repo is not in the table, then 0 is returned.
func getRepoID(db *sql.DB, r repo.Repo) (*uint64, error) {
	if db == nil {
		return nil, errors.New("nil database given")
	}

	var id *uint64
	// Clone URL is unique
	err := db.QueryRow("SELECT id FROM repositories WHERE clone_url=$1", r.GetCloneURL()).Scan(&id)
	switch {
	case err == sql.ErrNoRows:
		return nil, nil
	case err != nil:
		return nil, err
	}
	return id, nil
}

// genInsQuery generates a query string for an insertion in the database.
func genInsQuery(tableName string, fields ...string) string {
	var buf bytes.Buffer

	buf.WriteString(fmt.Sprintf("INSERT INTO %s(%s)\n",
		tableName, strings.Join(fields, ",")))
	buf.WriteString("VALUES(")

	for ind := range fields {
		if ind > 0 {
			buf.WriteString(",")
		}

		buf.WriteString(fmt.Sprintf("$%d", ind+1))
	}

	buf.WriteString(")\n")

	return buf.String()
}
<|endoftext|>