<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package golang contains Go buildpack library code.
package golang

import (
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/GoogleCloudPlatform/buildpacks/pkg/appengine"
	"github.com/GoogleCloudPlatform/buildpacks/pkg/cache"
	"github.com/GoogleCloudPlatform/buildpacks/pkg/env"
	gcp "github.com/GoogleCloudPlatform/buildpacks/pkg/gcpbuildpack"
	"github.com/buildpacks/libcnb"
	"github.com/Masterminds/semver"
)

const (
	// OutBin is the name of the final compiled binary produced by Go buildpacks.
	OutBin = "main"
	// BuildDirEnv is an environment variable that buildpacks can use to communicate the working directory to `go build`.
	BuildDirEnv = "GOOGLE_INTERNAL_BUILD_DIR"
	// The name of the layer where the GOPATH is stored
	goPathLayerName = "gopath"
	// The key used when a layers' cache is keyed off of the go mod
	goModCacheKey = "go-mod-sha"
)

var (
	// goVersionRegexp is used to parse `go version`'s output.
	goVersionRegexp = regexp.MustCompile(`^go version go(\d+(\.\d+){1,2})([a-z]+\d+)? .*$`)

	// goModVersionRegexp is used to get correct declaration of Go version from go.mod file.
	goModVersionRegexp = regexp.MustCompile(`(?m)^\s*go\s+(\d+(\.\d+){1,2})\s*$`)
)

// SupportsAppEngineApis is a Go buildpack specific function that returns true if App Engine API access is enabled
func SupportsAppEngineApis(ctx *gcp.Context) (bool, error) {
	if os.Getenv(env.Runtime) == "go111" {
		return true, nil
	}

	return appengine.ApisEnabled(ctx)
}

// SupportsAutoVendor returns true if the Go version supports automatic detection of the vendor directory.
// This feature is supported by Go 1.14 and higher.
func SupportsAutoVendor(ctx *gcp.Context) (bool, error) {
	return VersionMatches(ctx, ">=1.14.0")
}

// SupportsGoProxyFallback returns true if the Go version supports fallback in GOPROXY using the pipe character.
// This feature is supported by Go 1.15 and higher.
func SupportsGoProxyFallback(ctx *gcp.Context) (bool, error) {
	return VersionMatches(ctx, ">=1.15.0")
}

// SupportsGoCleanModCache returns true if the Go version supports `go clean -modcache` without loading the packages.
// The command fails if the packages aren't available for Go 1.12 and lower.
// The feature to skip loading the packages is only supported by Go 1.13 and higher.
// More information can be found at golang.org/issue/28680 and golang.org/issue/28459.
func SupportsGoCleanModCache(ctx *gcp.Context) (bool, error) {
	return VersionMatches(ctx, ">=1.13.0")
}

// VersionMatches checks if the installed version of Go and the version specified in go.mod match the given version range.
// The range string has the following format: https://github.com/blang/semver#ranges.
func VersionMatches(ctx *gcp.Context, versionRange string) (bool, error) {
	v, err := GoModVersion(ctx)
	if err != nil {
		return false, err
	}
	if v == "" {
		return false, nil
	}

	version, err := semver.NewVersion(v)
	if err != nil {
		return false, gcp.InternalErrorf("unable to parse go.mod version string %q: %s", v, err)
	}

	goVersionMatches, err := semver.NewConstraint(versionRange)
	if err != nil {
		return false, gcp.InternalErrorf("unable to parse version range %q: %s", v, err)
	}

	if !goVersionMatches.Check(version) {
		return false, nil
	}

	v, err = GoVersion(ctx)
	if err != nil {
		return false, err
	}

	version, err = semver.NewVersion(v)
	if err != nil {
		return false, gcp.InternalErrorf("unable to parse Go version string %q: %s", v, err)
	}

	return goVersionMatches.Check(version), nil
}

// GoVersion reads the version of the installed Go runtime.
func GoVersion(ctx *gcp.Context) (string, error) {
	v := readGoVersion(ctx)

	match := goVersionRegexp.FindStringSubmatch(v)
	if len(match) < 2 || match[1] == "" {
		return "", gcp.InternalErrorf("unable to find go version in %q", v)
	}

	return match[1], nil
}

// GoModVersion reads the version of Go from a go.mod file if present.
// If not present or if version isn't there returns an empty string.
func GoModVersion(ctx *gcp.Context) (string, error) {
	v, err := readGoMod(ctx)
	if err != nil {
		return "", fmt.Errorf("reading go.mod: %w", err)
	}
	if v == "" {
		return v, nil
	}

	match := goModVersionRegexp.FindStringSubmatch(v)
	if len(match) < 2 || match[1] == "" {
		return "", nil
	}

	return match[1], nil
}

// readGoVersion returns the output of `go version`.
// It can be overridden for testing.
var readGoVersion = func(ctx *gcp.Context) string {
	return ctx.Exec([]string{"go", "version"}).Stdout
}

// cleanModCache deletes the downloaded cached dependencies using `go clean -modcache`.
// The cached dependencies are written without write access and attempt
// to clear layer using ctx.ClearLayer(l) fails with permission denied errors.
// It can be overridden for testing.
var cleanModCache = func(ctx *gcp.Context) {
	ctx.Exec([]string{"go", "clean", "-modcache"})
}

// readGoMod reads the go.mod file if present. If not present, returns an empty string.
// It can be overridden for testing.
var readGoMod = func(ctx *gcp.Context) (string, error) {
	goModPath := goModPath(ctx)
	goModExists, err := ctx.FileExists(goModPath)
	if err != nil {
		return "", err
	}
	if !goModExists {
		return "", nil
	}
	bytes, err := ctx.ReadFile(goModPath)
	if err != nil {
		return "", err
	}
	return string(bytes), nil
}

// NewGoWorkspaceLayer returns a new layer for `go env GOPATH` or the go workspace. The
// layer is configured for caching if possible. It only supports caching for "go mod"
// based builds.
func NewGoWorkspaceLayer(ctx *gcp.Context) (*libcnb.Layer, error) {
	l, err := ctx.Layer(goPathLayerName, gcp.BuildLayer, gcp.CacheLayer, gcp.LaunchLayerIfDevMode)
	if err != nil {
		return nil, fmt.Errorf("creating %v layer: %w", goPathLayerName, err)
	}
	l.BuildEnvironment.Override("GOPATH", l.Path)
	l.BuildEnvironment.Override("GO111MODULE", "on")
	// Set GOPROXY to ensure no additional dependency is downloaded at built time.
	// All of them are downloaded here.
	l.BuildEnvironment.Override("GOPROXY", "off")

	shouldEnablePkgCache, err := SupportsGoCleanModCache(ctx)
	if err != nil {
		return nil, fmt.Errorf("checking for go pkg cache support: %w", err)
	}
	if !shouldEnablePkgCache {
		l.Cache = false
		return l, nil
	}

	sha, err := cache.Hash(ctx, cache.WithFiles(goModPath(ctx)))
	if err != nil {
		if os.IsNotExist(err) {
			// when go.mod doesn't exist, clear any previously cached bits and return an empty layer
			l.Cache = false
			cleanModCache(ctx)
			return l, nil
		}
		return nil, err
	}
	shaStr := fmt.Sprintf("%x", sha)
	if shaStr == ctx.GetMetadata(l, goModCacheKey) {
		ctx.Logf("GOPATH layer cache hit")
		ctx.CacheHit(goPathLayerName)
		return l, nil
	}
	ctx.Debugf("go.mod SHA has changed: clearing GOPATH layer's cache")
	cleanModCache(ctx)
	ctx.SetMetadata(l, goModCacheKey, shaStr)
	return l, nil
}

func goModPath(ctx *gcp.Context) string {
	return filepath.Join(ctx.ApplicationRoot(), "go.mod")
}

// ExecWithGoproxyFallback runs the given command with a GOPROXY fallback.
// Before Go 1.14, Go would fall back to direct only if a 404 or 410 error ocurred, for those
// versions, we explictly disable GOPROXY and try again on any error.
// For newer versions of Go, we take advantage of the "pipe" character which has the same effect.
func ExecWithGoproxyFallback(ctx *gcp.Context, cmd []string, opts ...gcp.ExecOption) (*gcp.ExecResult, error) {
	supportsGoProxy, err := SupportsGoProxyFallback(ctx)
	if err != nil {
		return nil, fmt.Errorf("checking for go proxy support: %w", err)
	}
	if supportsGoProxy {
		opts = append(opts, gcp.WithEnv("GOPROXY=https://proxy.golang.org|direct"))
		return ctx.Exec(cmd, opts...), nil
	}

	result, err := ctx.ExecWithErr(cmd, opts...)
	if err == nil {
		return result, nil
	}
	ctx.Warnf("%q failed. Retrying with GOSUMDB=off GOPROXY=direct. Error: %v", strings.Join(cmd, " "), err)

	opts = append(opts, gcp.WithEnv("GOSUMDB=off", "GOPROXY=direct"))
	return ctx.Exec(cmd, opts...), nil
}

// IsGo111Runtime returns true when the GOOGLE_RUNTIME is go111. This will be
// true when using GCF or GAE with go 1.11.
func IsGo111Runtime() bool {
	return os.Getenv(env.Runtime) == "go111"
}
<commit_msg>Replace a check for the go111 runtime with calling IsGo111Runtime()<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package golang contains Go buildpack library code.
package golang

import (
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/GoogleCloudPlatform/buildpacks/pkg/appengine"
	"github.com/GoogleCloudPlatform/buildpacks/pkg/cache"
	"github.com/GoogleCloudPlatform/buildpacks/pkg/env"
	gcp "github.com/GoogleCloudPlatform/buildpacks/pkg/gcpbuildpack"
	"github.com/buildpacks/libcnb"
	"github.com/Masterminds/semver"
)

const (
	// OutBin is the name of the final compiled binary produced by Go buildpacks.
	OutBin = "main"
	// BuildDirEnv is an environment variable that buildpacks can use to communicate the working directory to `go build`.
	BuildDirEnv = "GOOGLE_INTERNAL_BUILD_DIR"
	// The name of the layer where the GOPATH is stored
	goPathLayerName = "gopath"
	// The key used when a layers' cache is keyed off of the go mod
	goModCacheKey = "go-mod-sha"
)

var (
	// goVersionRegexp is used to parse `go version`'s output.
	goVersionRegexp = regexp.MustCompile(`^go version go(\d+(\.\d+){1,2})([a-z]+\d+)? .*$`)

	// goModVersionRegexp is used to get correct declaration of Go version from go.mod file.
	goModVersionRegexp = regexp.MustCompile(`(?m)^\s*go\s+(\d+(\.\d+){1,2})\s*$`)
)

// SupportsAppEngineApis is a Go buildpack specific function that returns true if App Engine API access is enabled
func SupportsAppEngineApis(ctx *gcp.Context) (bool, error) {
	if IsGo111Runtime() {
		return true, nil
	}

	return appengine.ApisEnabled(ctx)
}

// SupportsAutoVendor returns true if the Go version supports automatic detection of the vendor directory.
// This feature is supported by Go 1.14 and higher.
func SupportsAutoVendor(ctx *gcp.Context) (bool, error) {
	return VersionMatches(ctx, ">=1.14.0")
}

// SupportsGoProxyFallback returns true if the Go version supports fallback in GOPROXY using the pipe character.
// This feature is supported by Go 1.15 and higher.
func SupportsGoProxyFallback(ctx *gcp.Context) (bool, error) {
	return VersionMatches(ctx, ">=1.15.0")
}

// SupportsGoCleanModCache returns true if the Go version supports `go clean -modcache` without loading the packages.
// The command fails if the packages aren't available for Go 1.12 and lower.
// The feature to skip loading the packages is only supported by Go 1.13 and higher.
// More information can be found at golang.org/issue/28680 and golang.org/issue/28459.
func SupportsGoCleanModCache(ctx *gcp.Context) (bool, error) {
	return VersionMatches(ctx, ">=1.13.0")
}

// VersionMatches checks if the installed version of Go and the version specified in go.mod match the given version range.
// The range string has the following format: https://github.com/blang/semver#ranges.
func VersionMatches(ctx *gcp.Context, versionRange string) (bool, error) {
	v, err := GoModVersion(ctx)
	if err != nil {
		return false, err
	}
	if v == "" {
		return false, nil
	}

	version, err := semver.NewVersion(v)
	if err != nil {
		return false, gcp.InternalErrorf("unable to parse go.mod version string %q: %s", v, err)
	}

	goVersionMatches, err := semver.NewConstraint(versionRange)
	if err != nil {
		return false, gcp.InternalErrorf("unable to parse version range %q: %s", v, err)
	}

	if !goVersionMatches.Check(version) {
		return false, nil
	}

	v, err = GoVersion(ctx)
	if err != nil {
		return false, err
	}

	version, err = semver.NewVersion(v)
	if err != nil {
		return false, gcp.InternalErrorf("unable to parse Go version string %q: %s", v, err)
	}

	return goVersionMatches.Check(version), nil
}

// GoVersion reads the version of the installed Go runtime.
func GoVersion(ctx *gcp.Context) (string, error) {
	v := readGoVersion(ctx)

	match := goVersionRegexp.FindStringSubmatch(v)
	if len(match) < 2 || match[1] == "" {
		return "", gcp.InternalErrorf("unable to find go version in %q", v)
	}

	return match[1], nil
}

// GoModVersion reads the version of Go from a go.mod file if present.
// If not present or if version isn't there returns an empty string.
func GoModVersion(ctx *gcp.Context) (string, error) {
	v, err := readGoMod(ctx)
	if err != nil {
		return "", fmt.Errorf("reading go.mod: %w", err)
	}
	if v == "" {
		return v, nil
	}

	match := goModVersionRegexp.FindStringSubmatch(v)
	if len(match) < 2 || match[1] == "" {
		return "", nil
	}

	return match[1], nil
}

// readGoVersion returns the output of `go version`.
// It can be overridden for testing.
var readGoVersion = func(ctx *gcp.Context) string {
	return ctx.Exec([]string{"go", "version"}).Stdout
}

// cleanModCache deletes the downloaded cached dependencies using `go clean -modcache`.
// The cached dependencies are written without write access and attempt
// to clear layer using ctx.ClearLayer(l) fails with permission denied errors.
// It can be overridden for testing.
var cleanModCache = func(ctx *gcp.Context) {
	ctx.Exec([]string{"go", "clean", "-modcache"})
}

// readGoMod reads the go.mod file if present. If not present, returns an empty string.
// It can be overridden for testing.
var readGoMod = func(ctx *gcp.Context) (string, error) {
	goModPath := goModPath(ctx)
	goModExists, err := ctx.FileExists(goModPath)
	if err != nil {
		return "", err
	}
	if !goModExists {
		return "", nil
	}
	bytes, err := ctx.ReadFile(goModPath)
	if err != nil {
		return "", err
	}
	return string(bytes), nil
}

// NewGoWorkspaceLayer returns a new layer for `go env GOPATH` or the go workspace. The
// layer is configured for caching if possible. It only supports caching for "go mod"
// based builds.
func NewGoWorkspaceLayer(ctx *gcp.Context) (*libcnb.Layer, error) {
	l, err := ctx.Layer(goPathLayerName, gcp.BuildLayer, gcp.CacheLayer, gcp.LaunchLayerIfDevMode)
	if err != nil {
		return nil, fmt.Errorf("creating %v layer: %w", goPathLayerName, err)
	}
	l.BuildEnvironment.Override("GOPATH", l.Path)
	l.BuildEnvironment.Override("GO111MODULE", "on")
	// Set GOPROXY to ensure no additional dependency is downloaded at built time.
	// All of them are downloaded here.
	l.BuildEnvironment.Override("GOPROXY", "off")

	shouldEnablePkgCache, err := SupportsGoCleanModCache(ctx)
	if err != nil {
		return nil, fmt.Errorf("checking for go pkg cache support: %w", err)
	}
	if !shouldEnablePkgCache {
		l.Cache = false
		return l, nil
	}

	sha, err := cache.Hash(ctx, cache.WithFiles(goModPath(ctx)))
	if err != nil {
		if os.IsNotExist(err) {
			// when go.mod doesn't exist, clear any previously cached bits and return an empty layer
			l.Cache = false
			cleanModCache(ctx)
			return l, nil
		}
		return nil, err
	}
	shaStr := fmt.Sprintf("%x", sha)
	if shaStr == ctx.GetMetadata(l, goModCacheKey) {
		ctx.Logf("GOPATH layer cache hit")
		ctx.CacheHit(goPathLayerName)
		return l, nil
	}
	ctx.Debugf("go.mod SHA has changed: clearing GOPATH layer's cache")
	cleanModCache(ctx)
	ctx.SetMetadata(l, goModCacheKey, shaStr)
	return l, nil
}

func goModPath(ctx *gcp.Context) string {
	return filepath.Join(ctx.ApplicationRoot(), "go.mod")
}

// ExecWithGoproxyFallback runs the given command with a GOPROXY fallback.
// Before Go 1.14, Go would fall back to direct only if a 404 or 410 error ocurred, for those
// versions, we explictly disable GOPROXY and try again on any error.
// For newer versions of Go, we take advantage of the "pipe" character which has the same effect.
func ExecWithGoproxyFallback(ctx *gcp.Context, cmd []string, opts ...gcp.ExecOption) (*gcp.ExecResult, error) {
	supportsGoProxy, err := SupportsGoProxyFallback(ctx)
	if err != nil {
		return nil, fmt.Errorf("checking for go proxy support: %w", err)
	}
	if supportsGoProxy {
		opts = append(opts, gcp.WithEnv("GOPROXY=https://proxy.golang.org|direct"))
		return ctx.Exec(cmd, opts...), nil
	}

	result, err := ctx.ExecWithErr(cmd, opts...)
	if err == nil {
		return result, nil
	}
	ctx.Warnf("%q failed. Retrying with GOSUMDB=off GOPROXY=direct. Error: %v", strings.Join(cmd, " "), err)

	opts = append(opts, gcp.WithEnv("GOSUMDB=off", "GOPROXY=direct"))
	return ctx.Exec(cmd, opts...), nil
}

// IsGo111Runtime returns true when the GOOGLE_RUNTIME is go111. This will be
// true when using GCF or GAE with go 1.11.
func IsGo111Runtime() bool {
	return os.Getenv(env.Runtime) == "go111"
}
<|endoftext|>


<commit_before>package jobs

import (
	"container/list"
	"errors"
	"sync"
)

var (
	memBrokers   map[string]*MemBroker
	memBrokersMu sync.Mutex
)

type (
	// MemQueue is a queue in-memory implementation of the Queue interface.
	MemQueue struct {
		MaxCapacity int

		jobs *list.List
		run  bool
		jmu  sync.RWMutex

		ch chan Job
		cl chan bool
	}

	// MemBroker is an in-memory broker implementation of the Broker interface.
	MemBroker struct {
		domain string
		queues map[string]*MemQueue
	}

	// MemJob struct contains all the parameters of a job.
	MemJob struct {
		infos *JobInfos
		jobch chan *JobInfos
	}
)

// NewMemQueue creates and a new in-memory queue.
func NewMemQueue(domain, workerType string) *MemQueue {
	return &MemQueue{
		jobs: list.New(),
		ch:   make(chan Job),
		cl:   make(chan bool),
	}
}

// Enqueue into the queue
func (q *MemQueue) Enqueue(job Job) error {
	q.jmu.Lock()
	defer q.jmu.Unlock()
	q.jobs.PushBack(job)
	if !q.run {
		q.run = true
		go q.send()
	}
	return nil
}

func (q *MemQueue) send() {
	for {
		q.jmu.Lock()
		e := q.jobs.Front()
		if e == nil {
			q.run = false
			q.jmu.Unlock()
			return
		}
		q.jobs.Remove(e)
		q.jmu.Unlock()
		select {
		case q.ch <- e.Value.(Job):
			continue
		case <-q.cl:
			return
		}
	}
}

// Consume from the queue
func (q *MemQueue) Consume() (Job, error) {
	select {
	case job := <-q.ch:
		return job, nil
	case <-q.cl:
		return nil, ErrQueueClosed
	}
}

// Len returns the length of the queue
func (q *MemQueue) Len() int {
	q.jmu.RLock()
	defer q.jmu.RUnlock()
	return q.jobs.Len()
}

// Close closes the queue
func (q *MemQueue) Close() {
	close(q.cl)
}

// NewMemBroker creates a new in-memory broker system.
//
// The in-memory implementation of the job system has the specifity that
// workers are actually launched by the broker at its creation.
func NewMemBroker(domain string, ws WorkersList) Broker {
	memBrokersMu.Lock()
	defer memBrokersMu.Unlock()
	if memBrokers == nil {
		memBrokers = make(map[string]*MemBroker)
	}
	b, ok := memBrokers[domain]
	if ok {
		return b
	}
	queues := make(map[string]*MemQueue)
	for workerType, conf := range ws {
		q := NewMemQueue(domain, workerType)
		queues[workerType] = q
		w := &Worker{
			Domain: domain,
			Type:   workerType,
			Conf:   conf,
		}
		w.Start(q)
	}
	b = &MemBroker{
		domain: domain,
		queues: queues,
	}
	memBrokers[domain] = b
	return b
}

// Domain returns the broker's domain
func (b *MemBroker) Domain() string {
	return b.domain
}

// PushJob will produce a new Job with the given options and enqueue the job in
// the proper queue.
func (b *MemBroker) PushJob(req *JobRequest) (*JobInfos, <-chan *JobInfos, error) {
	workerType := req.WorkerType
	q, ok := b.queues[workerType]
	if !ok {
		return nil, nil, ErrUnknownWorker
	}
	jobch := make(chan *JobInfos, 2)
	j := &MemJob{
		infos: NewJobInfos(req),
		jobch: jobch,
	}
	if err := q.Enqueue(j); err != nil {
		return nil, nil, err
	}
	return j.Infos(), jobch, nil
}

// Infos returns the associated job infos
func (j *MemJob) Infos() *JobInfos {
	return j.infos
}

// AckConsumed sets the job infos state to Running an sends the new job infos
// on the channel.
func (j *MemJob) AckConsumed() error {
	j.infos.State = Running
	return j.asyncSend(false)
}

// Ack sets the job infos state to Done an sends the new job infos on the
// channel.
func (j *MemJob) Ack() error {
	j.infos.State = Done
	return j.asyncSend(true)
}

// Nack sets the job infos state to Errored, set the specified error has the
// error field and sends the new job infos on the channel.
func (j *MemJob) Nack(err error) error {
	j.infos.State = Errored
	j.infos.Error = err
	return j.asyncSend(true)
}

func (j *MemJob) asyncSend(closed bool) error {
	select {
	case j.jobch <- j.infos:
	default:
	}
	if closed {
		close(j.jobch)
	}
	return nil
}

// Marshal should not be used for a MemJob
func (j *MemJob) Marshal() ([]byte, error) {
	return nil, errors.New("should not be marshaled")
}

// Unmarshal should not be used for a MemJob
func (j *MemJob) Unmarshal() error {
	return errors.New("should not be unmarshaled")
}

var (
	_ Queue  = &MemQueue{}
	_ Broker = &MemBroker{}
	_ Job    = &MemJob{}
)
<commit_msg>Fix race condition on MemJobs infos field<commit_after>package jobs

import (
	"container/list"
	"errors"
	"sync"
)

var (
	memBrokers   map[string]*MemBroker
	memBrokersMu sync.Mutex
)

type (
	// MemQueue is a queue in-memory implementation of the Queue interface.
	MemQueue struct {
		MaxCapacity int

		jobs *list.List
		run  bool
		jmu  sync.RWMutex

		ch chan Job
		cl chan bool
	}

	// MemBroker is an in-memory broker implementation of the Broker interface.
	MemBroker struct {
		domain string
		queues map[string]*MemQueue
	}

	// MemJob struct contains all the parameters of a job.
	MemJob struct {
		infos *JobInfos
		infmu sync.RWMutex
		jobch chan *JobInfos
	}
)

// NewMemQueue creates and a new in-memory queue.
func NewMemQueue(domain, workerType string) *MemQueue {
	return &MemQueue{
		jobs: list.New(),
		ch:   make(chan Job),
		cl:   make(chan bool),
	}
}

// Enqueue into the queue
func (q *MemQueue) Enqueue(job Job) error {
	q.jmu.Lock()
	defer q.jmu.Unlock()
	q.jobs.PushBack(job)
	if !q.run {
		q.run = true
		go q.send()
	}
	return nil
}

func (q *MemQueue) send() {
	for {
		q.jmu.Lock()
		e := q.jobs.Front()
		if e == nil {
			q.run = false
			q.jmu.Unlock()
			return
		}
		q.jobs.Remove(e)
		q.jmu.Unlock()
		select {
		case q.ch <- e.Value.(Job):
			continue
		case <-q.cl:
			return
		}
	}
}

// Consume from the queue
func (q *MemQueue) Consume() (Job, error) {
	select {
	case job := <-q.ch:
		return job, nil
	case <-q.cl:
		return nil, ErrQueueClosed
	}
}

// Len returns the length of the queue
func (q *MemQueue) Len() int {
	q.jmu.RLock()
	defer q.jmu.RUnlock()
	return q.jobs.Len()
}

// Close closes the queue
func (q *MemQueue) Close() {
	close(q.cl)
}

// NewMemBroker creates a new in-memory broker system.
//
// The in-memory implementation of the job system has the specifity that
// workers are actually launched by the broker at its creation.
func NewMemBroker(domain string, ws WorkersList) Broker {
	memBrokersMu.Lock()
	defer memBrokersMu.Unlock()
	if memBrokers == nil {
		memBrokers = make(map[string]*MemBroker)
	}
	b, ok := memBrokers[domain]
	if ok {
		return b
	}
	queues := make(map[string]*MemQueue)
	for workerType, conf := range ws {
		q := NewMemQueue(domain, workerType)
		queues[workerType] = q
		w := &Worker{
			Domain: domain,
			Type:   workerType,
			Conf:   conf,
		}
		w.Start(q)
	}
	b = &MemBroker{
		domain: domain,
		queues: queues,
	}
	memBrokers[domain] = b
	return b
}

// Domain returns the broker's domain
func (b *MemBroker) Domain() string {
	return b.domain
}

// PushJob will produce a new Job with the given options and enqueue the job in
// the proper queue.
func (b *MemBroker) PushJob(req *JobRequest) (*JobInfos, <-chan *JobInfos, error) {
	workerType := req.WorkerType
	q, ok := b.queues[workerType]
	if !ok {
		return nil, nil, ErrUnknownWorker
	}
	jobch := make(chan *JobInfos, 2)
	infos := NewJobInfos(req)
	j := &MemJob{
		infos: infos,
		jobch: jobch,
	}
	if err := q.Enqueue(j); err != nil {
		return nil, nil, err
	}
	return infos, jobch, nil
}

// Infos returns the associated job infos
func (j *MemJob) Infos() *JobInfos {
	j.infmu.RLock()
	defer j.infmu.RUnlock()
	return j.infos
}

// AckConsumed sets the job infos state to Running an sends the new job infos
// on the channel.
func (j *MemJob) AckConsumed() error {
	j.infmu.Lock()
	job := *j.infos
	job.State = Running
	j.infos = &job
	j.infmu.Unlock()
	return j.asyncSend(job, false)
}

// Ack sets the job infos state to Done an sends the new job infos on the
// channel.
func (j *MemJob) Ack() error {
	j.infmu.Lock()
	job := *j.infos
	job.State = Done
	j.infos = &job
	j.infmu.Unlock()
	return j.asyncSend(job, true)
}

// Nack sets the job infos state to Errored, set the specified error has the
// error field and sends the new job infos on the channel.
func (j *MemJob) Nack(err error) error {
	j.infmu.Lock()
	job := *j.infos
	job.State = Errored
	job.Error = err
	j.infos = &job
	j.infmu.Unlock()
	return j.asyncSend(job, true)
}

func (j *MemJob) asyncSend(job JobInfos, closed bool) error {
	select {
	case j.jobch <- &job:
	default:
	}
	if closed {
		close(j.jobch)
	}
	return nil
}

// Marshal should not be used for a MemJob
func (j *MemJob) Marshal() ([]byte, error) {
	return nil, errors.New("should not be marshaled")
}

// Unmarshal should not be used for a MemJob
func (j *MemJob) Unmarshal() error {
	return errors.New("should not be unmarshaled")
}

var (
	_ Queue  = &MemQueue{}
	_ Broker = &MemBroker{}
	_ Job    = &MemJob{}
)
<|endoftext|>


<commit_before>package remote

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"path"
	"regexp"
	"runtime"
	"strings"

	"github.com/cozy/cozy-stack/pkg/config"
	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/logger"
	"github.com/cozy/echo"
)

var (
	// ErrNotFoundRemote is used when no request is defined for a doctype
	ErrNotFoundRemote = errors.New("the doctype has no request defined")
	// ErrInvalidRequest is used when we can't use the request defined by the
	// developer
	ErrInvalidRequest = errors.New("the request is not valid")
	// ErrRequestFailed is used when the connexion to the remote website can't
	// be established
	ErrRequestFailed = errors.New("can't connect to the remote host")
	// ErrInvalidVariables is used when the variables can't be extracted from
	// the request
	ErrInvalidVariables = errors.New("the variables are not valid")
	// ErrInvalidContentType is used when the response has a content-type that
	// we deny for security reasons
	ErrInvalidContentType = errors.New("the content-type for the response is not authorized")
)

const rawURL = "https://raw.githubusercontent.com/cozy/cozy-doctypes/master/%s/request"

// Doctype is used to describe a doctype, its request for a remote doctype for example
type Doctype struct {
	DocID   string `json:"_id,omitempty"`
	DocRev  string `json:"_rev,omitempty"`
	Request string `json:"request"`
}

// ID is used to implement the couchdb.Doc interface
func (d *Doctype) ID() string { return d.DocID }

// Rev is used to implement the couchdb.Doc interface
func (d *Doctype) Rev() string { return d.DocRev }

// SetID is used to implement the couchdb.Doc interface
func (d *Doctype) SetID(id string) { d.DocID = id }

// SetRev is used to implement the couchdb.Doc interface
func (d *Doctype) SetRev(rev string) { d.DocRev = rev }

// DocType implements couchdb.Doc
func (d *Doctype) DocType() string { return consts.Doctypes }

// Clone implements couchdb.Doc
func (d *Doctype) Clone() couchdb.Doc { cloned := *d; return &cloned }

// Request is used to log in couchdb a call to a remote website
type Request struct {
	DocID         string `json:"_id,omitempty"`
	DocRev        string `json:"_rev,omitempty"`
	RemoteDoctype string `json:"doctype"`
	Verb          string `json:"verb"`
	URL           string `json:"url"`
	ResponseCode  int    `json:"response_code"`
	ContentType   string `json:"content_type"`
}

// ID is used to implement the couchdb.Doc interface
func (r *Request) ID() string { return r.DocID }

// Rev is used to implement the couchdb.Doc interface
func (r *Request) Rev() string { return r.DocRev }

// SetID is used to implement the couchdb.Doc interface
func (r *Request) SetID(id string) { r.DocID = id }

// SetRev is used to implement the couchdb.Doc interface
func (r *Request) SetRev(rev string) { r.DocRev = rev }

// DocType implements couchdb.Doc
func (r *Request) DocType() string { return consts.RemoteRequests }

// Clone implements couchdb.Doc
func (r *Request) Clone() couchdb.Doc { cloned := *r; return &cloned }

// Remote is the struct used to call a remote website for a doctype
type Remote struct {
	Verb    string
	URL     *url.URL
	Headers map[string]string
	Body    string
}

var log = logger.WithNamespace("remote")

// ParseRawRequest takes a string and parse it as a remote struct.
// First line is verb and URL.
// Then, we have the headers.
// And for a POST, we have a blank line, and then the body.
func ParseRawRequest(doctype, raw string) (*Remote, error) {
	lines := strings.Split(raw, "\n")
	parts := strings.Split(lines[0], " ")
	if len(parts) != 2 {
		log.Infof("%s cannot be used as a remote doctype", doctype)
		return nil, ErrInvalidRequest
	}
	var remote Remote
	remote.Verb = parts[0]
	if remote.Verb != echo.GET && remote.Verb != echo.POST {
		log.Infof("Invalid verb for remote doctype %s: %s", doctype, remote.Verb)
		return nil, ErrInvalidRequest
	}
	u, err := url.Parse(parts[1])
	if err != nil {
		log.Infof("Invalid URL for remote doctype %s: %s", doctype, parts[1])
		return nil, ErrInvalidRequest
	}
	if u.Scheme != "https" && u.Scheme != "http" {
		log.Infof("Invalid scheme for remote doctype %s: %s", doctype, u.Scheme)
		return nil, ErrInvalidRequest
	}
	remote.URL = u
	remote.Headers = make(map[string]string)
	for i, line := range lines[1:] {
		if line == "" {
			if remote.Verb == echo.GET {
				continue
			}
			remote.Body = strings.Join(lines[i+2:], "\n")
			break
		}
		parts = strings.SplitN(line, ":", 2)
		if len(parts) != 2 {
			log.Infof("Invalid header for remote doctype %s: %s", doctype, line)
			return nil, ErrInvalidRequest
		}
		remote.Headers[parts[0]] = strings.TrimSpace(parts[1])
	}
	return &remote, nil
}

// Find finds the request defined for the given doctype
func Find(ins *instance.Instance, doctype string) (*Remote, error) {
	var raw string

	if config.GetConfig().Doctypes == "" {
		dt := Doctype{
			DocID: consts.Doctypes + "/" + doctype,
		}
		err := couchdb.GetDoc(ins, consts.Doctypes, dt.DocID, &dt)
		if err != nil {
			u := fmt.Sprintf(rawURL, doctype)
			res, err := http.Get(u)
			log.Debugf("Fetch remote doctype from %s\n", doctype)
			if err != nil {
				log.Infof("Request not found for remote doctype %s: %s", doctype, err)
				return nil, ErrNotFoundRemote
			}
			defer res.Body.Close()
			b, err := ioutil.ReadAll(res.Body)
			if err != nil {
				log.Infof("Request not found for remote doctype %s: %s", doctype, err)
				return nil, ErrNotFoundRemote
			}
			dt.Request = string(b)
			err = couchdb.CreateNamedDocWithDB(ins, &dt)
			if err != nil {
				log.Infof("Cannot save remote doctype %s: %s", doctype, err)
			}
		}
		raw = dt.Request
	} else {
		filename := path.Join(config.GetConfig().Doctypes, doctype, "request")
		bytes, err := ioutil.ReadFile(filename)
		if err != nil {
			return nil, ErrNotFoundRemote
		}
		raw = string(bytes)
	}

	return ParseRawRequest(doctype, raw)
}

// ExtractVariables extracts the variables:
// - from the query string for a GET
// - from the body formatted as JSON for a POST
func ExtractVariables(verb string, in *http.Request) (map[string]string, error) {
	vars := make(map[string]string)
	if verb == echo.GET {
		for k, v := range in.URL.Query() {
			vars[k] = v[0]
		}
	} else {
		err := json.NewDecoder(in.Body).Decode(&vars)
		if err != nil {
			return nil, err
		}
	}
	return vars, nil
}

var injectionRegexp = regexp.MustCompile(`{{\w+}}`)

func injectVar(src string, vars map[string]string) string {
	return injectionRegexp.ReplaceAllStringFunc(src, func(m string) string {
		m = strings.TrimLeft(m, "{{")
		m = strings.TrimRight(m, "}}")
		m = strings.TrimSpace(m)
		return vars[m]
	})
}

// InjectVariables replaces {{variable}} by its value in some fields of the
// remote struct
func InjectVariables(remote *Remote, vars map[string]string) {
	if strings.Contains(remote.URL.Path, "{{") {
		remote.URL.Path = injectVar(remote.URL.Path, vars)
	}
	if strings.Contains(remote.URL.RawQuery, "{{") {
		remote.URL.RawQuery = injectVar(remote.URL.RawQuery, vars)
	}
	for k, v := range remote.Headers {
		if strings.Contains(v, "{{") {
			remote.Headers[k] = injectVar(v, vars)
		}
	}
	if strings.Contains(remote.Body, "{{") {
		remote.Body = injectVar(remote.Body, vars)
	}
}

// ProxyTo calls the external website and proxy the reponse
func (remote *Remote) ProxyTo(doctype string, ins *instance.Instance, rw http.ResponseWriter, in *http.Request) error {
	vars, err := ExtractVariables(remote.Verb, in)
	if err != nil {
		log.Infof("Error on extracting variables: %s", err)
		return ErrInvalidVariables
	}
	InjectVariables(remote, vars)

	// Sanitize the remote URL
	if strings.Contains(remote.URL.Host, ":") {
		log.Infof("Invalid host for remote doctype %s: %s", doctype, remote.URL.Host)
		return ErrInvalidRequest
	}
	remote.URL.User = nil
	remote.URL.Fragment = ""

	req, err := http.NewRequest(remote.Verb, remote.URL.String(), nil)
	if err != nil {
		return ErrInvalidRequest
	}

	req.Header.Set("User-Agent", "cozy-stack "+config.Version+" ("+runtime.Version()+")")
	for k, v := range remote.Headers {
		req.Header.Set(k, v)
	}

	res, err := http.DefaultClient.Do(req)
	if err != nil {
		log.Infof("Error on request %s: %s", remote.URL.String(), err)
		return ErrRequestFailed
	}
	defer res.Body.Close()

	ctype := strings.SplitN(res.Header.Get("Content-Type"), ";", 2)[0] // Drop the charset
	if ctype != "application/json" && ctype != "text/xml" && ctype != "application/xml" {
		class := strings.SplitN(ctype, "/", 2)[0]
		if class != "image" && class != "audio" && class != "video" {
			log.Infof("request %s has a content-type that is not allowed: %s",
				remote.URL.String(), ctype)
			return ErrInvalidContentType
		}
	}

	logged := &Request{
		RemoteDoctype: doctype,
		Verb:          remote.Verb,
		URL:           remote.URL.String(),
		ResponseCode:  res.StatusCode,
		ContentType:   ctype,
	}
	err = couchdb.CreateDoc(ins, logged)
	if err != nil {
		log.Errorf("Can't save remote request: %s", err)
	}
	log.Debugf("Remote request: %#v\n", logged)

	rw.WriteHeader(res.StatusCode)
	copyHeader(rw.Header(), res.Header)
	_, err = io.Copy(rw, res.Body)
	if err != nil {
		log.Infof("Error on copying response from %s: %s", remote.URL.String(), err)
	}
	return nil
}

func copyHeader(dst, src http.Header) {
	for k, vv := range src {
		for _, v := range vv {
			dst.Add(k, v)
		}
	}
}

var (
	_ couchdb.Doc = (*Doctype)(nil)
	_ couchdb.Doc = (*Request)(nil)
)
<commit_msg>Use an HTTP client with a timeout for remote doctypes<commit_after>package remote

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"path"
	"regexp"
	"runtime"
	"strings"
	"time"

	"github.com/cozy/cozy-stack/pkg/config"
	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/logger"
	"github.com/cozy/echo"
)

var (
	// ErrNotFoundRemote is used when no request is defined for a doctype
	ErrNotFoundRemote = errors.New("the doctype has no request defined")
	// ErrInvalidRequest is used when we can't use the request defined by the
	// developer
	ErrInvalidRequest = errors.New("the request is not valid")
	// ErrRequestFailed is used when the connexion to the remote website can't
	// be established
	ErrRequestFailed = errors.New("can't connect to the remote host")
	// ErrInvalidVariables is used when the variables can't be extracted from
	// the request
	ErrInvalidVariables = errors.New("the variables are not valid")
	// ErrInvalidContentType is used when the response has a content-type that
	// we deny for security reasons
	ErrInvalidContentType = errors.New("the content-type for the response is not authorized")
)

const rawURL = "https://raw.githubusercontent.com/cozy/cozy-doctypes/master/%s/request"

// Doctype is used to describe a doctype, its request for a remote doctype for example
type Doctype struct {
	DocID   string `json:"_id,omitempty"`
	DocRev  string `json:"_rev,omitempty"`
	Request string `json:"request"`
}

// ID is used to implement the couchdb.Doc interface
func (d *Doctype) ID() string { return d.DocID }

// Rev is used to implement the couchdb.Doc interface
func (d *Doctype) Rev() string { return d.DocRev }

// SetID is used to implement the couchdb.Doc interface
func (d *Doctype) SetID(id string) { d.DocID = id }

// SetRev is used to implement the couchdb.Doc interface
func (d *Doctype) SetRev(rev string) { d.DocRev = rev }

// DocType implements couchdb.Doc
func (d *Doctype) DocType() string { return consts.Doctypes }

// Clone implements couchdb.Doc
func (d *Doctype) Clone() couchdb.Doc { cloned := *d; return &cloned }

// Request is used to log in couchdb a call to a remote website
type Request struct {
	DocID         string `json:"_id,omitempty"`
	DocRev        string `json:"_rev,omitempty"`
	RemoteDoctype string `json:"doctype"`
	Verb          string `json:"verb"`
	URL           string `json:"url"`
	ResponseCode  int    `json:"response_code"`
	ContentType   string `json:"content_type"`
}

// ID is used to implement the couchdb.Doc interface
func (r *Request) ID() string { return r.DocID }

// Rev is used to implement the couchdb.Doc interface
func (r *Request) Rev() string { return r.DocRev }

// SetID is used to implement the couchdb.Doc interface
func (r *Request) SetID(id string) { r.DocID = id }

// SetRev is used to implement the couchdb.Doc interface
func (r *Request) SetRev(rev string) { r.DocRev = rev }

// DocType implements couchdb.Doc
func (r *Request) DocType() string { return consts.RemoteRequests }

// Clone implements couchdb.Doc
func (r *Request) Clone() couchdb.Doc { cloned := *r; return &cloned }

// Remote is the struct used to call a remote website for a doctype
type Remote struct {
	Verb    string
	URL     *url.URL
	Headers map[string]string
	Body    string
}

var log = logger.WithNamespace("remote")

// ParseRawRequest takes a string and parse it as a remote struct.
// First line is verb and URL.
// Then, we have the headers.
// And for a POST, we have a blank line, and then the body.
func ParseRawRequest(doctype, raw string) (*Remote, error) {
	lines := strings.Split(raw, "\n")
	parts := strings.Split(lines[0], " ")
	if len(parts) != 2 {
		log.Infof("%s cannot be used as a remote doctype", doctype)
		return nil, ErrInvalidRequest
	}
	var remote Remote
	remote.Verb = parts[0]
	if remote.Verb != echo.GET && remote.Verb != echo.POST {
		log.Infof("Invalid verb for remote doctype %s: %s", doctype, remote.Verb)
		return nil, ErrInvalidRequest
	}
	u, err := url.Parse(parts[1])
	if err != nil {
		log.Infof("Invalid URL for remote doctype %s: %s", doctype, parts[1])
		return nil, ErrInvalidRequest
	}
	if u.Scheme != "https" && u.Scheme != "http" {
		log.Infof("Invalid scheme for remote doctype %s: %s", doctype, u.Scheme)
		return nil, ErrInvalidRequest
	}
	remote.URL = u
	remote.Headers = make(map[string]string)
	for i, line := range lines[1:] {
		if line == "" {
			if remote.Verb == echo.GET {
				continue
			}
			remote.Body = strings.Join(lines[i+2:], "\n")
			break
		}
		parts = strings.SplitN(line, ":", 2)
		if len(parts) != 2 {
			log.Infof("Invalid header for remote doctype %s: %s", doctype, line)
			return nil, ErrInvalidRequest
		}
		remote.Headers[parts[0]] = strings.TrimSpace(parts[1])
	}
	return &remote, nil
}

// Find finds the request defined for the given doctype
func Find(ins *instance.Instance, doctype string) (*Remote, error) {
	var raw string

	if config.GetConfig().Doctypes == "" {
		dt := Doctype{
			DocID: consts.Doctypes + "/" + doctype,
		}
		err := couchdb.GetDoc(ins, consts.Doctypes, dt.DocID, &dt)
		if err != nil {
			u := fmt.Sprintf(rawURL, doctype)
			res, err := http.Get(u)
			log.Debugf("Fetch remote doctype from %s\n", doctype)
			if err != nil {
				log.Infof("Request not found for remote doctype %s: %s", doctype, err)
				return nil, ErrNotFoundRemote
			}
			defer res.Body.Close()
			b, err := ioutil.ReadAll(res.Body)
			if err != nil {
				log.Infof("Request not found for remote doctype %s: %s", doctype, err)
				return nil, ErrNotFoundRemote
			}
			dt.Request = string(b)
			err = couchdb.CreateNamedDocWithDB(ins, &dt)
			if err != nil {
				log.Infof("Cannot save remote doctype %s: %s", doctype, err)
			}
		}
		raw = dt.Request
	} else {
		filename := path.Join(config.GetConfig().Doctypes, doctype, "request")
		bytes, err := ioutil.ReadFile(filename)
		if err != nil {
			return nil, ErrNotFoundRemote
		}
		raw = string(bytes)
	}

	return ParseRawRequest(doctype, raw)
}

// ExtractVariables extracts the variables:
// - from the query string for a GET
// - from the body formatted as JSON for a POST
func ExtractVariables(verb string, in *http.Request) (map[string]string, error) {
	vars := make(map[string]string)
	if verb == echo.GET {
		for k, v := range in.URL.Query() {
			vars[k] = v[0]
		}
	} else {
		err := json.NewDecoder(in.Body).Decode(&vars)
		if err != nil {
			return nil, err
		}
	}
	return vars, nil
}

var injectionRegexp = regexp.MustCompile(`{{\w+}}`)

func injectVar(src string, vars map[string]string) string {
	return injectionRegexp.ReplaceAllStringFunc(src, func(m string) string {
		m = strings.TrimLeft(m, "{{")
		m = strings.TrimRight(m, "}}")
		m = strings.TrimSpace(m)
		return vars[m]
	})
}

// InjectVariables replaces {{variable}} by its value in some fields of the
// remote struct
func InjectVariables(remote *Remote, vars map[string]string) {
	if strings.Contains(remote.URL.Path, "{{") {
		remote.URL.Path = injectVar(remote.URL.Path, vars)
	}
	if strings.Contains(remote.URL.RawQuery, "{{") {
		remote.URL.RawQuery = injectVar(remote.URL.RawQuery, vars)
	}
	for k, v := range remote.Headers {
		if strings.Contains(v, "{{") {
			remote.Headers[k] = injectVar(v, vars)
		}
	}
	if strings.Contains(remote.Body, "{{") {
		remote.Body = injectVar(remote.Body, vars)
	}
}

// ProxyTo calls the external website and proxy the reponse
func (remote *Remote) ProxyTo(doctype string, ins *instance.Instance, rw http.ResponseWriter, in *http.Request) error {
	vars, err := ExtractVariables(remote.Verb, in)
	if err != nil {
		log.Infof("Error on extracting variables: %s", err)
		return ErrInvalidVariables
	}
	InjectVariables(remote, vars)

	// Sanitize the remote URL
	if strings.Contains(remote.URL.Host, ":") {
		log.Infof("Invalid host for remote doctype %s: %s", doctype, remote.URL.Host)
		return ErrInvalidRequest
	}
	remote.URL.User = nil
	remote.URL.Fragment = ""

	req, err := http.NewRequest(remote.Verb, remote.URL.String(), nil)
	if err != nil {
		return ErrInvalidRequest
	}

	req.Header.Set("User-Agent", "cozy-stack "+config.Version+" ("+runtime.Version()+")")
	for k, v := range remote.Headers {
		req.Header.Set(k, v)
	}

	client := http.Client{Timeout: 20 * time.Second}
	res, err := client.Do(req)
	if err != nil {
		log.Infof("Error on request %s: %s", remote.URL.String(), err)
		return ErrRequestFailed
	}
	defer res.Body.Close()

	ctype := strings.SplitN(res.Header.Get("Content-Type"), ";", 2)[0] // Drop the charset
	if ctype != "application/json" && ctype != "text/xml" && ctype != "application/xml" {
		class := strings.SplitN(ctype, "/", 2)[0]
		if class != "image" && class != "audio" && class != "video" {
			log.Infof("request %s has a content-type that is not allowed: %s",
				remote.URL.String(), ctype)
			return ErrInvalidContentType
		}
	}

	logged := &Request{
		RemoteDoctype: doctype,
		Verb:          remote.Verb,
		URL:           remote.URL.String(),
		ResponseCode:  res.StatusCode,
		ContentType:   ctype,
	}
	err = couchdb.CreateDoc(ins, logged)
	if err != nil {
		log.Errorf("Can't save remote request: %s", err)
	}
	log.Debugf("Remote request: %#v\n", logged)

	rw.WriteHeader(res.StatusCode)
	copyHeader(rw.Header(), res.Header)
	_, err = io.Copy(rw, res.Body)
	if err != nil {
		log.Infof("Error on copying response from %s: %s", remote.URL.String(), err)
	}
	return nil
}

func copyHeader(dst, src http.Header) {
	for k, vv := range src {
		for _, v := range vv {
			dst.Add(k, v)
		}
	}
}

var (
	_ couchdb.Doc = (*Doctype)(nil)
	_ couchdb.Doc = (*Request)(nil)
)
<|endoftext|>


<commit_before>package server

import (
	"fmt"
	"github.com/Aptomi/aptomi/pkg/api"
	"github.com/Aptomi/aptomi/pkg/api/middleware"
	"github.com/Aptomi/aptomi/pkg/config"
	"github.com/Aptomi/aptomi/pkg/external"
	"github.com/Aptomi/aptomi/pkg/external/secrets"
	"github.com/Aptomi/aptomi/pkg/external/users"
	"github.com/Aptomi/aptomi/pkg/runtime"
	"github.com/Aptomi/aptomi/pkg/runtime/store"
	"github.com/Aptomi/aptomi/pkg/runtime/store/core"
	"github.com/Aptomi/aptomi/pkg/runtime/store/generic/bolt"
	"github.com/Aptomi/aptomi/pkg/server/ui"
	log "github.com/Sirupsen/logrus"
	"github.com/gorilla/handlers"
	"github.com/julienschmidt/httprouter"
	"github.com/rs/cors"
	"net/http"
	"os"
	"time"
)

// Server is Aptomi server. It serves UI front-end, API calls, as well as does policy resolution & continuous state enforcement
type Server struct {
	cfg              *config.Server
	backgroundErrors chan string

	externalData *external.Data
	store        store.Core

	httpServer *http.Server

	enforcementIdx uint
}

// NewServer creates a new Aptomi Server
func NewServer(cfg *config.Server) *Server {
	s := &Server{
		cfg:              cfg,
		backgroundErrors: make(chan string),
	}

	return s
}

// Start initializes Aptomi server, starts API & UI processing, and as well as runs the required background jobs for
// continuous policy resolution and state enforcement
func (server *Server) Start() {
	server.initStore()
	server.initExternalData()

	// See if policy initialization needs to happen on the first run
	server.initPolicyOnFirstRun()

	// Start API, UI and Enforcer
	server.startHTTPServer()
	server.startEnforcer()

	// Wait for jobs to complete (it essentially hangs forever)
	server.wait()
}

func (server *Server) initPolicyOnFirstRun() {
	policy, _, err := server.store.GetPolicy(runtime.LastGen)
	if err != nil {
		panic(fmt.Sprintf("error while getting latest policy: %s", err))
	}

	// if policy does not exist, let's create the first version (it should be created here, before we start the server)
	if policy == nil {
		log.Infof("Policy not found in the store (likely, it's a first run of Aptomi server). Creating empty policy")
		err := server.store.InitPolicy()
		if err != nil {
			panic(fmt.Sprintf("error while creating empty policy: %s", err))
		}
	}
}

func (server *Server) initExternalData() {
	var userLoaders []users.UserLoader
	for _, ldap := range server.cfg.Users.LDAP {
		userLoaders = append(userLoaders, users.NewUserLoaderFromLDAP(ldap, server.cfg.DomainAdminOverrides))
	}
	for _, file := range server.cfg.Users.File {
		userLoaders = append(userLoaders, users.NewUserLoaderFromFile(file, server.cfg.DomainAdminOverrides))
	}
	server.externalData = external.NewData(
		users.NewUserLoaderMultipleSources(userLoaders),
		secrets.NewSecretLoaderFromDir(server.cfg.SecretsDir),
	)
}

func (server *Server) initStore() {
	registry := runtime.NewRegistry().Append(store.Objects...)
	b := bolt.NewGenericStore(registry)
	err := b.Open(server.cfg.DB)
	if err != nil {
		panic(fmt.Sprintf("Can't open object store: %s", err))
	}
	server.store = core.NewStore(b)
}

func (server *Server) startHTTPServer() {
	router := httprouter.New()

	api.Serve(router, server.store, server.externalData)
	server.serveUI(router)

	var handler http.Handler = router

	// todo write to logrus
	handler = handlers.CombinedLoggingHandler(os.Stdout, handler) // todo(slukjanov): make it at least somehow configurable - for example, select file to write to with rotation
	handler = cors.Default().Handler(handler)
	handler = middleware.NewPanicHandler(handler)
	// todo(slukjanov): add configurable handlers.ProxyHeaders to f behind the nginx or any other proxy
	// todo(slukjanov): add compression handler and compress by default in client

	server.httpServer = &http.Server{
		Handler:      handler,
		Addr:         server.cfg.API.ListenAddr(),
		WriteTimeout: 5 * time.Second,
		ReadTimeout:  30 * time.Second,
	}

	// Start HTTP server
	server.runInBackground("HTTP Server / API", true, func() {
		panic(server.httpServer.ListenAndServe())
	})
}

func (server *Server) serveUI(router *httprouter.Router) {
	if !server.cfg.UI.Enable {
		log.Infof("UI isn't enabled. UI will not be served")
		return
	}

	fileServer := http.FileServer(ui.HTTP)

	router.GET("/", func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
		req.URL.Path = "/"
		fileServer.ServeHTTP(w, req)
	})

	router.GET("/index.html", func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
		req.URL.Path = "/"
		fileServer.ServeHTTP(w, req)
	})

	router.GET("/static/*filepath", func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
		req.URL.Path = "/static/" + ps.ByName("filepath")
		fileServer.ServeHTTP(w, req)
	})
}

func (server *Server) startEnforcer() {
	// Start policy enforcement job
	if !server.cfg.Enforcer.Disabled {
		server.runInBackground("Policy Enforcer", true, func() {
			panic(server.enforceLoop())
		})
	}
}
<commit_msg>Fix shadowed err declaration<commit_after>package server

import (
	"fmt"
	"github.com/Aptomi/aptomi/pkg/api"
	"github.com/Aptomi/aptomi/pkg/api/middleware"
	"github.com/Aptomi/aptomi/pkg/config"
	"github.com/Aptomi/aptomi/pkg/external"
	"github.com/Aptomi/aptomi/pkg/external/secrets"
	"github.com/Aptomi/aptomi/pkg/external/users"
	"github.com/Aptomi/aptomi/pkg/runtime"
	"github.com/Aptomi/aptomi/pkg/runtime/store"
	"github.com/Aptomi/aptomi/pkg/runtime/store/core"
	"github.com/Aptomi/aptomi/pkg/runtime/store/generic/bolt"
	"github.com/Aptomi/aptomi/pkg/server/ui"
	log "github.com/Sirupsen/logrus"
	"github.com/gorilla/handlers"
	"github.com/julienschmidt/httprouter"
	"github.com/rs/cors"
	"net/http"
	"os"
	"time"
)

// Server is Aptomi server. It serves UI front-end, API calls, as well as does policy resolution & continuous state enforcement
type Server struct {
	cfg              *config.Server
	backgroundErrors chan string

	externalData *external.Data
	store        store.Core

	httpServer *http.Server

	enforcementIdx uint
}

// NewServer creates a new Aptomi Server
func NewServer(cfg *config.Server) *Server {
	s := &Server{
		cfg:              cfg,
		backgroundErrors: make(chan string),
	}

	return s
}

// Start initializes Aptomi server, starts API & UI processing, and as well as runs the required background jobs for
// continuous policy resolution and state enforcement
func (server *Server) Start() {
	server.initStore()
	server.initExternalData()

	// See if policy initialization needs to happen on the first run
	server.initPolicyOnFirstRun()

	// Start API, UI and Enforcer
	server.startHTTPServer()
	server.startEnforcer()

	// Wait for jobs to complete (it essentially hangs forever)
	server.wait()
}

func (server *Server) initPolicyOnFirstRun() {
	policy, _, err := server.store.GetPolicy(runtime.LastGen)
	if err != nil {
		panic(fmt.Sprintf("error while getting latest policy: %s", err))
	}

	// if policy does not exist, let's create the first version (it should be created here, before we start the server)
	if policy == nil {
		log.Infof("Policy not found in the store (likely, it's a first run of Aptomi server). Creating empty policy")
		initErr := server.store.InitPolicy()
		if initErr != nil {
			panic(fmt.Sprintf("error while creating empty policy: %s", initErr))
		}
	}
}

func (server *Server) initExternalData() {
	var userLoaders []users.UserLoader
	for _, ldap := range server.cfg.Users.LDAP {
		userLoaders = append(userLoaders, users.NewUserLoaderFromLDAP(ldap, server.cfg.DomainAdminOverrides))
	}
	for _, file := range server.cfg.Users.File {
		userLoaders = append(userLoaders, users.NewUserLoaderFromFile(file, server.cfg.DomainAdminOverrides))
	}
	server.externalData = external.NewData(
		users.NewUserLoaderMultipleSources(userLoaders),
		secrets.NewSecretLoaderFromDir(server.cfg.SecretsDir),
	)
}

func (server *Server) initStore() {
	registry := runtime.NewRegistry().Append(store.Objects...)
	b := bolt.NewGenericStore(registry)
	err := b.Open(server.cfg.DB)
	if err != nil {
		panic(fmt.Sprintf("Can't open object store: %s", err))
	}
	server.store = core.NewStore(b)
}

func (server *Server) startHTTPServer() {
	router := httprouter.New()

	api.Serve(router, server.store, server.externalData)
	server.serveUI(router)

	var handler http.Handler = router

	// todo write to logrus
	handler = handlers.CombinedLoggingHandler(os.Stdout, handler) // todo(slukjanov): make it at least somehow configurable - for example, select file to write to with rotation
	handler = cors.Default().Handler(handler)
	handler = middleware.NewPanicHandler(handler)
	// todo(slukjanov): add configurable handlers.ProxyHeaders to f behind the nginx or any other proxy
	// todo(slukjanov): add compression handler and compress by default in client

	server.httpServer = &http.Server{
		Handler:      handler,
		Addr:         server.cfg.API.ListenAddr(),
		WriteTimeout: 5 * time.Second,
		ReadTimeout:  30 * time.Second,
	}

	// Start HTTP server
	server.runInBackground("HTTP Server / API", true, func() {
		panic(server.httpServer.ListenAndServe())
	})
}

func (server *Server) serveUI(router *httprouter.Router) {
	if !server.cfg.UI.Enable {
		log.Infof("UI isn't enabled. UI will not be served")
		return
	}

	fileServer := http.FileServer(ui.HTTP)

	router.GET("/", func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
		req.URL.Path = "/"
		fileServer.ServeHTTP(w, req)
	})

	router.GET("/index.html", func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
		req.URL.Path = "/"
		fileServer.ServeHTTP(w, req)
	})

	router.GET("/static/*filepath", func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
		req.URL.Path = "/static/" + ps.ByName("filepath")
		fileServer.ServeHTTP(w, req)
	})
}

func (server *Server) startEnforcer() {
	// Start policy enforcement job
	if !server.cfg.Enforcer.Disabled {
		server.runInBackground("Policy Enforcer", true, func() {
			panic(server.enforceLoop())
		})
	}
}
<|endoftext|>


<commit_before>package server

import (
	jnet "doozer/net"
	"doozer/paxos"
	"doozer/proto"
	"doozer/store"
	"doozer/util"
	"net"
	"os"
	"reflect"
	"strconv"
	"strings"
	"time"
)

const packetSize = 3000

const lease = 3e9 // ns == 3s

var (
	ErrBadPrefix = os.NewError("bad prefix in path")
	ErrNoWrite = os.NewError("no known writeable address")
	responded = os.NewError("already responded")
)

const (
	Ok = proto.Line("OK")
)

type conn struct {
	*proto.Conn
	c   net.Conn
	s   *Server
	cal bool
}

type Manager interface {
	paxos.Proposer
	PutFrom(string, paxos.Msg)
	Alpha() int
}

type Server struct {
	Addr         string
	St           *store.Store
	Mg           Manager
	Self, Prefix string
}

func (sv *Server) ListenAndServeUdp(outs chan paxos.Packet) os.Error {
	logger := util.NewLogger("udp server %s", sv.Addr)

	logger.Println("binding")
	u, err := net.ListenPacket("udp", sv.Addr)
	if err != nil {
		logger.Println(err)
		return err
	}
	defer u.Close()
	logger.Println("listening")

	err = sv.ServeUdp(u, outs)
	if err != nil {
		logger.Printf("%s: %s", u, err)
	}
	return err
}

func (sv *Server) ServeUdp(u jnet.Conn, outs chan paxos.Packet) os.Error {
	r := jnet.Ackify(u, outs)

	for p := range r {
		sv.Mg.PutFrom(p.Addr, p.Msg)
	}

	panic("unreachable")
}

var clg = util.NewLogger("cal")

func (s *Server) Serve(l net.Listener, cal chan int) os.Error {
	var ok bool
	for {
		if !ok {
			_, ok = <-cal
		}
		clg.Println(ok)

		rw, e := l.Accept()
		if e != nil {
			return e
		}
		c := &conn{proto.NewConn(rw), rw, s, ok}
		go c.serve()
	}

	panic("unreachable")
}

func (sv *Server) leader() string {
	parts, cas := sv.St.Get("/doozer/leader")
	if cas == store.Dir && cas == store.Missing {
		return ""
	}
	return parts[0]
}

func (sv *Server) addrFor(id string) string {
	parts, cas := sv.St.Get("/doozer/members/" + id)
	if cas == store.Dir && cas == store.Missing {
		return ""
	}
	return parts[0]
}

// Checks that path begins with the proper prefix and returns the short path
// without the prefix.
func (sv *Server) checkPath(path string) (string, os.Error) {
	logger := util.NewLogger("checkPath")
	if !strings.HasPrefix(path, sv.Prefix+"/") {
		logger.Printf("prefix %q not in %q", sv.Prefix+"/", path)
		return "", ErrBadPrefix
	}
	return path[len(sv.Prefix):], nil
}

// Repeatedly propose nop values until a successful read from `done`.
func (sv *Server) AdvanceUntil(done chan int) {
	for _, ok := <-done; !ok; _, ok = <-done {
		sv.Mg.Propose(store.Nop)
	}
}

func (c *conn) redirect(rid uint) {
	leader := c.s.leader()
	addr := c.s.addrFor(leader)
	if addr == "" {
		c.SendResponse(rid, proto.Last, ErrNoWrite)
	} else {
		c.SendResponse(rid, proto.Last, proto.Redirect(addr))
	}
}

func get(c *conn, _ uint, data interface{}) (interface{}, os.Error) {
	r := data.(*proto.ReqGet)
	shortPath, err := c.s.checkPath(r.Path)
	if err != nil {
		return nil, err
	}

	v, cas := c.s.St.Get(shortPath)
	return proto.ResGet{v, cas}, nil
}

func sget(c *conn, _ uint, data interface{}) (interface{}, os.Error) {
	r := data.(*proto.ReqGet)
	shortPath, err := c.s.checkPath(r.Path)
	if err != nil {
		return "", err
	}

	body, err := store.GetString(c.s.St.SyncPath(shortPath), shortPath), nil
	if err != nil {
		return nil, err
	}

	return body, nil
}

func set(c *conn, _ uint, data interface{}) (interface{}, os.Error) {
	r := data.(*proto.ReqSet)

	shortPath, err := c.s.checkPath(r.Path)
	if err != nil {
		return nil, err
	}

	seqn, _, err := paxos.Set(c.s.Mg, shortPath, r.Body, r.Cas)
	if err != nil {
		return nil, err
	}
	return seqn, nil
}

func del(c *conn, _ uint, data interface{}) (interface{}, os.Error) {
	r := data.(*proto.ReqDel)

	shortPath, err := c.s.checkPath(r.Path)
	if err != nil {
		return 0, err
	}

	seqn, err := paxos.Del(c.s.Mg, shortPath, r.Cas)
	if err != nil {
		return nil, err
	}

	return seqn, nil
}

func nop(c *conn, _ uint, data interface{}) (interface{}, os.Error) {
	c.s.Mg.Propose(store.Nop)
	return nil, nil
}

func join(c *conn, _ uint, data interface{}) (interface{}, os.Error) {
	r := data.(*proto.ReqJoin)
	key := "/doozer/members/" + r.Who
	seqn, _, err := paxos.Set(c.s.Mg, key, r.Addr, store.Missing)
	if err != nil {
		return nil, err
	}

	done := make(chan int)
	go c.s.AdvanceUntil(done)
	c.s.St.Sync(seqn + uint64(c.s.Mg.Alpha()))
	close(done)
	seqn, snap := c.s.St.Snapshot()
	return proto.ResJoin{seqn, snap}, nil
}

func checkin(c *conn, _ uint, data interface{}) (interface{}, os.Error) {
	r := data.(*proto.ReqCheckin)
	t := time.Nanoseconds() + lease
	_, cas, err := paxos.Set(c.s.Mg, "/session/"+r.Sid, strconv.Itoa64(t), r.Cas)
	if err != nil {
		return nil, err
	}
	return proto.ResCheckin{t, cas}, nil
}

func closeOp(c *conn, _ uint, data interface{}) (interface{}, os.Error) {
	err := c.CloseResponse(data.(uint))
	if err != nil {
		return nil, err
	}
	return Ok, nil
}

func watch(c *conn, id uint, data interface{}) (interface{}, os.Error) {
	glob := data.(string)
	// TODO check glob pattern for errors
	ch := c.s.St.Watch(glob)
	// TODO buffer (and possibly discard) events
	for ev := range ch {
		var r proto.ResWatch
		r.Path = ev.Path
		r.Body = ev.Body
		r.Cas = ev.Cas
		err := c.SendResponse(id, 0, r)
		if err == proto.ErrClosed {
			close(ch)
			err = nil
		}
		if err != nil {
			// TODO log error
			break
		}
	}
	return nil, responded
}

func indirect(x interface{}) interface{} {
	return reflect.Indirect(reflect.NewValue(x)).Interface()
}

type handler func(*conn, uint, interface{}) (interface{}, os.Error)

type op struct {
	p interface{}
	f handler

	redirect bool
}

var ops = map[string]op{
	// new stuff, see doc/proto.md
	"CLOSE": {p: new(uint), f: closeOp},
	"WATCH": {p: new(string), f: watch},

	// former stuff
	"get":     {p: new(*proto.ReqGet), f: get},
	"sget":    {p: new(*proto.ReqGet), f: sget},
	"set":     {p: new(*proto.ReqSet), f: set, redirect: true},
	"del":     {p: new(*proto.ReqDel), f: del, redirect: true},
	"nop":     {p: new(*[]interface{}), f: nop, redirect: true},
	"join":    {p: new(*proto.ReqJoin), f: join, redirect: true},
	"checkin": {p: new(*proto.ReqCheckin), f: checkin, redirect: true},
}

func (c *conn) handle(rid uint, f handler, data interface{}) {
	res, err := f(c, rid, data)
	if err == responded {
		return
	}

	if err != nil {
		c.SendResponse(rid, proto.Last, err)
	} else {
		c.SendResponse(rid, proto.Last, res)
	}
}

func (c *conn) serve() {
	logger := util.NewLogger("%v", c.c.RemoteAddr())
	logger.Println("accepted connection")
	for {
		rid, verb, data, err := c.ReadRequest()
		if err != nil {
			if err == os.EOF {
				logger.Println("connection closed by peer")
			} else {
				logger.Println(err)
			}
			return
		}

		rlogger := util.NewLogger("%v - req [%d]", c.c.RemoteAddr(), rid)

		if o, ok := ops[verb]; ok {
			rlogger.Printf("%s %v", verb, data)

			err := proto.Fit(data, o.p)
			if err != nil {
				c.SendResponse(rid, proto.Last, err)
				continue
			}

			if o.redirect && !c.cal {
				c.redirect(rid)
				continue
			}

			go c.handle(rid, o.f, indirect(o.p))
			continue
		}

		rlogger.Printf("unknown command <%s>", verb)
		c.SendResponse(rid, proto.Last, os.ErrorString(proto.InvalidCommand+" "+verb))
	}
}
<commit_msg>refactor<commit_after>package server

import (
	jnet "doozer/net"
	"doozer/paxos"
	"doozer/proto"
	"doozer/store"
	"doozer/util"
	"net"
	"os"
	"reflect"
	"strconv"
	"strings"
	"time"
)

const packetSize = 3000

const lease = 3e9 // ns == 3s

var (
	ErrBadPrefix = os.NewError("bad prefix in path")
	ErrNoWrite = os.NewError("no known writeable address")
	responded = os.NewError("already responded")
)

const (
	Ok = proto.Line("OK")
)

type conn struct {
	*proto.Conn
	c   net.Conn
	s   *Server
	cal bool
}

type Manager interface {
	paxos.Proposer
	PutFrom(string, paxos.Msg)
	Alpha() int
}

type Server struct {
	Addr         string
	St           *store.Store
	Mg           Manager
	Self, Prefix string
}

func (sv *Server) ListenAndServeUdp(outs chan paxos.Packet) os.Error {
	logger := util.NewLogger("udp server %s", sv.Addr)

	logger.Println("binding")
	u, err := net.ListenPacket("udp", sv.Addr)
	if err != nil {
		logger.Println(err)
		return err
	}
	defer u.Close()
	logger.Println("listening")

	err = sv.ServeUdp(u, outs)
	if err != nil {
		logger.Printf("%s: %s", u, err)
	}
	return err
}

func (sv *Server) ServeUdp(u jnet.Conn, outs chan paxos.Packet) os.Error {
	r := jnet.Ackify(u, outs)

	for p := range r {
		sv.Mg.PutFrom(p.Addr, p.Msg)
	}

	panic("unreachable")
}

var clg = util.NewLogger("cal")

func (s *Server) Serve(l net.Listener, cal chan int) os.Error {
	var ok bool
	for {
		if !ok {
			_, ok = <-cal
		}
		clg.Println(ok)

		rw, e := l.Accept()
		if e != nil {
			return e
		}
		c := &conn{proto.NewConn(rw), rw, s, ok}
		go c.serve()
	}

	panic("unreachable")
}

func (sv *Server) leader() string {
	parts, cas := sv.St.Get("/doozer/leader")
	if cas == store.Dir && cas == store.Missing {
		return ""
	}
	return parts[0]
}

func (sv *Server) addrFor(id string) string {
	parts, cas := sv.St.Get("/doozer/members/" + id)
	if cas == store.Dir && cas == store.Missing {
		return ""
	}
	return parts[0]
}

// Checks that path begins with the proper prefix and returns the short path
// without the prefix.
func (sv *Server) checkPath(path string) (string, os.Error) {
	logger := util.NewLogger("checkPath")
	if !strings.HasPrefix(path, sv.Prefix+"/") {
		logger.Printf("prefix %q not in %q", sv.Prefix+"/", path)
		return "", ErrBadPrefix
	}
	return path[len(sv.Prefix):], nil
}

// Repeatedly propose nop values until a successful read from `done`.
func (sv *Server) AdvanceUntil(done chan int) {
	for _, ok := <-done; !ok; _, ok = <-done {
		sv.Mg.Propose(store.Nop)
	}
}

func (c *conn) redirect(rid uint) {
	leader := c.s.leader()
	addr := c.s.addrFor(leader)
	if addr == "" {
		c.SendResponse(rid, proto.Last, ErrNoWrite)
	} else {
		c.SendResponse(rid, proto.Last, proto.Redirect(addr))
	}
}

func get(c *conn, _ uint, data interface{}) interface{} {
	r := data.(*proto.ReqGet)
	shortPath, err := c.s.checkPath(r.Path)
	if err != nil {
		return err
	}

	v, cas := c.s.St.Get(shortPath)
	return proto.ResGet{v, cas}
}

func sget(c *conn, _ uint, data interface{}) interface{} {
	r := data.(*proto.ReqGet)
	shortPath, err := c.s.checkPath(r.Path)
	if err != nil {
		return err
	}

	body, err := store.GetString(c.s.St.SyncPath(shortPath), shortPath), nil
	if err != nil {
		return err
	}

	return body
}

func set(c *conn, _ uint, data interface{}) interface{} {
	r := data.(*proto.ReqSet)

	shortPath, err := c.s.checkPath(r.Path)
	if err != nil {
		return err
	}

	seqn, _, err := paxos.Set(c.s.Mg, shortPath, r.Body, r.Cas)
	if err != nil {
		return err
	}
	return seqn
}

func del(c *conn, _ uint, data interface{}) interface{} {
	r := data.(*proto.ReqDel)

	shortPath, err := c.s.checkPath(r.Path)
	if err != nil {
		return err
	}

	seqn, err := paxos.Del(c.s.Mg, shortPath, r.Cas)
	if err != nil {
		return err
	}

	return seqn
}

func nop(c *conn, _ uint, data interface{}) interface{} {
	c.s.Mg.Propose(store.Nop)
	return nil
}

func join(c *conn, _ uint, data interface{}) interface{} {
	r := data.(*proto.ReqJoin)
	key := "/doozer/members/" + r.Who
	seqn, _, err := paxos.Set(c.s.Mg, key, r.Addr, store.Missing)
	if err != nil {
		return err
	}

	done := make(chan int)
	go c.s.AdvanceUntil(done)
	c.s.St.Sync(seqn + uint64(c.s.Mg.Alpha()))
	close(done)
	seqn, snap := c.s.St.Snapshot()
	return proto.ResJoin{seqn, snap}
}

func checkin(c *conn, _ uint, data interface{}) interface{} {
	r := data.(*proto.ReqCheckin)
	t := time.Nanoseconds() + lease
	_, cas, err := paxos.Set(c.s.Mg, "/session/"+r.Sid, strconv.Itoa64(t), r.Cas)
	if err != nil {
		return err
	}
	return proto.ResCheckin{t, cas}
}

func closeOp(c *conn, _ uint, data interface{}) interface{} {
	err := c.CloseResponse(data.(uint))
	if err != nil {
		return err
	}
	return Ok
}

func watch(c *conn, id uint, data interface{}) interface{} {
	glob := data.(string)
	// TODO check glob pattern for errors
	ch := c.s.St.Watch(glob)
	// TODO buffer (and possibly discard) events
	for ev := range ch {
		var r proto.ResWatch
		r.Path = ev.Path
		r.Body = ev.Body
		r.Cas = ev.Cas
		err := c.SendResponse(id, 0, r)
		if err == proto.ErrClosed {
			close(ch)
			err = nil
		}
		if err != nil {
			// TODO log error
			break
		}
	}
	return responded
}

func indirect(x interface{}) interface{} {
	return reflect.Indirect(reflect.NewValue(x)).Interface()
}

type handler func(*conn, uint, interface{}) interface{}

type op struct {
	p interface{}
	f handler

	redirect bool
}

var ops = map[string]op{
	// new stuff, see doc/proto.md
	"CLOSE": {p: new(uint), f: closeOp},
	"WATCH": {p: new(string), f: watch},

	// former stuff
	"get":     {p: new(*proto.ReqGet), f: get},
	"sget":    {p: new(*proto.ReqGet), f: sget},
	"set":     {p: new(*proto.ReqSet), f: set, redirect: true},
	"del":     {p: new(*proto.ReqDel), f: del, redirect: true},
	"nop":     {p: new(*[]interface{}), f: nop, redirect: true},
	"join":    {p: new(*proto.ReqJoin), f: join, redirect: true},
	"checkin": {p: new(*proto.ReqCheckin), f: checkin, redirect: true},
}

func (c *conn) handle(rid uint, f handler, data interface{}) {
	res := f(c, rid, data)
	if res == responded {
		return
	}

	c.SendResponse(rid, proto.Last, res)
}

func (c *conn) serve() {
	logger := util.NewLogger("%v", c.c.RemoteAddr())
	logger.Println("accepted connection")
	for {
		rid, verb, data, err := c.ReadRequest()
		if err != nil {
			if err == os.EOF {
				logger.Println("connection closed by peer")
			} else {
				logger.Println(err)
			}
			return
		}

		rlogger := util.NewLogger("%v - req [%d]", c.c.RemoteAddr(), rid)

		if o, ok := ops[verb]; ok {
			rlogger.Printf("%s %v", verb, data)

			err := proto.Fit(data, o.p)
			if err != nil {
				c.SendResponse(rid, proto.Last, err)
				continue
			}

			if o.redirect && !c.cal {
				c.redirect(rid)
				continue
			}

			go c.handle(rid, o.f, indirect(o.p))
			continue
		}

		rlogger.Printf("unknown command <%s>", verb)
		c.SendResponse(rid, proto.Last, os.ErrorString(proto.InvalidCommand+" "+verb))
	}
}
<|endoftext|>


<commit_before>/*
 * Mini Object Storage, (C) 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package fs

import (
	"bytes"
	"encoding/json"
	"io"
	"io/ioutil"
	"os"
	"path"
	"strings"
	"sync"

	mstorage "github.com/minio-io/minio/pkg/storage"
	"github.com/minio-io/minio/pkg/utils/policy"
)

type storage struct {
	root      string
	writeLock sync.Mutex
}

type SerializedMetadata struct {
	ContentType string
}

type MkdirFailedError struct{}

func (self MkdirFailedError) Error() string {
	return "Mkdir Failed"
}

func Start(root string) (chan<- string, <-chan error, *storage) {
	ctrlChannel := make(chan string)
	errorChannel := make(chan error)
	go start(ctrlChannel, errorChannel)
	return ctrlChannel, errorChannel, &storage{root: root}
}

func start(ctrlChannel <-chan string, errorChannel chan<- error) {
	close(errorChannel)
}

// Bucket Operations

func (storage *storage) ListBuckets() ([]mstorage.BucketMetadata, error) {
	files, err := ioutil.ReadDir(storage.root)
	if err != nil {
		return []mstorage.BucketMetadata{}, mstorage.EmbedError("bucket", "", err)
	}

	var metadataList []mstorage.BucketMetadata
	for _, file := range files {
		if !file.IsDir() {
			return []mstorage.BucketMetadata{}, mstorage.BackendCorrupted{Path: storage.root}
		}
		metadata := mstorage.BucketMetadata{
			Name:    file.Name(),
			Created: file.ModTime(), // TODO - provide real created time
		}
		metadataList = append(metadataList, metadata)
	}
	return metadataList, nil
}

func (storage *storage) StoreBucket(bucket string) error {
	storage.writeLock.Lock()
	defer storage.writeLock.Unlock()

	// verify bucket path legal
	if mstorage.IsValidBucket(bucket) == false {
		return mstorage.BucketNameInvalid{Bucket: bucket}
	}

	// get bucket path
	bucketDir := path.Join(storage.root, bucket)

	// check if bucket exists
	if _, err := os.Stat(bucketDir); err == nil {
		return mstorage.BucketExists{
			Bucket: bucket,
		}
	}

	// make bucket
	err := os.Mkdir(bucketDir, 0700)
	if err != nil {
		return mstorage.EmbedError(bucket, "", err)
	}
	return nil
}

func (storage *storage) GetBucketPolicy(bucket string) (interface{}, error) {
	storage.writeLock.Lock()
	defer storage.writeLock.Unlock()

	var p policy.BucketPolicy
	// verify bucket path legal
	if mstorage.IsValidBucket(bucket) == false {
		return policy.BucketPolicy{}, mstorage.BucketNameInvalid{Bucket: bucket}
	}

	// get bucket path
	bucketDir := path.Join(storage.root, bucket)
	// check if bucket exists
	if _, err := os.Stat(bucketDir); err != nil {
		return policy.BucketPolicy{}, mstorage.BucketNotFound{Bucket: bucket}
	}

	// get policy path
	bucketPolicy := path.Join(storage.root, bucket+"_policy.json")
	filestat, err := os.Stat(bucketPolicy)
	if filestat.IsDir() {
		return policy.BucketPolicy{}, mstorage.BackendCorrupted{Path: bucketPolicy}
	}

	if os.IsNotExist(err) {
		return policy.BucketPolicy{}, mstorage.BucketPolicyNotFound{Bucket: bucket}
	}

	file, err := os.OpenFile(bucketPolicy, os.O_RDONLY, 0666)
	defer file.Close()
	if err != nil {
		return policy.BucketPolicy{}, mstorage.EmbedError(bucket, "", err)
	}
	encoder := json.NewDecoder(file)
	err = encoder.Decode(&p)
	if err != nil {
		return policy.BucketPolicy{}, mstorage.EmbedError(bucket, "", err)
	}

	return p, nil

}

func (storage *storage) StoreBucketPolicy(bucket string, policy interface{}) error {
	storage.writeLock.Lock()
	defer storage.writeLock.Unlock()

	// verify bucket path legal
	if mstorage.IsValidBucket(bucket) == false {
		return mstorage.BucketNameInvalid{Bucket: bucket}
	}

	// get bucket path
	bucketDir := path.Join(storage.root, bucket)
	// check if bucket exists
	if _, err := os.Stat(bucketDir); err != nil {
		return mstorage.BucketNotFound{
			Bucket: bucket,
		}
	}

	// get policy path
	bucketPolicy := path.Join(storage.root, bucket+"_policy.json")
	filestat, _ := os.Stat(bucketPolicy)
	if filestat.IsDir() {
		return mstorage.BackendCorrupted{Path: bucketPolicy}
	}
	file, err := os.OpenFile(bucketPolicy, os.O_WRONLY|os.O_CREATE, 0600)
	defer file.Close()
	if err != nil {
		return mstorage.EmbedError(bucket, "", err)
	}
	encoder := json.NewEncoder(file)
	err = encoder.Encode(policy)
	if err != nil {
		return mstorage.EmbedError(bucket, "", err)
	}
	return nil
}

// Object Operations

func (storage *storage) CopyObjectToWriter(w io.Writer, bucket string, object string) (int64, error) {
	// validate bucket
	if mstorage.IsValidBucket(bucket) == false {
		return 0, mstorage.BucketNameInvalid{Bucket: bucket}
	}

	// validate object
	if mstorage.IsValidObject(object) == false {
		return 0, mstorage.ObjectNameInvalid{Bucket: bucket, Object: object}
	}

	objectPath := path.Join(storage.root, bucket, object)

	filestat, err := os.Stat(objectPath)
	switch err := err.(type) {
	case nil:
		{
			if filestat.IsDir() {
				return 0, mstorage.ObjectNotFound{Bucket: bucket, Object: object}
			}
		}
	default:
		{
			if os.IsNotExist(err) {
				return 0, mstorage.ObjectNotFound{Bucket: bucket, Object: object}
			} else {
				return 0, mstorage.EmbedError(bucket, object, err)
			}
		}
	}
	file, err := os.Open(objectPath)
	defer file.Close()
	if err != nil {
		return 0, mstorage.EmbedError(bucket, object, err)
	}
	count, err := io.Copy(w, file)
	if err != nil {
		return count, mstorage.EmbedError(bucket, object, err)
	}
	return count, nil
}

func (storage *storage) GetObjectMetadata(bucket string, object string) (mstorage.ObjectMetadata, error) {
	if mstorage.IsValidBucket(bucket) == false {
		return mstorage.ObjectMetadata{}, mstorage.BucketNameInvalid{Bucket: bucket}
	}

	if mstorage.IsValidObject(bucket) == false {
		return mstorage.ObjectMetadata{}, mstorage.ObjectNameInvalid{Bucket: bucket, Object: bucket}
	}

	objectPath := path.Join(storage.root, bucket, object)

	stat, err := os.Stat(objectPath)
	if os.IsNotExist(err) {
		return mstorage.ObjectMetadata{}, mstorage.ObjectNotFound{Bucket: bucket, Object: object}
	}

	_, err = os.Stat(objectPath + "$metadata")
	if os.IsNotExist(err) {
		return mstorage.ObjectMetadata{}, mstorage.ObjectNotFound{Bucket: bucket, Object: object}
	}

	file, err := os.Open(objectPath + "$metadata")
	defer file.Close()
	if err != nil {
		return mstorage.ObjectMetadata{}, mstorage.EmbedError(bucket, object, err)
	}

	metadataBuffer, err := ioutil.ReadAll(file)
	if err != nil {
		return mstorage.ObjectMetadata{}, mstorage.EmbedError(bucket, object, err)
	}

	var deserializedMetadata SerializedMetadata
	err = json.Unmarshal(metadataBuffer, &deserializedMetadata)
	if err != nil {
		return mstorage.ObjectMetadata{}, mstorage.EmbedError(bucket, object, err)
	}

	contentType := "application/octet-stream"
	if deserializedMetadata.ContentType != "" {
		contentType = deserializedMetadata.ContentType
	}
	contentType = strings.TrimSpace(contentType)

	metadata := mstorage.ObjectMetadata{
		Bucket:      bucket,
		Key:         object,
		Created:     stat.ModTime(),
		Size:        stat.Size(),
		ETag:        bucket + "#" + object,
		ContentType: contentType,
	}

	return metadata, nil
}

func (storage *storage) ListObjects(bucket, prefix string, count int) ([]mstorage.ObjectMetadata, bool, error) {
	if mstorage.IsValidBucket(bucket) == false {
		return []mstorage.ObjectMetadata{}, false, mstorage.BucketNameInvalid{Bucket: bucket}
	}
	if mstorage.IsValidObject(prefix) == false {
		return []mstorage.ObjectMetadata{}, false, mstorage.ObjectNameInvalid{Bucket: bucket, Object: prefix}
	}

	rootPrefix := path.Join(storage.root, bucket)

	// check bucket exists
	if _, err := os.Stat(rootPrefix); os.IsNotExist(err) {
		return []mstorage.ObjectMetadata{}, false, mstorage.BucketNotFound{Bucket: bucket}
	}

	files, err := ioutil.ReadDir(rootPrefix)
	if err != nil {
		return []mstorage.ObjectMetadata{}, false, mstorage.EmbedError("bucket", "", err)
	}

	var metadataList []mstorage.ObjectMetadata
	for _, file := range files {
		if !strings.HasSuffix(file.Name(), "$metadata") {
			if len(metadataList) >= count {
				return metadataList, true, nil
			}
			if strings.HasPrefix(file.Name(), prefix) {
				metadata := mstorage.ObjectMetadata{
					Bucket:  bucket,
					Key:     file.Name(),
					Created: file.ModTime(),
					Size:    file.Size(),
					ETag:    bucket + "#" + file.Name(),
				}
				metadataList = append(metadataList, metadata)
			}
		}
	}
	return metadataList, false, nil
}

func (storage *storage) StoreObject(bucket, key, contentType string, data io.Reader) error {
	// TODO Commits should stage then move instead of writing directly
	storage.writeLock.Lock()
	defer storage.writeLock.Unlock()

	// check bucket name valid
	if mstorage.IsValidBucket(bucket) == false {
		return mstorage.BucketNameInvalid{Bucket: bucket}
	}

	// check bucket exists
	if _, err := os.Stat(path.Join(storage.root, bucket)); os.IsNotExist(err) {
		return mstorage.BucketNotFound{Bucket: bucket}
	}

	// verify object path legal
	if mstorage.IsValidObject(key) == false {
		return mstorage.ObjectNameInvalid{Bucket: bucket, Object: key}
	}

	// verify content type
	if contentType == "" {
		contentType = "application/octet-stream"
	}
	contentType = strings.TrimSpace(contentType)

	// get object path
	objectPath := path.Join(storage.root, bucket, key)
	objectDir := path.Dir(objectPath)
	if _, err := os.Stat(objectDir); os.IsNotExist(err) {
		err = os.MkdirAll(objectDir, 0700)
		if err != nil {
			return mstorage.EmbedError(bucket, key, err)
		}
	}

	// check if object exists
	if _, err := os.Stat(objectPath); !os.IsNotExist(err) {
		return mstorage.ObjectExists{
			Bucket: bucket,
			Key:    key,
		}
	}

	// write object
	file, err := os.OpenFile(objectPath, os.O_WRONLY|os.O_CREATE, 0600)
	defer file.Close()
	if err != nil {
		return mstorage.EmbedError(bucket, key, err)
	}

	_, err = io.Copy(file, data)
	if err != nil {
		return mstorage.EmbedError(bucket, key, err)
	}

	// serialize metadata to json

	metadataBuffer, err := json.Marshal(SerializedMetadata{ContentType: contentType})
	if err != nil {
		return mstorage.EmbedError(bucket, key, err)
	}

	//
	file, err = os.OpenFile(objectPath+"$metadata", os.O_WRONLY|os.O_CREATE, 0600)
	defer file.Close()
	if err != nil {
		return mstorage.EmbedError(bucket, key, err)
	}

	_, err = io.Copy(file, bytes.NewBuffer(metadataBuffer))
	if err != nil {
		return mstorage.EmbedError(bucket, key, err)
	}

	return nil
}
<commit_msg>Skip policy files<commit_after>/*
 * Mini Object Storage, (C) 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package fs

import (
	"bytes"
	"encoding/json"
	"io"
	"io/ioutil"
	"os"
	"path"
	"strings"
	"sync"

	mstorage "github.com/minio-io/minio/pkg/storage"
	"github.com/minio-io/minio/pkg/utils/policy"
)

type storage struct {
	root      string
	writeLock sync.Mutex
}

type SerializedMetadata struct {
	ContentType string
}

type MkdirFailedError struct{}

func (self MkdirFailedError) Error() string {
	return "Mkdir Failed"
}

func Start(root string) (chan<- string, <-chan error, *storage) {
	ctrlChannel := make(chan string)
	errorChannel := make(chan error)
	go start(ctrlChannel, errorChannel)
	return ctrlChannel, errorChannel, &storage{root: root}
}

func start(ctrlChannel <-chan string, errorChannel chan<- error) {
	close(errorChannel)
}

// Bucket Operations

func (storage *storage) ListBuckets() ([]mstorage.BucketMetadata, error) {
	files, err := ioutil.ReadDir(storage.root)
	if err != nil {
		return []mstorage.BucketMetadata{}, mstorage.EmbedError("bucket", "", err)
	}

	var metadataList []mstorage.BucketMetadata
	for _, file := range files {
		// Skip policy files
		if strings.HasSuffix(file.Name(), "_policy.json") {
			continue
		}
		if !file.IsDir() {
			return []mstorage.BucketMetadata{}, mstorage.BackendCorrupted{Path: storage.root}
		}
		metadata := mstorage.BucketMetadata{
			Name:    file.Name(),
			Created: file.ModTime(), // TODO - provide real created time
		}
		metadataList = append(metadataList, metadata)
	}
	return metadataList, nil
}

func (storage *storage) StoreBucket(bucket string) error {
	storage.writeLock.Lock()
	defer storage.writeLock.Unlock()

	// verify bucket path legal
	if mstorage.IsValidBucket(bucket) == false {
		return mstorage.BucketNameInvalid{Bucket: bucket}
	}

	// get bucket path
	bucketDir := path.Join(storage.root, bucket)

	// check if bucket exists
	if _, err := os.Stat(bucketDir); err == nil {
		return mstorage.BucketExists{
			Bucket: bucket,
		}
	}

	// make bucket
	err := os.Mkdir(bucketDir, 0700)
	if err != nil {
		return mstorage.EmbedError(bucket, "", err)
	}
	return nil
}

func (storage *storage) GetBucketPolicy(bucket string) (interface{}, error) {
	storage.writeLock.Lock()
	defer storage.writeLock.Unlock()

	var p policy.BucketPolicy
	// verify bucket path legal
	if mstorage.IsValidBucket(bucket) == false {
		return policy.BucketPolicy{}, mstorage.BucketNameInvalid{Bucket: bucket}
	}

	// get bucket path
	bucketDir := path.Join(storage.root, bucket)
	// check if bucket exists
	if _, err := os.Stat(bucketDir); err != nil {
		return policy.BucketPolicy{}, mstorage.BucketNotFound{Bucket: bucket}
	}

	// get policy path
	bucketPolicy := path.Join(storage.root, bucket+"_policy.json")
	filestat, err := os.Stat(bucketPolicy)
	if filestat.IsDir() {
		return policy.BucketPolicy{}, mstorage.BackendCorrupted{Path: bucketPolicy}
	}

	if os.IsNotExist(err) {
		return policy.BucketPolicy{}, mstorage.BucketPolicyNotFound{Bucket: bucket}
	}

	file, err := os.OpenFile(bucketPolicy, os.O_RDONLY, 0666)
	defer file.Close()
	if err != nil {
		return policy.BucketPolicy{}, mstorage.EmbedError(bucket, "", err)
	}
	encoder := json.NewDecoder(file)
	err = encoder.Decode(&p)
	if err != nil {
		return policy.BucketPolicy{}, mstorage.EmbedError(bucket, "", err)
	}

	return p, nil

}

func (storage *storage) StoreBucketPolicy(bucket string, policy interface{}) error {
	storage.writeLock.Lock()
	defer storage.writeLock.Unlock()

	// verify bucket path legal
	if mstorage.IsValidBucket(bucket) == false {
		return mstorage.BucketNameInvalid{Bucket: bucket}
	}

	// get bucket path
	bucketDir := path.Join(storage.root, bucket)
	// check if bucket exists
	if _, err := os.Stat(bucketDir); err != nil {
		return mstorage.BucketNotFound{
			Bucket: bucket,
		}
	}

	// get policy path
	bucketPolicy := path.Join(storage.root, bucket+"_policy.json")
	filestat, _ := os.Stat(bucketPolicy)
	if filestat.IsDir() {
		return mstorage.BackendCorrupted{Path: bucketPolicy}
	}
	file, err := os.OpenFile(bucketPolicy, os.O_WRONLY|os.O_CREATE, 0600)
	defer file.Close()
	if err != nil {
		return mstorage.EmbedError(bucket, "", err)
	}
	encoder := json.NewEncoder(file)
	err = encoder.Encode(policy)
	if err != nil {
		return mstorage.EmbedError(bucket, "", err)
	}
	return nil
}

// Object Operations

func (storage *storage) CopyObjectToWriter(w io.Writer, bucket string, object string) (int64, error) {
	// validate bucket
	if mstorage.IsValidBucket(bucket) == false {
		return 0, mstorage.BucketNameInvalid{Bucket: bucket}
	}

	// validate object
	if mstorage.IsValidObject(object) == false {
		return 0, mstorage.ObjectNameInvalid{Bucket: bucket, Object: object}
	}

	objectPath := path.Join(storage.root, bucket, object)

	filestat, err := os.Stat(objectPath)
	switch err := err.(type) {
	case nil:
		{
			if filestat.IsDir() {
				return 0, mstorage.ObjectNotFound{Bucket: bucket, Object: object}
			}
		}
	default:
		{
			if os.IsNotExist(err) {
				return 0, mstorage.ObjectNotFound{Bucket: bucket, Object: object}
			} else {
				return 0, mstorage.EmbedError(bucket, object, err)
			}
		}
	}
	file, err := os.Open(objectPath)
	defer file.Close()
	if err != nil {
		return 0, mstorage.EmbedError(bucket, object, err)
	}
	count, err := io.Copy(w, file)
	if err != nil {
		return count, mstorage.EmbedError(bucket, object, err)
	}
	return count, nil
}

func (storage *storage) GetObjectMetadata(bucket string, object string) (mstorage.ObjectMetadata, error) {
	if mstorage.IsValidBucket(bucket) == false {
		return mstorage.ObjectMetadata{}, mstorage.BucketNameInvalid{Bucket: bucket}
	}

	if mstorage.IsValidObject(bucket) == false {
		return mstorage.ObjectMetadata{}, mstorage.ObjectNameInvalid{Bucket: bucket, Object: bucket}
	}

	objectPath := path.Join(storage.root, bucket, object)

	stat, err := os.Stat(objectPath)
	if os.IsNotExist(err) {
		return mstorage.ObjectMetadata{}, mstorage.ObjectNotFound{Bucket: bucket, Object: object}
	}

	_, err = os.Stat(objectPath + "$metadata")
	if os.IsNotExist(err) {
		return mstorage.ObjectMetadata{}, mstorage.ObjectNotFound{Bucket: bucket, Object: object}
	}

	file, err := os.Open(objectPath + "$metadata")
	defer file.Close()
	if err != nil {
		return mstorage.ObjectMetadata{}, mstorage.EmbedError(bucket, object, err)
	}

	metadataBuffer, err := ioutil.ReadAll(file)
	if err != nil {
		return mstorage.ObjectMetadata{}, mstorage.EmbedError(bucket, object, err)
	}

	var deserializedMetadata SerializedMetadata
	err = json.Unmarshal(metadataBuffer, &deserializedMetadata)
	if err != nil {
		return mstorage.ObjectMetadata{}, mstorage.EmbedError(bucket, object, err)
	}

	contentType := "application/octet-stream"
	if deserializedMetadata.ContentType != "" {
		contentType = deserializedMetadata.ContentType
	}
	contentType = strings.TrimSpace(contentType)

	metadata := mstorage.ObjectMetadata{
		Bucket:      bucket,
		Key:         object,
		Created:     stat.ModTime(),
		Size:        stat.Size(),
		ETag:        bucket + "#" + object,
		ContentType: contentType,
	}

	return metadata, nil
}

func (storage *storage) ListObjects(bucket, prefix string, count int) ([]mstorage.ObjectMetadata, bool, error) {
	if mstorage.IsValidBucket(bucket) == false {
		return []mstorage.ObjectMetadata{}, false, mstorage.BucketNameInvalid{Bucket: bucket}
	}
	if mstorage.IsValidObject(prefix) == false {
		return []mstorage.ObjectMetadata{}, false, mstorage.ObjectNameInvalid{Bucket: bucket, Object: prefix}
	}

	rootPrefix := path.Join(storage.root, bucket)

	// check bucket exists
	if _, err := os.Stat(rootPrefix); os.IsNotExist(err) {
		return []mstorage.ObjectMetadata{}, false, mstorage.BucketNotFound{Bucket: bucket}
	}

	files, err := ioutil.ReadDir(rootPrefix)
	if err != nil {
		return []mstorage.ObjectMetadata{}, false, mstorage.EmbedError("bucket", "", err)
	}

	var metadataList []mstorage.ObjectMetadata
	for _, file := range files {
		if !strings.HasSuffix(file.Name(), "$metadata") {
			if len(metadataList) >= count {
				return metadataList, true, nil
			}
			if strings.HasPrefix(file.Name(), prefix) {
				metadata := mstorage.ObjectMetadata{
					Bucket:  bucket,
					Key:     file.Name(),
					Created: file.ModTime(),
					Size:    file.Size(),
					ETag:    bucket + "#" + file.Name(),
				}
				metadataList = append(metadataList, metadata)
			}
		}
	}
	return metadataList, false, nil
}

func (storage *storage) StoreObject(bucket, key, contentType string, data io.Reader) error {
	// TODO Commits should stage then move instead of writing directly
	storage.writeLock.Lock()
	defer storage.writeLock.Unlock()

	// check bucket name valid
	if mstorage.IsValidBucket(bucket) == false {
		return mstorage.BucketNameInvalid{Bucket: bucket}
	}

	// check bucket exists
	if _, err := os.Stat(path.Join(storage.root, bucket)); os.IsNotExist(err) {
		return mstorage.BucketNotFound{Bucket: bucket}
	}

	// verify object path legal
	if mstorage.IsValidObject(key) == false {
		return mstorage.ObjectNameInvalid{Bucket: bucket, Object: key}
	}

	// verify content type
	if contentType == "" {
		contentType = "application/octet-stream"
	}
	contentType = strings.TrimSpace(contentType)

	// get object path
	objectPath := path.Join(storage.root, bucket, key)
	objectDir := path.Dir(objectPath)
	if _, err := os.Stat(objectDir); os.IsNotExist(err) {
		err = os.MkdirAll(objectDir, 0700)
		if err != nil {
			return mstorage.EmbedError(bucket, key, err)
		}
	}

	// check if object exists
	if _, err := os.Stat(objectPath); !os.IsNotExist(err) {
		return mstorage.ObjectExists{
			Bucket: bucket,
			Key:    key,
		}
	}

	// write object
	file, err := os.OpenFile(objectPath, os.O_WRONLY|os.O_CREATE, 0600)
	defer file.Close()
	if err != nil {
		return mstorage.EmbedError(bucket, key, err)
	}

	_, err = io.Copy(file, data)
	if err != nil {
		return mstorage.EmbedError(bucket, key, err)
	}

	// serialize metadata to json

	metadataBuffer, err := json.Marshal(SerializedMetadata{ContentType: contentType})
	if err != nil {
		return mstorage.EmbedError(bucket, key, err)
	}

	//
	file, err = os.OpenFile(objectPath+"$metadata", os.O_WRONLY|os.O_CREATE, 0600)
	defer file.Close()
	if err != nil {
		return mstorage.EmbedError(bucket, key, err)
	}

	_, err = io.Copy(file, bytes.NewBuffer(metadataBuffer))
	if err != nil {
		return mstorage.EmbedError(bucket, key, err)
	}

	return nil
}
<|endoftext|>