<commit_before>package servers

import (
	"fmt"

	"google.golang.org/grpc"

	jwt "github.com/dgrijalva/jwt-go"
	"github.com/ellcrys/util"
	middleware "github.com/grpc-ecosystem/go-grpc-middleware"
	"github.com/ncodes/patchain"
	"github.com/ncodes/patchain/cockroach/tables"
	"github.com/ncodes/patchain/object"
	"github.com/ncodes/safehold/servers/common"
	"github.com/ncodes/safehold/servers/oauth"
	"github.com/ncodes/safehold/types"
	"golang.org/x/net/context"
)

var (
	// CtxIdentity represents an authenticated identity
	CtxIdentity types.CtxKey = "identity"

	// CtxTokenClaims represents claims in an auth token
	CtxTokenClaims types.CtxKey = "token_claims"

	// ErrInvalidToken represents a error about an invalid token
	ErrInvalidToken = fmt.Errorf("permission denied. Invalid token")

	// methodsNotRequiringAuth includes the full method name of methods that
	// must not be processed by the auth interceptor
	methodsNotRequiringAuth = []string{
		"/proto_rpc.API/CreateIdentity",
	}

	// methodsRequiringAppToken includes the full method name of methods that
	// must pass app token authorization checks
	methodsRequiringAppToken = []string{
		"/proto_rpc.API/GetIdentity",
		"/proto_rpc.API/CreateObjects",
		"/proto_rpc.API/GetObjects",
		"/proto_rpc.API/CreateDBSession",
		"/proto_rpc.API/GetDBSession",
		"/proto_rpc.API/DeleteDBSession",
		"/proto_rpc.API/CountObjects",
		"/proto_rpc.API/CreateMapping",
		"/proto_rpc.API/GetMapping",
		"/proto_rpc.API/GetAllMapping",
	}
)

// Interceptors returns the API interceptors
func (s *RPC) Interceptors() grpc.UnaryServerInterceptor {
	return middleware.ChainUnaryServer(func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
		logRPC.Debugf("New request [method=%s]", info.FullMethod)
		return handler(ctx, req)
	}, s.authInterceptor, s.requiresAppTokenInterceptor)
}

// authInterceptor checks whether the request has valid access token
func (s *RPC) authInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {

	if util.InStringSlice(methodsNotRequiringAuth, info.FullMethod) {
		return handler(ctx, req)
	}

	tokenStr, err := util.GetAuthToken(ctx, "bearer")
	if err != nil {
		return nil, common.NewSingleAPIErr(400, common.CodeAuthorizationError, "", err.Error(), nil)
	}

	token, err := jwt.Parse(tokenStr, func(token *jwt.Token) (interface{}, error) {

		if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok {
			return nil, fmt.Errorf("unexpected signing method: %v", token.Header["alg"])
		}

		tokenType := token.Claims.(jwt.MapClaims)["type"]
		if tokenType == oauth.TokenTypeApp {
			return []byte(oauth.SigningSecret), nil
		}

		return nil, fmt.Errorf("unknown token type")
	})
	if err != nil {
		logRPC.Errorf("%+v", err)
		return nil, common.NewSingleAPIErr(400, common.CodeAuthorizationError, "", ErrInvalidToken.Error(), nil)
	}

	claims := token.Claims.(jwt.MapClaims)
	if err = claims.Valid(); err != nil {
		return nil, common.NewSingleAPIErr(400, common.CodeAuthorizationError, "", ErrInvalidToken.Error(), nil)
	}

	ctx = context.WithValue(ctx, CtxTokenClaims, claims)

	return handler(ctx, req)
}

// processAppTokenClaims checks whether a token claim is valid. It confirms the
// the identity associated with the claim and includes the identity in the context.
func (s *RPC) processAppTokenClaims(ctx context.Context, claims jwt.MapClaims) (context.Context, error) {

	// get identity with matching client id
	identity, err := s.object.GetLast(&tables.Object{
		QueryParams: patchain.KeyStartsWith(object.IdentityPrefix),
		Ref1:        claims["id"].(string),
	})
	if err != nil {
		logRPC.Errorf("%+v", err)
		return ctx, common.NewSingleAPIErr(400, common.CodeAuthorizationError, "", ErrInvalidToken.Error(), nil)
	}

	return context.WithValue(ctx, CtxIdentity, identity.ID), nil
}

// requiresAppTokenInterceptor checks if the claim produced by authInterceptor
// is a valid app token. This will only apply to methods that require app token. It immediately
// calls the next handler if the method does not require app token.
func (s *RPC) requiresAppTokenInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {

	var err error

	// verify claims for methods that require an app token
	if !util.InStringSlice(methodsRequiringAppToken, info.FullMethod) {
		return handler(ctx, req)
	}

	claims := ctx.Value(CtxTokenClaims).(jwt.MapClaims)
	if claims["type"] != oauth.TokenTypeApp {
		return nil, common.NewSingleAPIErr(400, "", "", "endpoint requires an app token", nil)
	}

	ctx, err = s.processAppTokenClaims(ctx, claims)
	if err != nil {
		return nil, err
	}

	return handler(ctx, req)
}
<commit_msg>Distinguish between `ErrNotFound` and a database error<commit_after>package servers

import (
	"fmt"

	"google.golang.org/grpc"

	jwt "github.com/dgrijalva/jwt-go"
	"github.com/ellcrys/util"
	middleware "github.com/grpc-ecosystem/go-grpc-middleware"
	"github.com/ncodes/patchain"
	"github.com/ncodes/patchain/cockroach/tables"
	"github.com/ncodes/patchain/object"
	"github.com/ncodes/safehold/servers/common"
	"github.com/ncodes/safehold/servers/oauth"
	"github.com/ncodes/safehold/types"
	"golang.org/x/net/context"
)

var (
	// CtxIdentity represents an authenticated identity
	CtxIdentity types.CtxKey = "identity"

	// CtxTokenClaims represents claims in an auth token
	CtxTokenClaims types.CtxKey = "token_claims"

	// ErrInvalidToken represents a error about an invalid token
	ErrInvalidToken = fmt.Errorf("permission denied. Invalid token")

	// methodsNotRequiringAuth includes the full method name of methods that
	// must not be processed by the auth interceptor
	methodsNotRequiringAuth = []string{
		"/proto_rpc.API/CreateIdentity",
	}

	// methodsRequiringAppToken includes the full method name of methods that
	// must pass app token authorization checks
	methodsRequiringAppToken = []string{
		"/proto_rpc.API/GetIdentity",
		"/proto_rpc.API/CreateObjects",
		"/proto_rpc.API/GetObjects",
		"/proto_rpc.API/CreateDBSession",
		"/proto_rpc.API/GetDBSession",
		"/proto_rpc.API/DeleteDBSession",
		"/proto_rpc.API/CountObjects",
		"/proto_rpc.API/CreateMapping",
		"/proto_rpc.API/GetMapping",
		"/proto_rpc.API/GetAllMapping",
	}
)

// Interceptors returns the API interceptors
func (s *RPC) Interceptors() grpc.UnaryServerInterceptor {
	return middleware.ChainUnaryServer(func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
		logRPC.Debugf("New request [method=%s]", info.FullMethod)
		return handler(ctx, req)
	}, s.authInterceptor, s.requiresAppTokenInterceptor)
}

// authInterceptor checks whether the request has valid access token
func (s *RPC) authInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {

	if util.InStringSlice(methodsNotRequiringAuth, info.FullMethod) {
		return handler(ctx, req)
	}

	tokenStr, err := util.GetAuthToken(ctx, "bearer")
	if err != nil {
		return nil, common.NewSingleAPIErr(400, common.CodeAuthorizationError, "", err.Error(), nil)
	}

	token, err := jwt.Parse(tokenStr, func(token *jwt.Token) (interface{}, error) {

		if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok {
			return nil, fmt.Errorf("unexpected signing method: %v", token.Header["alg"])
		}

		tokenType := token.Claims.(jwt.MapClaims)["type"]
		if tokenType == oauth.TokenTypeApp {
			return []byte(oauth.SigningSecret), nil
		}

		return nil, fmt.Errorf("unknown token type")
	})
	if err != nil {
		logRPC.Debugf("%+v", err)
		return nil, common.NewSingleAPIErr(400, common.CodeAuthorizationError, "", ErrInvalidToken.Error(), nil)
	}

	claims := token.Claims.(jwt.MapClaims)
	if err = claims.Valid(); err != nil {
		return nil, common.NewSingleAPIErr(400, common.CodeAuthorizationError, "", ErrInvalidToken.Error(), nil)
	}

	ctx = context.WithValue(ctx, CtxTokenClaims, claims)

	return handler(ctx, req)
}

// processAppTokenClaims checks whether a token claim is valid. It confirms the
// the identity associated with the claim and includes the identity in the context.
func (s *RPC) processAppTokenClaims(ctx context.Context, claims jwt.MapClaims) (context.Context, error) {

	// get identity with matching client id
	identity, err := s.object.GetLast(&tables.Object{
		QueryParams: patchain.KeyStartsWith(object.IdentityPrefix),
		Ref1:        claims["id"].(string),
	})
	if err != nil {
		if err == patchain.ErrNotFound {
			return ctx, common.NewSingleAPIErr(400, common.CodeAuthorizationError, "", ErrInvalidToken.Error(), nil)
		}
		logRPC.Errorf("%+v", err)
		return nil, common.ServerError
	}

	return context.WithValue(ctx, CtxIdentity, identity.ID), nil
}

// requiresAppTokenInterceptor checks if the claim produced by authInterceptor
// is a valid app token. This will only apply to methods that require app token. It immediately
// calls the next handler if the method does not require app token.
func (s *RPC) requiresAppTokenInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {

	var err error

	// verify claims for methods that require an app token
	if !util.InStringSlice(methodsRequiringAppToken, info.FullMethod) {
		return handler(ctx, req)
	}

	claims := ctx.Value(CtxTokenClaims).(jwt.MapClaims)
	if claims["type"] != oauth.TokenTypeApp {
		return nil, common.NewSingleAPIErr(400, "", "", "endpoint requires an app token", nil)
	}

	ctx, err = s.processAppTokenClaims(ctx, claims)
	if err != nil {
		return nil, err
	}

	return handler(ctx, req)
}
<|endoftext|>


<commit_before>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package manager

import (
	"flag"
	"fmt"
	"sync"
	"time"

	"github.com/docker/docker/pkg/units"
	"github.com/golang/glog"
	"github.com/google/cadvisor/container"
	"github.com/google/cadvisor/info"
	"github.com/google/cadvisor/storage"
)

// Housekeeping interval.
var HousekeepingInterval = flag.Duration("housekeeping_interval", 1*time.Second, "Interval between container housekeepings")
var maxHousekeepingInterval = flag.Duration("max_housekeeping_interval", 60*time.Second, "Largest interval to allow between container housekeepings")
var allowDynamicHousekeeping = flag.Bool("allow_dynamic_housekeeping", true, "Whether to allow the housekeeping interval to be dynamic")

type containerInfo struct {
	info.ContainerReference
	Subcontainers []info.ContainerReference
	Spec          info.ContainerSpec
}

type containerData struct {
	handler              container.ContainerHandler
	info                 containerInfo
	storageDriver        storage.StorageDriver
	lock                 sync.Mutex
	housekeepingInterval time.Duration

	// Whether to log the usage of this container when it is updated.
	logUsage bool

	// Tells the container to stop.
	stop chan bool
}

func (c *containerData) Start() error {
	go c.housekeeping()
	return nil
}

func (c *containerData) Stop() error {
	c.stop <- true
	return nil
}

func (c *containerData) GetInfo() (*containerInfo, error) {
	// TODO(vmarmol): Consider caching this.
	// Get spec and subcontainers.
	err := c.updateSpec()
	if err != nil {
		return nil, err
	}
	err = c.updateSubcontainers()
	if err != nil {
		return nil, err
	}

	// Make a copy of the info for the user.
	c.lock.Lock()
	defer c.lock.Unlock()
	return &c.info, nil
}

func newContainerData(containerName string, driver storage.StorageDriver, handler container.ContainerHandler, logUsage bool) (*containerData, error) {
	if driver == nil {
		return nil, fmt.Errorf("nil storage driver")
	}
	if handler == nil {
		return nil, fmt.Errorf("nil container handler")
	}
	ref, err := handler.ContainerReference()
	if err != nil {
		return nil, err
	}

	cont := &containerData{
		handler:              handler,
		storageDriver:        driver,
		housekeepingInterval: *HousekeepingInterval,
		logUsage:             logUsage,
		stop:                 make(chan bool, 1),
	}
	cont.info.ContainerReference = ref

	return cont, nil
}

// Determine when the next housekeeping should occur.
func (self *containerData) nextHousekeeping(lastHousekeeping time.Time) time.Time {
	if *allowDynamicHousekeeping {
		stats, err := self.storageDriver.RecentStats(self.info.Name, 2)
		if err != nil {
			glog.Warningf("Failed to get RecentStats(%q) while determining the next housekeeping: %v", self.info.Name, err)
		} else if len(stats) == 2 {
			// TODO(vishnuk): Use no processes as a signal.
			// Raise the interval if usage hasn't changed in the last housekeeping.
			if stats[0].StatsEq(stats[1]) && (self.housekeepingInterval < *maxHousekeepingInterval) {
				self.housekeepingInterval *= 2
				if self.housekeepingInterval > *maxHousekeepingInterval {
					self.housekeepingInterval = *maxHousekeepingInterval
				}
				glog.V(3).Infof("Raising housekeeping interval for %q to %v", self.info.Name, self.housekeepingInterval)
			} else if self.housekeepingInterval != *HousekeepingInterval {
				// Lower interval back to the baseline.
				self.housekeepingInterval = *HousekeepingInterval
				glog.V(3).Infof("Lowering housekeeping interval for %q to %v", self.info.Name, self.housekeepingInterval)
			}
		}
	}

	return lastHousekeeping.Add(self.housekeepingInterval)
}

func (c *containerData) housekeeping() {
	// Long housekeeping is either 100ms or half of the housekeeping interval.
	longHousekeeping := 100 * time.Millisecond
	if *HousekeepingInterval/2 < longHousekeeping {
		longHousekeeping = *HousekeepingInterval / 2
	}

	// Housekeep every second.
	glog.Infof("Start housekeeping for container %q\n", c.info.Name)
	lastHousekeeping := time.Now()
	for {
		select {
		case <-c.stop:
			// Stop housekeeping when signaled.
			return
		default:
			// Perform housekeeping.
			start := time.Now()
			c.housekeepingTick()

			// Log if housekeeping took too long.
			duration := time.Since(start)
			if duration >= longHousekeeping {
				glog.V(2).Infof("[%s] Housekeeping took %s", c.info.Name, duration)
			}
		}

		// Log usage if asked to do so.
		if c.logUsage {
			stats, err := c.storageDriver.RecentStats(c.info.Name, 2)
			if err != nil {
				glog.Infof("[%s] Failed to get recent stats for logging usage: %v", c.info.Name, err)
			} else if len(stats) < 2 {
				// Ignore, not enough stats yet.
			} else {
				usageCpuNs := stats[1].Cpu.Usage.Total - stats[0].Cpu.Usage.Total
				usageMemory := stats[1].Memory.Usage

				usageInCores := float64(usageCpuNs) / float64(stats[1].Timestamp.Sub(stats[0].Timestamp).Nanoseconds())
				usageInHuman := units.HumanSize(int64(usageMemory))
				glog.Infof("[%s] %.3f cores, %s of memory", c.info.Name, usageInCores, usageInHuman)
			}
		}

		// Schedule the next housekeeping. Sleep until that time.
		nextHousekeeping := c.nextHousekeeping(lastHousekeeping)
		if time.Now().Before(nextHousekeeping) {
			time.Sleep(nextHousekeeping.Sub(time.Now()))
		}
		lastHousekeeping = nextHousekeeping
	}
}

func (c *containerData) housekeepingTick() {
	err := c.updateStats()
	if err != nil {
		glog.Infof("Failed to update stats for container \"%s\": %s", c.info.Name, err)
	}
}

func (c *containerData) updateSpec() error {
	spec, err := c.handler.GetSpec()
	if err != nil {
		// Ignore errors if the container is dead.
		if !c.handler.Exists() {
			return nil
		}
		return err
	}
	c.lock.Lock()
	defer c.lock.Unlock()
	c.info.Spec = spec
	return nil
}

func (c *containerData) updateStats() error {
	stats, err := c.handler.GetStats()
	if err != nil {
		// Ignore errors if the container is dead.
		if !c.handler.Exists() {
			return nil
		}
		return err
	}
	if stats == nil {
		return nil
	}
	ref, err := c.handler.ContainerReference()
	if err != nil {
		// Ignore errors if the container is dead.
		if !c.handler.Exists() {
			return nil
		}
		return err
	}
	err = c.storageDriver.AddStats(ref, stats)
	if err != nil {
		return err
	}
	return nil
}

func (c *containerData) updateSubcontainers() error {
	subcontainers, err := c.handler.ListContainers(container.ListSelf)
	if err != nil {
		// Ignore errors if the container is dead.
		if !c.handler.Exists() {
			return nil
		}
		return err
	}
	c.lock.Lock()
	defer c.lock.Unlock()
	c.info.Subcontainers = subcontainers
	return nil
}
<commit_msg>Caches container info for 5 seconds before updating it<commit_after>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package manager

import (
	"flag"
	"fmt"
	"sync"
	"time"

	"github.com/docker/docker/pkg/units"
	"github.com/golang/glog"
	"github.com/google/cadvisor/container"
	"github.com/google/cadvisor/info"
	"github.com/google/cadvisor/storage"
)

// Housekeeping interval.
var HousekeepingInterval = flag.Duration("housekeeping_interval", 1*time.Second, "Interval between container housekeepings")
var maxHousekeepingInterval = flag.Duration("max_housekeeping_interval", 60*time.Second, "Largest interval to allow between container housekeepings")
var allowDynamicHousekeeping = flag.Bool("allow_dynamic_housekeeping", true, "Whether to allow the housekeeping interval to be dynamic")

type containerInfo struct {
	info.ContainerReference
	Subcontainers []info.ContainerReference
	Spec          info.ContainerSpec
}

type containerData struct {
	handler              container.ContainerHandler
	info                 containerInfo
	storageDriver        storage.StorageDriver
	lock                 sync.Mutex
	housekeepingInterval time.Duration
	lastUpdatedTime      time.Time

	// Whether to log the usage of this container when it is updated.
	logUsage bool

	// Tells the container to stop.
	stop chan bool
}

func (c *containerData) Start() error {
	go c.housekeeping()
	return nil
}

func (c *containerData) Stop() error {
	c.stop <- true
	return nil
}

func (c *containerData) GetInfo() (*containerInfo, error) {
	// Get spec and subcontainers.
	if time.Since(c.lastUpdatedTime) > 5 * time.Second {
		err := c.updateSpec()
		if err != nil {
			return nil, err
		}
		err = c.updateSubcontainers()
		if err != nil {
			return nil, err
		}
		c.lastUpdatedTime = time.Now()
	}
	// Make a copy of the info for the user.
	c.lock.Lock()
	defer c.lock.Unlock()
	return &c.info, nil
}

func newContainerData(containerName string, driver storage.StorageDriver, handler container.ContainerHandler, logUsage bool) (*containerData, error) {
	if driver == nil {
		return nil, fmt.Errorf("nil storage driver")
	}
	if handler == nil {
		return nil, fmt.Errorf("nil container handler")
	}
	ref, err := handler.ContainerReference()
	if err != nil {
		return nil, err
	}

	cont := &containerData{
		handler:              handler,
		storageDriver:        driver,
		housekeepingInterval: *HousekeepingInterval,
		logUsage:             logUsage,
		stop:                 make(chan bool, 1),
	}
	cont.info.ContainerReference = ref

	return cont, nil
}

// Determine when the next housekeeping should occur.
func (self *containerData) nextHousekeeping(lastHousekeeping time.Time) time.Time {
	if *allowDynamicHousekeeping {
		stats, err := self.storageDriver.RecentStats(self.info.Name, 2)
		if err != nil {
			glog.Warningf("Failed to get RecentStats(%q) while determining the next housekeeping: %v", self.info.Name, err)
		} else if len(stats) == 2 {
			// TODO(vishnuk): Use no processes as a signal.
			// Raise the interval if usage hasn't changed in the last housekeeping.
			if stats[0].StatsEq(stats[1]) && (self.housekeepingInterval < *maxHousekeepingInterval) {
				self.housekeepingInterval *= 2
				if self.housekeepingInterval > *maxHousekeepingInterval {
					self.housekeepingInterval = *maxHousekeepingInterval
				}
				glog.V(3).Infof("Raising housekeeping interval for %q to %v", self.info.Name, self.housekeepingInterval)
			} else if self.housekeepingInterval != *HousekeepingInterval {
				// Lower interval back to the baseline.
				self.housekeepingInterval = *HousekeepingInterval
				glog.V(3).Infof("Lowering housekeeping interval for %q to %v", self.info.Name, self.housekeepingInterval)
			}
		}
	}

	return lastHousekeeping.Add(self.housekeepingInterval)
}

func (c *containerData) housekeeping() {
	// Long housekeeping is either 100ms or half of the housekeeping interval.
	longHousekeeping := 100 * time.Millisecond
	if *HousekeepingInterval/2 < longHousekeeping {
		longHousekeeping = *HousekeepingInterval / 2
	}

	// Housekeep every second.
	glog.Infof("Start housekeeping for container %q\n", c.info.Name)
	lastHousekeeping := time.Now()
	for {
		select {
		case <-c.stop:
			// Stop housekeeping when signaled.
			return
		default:
			// Perform housekeeping.
			start := time.Now()
			c.housekeepingTick()

			// Log if housekeeping took too long.
			duration := time.Since(start)
			if duration >= longHousekeeping {
				glog.V(2).Infof("[%s] Housekeeping took %s", c.info.Name, duration)
			}
		}

		// Log usage if asked to do so.
		if c.logUsage {
			stats, err := c.storageDriver.RecentStats(c.info.Name, 2)
			if err != nil {
				glog.Infof("[%s] Failed to get recent stats for logging usage: %v", c.info.Name, err)
			} else if len(stats) < 2 {
				// Ignore, not enough stats yet.
			} else {
				usageCpuNs := stats[1].Cpu.Usage.Total - stats[0].Cpu.Usage.Total
				usageMemory := stats[1].Memory.Usage

				usageInCores := float64(usageCpuNs) / float64(stats[1].Timestamp.Sub(stats[0].Timestamp).Nanoseconds())
				usageInHuman := units.HumanSize(int64(usageMemory))
				glog.Infof("[%s] %.3f cores, %s of memory", c.info.Name, usageInCores, usageInHuman)
			}
		}

		// Schedule the next housekeeping. Sleep until that time.
		nextHousekeeping := c.nextHousekeeping(lastHousekeeping)
		if time.Now().Before(nextHousekeeping) {
			time.Sleep(nextHousekeeping.Sub(time.Now()))
		}
		lastHousekeeping = nextHousekeeping
	}
}

func (c *containerData) housekeepingTick() {
	err := c.updateStats()
	if err != nil {
		glog.Infof("Failed to update stats for container \"%s\": %s", c.info.Name, err)
	}
}

func (c *containerData) updateSpec() error {
	spec, err := c.handler.GetSpec()
	if err != nil {
		// Ignore errors if the container is dead.
		if !c.handler.Exists() {
			return nil
		}
		return err
	}
	c.lock.Lock()
	defer c.lock.Unlock()
	c.info.Spec = spec
	return nil
}

func (c *containerData) updateStats() error {
	stats, err := c.handler.GetStats()
	if err != nil {
		// Ignore errors if the container is dead.
		if !c.handler.Exists() {
			return nil
		}
		return err
	}
	if stats == nil {
		return nil
	}
	ref, err := c.handler.ContainerReference()
	if err != nil {
		// Ignore errors if the container is dead.
		if !c.handler.Exists() {
			return nil
		}
		return err
	}
	err = c.storageDriver.AddStats(ref, stats)
	if err != nil {
		return err
	}
	return nil
}

func (c *containerData) updateSubcontainers() error {
	subcontainers, err := c.handler.ListContainers(container.ListSelf)
	if err != nil {
		// Ignore errors if the container is dead.
		if !c.handler.Exists() {
			return nil
		}
		return err
	}
	c.lock.Lock()
	defer c.lock.Unlock()
	c.info.Subcontainers = subcontainers
	return nil
}
<|endoftext|>


<commit_before>/*

*/
package mandrake

import (
	"log"
	"encoding/json"

	"golang.org/x/exp/inotify"
	"github.com/hosom/gomandrake/config"
	"github.com/hosom/gomandrake/filemeta"
	"github.com/hosom/gomandrake/plugin"
)

// Mandrake is a wrapper struct for the bulk of the application logic
type Mandrake struct {
	AnalysisPipeline	chan string
	MonitoredDirectory	string
	Analyzers			[]plugin.AnalyzerCaller
	AnalyzerFilter		map[string][]plugin.AnalyzerCaller
}

// NewMandrake creates and returns a Mandrake struct utilizing a passed
// parsed configuration file to create the correct fields.
func NewMandrake(c config.Config) Mandrake {
	analyzers := []plugin.AnalyzerCaller{}
	filter := make(map[string][]plugin.AnalyzerCaller)
	for _, plug := range c.Analyzers {
		analyzer := plugin.NewAnalyzerCaller(plug)
		// Build a slice of all AnalyzerCaller structs
		analyzers = append(analyzers, analyzer)

		// Create a map to function as a mime_type filter for analyzers
		for _, mime := range analyzer.MimeFilter {
			filter[mime] = append(filter[mime], analyzer)
		}
	}

	return Mandrake{make(chan string), c.MonitoredDirectory, analyzers, filter}
}

// ListenAndServe starts the goroutines that perform all of the heavy lifting
// including Monitor() and DispatchAnalysis().
func (m Mandrake) ListenAndServe() {
	log.SetPrefix("[mandrake] ")
	log.Println(m.Analyzers[0])
	go m.DispatchAnalysis()
	m.Monitor()
}

// DispatchAnalysis intelligently sends a new file to registered plugins so
// that it can be analyzed.
func (m Mandrake) DispatchAnalysis() {
	for fpath := range m.AnalysisPipeline {
		fmeta, err := filemeta.NewFileMeta(fpath)
		if err != nil {
			log.Println(err)
		}

		// Create JSON filemeta object to pass to plugins so that plugins
		// receive basic contextual information about the file.
		fs, err := json.Marshal(fmeta)

		var analysis map[string]interface{}

		for _, analyzer := range m.AnalyzerFilter["all"] {
			result, err := analyzer.Analyze(string(fs))
			if err != nil {
				log.Print(err)
			}
			analysis[analyzer.Name] = make(map[string]interface{})
			analysis[analyzer.Name] = MapFromJSON(result)
		}

		for _, analyzer := range m.AnalyzerFilter[fmeta.Mime] {
			result, err := analyzer.Analyze(string(fs))
			if err != nil {
				log.Print(err)
			}
			analysis[analyzer.Name] = make(map[string]interface{})
			analysis[analyzer.Name] = MapFromJSON(result)
		}

		log.Println(json.Marshal(analysis))
		log.Println(string(fs))
		log.Printf("%s", fpath)
	}
}

// Monitor uses inotify to monitor the MonitoredDirectory for IN_CLOSE_WRITE
// events. Files written to the MonitoredDirectory will be sent to the
// analysis pipeline to be analyzed.
func (m Mandrake) Monitor() {
	log.Println("starting inotify watcher")
	watcher, err := inotify.NewWatcher()
	if err != nil {
		log.Fatal(err)
	}

	log.Printf("adding watcher to %s directory", m.MonitoredDirectory)
	err = watcher.AddWatch(m.MonitoredDirectory, inotify.IN_CLOSE_WRITE)
	if err != nil {
		log.Fatal(err)
	}

	for {
		select {
		case ev := <- watcher.Event:
			m.AnalysisPipeline <- ev.Name
		case err := <- watcher.Error:
			log.Printf("inotify error: %s", err)
		}
	}
}

// MapFromJSON accepts an anonymous JSON object as a string and returns the
// resulting Map
func MapFromJSON(s string) map[string]interface{} {
	log.Printf("Performing mapping with string: %s", s)
	var f interface{}
	json.Unmarshal([]byte(s), &f)
	m := f.(map[string]interface{})
	return m
}<commit_msg>attempt to fix crash<commit_after>/*

*/
package mandrake

import (
	"log"
	"encoding/json"

	"golang.org/x/exp/inotify"
	"github.com/hosom/gomandrake/config"
	"github.com/hosom/gomandrake/filemeta"
	"github.com/hosom/gomandrake/plugin"
)

// Mandrake is a wrapper struct for the bulk of the application logic
type Mandrake struct {
	AnalysisPipeline	chan string
	MonitoredDirectory	string
	Analyzers			[]plugin.AnalyzerCaller
	AnalyzerFilter		map[string][]plugin.AnalyzerCaller
}

// NewMandrake creates and returns a Mandrake struct utilizing a passed
// parsed configuration file to create the correct fields.
func NewMandrake(c config.Config) Mandrake {
	analyzers := []plugin.AnalyzerCaller{}
	filter := make(map[string][]plugin.AnalyzerCaller)
	for _, plug := range c.Analyzers {
		analyzer := plugin.NewAnalyzerCaller(plug)
		// Build a slice of all AnalyzerCaller structs
		analyzers = append(analyzers, analyzer)

		// Create a map to function as a mime_type filter for analyzers
		for _, mime := range analyzer.MimeFilter {
			filter[mime] = append(filter[mime], analyzer)
		}
	}

	return Mandrake{make(chan string), c.MonitoredDirectory, analyzers, filter}
}

// ListenAndServe starts the goroutines that perform all of the heavy lifting
// including Monitor() and DispatchAnalysis().
func (m Mandrake) ListenAndServe() {
	log.SetPrefix("[mandrake] ")
	log.Println(m.Analyzers[0])
	go m.DispatchAnalysis()
	m.Monitor()
}

// DispatchAnalysis intelligently sends a new file to registered plugins so
// that it can be analyzed.
func (m Mandrake) DispatchAnalysis() {
	for fpath := range m.AnalysisPipeline {
		fmeta, err := filemeta.NewFileMeta(fpath)
		if err != nil {
			log.Println(err)
		}

		// Create JSON filemeta object to pass to plugins so that plugins
		// receive basic contextual information about the file.
		fs, err := json.Marshal(fmeta)

		var analysis map[string]map[string]interface{}

		for _, analyzer := range m.AnalyzerFilter["all"] {
			result, err := analyzer.Analyze(string(fs))
			if err != nil {
				log.Print(err)
			}
			analysis[analyzer.Name] = MapFromJSON(result)
		}

		for _, analyzer := range m.AnalyzerFilter[fmeta.Mime] {
			result, err := analyzer.Analyze(string(fs))
			if err != nil {
				log.Print(err)
			}
			analysis[analyzer.Name] = MapFromJSON(result)
		}

		log.Println(json.Marshal(analysis))
		log.Println(string(fs))
		log.Printf("%s", fpath)
	}
}

// Monitor uses inotify to monitor the MonitoredDirectory for IN_CLOSE_WRITE
// events. Files written to the MonitoredDirectory will be sent to the
// analysis pipeline to be analyzed.
func (m Mandrake) Monitor() {
	log.Println("starting inotify watcher")
	watcher, err := inotify.NewWatcher()
	if err != nil {
		log.Fatal(err)
	}

	log.Printf("adding watcher to %s directory", m.MonitoredDirectory)
	err = watcher.AddWatch(m.MonitoredDirectory, inotify.IN_CLOSE_WRITE)
	if err != nil {
		log.Fatal(err)
	}

	for {
		select {
		case ev := <- watcher.Event:
			m.AnalysisPipeline <- ev.Name
		case err := <- watcher.Error:
			log.Printf("inotify error: %s", err)
		}
	}
}

// MapFromJSON accepts an anonymous JSON object as a string and returns the
// resulting Map
func MapFromJSON(s string) map[string]interface{} {
	log.Printf("Performing mapping with string: %s", s)
	var f interface{}
	json.Unmarshal([]byte(s), &f)
	m := f.(map[string]interface{})
	return m
}<|endoftext|>


<commit_before>package sgload

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"path"
	"regexp"
	"strings"
	"time"

	sgreplicate "github.com/couchbaselabs/sg-replicate"
	"github.com/peterbourgon/g2s"
)

var (
	// The sample rate -- set this to 0.1 if you only want
	// 10% of the samples to be pushed to statds, or .01 if you only
	// want 1% of the samples pushed to statsd.  Useful for
	// not overwhelming stats if you have too many samples.
	statsdSampleRate float32 = 1.0
)

type SGDataStore struct {
	SyncGatewayUrl       string
	SyncGatewayAdminPort int
	UserCreds            UserCred
	StatsdClient         *g2s.Statsd
}

func NewSGDataStore(sgUrl string, sgAdminPort int, statsdClient *g2s.Statsd) *SGDataStore {
	return &SGDataStore{
		SyncGatewayUrl:       sgUrl,
		SyncGatewayAdminPort: sgAdminPort,
		StatsdClient:         statsdClient,
	}
}

func (s *SGDataStore) SetUserCreds(u UserCred) {
	s.UserCreds = u
}

func (s SGDataStore) CreateUser(u UserCred, channelNames []string) error {

	adminUrl, err := s.sgAdminURL()
	if err != nil {
		return err
	}

	adminUrlUserEndpoint, err := addEndpointToUrl(adminUrl, "_user")
	if err != nil {
		return err
	}

	adminUrlUserEndpoint = addTrailingSlash(adminUrlUserEndpoint)

	userDoc := map[string]interface{}{}
	userDoc["name"] = u.Username
	userDoc["password"] = u.Password
	userDoc["admin_channels"] = channelNames

	docBytes, err := json.Marshal(userDoc)
	if err != nil {
		return err
	}
	buf := bytes.NewBuffer(docBytes)

	req, err := http.NewRequest("POST", adminUrlUserEndpoint, buf)

	req.Header.Set("Content-Type", "application/json")

	client := http.DefaultClient

	startTime := time.Now()
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	s.pushTimingStat("create_user", time.Since(startTime))

	if resp.StatusCode < 200 || resp.StatusCode > 201 {
		return fmt.Errorf("Unexpected response status for POST request: %d", resp.StatusCode)
	}

	defer resp.Body.Close()
	io.Copy(ioutil.Discard, resp.Body)

	return nil
}

func (s SGDataStore) sgAdminURL() (string, error) {

	sgAdminURL := s.SyncGatewayUrl

	parsedSgUrl, err := url.Parse(s.SyncGatewayUrl)
	if err != nil {
		log.Printf("Error parsing url: %v", err)
		return sgAdminURL, err
	}

	// find the port from the url
	host, port, err := splitHostPortWrapper(parsedSgUrl.Host)
	if err != nil {
		return sgAdminURL, err
	}

	if port != "" {
		// is there a port?
		// do a regex replace on :port on the original url
		r := regexp.MustCompile(port)
		sgAdminURL = r.ReplaceAllString(
			sgAdminURL,
			fmt.Sprintf("%d", s.SyncGatewayAdminPort),
		)

	} else {
		// is there no port?
		// do a regex replace of host with host:port
		r := regexp.MustCompile(host)
		hostWithAdminPort := fmt.Sprintf("%v:%v", host, s.SyncGatewayAdminPort)
		sgAdminURL = r.ReplaceAllString(
			sgAdminURL,
			hostWithAdminPort,
		)

	}

	// return it
	return sgAdminURL, nil

}

func (s SGDataStore) Changes(sinceVal Sincer, limit int) (changes sgreplicate.Changes, newSinceVal Sincer, err error) {

	changesFeedEndpoint, err := addEndpointToUrl(s.SyncGatewayUrl, "_changes")
	if err != nil {
		return sgreplicate.Changes{}, sinceVal, err
	}

	changesFeedParams := NewChangesFeedParams(sinceVal, limit)

	changesFeedUrl := fmt.Sprintf(
		"%s?%s",
		changesFeedEndpoint,
		changesFeedParams,
	)

	req, err := http.NewRequest("GET", changesFeedUrl, nil)
	s.addAuthIfNeeded(req)

	req.Header.Set("Content-Type", "application/json")

	client := http.DefaultClient

	startTime := time.Now()
	resp, err := client.Do(req)
	if err != nil {
		return sgreplicate.Changes{}, sinceVal, err
	}
	s.pushTimingStat("changes_feed", time.Since(startTime))
	if resp.StatusCode < 200 || resp.StatusCode > 201 {
		return sgreplicate.Changes{}, sinceVal, fmt.Errorf("Unexpected response status for POST request: %d", resp.StatusCode)
	}

	defer resp.Body.Close()
	decoder := json.NewDecoder(resp.Body)
	err = decoder.Decode(&changes)
	if err != nil {
		return sgreplicate.Changes{}, sinceVal, err
	}
	lastSequenceStr, ok := changes.LastSequence.(string)
	if !ok {
		return sgreplicate.Changes{}, sinceVal, fmt.Errorf("Could not convert changes.LastSequence to string")
	}
	lastSequenceSincer := StringSincer{
		Since: lastSequenceStr,
	}

	return changes, lastSequenceSincer, nil
}

// Create a single document in Sync Gateway
func (s SGDataStore) CreateDocument(d Document) (sgreplicate.DocumentRevisionPair, error) {

	docRevisionPairs, err := s.BulkCreateDocuments([]Document{d})
	if err != nil {
		return sgreplicate.DocumentRevisionPair{}, err
	}
	if len(docRevisionPairs) == 0 {
		return sgreplicate.DocumentRevisionPair{}, fmt.Errorf("Unexpected response")
	}
	return docRevisionPairs[0], nil

}

// Bulk create a set of documents in Sync Gateway
func (s SGDataStore) BulkCreateDocuments(docs []Document) ([]sgreplicate.DocumentRevisionPair, error) {

	defer s.pushCounter("create_document_counter", len(docs))

	// Set the "created_at" timestamp which is used to calculate the
	// gateload roundtrip time
	updateCreatedAtTimestamp(docs)

	bulkDocsResponse := []sgreplicate.DocumentRevisionPair{}

	bulkDocsEndpoint, err := addEndpointToUrl(s.SyncGatewayUrl, "_bulk_docs")
	if err != nil {
		return bulkDocsResponse, err
	}

	bulkDocs := BulkDocs{
		Documents: docs,
		NewEdits:  true,
	}
	docBytes, err := json.Marshal(bulkDocs)
	if err != nil {
		return bulkDocsResponse, err
	}
	buf := bytes.NewBuffer(docBytes)

	req, err := http.NewRequest("POST", bulkDocsEndpoint, buf)
	s.addAuthIfNeeded(req)

	req.Header.Set("Content-Type", "application/json")

	client := http.DefaultClient

	startTime := time.Now()
	resp, err := client.Do(req)
	if err != nil {
		return bulkDocsResponse, err
	}
	s.pushTimingStat("create_document", timeDeltaPerDocument(len(docs), time.Since(startTime)))
	if resp.StatusCode < 200 || resp.StatusCode > 201 {
		return bulkDocsResponse, fmt.Errorf("Unexpected response status for POST request: %d", resp.StatusCode)
	}

	defer resp.Body.Close()

	decoder := json.NewDecoder(resp.Body)
	if err = decoder.Decode(&bulkDocsResponse); err != nil {
		return bulkDocsResponse, err
	}

	// If any of the bulk docs had errors, return an error
	for _, docRevisionPair := range bulkDocsResponse {
		if docRevisionPair.Error != "" {
			return bulkDocsResponse, fmt.Errorf("%v", docRevisionPair.Error)
		}
	}

	return bulkDocsResponse, nil

}

func (s SGDataStore) BulkGetDocuments(r sgreplicate.BulkGetRequest) ([]sgreplicate.Document, error) {

	defer s.pushCounter("get_document_counter", len(r.Docs))

	bulkGetEndpoint, err := addEndpointToUrl(s.SyncGatewayUrl, "_bulk_get")
	if err != nil {
		return nil, err
	}

	bulkGetBytes, err := json.Marshal(r)
	if err != nil {
		return nil, fmt.Errorf("BulkGetDocuemnts failed to marshal request: %v", err)
	}

	buf := bytes.NewBuffer(bulkGetBytes)

	req, err := http.NewRequest("POST", bulkGetEndpoint, buf)
	s.addAuthIfNeeded(req)

	req.Header.Set("Content-Type", "application/json")

	client := http.DefaultClient

	startTime := time.Now()
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	s.pushTimingStat("get_document", timeDeltaPerDocument(len(r.Docs), time.Since(startTime)))
	if resp.StatusCode < 200 || resp.StatusCode > 201 {
		return nil, fmt.Errorf("Unexpected response status for POST request: %d", resp.StatusCode)
	}

	// Parse the response and make sure that we got all the docs we requested
	defer resp.Body.Close()
	loggerFunc := func(key string, format string, args ...interface{}) {
		logger.Warn("BulkGetResponse", "log", fmt.Sprintf(format, args))
	}
	documents, err := sgreplicate.ReadBulkGetResponse(resp, loggerFunc)
	if len(documents) != len(r.Docs) {
		return nil, fmt.Errorf("Expected %d docs, got %d docs", len(r.Docs), len(documents))
	}

	for _, doc := range documents {
		createAtRFC3339NanoIface, ok := doc.Body["created_at"]
		if !ok {
			logger.Warn("Document missing created_at field", "doc.Body", doc.Body)
			continue
		}
		createAtRFC3339NanoStr, ok := createAtRFC3339NanoIface.(string)
		if !ok {
			logger.Warn("Document created_at not a string", "doc.Body", doc.Body)
			continue
		}
		createAtRFC3339Nano, err := time.Parse(
			time.RFC3339Nano,
			createAtRFC3339NanoStr,
		)
		if err != nil {
			logger.Warn("Could not parse doc.created_at field into time", "createAtRFC3339Nano", createAtRFC3339Nano)
			continue
		}
		delta := time.Since(createAtRFC3339Nano)
		s.pushTimingStat("gateload_roundtrip", delta)
	}

	return documents, nil

}

// add BasicAuth header for user if needed
func (s SGDataStore) addAuthIfNeeded(req *http.Request) {
	if !s.UserCreds.Empty() {
		req.SetBasicAuth(s.UserCreds.Username, s.UserCreds.Password)
	}
}

func (s SGDataStore) pushTimingStat(key string, delta time.Duration) {
	if s.StatsdClient == nil {
		return
	}
	s.StatsdClient.Timing(
		statsdSampleRate,
		key,
		delta,
	)
}

func (s SGDataStore) pushCounter(key string, n int) {
	if s.StatsdClient == nil {
		return
	}
	s.StatsdClient.Counter(
		statsdSampleRate,
		key,
		n,
	)
}

func splitHostPortWrapper(host string) (string, string, error) {
	if !strings.Contains(host, ":") {
		return host, "", nil
	}

	return net.SplitHostPort(host)
}

func addEndpointToUrl(urlStr, endpoint string) (string, error) {
	u, err := url.Parse(urlStr)
	if err != nil {
		return "", err
	}
	u.Path = path.Join(u.Path, endpoint)
	return u.String(), nil
}

func addTrailingSlash(urlStr string) string {
	return fmt.Sprintf("%v/", urlStr)
}

func timeDeltaPerDocument(numDocs int, timeDeltaAllDocs time.Duration) time.Duration {
	if numDocs == 0 {
		return timeDeltaAllDocs
	}
	return time.Duration(int64(timeDeltaAllDocs) / int64(numDocs))
}
<commit_msg>Absorb BulkGetResponse logs instead of printing them as warnings<commit_after>package sgload

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"path"
	"regexp"
	"strings"
	"time"

	sgreplicate "github.com/couchbaselabs/sg-replicate"
	"github.com/peterbourgon/g2s"
)

var (
	// The sample rate -- set this to 0.1 if you only want
	// 10% of the samples to be pushed to statds, or .01 if you only
	// want 1% of the samples pushed to statsd.  Useful for
	// not overwhelming stats if you have too many samples.
	statsdSampleRate float32 = 1.0
)

type SGDataStore struct {
	SyncGatewayUrl       string
	SyncGatewayAdminPort int
	UserCreds            UserCred
	StatsdClient         *g2s.Statsd
}

func NewSGDataStore(sgUrl string, sgAdminPort int, statsdClient *g2s.Statsd) *SGDataStore {
	return &SGDataStore{
		SyncGatewayUrl:       sgUrl,
		SyncGatewayAdminPort: sgAdminPort,
		StatsdClient:         statsdClient,
	}
}

func (s *SGDataStore) SetUserCreds(u UserCred) {
	s.UserCreds = u
}

func (s SGDataStore) CreateUser(u UserCred, channelNames []string) error {

	adminUrl, err := s.sgAdminURL()
	if err != nil {
		return err
	}

	adminUrlUserEndpoint, err := addEndpointToUrl(adminUrl, "_user")
	if err != nil {
		return err
	}

	adminUrlUserEndpoint = addTrailingSlash(adminUrlUserEndpoint)

	userDoc := map[string]interface{}{}
	userDoc["name"] = u.Username
	userDoc["password"] = u.Password
	userDoc["admin_channels"] = channelNames

	docBytes, err := json.Marshal(userDoc)
	if err != nil {
		return err
	}
	buf := bytes.NewBuffer(docBytes)

	req, err := http.NewRequest("POST", adminUrlUserEndpoint, buf)

	req.Header.Set("Content-Type", "application/json")

	client := http.DefaultClient

	startTime := time.Now()
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	s.pushTimingStat("create_user", time.Since(startTime))

	if resp.StatusCode < 200 || resp.StatusCode > 201 {
		return fmt.Errorf("Unexpected response status for POST request: %d", resp.StatusCode)
	}

	defer resp.Body.Close()
	io.Copy(ioutil.Discard, resp.Body)

	return nil
}

func (s SGDataStore) sgAdminURL() (string, error) {

	sgAdminURL := s.SyncGatewayUrl

	parsedSgUrl, err := url.Parse(s.SyncGatewayUrl)
	if err != nil {
		log.Printf("Error parsing url: %v", err)
		return sgAdminURL, err
	}

	// find the port from the url
	host, port, err := splitHostPortWrapper(parsedSgUrl.Host)
	if err != nil {
		return sgAdminURL, err
	}

	if port != "" {
		// is there a port?
		// do a regex replace on :port on the original url
		r := regexp.MustCompile(port)
		sgAdminURL = r.ReplaceAllString(
			sgAdminURL,
			fmt.Sprintf("%d", s.SyncGatewayAdminPort),
		)

	} else {
		// is there no port?
		// do a regex replace of host with host:port
		r := regexp.MustCompile(host)
		hostWithAdminPort := fmt.Sprintf("%v:%v", host, s.SyncGatewayAdminPort)
		sgAdminURL = r.ReplaceAllString(
			sgAdminURL,
			hostWithAdminPort,
		)

	}

	// return it
	return sgAdminURL, nil

}

func (s SGDataStore) Changes(sinceVal Sincer, limit int) (changes sgreplicate.Changes, newSinceVal Sincer, err error) {

	changesFeedEndpoint, err := addEndpointToUrl(s.SyncGatewayUrl, "_changes")
	if err != nil {
		return sgreplicate.Changes{}, sinceVal, err
	}

	changesFeedParams := NewChangesFeedParams(sinceVal, limit)

	changesFeedUrl := fmt.Sprintf(
		"%s?%s",
		changesFeedEndpoint,
		changesFeedParams,
	)

	req, err := http.NewRequest("GET", changesFeedUrl, nil)
	s.addAuthIfNeeded(req)

	req.Header.Set("Content-Type", "application/json")

	client := http.DefaultClient

	startTime := time.Now()
	resp, err := client.Do(req)
	if err != nil {
		return sgreplicate.Changes{}, sinceVal, err
	}
	s.pushTimingStat("changes_feed", time.Since(startTime))
	if resp.StatusCode < 200 || resp.StatusCode > 201 {
		return sgreplicate.Changes{}, sinceVal, fmt.Errorf("Unexpected response status for POST request: %d", resp.StatusCode)
	}

	defer resp.Body.Close()
	decoder := json.NewDecoder(resp.Body)
	err = decoder.Decode(&changes)
	if err != nil {
		return sgreplicate.Changes{}, sinceVal, err
	}
	lastSequenceStr, ok := changes.LastSequence.(string)
	if !ok {
		return sgreplicate.Changes{}, sinceVal, fmt.Errorf("Could not convert changes.LastSequence to string")
	}
	lastSequenceSincer := StringSincer{
		Since: lastSequenceStr,
	}

	return changes, lastSequenceSincer, nil
}

// Create a single document in Sync Gateway
func (s SGDataStore) CreateDocument(d Document) (sgreplicate.DocumentRevisionPair, error) {

	docRevisionPairs, err := s.BulkCreateDocuments([]Document{d})
	if err != nil {
		return sgreplicate.DocumentRevisionPair{}, err
	}
	if len(docRevisionPairs) == 0 {
		return sgreplicate.DocumentRevisionPair{}, fmt.Errorf("Unexpected response")
	}
	return docRevisionPairs[0], nil

}

// Bulk create a set of documents in Sync Gateway
func (s SGDataStore) BulkCreateDocuments(docs []Document) ([]sgreplicate.DocumentRevisionPair, error) {

	defer s.pushCounter("create_document_counter", len(docs))

	// Set the "created_at" timestamp which is used to calculate the
	// gateload roundtrip time
	updateCreatedAtTimestamp(docs)

	bulkDocsResponse := []sgreplicate.DocumentRevisionPair{}

	bulkDocsEndpoint, err := addEndpointToUrl(s.SyncGatewayUrl, "_bulk_docs")
	if err != nil {
		return bulkDocsResponse, err
	}

	bulkDocs := BulkDocs{
		Documents: docs,
		NewEdits:  true,
	}
	docBytes, err := json.Marshal(bulkDocs)
	if err != nil {
		return bulkDocsResponse, err
	}
	buf := bytes.NewBuffer(docBytes)

	req, err := http.NewRequest("POST", bulkDocsEndpoint, buf)
	s.addAuthIfNeeded(req)

	req.Header.Set("Content-Type", "application/json")

	client := http.DefaultClient

	startTime := time.Now()
	resp, err := client.Do(req)
	if err != nil {
		return bulkDocsResponse, err
	}
	s.pushTimingStat("create_document", timeDeltaPerDocument(len(docs), time.Since(startTime)))
	if resp.StatusCode < 200 || resp.StatusCode > 201 {
		return bulkDocsResponse, fmt.Errorf("Unexpected response status for POST request: %d", resp.StatusCode)
	}

	defer resp.Body.Close()

	decoder := json.NewDecoder(resp.Body)
	if err = decoder.Decode(&bulkDocsResponse); err != nil {
		return bulkDocsResponse, err
	}

	// If any of the bulk docs had errors, return an error
	for _, docRevisionPair := range bulkDocsResponse {
		if docRevisionPair.Error != "" {
			return bulkDocsResponse, fmt.Errorf("%v", docRevisionPair.Error)
		}
	}

	return bulkDocsResponse, nil

}

func (s SGDataStore) BulkGetDocuments(r sgreplicate.BulkGetRequest) ([]sgreplicate.Document, error) {

	defer s.pushCounter("get_document_counter", len(r.Docs))

	bulkGetEndpoint, err := addEndpointToUrl(s.SyncGatewayUrl, "_bulk_get")
	if err != nil {
		return nil, err
	}

	bulkGetBytes, err := json.Marshal(r)
	if err != nil {
		return nil, fmt.Errorf("BulkGetDocuemnts failed to marshal request: %v", err)
	}

	buf := bytes.NewBuffer(bulkGetBytes)

	req, err := http.NewRequest("POST", bulkGetEndpoint, buf)
	s.addAuthIfNeeded(req)

	req.Header.Set("Content-Type", "application/json")

	client := http.DefaultClient

	startTime := time.Now()
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	s.pushTimingStat("get_document", timeDeltaPerDocument(len(r.Docs), time.Since(startTime)))
	if resp.StatusCode < 200 || resp.StatusCode > 201 {
		return nil, fmt.Errorf("Unexpected response status for POST request: %d", resp.StatusCode)
	}

	// Parse the response and make sure that we got all the docs we requested
	defer resp.Body.Close()

	// Logger function that ignores any logging from the ReadBulkGetResponse method
	loggerFunc := func(key string, format string, args ...interface{}) {}
	documents, err := sgreplicate.ReadBulkGetResponse(resp, loggerFunc)
	if len(documents) != len(r.Docs) {
		return nil, fmt.Errorf("Expected %d docs, got %d docs", len(r.Docs), len(documents))
	}

	for _, doc := range documents {
		createAtRFC3339NanoIface, ok := doc.Body["created_at"]
		if !ok {
			logger.Warn("Document missing created_at field", "doc.Body", doc.Body)
			continue
		}
		createAtRFC3339NanoStr, ok := createAtRFC3339NanoIface.(string)
		if !ok {
			logger.Warn("Document created_at not a string", "doc.Body", doc.Body)
			continue
		}
		createAtRFC3339Nano, err := time.Parse(
			time.RFC3339Nano,
			createAtRFC3339NanoStr,
		)
		if err != nil {
			logger.Warn("Could not parse doc.created_at field into time", "createAtRFC3339Nano", createAtRFC3339Nano)
			continue
		}
		delta := time.Since(createAtRFC3339Nano)
		s.pushTimingStat("gateload_roundtrip", delta)
	}

	return documents, nil

}

// add BasicAuth header for user if needed
func (s SGDataStore) addAuthIfNeeded(req *http.Request) {
	if !s.UserCreds.Empty() {
		req.SetBasicAuth(s.UserCreds.Username, s.UserCreds.Password)
	}
}

func (s SGDataStore) pushTimingStat(key string, delta time.Duration) {
	if s.StatsdClient == nil {
		return
	}
	s.StatsdClient.Timing(
		statsdSampleRate,
		key,
		delta,
	)
}

func (s SGDataStore) pushCounter(key string, n int) {
	if s.StatsdClient == nil {
		return
	}
	s.StatsdClient.Counter(
		statsdSampleRate,
		key,
		n,
	)
}

func splitHostPortWrapper(host string) (string, string, error) {
	if !strings.Contains(host, ":") {
		return host, "", nil
	}

	return net.SplitHostPort(host)
}

func addEndpointToUrl(urlStr, endpoint string) (string, error) {
	u, err := url.Parse(urlStr)
	if err != nil {
		return "", err
	}
	u.Path = path.Join(u.Path, endpoint)
	return u.String(), nil
}

func addTrailingSlash(urlStr string) string {
	return fmt.Sprintf("%v/", urlStr)
}

func timeDeltaPerDocument(numDocs int, timeDeltaAllDocs time.Duration) time.Duration {
	if numDocs == 0 {
		return timeDeltaAllDocs
	}
	return time.Duration(int64(timeDeltaAllDocs) / int64(numDocs))
}
<|endoftext|>


<commit_before>package resolvers

import (
	"github.com/etcinit/gonduit/requests"
	"github.com/etcinit/phabulous/app/factories"
	"github.com/jacobstr/confer"
)

// DifferentialResolver resolves phabricator revisions and diffs to a channel.
type DifferentialResolver struct {
	Config  *confer.Config            `inject:""`
	Factory *factories.GonduitFactory `inject:""`
}

// Resolve resolves the channel the message about a revision should be posted
// on.
func (c *DifferentialResolver) Resolve(phid string) (string, error) {
	conduit, err := c.Factory.Make()

	if err != nil {
		return "", err
	}

	results, err := conduit.DifferentialQuery(
		requests.DifferentialQueryRequest{
			PHIDs: []string{phid},
		},
	)

	if err != nil {
		return "", err
	}

	revision := (*results)[0]

	repos, err := conduit.RepositoryQuery(requests.RepositoryQueryRequest{
		PHIDs: []string{revision.RepositoryPHID},
		Order: "newest",
	})

	if err != nil {
		return "", err
	}

	// If the response contains a repository, try to match it
	if len(*repos) !=0 {
		channelMap := c.Config.GetStringMapString("channels.repositories")

		if channelName, ok := channelMap[(*repos)[0].Callsign]; ok == true {
			return channelName, nil
		}
	}

	// Look for a project if the repository didn't match
	if projects, ok := revision.Auxiliary["phabricator:projects"];
			ok == true && len(projects) != 0 {
		res, err := conduit.ProjectQuery(requests.ProjectQueryRequest{
			PHIDs: []string{projects[0]},
		})
		if err != nil {
			return "", err
		}
		if proj, ok := res.Data[projects[0]]; ok == true {
			channelMap := c.Config.GetStringMapString("channels.projects")
			if channelName, ok := channelMap[proj.Name]; ok == true {
				return channelName, nil
			}
		}
	}

	return "", nil
}
<commit_msg>style(resolvers): Run gofmt and clean up spacing<commit_after>package resolvers

import (
	"github.com/etcinit/gonduit/requests"
	"github.com/etcinit/phabulous/app/factories"
	"github.com/jacobstr/confer"
)

// DifferentialResolver resolves phabricator revisions and diffs to a channel.
type DifferentialResolver struct {
	Config  *confer.Config            `inject:""`
	Factory *factories.GonduitFactory `inject:""`
}

// Resolve resolves the channel the message about a revision should be posted
// on.
func (c *DifferentialResolver) Resolve(phid string) (string, error) {
	conduit, err := c.Factory.Make()

	if err != nil {
		return "", err
	}

	results, err := conduit.DifferentialQuery(
		requests.DifferentialQueryRequest{
			PHIDs: []string{phid},
		},
	)

	if err != nil {
		return "", err
	}

	revision := (*results)[0]

	repos, err := conduit.RepositoryQuery(requests.RepositoryQueryRequest{
		PHIDs: []string{revision.RepositoryPHID},
		Order: "newest",
	})

	if err != nil {
		return "", err
	}

	// If the response contains a repository, try to match it
	if len(*repos) != 0 {
		channelMap := c.Config.GetStringMapString("channels.repositories")

		if channelName, ok := channelMap[(*repos)[0].Callsign]; ok == true {
			return channelName, nil
		}
	}

	// Look for a project if the repository didn't match
	projects, ok := revision.Auxiliary["phabricator:projects"]

	if ok == true && len(projects) != 0 {
		res, err := conduit.ProjectQuery(requests.ProjectQueryRequest{
			PHIDs: []string{projects[0]},
		})

		if err != nil {
			return "", err
		}

		if proj, ok := res.Data[projects[0]]; ok == true {
			channelMap := c.Config.GetStringMapString("channels.projects")

			if channelName, ok := channelMap[proj.Name]; ok == true {
				return channelName, nil
			}
		}
	}

	return "", nil
}
<|endoftext|>


<commit_before>package http_api

import (
	"encoding/hex"
	"fmt"
	"net/http"
	"testing"

	"github.com/jarcoal/httpmock"
	"github.com/uniqush/uniqush-push/push"
	"github.com/uniqush/uniqush-push/srv/apns/common"
)

const (
	authToken = "test_auth_token"
	keyFile   = "../apns-test/localhost.p8"
	keyID     = "FD8789SD9"
	teamID    = "JVNS20943"
	bundleID  = "com.example.test"
)

var (
	pushServiceProvider = &push.PushServiceProvider{
		push.PushPeer{
			VolatileData: map[string]string{
				"addr": "https://api.development.push.apple.com",
			},
			FixedData: map[string]string{
				"p8":       keyFile,
				"keyid":    keyID,
				"teamid":   teamID,
				"bundleid": bundleID,
			},
		},
	}
	devToken = []byte("test_device_token")
	payload  = []byte(`{"alert":"test_message"}`)
	apiURL   = fmt.Sprintf("%s/3/device/%s", pushServiceProvider.VolatileData["addr"], hex.EncodeToString(devToken))
)

type MockJWTManager struct{}

func (*MockJWTManager) GenerateToken() (string, error) {
	return authToken, nil
}

func mockAPNSRequest(fn func(r *http.Request) (*http.Response, error)) {
	httpmock.RegisterResponder("POST", apiURL, fn)
}

func newPushRequest() (*common.PushRequest, chan push.PushError, chan *common.APNSResult) {
	errChan := make(chan push.PushError)
	resChan := make(chan *common.APNSResult, 1)
	request := &common.PushRequest{
		PSP:       pushServiceProvider,
		Devtokens: [][]byte{devToken},
		Payload:   payload,
		ErrChan:   errChan,
		ResChan:   resChan,
	}

	return request, errChan, resChan
}

var mockJWTManager = &MockJWTManager{}

func TestAddRequestPushSuccessful(t *testing.T) {
	httpmock.Activate()
	defer httpmock.DeactivateAndReset()

	request, _, resChan := newPushRequest()
	mockAPNSRequest(func(r *http.Request) (*http.Response, error) {
		// Return empty body
		return httpmock.NewBytesResponse(http.StatusOK, nil), nil
	})

	common.SetJWTManagerSingleton(mockJWTManager)
	NewRequestProcessor().AddRequest(request)

	res := <-resChan
	if res.MsgId == 0 {
		t.Fatal("Expected non-zero message id, got zero")
	}
}

func TestAddRequestPushFailConnectionError(t *testing.T) {
	httpmock.Activate()
	defer httpmock.DeactivateAndReset()

	request, errChan, _ := newPushRequest()
	common.SetJWTManagerSingleton(mockJWTManager)
	mockAPNSRequest(func(r *http.Request) (*http.Response, error) {
		return nil, fmt.Errorf("No connection")
	})

	NewRequestProcessor().AddRequest(request)

	err := <-errChan
	if _, ok := err.(*push.ConnectionError); !ok {
		t.Fatal("Expected Connection error, got", err)
	}
}

func TestAddRequestPushFailNotificationError(t *testing.T) {
	httpmock.Activate()
	defer httpmock.DeactivateAndReset()

	request, errChan, _ := newPushRequest()
	common.SetJWTManagerSingleton(mockJWTManager)
	mockAPNSRequest(func(r *http.Request) (*http.Response, error) {
		response := &APNSErrorResponse{
			Reason: "BadDeviceToken",
		}
		return httpmock.NewJsonResponse(http.StatusBadRequest, response)
	})

	NewRequestProcessor().AddRequest(request)

	err := <-errChan
	if _, ok := err.(*push.BadNotification); !ok {
		t.Fatal("Expected BadNotification error, got", err)
	}
}

func TestGetMaxPayloadSize(t *testing.T) {
	maxPayloadSize := NewRequestProcessor().GetMaxPayloadSize()
	if maxPayloadSize != 4096 {
		t.Fatalf("Wrong max payload, expected `4096`, got `%d`", maxPayloadSize)
	}
}
<commit_msg>Add request header and payload validation in unit test<commit_after>package http_api

import (
	"encoding/hex"
	"fmt"
	"net/http"
	"testing"

	"io/ioutil"

	"bytes"

	"github.com/jarcoal/httpmock"
	"github.com/uniqush/uniqush-push/push"
	"github.com/uniqush/uniqush-push/srv/apns/common"
)

const (
	authToken = "test_auth_token"
	keyFile   = "../apns-test/localhost.p8"
	keyID     = "FD8789SD9"
	teamID    = "JVNS20943"
	bundleID  = "com.example.test"
)

var (
	pushServiceProvider = &push.PushServiceProvider{
		push.PushPeer{
			VolatileData: map[string]string{
				"addr": "https://api.development.push.apple.com",
			},
			FixedData: map[string]string{
				"p8":       keyFile,
				"keyid":    keyID,
				"teamid":   teamID,
				"bundleid": bundleID,
			},
		},
	}
	devToken = []byte("test_device_token")
	payload  = []byte(`{"alert":"test_message"}`)
	apiURL   = fmt.Sprintf("%s/3/device/%s", pushServiceProvider.VolatileData["addr"], hex.EncodeToString(devToken))
)

type MockJWTManager struct{}

func (*MockJWTManager) GenerateToken() (string, error) {
	return authToken, nil
}

func mockAPNSRequest(fn func(r *http.Request) (*http.Response, error)) {
	httpmock.RegisterResponder("POST", apiURL, fn)
}

func newPushRequest() (*common.PushRequest, chan push.PushError, chan *common.APNSResult) {
	errChan := make(chan push.PushError)
	resChan := make(chan *common.APNSResult, 1)
	request := &common.PushRequest{
		PSP:       pushServiceProvider,
		Devtokens: [][]byte{devToken},
		Payload:   payload,
		ErrChan:   errChan,
		ResChan:   resChan,
	}

	return request, errChan, resChan
}

var mockJWTManager = &MockJWTManager{}

func TestAddRequestPushSuccessful(t *testing.T) {
	httpmock.Activate()
	defer httpmock.DeactivateAndReset()

	request, _, resChan := newPushRequest()
	mockAPNSRequest(func(r *http.Request) (*http.Response, error) {
		if len(r.Header["authorization"]) == 0 {
			t.Error("Missing authorization header")
		}
		if len(r.Header["apns-expiration"]) == 0 {
			t.Error("Missing apns-expiration header")
		}
		if len(r.Header["apns-priority"]) == 0 {
			t.Error("Missing apns-priority header")
		}
		if len(r.Header["apns-topic"]) == 0 {
			t.Error("Missing apns-topic header")
		}
		requestBody, err := ioutil.ReadAll(r.Body)
		if err != nil {
			t.Error("Error reading request body:", err)
		}
		if bytes.Compare(requestBody, payload) != 0 {
			t.Errorf("Wrong message payload, expected `%v`, got `%v`", payload, requestBody)
		}
		// Return empty body
		return httpmock.NewBytesResponse(http.StatusOK, nil), nil
	})

	common.SetJWTManagerSingleton(mockJWTManager)
	NewRequestProcessor().AddRequest(request)

	res := <-resChan
	if res.MsgId == 0 {
		t.Fatal("Expected non-zero message id, got zero")
	}
}

func TestAddRequestPushFailConnectionError(t *testing.T) {
	httpmock.Activate()
	defer httpmock.DeactivateAndReset()

	request, errChan, _ := newPushRequest()
	common.SetJWTManagerSingleton(mockJWTManager)
	mockAPNSRequest(func(r *http.Request) (*http.Response, error) {
		return nil, fmt.Errorf("No connection")
	})

	NewRequestProcessor().AddRequest(request)

	err := <-errChan
	if _, ok := err.(*push.ConnectionError); !ok {
		t.Fatal("Expected Connection error, got", err)
	}
}

func TestAddRequestPushFailNotificationError(t *testing.T) {
	httpmock.Activate()
	defer httpmock.DeactivateAndReset()

	request, errChan, _ := newPushRequest()
	common.SetJWTManagerSingleton(mockJWTManager)
	mockAPNSRequest(func(r *http.Request) (*http.Response, error) {
		response := &APNSErrorResponse{
			Reason: "BadDeviceToken",
		}
		return httpmock.NewJsonResponse(http.StatusBadRequest, response)
	})

	NewRequestProcessor().AddRequest(request)

	err := <-errChan
	if _, ok := err.(*push.BadNotification); !ok {
		t.Fatal("Expected BadNotification error, got", err)
	}
}

func TestGetMaxPayloadSize(t *testing.T) {
	maxPayloadSize := NewRequestProcessor().GetMaxPayloadSize()
	if maxPayloadSize != 4096 {
		t.Fatalf("Wrong max payload, expected `4096`, got `%d`", maxPayloadSize)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/op/go-logging"
	"os"
	"io"
	"io/ioutil"
	"net/http"
	// "flag"
	"fmt"
	"bufio"
	"runtime"
	"strings"
	// "github.com/droundy/goopt"
	"github.com/andrew-d/go-termutil"
)

var log = logging.MustGetLogger("dfpp")
var format = "%{color}%{time:2006-01-02T15:04:05.000Z07:00} %{level:-5s} [%{shortfile}]%{color:reset} %{message}"

func main() {
	if termutil.Isatty(os.Stdin.Fd()) {
		Usage()
		os.Exit(0)
    }

	logBackend := logging.NewLogBackend(os.Stderr, "", 0)
	logging.SetBackend(
		logging.NewBackendFormatter(
			logBackend,
			logging.MustStringFormatter(format),
		),
	)
	logging.SetLevel(logging.DEBUG, "")

	for line := range InstructionScanner(os.Stdin) {
		parts := strings.Fields(line)
		if len(parts) > 0 {
			instruction := parts[0]
			if instruction == "INCLUDE" {
				ProcessInclude(line, parts)
				continue;
			}
		}
		fmt.Println(line)
	}
}

func InstructionScanner(input io.Reader) chan string {
	ch := make(chan string)
	go func() {
		scanner := bufio.NewScanner(input)
		for scanner.Scan() {
			line := scanner.Text();
			for len(line)>0 && line[len(line)-1] == '\\' {
				scanner.Scan();
				line += "\n" + scanner.Text()
			}
			ch <- line
		}
		close(ch)
	}()
	return ch
}

func ProcessInclude(line string, fields []string) {
	merge := false
	exclude := make(map[string]bool)
	include := make(map[string]bool)

	uris := make([]string,0, len(fields)-1)
	for _, field := range fields {
		if _, err := os.Stat(field); err == nil {
			uris = append(uris, field)
			continue;
		}
		clude := include
		if field[0] == '-' {
			clude = exclude
			field = field[1:]
		}

		switch field {
        case "\\": continue
		case "INCLUDE": continue
		case "MERGE": merge = true
		case "ADD": fallthrough
		case "CMD": fallthrough
		case "COPY": fallthrough
		case "ENTRYPOINT": fallthrough
		case "EVN": fallthrough
		case "EXPOSE": fallthrough
		case "FROM": fallthrough
		case "LABEL": fallthrough
		case "MAINTAINER": fallthrough
		case "ONBUILD": fallthrough
		case "RUN": fallthrough
		case "USER": fallthrough
		case "VOLUME": fallthrough
		case "WORKDIR": clude[field] = true
		default: uris = append(uris, field)
		}
	}

	docs := make([]string, 0, len(uris))
	for _, uri := range uris {
		if _, err := os.Stat(uri); err == nil {
			content, err := ioutil.ReadFile(uri)
			if err != nil {
				log.Error("Failed to read %s: %s", uri, err)
				os.Exit(1)
			}
			docs = append(docs, string(content))
		} else {
			req, _ := http.NewRequest("GET", uri, nil)
			ua := &http.Client{}
			if resp, err := ua.Do(req); err != nil {
				log.Error("Failed to %s %s: %s", req.Method, req.URL.String(), err)
				os.Exit(1)
			} else {
				if resp.StatusCode < 200 || resp.StatusCode >= 300 && resp.StatusCode != 401 {
					log.Error("response status: %s", resp.Status)
				}
				runtime.SetFinalizer(resp, func(r *http.Response) {
					r.Body.Close()
				})
				if buf, err := ioutil.ReadAll(resp.Body); err == nil {
					docs = append(docs, string(buf))
				}
			}
		}
	}
	Merge(merge, docs, include, exclude)
}

func Merge(merge bool, docs []string, include, exclude map[string]bool) {
	result := make([]*string, 0)
	ops := make(map[string]*string)
	for _, doc := range docs {
		for line := range InstructionScanner( strings.NewReader(doc) ) {
			fields := strings.Fields(line)
			if len(fields) > 0 {
				op := fields[0]
				if _, ok := exclude[op]; ok {
					continue
				}
				if _, ok := include[op]; len(include) > 0 && !ok {
					continue
				}
				details := strings.TrimPrefix(line, fields[0]+" ")

				if sref, ok := ops[op]; merge && ok {
					if op == "ENV" || op == "LABEL" {
						*sref += " \\\n" + strings.Repeat(" ", len(op)+1) + details
					} else if op == "RUN" {
						*sref += " && \\\n    " + details

						// squash redundent apt-get updates
						squash := "apt-get update"
						if ix := strings.Index(*sref, squash); ix >= 0 {
							rest := strings.Replace((*sref)[ix+len(squash):], squash, "echo skipping redundent apt-get-update", -1);
							*sref = (*sref)[0:ix+len(squash)] + rest
						}

					} else {
						dup := string(line)
						result = append(result, &dup)
					}
				} else {
					dup := string(line)
					result = append(result, &dup)
					ops[op] = result[len(result)-1]
				}
			}
		}
	}
	for _, line := range result {
		fmt.Println(*line)
	}
}

func Usage() {
	fmt.Print(`
NAME
    dfpp - Dockerfile preprocessor

SYNOPSIS
        $ dfpp Dockerfile.pre > Dockerfile

        # Dockerfile Syntax:
        INCLUDE ./Dockerfile.inc
        INCLUDE http://path/to/Dockerfile.inc
        INCLUDE ./Dockerfile.inc http://path/to/Dockerfile.inc

        INCLUDE MERGE a.inc b.inc

        # include only RUN instructions
        INCLUDE RUN a.inc b.inc

        # include only RUN and ENV instructions
        INCLUDE RUN ENV a.inc b.inc

        # include only RUN and ENV instructions but merge them
        INCLUDE MERGE RUN ENV a.inc b.inc

        # exclude FROM instructions
        INCLUDE -FROM a.inc b.inc

DESCRIPTION
    "dfpp" was written to allow simple pre-processing of Dockerfiles to add
    capabilities currently unsupported by docker build.

INSTRUCTIONS
  INCLUDE [MERGE] [FILTERS] [file|uri] ...
    This will inline a file or uri into the Dockerfile being generated.

   MERGE
    When including multiple Dockerfile snippets this will attempt to merge
    common instructions. Currently only ENV, LABEL and RUN are merged,
    otherwise multiple instructions will be repeated. RUN instructions are
    merged with "&&" while other instructions are merged with a space.

   FILTERS
   [-]ADD
    Include or Exclude ADD instructions from inlined Dockerfile snippets

   [-]CMD
    Include or Exclude CMD instructions from inlined Dockerfile snippets

   [-]COPY
    Include or Exclude COPY instructions from inlined Dockerfile snippets

   [-]ENTRYPOINT
    Include or Exclude ENTRYPOINT instructions from inlined Dockerfile
    snippets

   [-]ENV
    Include or Exclude ENV instructions from inlined Dockerfile snippets

   [-]EXPOSE
    Include or Exclude EXPOSE instructions from inlined Dockerfile snippets

   [-]FROM
    Include or Exclude FROM instructions from inlined Dockerfile snippets

   [-]INCLUDE
    Include or Exclude INCLUDE instructions from inlined Dockerfile snippets

   [-]LABEL
    Include or Exclude LABEL instructions from inlined Dockerfile snippets

   [-]MAINTAINER
    Include or Exclude MAINTAINER instructions from inlined Dockerfile
    snippets

   [-]ONBUILD
    Include or Exclude ONBUILD instructions from inlined Dockerfile snippets

   [-]RUN
    Include or Exclude RUN instructions from inlined Dockerfile snippets

   [-]USER
    Include or Exclude USER instructions from inlined Dockerfile snippets

   [-]VOLUME
    Include or Exclude VOLUME instructions from inlined Dockerfile snippets

   [-]WORKDIR
    Include or Exclude WORKDIR instructions from inlined Dockerfile snippets

AUTHOR
    2015, Cory Bennett <github@corybennett.org>

SOURCE
    The Source is available at github:
    https://github.com/coryb/dfpp

COPYRIGHT and LICENSE
    Copyright (c) 2015 Netflix Inc. All rights reserved. The copyrights to
    the contents of this file are licensed under the Apache License, Version 2.0
`)
}
<commit_msg>remove unused imports<commit_after>package main

import (
	"github.com/op/go-logging"
	"os"
	"io"
	"io/ioutil"
	"net/http"
	"fmt"
	"bufio"
	"runtime"
	"strings"
	"github.com/andrew-d/go-termutil"
)

var log = logging.MustGetLogger("dfpp")
var format = "%{color}%{time:2006-01-02T15:04:05.000Z07:00} %{level:-5s} [%{shortfile}]%{color:reset} %{message}"

func main() {
	if termutil.Isatty(os.Stdin.Fd()) {
		Usage()
		os.Exit(0)
    }

	logBackend := logging.NewLogBackend(os.Stderr, "", 0)
	logging.SetBackend(
		logging.NewBackendFormatter(
			logBackend,
			logging.MustStringFormatter(format),
		),
	)
	logging.SetLevel(logging.DEBUG, "")

	for line := range InstructionScanner(os.Stdin) {
		parts := strings.Fields(line)
		if len(parts) > 0 {
			instruction := parts[0]
			if instruction == "INCLUDE" {
				ProcessInclude(line, parts)
				continue;
			}
		}
		fmt.Println(line)
	}
}

func InstructionScanner(input io.Reader) chan string {
	ch := make(chan string)
	go func() {
		scanner := bufio.NewScanner(input)
		for scanner.Scan() {
			line := scanner.Text();
			for len(line)>0 && line[len(line)-1] == '\\' {
				scanner.Scan();
				line += "\n" + scanner.Text()
			}
			ch <- line
		}
		close(ch)
	}()
	return ch
}

func ProcessInclude(line string, fields []string) {
	merge := false
	exclude := make(map[string]bool)
	include := make(map[string]bool)

	uris := make([]string,0, len(fields)-1)
	for _, field := range fields {
		if _, err := os.Stat(field); err == nil {
			uris = append(uris, field)
			continue;
		}
		clude := include
		if field[0] == '-' {
			clude = exclude
			field = field[1:]
		}

		switch field {
        case "\\": continue
		case "INCLUDE": continue
		case "MERGE": merge = true
		case "ADD": fallthrough
		case "CMD": fallthrough
		case "COPY": fallthrough
		case "ENTRYPOINT": fallthrough
		case "EVN": fallthrough
		case "EXPOSE": fallthrough
		case "FROM": fallthrough
		case "LABEL": fallthrough
		case "MAINTAINER": fallthrough
		case "ONBUILD": fallthrough
		case "RUN": fallthrough
		case "USER": fallthrough
		case "VOLUME": fallthrough
		case "WORKDIR": clude[field] = true
		default: uris = append(uris, field)
		}
	}

	docs := make([]string, 0, len(uris))
	for _, uri := range uris {
		if _, err := os.Stat(uri); err == nil {
			content, err := ioutil.ReadFile(uri)
			if err != nil {
				log.Error("Failed to read %s: %s", uri, err)
				os.Exit(1)
			}
			docs = append(docs, string(content))
		} else {
			req, _ := http.NewRequest("GET", uri, nil)
			ua := &http.Client{}
			if resp, err := ua.Do(req); err != nil {
				log.Error("Failed to %s %s: %s", req.Method, req.URL.String(), err)
				os.Exit(1)
			} else {
				if resp.StatusCode < 200 || resp.StatusCode >= 300 && resp.StatusCode != 401 {
					log.Error("response status: %s", resp.Status)
				}
				runtime.SetFinalizer(resp, func(r *http.Response) {
					r.Body.Close()
				})
				if buf, err := ioutil.ReadAll(resp.Body); err == nil {
					docs = append(docs, string(buf))
				}
			}
		}
	}
	Merge(merge, docs, include, exclude)
}

func Merge(merge bool, docs []string, include, exclude map[string]bool) {
	result := make([]*string, 0)
	ops := make(map[string]*string)
	for _, doc := range docs {
		for line := range InstructionScanner( strings.NewReader(doc) ) {
			fields := strings.Fields(line)
			if len(fields) > 0 {
				op := fields[0]
				if _, ok := exclude[op]; ok {
					continue
				}
				if _, ok := include[op]; len(include) > 0 && !ok {
					continue
				}
				details := strings.TrimPrefix(line, fields[0]+" ")

				if sref, ok := ops[op]; merge && ok {
					if op == "ENV" || op == "LABEL" {
						*sref += " \\\n" + strings.Repeat(" ", len(op)+1) + details
					} else if op == "RUN" {
						*sref += " && \\\n    " + details

						// squash redundent apt-get updates
						squash := "apt-get update"
						if ix := strings.Index(*sref, squash); ix >= 0 {
							rest := strings.Replace((*sref)[ix+len(squash):], squash, "echo skipping redundent apt-get-update", -1);
							*sref = (*sref)[0:ix+len(squash)] + rest
						}

					} else {
						dup := string(line)
						result = append(result, &dup)
					}
				} else {
					dup := string(line)
					result = append(result, &dup)
					ops[op] = result[len(result)-1]
				}
			}
		}
	}
	for _, line := range result {
		fmt.Println(*line)
	}
}

func Usage() {
	fmt.Print(`
NAME
    dfpp - Dockerfile preprocessor

SYNOPSIS
        $ dfpp Dockerfile.pre > Dockerfile

        # Dockerfile Syntax:
        INCLUDE ./Dockerfile.inc
        INCLUDE http://path/to/Dockerfile.inc
        INCLUDE ./Dockerfile.inc http://path/to/Dockerfile.inc

        INCLUDE MERGE a.inc b.inc

        # include only RUN instructions
        INCLUDE RUN a.inc b.inc

        # include only RUN and ENV instructions
        INCLUDE RUN ENV a.inc b.inc

        # include only RUN and ENV instructions but merge them
        INCLUDE MERGE RUN ENV a.inc b.inc

        # exclude FROM instructions
        INCLUDE -FROM a.inc b.inc

DESCRIPTION
    "dfpp" was written to allow simple pre-processing of Dockerfiles to add
    capabilities currently unsupported by docker build.

INSTRUCTIONS
  INCLUDE [MERGE] [FILTERS] [file|uri] ...
    This will inline a file or uri into the Dockerfile being generated.

   MERGE
    When including multiple Dockerfile snippets this will attempt to merge
    common instructions. Currently only ENV, LABEL and RUN are merged,
    otherwise multiple instructions will be repeated. RUN instructions are
    merged with "&&" while other instructions are merged with a space.

   FILTERS
   [-]ADD
    Include or Exclude ADD instructions from inlined Dockerfile snippets

   [-]CMD
    Include or Exclude CMD instructions from inlined Dockerfile snippets

   [-]COPY
    Include or Exclude COPY instructions from inlined Dockerfile snippets

   [-]ENTRYPOINT
    Include or Exclude ENTRYPOINT instructions from inlined Dockerfile
    snippets

   [-]ENV
    Include or Exclude ENV instructions from inlined Dockerfile snippets

   [-]EXPOSE
    Include or Exclude EXPOSE instructions from inlined Dockerfile snippets

   [-]FROM
    Include or Exclude FROM instructions from inlined Dockerfile snippets

   [-]INCLUDE
    Include or Exclude INCLUDE instructions from inlined Dockerfile snippets

   [-]LABEL
    Include or Exclude LABEL instructions from inlined Dockerfile snippets

   [-]MAINTAINER
    Include or Exclude MAINTAINER instructions from inlined Dockerfile
    snippets

   [-]ONBUILD
    Include or Exclude ONBUILD instructions from inlined Dockerfile snippets

   [-]RUN
    Include or Exclude RUN instructions from inlined Dockerfile snippets

   [-]USER
    Include or Exclude USER instructions from inlined Dockerfile snippets

   [-]VOLUME
    Include or Exclude VOLUME instructions from inlined Dockerfile snippets

   [-]WORKDIR
    Include or Exclude WORKDIR instructions from inlined Dockerfile snippets

AUTHOR
    2015, Cory Bennett <github@corybennett.org>

SOURCE
    The Source is available at github:
    https://github.com/coryb/dfpp

COPYRIGHT and LICENSE
    Copyright (c) 2015 Netflix Inc. All rights reserved. The copyrights to
    the contents of this file are licensed under the Apache License, Version 2.0
`)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"github.com/gorilla/mux"
	"io/ioutil"
	"log"
	"net/http"
	"reflect"
	"strconv"
	"time"
	"validators"
)

type Chunk struct {
	Tool string
	Version string
	Data []byte
}

var funcs = map[string]interface{} {
	"fathom": validators.Fathom,
	"ndt": validators.Ndt,
}

var port = flag.Int("port", 4242, "the port to listen on")

func (c *Chunk) validate() error {
	fun := funcs[c.Tool]
	if fun == nil {
		return errors.New("No validator for tool")
	}
	f := reflect.ValueOf(fun)
	in := make([]reflect.Value, 2)
	in[0] = reflect.ValueOf(c.Version)
	in[1] = reflect.ValueOf(c.Data)
	valid := f.Call(in)
	if len(valid) != 1 {
		return errors.New("Unexpected return from validation function")
	}
	if !valid[0].IsNil() {
		return valid[0].Interface().(error)
	}
	return nil
}

func (c *Chunk) save() error {
	// Use tool/version to lookup data format restrictions and apply them.
	// Eg: XML, JSON, content length, etc.
	err := c.validate()
	if err != nil {
		return err
	}

  // TODO(dominic): Save to
  // /var/spool/<tool>/YYYY/MM/DD/<iso8601>_<tool>_<version>
	filename := c.Tool + "-" + c.Version + "." +
		strconv.FormatInt(time.Now().Unix(), 10)
	return ioutil.WriteFile(filename, c.Data, 0600)
}

func main() {
	flag.Parse()
	r := mux.NewRouter()
	r.HandleFunc("/", root).Methods("GET")
	r.HandleFunc("/{tool}", tool).Methods("GET")
	r.HandleFunc("/{tool}/{version}", tool_and_version)
	http.Handle("/", r)
	log.Fatal(http.ListenAndServe(":" + strconv.Itoa(*port), nil))
}

func root(w http.ResponseWriter, r *http.Request) {
	// TODO(dominic): Return information on pipeline and links to tool
	// 		  metrics.
	fmt.Fprint(w, "Info on pipeline and links to tools will be here")
}

func tool(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	tool := vars["tool"]

	// TODO(dominic): return metrics for tool
	fmt.Fprintf(w, "Metrics for tool '%s' will be here", tool)
}

func tool_and_version(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	tool := vars["tool"]
	version := vars["version"]

	if r.Method == "GET" {
		// TODO(dominic): return metrics for tool and version
		fmt.Fprintf(w, "Metrics for tool '%s-%s' will be here",
			    tool, version)
	} else if r.Method == "POST" {
		fmt.Fprintf(w, "Data POSTED for tool '%s-%s'", tool, version)
		// Read body
		buf := new(bytes.Buffer)
		buf.ReadFrom(r.Body)
		// And save to disk
		chunk := &Chunk{Tool: tool, Version: version, Data: buf.Bytes()}
		err := chunk.save()
		if err != nil {
			http.Error(w, err.Error(),
				   http.StatusInternalServerError)
			return
		}
		w.WriteHeader(http.StatusFound)
	}
}
<commit_msg>Save to rsync folder<commit_after>package main

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"github.com/gorilla/mux"
	"io/ioutil"
	"log"
	"net/http"
	"reflect"
	"strconv"
	"time"
	"validators"
)

type Chunk struct {
	Tool string
	Version string
	Data []byte
}

var funcs = map[string]interface{} {
	"fathom": validators.Fathom,
	"ndt": validators.Ndt,
}

var port = flag.Int("port", 4242, "the port to listen on")

func (c *Chunk) validate() error {
	fun := funcs[c.Tool]
	if fun == nil {
		return errors.New("No validator for tool")
	}
	f := reflect.ValueOf(fun)
	in := make([]reflect.Value, 2)
	in[0] = reflect.ValueOf(c.Version)
	in[1] = reflect.ValueOf(c.Data)
	valid := f.Call(in)
	if len(valid) != 1 {
		return errors.New("Unexpected return from validation function")
	}
	if !valid[0].IsNil() {
		return valid[0].Interface().(error)
	}
	return nil
}

func (c *Chunk) save() error {
	// Use tool/version to lookup data format restrictions and apply them.
	// Eg: XML, JSON, content length, etc.
	err := c.validate()
	if err != nil {
		return err
	}

	now := time.Now()

	// /var/spool/<tool>/YYYY/MM/DD/<iso8601>_<tool>_<version>
	filename := "/var/spool/" + c.Tool + "/" +
		strconv.FormatInt(now.Year(), 10) + "/" +
		fmt.Sprintf("%02d", now.Month()) + "/" +
		fmt.Sprintf("%02d", now.Day()) + "/" +
		strconv.FormatInt(now.Unix(), 10) + "_" + c.Tool + "_" + c.Version
	return ioutil.WriteFile(filename, c.Data, 0600)
}

func main() {
	flag.Parse()
	r := mux.NewRouter()
	r.HandleFunc("/", root).Methods("GET")
	r.HandleFunc("/{tool}", tool).Methods("GET")
	r.HandleFunc("/{tool}/{version}", tool_and_version)
	http.Handle("/", r)
	log.Fatal(http.ListenAndServe(":" + strconv.Itoa(*port), nil))
}

func root(w http.ResponseWriter, r *http.Request) {
	// TODO(dominic): Return information on pipeline and links to tool
	// 		  metrics.
	fmt.Fprint(w, "Info on pipeline and links to tools will be here")
}

func tool(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	tool := strings.ToLower(vars["tool"])

	// TODO(dominic): return metrics for tool
	fmt.Fprintf(w, "Metrics for tool '%s' will be here", tool)
}

func tool_and_version(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	tool := strings.ToLower(vars["tool"])
	version := strings.ToLower(vars["version"])

	if r.Method == "GET" {
		// TODO(dominic): return metrics for tool and version
		fmt.Fprintf(w, "Metrics for tool '%s-%s' will be here",
			    tool, version)
	} else if r.Method == "POST" {
		fmt.Fprintf(w, "Data POSTED for tool '%s-%s'", tool, version)
		// Read body
		buf := new(bytes.Buffer)
		buf.ReadFrom(r.Body)
		// And save to disk
		chunk := &Chunk{Tool: tool, Version: version, Data: buf.Bytes()}
		err := chunk.save()
		if err != nil {
			http.Error(w, err.Error(),
				   http.StatusInternalServerError)
			return
		}
		w.WriteHeader(http.StatusFound)
	}
}
<|endoftext|>