<commit_before>package logbroker

import (
	"errors"
	"fmt"
	"io"
	"sync"

	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"

	"github.com/Sirupsen/logrus"
	"github.com/docker/go-events"
	"github.com/docker/swarmkit/api"
	"github.com/docker/swarmkit/ca"
	"github.com/docker/swarmkit/identity"
	"github.com/docker/swarmkit/log"
	"github.com/docker/swarmkit/manager/state/store"
	"github.com/docker/swarmkit/watch"
	"golang.org/x/net/context"
)

var (
	errAlreadyRunning = errors.New("broker is already running")
	errNotRunning     = errors.New("broker is not running")
)

type logMessage struct {
	*api.PublishLogsMessage
	completed bool
	err       error
}

// LogBroker coordinates log subscriptions to services and tasks. Clients can
// publish and subscribe to logs channels.
//
// Log subscriptions are pushed to the work nodes by creating log subscsription
// tasks. As such, the LogBroker also acts as an orchestrator of these tasks.
type LogBroker struct {
	mu                sync.RWMutex
	logQueue          *watch.Queue
	subscriptionQueue *watch.Queue

	registeredSubscriptions map[string]*subscription
	connectedNodes          map[string]struct{}

	pctx      context.Context
	cancelAll context.CancelFunc

	store *store.MemoryStore
}

// New initializes and returns a new LogBroker
func New(store *store.MemoryStore) *LogBroker {
	return &LogBroker{
		store: store,
	}
}

// Run the log broker
func (lb *LogBroker) Run(ctx context.Context) error {
	lb.mu.Lock()

	if lb.cancelAll != nil {
		lb.mu.Unlock()
		return errAlreadyRunning
	}

	lb.pctx, lb.cancelAll = context.WithCancel(ctx)
	lb.logQueue = watch.NewQueue()
	lb.subscriptionQueue = watch.NewQueue()
	lb.registeredSubscriptions = make(map[string]*subscription)
	lb.connectedNodes = make(map[string]struct{})
	lb.mu.Unlock()

	select {
	case <-lb.pctx.Done():
		return lb.pctx.Err()
	}
}

// Stop stops the log broker
func (lb *LogBroker) Stop() error {
	lb.mu.Lock()
	defer lb.mu.Unlock()

	if lb.cancelAll == nil {
		return errNotRunning
	}
	lb.cancelAll()
	lb.cancelAll = nil

	lb.logQueue.Close()
	lb.subscriptionQueue.Close()

	return nil
}

func validateSelector(selector *api.LogSelector) error {
	if selector == nil {
		return grpc.Errorf(codes.InvalidArgument, "log selector must be provided")
	}

	if len(selector.ServiceIDs) == 0 && len(selector.TaskIDs) == 0 && len(selector.NodeIDs) == 0 {
		return grpc.Errorf(codes.InvalidArgument, "log selector must not be empty")
	}

	return nil
}

func (lb *LogBroker) newSubscription(selector *api.LogSelector, options *api.LogSubscriptionOptions) *subscription {
	lb.mu.RLock()
	defer lb.mu.RUnlock()

	subscription := newSubscription(lb.store, &api.SubscriptionMessage{
		ID:       identity.NewID(),
		Selector: selector,
		Options:  options,
	}, lb.subscriptionQueue)

	return subscription
}

func (lb *LogBroker) getSubscription(id string) *subscription {
	lb.mu.RLock()
	defer lb.mu.RUnlock()

	subscription, ok := lb.registeredSubscriptions[id]
	if !ok {
		return nil
	}
	return subscription
}

func (lb *LogBroker) registerSubscription(subscription *subscription) {
	lb.mu.Lock()
	defer lb.mu.Unlock()

	lb.registeredSubscriptions[subscription.message.ID] = subscription
	lb.subscriptionQueue.Publish(subscription)

	// Mark nodes that won't receive the message as done.
	for _, node := range subscription.Nodes() {
		if _, ok := lb.connectedNodes[node]; !ok {
			subscription.Done(node, fmt.Errorf("node %s is not available", node))
		}
	}
}

func (lb *LogBroker) unregisterSubscription(subscription *subscription) {
	lb.mu.Lock()
	defer lb.mu.Unlock()

	delete(lb.registeredSubscriptions, subscription.message.ID)

	subscription.Close()
	lb.subscriptionQueue.Publish(subscription)
}

// watchSubscriptions grabs all current subscriptions and notifies of any
// subscription change for this node.
//
// Subscriptions may fire multiple times and the caller has to protect against
// dupes.
func (lb *LogBroker) watchSubscriptions(nodeID string) ([]*subscription, chan events.Event, func()) {
	lb.mu.RLock()
	defer lb.mu.RUnlock()

	// Watch for subscription changes for this node.
	ch, cancel := lb.subscriptionQueue.CallbackWatch(events.MatcherFunc(func(event events.Event) bool {
		s := event.(*subscription)
		return s.Contains(nodeID)
	}))

	// Grab current subscriptions.
	var subscriptions []*subscription
	for _, s := range lb.registeredSubscriptions {
		if s.Contains(nodeID) {
			subscriptions = append(subscriptions, s)
		}
	}

	return subscriptions, ch, cancel
}

func (lb *LogBroker) subscribe(id string) (chan events.Event, func()) {
	lb.mu.RLock()
	defer lb.mu.RUnlock()

	return lb.logQueue.CallbackWatch(events.MatcherFunc(func(event events.Event) bool {
		publish := event.(*logMessage)
		return publish.SubscriptionID == id
	}))
}

func (lb *LogBroker) publish(log *api.PublishLogsMessage) {
	lb.mu.RLock()
	defer lb.mu.RUnlock()

	lb.logQueue.Publish(&logMessage{PublishLogsMessage: log})
}

// SubscribeLogs creates a log subscription and streams back logs
func (lb *LogBroker) SubscribeLogs(request *api.SubscribeLogsRequest, stream api.Logs_SubscribeLogsServer) error {
	ctx := stream.Context()

	if err := validateSelector(request.Selector); err != nil {
		return err
	}

	subscription := lb.newSubscription(request.Selector, request.Options)
	subscription.Run(lb.pctx)
	defer subscription.Stop()

	log := log.G(ctx).WithFields(
		logrus.Fields{
			"method":          "(*LogBroker).SubscribeLogs",
			"subscription.id": subscription.message.ID,
		},
	)
	log.Debug("subscribed")

	publishCh, publishCancel := lb.subscribe(subscription.message.ID)
	defer publishCancel()

	lb.registerSubscription(subscription)
	defer lb.unregisterSubscription(subscription)

	completed := subscription.Wait(ctx)
	for {
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-lb.pctx.Done():
			return lb.pctx.Err()
		case event := <-publishCh:
			publish := event.(*logMessage)
			if publish.completed {
				return publish.err
			}
			if err := stream.Send(&api.SubscribeLogsMessage{
				Messages: publish.Messages,
			}); err != nil {
				return err
			}
		case <-completed:
			completed = nil
			lb.logQueue.Publish(&logMessage{
				PublishLogsMessage: &api.PublishLogsMessage{
					SubscriptionID: subscription.message.ID,
				},
				completed: true,
				err:       subscription.Err(),
			})
		}
	}
}

func (lb *LogBroker) nodeConnected(nodeID string) {
	lb.mu.Lock()
	defer lb.mu.Unlock()

	lb.connectedNodes[nodeID] = struct{}{}
}

func (lb *LogBroker) nodeDisconnected(nodeID string) {
	lb.mu.Lock()
	defer lb.mu.Unlock()

	delete(lb.connectedNodes, nodeID)
}

// ListenSubscriptions returns a stream of matching subscriptions for the current node
func (lb *LogBroker) ListenSubscriptions(request *api.ListenSubscriptionsRequest, stream api.LogBroker_ListenSubscriptionsServer) error {
	remote, err := ca.RemoteNode(stream.Context())
	if err != nil {
		return err
	}

	lb.nodeConnected(remote.NodeID)
	defer lb.nodeDisconnected(remote.NodeID)

	log := log.G(stream.Context()).WithFields(
		logrus.Fields{
			"method": "(*LogBroker).ListenSubscriptions",
			"node":   remote.NodeID,
		},
	)
	subscriptions, subscriptionCh, subscriptionCancel := lb.watchSubscriptions(remote.NodeID)
	defer subscriptionCancel()

	log.Debug("node registered")

	activeSubscriptions := make(map[string]*subscription)
	defer func() {
		// If the worker quits, mark all active subscriptions as finished.
		for _, subscription := range activeSubscriptions {
			subscription.Done(remote.NodeID, fmt.Errorf("node %s disconnected unexpectedly", remote.NodeID))
		}
	}()

	// Start by sending down all active subscriptions.
	for _, subscription := range subscriptions {
		select {
		case <-stream.Context().Done():
			return stream.Context().Err()
		case <-lb.pctx.Done():
			return nil
		default:
		}

		if err := stream.Send(subscription.message); err != nil {
			log.Error(err)
			return err
		}
		activeSubscriptions[subscription.message.ID] = subscription
	}

	// Send down new subscriptions.
	for {
		select {
		case v := <-subscriptionCh:
			subscription := v.(*subscription)

			if subscription.Closed() {
				log.WithField("subscription.id", subscription.message.ID).Debug("subscription closed")
				delete(activeSubscriptions, subscription.message.ID)
			} else {
				// Avoid sending down the same subscription multiple times
				if _, ok := activeSubscriptions[subscription.message.ID]; ok {
					continue
				}
				activeSubscriptions[subscription.message.ID] = subscription
				log.WithField("subscription.id", subscription.message.ID).Debug("subscription added")
			}
			if err := stream.Send(subscription.message); err != nil {
				log.Error(err)
				return err
			}
		case <-stream.Context().Done():
			return stream.Context().Err()
		case <-lb.pctx.Done():
			return nil
		}
	}
}

// PublishLogs publishes log messages for a given subscription
func (lb *LogBroker) PublishLogs(stream api.LogBroker_PublishLogsServer) (err error) {
	remote, err := ca.RemoteNode(stream.Context())
	if err != nil {
		return err
	}

	var currentSubscription *subscription
	defer func() {
		if currentSubscription != nil {
			currentSubscription.Done(remote.NodeID, err)
		}
	}()

	for {
		logMsg, err := stream.Recv()
		if err == io.EOF {
			return stream.SendAndClose(&api.PublishLogsResponse{})
		}
		if err != nil {
			return err
		}

		if logMsg.SubscriptionID == "" {
			return grpc.Errorf(codes.InvalidArgument, "missing subscription ID")
		}

		if currentSubscription == nil {
			currentSubscription = lb.getSubscription(logMsg.SubscriptionID)
			if currentSubscription == nil {
				return grpc.Errorf(codes.NotFound, "unknown subscription ID")
			}
		} else {
			if logMsg.SubscriptionID != currentSubscription.message.ID {
				return grpc.Errorf(codes.InvalidArgument, "different subscription IDs in the same session")
			}
		}

		// if we have a close message, close out the subscription
		if logMsg.Close {
			// Mark done and then set to nil so if we error after this point,
			// we don't try to close again in the defer
			currentSubscription.Done(remote.NodeID, err)
			currentSubscription = nil
			return nil
		}

		// Make sure logs are emitted using the right Node ID to avoid impersonation.
		for _, msg := range logMsg.Messages {
			if msg.Context.NodeID != remote.NodeID {
				return grpc.Errorf(codes.PermissionDenied, "invalid NodeID: expected=%s;received=%s", remote.NodeID, msg.Context.NodeID)
			}
		}

		lb.publish(logMsg)
	}
}
<commit_msg>Fix possible race in ListenSubscriptions<commit_after>package logbroker

import (
	"errors"
	"fmt"
	"io"
	"sync"

	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"

	"github.com/Sirupsen/logrus"
	"github.com/docker/go-events"
	"github.com/docker/swarmkit/api"
	"github.com/docker/swarmkit/ca"
	"github.com/docker/swarmkit/identity"
	"github.com/docker/swarmkit/log"
	"github.com/docker/swarmkit/manager/state/store"
	"github.com/docker/swarmkit/watch"
	"golang.org/x/net/context"
)

var (
	errAlreadyRunning = errors.New("broker is already running")
	errNotRunning     = errors.New("broker is not running")
)

type logMessage struct {
	*api.PublishLogsMessage
	completed bool
	err       error
}

// LogBroker coordinates log subscriptions to services and tasks. Clients can
// publish and subscribe to logs channels.
//
// Log subscriptions are pushed to the work nodes by creating log subscsription
// tasks. As such, the LogBroker also acts as an orchestrator of these tasks.
type LogBroker struct {
	mu                sync.RWMutex
	logQueue          *watch.Queue
	subscriptionQueue *watch.Queue

	registeredSubscriptions map[string]*subscription
	subscriptionsByNode     map[string]map[*subscription]struct{}

	pctx      context.Context
	cancelAll context.CancelFunc

	store *store.MemoryStore
}

// New initializes and returns a new LogBroker
func New(store *store.MemoryStore) *LogBroker {
	return &LogBroker{
		store: store,
	}
}

// Run the log broker
func (lb *LogBroker) Run(ctx context.Context) error {
	lb.mu.Lock()

	if lb.cancelAll != nil {
		lb.mu.Unlock()
		return errAlreadyRunning
	}

	lb.pctx, lb.cancelAll = context.WithCancel(ctx)
	lb.logQueue = watch.NewQueue()
	lb.subscriptionQueue = watch.NewQueue()
	lb.registeredSubscriptions = make(map[string]*subscription)
	lb.subscriptionsByNode = make(map[string]map[*subscription]struct{})
	lb.mu.Unlock()

	select {
	case <-lb.pctx.Done():
		return lb.pctx.Err()
	}
}

// Stop stops the log broker
func (lb *LogBroker) Stop() error {
	lb.mu.Lock()
	defer lb.mu.Unlock()

	if lb.cancelAll == nil {
		return errNotRunning
	}
	lb.cancelAll()
	lb.cancelAll = nil

	lb.logQueue.Close()
	lb.subscriptionQueue.Close()

	return nil
}

func validateSelector(selector *api.LogSelector) error {
	if selector == nil {
		return grpc.Errorf(codes.InvalidArgument, "log selector must be provided")
	}

	if len(selector.ServiceIDs) == 0 && len(selector.TaskIDs) == 0 && len(selector.NodeIDs) == 0 {
		return grpc.Errorf(codes.InvalidArgument, "log selector must not be empty")
	}

	return nil
}

func (lb *LogBroker) newSubscription(selector *api.LogSelector, options *api.LogSubscriptionOptions) *subscription {
	lb.mu.RLock()
	defer lb.mu.RUnlock()

	subscription := newSubscription(lb.store, &api.SubscriptionMessage{
		ID:       identity.NewID(),
		Selector: selector,
		Options:  options,
	}, lb.subscriptionQueue)

	return subscription
}

func (lb *LogBroker) getSubscription(id string) *subscription {
	lb.mu.RLock()
	defer lb.mu.RUnlock()

	subscription, ok := lb.registeredSubscriptions[id]
	if !ok {
		return nil
	}
	return subscription
}

func (lb *LogBroker) registerSubscription(subscription *subscription) {
	lb.mu.Lock()
	defer lb.mu.Unlock()

	lb.registeredSubscriptions[subscription.message.ID] = subscription
	lb.subscriptionQueue.Publish(subscription)

	for _, node := range subscription.Nodes() {
		if _, ok := lb.subscriptionsByNode[node]; !ok {
			// Mark nodes that won't receive the message as done.
			subscription.Done(node, fmt.Errorf("node %s is not available", node))
		} else {
			// otherwise, add the subscription to the node's subscriptions list
			lb.subscriptionsByNode[node][subscription] = struct{}{}
		}
	}
}

func (lb *LogBroker) unregisterSubscription(subscription *subscription) {
	lb.mu.Lock()
	defer lb.mu.Unlock()

	delete(lb.registeredSubscriptions, subscription.message.ID)

	// remove the subscription from all of the nodes
	for _, node := range subscription.Nodes() {
		// but only if a node exists
		if _, ok := lb.subscriptionsByNode[node]; ok {
			delete(lb.subscriptionsByNode[node], subscription)
		}
	}

	subscription.Close()
	lb.subscriptionQueue.Publish(subscription)
}

// watchSubscriptions grabs all current subscriptions and notifies of any
// subscription change for this node.
//
// Subscriptions may fire multiple times and the caller has to protect against
// dupes.
func (lb *LogBroker) watchSubscriptions(nodeID string) ([]*subscription, chan events.Event, func()) {
	lb.mu.RLock()
	defer lb.mu.RUnlock()

	// Watch for subscription changes for this node.
	ch, cancel := lb.subscriptionQueue.CallbackWatch(events.MatcherFunc(func(event events.Event) bool {
		s := event.(*subscription)
		return s.Contains(nodeID)
	}))

	// Grab current subscriptions.
	var subscriptions []*subscription
	for _, s := range lb.registeredSubscriptions {
		if s.Contains(nodeID) {
			subscriptions = append(subscriptions, s)
		}
	}

	return subscriptions, ch, cancel
}

func (lb *LogBroker) subscribe(id string) (chan events.Event, func()) {
	lb.mu.RLock()
	defer lb.mu.RUnlock()

	return lb.logQueue.CallbackWatch(events.MatcherFunc(func(event events.Event) bool {
		publish := event.(*logMessage)
		return publish.SubscriptionID == id
	}))
}

func (lb *LogBroker) publish(log *api.PublishLogsMessage) {
	lb.mu.RLock()
	defer lb.mu.RUnlock()

	lb.logQueue.Publish(&logMessage{PublishLogsMessage: log})
}

// markDone wraps (*Subscription).Done() so that the removal of the sub from
// the node's subscription list is possible
func (lb *LogBroker) markDone(sub *subscription, nodeID string, err error) {
	lb.mu.Lock()
	defer lb.mu.Unlock()

	// remove the subscription from the node's subscription list, if it exists
	if _, ok := lb.subscriptionsByNode[nodeID]; ok {
		delete(lb.subscriptionsByNode[nodeID], sub)
	}

	// mark the sub as done
	sub.Done(nodeID, err)
}

// SubscribeLogs creates a log subscription and streams back logs
func (lb *LogBroker) SubscribeLogs(request *api.SubscribeLogsRequest, stream api.Logs_SubscribeLogsServer) error {
	ctx := stream.Context()

	if err := validateSelector(request.Selector); err != nil {
		return err
	}

	subscription := lb.newSubscription(request.Selector, request.Options)
	subscription.Run(lb.pctx)
	defer subscription.Stop()

	log := log.G(ctx).WithFields(
		logrus.Fields{
			"method":          "(*LogBroker).SubscribeLogs",
			"subscription.id": subscription.message.ID,
		},
	)
	log.Debug("subscribed")

	publishCh, publishCancel := lb.subscribe(subscription.message.ID)
	defer publishCancel()

	lb.registerSubscription(subscription)
	defer lb.unregisterSubscription(subscription)

	completed := subscription.Wait(ctx)
	for {
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-lb.pctx.Done():
			return lb.pctx.Err()
		case event := <-publishCh:
			publish := event.(*logMessage)
			if publish.completed {
				return publish.err
			}
			if err := stream.Send(&api.SubscribeLogsMessage{
				Messages: publish.Messages,
			}); err != nil {
				return err
			}
		case <-completed:
			completed = nil
			lb.logQueue.Publish(&logMessage{
				PublishLogsMessage: &api.PublishLogsMessage{
					SubscriptionID: subscription.message.ID,
				},
				completed: true,
				err:       subscription.Err(),
			})
		}
	}
}

func (lb *LogBroker) nodeConnected(nodeID string) {
	lb.mu.Lock()
	defer lb.mu.Unlock()

	if _, ok := lb.subscriptionsByNode[nodeID]; !ok {
		lb.subscriptionsByNode[nodeID] = make(map[*subscription]struct{})
	}
}

func (lb *LogBroker) nodeDisconnected(nodeID string) {
	lb.mu.Lock()
	defer lb.mu.Unlock()

	for sub := range lb.subscriptionsByNode[nodeID] {
		sub.Done(nodeID, fmt.Errorf("node %s disconnected unexpectedly", nodeID))
	}
	delete(lb.subscriptionsByNode, nodeID)
}

// ListenSubscriptions returns a stream of matching subscriptions for the current node
func (lb *LogBroker) ListenSubscriptions(request *api.ListenSubscriptionsRequest, stream api.LogBroker_ListenSubscriptionsServer) error {
	remote, err := ca.RemoteNode(stream.Context())
	if err != nil {
		return err
	}

	lb.nodeConnected(remote.NodeID)
	defer lb.nodeDisconnected(remote.NodeID)

	log := log.G(stream.Context()).WithFields(
		logrus.Fields{
			"method": "(*LogBroker).ListenSubscriptions",
			"node":   remote.NodeID,
		},
	)
	subscriptions, subscriptionCh, subscriptionCancel := lb.watchSubscriptions(remote.NodeID)
	defer subscriptionCancel()

	log.Debug("node registered")

	activeSubscriptions := make(map[string]*subscription)

	// Start by sending down all active subscriptions.
	for _, subscription := range subscriptions {
		select {
		case <-stream.Context().Done():
			return stream.Context().Err()
		case <-lb.pctx.Done():
			return nil
		default:
		}

		if err := stream.Send(subscription.message); err != nil {
			log.Error(err)
			return err
		}
		activeSubscriptions[subscription.message.ID] = subscription
	}

	// Send down new subscriptions.
	for {
		select {
		case v := <-subscriptionCh:
			subscription := v.(*subscription)

			if subscription.Closed() {
				delete(activeSubscriptions, subscription.message.ID)
			} else {
				// Avoid sending down the same subscription multiple times
				if _, ok := activeSubscriptions[subscription.message.ID]; ok {
					continue
				}
				activeSubscriptions[subscription.message.ID] = subscription
			}
			if err := stream.Send(subscription.message); err != nil {
				log.Error(err)
				return err
			}
		case <-stream.Context().Done():
			return stream.Context().Err()
		case <-lb.pctx.Done():
			return nil
		}
	}
}

// PublishLogs publishes log messages for a given subscription
func (lb *LogBroker) PublishLogs(stream api.LogBroker_PublishLogsServer) (err error) {
	remote, err := ca.RemoteNode(stream.Context())
	if err != nil {
		return err
	}

	var currentSubscription *subscription
	defer func() {
		if currentSubscription != nil {
			lb.markDone(currentSubscription, remote.NodeID, err)
		}
	}()

	for {
		logMsg, err := stream.Recv()
		if err == io.EOF {
			return stream.SendAndClose(&api.PublishLogsResponse{})
		}
		if err != nil {
			return err
		}

		if logMsg.SubscriptionID == "" {
			return grpc.Errorf(codes.InvalidArgument, "missing subscription ID")
		}

		if currentSubscription == nil {
			currentSubscription = lb.getSubscription(logMsg.SubscriptionID)
			if currentSubscription == nil {
				return grpc.Errorf(codes.NotFound, "unknown subscription ID")
			}
		} else {
			if logMsg.SubscriptionID != currentSubscription.message.ID {
				return grpc.Errorf(codes.InvalidArgument, "different subscription IDs in the same session")
			}
		}

		// if we have a close message, close out the subscription
		if logMsg.Close {
			// Mark done and then set to nil so if we error after this point,
			// we don't try to close again in the defer
			lb.markDone(currentSubscription, remote.NodeID, err)
			currentSubscription = nil
			return nil
		}

		// Make sure logs are emitted using the right Node ID to avoid impersonation.
		for _, msg := range logMsg.Messages {
			if msg.Context.NodeID != remote.NodeID {
				return grpc.Errorf(codes.PermissionDenied, "invalid NodeID: expected=%s;received=%s", remote.NodeID, msg.Context.NodeID)
			}
		}

		lb.publish(logMsg)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"math"

	"github.com/peterstace/grayt"
)

func main() {
	r := grayt.NewRunner()
	r.PxWide = 800
	r.PxHigh = 800
	r.Run(scene())
}

// Cornell Box
//
// (0,y,-1) +---------+ (1,y,-1)
//          | BB      |
//          | BB  BB  |
//          |     BB  |
// (0,y,0)  +         + (1,y,0)
//            \     /
//             \   /
//              \ /
//               C (0.5,0.5,D)
//
//
// SINE(0.5*T) = 0.5 / SQRT(0.5^2 + D^2)
// T = 2*ARCSINE(0.5/SQRT(0.25 + D^2))

var (
	up    = grayt.Vect(0.0, 1.0, 0.0)
	down  = grayt.Vect(0.0, -1.0, 0.0)
	left  = grayt.Vect(-1.0, 0.0, 0.0)
	right = grayt.Vect(1.0, 0.0, 0.0)
	back  = grayt.Vect(0.0, 0.0, 1.0)
	zero  = grayt.Vect(0.0, 0.0, 0.0)
	one   = grayt.Vect(1.0, 1.0, -1.0)
)

var (
	white = grayt.Material{Colour: grayt.White, Emittance: 0.0}
	green = grayt.Material{Colour: grayt.Green, Emittance: 0.0}
	red   = grayt.Material{Colour: grayt.Red, Emittance: 0.0}
	blue  = grayt.Material{Colour: grayt.Blue, Emittance: 0.0}
)

func scene() grayt.Scene {
	ee := []grayt.Entity{{
		grayt.Material{Colour: grayt.White, Emittance: 5},
		[]grayt.SurfaceFactory{grayt.Sphere{C: grayt.Vect(0.5, 1.0, -0.5), R: 0.25}},
	}}
	ee = append(ee, box()...)
	ee = append(ee, grayt.Entity{Material: white, SurfaceFactories: tallBlock()})
	ee = append(ee, grayt.Entity{Material: white, SurfaceFactories: shortBlock()})
	return grayt.Scene{Camera: cam(), Entities: ee}
}

func cam() grayt.Camera {
	const D = 1.3 // Estimated.
	return grayt.NewRectilinearCamera(grayt.CameraConfig{
		Location:      grayt.Vect(0.5, 0.5, D),
		ViewDirection: grayt.Vect(0.0, 0.0, -1.0),
		UpDirection:   up,
		FieldOfView:   2 * math.Asin(0.5/math.Sqrt(0.25+D*D)),
		FocalLength:   0.5 + D,
		FocalRatio:    math.MaxFloat64, //math.Inf(1),
	})
}

func box() []grayt.Entity {
	return []grayt.Entity{
		{Material: white, SurfaceFactories: []grayt.SurfaceFactory{grayt.Plane{N: up, X: zero}}},
		{Material: white, SurfaceFactories: []grayt.SurfaceFactory{grayt.Plane{N: down, X: one}}},
		{Material: red, SurfaceFactories: []grayt.SurfaceFactory{grayt.Plane{N: right, X: zero}}},
		{Material: green, SurfaceFactories: []grayt.SurfaceFactory{grayt.Plane{N: left, X: one}}},
		{Material: white, SurfaceFactories: []grayt.SurfaceFactory{grayt.Plane{N: back, X: one}}},
		{Material: white, SurfaceFactories: []grayt.SurfaceFactory{grayt.AlignedBox{Corner1: grayt.Vect(0.1, 0.0, -0.1), Corner2: grayt.Vect(0.9, 0.1, -0.9)}}},
	}
}

func shortBlock() []grayt.SurfaceFactory {
	var (
		// Left/Right, Top/Bottom, Front/Back.
		LBF = grayt.Vect(0.76, 0.00, -0.12)
		LBB = grayt.Vect(0.85, 0.00, -0.41)
		RBF = grayt.Vect(0.47, 0.00, -0.21)
		RBB = grayt.Vect(0.56, 0.00, -0.49)
		LTF = grayt.Vect(0.76, 0.30, -0.12)
		LTB = grayt.Vect(0.85, 0.30, -0.41)
		RTF = grayt.Vect(0.47, 0.30, -0.21)
		RTB = grayt.Vect(0.56, 0.30, -0.49)
	)
	return []grayt.SurfaceFactory{
		grayt.Square{V1: LTF, V2: LTB, V3: RTB, V4: RTF},
		grayt.Square{V1: LBF, V2: RBF, V3: RTF, V4: LTF},
		grayt.Square{V1: LBB, V2: RBB, V3: RTB, V4: LTB},
		grayt.Square{V1: LBF, V2: LBB, V3: LTB, V4: LTF},
		grayt.Square{V1: RBF, V2: RBB, V3: RTB, V4: RTF},
	}
}

func tallBlock() []grayt.SurfaceFactory {
	var (
		// Left/Right, Top/Bottom, Front/Back.
		LBF = grayt.Vect(0.52, 0.00, -0.54)
		LBB = grayt.Vect(0.43, 0.00, -0.83)
		RBF = grayt.Vect(0.23, 0.00, -0.45)
		RBB = grayt.Vect(0.14, 0.00, -0.74)
		LTF = grayt.Vect(0.52, 0.60, -0.54)
		LTB = grayt.Vect(0.43, 0.60, -0.83)
		RTF = grayt.Vect(0.23, 0.60, -0.45)
		RTB = grayt.Vect(0.14, 0.60, -0.74)
	)
	return []grayt.SurfaceFactory{
		grayt.Square{V1: LTF, V2: LTB, V3: RTB, V4: RTF},
		grayt.Square{V1: LBF, V2: RBF, V3: RTF, V4: LTF},
		grayt.Square{V1: LBB, V2: RBB, V3: RTB, V4: LTB},
		grayt.Square{V1: LBF, V2: LBB, V3: LTB, V4: LTF},
		grayt.Square{V1: RBF, V2: RBB, V3: RTB, V4: RTF},
	}
}
<commit_msg>Use infinite focal ratio<commit_after>package main

import (
	"math"

	"github.com/peterstace/grayt"
)

func main() {
	r := grayt.NewRunner()
	r.PxWide = 800
	r.PxHigh = 800
	r.Run(scene())
}

// Cornell Box
//
// (0,y,-1) +---------+ (1,y,-1)
//          | BB      |
//          | BB  BB  |
//          |     BB  |
// (0,y,0)  +         + (1,y,0)
//            \     /
//             \   /
//              \ /
//               C (0.5,0.5,D)
//
//
// SINE(0.5*T) = 0.5 / SQRT(0.5^2 + D^2)
// T = 2*ARCSINE(0.5/SQRT(0.25 + D^2))

var (
	up    = grayt.Vect(0.0, 1.0, 0.0)
	down  = grayt.Vect(0.0, -1.0, 0.0)
	left  = grayt.Vect(-1.0, 0.0, 0.0)
	right = grayt.Vect(1.0, 0.0, 0.0)
	back  = grayt.Vect(0.0, 0.0, 1.0)
	zero  = grayt.Vect(0.0, 0.0, 0.0)
	one   = grayt.Vect(1.0, 1.0, -1.0)
)

var (
	white = grayt.Material{Colour: grayt.White, Emittance: 0.0}
	green = grayt.Material{Colour: grayt.Green, Emittance: 0.0}
	red   = grayt.Material{Colour: grayt.Red, Emittance: 0.0}
	blue  = grayt.Material{Colour: grayt.Blue, Emittance: 0.0}
)

func scene() grayt.Scene {
	ee := []grayt.Entity{{
		grayt.Material{Colour: grayt.White, Emittance: 5},
		[]grayt.SurfaceFactory{grayt.Sphere{C: grayt.Vect(0.5, 1.0, -0.5), R: 0.25}},
	}}
	ee = append(ee, box()...)
	ee = append(ee, grayt.Entity{Material: white, SurfaceFactories: tallBlock()})
	ee = append(ee, grayt.Entity{Material: white, SurfaceFactories: shortBlock()})
	return grayt.Scene{Camera: cam(), Entities: ee}
}

func cam() grayt.Camera {
	const D = 1.3 // Estimated.
	return grayt.NewRectilinearCamera(grayt.CameraConfig{
		Location:      grayt.Vect(0.5, 0.5, D),
		ViewDirection: grayt.Vect(0.0, 0.0, -1.0),
		UpDirection:   up,
		FieldOfView:   2 * math.Asin(0.5/math.Sqrt(0.25+D*D)),
		FocalLength:   0.5 + D,
		FocalRatio:    math.Inf(+1),
	})
}

func box() []grayt.Entity {
	return []grayt.Entity{
		{Material: white, SurfaceFactories: []grayt.SurfaceFactory{grayt.Plane{N: up, X: zero}}},
		{Material: white, SurfaceFactories: []grayt.SurfaceFactory{grayt.Plane{N: down, X: one}}},
		{Material: red, SurfaceFactories: []grayt.SurfaceFactory{grayt.Plane{N: right, X: zero}}},
		{Material: green, SurfaceFactories: []grayt.SurfaceFactory{grayt.Plane{N: left, X: one}}},
		{Material: white, SurfaceFactories: []grayt.SurfaceFactory{grayt.Plane{N: back, X: one}}},
		{Material: white, SurfaceFactories: []grayt.SurfaceFactory{grayt.AlignedBox{Corner1: grayt.Vect(0.1, 0.0, -0.1), Corner2: grayt.Vect(0.9, 0.1, -0.9)}}},
	}
}

func shortBlock() []grayt.SurfaceFactory {
	var (
		// Left/Right, Top/Bottom, Front/Back.
		LBF = grayt.Vect(0.76, 0.00, -0.12)
		LBB = grayt.Vect(0.85, 0.00, -0.41)
		RBF = grayt.Vect(0.47, 0.00, -0.21)
		RBB = grayt.Vect(0.56, 0.00, -0.49)
		LTF = grayt.Vect(0.76, 0.30, -0.12)
		LTB = grayt.Vect(0.85, 0.30, -0.41)
		RTF = grayt.Vect(0.47, 0.30, -0.21)
		RTB = grayt.Vect(0.56, 0.30, -0.49)
	)
	return []grayt.SurfaceFactory{
		grayt.Square{V1: LTF, V2: LTB, V3: RTB, V4: RTF},
		grayt.Square{V1: LBF, V2: RBF, V3: RTF, V4: LTF},
		grayt.Square{V1: LBB, V2: RBB, V3: RTB, V4: LTB},
		grayt.Square{V1: LBF, V2: LBB, V3: LTB, V4: LTF},
		grayt.Square{V1: RBF, V2: RBB, V3: RTB, V4: RTF},
	}
}

func tallBlock() []grayt.SurfaceFactory {
	var (
		// Left/Right, Top/Bottom, Front/Back.
		LBF = grayt.Vect(0.52, 0.00, -0.54)
		LBB = grayt.Vect(0.43, 0.00, -0.83)
		RBF = grayt.Vect(0.23, 0.00, -0.45)
		RBB = grayt.Vect(0.14, 0.00, -0.74)
		LTF = grayt.Vect(0.52, 0.60, -0.54)
		LTB = grayt.Vect(0.43, 0.60, -0.83)
		RTF = grayt.Vect(0.23, 0.60, -0.45)
		RTB = grayt.Vect(0.14, 0.60, -0.74)
	)
	return []grayt.SurfaceFactory{
		grayt.Square{V1: LTF, V2: LTB, V3: RTB, V4: RTF},
		grayt.Square{V1: LBF, V2: RBF, V3: RTF, V4: LTF},
		grayt.Square{V1: LBB, V2: RBB, V3: RTB, V4: LTB},
		grayt.Square{V1: LBF, V2: LBB, V3: LTB, V4: LTF},
		grayt.Square{V1: RBF, V2: RBB, V3: RTB, V4: RTF},
	}
}
<|endoftext|>


<commit_before>//go:build go1.16
// +build go1.16

package embed

import (
	"errors"
	"io/fs"
	"path/filepath"
	"strings"

	"github.com/webx-top/echo"
)

func NewFileSystems() FileSystems {
	return FileSystems{}
}

type FileSystems []fs.FS

func (f FileSystems) Open(name string) (file fs.File, err error) {
	for _, fileSystem := range f {
		file, err = fileSystem.Open(name)
		if err == nil || !errors.Is(err, fs.ErrNotExist) {
			return
		}
	}
	return
}

func (f FileSystems) Size() int {
	return len(f)
}

func (f FileSystems) IsEmpty() bool {
	return f.Size() == 0
}

func (f *FileSystems) Register(fileSystem fs.FS) {
	*f = append(*f, fileSystem)
}

// File
// e.Get(`/*`, File(customFS))
func File(fs FileSystems, configs ...Config) func(c echo.Context) error {
	config := DefaultConfig
	if len(configs) > 0 {
		config = configs[0]
		if len(config.Index) == 0 {
			config.Index = DefaultConfig.Index
		}
		if len(config.Prefix) > 0 {
			config.Prefix = strings.TrimPrefix(config.Prefix, `/`)
		}
		if len(config.Prefix) > 0 {
			if !strings.HasSuffix(config.Prefix, `/`) {
				config.Prefix += `/`
			}
		}
		if config.FilePath == nil {
			config.FilePath = DefaultConfig.FilePath
		}
	}
	return func(c echo.Context) error {
		file, err := config.FilePath(c)
		if err != nil {
			return err
		}
		if len(file) == 0 {
			file = config.Index
		}
		if len(config.Prefix) > 0 {
			file = config.Prefix + file
		}
		f, err := fs.Open(file)
		if err != nil {
			return echo.ErrNotFound
		}
		defer func() {
			if f != nil {
				f.Close()
			}
		}()
		fi, err := f.Stat()
		if err != nil {
			return err
		}
		if fi.IsDir() {
			f.Close()

			file = filepath.Join(file, config.Index)
			if f, err = fs.Open(file); err != nil {
				return echo.ErrNotFound
			}

			if fi, err = f.Stat(); err != nil {
				return err
			}
		}
		return c.ServeContent(f, fi.Name(), fi.ModTime())
	}
}
<commit_msg>update<commit_after>//go:build go1.16
// +build go1.16

package embed

import (
	"errors"
	"io/fs"
	"path/filepath"
	"strings"

	"github.com/webx-top/echo"
)

func NewFileSystems() FileSystems {
	return FileSystems{}
}

type FileSystems []fs.FS

func (f FileSystems) Open(name string) (file fs.File, err error) {
	for _, fileSystem := range f {
		file, err = fileSystem.Open(name)
		if err == nil || !errors.Is(err, fs.ErrNotExist) {
			return
		}
	}
	return
}

func (f FileSystems) ReadFile(name string) (content []byte, err error) {
	for _, fileSystem := range f {
		content, err = fs.ReadFile(fileSystem, name)
		if err == nil || !errors.Is(err, fs.ErrNotExist) {
			return
		}
	}
	return
}

func (f FileSystems) ReadDir(name string) (dirs []fs.DirEntry, err error) {
	for _, fileSystem := range f {
		dirs, err = fs.ReadDir(fileSystem, name)
		if err == nil || !errors.Is(err, fs.ErrNotExist) {
			return
		}
	}
	return
}

func (f FileSystems) Sub(name string) (sub fs.FS, err error) {
	for _, fileSystem := range f {
		sub, err = fs.Sub(fileSystem, name)
		if err == nil || !errors.Is(err, fs.ErrNotExist) {
			return
		}
	}
	return
}

func (f FileSystems) WalkDir(name string, fn fs.WalkDirFunc) (err error) {
	for _, fileSystem := range f {
		err = fs.WalkDir(fileSystem, name, fn)
		if err == nil || !errors.Is(err, fs.ErrNotExist) {
			return
		}
	}
	return
}

func (f FileSystems) Stat(name string) (fi fs.FileInfo, err error) {
	for _, fileSystem := range f {
		fi, err = fs.Stat(fileSystem, name)
		if err == nil || !errors.Is(err, fs.ErrNotExist) {
			return
		}
	}
	return
}

func (f FileSystems) Glob(pattern string) (matches []string, err error) {
	for _, fileSystem := range f {
		matches, err = fs.Glob(fileSystem, pattern)
		if err == nil || !errors.Is(err, fs.ErrNotExist) {
			return
		}
	}
	return
}

func (f FileSystems) Size() int {
	return len(f)
}

func (f FileSystems) IsEmpty() bool {
	return f.Size() == 0
}

func (f *FileSystems) Register(fileSystem fs.FS) {
	*f = append(*f, fileSystem)
}

// File
// e.Get(`/*`, File(customFS))
func File(fs FileSystems, configs ...Config) func(c echo.Context) error {
	config := DefaultConfig
	if len(configs) > 0 {
		config = configs[0]
		if len(config.Index) == 0 {
			config.Index = DefaultConfig.Index
		}
		if len(config.Prefix) > 0 {
			config.Prefix = strings.TrimPrefix(config.Prefix, `/`)
		}
		if len(config.Prefix) > 0 {
			if !strings.HasSuffix(config.Prefix, `/`) {
				config.Prefix += `/`
			}
		}
		if config.FilePath == nil {
			config.FilePath = DefaultConfig.FilePath
		}
	}
	return func(c echo.Context) error {
		file, err := config.FilePath(c)
		if err != nil {
			return err
		}
		if len(file) == 0 {
			file = config.Index
		}
		if len(config.Prefix) > 0 {
			file = config.Prefix + file
		}
		f, err := fs.Open(file)
		if err != nil {
			return echo.ErrNotFound
		}
		defer func() {
			if f != nil {
				f.Close()
			}
		}()
		fi, err := f.Stat()
		if err != nil {
			return err
		}
		if fi.IsDir() {
			f.Close()

			file = filepath.Join(file, config.Index)
			if f, err = fs.Open(file); err != nil {
				return echo.ErrNotFound
			}

			if fi, err = f.Stat(); err != nil {
				return err
			}
		}
		return c.ServeContent(f, fi.Name(), fi.ModTime())
	}
}
<|endoftext|>


<commit_before>// Copyright 2022 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

// Package cuttlefish allows to use Cuttlefish Android emulators hosted on Google Compute Engine
// (GCE) virtual machines as VMs. It is assumed that syz-manager also runs on GCE as VMs are
// created in the current project/zone.
//
// See https://cloud.google.com/compute/docs for details.
// In particular, how to build GCE-compatible images:
// https://cloud.google.com/compute/docs/tutorials/building-images
package cuttlefish

import (
	"fmt"
	"path/filepath"
	"time"

	"github.com/google/syzkaller/pkg/log"
	"github.com/google/syzkaller/pkg/report"
	"github.com/google/syzkaller/vm/vmimpl"
)

const (
	deviceRoot = "/data/fuzz"
)

func init() {
	vmimpl.Register("cuttlefish", ctor, true)
}

type Pool struct {
	env     *vmimpl.Env
	gcePool vmimpl.Pool
}

type instance struct {
	debug   bool
	gceInst vmimpl.Instance
}

func ctor(env *vmimpl.Env) (vmimpl.Pool, error) {
	gcePool, err := vmimpl.Types["gce"].Ctor(env)
	if err != nil {
		return nil, fmt.Errorf("failed to create underlying GCE pool: %s", err)
	}

	pool := &Pool{
		env:     env,
		gcePool: gcePool,
	}

	return pool, nil
}

func (pool *Pool) Count() int {
	return pool.gcePool.Count()
}

func (pool *Pool) Create(workdir string, index int) (vmimpl.Instance, error) {
	gceInst, err := pool.gcePool.Create(workdir, index)
	if err != nil {
		return nil, fmt.Errorf("failed to create underlying gce instance: %s", err)
	}

	inst := &instance{
		debug:   pool.env.Debug,
		gceInst: gceInst,
	}

	// Start a Cuttlefish device on the GCE instance
	// TODO: pass it the specific kernel artifact using -kernel_path and -initramfs_path flags
	if err := inst.runOnHost(10*time.Minute, "./bin/launch_cvd -daemon"); err != nil {
		return nil, fmt.Errorf("failed to start cuttlefish: %s", err)
	}

	if err := inst.runOnHost(10*time.Minute, "adb wait-for-device"); err != nil {
		return nil, fmt.Errorf("failed while waiting for device: %s", err)
	}

	if err := inst.runOnHost(5*time.Minute, "adb root"); err != nil {
		return nil, fmt.Errorf("failed to get root access to device: %s", err)
	}

	return inst, nil
}

func (inst *instance) runOnHost(timeout time.Duration, cmd string) error {
	outc, errc, err := inst.gceInst.Run(timeout, nil, cmd)
	if err != nil {
		return fmt.Errorf("failed to run command: %s", err)
	}

	for {
		select {
		case <-vmimpl.Shutdown:
			return nil
		case err := <-errc:
			if err != nil {
				return fmt.Errorf("error while running: %s", err)
			}
			return nil
		case out, ok := <-outc:
			if ok && inst.debug {
				log.Logf(1, "%s", out)
			}
		}
	}
}

func (inst *instance) Copy(hostSrc string) (string, error) {
	gceDst, err := inst.gceInst.Copy(hostSrc)
	if err != nil {
		return "", fmt.Errorf("error copying to worker instance: %s", err)
	}

	deviceDst := filepath.Join(deviceRoot, filepath.Base(hostSrc))
	pushCmd := fmt.Sprintf("adb push %s %s", gceDst, deviceDst)

	if err := inst.runOnHost(5*time.Minute, pushCmd); err != nil {
		return "", fmt.Errorf("error pushing to device: %s", err)
	}

	return deviceDst, nil
}

func (inst *instance) Forward(port int) (string, error) {
	return "", fmt.Errorf("not implemented")
}

func (inst *instance) Close() {
	inst.gceInst.Close()
}

func (inst *instance) Run(timeout time.Duration, stop <-chan bool, command string) (
	<-chan []byte, <-chan error, error) {
	return inst.gceInst.Run(timeout, stop, fmt.Sprintf("adb shell %s", command))
}

func (inst *instance) Diagnose(rep *report.Report) ([]byte, bool) {
	return nil, false
}
<commit_msg>vm/cuttlefish: implement forward function<commit_after>// Copyright 2022 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

// Package cuttlefish allows to use Cuttlefish Android emulators hosted on Google Compute Engine
// (GCE) virtual machines as VMs. It is assumed that syz-manager also runs on GCE as VMs are
// created in the current project/zone.
//
// See https://cloud.google.com/compute/docs for details.
// In particular, how to build GCE-compatible images:
// https://cloud.google.com/compute/docs/tutorials/building-images
package cuttlefish

import (
	"fmt"
	"path/filepath"
	"time"

	"github.com/google/syzkaller/pkg/log"
	"github.com/google/syzkaller/pkg/report"
	"github.com/google/syzkaller/vm/vmimpl"
)

const (
	deviceRoot = "/data/fuzz"
)

func init() {
	vmimpl.Register("cuttlefish", ctor, true)
}

type Pool struct {
	env     *vmimpl.Env
	gcePool vmimpl.Pool
}

type instance struct {
	debug   bool
	gceInst vmimpl.Instance
}

func ctor(env *vmimpl.Env) (vmimpl.Pool, error) {
	gcePool, err := vmimpl.Types["gce"].Ctor(env)
	if err != nil {
		return nil, fmt.Errorf("failed to create underlying GCE pool: %s", err)
	}

	pool := &Pool{
		env:     env,
		gcePool: gcePool,
	}

	return pool, nil
}

func (pool *Pool) Count() int {
	return pool.gcePool.Count()
}

func (pool *Pool) Create(workdir string, index int) (vmimpl.Instance, error) {
	gceInst, err := pool.gcePool.Create(workdir, index)
	if err != nil {
		return nil, fmt.Errorf("failed to create underlying gce instance: %s", err)
	}

	inst := &instance{
		debug:   pool.env.Debug,
		gceInst: gceInst,
	}

	// Start a Cuttlefish device on the GCE instance
	// TODO: pass it the specific kernel artifact using -kernel_path and -initramfs_path flags
	if err := inst.runOnHost(10*time.Minute, "./bin/launch_cvd -daemon", false); err != nil {
		return nil, fmt.Errorf("failed to start cuttlefish: %s", err)
	}

	if err := inst.runOnHost(10*time.Minute, "adb wait-for-device", false); err != nil {
		return nil, fmt.Errorf("failed while waiting for device: %s", err)
	}

	if err := inst.runOnHost(5*time.Minute, "adb root", false); err != nil {
		return nil, fmt.Errorf("failed to get root access to device: %s", err)
	}

	return inst, nil
}

func (inst *instance) runOnHost(timeout time.Duration, cmd string, background bool) error {
	taskTimeout := timeout
	if background {
		taskTimeout = 24 * time.Hour
	}

	outc, errc, err := inst.gceInst.Run(taskTimeout, nil, cmd)
	if err != nil {
		return fmt.Errorf("failed to run command: %s", err)
	}

	timer := time.NewTimer(timeout)
	for {
		select {
		case <-vmimpl.Shutdown:
			return nil
		case err := <-errc:
			if err != nil {
				return fmt.Errorf("error while running: %s", err)
			}
			return nil
		case out, ok := <-outc:
			if ok && inst.debug {
				log.Logf(1, "%s", out)
			}
		case <-timer.C:
			// When running processes in the background, we assume that everything is fine if the timer
			// expires and it hasn't given any errors.
			if background {
				return nil
			}
		}
	}
}

func (inst *instance) Copy(hostSrc string) (string, error) {
	gceDst, err := inst.gceInst.Copy(hostSrc)
	if err != nil {
		return "", fmt.Errorf("error copying to worker instance: %s", err)
	}

	deviceDst := filepath.Join(deviceRoot, filepath.Base(hostSrc))
	pushCmd := fmt.Sprintf("adb push %s %s", gceDst, deviceDst)

	if err := inst.runOnHost(5*time.Minute, pushCmd, false); err != nil {
		return "", fmt.Errorf("error pushing to device: %s", err)
	}

	return deviceDst, nil
}

func (inst *instance) Forward(port int) (string, error) {
	hostForward, err := inst.gceInst.Forward(port)
	if err != nil {
		return "", fmt.Errorf("failed to get IP/port from GCE instance: %s", err)
	}
	cmd := fmt.Sprintf("socat TCP-LISTEN:%d,fork TCP:%s", port, hostForward)
	if err := inst.runOnHost(60*time.Second, cmd, true); err != nil {
		return "", fmt.Errorf("unable to forward port on host: %s", err)
	}

	for i := 0; i < 100; i++ {
		devicePort := vmimpl.RandomPort()
		cmd := fmt.Sprintf("adb reverse tcp:%d tcp:%d", devicePort, port)
		err = inst.runOnHost(10*time.Second, cmd, false)
		if err == nil {
			return fmt.Sprintf("127.0.0.1:%d", devicePort), nil
		}
	}

	return "", fmt.Errorf("unable to forward port on device: %s", err)
}

func (inst *instance) Close() {
	inst.gceInst.Close()
}

func (inst *instance) Run(timeout time.Duration, stop <-chan bool, command string) (
	<-chan []byte, <-chan error, error) {
	return inst.gceInst.Run(timeout, stop, fmt.Sprintf("adb shell %s", command))
}

func (inst *instance) Diagnose(rep *report.Report) ([]byte, bool) {
	return nil, false
}
<|endoftext|>


<commit_before>package buse

import (
	"fmt"
	"io"
	"os"
	"os/exec"
	"path"
	"strings"
	"syscall"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/libopenstorage/openstorage/api"
	"github.com/libopenstorage/openstorage/cluster"
	"github.com/libopenstorage/openstorage/volume"
	"github.com/pborman/uuid"
	"github.com/portworx/kvdb"
)

const (
	Name          = "buse"
	Type          = api.Block
	BuseDBKey     = "OpenStorageBuseKey"
	BuseMountPath = "/var/lib/openstorage/buse/"
)

// Implements the open storage volume interface.
type driver struct {
	*volume.IoNotSupported
	*volume.DefaultEnumerator
	buseDevices map[string]*buseDev
}

// Implements the Device interface.
type buseDev struct {
	file string
	f    *os.File
	nbd  *NBD
}

func (d *buseDev) ReadAt(b []byte, off int64) (n int, err error) {
	return d.f.ReadAt(b, off)
}

func (d *buseDev) WriteAt(b []byte, off int64) (n int, err error) {
	return d.f.WriteAt(b, off)
}

func copyFile(source string, dest string) (err error) {
	sourcefile, err := os.Open(source)
	if err != nil {
		return err
	}

	defer sourcefile.Close()

	destfile, err := os.Create(dest)
	if err != nil {
		return err
	}

	defer destfile.Close()

	_, err = io.Copy(destfile, sourcefile)
	if err == nil {
		sourceinfo, err := os.Stat(source)
		if err != nil {
			err = os.Chmod(dest, sourceinfo.Mode())
		}

	}

	return
}

func Init(params volume.DriverParams) (volume.VolumeDriver, error) {
	inst := &driver{
		IoNotSupported:    &volume.IoNotSupported{},
		DefaultEnumerator: volume.NewDefaultEnumerator(Name, kvdb.Instance()),
	}

	inst.buseDevices = make(map[string]*buseDev)

	err := os.MkdirAll(BuseMountPath, 0744)
	if err != nil {
		return nil, err
	}

	volumeInfo, err := inst.DefaultEnumerator.Enumerate(
		api.VolumeLocator{},
		nil)
	if err == nil {
		for _, info := range volumeInfo {
			if info.Status == "" {
				info.Status = api.Up
				inst.UpdateVol(&info)
			}
		}
	} else {
		logrus.Println("Could not enumerate Volumes, ", err)
	}

	c, err := cluster.Inst()
	if err != nil {
		logrus.Println("BUSE initializing in single node mode")
	} else {
		logrus.Println("BUSE initializing in clustered mode")
		c.AddEventListener(inst)
	}

	logrus.Println("BUSE initialized and driver mounted at: ", BuseMountPath)
	return inst, nil
}

//
// These functions below implement the volume driver interface.
//

func (d *driver) String() string {
	return Name
}

func (d *driver) Type() api.DriverType {
	return Type
}

// Status diagnostic information
func (d *driver) Status() [][2]string {
	return [][2]string{}
}

func (d *driver) Create(locator api.VolumeLocator, source *api.Source, spec *api.VolumeSpec) (api.VolumeID, error) {
	volumeID := uuid.New()
	volumeID = strings.TrimSuffix(volumeID, "\n")

	if spec.Size == 0 {
		return api.BadVolumeID, fmt.Errorf("Volume size cannot be zero", "buse")
	}

	if spec.Format == "" {
		return api.BadVolumeID, fmt.Errorf("Missing volume format", "buse")
	}

	// Create a file on the local buse path with this UUID.
	buseFile := path.Join(BuseMountPath, string(volumeID))
	f, err := os.Create(buseFile)
	if err != nil {
		logrus.Println(err)
		return api.BadVolumeID, err
	}

	err = f.Truncate(int64(spec.Size))
	if err != nil {
		logrus.Println(err)
		return api.BadVolumeID, err
	}

	bd := &buseDev{
		file: buseFile,
		f:    f}

	nbd := Create(bd, int64(spec.Size))
	bd.nbd = nbd

	logrus.Infof("Connecting to NBD...")
	dev, err := bd.nbd.Connect()
	if err != nil {
		logrus.Println(err)
		return api.BadVolumeID, err
	}

	logrus.Infof("Formatting %s with %v", dev, spec.Format)
	cmd := "/sbin/mkfs." + string(spec.Format)
	o, err := exec.Command(cmd, dev).Output()
	if err != nil {
		logrus.Warnf("Failed to run command %v %v: %v", cmd, dev, o)
		return api.BadVolumeID, err
	}

	logrus.Infof("BUSE mapped NBD device %s (size=%v) to block file %s", dev, spec.Size, buseFile)

	v := &api.Volume{
		ID:         api.VolumeID(volumeID),
		Source:     source,
		Locator:    locator,
		Ctime:      time.Now(),
		Spec:       spec,
		LastScan:   time.Now(),
		Format:     spec.Format,
		State:      api.VolumeAvailable,
		Status:     api.Up,
		DevicePath: dev,
	}

	d.buseDevices[dev] = bd

	err = d.CreateVol(v)
	if err != nil {
		return api.BadVolumeID, err
	}
	return v.ID, err
}

func (d *driver) Delete(volumeID api.VolumeID) error {
	v, err := d.GetVol(volumeID)
	if err != nil {
		logrus.Println(err)
		return err
	}

	bd, ok := d.buseDevices[v.DevicePath]
	if !ok {
		err = fmt.Errorf("Cannot locate a BUSE device for %s", v.DevicePath)
		logrus.Println(err)
		return err
	}

	// Clean up buse block file and close the NBD connection.
	os.Remove(bd.file)
	bd.f.Close()
	bd.nbd.Disconnect()

	logrus.Infof("BUSE deleted volume %v at NBD device %s", volumeID, v.DevicePath)

	err = d.DeleteVol(volumeID)
	if err != nil {
		logrus.Println(err)
		return err
	}

	return nil
}

func (d *driver) Mount(volumeID api.VolumeID, mountpath string) error {
	v, err := d.GetVol(volumeID)
	if err != nil {
		return fmt.Errorf("Failed to locate volume %q", string(volumeID))
	}
	err = syscall.Mount(v.DevicePath, mountpath, string(v.Spec.Format), 0, "")
	if err != nil {
		logrus.Errorf("Mounting %s on %s failed because of %v", v.DevicePath, mountpath, err)
		return fmt.Errorf("Failed to mount %v at %v: %v", v.DevicePath, mountpath, err)
	}

	logrus.Infof("BUSE mounted NBD device %s at %s", v.DevicePath, mountpath)

	v.AttachPath = mountpath
	err = d.UpdateVol(v)

	return nil
}

func (d *driver) Unmount(volumeID api.VolumeID, mountpath string) error {
	v, err := d.GetVol(volumeID)
	if err != nil {
		return err
	}
	if v.AttachPath == "" {
		return fmt.Errorf("Device %v not mounted", volumeID)
	}
	err = syscall.Unmount(v.AttachPath, 0)
	if err != nil {
		return err
	}
	v.AttachPath = ""
	err = d.UpdateVol(v)
	return err
}

func (d *driver) Snapshot(volumeID api.VolumeID, readonly bool, locator api.VolumeLocator) (api.VolumeID, error) {
	volIDs := make([]api.VolumeID, 1)
	volIDs[0] = volumeID
	vols, err := d.Inspect(volIDs)
	if err != nil {
		return api.BadVolumeID, nil
	}

	source := &api.Source{Parent: volumeID}
	newVolumeID, err := d.Create(locator, source, vols[0].Spec)
	if err != nil {
		return api.BadVolumeID, nil
	}

	// BUSE does not support snapshots, so just copy the block files.
	err = copyFile(BuseMountPath+string(volumeID), BuseMountPath+string(newVolumeID))
	if err != nil {
		d.Delete(newVolumeID)
		return api.BadVolumeID, nil
	}

	return newVolumeID, nil
}

func (d *driver) Set(volumeID api.VolumeID, locator *api.VolumeLocator, spec *api.VolumeSpec) error {
	if spec != nil {
		return volume.ErrNotSupported
	}
	v, err := d.GetVol(volumeID)
	if err != nil {
		return err
	}
	if locator != nil {
		v.Locator = *locator
	}
	err = d.UpdateVol(v)
	return err
}

func (d *driver) Attach(volumeID api.VolumeID) (string, error) {
	// Nothing to do on attach.
	return path.Join(BuseMountPath, string(volumeID)), nil
}

func (d *driver) Detach(volumeID api.VolumeID) error {
	// Nothing to do on detach.
	return nil
}

func (d *driver) Stats(volumeID api.VolumeID) (api.Stats, error) {
	return api.Stats{}, volume.ErrNotSupported
}

func (d *driver) Alerts(volumeID api.VolumeID) (api.Alerts, error) {
	return api.Alerts{}, volume.ErrNotSupported
}

func (d *driver) Shutdown() {
	logrus.Printf("%s Shutting down", Name)
	syscall.Unmount(BuseMountPath, 0)
}

func (d *driver) ClusterInit(self *api.Node, db *cluster.Database) error {
	return nil
}

func (d *driver) Init(self *api.Node, db *cluster.Database) error {
	return nil
}

func (d *driver) Join(self *api.Node, db *cluster.Database) error {
	return nil
}

func (d *driver) Add(self *api.Node) error {
	return nil
}

func (d *driver) Remove(self *api.Node) error {
	return nil
}

func (d *driver) Update(self *api.Node) error {
	return nil
}

func (d *driver) Leave(self *api.Node) error {
	return nil
}

func init() {

	// Register ourselves as an openstorage volume driver.
	volume.Register(Name, Init)
}
<commit_msg>add volume/drivers/common/common.go<commit_after>package buse

import (
	"fmt"
	"io"
	"os"
	"os/exec"
	"path"
	"strings"
	"syscall"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/libopenstorage/openstorage/api"
	"github.com/libopenstorage/openstorage/cluster"
	"github.com/libopenstorage/openstorage/volume"
	"github.com/pborman/uuid"
	"github.com/portworx/kvdb"
)

const (
	Name          = "buse"
	Type          = api.DriverType_DRIVER_TYPE_BLOCK
	BuseDBKey     = "OpenStorageBuseKey"
	BuseMountPath = "/var/lib/openstorage/buse/"
)

// Implements the open storage volume interface.
type driver struct {
	*volume.IoNotSupported
	*volume.DefaultEnumerator
	buseDevices map[string]*buseDev
}

// Implements the Device interface.
type buseDev struct {
	file string
	f    *os.File
	nbd  *NBD
}

func (d *buseDev) ReadAt(b []byte, off int64) (n int, err error) {
	return d.f.ReadAt(b, off)
}

func (d *buseDev) WriteAt(b []byte, off int64) (n int, err error) {
	return d.f.WriteAt(b, off)
}

func copyFile(source string, dest string) (err error) {
	sourcefile, err := os.Open(source)
	if err != nil {
		return err
	}

	defer sourcefile.Close()

	destfile, err := os.Create(dest)
	if err != nil {
		return err
	}

	defer destfile.Close()

	_, err = io.Copy(destfile, sourcefile)
	if err == nil {
		sourceinfo, err := os.Stat(source)
		if err != nil {
			err = os.Chmod(dest, sourceinfo.Mode())
		}

	}

	return
}

func Init(params volume.DriverParams) (volume.VolumeDriver, error) {
	inst := &driver{
		IoNotSupported:    &volume.IoNotSupported{},
		DefaultEnumerator: volume.NewDefaultEnumerator(Name, kvdb.Instance()),
	}

	inst.buseDevices = make(map[string]*buseDev)

	err := os.MkdirAll(BuseMountPath, 0744)
	if err != nil {
		return nil, err
	}

	volumeInfo, err := inst.DefaultEnumerator.Enumerate(
		api.VolumeLocator{},
		nil)
	if err == nil {
		for _, info := range volumeInfo {
			if info.Status == "" {
				info.Status = api.Up
				inst.UpdateVol(&info)
			}
		}
	} else {
		logrus.Println("Could not enumerate Volumes, ", err)
	}

	c, err := cluster.Inst()
	if err != nil {
		logrus.Println("BUSE initializing in single node mode")
	} else {
		logrus.Println("BUSE initializing in clustered mode")
		c.AddEventListener(inst)
	}

	logrus.Println("BUSE initialized and driver mounted at: ", BuseMountPath)
	return inst, nil
}

//
// These functions below implement the volume driver interface.
//

func (d *driver) String() string {
	return Name
}

func (d *driver) Type() api.DriverType {
	return Type
}

// Status diagnostic information
func (d *driver) Status() [][2]string {
	return [][2]string{}
}

func (d *driver) Create(locator api.VolumeLocator, source *api.Source, spec *api.VolumeSpec) (string, error) {
	volumeID := uuid.New()
	volumeID = strings.TrimSuffix(volumeID, "\n")

	if spec.Size == 0 {
		return "", fmt.Errorf("Volume size cannot be zero", "buse")
	}

	if spec.Format == "" {
		return "", fmt.Errorf("Missing volume format", "buse")
	}

	// Create a file on the local buse path with this UUID.
	buseFile := path.Join(BuseMountPath, volumeID)
	f, err := os.Create(buseFile)
	if err != nil {
		logrus.Println(err)
		return "", err
	}

	err = f.Truncate(int64(spec.Size))
	if err != nil {
		logrus.Println(err)
		return "", err
	}

	bd := &buseDev{
		file: buseFile,
		f:    f}

	nbd := Create(bd, int64(spec.Size))
	bd.nbd = nbd

	logrus.Infof("Connecting to NBD...")
	dev, err := bd.nbd.Connect()
	if err != nil {
		logrus.Println(err)
		return "", err
	}

	logrus.Infof("Formatting %s with %v", dev, spec.Format)
	cmd := "/sbin/mkfs." + spec.Format
	o, err := exec.Command(cmd, dev).Output()
	if err != nil {
		logrus.Warnf("Failed to run command %v %v: %v", cmd, dev, o)
		return "", err
	}

	logrus.Infof("BUSE mapped NBD device %s (size=%v) to block file %s", dev, spec.Size, buseFile)

	v := &api.Volume{
		ID:         volumeID,
		Source:     source,
		Locator:    locator,
		Ctime:      time.Now(),
		Spec:       spec,
		LastScan:   time.Now(),
		Format:     spec.Format,
		State:      api.VolumeAvailable,
		Status:     api.Up,
		DevicePath: dev,
	}

	d.buseDevices[dev] = bd

	err = d.CreateVol(v)
	if err != nil {
		return "", err
	}
	return v.ID, err
}

func (d *driver) Delete(volumeID string) error {
	v, err := d.GetVol(volumeID)
	if err != nil {
		logrus.Println(err)
		return err
	}

	bd, ok := d.buseDevices[v.DevicePath]
	if !ok {
		err = fmt.Errorf("Cannot locate a BUSE device for %s", v.DevicePath)
		logrus.Println(err)
		return err
	}

	// Clean up buse block file and close the NBD connection.
	os.Remove(bd.file)
	bd.f.Close()
	bd.nbd.Disconnect()

	logrus.Infof("BUSE deleted volume %v at NBD device %s", volumeID, v.DevicePath)

	err = d.DeleteVol(volumeID)
	if err != nil {
		logrus.Println(err)
		return err
	}

	return nil
}

func (d *driver) Mount(volumeID string, mountpath string) error {
	v, err := d.GetVol(volumeID)
	if err != nil {
		return fmt.Errorf("Failed to locate volume %q", volumeID)
	}
	err = syscall.Mount(v.DevicePath, mountpath, v.Spec.Format, 0, "")
	if err != nil {
		logrus.Errorf("Mounting %s on %s failed because of %v", v.DevicePath, mountpath, err)
		return fmt.Errorf("Failed to mount %v at %v: %v", v.DevicePath, mountpath, err)
	}

	logrus.Infof("BUSE mounted NBD device %s at %s", v.DevicePath, mountpath)

	v.AttachPath = mountpath
	err = d.UpdateVol(v)

	return nil
}

func (d *driver) Unmount(volumeID string, mountpath string) error {
	v, err := d.GetVol(volumeID)
	if err != nil {
		return err
	}
	if v.AttachPath == "" {
		return fmt.Errorf("Device %v not mounted", volumeID)
	}
	err = syscall.Unmount(v.AttachPath, 0)
	if err != nil {
		return err
	}
	v.AttachPath = ""
	err = d.UpdateVol(v)
	return err
}

func (d *driver) Snapshot(volumeID string, readonly bool, locator api.VolumeLocator) (api.VolumeID, error) {
	volIDs := make([]string, 1)
	volIDs[0] = volumeID
	vols, err := d.Inspect(volIDs)
	if err != nil {
		return "", nil
	}

	source := &api.Source{Parent: volumeID}
	newVolumeID, err := d.Create(locator, source, vols[0].Spec)
	if err != nil {
		return "", nil
	}

	// BUSE does not support snapshots, so just copy the block files.
	err = copyFile(BuseMountPath+volumeID, BuseMountPath+newVolumeID)
	if err != nil {
		d.Delete(newVolumeID)
		return "", nil
	}

	return newVolumeID, nil
}

func (d *driver) Set(volumeID string, locator *api.VolumeLocator, spec *api.VolumeSpec) error {
	if spec != nil {
		return volume.ErrNotSupported
	}
	v, err := d.GetVol(volumeID)
	if err != nil {
		return err
	}
	if locator != nil {
		v.Locator = *locator
	}
	err = d.UpdateVol(v)
	return err
}

func (d *driver) Attach(volumeID string) (string, error) {
	// Nothing to do on attach.
	return path.Join(BuseMountPath, volumeID), nil
}

func (d *driver) Detach(volumeID string) error {
	// Nothing to do on detach.
	return nil
}

func (d *driver) Stats(volumeID string) (*api.Stats, error) {
	return nil, volume.ErrNotSupported
}

func (d *driver) Alerts(volumeID string) (*api.Alerts, error) {
	return nil, volume.ErrNotSupported
}

func (d *driver) Shutdown() {
	logrus.Printf("%s Shutting down", Name)
	syscall.Unmount(BuseMountPath, 0)
}

func (d *driver) ClusterInit(self *api.Node, db *cluster.Database) error {
	return nil
}

func (d *driver) Init(self *api.Node, db *cluster.Database) error {
	return nil
}

func (d *driver) Join(self *api.Node, db *cluster.Database) error {
	return nil
}

func (d *driver) Add(self *api.Node) error {
	return nil
}

func (d *driver) Remove(self *api.Node) error {
	return nil
}

func (d *driver) Update(self *api.Node) error {
	return nil
}

func (d *driver) Leave(self *api.Node) error {
	return nil
}

func init() {

	// Register ourselves as an openstorage volume driver.
	volume.Register(Name, Init)
}
<|endoftext|>


<commit_before>// Copyright ©2017 The Gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gonum

import (
	"testing"

	"gonum.org/v1/gonum/blas/testblas"
)

func TestZgemv(t *testing.T) {
	testblas.ZgemvTest(t, impl)
}

func TestZgerc(t *testing.T) {
	testblas.ZgercTest(t, impl)
}

func TestZgeru(t *testing.T) {
	testblas.ZgeruTest(t, impl)
}

func TestZhemv(t *testing.T) {
	testblas.ZhemvTest(t, impl)
}

func TestZher(t *testing.T) {
	testblas.ZherTest(t, impl)
}

func TestZher2(t *testing.T) {
	testblas.Zher2Test(t, impl)
}

func TestZhpmv(t *testing.T) {
	testblas.ZhpmvTest(t, impl)
}

func TestZhpr(t *testing.T) {
	testblas.ZhprTest(t, impl)
}

func TestZhpr2(t *testing.T) {
	testblas.Zhpr2Test(t, impl)
}

func TestZtrmv(t *testing.T) {
	testblas.ZtrmvTest(t, impl)
}

func TestZtrsv(t *testing.T) {
	testblas.ZtrsvTest(t, impl)
}
<commit_msg>blas/gonum: enable test for Ztpmv<commit_after>// Copyright ©2017 The Gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gonum

import (
	"testing"

	"gonum.org/v1/gonum/blas/testblas"
)

func TestZgemv(t *testing.T) {
	testblas.ZgemvTest(t, impl)
}

func TestZgerc(t *testing.T) {
	testblas.ZgercTest(t, impl)
}

func TestZgeru(t *testing.T) {
	testblas.ZgeruTest(t, impl)
}

func TestZhemv(t *testing.T) {
	testblas.ZhemvTest(t, impl)
}

func TestZher(t *testing.T) {
	testblas.ZherTest(t, impl)
}

func TestZher2(t *testing.T) {
	testblas.Zher2Test(t, impl)
}

func TestZhpmv(t *testing.T) {
	testblas.ZhpmvTest(t, impl)
}

func TestZhpr(t *testing.T) {
	testblas.ZhprTest(t, impl)
}

func TestZhpr2(t *testing.T) {
	testblas.Zhpr2Test(t, impl)
}

func TestZtpmv(t *testing.T) {
	testblas.ZtpmvTest(t, impl)
}

func TestZtrmv(t *testing.T) {
	testblas.ZtrmvTest(t, impl)
}

func TestZtrsv(t *testing.T) {
	testblas.ZtrsvTest(t, impl)
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/md5"
	"database/sql"
	"fmt"
	"log"
	"os"
	"os/exec"
	"path"
	"sync"

	_ "github.com/mattn/go-sqlite3"
)

const (
	thumbsDir      = "public/thumbs"
	bigThumbSize   = "1000x1000"
	smallThumbSize = "200x200"
	workers        = 4
)

func generateSmallThumb(photoPath, identifier string) (thumbPath string, err error) {
	thumbPath = path.Join(thumbsDir, fmt.Sprintf("%s_small.jpg", identifier))
	if _, err = os.Stat(thumbPath); os.IsNotExist(err) { // file does not exist
		err = exec.Command(
			"convert", photoPath,
			"-auto-orient",
			"-strip",
			"-resize", smallThumbSize+"^",
			"-gravity", "center",
			"-extent", smallThumbSize,
			thumbPath).Run()
	}
	return
}

func generateBigThumb(photoPath, identifier string) (thumbPath string, err error) {
	thumbPath = path.Join(thumbsDir, fmt.Sprintf("%s_big.jpg", identifier))
	if _, err = os.Stat(thumbPath); os.IsNotExist(err) { // file does not exist
		err = exec.Command(
			"convert", photoPath,
			"-auto-orient",
			"-strip",
			"-resize", bigThumbSize,
			thumbPath).Run()
	}
	return
}

func generateThumb(ch chan string, wg *sync.WaitGroup) {
	defer wg.Done()

	for photoPath := range ch {
		identifier := fmt.Sprintf("%x", md5.Sum([]byte(photoPath)))

		bigThumbPath, err := generateBigThumb(photoPath, identifier)
		if err != nil { // error
			return
		}

		fmt.Println(bigThumbPath)

		smallThumbPath, err := generateSmallThumb(bigThumbPath, identifier)
		if err == nil { // success
			fmt.Println(smallThumbPath)
		}
	}
}

func main() {
	var wg sync.WaitGroup
	ch := make(chan string)

	db, err := sql.Open("sqlite3", "thyme.db")
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	rows, err := db.Query(`
	SELECT path FROM photos
	JOIN sets ON photos.set_id = sets.id
	ORDER BY sets.taken_at DESC, photos.taken_at ASC
	`)
	if err != nil {
		log.Fatal(err)
	}
	defer rows.Close()

	if err := os.MkdirAll(thumbsDir, os.ModeDir|0755); err != nil {
		log.Fatal(err)
	}

	for i := 0; i < workers; i++ {
		wg.Add(1)
		go generateThumb(ch, &wg)
	}

	for rows.Next() {
		var photoPath string
		rows.Scan(&photoPath)
		ch <- photoPath
	}

	close(ch)
	wg.Wait()

	if err := rows.Err(); err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Ensure number of workers is at least 1<commit_after>package main

import (
	"crypto/md5"
	"database/sql"
	"fmt"
	"log"
	"os"
	"os/exec"
	"path"
	"sync"

	_ "github.com/mattn/go-sqlite3"
)

const (
	thumbsDir      = "public/thumbs"
	bigThumbSize   = "1000x1000"
	smallThumbSize = "200x200"
	workers        = 4 // min: 1
)

func generateSmallThumb(photoPath, identifier string) (thumbPath string, err error) {
	thumbPath = path.Join(thumbsDir, fmt.Sprintf("%s_small.jpg", identifier))
	if _, err = os.Stat(thumbPath); os.IsNotExist(err) { // file does not exist
		err = exec.Command(
			"convert", photoPath,
			"-auto-orient",
			"-strip",
			"-resize", smallThumbSize+"^",
			"-gravity", "center",
			"-extent", smallThumbSize,
			thumbPath).Run()
	}
	return
}

func generateBigThumb(photoPath, identifier string) (thumbPath string, err error) {
	thumbPath = path.Join(thumbsDir, fmt.Sprintf("%s_big.jpg", identifier))
	if _, err = os.Stat(thumbPath); os.IsNotExist(err) { // file does not exist
		err = exec.Command(
			"convert", photoPath,
			"-auto-orient",
			"-strip",
			"-resize", bigThumbSize,
			thumbPath).Run()
	}
	return
}

func generateThumb(ch chan string, wg *sync.WaitGroup) {
	defer wg.Done()

	for photoPath := range ch {
		identifier := fmt.Sprintf("%x", md5.Sum([]byte(photoPath)))

		bigThumbPath, err := generateBigThumb(photoPath, identifier)
		if err != nil { // error
			return
		}

		fmt.Println(bigThumbPath)

		smallThumbPath, err := generateSmallThumb(bigThumbPath, identifier)
		if err == nil { // success
			fmt.Println(smallThumbPath)
		}
	}
}

func main() {
	var wg sync.WaitGroup

	if workers < 1 {
		log.Fatal("number of workers must be at least 1")
	}

	ch := make(chan string)

	db, err := sql.Open("sqlite3", "thyme.db")
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	rows, err := db.Query(`
	SELECT path FROM photos
	JOIN sets ON photos.set_id = sets.id
	ORDER BY sets.taken_at DESC, photos.taken_at ASC
	`)
	if err != nil {
		log.Fatal(err)
	}
	defer rows.Close()

	if err := os.MkdirAll(thumbsDir, os.ModeDir|0755); err != nil {
		log.Fatal(err)
	}

	for i := 0; i < workers; i++ {
		wg.Add(1)
		go generateThumb(ch, &wg)
	}

	for rows.Next() {
		var photoPath string
		rows.Scan(&photoPath)
		ch <- photoPath
	}

	close(ch)
	wg.Wait()

	if err := rows.Err(); err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package backend

import (
	"encoding/json"
	"errors"
	"fmt"
	"strings"
)

const (
	AppName = "Imperial Splendour: Rise of the Republic"

	appDataPath  = "The Creative Assembly/Empire/scripts/"
	modPath      = "IS_Files/"
	dataPath     = "data/"
	campaignPath = dataPath + "campaigns/imperial_splendour/"

	userScript   = "user.empire_script.txt"
	fileListFile = "IS_FileList.txt"
	infoFile     = "IS_info.json"

	websiteURL  = "https://imperialsplendour.com/"
	etwSteamURI = "steam://rungameid/10500"
)

type API struct {
	logger  Logger
	browser Browser
	window  Window
	Sh      Handler
	dirs    struct {
		etw     string
		appData string
	}
	info info
}

type modFiles struct {
	dataFiles     []string
	campaignFiles []string
}

type info struct {
	IsActive           bool   `json:"isActive"`
	Version            string `json:"version"`
	UserScriptChecksum string `json:"usChecksum"`
}

type fileTuple struct {
	file string
	dir  string
}

func (a *API) setStatus(isActive bool) error {
	newInfo := a.info
	newInfo.IsActive = isActive

	newInfoJSON, err := json.MarshalIndent(newInfo, "", "\t")
	if err != nil {
		return err
	}

	if err = a.Sh.WriteFile(a.dirs.etw+modPath+infoFile, newInfoJSON); err != nil {
		return err
	}

	a.info.IsActive = isActive
	return nil
}

func (a *API) readFileList() (*modFiles, error) {
	modFiles := modFiles{
		dataFiles:     []string{},
		campaignFiles: []string{},
	}

	fileBlob, err := a.Sh.ReadFile(a.dirs.etw + modPath + fileListFile)
	if err != nil {
		return nil, err
	}

	var errMsg string
	for _, file := range strings.Split(string(fileBlob), "\n") {
		if strings.HasSuffix(file, ".pack") {
			modFiles.dataFiles = append(modFiles.dataFiles, file)
		} else if strings.HasSuffix(file, ".tga") || strings.HasSuffix(file, ".esf") || strings.HasSuffix(file, ".lua") {
			modFiles.campaignFiles = append(modFiles.campaignFiles, file)
		} else if file != "" {
			errMsg = errMsg + fmt.Sprintf("Unknown file '%s' found in file list\n", file)
		}
	}
	if errMsg != "" {
		return nil, errors.New(errMsg)
	}
	return &modFiles, nil
}

func (a *API) moveFile(source, destination string) error {
	a.logger.Debugf("Moving from %s to %s", source, destination)
	return a.Sh.MoveFile(source, destination)
}

func (a *API) rollbackActivation(fileList []fileTuple) error {
	hasError := false
	for _, tuple := range fileList {
		if err := a.moveFile(tuple.dir+tuple.file, a.dirs.etw+modPath+tuple.file); err != nil {
			a.logger.Warnf("%v", err)
			hasError = true
		}
	}

	if hasError {
		return errors.New("RollbackError")
	}
	return nil
}

func (a *API) activateImpSplen() error {
	a.logger.Debug("Activating ImpSplen")
	var filesMoved []fileTuple

	files, err := a.readFileList()
	if err != nil {
		a.logger.Warnf("%v", err)
		return errors.New("FileListError")
	}

	a.logger.Debug("Moving data files")
	for _, file := range (*files).dataFiles {
		if err := a.moveFile(a.dirs.etw+modPath+file, a.dirs.etw+dataPath+file); err != nil {
			a.logger.Warnf("%v", err)
			if err = a.rollbackActivation(filesMoved); err != nil {
			return err
		}
			return errors.New("ActivationError")
		}
		filesMoved = append(filesMoved, fileTuple{file, a.dirs.etw + dataPath})
	}

	a.logger.Debug("Moving campaign files")
	for _, file := range (*files).campaignFiles {
		if err := a.moveFile(a.dirs.etw+modPath+file, a.dirs.etw+campaignPath+file); err != nil {
			a.logger.Warnf("%v", err)
			if err = a.rollbackActivation(filesMoved); err != nil {
			return err
		}
			return errors.New("ActivationError")
		}
		filesMoved = append(filesMoved, fileTuple{file, a.dirs.etw + campaignPath})
	}

	a.logger.Debug("Moving User Script")
	if err = a.moveFile(a.dirs.etw+modPath+userScript, a.dirs.appData+userScript); err != nil {
		a.logger.Warnf("%v", err)
		if err = a.rollbackActivation(filesMoved); err != nil {
		return err
	}
		return errors.New("ActivationError")
	}
	filesMoved = append(filesMoved, fileTuple{userScript, a.dirs.appData})

	if err = a.setStatus(true); err != nil {
		a.logger.Warnf("%v", err)
		if err = a.rollbackActivation(filesMoved); err != nil {
		return err
	}
		return errors.New("StatusUpdateError")
	}
	a.logger.Debug("ImpSplen activated")
	return nil
}

func (a *API) deactivateImpSplen() error {
	a.logger.Debug("Deactivating ImpSplen")

	files, err := a.readFileList()
	if err != nil {
		a.logger.Warnf("%v", err)
		return errors.New("FileListError")
	}

	hasError := false
	a.logger.Debug("Moving data files")
	for _, file := range files.dataFiles {
		if err := a.moveFile(a.dirs.etw+dataPath+file, a.dirs.etw+modPath+file); err != nil {
			a.logger.Warnf("%v", err)
			hasError = true
		}
	}

	a.logger.Debug("Moving campaign files")
	for _, file := range files.campaignFiles {
		if err := a.moveFile(a.dirs.etw+campaignPath+file, a.dirs.etw+modPath+file); err != nil {
			a.logger.Warnf("%v", err)
			hasError = true
		}
	}

	a.logger.Debug("Moving User Script")
	if err = a.moveFile(a.dirs.appData+userScript, a.dirs.etw+modPath+userScript); err != nil {
		a.logger.Warnf("%v", err)
		hasError = true
	}

	if err := a.setStatus(false); err != nil {
		a.logger.Warnf("%v", err)
		return errors.New("StatusUpdateError")
	}

	a.logger.Debug("ImpSplen deactivated")
	if hasError {
		return errors.New("DeactivationError")
	}
	return nil
}

func (a *API) deleteAllFiles() error {
	return a.Sh.Remove(a.dirs.etw + modPath)
}
<commit_msg>Refactor rollback<commit_after>package backend

import (
	"encoding/json"
	"errors"
	"fmt"
	"strings"
)

const (
	AppName = "Imperial Splendour: Rise of the Republic"

	appDataPath  = "The Creative Assembly/Empire/scripts/"
	modPath      = "IS_Files/"
	dataPath     = "data/"
	campaignPath = dataPath + "campaigns/imperial_splendour/"

	userScript   = "user.empire_script.txt"
	fileListFile = "IS_FileList.txt"
	infoFile     = "IS_info.json"

	websiteURL  = "https://imperialsplendour.com/"
	etwSteamURI = "steam://rungameid/10500"
)

type API struct {
	logger  Logger
	browser Browser
	window  Window
	Sh      Handler
	dirs    struct {
		etw     string
		appData string
	}
	info info
}

type modFiles struct {
	dataFiles     []string
	campaignFiles []string
}

type info struct {
	IsActive           bool   `json:"isActive"`
	Version            string `json:"version"`
	UserScriptChecksum string `json:"usChecksum"`
}

type fileTuple struct {
	file string
	dir  string
}

func (a *API) setStatus(isActive bool) error {
	newInfo := a.info
	newInfo.IsActive = isActive

	newInfoJSON, err := json.MarshalIndent(newInfo, "", "\t")
	if err != nil {
		return err
	}

	if err = a.Sh.WriteFile(a.dirs.etw+modPath+infoFile, newInfoJSON); err != nil {
		return err
	}

	a.info.IsActive = isActive
	return nil
}

func (a *API) readFileList() (*modFiles, error) {
	modFiles := modFiles{
		dataFiles:     []string{},
		campaignFiles: []string{},
	}

	fileBlob, err := a.Sh.ReadFile(a.dirs.etw + modPath + fileListFile)
	if err != nil {
		return nil, err
	}

	var errMsg string
	for _, file := range strings.Split(string(fileBlob), "\n") {
		if strings.HasSuffix(file, ".pack") {
			modFiles.dataFiles = append(modFiles.dataFiles, file)
		} else if strings.HasSuffix(file, ".tga") || strings.HasSuffix(file, ".esf") || strings.HasSuffix(file, ".lua") {
			modFiles.campaignFiles = append(modFiles.campaignFiles, file)
		} else if file != "" {
			errMsg = errMsg + fmt.Sprintf("Unknown file '%s' found in file list\n", file)
		}
	}
	if errMsg != "" {
		return nil, errors.New(errMsg)
	}
	return &modFiles, nil
}

func (a *API) moveFile(source, destination string) error {
	a.logger.Debugf("Moving from %s to %s", source, destination)
	return a.Sh.MoveFile(source, destination)
}

func (a *API) rollbackActivation(fileList []fileTuple) error {
	hasError := false
	for _, tuple := range fileList {
		if err := a.moveFile(tuple.dir+tuple.file, a.dirs.etw+modPath+tuple.file); err != nil {
			a.logger.Warnf("%v", err)
			hasError = true
		}
	}

	if hasError {
		return errors.New("RollbackError")
	}
	return nil
}

func (a *API) activateFile(fileName string, targetDir string, activeFiles *[]fileTuple) error {
	if err := a.moveFile(a.dirs.etw+modPath+fileName, targetDir+fileName); err != nil {
		a.logger.Warnf("%v", err)
		if err = a.rollbackActivation(*activeFiles); err != nil {
			return err
		}
		return errors.New("ActivationError")
	}

	*activeFiles = append(*activeFiles, fileTuple{fileName, targetDir})
	return nil
}

func (a *API) activateImpSplen() error {
	a.logger.Debug("Activating ImpSplen")
	var filesMoved []fileTuple

	files, err := a.readFileList()
	if err != nil {
		a.logger.Warnf("%v", err)
		return errors.New("FileListError")
	}

	a.logger.Debug("Moving data files")
	for _, file := range (*files).dataFiles {
		if err := a.activateFile(file, a.dirs.etw+dataPath, &filesMoved); err != nil {
			return err
		}
	}

	a.logger.Debug("Moving campaign files")
	for _, file := range (*files).campaignFiles {
		if err := a.activateFile(file, a.dirs.etw+campaignPath, &filesMoved); err != nil {
			return err
		}
	}

	a.logger.Debug("Moving User Script")
	if err = a.activateFile(userScript, a.dirs.appData, &filesMoved); err != nil {
		return err
	}

	if err = a.setStatus(true); err != nil {
		a.logger.Warnf("%v", err)
		if err = a.rollbackActivation(filesMoved); err != nil {
			return err
		}
		return errors.New("StatusUpdateError")
	}
	a.logger.Debug("ImpSplen activated")
	return nil
}

func (a *API) deactivateImpSplen() error {
	a.logger.Debug("Deactivating ImpSplen")

	files, err := a.readFileList()
	if err != nil {
		a.logger.Warnf("%v", err)
		return errors.New("FileListError")
	}

	hasError := false
	a.logger.Debug("Moving data files")
	for _, file := range files.dataFiles {
		if err := a.moveFile(a.dirs.etw+dataPath+file, a.dirs.etw+modPath+file); err != nil {
			a.logger.Warnf("%v", err)
			hasError = true
		}
	}

	a.logger.Debug("Moving campaign files")
	for _, file := range files.campaignFiles {
		if err := a.moveFile(a.dirs.etw+campaignPath+file, a.dirs.etw+modPath+file); err != nil {
			a.logger.Warnf("%v", err)
			hasError = true
		}
	}

	a.logger.Debug("Moving User Script")
	if err = a.moveFile(a.dirs.appData+userScript, a.dirs.etw+modPath+userScript); err != nil {
		a.logger.Warnf("%v", err)
		hasError = true
	}

	if err := a.setStatus(false); err != nil {
		a.logger.Warnf("%v", err)
		return errors.New("StatusUpdateError")
	}

	a.logger.Debug("ImpSplen deactivated")
	if hasError {
		return errors.New("DeactivationError")
	}
	return nil
}

func (a *API) deleteAllFiles() error {
	return a.Sh.Remove(a.dirs.etw + modPath)
}
<|endoftext|>


<commit_before>package main

// This script generates Go code from Protocol Buffers specifications. It uses
// the "gogo" Go code generator (https://github.com/gogo/protobuf). The
// generated Go code depends only on pure Go libraries, so it doesn't need the
// standard C++-based Protocol Buffers installation available to compile. Thus,
// since we check-in the generated code, users can build Mutagen without the
// need to install anything other than Go, and there is no need to run this
// script as part of the normal build process.
//
// If you do want to run this script (say after modifying a .proto file), you'll
// need Protocol Buffers 3+ installed (the C++ version which includes protoc -
// https://github.com/google/protobuf) and the "protoc-gen-gogo" generator
// binary installed (see the "gogo" Go code generator link for installation
// instructions). You will need to ensure that you have a version of the code
// generator installed that corresponds to the vendored runtime package,
// otherwise the generated code may be incompatible.
//
// TODO: Can this script compile the protoc-gen-gogo binary directly from the
// vendor directory (into a temporary directory) and add it to the path of the
// protoc executable? That would remove the need for manually keeping the
// generator installation in sync with the runtime that's vendored.

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"

	"github.com/pkg/errors"

	"github.com/havoc-io/mutagen/cmd"
)

var subdirectories = []struct {
	path  string
	files []string
}{
	{"session", []string{"session.proto"}},
	{"sync", []string{"cache.proto", "entry.proto"}},
	{"url", []string{"url.proto"}},
}

func main() {
	// Compute the path to the Mutagen source directory.
	_, file, _, ok := runtime.Caller(0)
	if !ok {
		cmd.Fatal(errors.New("unable to compute script path"))
	}
	mutagenSource := filepath.Dir(filepath.Dir(file))

	// Compute the vendoring path.
	vendor := filepath.Join(mutagenSource, "vendor")

	// Compute the GOPATH src directory.
	gopathSrc := filepath.Dir(filepath.Dir(filepath.Dir(mutagenSource)))

	// Process subdirectories.
	for _, s := range subdirectories {
		// Compute the subdirectory path.
		subdirectory := filepath.Join(mutagenSource, s.path)

		// Print directory information.
		fmt.Println("Processing", subdirectory)

		// Execute the Protocol Buffers compiler using the gogo generator.
		// HACK: We specify include paths so that we can reference definitions
		// between packages, but this means that we also end up needing to
		// specify -I., because for some reason the Protocol Buffers compiler is
		// too stupid to include this automatically. If you don't believe me,
		// try removing that argument and the compiler will literally print a
		// message telling you how "stupid" it is.
		// HACK: We have to use a special argument to the "gogo" generator to
		// have it point to the "gogo" timestamp implementation. The default
		// timestamp.proto file that protoc imports sets go_package to
		// github.com/golang/protobuf/ptypes/timestamp, so we just override
		// that to point to the message implementation in the "gogo"
		// repository).
		arguments := make([]string, 0, len(s.files)+1)
		arguments = append(arguments, "-I.")
		arguments = append(arguments, fmt.Sprintf("-I%s", vendor))
		arguments = append(arguments, fmt.Sprintf("-I%s", gopathSrc))
		timestampRedirect := "Mgoogle/protobuf/timestamp.proto=github.com/gogo/protobuf/types"
		arguments = append(arguments, fmt.Sprintf("--gogo_out=%s:.", timestampRedirect))
		arguments = append(arguments, s.files...)
		protoc := exec.Command("protoc", arguments...)
		protoc.Dir = subdirectory
		protoc.Stdin = os.Stdin
		protoc.Stdout = os.Stdout
		protoc.Stderr = os.Stderr
		if err := protoc.Run(); err != nil {
			cmd.Fatal(errors.Wrap(err, "protoc execution failed"))
		}
	}
}
<commit_msg>Modified generate.go to build protoc-gen-gogo from vendored sources.<commit_after>package main

// This script generates Go code from Protocol Buffers specifications in the
// Mutagen source tree. It uses the "protoc-gen-gogo" Go code generator
// (https://github.com/gogo/protobuf). It builds this generator from the
// vendored sources to ensure it matches the version of the runtime code that
// goes into the final binaries.
//
// The generated Go code depends only on pure Go libraries, so it doesn't need
// the standard C++-based Protocol Buffers installation available to compile.
// Thus, since we check-in the generated code, users can build Mutagen without
// the need to install anything other than Go, and there is no need to run this
// script as part of the normal build process.
//
// If you do want to run this script (say after modifying a .proto file), then
// you'll need the C++ version of Protocol Buffers 3+
// (https://github.com/google/protobuf) installed with the protoc compiler in
// your path.

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"

	"github.com/pkg/errors"

	"github.com/havoc-io/mutagen/cmd"
	"github.com/havoc-io/mutagen/environment"
)

var subdirectories = []struct {
	path  string
	files []string
}{
	{"session", []string{"session.proto"}},
	{"sync", []string{"cache.proto", "entry.proto"}},
	{"url", []string{"url.proto"}},
}

func main() {
	// Create a temporary directory in which we can build the generator and
	// defer its removal.
	generatorPath, err := ioutil.TempDir("", "mutagen_generate")
	if err != nil {
		cmd.Fatal(errors.New("unable to create directory for generator build"))
	}
	defer os.RemoveAll(generatorPath)

	// Print status.
	fmt.Println("Building generator")

	// Build the generator.
	generatorBuild := exec.Command(
		"go",
		"build",
		"github.com/havoc-io/mutagen/vendor/github.com/gogo/protobuf/protoc-gen-gogo",
	)
	generatorBuild.Dir = generatorPath
	generatorBuild.Stdin = os.Stdin
	generatorBuild.Stdout = os.Stdout
	generatorBuild.Stderr = os.Stderr
	if err := generatorBuild.Run(); err != nil {
		cmd.Fatal(errors.Wrap(err, "generator build failed"))
	}

	// Create an environment with the generator injected into the path.
	protocEnvironmentMap := environment.CopyCurrent()
	if existingPath := protocEnvironmentMap["PATH"]; existingPath != "" {
		protocEnvironmentMap["PATH"] = fmt.Sprintf(
			"%s%s%s",
			generatorPath,
			string(os.PathListSeparator),
			existingPath,
		)
	} else {
		protocEnvironmentMap["PATH"] = generatorPath
	}
	protocEnvironment := environment.Format(protocEnvironmentMap)

	// Compute the path to the Mutagen source directory.
	_, file, _, ok := runtime.Caller(0)
	if !ok {
		cmd.Fatal(errors.New("unable to compute script path"))
	}
	mutagenSource := filepath.Dir(filepath.Dir(file))

	// Compute the vendoring path.
	vendor := filepath.Join(mutagenSource, "vendor")

	// Compute the GOPATH src directory.
	gopathSrc := filepath.Dir(filepath.Dir(filepath.Dir(mutagenSource)))

	// Process subdirectories.
	for _, s := range subdirectories {
		// Compute the subdirectory path.
		subdirectory := filepath.Join(mutagenSource, s.path)

		// Print directory information.
		fmt.Println("Processing", subdirectory)

		// Execute the Protocol Buffers compiler using the gogo generator.
		// HACK: We specify include paths so that we can reference definitions
		// between packages, but this means that we also end up needing to
		// specify -I., because for some reason the Protocol Buffers compiler is
		// too stupid to include this automatically. If you don't believe me,
		// try removing that argument and the compiler will literally print a
		// message telling you how "stupid" it is.
		// HACK: We have to use a special argument to the "gogo" generator to
		// have it point to the "gogo" timestamp implementation. The default
		// timestamp.proto file that protoc imports sets go_package to
		// github.com/golang/protobuf/ptypes/timestamp, so we just override
		// that to point to the message implementation in the "gogo"
		// repository).
		arguments := make([]string, 0, len(s.files)+1)
		arguments = append(arguments, "-I.")
		arguments = append(arguments, fmt.Sprintf("-I%s", vendor))
		arguments = append(arguments, fmt.Sprintf("-I%s", gopathSrc))
		timestampRedirect := "Mgoogle/protobuf/timestamp.proto=github.com/gogo/protobuf/types"
		arguments = append(arguments, fmt.Sprintf("--gogo_out=%s:.", timestampRedirect))
		arguments = append(arguments, s.files...)
		protoc := exec.Command("protoc", arguments...)
		protoc.Dir = subdirectory
		protoc.Env = protocEnvironment
		protoc.Stdin = os.Stdin
		protoc.Stdout = os.Stdout
		protoc.Stderr = os.Stderr
		if err := protoc.Run(); err != nil {
			cmd.Fatal(errors.Wrap(err, "protoc execution failed"))
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"html/template"
	"log"
	"net/http"
	"strings"

	"github.com/fjukstad/kegg"
)

type InputList struct {
	Input []string
}

type Selection struct {
	Selection []string
}

type Page struct {
	Title string
	Body  []byte
}

var defaultTemplatePaths = []string{
	"templates/_base.html",
	"templates/header.html",
	"templates/navbar.html",
	"templates/footer.html",
}

var emptyTemplatePath = []string{"templates/_empty_page.html",
	"templates/header.html",
}

var indexTemplatePath = "templates/index.html"
var aboutTemplatePath = "templates/about.html"
var browserTemplatePath = "templates/browser.html"
var browserVisualizationTemplatePath = "templates/visualization.html"
var geneVisualizationTemplatePath = "templates/genevis.html"

var indexTemplate = template.Must(template.ParseFiles(
	append(defaultTemplatePaths, indexTemplatePath)...,
))
var aboutTemplate = template.Must(template.ParseFiles(
	append(defaultTemplatePaths, aboutTemplatePath)...,
))
var browserTemplate = template.Must(template.ParseFiles(
	append(defaultTemplatePaths, browserTemplatePath)...,
))

var browserVisualizationTemplate = template.Must(template.ParseFiles(
	append(defaultTemplatePaths, browserVisualizationTemplatePath)...,
))

var geneVisualizationTemplate = template.Must(template.ParseFiles(
	append(defaultTemplatePaths, geneVisualizationTemplatePath)...,
))

func renderTemplate(t *template.Template, w http.ResponseWriter,
	d interface{}) {

	// Cross domain requests in browser are a go go
	w.Header().Set("Access-Control-Allow-Origin", "*")

	// Applies template to dataobject d
	err := t.Execute(w, d)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func indexHandler(w http.ResponseWriter, r *http.Request) {
	renderTemplate(indexTemplate, w, nil)
}

func aboutHandler(w http.ResponseWriter, r *http.Request) {
	renderTemplate(aboutTemplate, w, nil)
}

const lenPath = len("/browser/")

func browserHandler(w http.ResponseWriter, r *http.Request) {
	// Get selected pathways (if any)
	path := r.URL.Path[lenPath:]
	log.Print("title of page:", path)

	// if user has selected pathways render a visualization
	if strings.Contains(path, "pathwaySelect") {
		selection := Selection{parseInput(path, "pathwaySelect")}
		renderTemplate(browserVisualizationTemplate, w, selection)
		return
	} else if strings.Contains(path, "geneSelect") {
		selection := Selection{parseInput(path, "geneSelect")}
		renderTemplate(geneVisualizationTemplate, w, selection)
		return

	}

	// if user has not selected any pathways, fetch
	// availible pathways from db and display them to the user
	ids := kegg.GetAllHumanPathways()
	//pathways := kegg.ReadablePathwayNames(ids)
	input := InputList{ids}

	renderTemplate(browserTemplate, w, input)
}

func parseInput(input, separator string) []string {
	// Remove any unwanted characters
	a := strings.Replace(input, "%3A", ":", -1)
	a = strings.Replace(a, "&", "", -1)
	a = strings.Replace(a, "=", "", -1)

	// Split into separate hsa:... strings
	b := strings.Split(a, separator)

	// Clear out first empty item
	b = b[1:len(b)]

	return b

}

func parseGeneInput(input string) []string {

	// Remove any unwanted characters
	a := strings.Replace(input, "%3A", ":", -1)
	a = strings.Replace(a, "&", "", -1)
	a = strings.Replace(a, "=", "", -1)

	// Split into separate hsa:... strings
	b := strings.Split(a, "geneSelect")

	// Clear out first empty item
	b = b[1:len(b)]

	return b

}

func main() {

	// cmd line flags
	var ip = flag.String("ip", "localhost", "ip to run on")
	var port = flag.String("port", ":8000", "port to run on")
	flag.Parse()

	address := *ip + *port

	http.HandleFunc("/", indexHandler)
	http.HandleFunc("/about", aboutHandler)
	http.HandleFunc("/browser/", browserHandler)

	// Handling requests for css files
	http.Handle("/css/", http.StripPrefix("/css/",
		http.FileServer(http.Dir("css"))))

	// Handling requests for js files
	http.Handle("/js/", http.StripPrefix("/js/",
		http.FileServer(http.Dir("js"))))

	// Handling requests for any public file
	http.Handle("/public/", http.StripPrefix("/public/",
		http.FileServer(http.Dir("public"))))

	// lib directory, d3 jquery and so on.
	http.Handle("/lib/", http.StripPrefix("/lib/",
		http.FileServer(http.Dir("lib"))))

	fmt.Println("Webserver started on", address)
	err := http.ListenAndServe(address, nil)
	if err != nil {
		log.Panic("Could not start webapp! ", err.Error())
	}

}
<commit_msg>ip ports and that<commit_after>package main

import (
	"flag"
	"fmt"
	"html/template"
	"log"
	"net/http"
	"strings"

	"github.com/fjukstad/kegg"
)

type InputList struct {
	Input []string
}

type Selection struct {
	Selection []string
}

type Page struct {
	Title string
	Body  []byte
}

var defaultTemplatePaths = []string{
	"templates/_base.html",
	"templates/header.html",
	"templates/navbar.html",
	"templates/footer.html",
}

var emptyTemplatePath = []string{"templates/_empty_page.html",
	"templates/header.html",
}

var indexTemplatePath = "templates/index.html"
var aboutTemplatePath = "templates/about.html"
var browserTemplatePath = "templates/browser.html"
var browserVisualizationTemplatePath = "templates/visualization.html"
var geneVisualizationTemplatePath = "templates/genevis.html"

var indexTemplate = template.Must(template.ParseFiles(
	append(defaultTemplatePaths, indexTemplatePath)...,
))
var aboutTemplate = template.Must(template.ParseFiles(
	append(defaultTemplatePaths, aboutTemplatePath)...,
))
var browserTemplate = template.Must(template.ParseFiles(
	append(defaultTemplatePaths, browserTemplatePath)...,
))

var browserVisualizationTemplate = template.Must(template.ParseFiles(
	append(defaultTemplatePaths, browserVisualizationTemplatePath)...,
))

var geneVisualizationTemplate = template.Must(template.ParseFiles(
	append(defaultTemplatePaths, geneVisualizationTemplatePath)...,
))

func renderTemplate(t *template.Template, w http.ResponseWriter,
	d interface{}) {

	// Cross domain requests in browser are a go go
	w.Header().Set("Access-Control-Allow-Origin", "*")

	// Applies template to dataobject d
	err := t.Execute(w, d)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func indexHandler(w http.ResponseWriter, r *http.Request) {
	renderTemplate(indexTemplate, w, nil)
}

func aboutHandler(w http.ResponseWriter, r *http.Request) {
	renderTemplate(aboutTemplate, w, nil)
}

const lenPath = len("/browser/")

func browserHandler(w http.ResponseWriter, r *http.Request) {
	// Get selected pathways (if any)
	path := r.URL.Path[lenPath:]
	log.Print("title of page:", path)

	// if user has selected pathways render a visualization
	if strings.Contains(path, "pathwaySelect") {
		selection := Selection{parseInput(path, "pathwaySelect")}
		renderTemplate(browserVisualizationTemplate, w, selection)
		return
	} else if strings.Contains(path, "geneSelect") {
		selection := Selection{parseInput(path, "geneSelect")}
		renderTemplate(geneVisualizationTemplate, w, selection)
		return

	}

	// if user has not selected any pathways, fetch
	// availible pathways from db and display them to the user
	ids := kegg.GetAllHumanPathways()
	//pathways := kegg.ReadablePathwayNames(ids)
	input := InputList{ids}

	renderTemplate(browserTemplate, w, input)
}

func parseInput(input, separator string) []string {
	// Remove any unwanted characters
	a := strings.Replace(input, "%3A", ":", -1)
	a = strings.Replace(a, "&", "", -1)
	a = strings.Replace(a, "=", "", -1)

	// Split into separate hsa:... strings
	b := strings.Split(a, separator)

	// Clear out first empty item
	b = b[1:len(b)]

	return b

}

func parseGeneInput(input string) []string {

	// Remove any unwanted characters
	a := strings.Replace(input, "%3A", ":", -1)
	a = strings.Replace(a, "&", "", -1)
	a = strings.Replace(a, "=", "", -1)

	// Split into separate hsa:... strings
	b := strings.Split(a, "geneSelect")

	// Clear out first empty item
	b = b[1:len(b)]

	return b

}

func main() {

	// cmd line flags
	var ip = flag.String("ip", "localhost", "ip to run on")
	var port = flag.String("port", ":8000", "port to run on")
	flag.Parse()

	address := *ip + *port
	address = *port

	http.HandleFunc("/", indexHandler)
	http.HandleFunc("/about", aboutHandler)
	http.HandleFunc("/browser/", browserHandler)

	// Handling requests for css files
	http.Handle("/css/", http.StripPrefix("/css/",
		http.FileServer(http.Dir("css"))))

	// Handling requests for js files
	http.Handle("/js/", http.StripPrefix("/js/",
		http.FileServer(http.Dir("js"))))

	// Handling requests for any public file
	http.Handle("/public/", http.StripPrefix("/public/",
		http.FileServer(http.Dir("public"))))

	// lib directory, d3 jquery and so on.
	http.Handle("/lib/", http.StripPrefix("/lib/",
		http.FileServer(http.Dir("lib"))))

	fmt.Println("Webserver started on", address)
	err := http.ListenAndServe(address, nil)
	if err != nil {
		log.Panic("Could not start webapp! ", err.Error())
	}

}
<|endoftext|>


<commit_before>package common

import (
	"errors"
	"fmt"
	"github.com/mitchellh/goamz/ec2"
	"github.com/mitchellh/multistep"
	"log"
	"time"
)

// StateRefreshFunc is a function type used for StateChangeConf that is
// responsible for refreshing the item being watched for a state change.
//
// It returns three results. `result` is any object that will be returned
// as the final object after waiting for state change. This allows you to
// return the final updated object, for example an EC2 instance after refreshing
// it.
//
// `state` is the latest state of that object. And `err` is any error that
// may have happened while refreshing the state.
type StateRefreshFunc func() (result interface{}, state string, err error)

// StateChangeConf is the configuration struct used for `WaitForState`.
type StateChangeConf struct {
	Conn      *ec2.EC2
	Pending   []string
	Refresh   StateRefreshFunc
	StepState map[string]interface{}
	Target    string
}

// InstanceStateRefreshFunc returns a StateRefreshFunc that is used to watch
// an EC2 instance.
func InstanceStateRefreshFunc(conn *ec2.EC2, i *ec2.Instance) StateRefreshFunc {
	return func() (interface{}, string, error) {
		resp, err := conn.Instances([]string{i.InstanceId}, ec2.NewFilter())
		if err != nil {
			return nil, "", err
		}

		i = &resp.Reservations[0].Instances[0]
		return i, i.State.Name, nil
	}
}

// WaitForState watches an object and waits for it to achieve a certain
// state.
func WaitForState(conf *StateChangeConf) (i interface{}, err error) {
	log.Printf("Waiting for state to become: %s", conf.Target)

	for {
		var currentState string
		i, currentState, err = conf.Refresh()
		if err != nil {
			return
		}

		if currentState == conf.Target {
			return
		}

		if conf.StepState != nil {
			if _, ok := conf.StepState[multistep.StateCancelled]; ok {
				return nil, errors.New("interrupted")
			}
		}

		found := false
		for _, allowed := range conf.Pending {
			if currentState == allowed {
				found = true
				break
			}
		}

		if !found {
			fmt.Errorf("unexpected state '%s', wanted target '%s'", currentState, conf.Target)
			return
		}

		time.Sleep(2 * time.Second)
	}

	return
}
<commit_msg>Log error messages from goamz.<commit_after>package common

import (
	"errors"
	"fmt"
	"github.com/mitchellh/goamz/ec2"
	"github.com/mitchellh/multistep"
	"log"
	"time"
)

// StateRefreshFunc is a function type used for StateChangeConf that is
// responsible for refreshing the item being watched for a state change.
//
// It returns three results. `result` is any object that will be returned
// as the final object after waiting for state change. This allows you to
// return the final updated object, for example an EC2 instance after refreshing
// it.
//
// `state` is the latest state of that object. And `err` is any error that
// may have happened while refreshing the state.
type StateRefreshFunc func() (result interface{}, state string, err error)

// StateChangeConf is the configuration struct used for `WaitForState`.
type StateChangeConf struct {
	Conn      *ec2.EC2
	Pending   []string
	Refresh   StateRefreshFunc
	StepState map[string]interface{}
	Target    string
}

// InstanceStateRefreshFunc returns a StateRefreshFunc that is used to watch
// an EC2 instance.
func InstanceStateRefreshFunc(conn *ec2.EC2, i *ec2.Instance) StateRefreshFunc {
	return func() (interface{}, string, error) {
		resp, err := conn.Instances([]string{i.InstanceId}, ec2.NewFilter())
		if err != nil {
			log.Printf("Error on InstanceStateRefresh: %s", err)
			return nil, "", err
		}

		i = &resp.Reservations[0].Instances[0]
		return i, i.State.Name, nil
	}
}

// WaitForState watches an object and waits for it to achieve a certain
// state.
func WaitForState(conf *StateChangeConf) (i interface{}, err error) {
	log.Printf("Waiting for state to become: %s", conf.Target)

	for {
		var currentState string
		i, currentState, err = conf.Refresh()
		if err != nil {
			return
		}

		if currentState == conf.Target {
			return
		}

		if conf.StepState != nil {
			if _, ok := conf.StepState[multistep.StateCancelled]; ok {
				return nil, errors.New("interrupted")
			}
		}

		found := false
		for _, allowed := range conf.Pending {
			if currentState == allowed {
				found = true
				break
			}
		}

		if !found {
			fmt.Errorf("unexpected state '%s', wanted target '%s'", currentState, conf.Target)
			return
		}

		time.Sleep(2 * time.Second)
	}

	return
}
<|endoftext|>