<commit_before>package poller

import (
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/swf"
	"github.com/juju/errors"
	"github.com/pborman/uuid"
	. "github.com/sclasen/swfsm/log"
	. "github.com/sclasen/swfsm/sugar"
)

// SWFOps is the subset of the swf.SWF api used by pollers
type DecisionOps interface {
	PollForDecisionTaskPages(*swf.PollForDecisionTaskInput, func(*swf.PollForDecisionTaskOutput, bool) bool) error
}

type ActivityOps interface {
	PollForActivityTask(req *swf.PollForActivityTaskInput) (resp *swf.PollForActivityTaskOutput, err error)
}

// NewDecisionTaskPoller returns a DecisionTaskPoller whick can be used to poll the given task list.
func NewDecisionTaskPoller(dwc DecisionOps, domain string, identity string, taskList string) *DecisionTaskPoller {
	return &DecisionTaskPoller{
		client:   dwc,
		Domain:   domain,
		Identity: identity,
		TaskList: taskList,
	}
}

// DecisionTaskPoller polls a given task list in a domain for decision tasks.
type DecisionTaskPoller struct {
	client   DecisionOps
	Identity string
	Domain   string
	TaskList string
}

// Poll polls the task list for a task. If there is no task available, nil is
// returned. If an error is encountered, no task is returned.
func (p *DecisionTaskPoller) Poll(taskReady func(*swf.PollForDecisionTaskOutput) bool) (*swf.PollForDecisionTaskOutput, error) {
	var (
		resp   *swf.PollForDecisionTaskOutput
		page   int
		pollId = uuid.New()
	)

	eachPage := func(out *swf.PollForDecisionTaskOutput, _ bool) bool {
		page++

		var (
			firstEventId *int64
			lastEventId  *int64
			workflowId   string
		)

		if len(out.Events) > 0 {
			firstEventId = out.Events[0].EventId
			lastEventId = out.Events[len(out.Events)-1].EventId
		}

		if out.WorkflowExecution != nil {
			workflowId = LS(out.WorkflowExecution.WorkflowId)
		} else {
			workflowId = "no-workflow-execution"
		}

		Log.Printf("component=DecisionTaskPoller at=decision-task-page poll-id=%q task-list=%q workflow=%q page=%d "+
			"PreviousStartedEventId=%s StartedEventId=%s NumEvents=%d FirstEventId=%s LastEventId=%s",
			pollId, p.TaskList, workflowId, page,
			LL(out.PreviousStartedEventId), LL(out.StartedEventId), len(out.Events), LL(firstEventId), LL(lastEventId))

		if resp == nil {
			resp = out
		} else {
			resp.Events = append(resp.Events, out.Events...)
		}

		return !taskReady(resp)
	}

	err := p.client.PollForDecisionTaskPages(&swf.PollForDecisionTaskInput{
		Domain:       aws.String(p.Domain),
		Identity:     aws.String(p.Identity),
		ReverseOrder: aws.Bool(true),
		TaskList:     &swf.TaskList{Name: aws.String(p.TaskList)},
	}, eachPage)

	if err != nil {
		Log.Printf("component=DecisionTaskPoller poll-id=%q task-list=%q at=error error=%s",
			pollId, p.TaskList, err.Error())
		return nil, errors.Trace(err)
	}
	if resp != nil && resp.TaskToken != nil {
		Log.Printf("component=DecisionTaskPoller poll-id=%q at=decision-task-received task-list=%q workflow=%q",
			pollId, p.TaskList, LS(resp.WorkflowExecution.WorkflowId))
		p.logTaskLatency(resp)
		return resp, nil
	}
	Log.Printf("component=DecisionTaskPoller at=decision-task-empty-response poll-id=%q task-list=%q", pollId, p.TaskList)
	return nil, nil
}

// PollUntilShutdownBy will poll until signaled to shutdown by the PollerShutdownManager. this func blocks, so run it in a goroutine if necessary.
// The implementation calls Poll() and invokes the callback whenever a valid PollForDecisionTaskResponse is received.
func (p *DecisionTaskPoller) PollUntilShutdownBy(mgr *ShutdownManager, pollerName string, onTask func(*swf.PollForDecisionTaskOutput), taskReady func(*swf.PollForDecisionTaskOutput) bool) {
	stop := make(chan bool, 1)
	stopAck := make(chan bool, 1)
	mgr.Register(pollerName, stop, stopAck)
	for {
		select {
		case <-stop:
			Log.Printf("component=DecisionTaskPoller fn=PollUntilShutdownBy at=received-stop action=shutting-down poller=%s task-list=%q", pollerName, p.TaskList)
			stopAck <- true
			return
		default:
			task, err := p.Poll(taskReady)
			if err != nil {
				Log.Printf("component=DecisionTaskPoller fn=PollUntilShutdownBy at=poll-err poller=%s task-list=%q error=%q", pollerName, p.TaskList, err)
				continue
			}
			if task == nil {
				Log.Printf("component=DecisionTaskPoller fn=PollUntilShutdownBy at=poll-no-task poller=%s task-list=%q", pollerName, p.TaskList)
				continue
			}
			onTask(task)
		}
	}
}

func (p *DecisionTaskPoller) logTaskLatency(resp *swf.PollForDecisionTaskOutput) {
	for _, e := range resp.Events {
		if e.EventId == resp.StartedEventId {
			elapsed := time.Since(*e.EventTimestamp)
			Log.Printf("component=DecisionTaskPoller at=decision-task-latency latency=%s workflow=%s", elapsed, LS(resp.WorkflowType.Name))
		}
	}
}

// NewActivityTaskPoller returns an ActivityTaskPoller.
func NewActivityTaskPoller(awc ActivityOps, domain string, identity string, taskList string) *ActivityTaskPoller {
	return &ActivityTaskPoller{
		client:   awc,
		Domain:   domain,
		Identity: identity,
		TaskList: taskList,
	}
}

// ActivityTaskPoller polls a given task list in a domain for activity tasks, and sends tasks on its Tasks channel.
type ActivityTaskPoller struct {
	client   ActivityOps
	Identity string
	Domain   string
	TaskList string
}

// Poll polls the task list for a task. If there is no task, nil is returned.
// If an error is encountered, no task is returned.
func (p *ActivityTaskPoller) Poll() (*swf.PollForActivityTaskOutput, error) {
	resp, err := p.client.PollForActivityTask(&swf.PollForActivityTaskInput{
		Domain:   aws.String(p.Domain),
		Identity: aws.String(p.Identity),
		TaskList: &swf.TaskList{Name: aws.String(p.TaskList)},
	})
	if err != nil {
		Log.Printf("component=ActivityTaskPoller at=error error=%s", err.Error())
		return nil, errors.Trace(err)
	}
	if resp.TaskToken != nil {
		Log.Printf("component=ActivityTaskPoller at=activity-task-received activity=%s", LS(resp.ActivityType.Name))
		return resp, nil
	}
	Log.Println("component=ActivityTaskPoller at=activity-task-empty-response")
	return nil, nil
}

// PollUntilShutdownBy will poll until signaled to shutdown by the ShutdownManager. this func blocks, so run it in a goroutine if necessary.
// The implementation calls Poll() and invokes the callback whenever a valid PollForActivityTaskResponse is received.
func (p *ActivityTaskPoller) PollUntilShutdownBy(mgr *ShutdownManager, pollerName string, onTask func(*swf.PollForActivityTaskOutput)) {
	stop := make(chan bool, 1)
	stopAck := make(chan bool, 1)
	mgr.Register(pollerName, stop, stopAck)
	for {
		select {
		case <-stop:
			Log.Printf("component=ActivityTaskPoller fn=PollUntilShutdownBy at=received-stop action=shutting-down poller=%s task-list=%q", pollerName, p.TaskList)
			stopAck <- true
			return
		default:
			task, err := p.Poll()
			if err != nil {
				Log.Printf("component=ActivityTaskPoller fn=PollUntilShutdownBy at=poll-err poller=%s task-list=%q error=%q", pollerName, p.TaskList, err)
				continue
			}
			if task == nil {
				Log.Printf("component=ActivityTaskPoller fn=PollUntilShutdownBy at=poll-no-task poller=%s task-list=%q", pollerName, p.TaskList)
				continue
			}
			onTask(task)
		}
	}
}

// ShutdownManager facilitates cleanly shutting down pollers when the application decides to exit. When StopPollers() is called it will
// send to each of the stopChan that have been registered, then recieve from each of the ackChan that have been registered. At this point StopPollers() returns.
type ShutdownManager struct {
	registeredPollers map[string]*registeredPoller
}

type registeredPoller struct {
	name           string
	stopChannel    chan bool
	stopAckChannel chan bool
}

// NewShutdownManager creates a ShutdownManager
func NewShutdownManager() *ShutdownManager {

	mgr := &ShutdownManager{
		registeredPollers: make(map[string]*registeredPoller),
	}

	return mgr

}

//StopPollers blocks until it is able to stop all the registered pollers, which can take up to 60 seconds.
func (p *ShutdownManager) StopPollers() {
	Log.Printf("component=PollerShutdownManager at=stop-pollers")
	for _, r := range p.registeredPollers {
		Log.Printf("component=PollerShutdownManager at=sending-stop name=%s", r.name)
		r.stopChannel <- true
	}
	for _, r := range p.registeredPollers {
		Log.Printf("component=PollerShutdownManager at=awaiting-stop-ack name=%s", r.name)
		<-r.stopAckChannel
		Log.Printf("component=PollerShutdownManager at=stop-ack name=%s", r.name)
	}
}

// Register registers a named pair of channels to the shutdown manager. Buffered channels please!
func (p *ShutdownManager) Register(name string, stopChan chan bool, ackChan chan bool) {
	p.registeredPollers[name] = &registeredPoller{name, stopChan, ackChan}
}

// Deregister removes a registered pair of channels from the shutdown manager.
func (p *ShutdownManager) Deregister(name string) {
	delete(p.registeredPollers, name)
}
<commit_msg>guard registered pollers map with mutex<commit_after>package poller

import (
	"sync"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/swf"
	"github.com/juju/errors"
	"github.com/pborman/uuid"
	. "github.com/sclasen/swfsm/log"
	. "github.com/sclasen/swfsm/sugar"
)

// SWFOps is the subset of the swf.SWF api used by pollers
type DecisionOps interface {
	PollForDecisionTaskPages(*swf.PollForDecisionTaskInput, func(*swf.PollForDecisionTaskOutput, bool) bool) error
}

type ActivityOps interface {
	PollForActivityTask(req *swf.PollForActivityTaskInput) (resp *swf.PollForActivityTaskOutput, err error)
}

// NewDecisionTaskPoller returns a DecisionTaskPoller whick can be used to poll the given task list.
func NewDecisionTaskPoller(dwc DecisionOps, domain string, identity string, taskList string) *DecisionTaskPoller {
	return &DecisionTaskPoller{
		client:   dwc,
		Domain:   domain,
		Identity: identity,
		TaskList: taskList,
	}
}

// DecisionTaskPoller polls a given task list in a domain for decision tasks.
type DecisionTaskPoller struct {
	client   DecisionOps
	Identity string
	Domain   string
	TaskList string
}

// Poll polls the task list for a task. If there is no task available, nil is
// returned. If an error is encountered, no task is returned.
func (p *DecisionTaskPoller) Poll(taskReady func(*swf.PollForDecisionTaskOutput) bool) (*swf.PollForDecisionTaskOutput, error) {
	var (
		resp   *swf.PollForDecisionTaskOutput
		page   int
		pollId = uuid.New()
	)

	eachPage := func(out *swf.PollForDecisionTaskOutput, _ bool) bool {
		page++

		var (
			firstEventId *int64
			lastEventId  *int64
			workflowId   string
		)

		if len(out.Events) > 0 {
			firstEventId = out.Events[0].EventId
			lastEventId = out.Events[len(out.Events)-1].EventId
		}

		if out.WorkflowExecution != nil {
			workflowId = LS(out.WorkflowExecution.WorkflowId)
		} else {
			workflowId = "no-workflow-execution"
		}

		Log.Printf("component=DecisionTaskPoller at=decision-task-page poll-id=%q task-list=%q workflow=%q page=%d "+
			"PreviousStartedEventId=%s StartedEventId=%s NumEvents=%d FirstEventId=%s LastEventId=%s",
			pollId, p.TaskList, workflowId, page,
			LL(out.PreviousStartedEventId), LL(out.StartedEventId), len(out.Events), LL(firstEventId), LL(lastEventId))

		if resp == nil {
			resp = out
		} else {
			resp.Events = append(resp.Events, out.Events...)
		}

		return !taskReady(resp)
	}

	err := p.client.PollForDecisionTaskPages(&swf.PollForDecisionTaskInput{
		Domain:       aws.String(p.Domain),
		Identity:     aws.String(p.Identity),
		ReverseOrder: aws.Bool(true),
		TaskList:     &swf.TaskList{Name: aws.String(p.TaskList)},
	}, eachPage)

	if err != nil {
		Log.Printf("component=DecisionTaskPoller poll-id=%q task-list=%q at=error error=%s",
			pollId, p.TaskList, err.Error())
		return nil, errors.Trace(err)
	}
	if resp != nil && resp.TaskToken != nil {
		Log.Printf("component=DecisionTaskPoller poll-id=%q at=decision-task-received task-list=%q workflow=%q",
			pollId, p.TaskList, LS(resp.WorkflowExecution.WorkflowId))
		p.logTaskLatency(resp)
		return resp, nil
	}
	Log.Printf("component=DecisionTaskPoller at=decision-task-empty-response poll-id=%q task-list=%q", pollId, p.TaskList)
	return nil, nil
}

// PollUntilShutdownBy will poll until signaled to shutdown by the PollerShutdownManager. this func blocks, so run it in a goroutine if necessary.
// The implementation calls Poll() and invokes the callback whenever a valid PollForDecisionTaskResponse is received.
func (p *DecisionTaskPoller) PollUntilShutdownBy(mgr *ShutdownManager, pollerName string, onTask func(*swf.PollForDecisionTaskOutput), taskReady func(*swf.PollForDecisionTaskOutput) bool) {
	stop := make(chan bool, 1)
	stopAck := make(chan bool, 1)
	mgr.Register(pollerName, stop, stopAck)
	for {
		select {
		case <-stop:
			Log.Printf("component=DecisionTaskPoller fn=PollUntilShutdownBy at=received-stop action=shutting-down poller=%s task-list=%q", pollerName, p.TaskList)
			stopAck <- true
			return
		default:
			task, err := p.Poll(taskReady)
			if err != nil {
				Log.Printf("component=DecisionTaskPoller fn=PollUntilShutdownBy at=poll-err poller=%s task-list=%q error=%q", pollerName, p.TaskList, err)
				continue
			}
			if task == nil {
				Log.Printf("component=DecisionTaskPoller fn=PollUntilShutdownBy at=poll-no-task poller=%s task-list=%q", pollerName, p.TaskList)
				continue
			}
			onTask(task)
		}
	}
}

func (p *DecisionTaskPoller) logTaskLatency(resp *swf.PollForDecisionTaskOutput) {
	for _, e := range resp.Events {
		if e.EventId == resp.StartedEventId {
			elapsed := time.Since(*e.EventTimestamp)
			Log.Printf("component=DecisionTaskPoller at=decision-task-latency latency=%s workflow=%s", elapsed, LS(resp.WorkflowType.Name))
		}
	}
}

// NewActivityTaskPoller returns an ActivityTaskPoller.
func NewActivityTaskPoller(awc ActivityOps, domain string, identity string, taskList string) *ActivityTaskPoller {
	return &ActivityTaskPoller{
		client:   awc,
		Domain:   domain,
		Identity: identity,
		TaskList: taskList,
	}
}

// ActivityTaskPoller polls a given task list in a domain for activity tasks, and sends tasks on its Tasks channel.
type ActivityTaskPoller struct {
	client   ActivityOps
	Identity string
	Domain   string
	TaskList string
}

// Poll polls the task list for a task. If there is no task, nil is returned.
// If an error is encountered, no task is returned.
func (p *ActivityTaskPoller) Poll() (*swf.PollForActivityTaskOutput, error) {
	resp, err := p.client.PollForActivityTask(&swf.PollForActivityTaskInput{
		Domain:   aws.String(p.Domain),
		Identity: aws.String(p.Identity),
		TaskList: &swf.TaskList{Name: aws.String(p.TaskList)},
	})
	if err != nil {
		Log.Printf("component=ActivityTaskPoller at=error error=%s", err.Error())
		return nil, errors.Trace(err)
	}
	if resp.TaskToken != nil {
		Log.Printf("component=ActivityTaskPoller at=activity-task-received activity=%s", LS(resp.ActivityType.Name))
		return resp, nil
	}
	Log.Println("component=ActivityTaskPoller at=activity-task-empty-response")
	return nil, nil
}

// PollUntilShutdownBy will poll until signaled to shutdown by the ShutdownManager. this func blocks, so run it in a goroutine if necessary.
// The implementation calls Poll() and invokes the callback whenever a valid PollForActivityTaskResponse is received.
func (p *ActivityTaskPoller) PollUntilShutdownBy(mgr *ShutdownManager, pollerName string, onTask func(*swf.PollForActivityTaskOutput)) {
	stop := make(chan bool, 1)
	stopAck := make(chan bool, 1)
	mgr.Register(pollerName, stop, stopAck)
	for {
		select {
		case <-stop:
			Log.Printf("component=ActivityTaskPoller fn=PollUntilShutdownBy at=received-stop action=shutting-down poller=%s task-list=%q", pollerName, p.TaskList)
			stopAck <- true
			return
		default:
			task, err := p.Poll()
			if err != nil {
				Log.Printf("component=ActivityTaskPoller fn=PollUntilShutdownBy at=poll-err poller=%s task-list=%q error=%q", pollerName, p.TaskList, err)
				continue
			}
			if task == nil {
				Log.Printf("component=ActivityTaskPoller fn=PollUntilShutdownBy at=poll-no-task poller=%s task-list=%q", pollerName, p.TaskList)
				continue
			}
			onTask(task)
		}
	}
}

// ShutdownManager facilitates cleanly shutting down pollers when the application decides to exit. When StopPollers() is called it will
// send to each of the stopChan that have been registered, then recieve from each of the ackChan that have been registered. At this point StopPollers() returns.
type ShutdownManager struct {
	rpMu              sync.Mutex // protects registeredPollers
	registeredPollers map[string]*registeredPoller
}

type registeredPoller struct {
	name           string
	stopChannel    chan bool
	stopAckChannel chan bool
}

// NewShutdownManager creates a ShutdownManager
func NewShutdownManager() *ShutdownManager {

	mgr := &ShutdownManager{
		registeredPollers: make(map[string]*registeredPoller),
	}

	return mgr

}

//StopPollers blocks until it is able to stop all the registered pollers, which can take up to 60 seconds.
func (p *ShutdownManager) StopPollers() {
	p.rpMu.Lock()
	defer p.rpMu.Unlock()

	Log.Printf("component=PollerShutdownManager at=stop-pollers")
	for _, r := range p.registeredPollers {
		Log.Printf("component=PollerShutdownManager at=sending-stop name=%s", r.name)
		r.stopChannel <- true
	}
	for _, r := range p.registeredPollers {
		Log.Printf("component=PollerShutdownManager at=awaiting-stop-ack name=%s", r.name)
		<-r.stopAckChannel
		Log.Printf("component=PollerShutdownManager at=stop-ack name=%s", r.name)
	}
}

// Register registers a named pair of channels to the shutdown manager. Buffered channels please!
func (p *ShutdownManager) Register(name string, stopChan chan bool, ackChan chan bool) {
	p.rpMu.Lock()
	defer p.rpMu.Unlock()
	p.registeredPollers[name] = &registeredPoller{name, stopChan, ackChan}
}

// Deregister removes a registered pair of channels from the shutdown manager.
func (p *ShutdownManager) Deregister(name string) {
	p.rpMu.Lock()
	defer p.rpMu.Unlock()
	delete(p.registeredPollers, name)
}
<|endoftext|>


<commit_before>package fetcher

import (
	"os"
	"sync"

	"github.com/cloudfoundry-incubator/bbs"
	"github.com/cloudfoundry-incubator/bbs/models"
	"github.com/cloudfoundry/noaa"
)

type Fetcher interface {
	Fetch() (Data, error)
}

type fetcher struct {
	bbsClient  bbs.Client
	noaaClient *noaa.Consumer
}

func NewFetcher(bbsClient bbs.Client, noaaClient *noaa.Consumer) Fetcher {
	return &fetcher{
		bbsClient:  bbsClient,
		noaaClient: noaaClient,
	}
}

func (f *fetcher) Fetch() (Data, error) {
	domains, err := f.bbsClient.Domains()
	if err != nil {
		return Data{}, err
	}

	tasks, err := f.bbsClient.Tasks()
	if err != nil {
		return Data{}, err
	}

	lrps, err := f.fetchLRPs()
	if err != nil {
		return Data{}, err
	}

	return Data{
		Domains: domains,
		Tasks:   tasks,
		LRPs:    lrps,
	}, nil
}

func (f *fetcher) fetchLRPs() (map[string]*LRP, error) {
	lrps := map[string]*LRP{}

	desiredLRPs, err := f.bbsClient.DesiredLRPs(models.DesiredLRPFilter{})
	if err != nil {
		return nil, err
	}

	for _, desiredLRP := range desiredLRPs {
		lrps[desiredLRP.ProcessGuid] = &LRP{Desired: desiredLRP}
	}

	actualLRPGroups, err := f.bbsClient.ActualLRPGroups(models.ActualLRPFilter{})
	if err != nil {
		return nil, err
	}

	for _, actualLRPGroup := range actualLRPGroups {
		actualLRP, evacuating := actualLRPGroup.Resolve()

		lrp, ok := lrps[actualLRP.ProcessGuid]
		if !ok {
			continue
		}

		actual := Actual{ActualLRP: actualLRP, Evacuating: evacuating}
		lrp.Actuals = append(lrp.Actuals, &actual)
	}

	wg := sync.WaitGroup{}
	authToken := os.Getenv("OAUTH_TOKEN")
	for _, lrp := range lrps {
		wg.Add(1)
		go func() {
			defer wg.Done()
			containerMetrics, err := f.noaaClient.ContainerMetrics(lrp.Desired.LogGuid, authToken)
			if err != nil {
				return
			}

			for _, metrics := range containerMetrics {
				for _, actual := range lrp.Actuals {
					if actual.ActualLRP.Index == metrics.GetInstanceIndex() {
						containerMetrics := ContainerMetrics{
							CPU:    metrics.GetCpuPercentage(),
							Memory: metrics.GetMemoryBytes(),
							Disk:   metrics.GetDiskBytes(),
						}

						actual.Metrics = containerMetrics
					}
				}
			}
		}()
	}

	wg.Wait()

	return lrps, nil
}
<commit_msg>fix jim<commit_after>package fetcher

import (
	"os"
	"sync"

	"github.com/cloudfoundry-incubator/bbs"
	"github.com/cloudfoundry-incubator/bbs/models"
	"github.com/cloudfoundry/noaa"
)

type Fetcher interface {
	Fetch() (Data, error)
}

type fetcher struct {
	bbsClient  bbs.Client
	noaaClient *noaa.Consumer
}

func NewFetcher(bbsClient bbs.Client, noaaClient *noaa.Consumer) Fetcher {
	return &fetcher{
		bbsClient:  bbsClient,
		noaaClient: noaaClient,
	}
}

func (f *fetcher) Fetch() (Data, error) {
	domains, err := f.bbsClient.Domains()
	if err != nil {
		return Data{}, err
	}

	tasks, err := f.bbsClient.Tasks()
	if err != nil {
		return Data{}, err
	}

	lrps, err := f.fetchLRPs()
	if err != nil {
		return Data{}, err
	}

	return Data{
		Domains: domains,
		Tasks:   tasks,
		LRPs:    lrps,
	}, nil
}

func (f *fetcher) fetchLRPs() (map[string]*LRP, error) {
	lrps := map[string]*LRP{}

	desiredLRPs, err := f.bbsClient.DesiredLRPs(models.DesiredLRPFilter{})
	if err != nil {
		return nil, err
	}

	for _, desiredLRP := range desiredLRPs {
		lrps[desiredLRP.ProcessGuid] = &LRP{Desired: desiredLRP}
	}

	actualLRPGroups, err := f.bbsClient.ActualLRPGroups(models.ActualLRPFilter{})
	if err != nil {
		return nil, err
	}

	for _, actualLRPGroup := range actualLRPGroups {
		actualLRP, evacuating := actualLRPGroup.Resolve()

		lrp, ok := lrps[actualLRP.ProcessGuid]
		if !ok {
			continue
		}

		actual := Actual{ActualLRP: actualLRP, Evacuating: evacuating}
		lrp.Actuals = append(lrp.Actuals, &actual)
	}

	wg := sync.WaitGroup{}
	authToken := os.Getenv("OAUTH_TOKEN")
	for _, lrp := range lrps {
		lrp := lrp
		wg.Add(1)
		go func() {
			defer wg.Done()
			containerMetrics, err := f.noaaClient.ContainerMetrics(lrp.Desired.LogGuid, authToken)
			if err != nil {
				return
			}

			for _, metrics := range containerMetrics {
				for _, actual := range lrp.Actuals {
					if actual.ActualLRP.Index == metrics.GetInstanceIndex() {
						containerMetrics := ContainerMetrics{
							CPU:    metrics.GetCpuPercentage(),
							Memory: metrics.GetMemoryBytes(),
							Disk:   metrics.GetDiskBytes(),
						}

						actual.Metrics = containerMetrics
					}
				}
			}
		}()
	}

	wg.Wait()

	return lrps, nil
}
<|endoftext|>


<commit_before>package commands

import (
	"bytes"
	"fmt"
	"go/build"
	"os"
	"runtime"
	"strconv"

	"nvim-go/gb"
	"nvim-go/nvim"

	log "github.com/Sirupsen/logrus"
	"github.com/garyburd/neovim-go/vim"
	"github.com/garyburd/neovim-go/vim/plugin"
	"github.com/rogpeppe/godef/go/ast"
	"github.com/rogpeppe/godef/go/parser"
	"github.com/rogpeppe/godef/go/printer"
	"github.com/rogpeppe/godef/go/token"
	"github.com/rogpeppe/godef/go/types"
)

func init() {
	plugin.HandleCommand("Godef", &plugin.CommandOptions{Range: "%", Eval: "expand('%:p:h:h:h:h')"}, Def)
}

func Def(v *vim.Vim, r [2]int, file string) error {
	defer gb.WithGoBuildForPath(file)()

	b, err := v.CurrentBuffer()
	if err != nil {
		return v.WriteErr("cannot get current buffer")
	}

	buf, err := v.BufferLineSlice(b, 0, -1, true, true)
	if err != nil {
		return err
	}
	src := bytes.Join(buf, []byte{'\n'})

	filename, err := v.BufferName(b)
	if err != nil {
		return v.WriteErr("cannot get current buffer name")
	}

	searchpos, err := nvim.ByteOffset(v)
	if err != nil {
		return v.WriteErr("cannot get current buffer byte offset")
	}

	pkgScope := ast.NewScope(parser.Universe)
	f, err := parser.ParseFile(types.FileSet, filename, src, 0, pkgScope)
	if f == nil {
		nvim.Echomsg(v, "cannot parse %s: %v", filename, err)
	}

	// var o ast.Node
	// switch {
	// case searchpos >= 0:
	o := findIdentifier(v, f, searchpos)

	// default:
	// 	fmt.Fprintf(os.Stderr, "no expression or offset specified\n")
	// 	flag.Usage()
	// 	os.Exit(2)
	// }
	switch e := o.(type) {
	case *ast.ImportSpec:
		path := importPath(e)
		pkg, err := build.Default.Import(path, "", build.FindOnly)
		if err != nil {
			fail("error finding import path for %s: %s", path, err)
		}
		fmt.Println(pkg.Dir)
	case ast.Expr:
		// add declarations from other files in the local package and try again
		// pkg, err := parseLocalPackage(filename, f, pkgScope)
		// if pkg == nil {
		// 	fmt.Printf("parseLocalPackage error: %v\n", err)
		// }
		// if flag.NArg() > 0 {
		// 	// Reading declarations in other files might have
		// 	// resolved the original expression.
		// 	e = parseExpr(f.Scope, flag.Arg(0)).(ast.Expr)
		// }
		if obj, _ := types.ExprType(e, types.DefaultImporter); obj != nil {
			out := types.FileSet.Position(types.DeclPos(obj))
			// done(obj, typ)
			// out.Column--
			log.Debugln("def out.String():", out.String())
			log.Debugln("def out.Filename:", out.Filename)
			log.Debugln("def out.Line:", out.Line)
			log.Debugln("def out.Column:", out.Column)
			log.Debugln("def out.Offset:", out.Offset)
			log.Debugln("def out.IsValid():", out.IsValid())

			v.Command("silent lexpr '" + fmt.Sprintf("%v", out) + "'")
			w, err := v.CurrentWindow()
			if err != nil {
				log.Debugln(err)
			}
			v.SetWindowCursor(w, [2]int{out.Line, out.Column - 1})
			// v.Command("silent lgetexpr '" + fmt.Sprintf("%v", out) + "'")
			// v.Command("sil ll 1")
			// v.Feedkeys("zz", "normal", false)
		}
		// fail("no declaration found for %v", pretty{e})
	}
	return nil
}

func fail(s string, a ...interface{}) {
	fmt.Fprint(os.Stderr, "godef: "+fmt.Sprintf(s, a...)+"\n")
	os.Exit(2)
}

func importPath(n *ast.ImportSpec) string {
	p, err := strconv.Unquote(n.Path.Value)
	if err != nil {
		fail("invalid string literal %q in ast.ImportSpec", n.Path.Value)
	}
	return p
}

// findIdentifier looks for an identifier at byte-offset searchpos
// inside the parsed source represented by node.
// If it is part of a selector expression, it returns
// that expression rather than the identifier itself.
//
// As a special case, if it finds an import
// spec, it returns ImportSpec.
//
func findIdentifier(v *vim.Vim, f *ast.File, searchpos int) ast.Node {
	ec := make(chan ast.Node)
	found := func(startPos, endPos token.Pos) bool {
		start := types.FileSet.Position(startPos).Offset
		end := start + int(endPos-startPos)
		return start <= searchpos && searchpos <= end
	}
	go func() {
		var visit func(ast.Node) bool
		visit = func(n ast.Node) bool {
			var startPos token.Pos
			switch n := n.(type) {
			default:
				return true
			case *ast.Ident:
				startPos = n.NamePos
			case *ast.SelectorExpr:
				startPos = n.Sel.NamePos
			case *ast.ImportSpec:
				startPos = n.Pos()
			case *ast.StructType:
				// If we find an anonymous bare field in a
				// struct type, its definition points to itself,
				// but we actually want to go elsewhere,
				// so assume (dubiously) that the expression
				// works globally and return a new node for it.
				for _, field := range n.Fields.List {
					if field.Names != nil {
						continue
					}
					t := field.Type
					if pt, ok := field.Type.(*ast.StarExpr); ok {
						t = pt.X
					}
					if id, ok := t.(*ast.Ident); ok {
						if found(id.NamePos, id.End()) {
							ec <- parseExpr(f.Scope, id.Name)
							runtime.Goexit()
						}
					}
				}
				return true
			}
			if found(startPos, n.End()) {
				ec <- n
				runtime.Goexit()
			}
			return true
		}
		ast.Walk(FVisitor(visit), f)
		ec <- nil
	}()
	ev := <-ec
	if ev == nil {
		log.Debugln("def: no identifier found")
		nvim.Echomsg(v, "def: no identifier found")
		// fail("no identifier found")
	}
	return ev
}

// type orderedObjects []*ast.Object
//
// func (o orderedObjects) Less(i, j int) bool { return o[i].Name < o[j].Name }
// func (o orderedObjects) Len() int           { return len(o) }
// func (o orderedObjects) Swap(i, j int)      { o[i], o[j] = o[j], o[i] }

// func done(v *vim.Vim, obj *ast.Object, typ types.Type) token.Position {
// defer os.Exit(0)
// pos := types.FileSet.Position(types.DeclPos(obj))
// fmt.Printf("%v\n", pos)
// if typ.Kind == ast.Bad {
// 	return []
// }
// fmt.Printf("%s\n", strings.Replace(typeStr(obj, typ), "\n", "\n\t", -1))
// return pos
// }

func typeStr(obj *ast.Object, typ types.Type) string {
	switch obj.Kind {
	case ast.Fun, ast.Var:
		return fmt.Sprintf("%s %v", obj.Name, prettyType{typ})
	case ast.Pkg:
		return fmt.Sprintf("import (%s %s)", obj.Name, typ.Node.(*ast.ImportSpec).Path.Value)
	case ast.Con:
		if decl, ok := obj.Decl.(*ast.ValueSpec); ok {
			return fmt.Sprintf("const %s %v = %s", obj.Name, prettyType{typ}, pretty{decl.Values[0]})
		}
		return fmt.Sprintf("const %s %v", obj.Name, prettyType{typ})
	case ast.Lbl:
		return fmt.Sprintf("label %s", obj.Name)
	case ast.Typ:
		typ = typ.Underlying(false, types.DefaultImporter)
		return fmt.Sprintf("type %s %v", obj.Name, prettyType{typ})
	}
	return fmt.Sprintf("unknown %s %v", obj.Name, typ.Kind)
}

func parseExpr(s *ast.Scope, expr string) ast.Expr {
	n, err := parser.ParseExpr(types.FileSet, "<arg>", expr, s)
	if err != nil {
		fail("cannot parse expression: %v", err)
	}
	switch n := n.(type) {
	case *ast.Ident, *ast.SelectorExpr:
		return n
	}
	fail("no identifier found in expression")
	return nil
}

type FVisitor func(n ast.Node) bool

func (f FVisitor) Visit(n ast.Node) ast.Visitor {
	if f(n) {
		return f
	}
	return nil
}

// var errNoPkgFiles = errors.New("no more package files found")

// parseLocalPackage reads and parses all go files from the
// current directory that implement the same package name
// the principal source file, except the original source file
// itself, which will already have been parsed.
//
// func parseLocalPackage(filename string, src *ast.File, pkgScope *ast.Scope) (*ast.Package, error) {
// 	pkg := &ast.Package{src.Name.Name, pkgScope, nil, map[string]*ast.File{filename: src}}
// 	d, f := filepath.Split(filename)
// 	if d == "" {
// 		d = "./"
// 	}
// 	fd, err := os.Open(d)
// 	if err != nil {
// 		return nil, errNoPkgFiles
// 	}
// 	defer fd.Close()
//
// 	list, err := fd.Readdirnames(-1)
// 	if err != nil {
// 		return nil, errNoPkgFiles
// 	}
//
// 	for _, pf := range list {
// 		file := filepath.Join(d, pf)
// 		if !strings.HasSuffix(pf, ".go") ||
// 			pf == f ||
// 			pkgName(file) != pkg.Name {
// 			continue
// 		}
// 		src, err := parser.ParseFile(types.FileSet, file, nil, 0, pkg.Scope)
// 		if err == nil {
// 			pkg.Files[file] = src
// 		}
// 	}
// 	if len(pkg.Files) == 1 {
// 		return nil, errNoPkgFiles
// 	}
// 	return pkg, nil
// }

// pkgName returns the package name implemented by the go source filename
func pkgName(filename string) string {
	prog, _ := parser.ParseFile(types.FileSet, filename, nil, parser.PackageClauseOnly, nil)
	if prog != nil {
		return prog.Name.Name
	}
	return ""
}

func hasSuffix(s, suff string) bool {
	return len(s) >= len(suff) && s[len(s)-len(suff):] == suff
}

type pretty struct {
	n interface{}
}

func (p pretty) String() string {
	var b bytes.Buffer
	printer.Fprint(&b, types.FileSet, p.n)
	return b.String()
}

type prettyType struct {
	n types.Type
}

func (p prettyType) String() string {
	// TODO print path package when appropriate.
	// Current issues with using p.n.Pkg:
	//	- we should actually print the local package identifier
	//	rather than the package path when possible.
	//	- p.n.Pkg is non-empty even when
	//	the type is not relative to the package.
	return pretty{p.n.Node}.String()
}
<commit_msg>Cleanup unused code<commit_after>package commands

import (
	"bytes"
	"fmt"
	"go/build"
	"os"
	"runtime"
	"strconv"

	"nvim-go/gb"
	"nvim-go/nvim"

	log "github.com/Sirupsen/logrus"
	"github.com/garyburd/neovim-go/vim"
	"github.com/garyburd/neovim-go/vim/plugin"
	"github.com/rogpeppe/godef/go/ast"
	"github.com/rogpeppe/godef/go/parser"
	"github.com/rogpeppe/godef/go/printer"
	"github.com/rogpeppe/godef/go/token"
	"github.com/rogpeppe/godef/go/types"
)

func init() {
	plugin.HandleCommand("Godef", &plugin.CommandOptions{Range: "%", Eval: "expand('%:p:h:h:h:h')"}, Def)
}

func Def(v *vim.Vim, r [2]int, file string) error {
	defer gb.WithGoBuildForPath(file)()

	b, err := v.CurrentBuffer()
	if err != nil {
		return v.WriteErr("cannot get current buffer")
	}

	buf, err := v.BufferLineSlice(b, 0, -1, true, true)
	if err != nil {
		return err
	}
	src := bytes.Join(buf, []byte{'\n'})

	filename, err := v.BufferName(b)
	if err != nil {
		return v.WriteErr("cannot get current buffer name")
	}

	searchpos, err := nvim.ByteOffset(v)
	if err != nil {
		return v.WriteErr("cannot get current buffer byte offset")
	}

	pkgScope := ast.NewScope(parser.Universe)
	f, err := parser.ParseFile(types.FileSet, filename, src, 0, pkgScope)
	if f == nil {
		nvim.Echomsg(v, "cannot parse %s: %v", filename, err)
	}

	o := findIdentifier(v, f, searchpos)

	switch e := o.(type) {
	case *ast.ImportSpec:
		path := importPath(e)
		pkg, err := build.Default.Import(path, "", build.FindOnly)
		if err != nil {
			fail("error finding import path for %s: %s", path, err)
		}
		fmt.Println(pkg.Dir)
	case ast.Expr:
		if obj, _ := types.ExprType(e, types.DefaultImporter); obj != nil {
			out := types.FileSet.Position(types.DeclPos(obj))
			log.Debugln("def out.String():", out.String())
			log.Debugln("def out.Filename:", out.Filename)
			log.Debugln("def out.Line:", out.Line)
			log.Debugln("def out.Column:", out.Column)
			log.Debugln("def out.Offset:", out.Offset)
			log.Debugln("def out.IsValid():", out.IsValid())

			v.Command("silent lexpr '" + fmt.Sprintf("%v", out) + "'")
			w, err := v.CurrentWindow()
			if err != nil {
				log.Debugln(err)
			}
			v.SetWindowCursor(w, [2]int{out.Line, out.Column - 1})
			// v.Command("silent lgetexpr '" + fmt.Sprintf("%v", out) + "'")
			// v.Command("sil ll 1")
			// v.Feedkeys("zz", "normal", false)
		}
		// fail("no declaration found for %v", pretty{e})
	}
	return nil
}

func fail(s string, a ...interface{}) {
	fmt.Fprint(os.Stderr, "godef: "+fmt.Sprintf(s, a...)+"\n")
	os.Exit(2)
}

func importPath(n *ast.ImportSpec) string {
	p, err := strconv.Unquote(n.Path.Value)
	if err != nil {
		fail("invalid string literal %q in ast.ImportSpec", n.Path.Value)
	}
	return p
}

// findIdentifier looks for an identifier at byte-offset searchpos
// inside the parsed source represented by node.
// If it is part of a selector expression, it returns
// that expression rather than the identifier itself.
//
// As a special case, if it finds an import spec, it returns ImportSpec.
func findIdentifier(v *vim.Vim, f *ast.File, searchpos int) ast.Node {
	ec := make(chan ast.Node)
	found := func(startPos, endPos token.Pos) bool {
		start := types.FileSet.Position(startPos).Offset
		end := start + int(endPos-startPos)
		return start <= searchpos && searchpos <= end
	}
	go func() {
		var visit func(ast.Node) bool
		visit = func(n ast.Node) bool {
			var startPos token.Pos
			switch n := n.(type) {
			default:
				return true
			case *ast.Ident:
				startPos = n.NamePos
			case *ast.SelectorExpr:
				startPos = n.Sel.NamePos
			case *ast.ImportSpec:
				startPos = n.Pos()
			case *ast.StructType:
				// If we find an anonymous bare field in a
				// struct type, its definition points to itself,
				// but we actually want to go elsewhere,
				// so assume (dubiously) that the expression
				// works globally and return a new node for it.
				for _, field := range n.Fields.List {
					if field.Names != nil {
						continue
					}
					t := field.Type
					if pt, ok := field.Type.(*ast.StarExpr); ok {
						t = pt.X
					}
					if id, ok := t.(*ast.Ident); ok {
						if found(id.NamePos, id.End()) {
							ec <- parseExpr(f.Scope, id.Name)
							runtime.Goexit()
						}
					}
				}
				return true
			}
			if found(startPos, n.End()) {
				ec <- n
				runtime.Goexit()
			}
			return true
		}
		ast.Walk(FVisitor(visit), f)
		ec <- nil
	}()
	ev := <-ec
	if ev == nil {
		log.Debugln("def: no identifier found")
		nvim.Echomsg(v, "def: no identifier found")
		// fail("no identifier found")
	}
	return ev
}

func typeStr(obj *ast.Object, typ types.Type) string {
	switch obj.Kind {
	case ast.Fun, ast.Var:
		return fmt.Sprintf("%s %v", obj.Name, prettyType{typ})
	case ast.Pkg:
		return fmt.Sprintf("import (%s %s)", obj.Name, typ.Node.(*ast.ImportSpec).Path.Value)
	case ast.Con:
		if decl, ok := obj.Decl.(*ast.ValueSpec); ok {
			return fmt.Sprintf("const %s %v = %s", obj.Name, prettyType{typ}, pretty{decl.Values[0]})
		}
		return fmt.Sprintf("const %s %v", obj.Name, prettyType{typ})
	case ast.Lbl:
		return fmt.Sprintf("label %s", obj.Name)
	case ast.Typ:
		typ = typ.Underlying(false, types.DefaultImporter)
		return fmt.Sprintf("type %s %v", obj.Name, prettyType{typ})
	}
	return fmt.Sprintf("unknown %s %v", obj.Name, typ.Kind)
}

func parseExpr(s *ast.Scope, expr string) ast.Expr {
	n, err := parser.ParseExpr(types.FileSet, "<arg>", expr, s)
	if err != nil {
		fail("cannot parse expression: %v", err)
	}
	switch n := n.(type) {
	case *ast.Ident, *ast.SelectorExpr:
		return n
	}
	fail("no identifier found in expression")
	return nil
}

type FVisitor func(n ast.Node) bool

func (f FVisitor) Visit(n ast.Node) ast.Visitor {
	if f(n) {
		return f
	}
	return nil
}

// pkgName returns the package name implemented by the go source filename
func pkgName(filename string) string {
	prog, _ := parser.ParseFile(types.FileSet, filename, nil, parser.PackageClauseOnly, nil)
	if prog != nil {
		return prog.Name.Name
	}
	return ""
}

func hasSuffix(s, suff string) bool {
	return len(s) >= len(suff) && s[len(s)-len(suff):] == suff
}

type pretty struct {
	n interface{}
}

func (p pretty) String() string {
	var b bytes.Buffer
	printer.Fprint(&b, types.FileSet, p.n)
	return b.String()
}

type prettyType struct {
	n types.Type
}

func (p prettyType) String() string {
	// TODO print path package when appropriate.
	// Current issues with using p.n.Pkg:
	//	- we should actually print the local package identifier
	//	rather than the package path when possible.
	//	- p.n.Pkg is non-empty even when
	//	the type is not relative to the package.
	return pretty{p.n.Node}.String()
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"context"
	"crypto/md5"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"cloud.google.com/go/storage"
	"github.com/gorilla/mux"
	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/common"
	"go.skia.org/infra/go/exec"
	"go.skia.org/infra/go/httputils"
	"go.skia.org/infra/go/iap"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/util"
	"google.golang.org/api/option"
)

const (
	// BUCKET is the Cloud Storage bucket we store files in.
	BUCKET = "skottie-renderer"
)

// flags
var (
	local        = flag.Bool("local", false, "Running locally if true. As opposed to in production.")
	port         = flag.String("port", ":8000", "HTTP service address (e.g., ':8000')")
	promPort     = flag.String("prom_port", ":20000", "Metrics service address (e.g., ':10110')")
	resourcesDir = flag.String("resources_dir", "", "The directory to find templates, JS, and CSS files. If blank the current directory will be used.")
	skottieTool  = flag.String("skottie_tool", "", "Absolute path to the skottie_tool executable.")
)

var (
	invalidRequestErr = errors.New("")
)

// Server is the state of the server.
type Server struct {
	bucket    *storage.BucketHandle
	templates *template.Template
}

func New() (*Server, error) {
	if *resourcesDir == "" {
		_, filename, _, _ := runtime.Caller(0)
		*resourcesDir = filepath.Join(filepath.Dir(filename), "../../dist")
	}

	ts, err := auth.NewDefaultTokenSource(*local, storage.ScopeFullControl)
	if err != nil {
		return nil, fmt.Errorf("Failed to get token source: %s", err)
	}
	client := auth.ClientFromTokenSource(ts)
	storageClient, err := storage.NewClient(context.Background(), option.WithHTTPClient(client))
	if err != nil {
		return nil, fmt.Errorf("Problem creating storage client: %s", err)
	}

	srv := &Server{
		bucket: storageClient.Bucket(BUCKET),
	}
	srv.loadTemplates()
	return srv, nil
}

func (srv *Server) loadTemplates() {
	srv.templates = template.Must(template.New("").Delims("{%", "%}").ParseFiles(
		filepath.Join(*resourcesDir, "index.html"),
	))
}

// createWebm runs ffmpeg over the images in the given dir.
func createWebm(ctx context.Context, dir string, fps int) error {
	// ffmpeg -r $FPS -pattern_type glob -i '*.png' -c:v libvpx-vp9 -lossless 1 lottie.webm
	name := "ffmpeg"
	args := []string{
		"-r", fmt.Sprintf("%d", fps),
		"-pattern_type", "glob", "-i", "*.png",
		"-c:v", "libvpx-vp9",
		"-lossless", "1",
		"lottie.webm",
	}
	output := &bytes.Buffer{}
	runCmd := &exec.Command{
		Name:      name,
		Args:      args,
		Dir:       dir,
		LogStderr: true,
		Stdout:    output,
	}
	if err := exec.Run(ctx, runCmd); err != nil {
		return fmt.Errorf("ffmpeg failed %#v: %s", *runCmd, err)
	}

	return nil
}
func (srv *Server) mainHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/html")
	if *local {
		srv.loadTemplates()
	}
	if err := srv.templates.ExecuteTemplate(w, "index.html", nil); err != nil {
		sklog.Errorf("Failed to expand template: %s", err)
	}
}

func (srv *Server) jsonHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json")
	hash := mux.Vars(r)["hash"]
	path := strings.Join([]string{hash, "lottie.json"}, "/")
	reader, err := srv.bucket.Object(path).NewReader(r.Context())
	if err != nil {
		httputils.ReportError(w, r, err, "Can't load file from GCS")
		return
	}
	if _, err = io.Copy(w, reader); err != nil {
		httputils.ReportError(w, r, err, "Failed to write JSON file.")
		return
	}
}

func (srv *Server) webmHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "video/webm")
	hash := mux.Vars(r)["hash"]
	path := strings.Join([]string{hash, "lottie.webm"}, "/")
	reader, err := srv.bucket.Object(path).NewReader(r.Context())
	if err != nil {
		httputils.ReportError(w, r, err, "Can't load file from GCS")
		return
	}
	if _, err = io.Copy(w, reader); err != nil {
		httputils.ReportError(w, r, err, "Failed to write webm file.")
		return
	}
}

type UploadRequest struct {
	Lottie interface{} `json:"lottie"`
	Width  int         `json:"width"`
	Height int         `json:"height"`
	FPS    int         `json:"fps"`
}

type UploadResponse struct {
	Hash string `json:"hash"`
}

func (req *UploadRequest) validate(w http.ResponseWriter) error {
	if req.FPS < 1 || req.FPS > 120 {
		http.Error(w, "FPS must be betwee 1 and 120.", http.StatusBadRequest)
		return invalidRequestErr
	}
	if req.Width < 1 || req.Width > 1024 {
		http.Error(w, "Width must be betwee 1 and 1024.", http.StatusBadRequest)
		return invalidRequestErr
	}
	if req.Height < 1 || req.Height > 1024 {
		http.Error(w, "Height must be betwee 1 and 1024.", http.StatusBadRequest)
		return invalidRequestErr
	}
	return nil
}

func (srv *Server) uploadHandler(w http.ResponseWriter, r *http.Request) {
	ctx := r.Context()
	// Extract json file.
	defer util.Close(r.Body)
	var req UploadRequest
	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
		httputils.ReportError(w, r, err, "Error decoding JSON.")
		return
	}
	if err := req.validate(w); err != nil {
		return
	}

	source, err := ioutil.TempDir("", "source")
	if err != nil {
		httputils.ReportError(w, r, err, "Can't create temp space.")
		return
	}
	defer util.RemoveAll(source)
	dest, err := ioutil.TempDir("", "dest")
	if err != nil {
		httputils.ReportError(w, r, err, "Can't create temp space.")
		return
	}
	defer util.RemoveAll(dest)
	sourceFullPath := filepath.Join(source, "lottie.json")
	destFullPath := filepath.Join(dest, "lottie.webm")

	b, err := json.Marshal(req.Lottie)
	if err != nil {
		httputils.ReportError(w, r, err, "Can't re-encode lottie file.")
		return
	}
	if err := ioutil.WriteFile(sourceFullPath, b, 0644); err != nil {
		httputils.ReportError(w, r, err, "Can't write file.")
		return
	}
	// Run through skottie_tool.
	toolResults, err := exec.RunSimple(ctx, fmt.Sprintf("%s --input %s --writePath %s --width %d --height %d --fps %d", *skottieTool, sourceFullPath, dest, req.Width, req.Height, req.FPS))
	if err != nil {
		sklog.Warningf("Failed running: %q", toolResults)
		httputils.ReportError(w, r, err, "Failed running skottie_tool.")
		return
	}
	// Run results of that through ffmpeg to create webm.
	if err := createWebm(ctx, dest, req.FPS); err != nil {
		httputils.ReportError(w, r, err, "Failed building webm.")
		return
	}

	// Calculate md5 of file.
	// TODO(jcgregorio) include options in md5 calculation once they're added to the UI.
	h := md5.New()
	b, err = json.Marshal(req)
	if err != nil {
		httputils.ReportError(w, r, err, "Can't re-encode request.")
		return
	}
	if _, err = h.Write(b); err != nil {
		httputils.ReportError(w, r, err, "Failed calculating hash.")
		return
	}
	hash := fmt.Sprintf("%x", h.Sum(nil))

	// Write JSON file.
	path := strings.Join([]string{hash, "lottie.json"}, "/")
	wr := srv.bucket.Object(path).NewWriter(ctx)
	defer util.Close(wr)
	wr.ObjectAttrs.ContentEncoding = "application/json"
	if _, err := wr.Write(b); err != nil {
		httputils.ReportError(w, r, err, "Failed writing JSON to GCS.")
		return
	}

	// Write webm file.
	path = strings.Join([]string{hash, "lottie.webm"}, "/")
	wr = srv.bucket.Object(path).NewWriter(ctx)
	defer util.Close(wr)
	wr.ObjectAttrs.ContentEncoding = "video/webm"
	f, err := os.Open(destFullPath)
	if err != nil {
		httputils.ReportError(w, r, err, "Failed opening webm.")
		return
	}
	_, err = io.Copy(wr, f)
	if err != nil {
		httputils.ReportError(w, r, err, "Failed writing webm to GCS.")
		return
	}

	resp := UploadResponse{
		Hash: hash,
	}
	if err := json.NewEncoder(w).Encode(resp); err != nil {
		sklog.Errorf("Failed to write response: %s", err)
	}
}

func main() {
	common.InitWithMust(
		"skottie",
		common.PrometheusOpt(promPort),
	)

	if *skottieTool == "" {
		sklog.Fatal("The --skottie_tool flag is required.")
	}

	srv, err := New()
	if err != nil {
		sklog.Fatalf("Failed to start: %s", err)
	}

	r := mux.NewRouter()
	r.HandleFunc("/{hash:[0-9A-Za-z]*}", srv.mainHandler)

	r.HandleFunc("/_/i/{hash:[0-9A-Za-z]+}", srv.webmHandler)
	r.HandleFunc("/_/j/{hash:[0-9A-Za-z]+}", srv.jsonHandler)
	r.HandleFunc("/_/upload", srv.uploadHandler)

	r.PathPrefix("/static/").Handler(http.StripPrefix("/static/", http.HandlerFunc(httputils.MakeResourceHandler(*resourcesDir))))

	// TODO(jcgregorio) Implement CSRF.
	h := httputils.LoggingGzipRequestResponse(r)
	if !*local {
		h = iap.None(h)
	}

	http.Handle("/", h)
	sklog.Infoln("Ready to serve.")
	sklog.Fatal(http.ListenAndServe(*port, nil))
}
<commit_msg>[skottie] Fix typo.<commit_after>package main

import (
	"bytes"
	"context"
	"crypto/md5"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"cloud.google.com/go/storage"
	"github.com/gorilla/mux"
	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/common"
	"go.skia.org/infra/go/exec"
	"go.skia.org/infra/go/httputils"
	"go.skia.org/infra/go/iap"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/util"
	"google.golang.org/api/option"
)

const (
	// BUCKET is the Cloud Storage bucket we store files in.
	BUCKET = "skottie-renderer"
)

// flags
var (
	local        = flag.Bool("local", false, "Running locally if true. As opposed to in production.")
	port         = flag.String("port", ":8000", "HTTP service address (e.g., ':8000')")
	promPort     = flag.String("prom_port", ":20000", "Metrics service address (e.g., ':10110')")
	resourcesDir = flag.String("resources_dir", "", "The directory to find templates, JS, and CSS files. If blank the current directory will be used.")
	skottieTool  = flag.String("skottie_tool", "", "Absolute path to the skottie_tool executable.")
)

var (
	invalidRequestErr = errors.New("")
)

// Server is the state of the server.
type Server struct {
	bucket    *storage.BucketHandle
	templates *template.Template
}

func New() (*Server, error) {
	if *resourcesDir == "" {
		_, filename, _, _ := runtime.Caller(0)
		*resourcesDir = filepath.Join(filepath.Dir(filename), "../../dist")
	}

	ts, err := auth.NewDefaultTokenSource(*local, storage.ScopeFullControl)
	if err != nil {
		return nil, fmt.Errorf("Failed to get token source: %s", err)
	}
	client := auth.ClientFromTokenSource(ts)
	storageClient, err := storage.NewClient(context.Background(), option.WithHTTPClient(client))
	if err != nil {
		return nil, fmt.Errorf("Problem creating storage client: %s", err)
	}

	srv := &Server{
		bucket: storageClient.Bucket(BUCKET),
	}
	srv.loadTemplates()
	return srv, nil
}

func (srv *Server) loadTemplates() {
	srv.templates = template.Must(template.New("").Delims("{%", "%}").ParseFiles(
		filepath.Join(*resourcesDir, "index.html"),
	))
}

// createWebm runs ffmpeg over the images in the given dir.
func createWebm(ctx context.Context, dir string, fps int) error {
	// ffmpeg -r $FPS -pattern_type glob -i '*.png' -c:v libvpx-vp9 -lossless 1 lottie.webm
	name := "ffmpeg"
	args := []string{
		"-r", fmt.Sprintf("%d", fps),
		"-pattern_type", "glob", "-i", "*.png",
		"-c:v", "libvpx-vp9",
		"-lossless", "1",
		"lottie.webm",
	}
	output := &bytes.Buffer{}
	runCmd := &exec.Command{
		Name:      name,
		Args:      args,
		Dir:       dir,
		LogStderr: true,
		Stdout:    output,
	}
	if err := exec.Run(ctx, runCmd); err != nil {
		return fmt.Errorf("ffmpeg failed %#v: %s", *runCmd, err)
	}

	return nil
}
func (srv *Server) mainHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/html")
	if *local {
		srv.loadTemplates()
	}
	if err := srv.templates.ExecuteTemplate(w, "index.html", nil); err != nil {
		sklog.Errorf("Failed to expand template: %s", err)
	}
}

func (srv *Server) jsonHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json")
	hash := mux.Vars(r)["hash"]
	path := strings.Join([]string{hash, "lottie.json"}, "/")
	reader, err := srv.bucket.Object(path).NewReader(r.Context())
	if err != nil {
		httputils.ReportError(w, r, err, "Can't load file from GCS")
		return
	}
	if _, err = io.Copy(w, reader); err != nil {
		httputils.ReportError(w, r, err, "Failed to write JSON file.")
		return
	}
}

func (srv *Server) webmHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "video/webm")
	hash := mux.Vars(r)["hash"]
	path := strings.Join([]string{hash, "lottie.webm"}, "/")
	reader, err := srv.bucket.Object(path).NewReader(r.Context())
	if err != nil {
		httputils.ReportError(w, r, err, "Can't load file from GCS")
		return
	}
	if _, err = io.Copy(w, reader); err != nil {
		httputils.ReportError(w, r, err, "Failed to write webm file.")
		return
	}
}

type UploadRequest struct {
	Lottie interface{} `json:"lottie"`
	Width  int         `json:"width"`
	Height int         `json:"height"`
	FPS    int         `json:"fps"`
}

type UploadResponse struct {
	Hash string `json:"hash"`
}

func (req *UploadRequest) validate(w http.ResponseWriter) error {
	if req.FPS < 1 || req.FPS > 120 {
		http.Error(w, "FPS must be between 1 and 120.", http.StatusBadRequest)
		return invalidRequestErr
	}
	if req.Width < 1 || req.Width > 1024 {
		http.Error(w, "Width must be between 1 and 1024.", http.StatusBadRequest)
		return invalidRequestErr
	}
	if req.Height < 1 || req.Height > 1024 {
		http.Error(w, "Height must be between 1 and 1024.", http.StatusBadRequest)
		return invalidRequestErr
	}
	return nil
}

func (srv *Server) uploadHandler(w http.ResponseWriter, r *http.Request) {
	ctx := r.Context()
	// Extract json file.
	defer util.Close(r.Body)
	var req UploadRequest
	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
		httputils.ReportError(w, r, err, "Error decoding JSON.")
		return
	}
	if err := req.validate(w); err != nil {
		return
	}

	source, err := ioutil.TempDir("", "source")
	if err != nil {
		httputils.ReportError(w, r, err, "Can't create temp space.")
		return
	}
	defer util.RemoveAll(source)
	dest, err := ioutil.TempDir("", "dest")
	if err != nil {
		httputils.ReportError(w, r, err, "Can't create temp space.")
		return
	}
	defer util.RemoveAll(dest)
	sourceFullPath := filepath.Join(source, "lottie.json")
	destFullPath := filepath.Join(dest, "lottie.webm")

	b, err := json.Marshal(req.Lottie)
	if err != nil {
		httputils.ReportError(w, r, err, "Can't re-encode lottie file.")
		return
	}
	if err := ioutil.WriteFile(sourceFullPath, b, 0644); err != nil {
		httputils.ReportError(w, r, err, "Can't write file.")
		return
	}
	// Run through skottie_tool.
	toolResults, err := exec.RunSimple(ctx, fmt.Sprintf("%s --input %s --writePath %s --width %d --height %d --fps %d", *skottieTool, sourceFullPath, dest, req.Width, req.Height, req.FPS))
	if err != nil {
		sklog.Warningf("Failed running: %q", toolResults)
		httputils.ReportError(w, r, err, "Failed running skottie_tool.")
		return
	}
	// Run results of that through ffmpeg to create webm.
	if err := createWebm(ctx, dest, req.FPS); err != nil {
		httputils.ReportError(w, r, err, "Failed building webm.")
		return
	}

	// Calculate md5 of file.
	// TODO(jcgregorio) include options in md5 calculation once they're added to the UI.
	h := md5.New()
	b, err = json.Marshal(req)
	if err != nil {
		httputils.ReportError(w, r, err, "Can't re-encode request.")
		return
	}
	if _, err = h.Write(b); err != nil {
		httputils.ReportError(w, r, err, "Failed calculating hash.")
		return
	}
	hash := fmt.Sprintf("%x", h.Sum(nil))

	// Write JSON file.
	path := strings.Join([]string{hash, "lottie.json"}, "/")
	wr := srv.bucket.Object(path).NewWriter(ctx)
	defer util.Close(wr)
	wr.ObjectAttrs.ContentEncoding = "application/json"
	if _, err := wr.Write(b); err != nil {
		httputils.ReportError(w, r, err, "Failed writing JSON to GCS.")
		return
	}

	// Write webm file.
	path = strings.Join([]string{hash, "lottie.webm"}, "/")
	wr = srv.bucket.Object(path).NewWriter(ctx)
	defer util.Close(wr)
	wr.ObjectAttrs.ContentEncoding = "video/webm"
	f, err := os.Open(destFullPath)
	if err != nil {
		httputils.ReportError(w, r, err, "Failed opening webm.")
		return
	}
	_, err = io.Copy(wr, f)
	if err != nil {
		httputils.ReportError(w, r, err, "Failed writing webm to GCS.")
		return
	}

	resp := UploadResponse{
		Hash: hash,
	}
	if err := json.NewEncoder(w).Encode(resp); err != nil {
		sklog.Errorf("Failed to write response: %s", err)
	}
}

func main() {
	common.InitWithMust(
		"skottie",
		common.PrometheusOpt(promPort),
	)

	if *skottieTool == "" {
		sklog.Fatal("The --skottie_tool flag is required.")
	}

	srv, err := New()
	if err != nil {
		sklog.Fatalf("Failed to start: %s", err)
	}

	r := mux.NewRouter()
	r.HandleFunc("/{hash:[0-9A-Za-z]*}", srv.mainHandler)

	r.HandleFunc("/_/i/{hash:[0-9A-Za-z]+}", srv.webmHandler)
	r.HandleFunc("/_/j/{hash:[0-9A-Za-z]+}", srv.jsonHandler)
	r.HandleFunc("/_/upload", srv.uploadHandler)

	r.PathPrefix("/static/").Handler(http.StripPrefix("/static/", http.HandlerFunc(httputils.MakeResourceHandler(*resourcesDir))))

	// TODO(jcgregorio) Implement CSRF.
	h := httputils.LoggingGzipRequestResponse(r)
	if !*local {
		h = iap.None(h)
	}

	http.Handle("/", h)
	sklog.Infoln("Ready to serve.")
	sklog.Fatal(http.ListenAndServe(*port, nil))
}
<|endoftext|>


<commit_before>package node

import (
	"fmt"
	"runtime"
	"time"

	"github.com/anycable/anycable-go/common"
	"github.com/anycable/anycable-go/metrics"
	"github.com/anycable/anycable-go/ws"
	"github.com/apex/log"
)

const (
	// serverRestartReason is the disconnect reason on shutdown
	serverRestartReason    = "server_restart"
	remoteDisconnectReason = "remote"

	metricsGoroutines      = "goroutines_num"
	metricsMemSys          = "mem_sys_bytes"
	metricsClientsNum      = "clients_num"
	metricsUniqClientsNum  = "clients_uniq_num"
	metricsStreamsNum      = "broadcast_streams_num"
	metricsDisconnectQueue = "disconnect_queue_size"

	metricsFailedAuths           = "failed_auths_total"
	metricsReceivedMsg           = "client_msg_total"
	metricsFailedCommandReceived = "failed_client_msg_total"
	metricsBroadcastMsg          = "broadcast_msg_total"
	metricsUnknownBroadcast      = "failed_broadcast_msg_total"

	metricsSentMsg    = "server_msg_total"
	metricsFailedSent = "failed_server_msg_total"

	metricsDataSent     = "data_sent_total"
	metricsDataReceived = "data_rcvd_total"
)

// AppNode describes a basic node interface
type AppNode interface {
	HandlePubSub(msg []byte)
	Authenticate(s *Session) (*common.ConnectResult, error)
	Subscribe(s *Session, msg *common.Message) (*common.CommandResult, error)
	Unsubscribe(s *Session, msg *common.Message) (*common.CommandResult, error)
	Perform(s *Session, msg *common.Message) (*common.CommandResult, error)
	Disconnect(s *Session) error
}

// Connection represents underlying connection
type Connection interface {
	Write(msg []byte, deadline time.Time) error
	WriteBinary(msg []byte, deadline time.Time) error
	Read() ([]byte, error)
	Close(code int, reason string)
}

// Node represents the whole application
type Node struct {
	Metrics *metrics.Metrics

	config       *Config
	hub          *Hub
	controller   Controller
	disconnector Disconnector
	shutdownCh   chan struct{}
	log          *log.Entry
}

// NewNode builds new node struct
func NewNode(controller Controller, metrics *metrics.Metrics, config *Config) *Node {
	node := &Node{
		Metrics:    metrics,
		config:     config,
		controller: controller,
		shutdownCh: make(chan struct{}),
		log:        log.WithFields(log.Fields{"context": "node"}),
	}

	node.hub = NewHub(config.HubGopoolSize)

	node.registerMetrics()

	return node
}

// Start runs all the required goroutines
func (n *Node) Start() error {
	go n.hub.Run()
	go n.collectStats()

	return nil
}

// SetDisconnector set disconnector for the node
func (n *Node) SetDisconnector(d Disconnector) {
	n.disconnector = d
}

// HandleCommand parses incoming message from client and
// execute the command (if recognized)
func (n *Node) HandleCommand(s *Session, msg *common.Message) (err error) {
	s.Log.Debugf("Incoming message: %s", msg)
	switch msg.Command {
	case "subscribe":
		_, err = n.Subscribe(s, msg)
	case "unsubscribe":
		_, err = n.Unsubscribe(s, msg)
	case "message":
		_, err = n.Perform(s, msg)
	default:
		err = fmt.Errorf("Unknown command: %s", msg.Command)
	}

	return
}

// HandlePubSub parses incoming pubsub message and broadcast it
func (n *Node) HandlePubSub(raw []byte) {
	msg, err := common.PubSubMessageFromJSON(raw)

	if err != nil {
		n.Metrics.Counter(metricsUnknownBroadcast).Inc()
		n.log.Warnf("Failed to parse pubsub message '%s' with error: %v", raw, err)
		return
	}

	switch v := msg.(type) {
	case common.StreamMessage:
		n.Broadcast(&v)
	case common.RemoteDisconnectMessage:
		n.RemoteDisconnect(&v)
	}
}

// Shutdown stops all services (hub, controller)
func (n *Node) Shutdown() (err error) {
	close(n.shutdownCh)

	if n.hub != nil {
		n.hub.Shutdown()

		active := n.hub.Size()

		if active > 0 {
			n.log.Infof("Closing active connections: %d", active)
			disconnectMessage := newDisconnectMessage(serverRestartReason, true)
			// Close all registered sessions
			n.hub.sessionsMu.RLock()
			for _, session := range n.hub.sessions {
				session.Send(disconnectMessage)
				session.Disconnect("Shutdown", ws.CloseGoingAway)
			}
			n.hub.sessionsMu.RUnlock()

			n.log.Info("All active connections closed")

			// Wait to make sure that disconnect queue is not empty
			time.Sleep(time.Second)
		}
	}

	if n.disconnector != nil {
		err := n.disconnector.Shutdown()

		if err != nil {
			n.log.Warnf("%v", err)
		}
	}

	if n.controller != nil {
		err := n.controller.Shutdown()

		if err != nil {
			n.log.Warnf("%v", err)
		}
	}

	return
}

// Authenticate calls controller to perform authentication.
// If authentication is successful, session is registered with a hub.
func (n *Node) Authenticate(s *Session) (res *common.ConnectResult, err error) {
	res, err = n.controller.Authenticate(s.UID, s.env)

	if err == nil {
		s.Identifiers = res.Identifier
		s.Connected = true

		n.hub.addSession(s)
	} else {
		n.Metrics.Counter(metricsFailedAuths).Inc()
	}

	if res != nil {
		n.handleCallReply(s, res.ToCallResult())
	}

	if err != nil {
		s.Disconnect("Auth Error", ws.CloseInternalServerErr)
	}

	return
}

// Subscribe subscribes session to a channel
func (n *Node) Subscribe(s *Session, msg *common.Message) (res *common.CommandResult, err error) {
	s.smu.Lock()

	if _, ok := s.subscriptions[msg.Identifier]; ok {
		s.smu.Unlock()
		err = fmt.Errorf("Already subscribed to %s", msg.Identifier)
		return
	}

	res, err = n.controller.Subscribe(s.UID, s.env, s.Identifiers, msg.Identifier)

	if err != nil {
		s.Log.Errorf("Subscribe error: %v", err)
	} else {
		s.subscriptions[msg.Identifier] = true
		s.Log.Debugf("Subscribed to channel: %s", msg.Identifier)
	}

	s.smu.Unlock()

	if res != nil {
		n.handleCommandReply(s, msg, res)
	}

	return
}

// Unsubscribe unsubscribes session from a channel
func (n *Node) Unsubscribe(s *Session, msg *common.Message) (res *common.CommandResult, err error) {
	s.smu.Lock()

	if _, ok := s.subscriptions[msg.Identifier]; !ok {
		s.smu.Unlock()
		err = fmt.Errorf("Unknown subscription %s", msg.Identifier)
		return
	}

	res, err = n.controller.Unsubscribe(s.UID, s.env, s.Identifiers, msg.Identifier)

	if err != nil {
		s.Log.Errorf("Unsubscribe error: %v", err)
	} else {
		// Make sure to remove all streams subscriptions
		res.StopAllStreams = true

		delete(s.subscriptions, msg.Identifier)

		s.Log.Debugf("Unsubscribed from channel: %s", msg.Identifier)
	}

	s.smu.Unlock()

	if res != nil {
		n.handleCommandReply(s, msg, res)
	}

	return
}

// Perform executes client command
func (n *Node) Perform(s *Session, msg *common.Message) (res *common.CommandResult, err error) {
	s.smu.Lock()

	if _, ok := s.subscriptions[msg.Identifier]; !ok {
		s.smu.Unlock()
		err = fmt.Errorf("Unknown subscription %s", msg.Identifier)
		return
	}

	s.smu.Unlock()

	res, err = n.controller.Perform(s.UID, s.env, s.Identifiers, msg.Identifier, msg.Data)

	if err != nil {
		s.Log.Errorf("Perform error: %v", err)
	} else {
		s.Log.Debugf("Perform result: %v", res)
	}

	if res != nil {
		n.handleCommandReply(s, msg, res)
	}

	return
}

// Broadcast message to stream
func (n *Node) Broadcast(msg *common.StreamMessage) {
	n.Metrics.Counter(metricsBroadcastMsg).Inc()
	n.log.Debugf("Incoming pubsub message: %v", msg)
	n.hub.BroadcastMessage(msg)
}

// Disconnect adds session to disconnector queue and unregister session from hub
func (n *Node) Disconnect(s *Session) error {
	n.hub.RemoveSession(s)
	return n.disconnector.Enqueue(s)
}

// DisconnectNow execute disconnect on controller
func (n *Node) DisconnectNow(s *Session) error {
	sessionSubscriptions := subscriptionsList(s.subscriptions)

	s.Log.Debugf("Disconnect %s %s %v %v", s.Identifiers, s.env.URL, s.env.Headers, sessionSubscriptions)

	err := n.controller.Disconnect(
		s.UID,
		s.env,
		s.Identifiers,
		sessionSubscriptions,
	)

	if err != nil {
		s.Log.Errorf("Disconnect error: %v", err)
	}

	return err
}

// RemoteDisconnect find a session by identifier and closes it
func (n *Node) RemoteDisconnect(msg *common.RemoteDisconnectMessage) {
	n.Metrics.Counter(metricsBroadcastMsg).Inc()
	n.log.Debugf("Incoming pubsub command: %v", msg)
	n.hub.RemoteDisconnect(msg)
}

func transmit(s *Session, transmissions []string) {
	for _, msg := range transmissions {
		s.SendJSONTransmission(msg)
	}
}

func (n *Node) handleCommandReply(s *Session, msg *common.Message, reply *common.CommandResult) {
	if reply.Disconnect {
		defer s.Disconnect("Command Failed", ws.CloseAbnormalClosure)
	}

	if reply.StopAllStreams {
		n.hub.RemoveAllSubscriptions(s.UID, msg.Identifier)
	} else if reply.StoppedStreams != nil {
		for _, stream := range reply.StoppedStreams {
			n.hub.RemoveSubscription(s.UID, msg.Identifier, stream)
		}
	}

	if reply.Streams != nil {
		for _, stream := range reply.Streams {
			n.hub.subscribeSession(s.UID, stream, msg.Identifier)
		}
	}

	if reply.IState != nil {
		s.smu.Lock()
		s.env.MergeChannelState(msg.Identifier, &reply.IState)
		s.smu.Unlock()
	}

	n.handleCallReply(s, reply.ToCallResult())
}

func (n *Node) handleCallReply(s *Session, reply *common.CallResult) {
	if reply.CState != nil {
		s.smu.Lock()
		s.env.MergeConnectionState(&reply.CState)
		s.smu.Unlock()
	}

	if reply.Broadcasts != nil {
		for _, broadcast := range reply.Broadcasts {
			n.Broadcast(broadcast)
		}
	}

	if reply.Transmissions != nil {
		transmit(s, reply.Transmissions)
	}
}

func (n *Node) collectStats() {
	statsCollectInterval := time.Duration(n.config.StatsRefreshInterval) * time.Second

	for {
		select {
		case <-n.shutdownCh:
			return
		case <-time.After(statsCollectInterval):
			n.collectStatsOnce()
		}
	}
}

func (n *Node) collectStatsOnce() {
	n.Metrics.Gauge(metricsGoroutines).Set(runtime.NumGoroutine())

	var m runtime.MemStats
	runtime.ReadMemStats(&m)
	n.Metrics.Gauge(metricsMemSys).Set64(m.Sys)

	n.Metrics.Gauge(metricsClientsNum).Set(n.hub.Size())
	n.Metrics.Gauge(metricsUniqClientsNum).Set(n.hub.UniqSize())
	n.Metrics.Gauge(metricsStreamsNum).Set(n.hub.StreamsSize())
	n.Metrics.Gauge(metricsDisconnectQueue).Set(n.disconnector.Size())
}

func (n *Node) registerMetrics() {
	n.Metrics.RegisterGauge(metricsGoroutines, "The number of Go routines")
	n.Metrics.RegisterGauge(metricsMemSys, "The total bytes of memory obtained from the OS")

	n.Metrics.RegisterGauge(metricsClientsNum, "The number of active clients")
	n.Metrics.RegisterGauge(metricsUniqClientsNum, "The number of unique clients (with respect to connection identifiers)")
	n.Metrics.RegisterGauge(metricsStreamsNum, "The number of active broadcasting streams")
	n.Metrics.RegisterGauge(metricsDisconnectQueue, "The size of delayed disconnect")

	n.Metrics.RegisterCounter(metricsFailedAuths, "The total number of failed authentication attempts")
	n.Metrics.RegisterCounter(metricsReceivedMsg, "The total number of received messages from clients")
	n.Metrics.RegisterCounter(metricsFailedCommandReceived, "The total number of unrecognized messages received from clients")
	n.Metrics.RegisterCounter(metricsBroadcastMsg, "The total number of messages received through PubSub (for broadcast)")
	n.Metrics.RegisterCounter(metricsUnknownBroadcast, "The total number of unrecognized messages received through PubSub")

	n.Metrics.RegisterCounter(metricsSentMsg, "The total number of messages sent to clients")
	n.Metrics.RegisterCounter(metricsFailedSent, "The total number of messages failed to send to clients")

	n.Metrics.RegisterCounter(metricsDataSent, "The total amount of bytes sent to clients")
	n.Metrics.RegisterCounter(metricsDataReceived, "The total amount of bytes received from clients")
}

func subscriptionsList(m map[string]bool) []string {
	keys := make([]string, len(m))
	i := 0

	for k := range m {
		keys[i] = k
		i++
	}
	return keys
}

func newDisconnectMessage(reason string, reconnect bool) *common.DisconnectMessage {
	return &common.DisconnectMessage{Type: "disconnect", Reason: reason, Reconnect: reconnect}
}
<commit_msg>fix: prevent double node.Shutdown<commit_after>package node

import (
	"errors"
	"fmt"
	"runtime"
	"time"

	"github.com/anycable/anycable-go/common"
	"github.com/anycable/anycable-go/metrics"
	"github.com/anycable/anycable-go/ws"
	"github.com/apex/log"
)

const (
	// serverRestartReason is the disconnect reason on shutdown
	serverRestartReason    = "server_restart"
	remoteDisconnectReason = "remote"

	metricsGoroutines      = "goroutines_num"
	metricsMemSys          = "mem_sys_bytes"
	metricsClientsNum      = "clients_num"
	metricsUniqClientsNum  = "clients_uniq_num"
	metricsStreamsNum      = "broadcast_streams_num"
	metricsDisconnectQueue = "disconnect_queue_size"

	metricsFailedAuths           = "failed_auths_total"
	metricsReceivedMsg           = "client_msg_total"
	metricsFailedCommandReceived = "failed_client_msg_total"
	metricsBroadcastMsg          = "broadcast_msg_total"
	metricsUnknownBroadcast      = "failed_broadcast_msg_total"

	metricsSentMsg    = "server_msg_total"
	metricsFailedSent = "failed_server_msg_total"

	metricsDataSent     = "data_sent_total"
	metricsDataReceived = "data_rcvd_total"
)

// AppNode describes a basic node interface
type AppNode interface {
	HandlePubSub(msg []byte)
	Authenticate(s *Session) (*common.ConnectResult, error)
	Subscribe(s *Session, msg *common.Message) (*common.CommandResult, error)
	Unsubscribe(s *Session, msg *common.Message) (*common.CommandResult, error)
	Perform(s *Session, msg *common.Message) (*common.CommandResult, error)
	Disconnect(s *Session) error
}

// Connection represents underlying connection
type Connection interface {
	Write(msg []byte, deadline time.Time) error
	WriteBinary(msg []byte, deadline time.Time) error
	Read() ([]byte, error)
	Close(code int, reason string)
}

// Node represents the whole application
type Node struct {
	Metrics *metrics.Metrics

	config       *Config
	hub          *Hub
	controller   Controller
	disconnector Disconnector
	shutdownCh   chan struct{}
	log          *log.Entry
}

// NewNode builds new node struct
func NewNode(controller Controller, metrics *metrics.Metrics, config *Config) *Node {
	node := &Node{
		Metrics:    metrics,
		config:     config,
		controller: controller,
		shutdownCh: make(chan struct{}),
		log:        log.WithFields(log.Fields{"context": "node"}),
	}

	node.hub = NewHub(config.HubGopoolSize)

	node.registerMetrics()

	return node
}

// Start runs all the required goroutines
func (n *Node) Start() error {
	go n.hub.Run()
	go n.collectStats()

	return nil
}

// SetDisconnector set disconnector for the node
func (n *Node) SetDisconnector(d Disconnector) {
	n.disconnector = d
}

// HandleCommand parses incoming message from client and
// execute the command (if recognized)
func (n *Node) HandleCommand(s *Session, msg *common.Message) (err error) {
	s.Log.Debugf("Incoming message: %s", msg)
	switch msg.Command {
	case "subscribe":
		_, err = n.Subscribe(s, msg)
	case "unsubscribe":
		_, err = n.Unsubscribe(s, msg)
	case "message":
		_, err = n.Perform(s, msg)
	default:
		err = fmt.Errorf("Unknown command: %s", msg.Command)
	}

	return
}

// HandlePubSub parses incoming pubsub message and broadcast it
func (n *Node) HandlePubSub(raw []byte) {
	msg, err := common.PubSubMessageFromJSON(raw)

	if err != nil {
		n.Metrics.Counter(metricsUnknownBroadcast).Inc()
		n.log.Warnf("Failed to parse pubsub message '%s' with error: %v", raw, err)
		return
	}

	switch v := msg.(type) {
	case common.StreamMessage:
		n.Broadcast(&v)
	case common.RemoteDisconnectMessage:
		n.RemoteDisconnect(&v)
	}
}

// Shutdown stops all services (hub, controller)
func (n *Node) Shutdown() (err error) {
	if n.shutdownCh == nil {
		return errors.New("Already shut down")
	}

	close(n.shutdownCh)
	n.shutdownCh = nil

	if n.hub != nil {
		n.hub.Shutdown()

		active := n.hub.Size()

		if active > 0 {
			n.log.Infof("Closing active connections: %d", active)
			disconnectMessage := newDisconnectMessage(serverRestartReason, true)
			// Close all registered sessions
			n.hub.sessionsMu.RLock()
			for _, session := range n.hub.sessions {
				session.Send(disconnectMessage)
				session.Disconnect("Shutdown", ws.CloseGoingAway)
			}
			n.hub.sessionsMu.RUnlock()

			n.log.Info("All active connections closed")

			// Wait to make sure that disconnect queue is not empty
			time.Sleep(time.Second)
		}
	}

	if n.disconnector != nil {
		err := n.disconnector.Shutdown()

		if err != nil {
			n.log.Warnf("%v", err)
		}
	}

	if n.controller != nil {
		err := n.controller.Shutdown()

		if err != nil {
			n.log.Warnf("%v", err)
		}
	}

	return
}

// Authenticate calls controller to perform authentication.
// If authentication is successful, session is registered with a hub.
func (n *Node) Authenticate(s *Session) (res *common.ConnectResult, err error) {
	res, err = n.controller.Authenticate(s.UID, s.env)

	if err == nil {
		s.Identifiers = res.Identifier
		s.Connected = true

		n.hub.addSession(s)
	} else {
		n.Metrics.Counter(metricsFailedAuths).Inc()
	}

	if res != nil {
		n.handleCallReply(s, res.ToCallResult())
	}

	if err != nil {
		s.Disconnect("Auth Error", ws.CloseInternalServerErr)
	}

	return
}

// Subscribe subscribes session to a channel
func (n *Node) Subscribe(s *Session, msg *common.Message) (res *common.CommandResult, err error) {
	s.smu.Lock()

	if _, ok := s.subscriptions[msg.Identifier]; ok {
		s.smu.Unlock()
		err = fmt.Errorf("Already subscribed to %s", msg.Identifier)
		return
	}

	res, err = n.controller.Subscribe(s.UID, s.env, s.Identifiers, msg.Identifier)

	if err != nil {
		s.Log.Errorf("Subscribe error: %v", err)
	} else {
		s.subscriptions[msg.Identifier] = true
		s.Log.Debugf("Subscribed to channel: %s", msg.Identifier)
	}

	s.smu.Unlock()

	if res != nil {
		n.handleCommandReply(s, msg, res)
	}

	return
}

// Unsubscribe unsubscribes session from a channel
func (n *Node) Unsubscribe(s *Session, msg *common.Message) (res *common.CommandResult, err error) {
	s.smu.Lock()

	if _, ok := s.subscriptions[msg.Identifier]; !ok {
		s.smu.Unlock()
		err = fmt.Errorf("Unknown subscription %s", msg.Identifier)
		return
	}

	res, err = n.controller.Unsubscribe(s.UID, s.env, s.Identifiers, msg.Identifier)

	if err != nil {
		s.Log.Errorf("Unsubscribe error: %v", err)
	} else {
		// Make sure to remove all streams subscriptions
		res.StopAllStreams = true

		delete(s.subscriptions, msg.Identifier)

		s.Log.Debugf("Unsubscribed from channel: %s", msg.Identifier)
	}

	s.smu.Unlock()

	if res != nil {
		n.handleCommandReply(s, msg, res)
	}

	return
}

// Perform executes client command
func (n *Node) Perform(s *Session, msg *common.Message) (res *common.CommandResult, err error) {
	s.smu.Lock()

	if _, ok := s.subscriptions[msg.Identifier]; !ok {
		s.smu.Unlock()
		err = fmt.Errorf("Unknown subscription %s", msg.Identifier)
		return
	}

	s.smu.Unlock()

	res, err = n.controller.Perform(s.UID, s.env, s.Identifiers, msg.Identifier, msg.Data)

	if err != nil {
		s.Log.Errorf("Perform error: %v", err)
	} else {
		s.Log.Debugf("Perform result: %v", res)
	}

	if res != nil {
		n.handleCommandReply(s, msg, res)
	}

	return
}

// Broadcast message to stream
func (n *Node) Broadcast(msg *common.StreamMessage) {
	n.Metrics.Counter(metricsBroadcastMsg).Inc()
	n.log.Debugf("Incoming pubsub message: %v", msg)
	n.hub.BroadcastMessage(msg)
}

// Disconnect adds session to disconnector queue and unregister session from hub
func (n *Node) Disconnect(s *Session) error {
	n.hub.RemoveSession(s)
	return n.disconnector.Enqueue(s)
}

// DisconnectNow execute disconnect on controller
func (n *Node) DisconnectNow(s *Session) error {
	sessionSubscriptions := subscriptionsList(s.subscriptions)

	s.Log.Debugf("Disconnect %s %s %v %v", s.Identifiers, s.env.URL, s.env.Headers, sessionSubscriptions)

	err := n.controller.Disconnect(
		s.UID,
		s.env,
		s.Identifiers,
		sessionSubscriptions,
	)

	if err != nil {
		s.Log.Errorf("Disconnect error: %v", err)
	}

	return err
}

// RemoteDisconnect find a session by identifier and closes it
func (n *Node) RemoteDisconnect(msg *common.RemoteDisconnectMessage) {
	n.Metrics.Counter(metricsBroadcastMsg).Inc()
	n.log.Debugf("Incoming pubsub command: %v", msg)
	n.hub.RemoteDisconnect(msg)
}

func transmit(s *Session, transmissions []string) {
	for _, msg := range transmissions {
		s.SendJSONTransmission(msg)
	}
}

func (n *Node) handleCommandReply(s *Session, msg *common.Message, reply *common.CommandResult) {
	if reply.Disconnect {
		defer s.Disconnect("Command Failed", ws.CloseAbnormalClosure)
	}

	if reply.StopAllStreams {
		n.hub.RemoveAllSubscriptions(s.UID, msg.Identifier)
	} else if reply.StoppedStreams != nil {
		for _, stream := range reply.StoppedStreams {
			n.hub.RemoveSubscription(s.UID, msg.Identifier, stream)
		}
	}

	if reply.Streams != nil {
		for _, stream := range reply.Streams {
			n.hub.subscribeSession(s.UID, stream, msg.Identifier)
		}
	}

	if reply.IState != nil {
		s.smu.Lock()
		s.env.MergeChannelState(msg.Identifier, &reply.IState)
		s.smu.Unlock()
	}

	n.handleCallReply(s, reply.ToCallResult())
}

func (n *Node) handleCallReply(s *Session, reply *common.CallResult) {
	if reply.CState != nil {
		s.smu.Lock()
		s.env.MergeConnectionState(&reply.CState)
		s.smu.Unlock()
	}

	if reply.Broadcasts != nil {
		for _, broadcast := range reply.Broadcasts {
			n.Broadcast(broadcast)
		}
	}

	if reply.Transmissions != nil {
		transmit(s, reply.Transmissions)
	}
}

func (n *Node) collectStats() {
	statsCollectInterval := time.Duration(n.config.StatsRefreshInterval) * time.Second

	for {
		select {
		case <-n.shutdownCh:
			return
		case <-time.After(statsCollectInterval):
			n.collectStatsOnce()
		}
	}
}

func (n *Node) collectStatsOnce() {
	n.Metrics.Gauge(metricsGoroutines).Set(runtime.NumGoroutine())

	var m runtime.MemStats
	runtime.ReadMemStats(&m)
	n.Metrics.Gauge(metricsMemSys).Set64(m.Sys)

	n.Metrics.Gauge(metricsClientsNum).Set(n.hub.Size())
	n.Metrics.Gauge(metricsUniqClientsNum).Set(n.hub.UniqSize())
	n.Metrics.Gauge(metricsStreamsNum).Set(n.hub.StreamsSize())
	n.Metrics.Gauge(metricsDisconnectQueue).Set(n.disconnector.Size())
}

func (n *Node) registerMetrics() {
	n.Metrics.RegisterGauge(metricsGoroutines, "The number of Go routines")
	n.Metrics.RegisterGauge(metricsMemSys, "The total bytes of memory obtained from the OS")

	n.Metrics.RegisterGauge(metricsClientsNum, "The number of active clients")
	n.Metrics.RegisterGauge(metricsUniqClientsNum, "The number of unique clients (with respect to connection identifiers)")
	n.Metrics.RegisterGauge(metricsStreamsNum, "The number of active broadcasting streams")
	n.Metrics.RegisterGauge(metricsDisconnectQueue, "The size of delayed disconnect")

	n.Metrics.RegisterCounter(metricsFailedAuths, "The total number of failed authentication attempts")
	n.Metrics.RegisterCounter(metricsReceivedMsg, "The total number of received messages from clients")
	n.Metrics.RegisterCounter(metricsFailedCommandReceived, "The total number of unrecognized messages received from clients")
	n.Metrics.RegisterCounter(metricsBroadcastMsg, "The total number of messages received through PubSub (for broadcast)")
	n.Metrics.RegisterCounter(metricsUnknownBroadcast, "The total number of unrecognized messages received through PubSub")

	n.Metrics.RegisterCounter(metricsSentMsg, "The total number of messages sent to clients")
	n.Metrics.RegisterCounter(metricsFailedSent, "The total number of messages failed to send to clients")

	n.Metrics.RegisterCounter(metricsDataSent, "The total amount of bytes sent to clients")
	n.Metrics.RegisterCounter(metricsDataReceived, "The total amount of bytes received from clients")
}

func subscriptionsList(m map[string]bool) []string {
	keys := make([]string, len(m))
	i := 0

	for k := range m {
		keys[i] = k
		i++
	}
	return keys
}

func newDisconnectMessage(reason string, reconnect bool) *common.DisconnectMessage {
	return &common.DisconnectMessage{Type: "disconnect", Reason: reason, Reconnect: reconnect}
}
<|endoftext|>


<commit_before>// Copyright 2012 Xing Xing <mikespook@gmail.com>.
// All rights reserved.
// Use of this source code is governed by a commercial
// license that can be found in the LICENSE file.

package node

import (
    "os"
    "fmt"
    "sync"
    "time"
    "encoding/json"
    "github.com/ha/doozer"
    "github.com/mikespook/golib/funcmap"
    "github.com/mikespook/golib/log"
)

const (
    MaxErrorCount = 5
    FuncPoolSize = 30
    WireFile = "%s/wire"
    DefaultRegion = "/z-node"
    NodeFile = DefaultRegion + "/node/%s/%d"
    NodeInfo = DefaultRegion + "/info/%s/%d"
)

type ZNode struct {
    ErrHandler ErrorHandlerFunc

    conn *doozer.Conn
    uri, buri, hostname string
    regions []string
    pid int
    rev int64
    fmap funcmap.Funcs
    w sync.WaitGroup
}

type ZFunc struct {
    Name string
    Params []interface{}
}

func New(hostname string, regions ... string) (node *ZNode) {
    if len(regions) == 0 {
        regions = []string{DefaultRegion}
    }
    for i, _ := range regions {
       if regions[i][0] != '/' {
            regions[i] = "/" + regions[i]
       }
    }
    return &ZNode {
        regions: regions,
        hostname: hostname,
        pid: os.Getpid(),
        fmap: funcmap.New(FuncPoolSize),
    }
}

func (node *ZNode) Bind(name string, fn interface{}) (err error) {
    return node.fmap.Bind(name, fn)
}

func (node *ZNode) Start(uri, buri string) (err error) {
    node.uri = uri
    node.buri = buri

    if node.conn, err = doozer.DialUri(node.uri, node.buri); err != nil {
        return
    }
    if node.rev, err = node.conn.Rev(); err != nil {
        return
    }
    if node.rev, err = node.conn.Set(
        node.infopath(),
        node.rev, []byte(time.Now().String()));
        err != nil {
        return
    }
    node.watchSelf()
    node.watchWire()
    return
}

func (node *ZNode) Close() {
    if err := node.conn.Del(
        node.infopath(),
        node.rev); err != nil {
        node.err(err)
    }
    if err := node.conn.Del(
        node.nodepath(),
        node.rev); err != nil {
        node.err(err)
    }
    node.conn.Close()
    zNodeMod.Decref()
}

func (node *ZNode) Wait() {
    node.w.Wait()
}

func (node *ZNode) err(err error) {
    if node.ErrHandler != nil {
        node.ErrHandler(err)
    }
}

func (node *ZNode) watch(file string) {
    defer node.w.Done()
    for i := 0; i < MaxErrorCount; i ++ {
        ev, err := node.conn.Wait(file, node.rev)
        if err != nil {
            if err == doozer.ErrClosed {
                break
            }
            node.err(err)
            continue
        }
        node.rev = ev.Rev + 1
        if ev.IsDel() {
            break
        }
        if ev.IsSet() {
            var fn ZFunc
            if err := json.Unmarshal(ev.Body, &fn); err != nil {
                node.err(err)
                continue
            }
            go node.Call(fn.Name, fn.Params ...)
            i = 0
        }
    }
}

func (node *ZNode) Call(name string, params ... interface{}) {
    if _, ok := node.fmap[name]; ok {
        log.Messagef("Call Go function %s, %t supplied.", name, params)
        if _, err := node.fmap.Call(name, params ...); err != nil {
            node.err(err)
        }
        return
    }
    log.Messagef("Call Python script %s, %t supplied.", name, params)
    if err := execPython(name, params ...); err != nil {
        node.err(err)
    }
}

func (node *ZNode) infopath() string {
    return fmt.Sprintf(NodeInfo, node.hostname, node.pid)
}

func (node *ZNode) nodepath() string {
    return fmt.Sprintf(NodeFile, node.hostname, node.pid)
}

func (node *ZNode) wirepath(region string) string {
    return fmt.Sprintf(WireFile, region)
}

func (node *ZNode) watchSelf() {
    node.w.Add(1)
    go node.watch(node.nodepath())
}

func (node *ZNode) watchWire() {
    for _, v := range node.regions {
        node.w.Add(1)
        go node.watch(node.wirepath(v))
    }
}
<commit_msg>fixed deleting path issue<commit_after>// Copyright 2012 Xing Xing <mikespook@gmail.com>.
// All rights reserved.
// Use of this source code is governed by a commercial
// license that can be found in the LICENSE file.

package node

import (
    "os"
    "fmt"
    "sync"
    "time"
    "encoding/json"
    "github.com/ha/doozer"
    "github.com/mikespook/golib/funcmap"
    "github.com/mikespook/golib/log"
)

const (
    MaxErrorCount = 5
    FuncPoolSize = 30
    WireFile = "%s/wire"
    DefaultRegion = "/z-node"
    NodeFile = DefaultRegion + "/node/%s/%d"
    NodeInfo = DefaultRegion + "/info/%s/%d"
)

type ZNode struct {
    ErrHandler ErrorHandlerFunc

    conn *doozer.Conn
    uri, buri, hostname string
    regions []string
    pid int
    rev int64
    fmap funcmap.Funcs
    w sync.WaitGroup
}

type ZFunc struct {
    Name string
    Params []interface{}
}

func New(hostname string, regions ... string) (node *ZNode) {
    if len(regions) == 0 {
        regions = []string{DefaultRegion}
    }
    for i, _ := range regions {
       if regions[i][0] != '/' {
            regions[i] = "/" + regions[i]
       }
    }
    return &ZNode {
        regions: regions,
        hostname: hostname,
        pid: os.Getpid(),
        fmap: funcmap.New(FuncPoolSize),
    }
}

func (node *ZNode) Bind(name string, fn interface{}) (err error) {
    return node.fmap.Bind(name, fn)
}

func (node *ZNode) Start(uri, buri string) (err error) {
    node.uri = uri
    node.buri = buri

    if node.conn, err = doozer.DialUri(node.uri, node.buri); err != nil {
        return
    }
    if node.rev, err = node.conn.Rev(); err != nil {
        return
    }
    if node.rev, err = node.conn.Set(
        node.infopath(),
        node.rev, []byte(time.Now().String()));
        err != nil {
        return
    }
    node.watchSelf()
    node.watchWire()
    return
}

func (node *ZNode) Close() {
    if err := node.conn.Del(
        node.infopath(),
        node.rev); err != nil {
        node.err(err)
    }
    if err := node.conn.Del(
        node.nodepath(),
        node.rev); err != nil {
        node.err(err)
    }
    node.conn.Close()
    zNodeMod.Decref()
}

func (node *ZNode) Wait() {
    node.w.Wait()
}

func (node *ZNode) err(err error) {
    if node.ErrHandler != nil {
        node.ErrHandler(err)
    }
}

func (node *ZNode) watch(file string) {
    defer node.w.Done()
    for i := 0; i < MaxErrorCount; i ++ {
        ev, err := node.conn.Wait(file, node.rev)
        if err != nil {
            if err == doozer.ErrClosed {
                break
            }
            node.err(err)
            continue
        }
        node.rev = ev.Rev + 1
        if ev.IsSet() {
            var fn ZFunc
            if err := json.Unmarshal(ev.Body, &fn); err != nil {
                node.err(err)
                continue
            }
            go node.Call(fn.Name, fn.Params ...)
            i = 0
        }
    }
}

func (node *ZNode) Call(name string, params ... interface{}) {
    if _, ok := node.fmap[name]; ok {
        log.Messagef("Call Go function %s, %t supplied.", name, params)
        if _, err := node.fmap.Call(name, params ...); err != nil {
            node.err(err)
        }
        return
    }
    log.Messagef("Call Python script %s, %t supplied.", name, params)
    if err := execPython(name, params ...); err != nil {
        node.err(err)
    }
}

func (node *ZNode) infopath() string {
    return fmt.Sprintf(NodeInfo, node.hostname, node.pid)
}

func (node *ZNode) nodepath() string {
    return fmt.Sprintf(NodeFile, node.hostname, node.pid)
}

func (node *ZNode) wirepath(region string) string {
    return fmt.Sprintf(WireFile, region)
}

func (node *ZNode) watchSelf() {
    node.w.Add(1)
    go node.watch(node.nodepath())
}

func (node *ZNode) watchWire() {
    for _, v := range node.regions {
        node.w.Add(1)
        go node.watch(node.wirepath(v))
    }
}
<|endoftext|>


<commit_before>package corerepo

import (
	"fmt"
	"time"

	context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"

	"github.com/ipfs/go-ipfs/core"
	"github.com/ipfs/go-ipfs/merkledag"
	path "github.com/ipfs/go-ipfs/path"
	u "github.com/ipfs/go-ipfs/util"
)

func Pin(n *core.IpfsNode, paths []string, recursive bool) ([]u.Key, error) {

	dagnodes := make([]*merkledag.Node, 0)
	for _, fpath := range paths {
		dagnode, err := core.Resolve(n, path.Path(fpath))
		if err != nil {
			return nil, fmt.Errorf("pin: %s", err)
		}
		dagnodes = append(dagnodes, dagnode)
	}

	var out []u.Key
	for _, dagnode := range dagnodes {
		k, err := dagnode.Key()
		if err != nil {
			return nil, err
		}

		ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
		defer cancel()
		err = n.Pinning.Pin(ctx, dagnode, recursive)
		if err != nil {
			return nil, fmt.Errorf("pin: %s", err)
		}
		out = append(out, k)
	}

	err := n.Pinning.Flush()
	if err != nil {
		return nil, err
	}

	return out, nil
}

func Unpin(n *core.IpfsNode, paths []string, recursive bool) ([]u.Key, error) {

	dagnodes := make([]*merkledag.Node, 0)
	for _, fpath := range paths {
		dagnode, err := core.Resolve(n, path.Path(fpath))
		if err != nil {
			return nil, err
		}
		dagnodes = append(dagnodes, dagnode)
	}

	var unpinned []u.Key
	for _, dagnode := range dagnodes {
		k, _ := dagnode.Key()

		ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
		defer cancel()
		err := n.Pinning.Unpin(ctx, k, recursive)
		if err != nil {
			return nil, err
		}
		unpinned = append(unpinned, k)
	}

	err := n.Pinning.Flush()
	if err != nil {
		return nil, err
	}
	return unpinned, nil
}
<commit_msg>core/corerepo/pinning: Add a package comment<commit_after>/*
Package corerepo provides pinning and garbage collection for local
IPFS block services.

IPFS nodes will keep local copies of any object that have either been
added or requested locally.  Not all of these objects are worth
preserving forever though, so the node adminstrator can pin objects
they want to keep and unpin objects that they don't care about.

Garbage collection sweeps iterate through the local block store
removing objects that aren't pinned, which frees storage space for new
objects.
*/
package corerepo

import (
	"fmt"
	"time"

	context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"

	"github.com/ipfs/go-ipfs/core"
	"github.com/ipfs/go-ipfs/merkledag"
	path "github.com/ipfs/go-ipfs/path"
	u "github.com/ipfs/go-ipfs/util"
)

func Pin(n *core.IpfsNode, paths []string, recursive bool) ([]u.Key, error) {

	dagnodes := make([]*merkledag.Node, 0)
	for _, fpath := range paths {
		dagnode, err := core.Resolve(n, path.Path(fpath))
		if err != nil {
			return nil, fmt.Errorf("pin: %s", err)
		}
		dagnodes = append(dagnodes, dagnode)
	}

	var out []u.Key
	for _, dagnode := range dagnodes {
		k, err := dagnode.Key()
		if err != nil {
			return nil, err
		}

		ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
		defer cancel()
		err = n.Pinning.Pin(ctx, dagnode, recursive)
		if err != nil {
			return nil, fmt.Errorf("pin: %s", err)
		}
		out = append(out, k)
	}

	err := n.Pinning.Flush()
	if err != nil {
		return nil, err
	}

	return out, nil
}

func Unpin(n *core.IpfsNode, paths []string, recursive bool) ([]u.Key, error) {

	dagnodes := make([]*merkledag.Node, 0)
	for _, fpath := range paths {
		dagnode, err := core.Resolve(n, path.Path(fpath))
		if err != nil {
			return nil, err
		}
		dagnodes = append(dagnodes, dagnode)
	}

	var unpinned []u.Key
	for _, dagnode := range dagnodes {
		k, _ := dagnode.Key()

		ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
		defer cancel()
		err := n.Pinning.Unpin(ctx, k, recursive)
		if err != nil {
			return nil, err
		}
		unpinned = append(unpinned, k)
	}

	err := n.Pinning.Flush()
	if err != nil {
		return nil, err
	}
	return unpinned, nil
}
<|endoftext|>


<commit_before>package post

import (
	bf "github.com/russross/blackfriday"
	//blackfriday2 "gopkg.in/russross/blackfriday.v2"
)

// These are copied from
// https://github.com/russross/blackfriday/blob/5b2fb1b893850a20a483145b676af75eaad51a5d/markdown.go#L48
const (
	commonHtmlFlags = 0 |
		bf.HTML_USE_XHTML |
		bf.HTML_USE_SMARTYPANTS |
		bf.HTML_SMARTYPANTS_FRACTIONS |
		bf.HTML_SMARTYPANTS_DASHES |
		bf.HTML_SMARTYPANTS_LATEX_DASHES

	commonExtensions = 0 |
		bf.EXTENSION_NO_INTRA_EMPHASIS |
		bf.EXTENSION_TABLES |
		bf.EXTENSION_FENCED_CODE |
		bf.EXTENSION_AUTOLINK |
		bf.EXTENSION_STRIKETHROUGH |
		bf.EXTENSION_SPACE_HEADERS |
		bf.EXTENSION_HEADER_IDS |
		bf.EXTENSION_BACKSLASH_LINE_BREAK |
		bf.EXTENSION_DEFINITION_LISTS |
		bf.EXTENSION_AUTO_HEADER_IDS
)

func compileMarkdownToHTML(input []byte) []byte {
	// Use the default extensions
	//return bf.MarkdownCommon(input);

	// Use custom extensions (defaults + extra)
	renderer := bf.HtmlRenderer(commonHtmlFlags, "", "")
	return bf.Markdown(input, renderer, commonExtensions)
}
<commit_msg>Upgrade to blackfriday v2<commit_after>package post

import (
	bf2 "gopkg.in/russross/blackfriday.v2"
)

// These are copied from
// https://github.com/russross/blackfriday/blob/v2/markdown.go#L32
func compileMarkdownToHTML(input []byte) []byte {
	return bf2.Run(input, bf2.WithExtensions(bf2.CommonExtensions|
		bf2.AutoHeadingIDs))
}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build linux,!android darwin,!arm,!arm64

package sensor

import (
	"errors"
	"time"
)

type manager struct {
}

func (m *manager) initialize() {
}

func (m *manager) enable(t Type, delay time.Duration) error {
	return errors.New("sensor: no sensors available")
}

func (m *manager) disable(t Type) error {
	return errors.New("sensor: no sensors available")
}

func (m *manager) read(e []Event) (n int, err error) {
	return 0, errors.New("sensor: no sensor data available")
}

func (m *manager) close() error {
	return nil
}
<commit_msg>Fix the broken build for non-darwin and non-android.<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build linux,!android darwin,!arm,!arm64

package sensor

import (
	"errors"
	"time"
)

type manager struct{}

func (m *manager) initialize() {}

func (m *manager) enable(s sender, t Type, delay time.Duration) error {
	return errors.New("sensor: no sensors available")
}

func (m *manager) disable(t Type) error {
	return errors.New("sensor: no sensors available")
}

func (m *manager) close() error {
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package image

// Color can convert itself to alpha-premultiplied RGBA, with a possible loss
// of precision. Each value ranges within [0, 0xFFFF], but is represented by a
// uint32 so that multiplying by a blend factor up to 0xFFFF will not overflow.
type Color interface {
	RGBA() (r, g, b, a uint32)
}

// RGBAColor represents a traditional 32-bit alpha-premultiplied color,
// having 8 bits for each of red, green, blue and alpha.
type RGBAColor struct {
	R, G, B, A uint8
}

func (c RGBAColor) RGBA() (r, g, b, a uint32) {
	r = uint32(c.R)
	r |= r << 8
	g = uint32(c.G)
	g |= g << 8
	b = uint32(c.B)
	b |= b << 8
	a = uint32(c.A)
	a |= a << 8
	return
}

// RGBA64Color represents a 64-bit alpha-premultiplied color,
// having 16 bits for each of red, green, blue and alpha.
type RGBA64Color struct {
	R, G, B, A uint16
}

func (c RGBA64Color) RGBA() (r, g, b, a uint32) {
	return uint32(c.R), uint32(c.G), uint32(c.B), uint32(c.A)
}

// NRGBAColor represents a non-alpha-premultiplied 32-bit color.
type NRGBAColor struct {
	R, G, B, A uint8
}

func (c NRGBAColor) RGBA() (r, g, b, a uint32) {
	r = uint32(c.R)
	r |= r << 8
	r *= uint32(c.A)
	r /= 0xff
	g = uint32(c.G)
	g |= g << 8
	g *= uint32(c.A)
	g /= 0xff
	b = uint32(c.B)
	b |= b << 8
	b *= uint32(c.A)
	b /= 0xff
	a = uint32(c.A)
	a |= a << 8
	return
}

// NRGBA64Color represents a non-alpha-premultiplied 64-bit color,
// having 16 bits for each of red, green, blue and alpha.
type NRGBA64Color struct {
	R, G, B, A uint16
}

func (c NRGBA64Color) RGBA() (r, g, b, a uint32) {
	r = uint32(c.R)
	r *= uint32(c.A)
	r /= 0xffff
	g = uint32(c.G)
	g *= uint32(c.A)
	g /= 0xffff
	b = uint32(c.B)
	b *= uint32(c.A)
	b /= 0xffff
	a = uint32(c.A)
	return
}

// AlphaColor represents an 8-bit alpha.
type AlphaColor struct {
	A uint8
}

func (c AlphaColor) RGBA() (r, g, b, a uint32) {
	a = uint32(c.A)
	a |= a << 8
	return a, a, a, a
}

// Alpha16Color represents a 16-bit alpha.
type Alpha16Color struct {
	A uint16
}

func (c Alpha16Color) RGBA() (r, g, b, a uint32) {
	a = uint32(c.A)
	return a, a, a, a
}

// GrayColor represents an 8-bit grayscale color.
type GrayColor struct {
	Y uint8
}

func (c GrayColor) RGBA() (r, g, b, a uint32) {
	y := uint32(c.Y)
	y |= y << 8
	return y, y, y, 0xffff
}

// Gray16Color represents a 16-bit grayscale color.
type Gray16Color struct {
	Y uint16
}

func (c Gray16Color) RGBA() (r, g, b, a uint32) {
	y := uint32(c.Y)
	return y, y, y, 0xffff
}

// ColorModel can convert foreign Colors, with a possible loss of precision,
// to a Color from its own color model.
type ColorModel interface {
	Convert(c Color) Color
}

// The ColorModelFunc type is an adapter to allow the use of an ordinary
// color conversion function as a ColorModel.  If f is such a function,
// ColorModelFunc(f) is a ColorModel object that invokes f to implement
// the conversion.
type ColorModelFunc func(Color) Color

func (f ColorModelFunc) Convert(c Color) Color {
	return f(c)
}

func toRGBAColor(c Color) Color {
	if _, ok := c.(RGBAColor); ok {
		return c
	}
	r, g, b, a := c.RGBA()
	return RGBAColor{uint8(r >> 8), uint8(g >> 8), uint8(b >> 8), uint8(a >> 8)}
}

func toRGBA64Color(c Color) Color {
	if _, ok := c.(RGBA64Color); ok {
		return c
	}
	r, g, b, a := c.RGBA()
	return RGBA64Color{uint16(r), uint16(g), uint16(b), uint16(a)}
}

func toNRGBAColor(c Color) Color {
	if _, ok := c.(NRGBAColor); ok {
		return c
	}
	r, g, b, a := c.RGBA()
	if a == 0xffff {
		return NRGBAColor{uint8(r >> 8), uint8(g >> 8), uint8(b >> 8), 0xff}
	}
	if a == 0 {
		return NRGBAColor{0, 0, 0, 0}
	}
	// Since Color.RGBA returns a alpha-premultiplied color, we should have r <= a && g <= a && b <= a.
	r = (r * 0xffff) / a
	g = (g * 0xffff) / a
	b = (b * 0xffff) / a
	return NRGBAColor{uint8(r >> 8), uint8(g >> 8), uint8(b >> 8), uint8(a >> 8)}
}

func toNRGBA64Color(c Color) Color {
	if _, ok := c.(NRGBA64Color); ok {
		return c
	}
	r, g, b, a := c.RGBA()
	if a == 0xffff {
		return NRGBA64Color{uint16(r), uint16(g), uint16(b), 0xffff}
	}
	if a == 0 {
		return NRGBA64Color{0, 0, 0, 0}
	}
	// Since Color.RGBA returns a alpha-premultiplied color, we should have r <= a && g <= a && b <= a.
	r = (r * 0xffff) / a
	g = (g * 0xffff) / a
	b = (b * 0xffff) / a
	return NRGBA64Color{uint16(r), uint16(g), uint16(b), uint16(a)}
}

func toAlphaColor(c Color) Color {
	if _, ok := c.(AlphaColor); ok {
		return c
	}
	_, _, _, a := c.RGBA()
	return AlphaColor{uint8(a >> 8)}
}

func toAlpha16Color(c Color) Color {
	if _, ok := c.(Alpha16Color); ok {
		return c
	}
	_, _, _, a := c.RGBA()
	return Alpha16Color{uint16(a)}
}

func toGrayColor(c Color) Color {
	if _, ok := c.(GrayColor); ok {
		return c
	}
	r, g, b, _ := c.RGBA()
	y := (299*r + 587*g + 114*b + 500) / 1000
	return GrayColor{uint8(y >> 8)}
}

func toGray16Color(c Color) Color {
	if _, ok := c.(Gray16Color); ok {
		return c
	}
	r, g, b, _ := c.RGBA()
	y := (299*r + 587*g + 114*b + 500) / 1000
	return Gray16Color{uint16(y)}
}

// The ColorModel associated with RGBAColor.
var RGBAColorModel ColorModel = ColorModelFunc(toRGBAColor)

// The ColorModel associated with RGBA64Color.
var RGBA64ColorModel ColorModel = ColorModelFunc(toRGBA64Color)

// The ColorModel associated with NRGBAColor.
var NRGBAColorModel ColorModel = ColorModelFunc(toNRGBAColor)

// The ColorModel associated with NRGBA64Color.
var NRGBA64ColorModel ColorModel = ColorModelFunc(toNRGBA64Color)

// The ColorModel associated with AlphaColor.
var AlphaColorModel ColorModel = ColorModelFunc(toAlphaColor)

// The ColorModel associated with Alpha16Color.
var Alpha16ColorModel ColorModel = ColorModelFunc(toAlpha16Color)

// The ColorModel associated with GrayColor.
var GrayColorModel ColorModel = ColorModelFunc(toGrayColor)

// The ColorModel associated with Gray16Color.
var Gray16ColorModel ColorModel = ColorModelFunc(toGray16Color)
<commit_msg>image: delete obsolete color.go<commit_after><|endoftext|>