<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package discoverspaces

import (
	"github.com/juju/errors"
	"github.com/juju/loggo"
	"github.com/juju/names"
	"github.com/juju/utils/set"

	"github.com/juju/juju/apiserver/params"
	"github.com/juju/juju/environs"
	"github.com/juju/juju/worker"
	"github.com/juju/juju/worker/catacomb"
	"github.com/juju/juju/worker/gate"
)

// Facade exposes the relevant capabilities of a *discoverspaces.API; it's
// a bit raw but at least it's easily mockable.
type Facade interface {
	CreateSpaces(params.CreateSpacesParams) (params.ErrorResults, error)
	AddSubnets(params.AddSubnetsParams) (params.ErrorResults, error)
	ListSpaces() (params.DiscoverSpacesResults, error)
	ListSubnets(params.SubnetsFilters) (params.ListSubnetsResults, error)
}

// NameFunc returns a string derived from base that is not contained in used.
type NameFunc func(base string, used set.Strings) string

// Config defines the operation of a space discovery worker.
type Config struct {

	// Facade exposes the capabilities of a controller.
	Facade Facade

	// Environ exposes the capabilities of a compute substrate.
	Environ environs.Environ

	// NewName is used to sanitise, and make unique, space names as
	// reported by an Environ (for use in juju, via the Facade). You
	// should probably set it to ConvertSpaceName.
	NewName NameFunc

	// Unlocker, if not nil, will be unlocked when the first discovery
	// attempt completes successfully.
	Unlocker gate.Unlocker
}

// Validate returns an error if the config cannot be expected to
// drive a functional worker.
func (config Config) Validate() error {
	if config.Facade == nil {
		return errors.NotValidf("nil Facade")
	}
	if config.Environ == nil {
		return errors.NotValidf("nil Environ")
	}
	if config.NewName == nil {
		return errors.NotValidf("nil NewName")
	}
	// missing Unlocker gate just means "don't bother notifying"
	return nil
}

var logger = loggo.GetLogger("juju.worker.discoverspaces")

type discoverspacesWorker struct {
	catacomb catacomb.Catacomb
	config   Config
}

// NewWorker returns a worker that will attempt to discover the
// configured Environ's spaces, and update the controller via the
// configured Facade. Names are sanitised with NewName, and any
// supplied Unlocker will be Unlock()ed when the first complete
// discovery and update succeeds.
//
// Once that update completes, the worker just waits to be Kill()ed.
// We should probably poll for changes, really, but I'm making an
// effort to preserve existing behaviour where possible.
func NewWorker(config Config) (worker.Worker, error) {
	if err := config.Validate(); err != nil {
		return nil, errors.Trace(err)
	}
	dw := &discoverspacesWorker{
		config: config,
	}
	err := catacomb.Invoke(catacomb.Plan{
		Site: &dw.catacomb,
		Work: dw.loop,
	})
	if err != nil {
		return nil, errors.Trace(err)
	}
	return dw, nil
}

// Kill is part of the worker.Worker interface.
func (dw *discoverspacesWorker) Kill() {
	dw.catacomb.Kill(nil)
}

// Wait is part of the worker.Worker interface.
func (dw *discoverspacesWorker) Wait() error {
	return dw.catacomb.Wait()
}

func (dw *discoverspacesWorker) loop() (err error) {

	// TODO(mfoord): we'll have a watcher here checking if we need to
	// update the spaces/subnets definition.
	// TODO(fwereade): for now, use a changes channel that apes the
	// standard initial event behaviour, so we can make the loop
	// follow the standard structure.
	changes := make(chan struct{}, 1)
	changes <- struct{}{}

	gate := dw.config.Unlocker
	for {
		select {
		case <-dw.catacomb.Dying():
			return dw.catacomb.ErrDying()
		case <-changes:
			if err := dw.handleSubnets(); err != nil {
				return errors.Trace(err)
			}
			logger.Debugf("space discovery complete")
			if gate != nil {
				gate.Unlock()
				gate = nil
			}
		}
	}
}

func (dw *discoverspacesWorker) handleSubnets() error {
	environ, ok := environs.SupportsNetworking(dw.config.Environ)
	if !ok {
		logger.Debugf("not a networking environ")
		return nil
	}
	if supported, err := environ.SupportsSpaceDiscovery(); err != nil {
		return errors.Trace(err)
	} else if !supported {
		logger.Debugf("environ does not support space discovery")
		return nil
	}
	providerSpaces, err := environ.Spaces()
	if err != nil {
		return errors.Trace(err)
	}

	facade := dw.config.Facade
	listSpacesResult, err := facade.ListSpaces()
	if err != nil {
		return errors.Trace(err)
	}
	stateSubnets, err := facade.ListSubnets(params.SubnetsFilters{})
	if err != nil {
		return errors.Trace(err)
	}

	stateSubnetIds := make(set.Strings)
	for _, subnet := range stateSubnets.Results {
		stateSubnetIds.Add(subnet.ProviderId)
	}
	stateSpaceMap := make(map[string]params.ProviderSpace)
	spaceNames := make(set.Strings)
	for _, space := range listSpacesResult.Results {
		stateSpaceMap[space.ProviderId] = space
		spaceNames.Add(space.Name)
	}

	// TODO(mfoord): we need to delete spaces and subnets that no longer
	// exist, so long as they're not in use.
	var createSpacesArgs params.CreateSpacesParams
	var addSubnetsArgs params.AddSubnetsParams
	for _, space := range providerSpaces {
		// Check if the space is already in state, in which case we know
		// its name.
		stateSpace, ok := stateSpaceMap[string(space.ProviderId)]
		var spaceTag names.SpaceTag
		if ok {
			spaceName := stateSpace.Name
			if !names.IsValidSpace(spaceName) {
				// Can only happen if an invalid name is stored
				// in state.
				logger.Errorf("space %q has an invalid name, ignoring", spaceName)
				continue

			}
			spaceTag = names.NewSpaceTag(spaceName)

		} else {
			// The space is new, we need to create a valid name for it
			// in state.
			spaceName := string(space.Name)
			// Convert the name into a valid name that isn't already in
			// use.
			spaceName = dw.config.NewName(spaceName, spaceNames)
			spaceNames.Add(spaceName)
			spaceTag = names.NewSpaceTag(spaceName)
			// We need to create the space.
			createSpacesArgs.Spaces = append(createSpacesArgs.Spaces, params.CreateSpaceParams{
				Public:     false,
				SpaceTag:   spaceTag.String(),
				ProviderId: string(space.ProviderId),
			})
		}
		// TODO(mfoord): currently no way of removing subnets, or
		// changing the space they're in, so we can only add ones we
		// don't already know about.
		for _, subnet := range space.Subnets {
			if stateSubnetIds.Contains(string(subnet.ProviderId)) {
				continue
			}
			zones := subnet.AvailabilityZones
			if len(zones) == 0 {
				logger.Debugf("provider does not specify zones for subnet %q; using 'default' zone as fallback")
				zones = []string{"default"}
			}
			addSubnetsArgs.Subnets = append(addSubnetsArgs.Subnets, params.AddSubnetParams{
				SubnetProviderId: string(subnet.ProviderId),
				SpaceTag:         spaceTag.String(),
				Zones:            zones,
			})
		}
	}

	if err := dw.createSpacesFromArgs(createSpacesArgs); err != nil {
		return errors.Trace(err)
	}

	if err := dw.addSubnetsFromArgs(addSubnetsArgs); err != nil {
		return errors.Trace(err)
	}

	return nil
}

func (dw *discoverspacesWorker) createSpacesFromArgs(createSpacesArgs params.CreateSpacesParams) error {
	facade := dw.config.Facade

	expectedNumCreated := len(createSpacesArgs.Spaces)
	if expectedNumCreated > 0 {
		result, err := facade.CreateSpaces(createSpacesArgs)
		if err != nil {
			return errors.Annotate(err, "creating spaces failed")
		}
		if len(result.Results) != expectedNumCreated {
			return errors.Errorf(
				"unexpected response from CreateSpaces: expected %d results, got %d",
				expectedNumCreated, len(result.Results),
			)
		}
		for _, res := range result.Results {
			if res.Error != nil {
				return errors.Annotate(res.Error, "creating space failed")
			}
		}
		logger.Debugf("discovered and imported %d spaces: %v", expectedNumCreated, createSpacesArgs)
	} else {
		logger.Debugf("no unknown spaces discovered for import")
	}

	return nil
}

func (dw *discoverspacesWorker) addSubnetsFromArgs(addSubnetsArgs params.AddSubnetsParams) error {
	facade := dw.config.Facade

	expectedNumAdded := len(addSubnetsArgs.Subnets)
	if expectedNumAdded > 0 {
		result, err := facade.AddSubnets(addSubnetsArgs)
		if err != nil {
			return errors.Annotate(err, "adding subnets failed")
		}
		if len(result.Results) != expectedNumAdded {
			return errors.Errorf(
				"unexpected response from AddSubnets: expected %d results, got %d",
				expectedNumAdded, len(result.Results),
			)
		}
		for _, res := range result.Results {
			if res.Error != nil {
				return errors.Annotate(res.Error, "adding subnet failed")
			}
		}
		logger.Debugf("discovered and imported %d subnets: %v", expectedNumAdded, addSubnetsArgs)
	} else {
		logger.Debugf("no unknown subnets discovered for import")
	}

	return nil
}
<commit_msg>Fixed a go vet issue and reduced log level of the message<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package discoverspaces

import (
	"github.com/juju/errors"
	"github.com/juju/loggo"
	"github.com/juju/names"
	"github.com/juju/utils/set"

	"github.com/juju/juju/apiserver/params"
	"github.com/juju/juju/environs"
	"github.com/juju/juju/worker"
	"github.com/juju/juju/worker/catacomb"
	"github.com/juju/juju/worker/gate"
)

// Facade exposes the relevant capabilities of a *discoverspaces.API; it's
// a bit raw but at least it's easily mockable.
type Facade interface {
	CreateSpaces(params.CreateSpacesParams) (params.ErrorResults, error)
	AddSubnets(params.AddSubnetsParams) (params.ErrorResults, error)
	ListSpaces() (params.DiscoverSpacesResults, error)
	ListSubnets(params.SubnetsFilters) (params.ListSubnetsResults, error)
}

// NameFunc returns a string derived from base that is not contained in used.
type NameFunc func(base string, used set.Strings) string

// Config defines the operation of a space discovery worker.
type Config struct {

	// Facade exposes the capabilities of a controller.
	Facade Facade

	// Environ exposes the capabilities of a compute substrate.
	Environ environs.Environ

	// NewName is used to sanitise, and make unique, space names as
	// reported by an Environ (for use in juju, via the Facade). You
	// should probably set it to ConvertSpaceName.
	NewName NameFunc

	// Unlocker, if not nil, will be unlocked when the first discovery
	// attempt completes successfully.
	Unlocker gate.Unlocker
}

// Validate returns an error if the config cannot be expected to
// drive a functional worker.
func (config Config) Validate() error {
	if config.Facade == nil {
		return errors.NotValidf("nil Facade")
	}
	if config.Environ == nil {
		return errors.NotValidf("nil Environ")
	}
	if config.NewName == nil {
		return errors.NotValidf("nil NewName")
	}
	// missing Unlocker gate just means "don't bother notifying"
	return nil
}

var logger = loggo.GetLogger("juju.worker.discoverspaces")

type discoverspacesWorker struct {
	catacomb catacomb.Catacomb
	config   Config
}

// NewWorker returns a worker that will attempt to discover the
// configured Environ's spaces, and update the controller via the
// configured Facade. Names are sanitised with NewName, and any
// supplied Unlocker will be Unlock()ed when the first complete
// discovery and update succeeds.
//
// Once that update completes, the worker just waits to be Kill()ed.
// We should probably poll for changes, really, but I'm making an
// effort to preserve existing behaviour where possible.
func NewWorker(config Config) (worker.Worker, error) {
	if err := config.Validate(); err != nil {
		return nil, errors.Trace(err)
	}
	dw := &discoverspacesWorker{
		config: config,
	}
	err := catacomb.Invoke(catacomb.Plan{
		Site: &dw.catacomb,
		Work: dw.loop,
	})
	if err != nil {
		return nil, errors.Trace(err)
	}
	return dw, nil
}

// Kill is part of the worker.Worker interface.
func (dw *discoverspacesWorker) Kill() {
	dw.catacomb.Kill(nil)
}

// Wait is part of the worker.Worker interface.
func (dw *discoverspacesWorker) Wait() error {
	return dw.catacomb.Wait()
}

func (dw *discoverspacesWorker) loop() (err error) {

	// TODO(mfoord): we'll have a watcher here checking if we need to
	// update the spaces/subnets definition.
	// TODO(fwereade): for now, use a changes channel that apes the
	// standard initial event behaviour, so we can make the loop
	// follow the standard structure.
	changes := make(chan struct{}, 1)
	changes <- struct{}{}

	gate := dw.config.Unlocker
	for {
		select {
		case <-dw.catacomb.Dying():
			return dw.catacomb.ErrDying()
		case <-changes:
			if err := dw.handleSubnets(); err != nil {
				return errors.Trace(err)
			}
			logger.Debugf("space discovery complete")
			if gate != nil {
				gate.Unlock()
				gate = nil
			}
		}
	}
}

func (dw *discoverspacesWorker) handleSubnets() error {
	environ, ok := environs.SupportsNetworking(dw.config.Environ)
	if !ok {
		logger.Debugf("not a networking environ")
		return nil
	}
	if supported, err := environ.SupportsSpaceDiscovery(); err != nil {
		return errors.Trace(err)
	} else if !supported {
		logger.Debugf("environ does not support space discovery")
		return nil
	}
	providerSpaces, err := environ.Spaces()
	if err != nil {
		return errors.Trace(err)
	}

	facade := dw.config.Facade
	listSpacesResult, err := facade.ListSpaces()
	if err != nil {
		return errors.Trace(err)
	}
	stateSubnets, err := facade.ListSubnets(params.SubnetsFilters{})
	if err != nil {
		return errors.Trace(err)
	}

	stateSubnetIds := make(set.Strings)
	for _, subnet := range stateSubnets.Results {
		stateSubnetIds.Add(subnet.ProviderId)
	}
	stateSpaceMap := make(map[string]params.ProviderSpace)
	spaceNames := make(set.Strings)
	for _, space := range listSpacesResult.Results {
		stateSpaceMap[space.ProviderId] = space
		spaceNames.Add(space.Name)
	}

	// TODO(mfoord): we need to delete spaces and subnets that no longer
	// exist, so long as they're not in use.
	var createSpacesArgs params.CreateSpacesParams
	var addSubnetsArgs params.AddSubnetsParams
	for _, space := range providerSpaces {
		// Check if the space is already in state, in which case we know
		// its name.
		stateSpace, ok := stateSpaceMap[string(space.ProviderId)]
		var spaceTag names.SpaceTag
		if ok {
			spaceName := stateSpace.Name
			if !names.IsValidSpace(spaceName) {
				// Can only happen if an invalid name is stored
				// in state.
				logger.Errorf("space %q has an invalid name, ignoring", spaceName)
				continue

			}
			spaceTag = names.NewSpaceTag(spaceName)

		} else {
			// The space is new, we need to create a valid name for it
			// in state.
			spaceName := string(space.Name)
			// Convert the name into a valid name that isn't already in
			// use.
			spaceName = dw.config.NewName(spaceName, spaceNames)
			spaceNames.Add(spaceName)
			spaceTag = names.NewSpaceTag(spaceName)
			// We need to create the space.
			createSpacesArgs.Spaces = append(createSpacesArgs.Spaces, params.CreateSpaceParams{
				Public:     false,
				SpaceTag:   spaceTag.String(),
				ProviderId: string(space.ProviderId),
			})
		}
		// TODO(mfoord): currently no way of removing subnets, or
		// changing the space they're in, so we can only add ones we
		// don't already know about.
		for _, subnet := range space.Subnets {
			if stateSubnetIds.Contains(string(subnet.ProviderId)) {
				continue
			}
			zones := subnet.AvailabilityZones
			if len(zones) == 0 {
				logger.Tracef(
					"provider does not specify zones for subnet %q; using 'default' zone as fallback",
					subnet.CIDR,
				)
				zones = []string{"default"}
			}
			addSubnetsArgs.Subnets = append(addSubnetsArgs.Subnets, params.AddSubnetParams{
				SubnetProviderId: string(subnet.ProviderId),
				SpaceTag:         spaceTag.String(),
				Zones:            zones,
			})
		}
	}

	if err := dw.createSpacesFromArgs(createSpacesArgs); err != nil {
		return errors.Trace(err)
	}

	if err := dw.addSubnetsFromArgs(addSubnetsArgs); err != nil {
		return errors.Trace(err)
	}

	return nil
}

func (dw *discoverspacesWorker) createSpacesFromArgs(createSpacesArgs params.CreateSpacesParams) error {
	facade := dw.config.Facade

	expectedNumCreated := len(createSpacesArgs.Spaces)
	if expectedNumCreated > 0 {
		result, err := facade.CreateSpaces(createSpacesArgs)
		if err != nil {
			return errors.Annotate(err, "creating spaces failed")
		}
		if len(result.Results) != expectedNumCreated {
			return errors.Errorf(
				"unexpected response from CreateSpaces: expected %d results, got %d",
				expectedNumCreated, len(result.Results),
			)
		}
		for _, res := range result.Results {
			if res.Error != nil {
				return errors.Annotate(res.Error, "creating space failed")
			}
		}
		logger.Debugf("discovered and imported %d spaces: %v", expectedNumCreated, createSpacesArgs)
	} else {
		logger.Debugf("no unknown spaces discovered for import")
	}

	return nil
}

func (dw *discoverspacesWorker) addSubnetsFromArgs(addSubnetsArgs params.AddSubnetsParams) error {
	facade := dw.config.Facade

	expectedNumAdded := len(addSubnetsArgs.Subnets)
	if expectedNumAdded > 0 {
		result, err := facade.AddSubnets(addSubnetsArgs)
		if err != nil {
			return errors.Annotate(err, "adding subnets failed")
		}
		if len(result.Results) != expectedNumAdded {
			return errors.Errorf(
				"unexpected response from AddSubnets: expected %d results, got %d",
				expectedNumAdded, len(result.Results),
			)
		}
		for _, res := range result.Results {
			if res.Error != nil {
				return errors.Annotate(res.Error, "adding subnet failed")
			}
		}
		logger.Debugf("discovered and imported %d subnets: %v", expectedNumAdded, addSubnetsArgs)
	} else {
		logger.Debugf("no unknown subnets discovered for import")
	}

	return nil
}
<|endoftext|>


<commit_before>package irelate

import (
	"fmt"
	"io"
	"log"
	"os"
	"runtime"
	"sort"

	"github.com/brentp/irelate/interfaces"
)

func getStart(v interfaces.Relatable, s int) int {
	if ci, ok := v.(interfaces.CIFace); ok {
		a, _, ok := ci.CIPos()
		if ok && int(a) < s {
			return int(a)
		}
	}
	return s
}

func getEnd(v interfaces.Relatable, e int) int {
	if ci, ok := v.(interfaces.CIFace); ok {
		_, b, ok := ci.CIEnd()
		if ok && int(b) > e {
			return int(e)
		}
	}
	return e
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}

func max(a, b int) int {
	if a > b {
		return a
	}
	return b
}

type sliceIt struct {
	slice []interfaces.Relatable
	i     int
}

func (s *sliceIt) Next() (interfaces.Relatable, error) {
	if s.i < len(s.slice) {
		v := s.slice[s.i]
		s.i += 1
		return v, nil
	}
	s.slice = nil
	return nil, io.EOF

}
func (s *sliceIt) Close() error {
	return nil
}

func sliceToIterator(A []interfaces.Relatable) interfaces.RelatableIterator {
	return &sliceIt{A, 0}
}

// islice makes []interfaces.Relatable sortable.
type islice []interfaces.Relatable

func (i islice) Len() int {
	return len(i)
}

func (i islice) Less(a, b int) bool {
	if i[a].Start() < i[b].Start() {
		return true
	}
	if i[a].Start() == i[b].Start() && i[a].End() <= i[b].End() {
		return true
	}
	return false
}

func (is islice) Swap(i, j int) {
	is[i], is[j] = is[j], is[i]
}

type pos struct {
	chrom string
	start int
	end   int
}

func (p pos) Chrom() string {
	return p.chrom
}
func (p pos) Start() uint32 {
	return uint32(p.start)
}
func (p pos) End() uint32 {
	return uint32(p.end)
}

// make a set of streams ready to be sent to irelate.
func makeStreams(receiver chan []interfaces.RelatableIterator, mustSort bool, A []interfaces.Relatable, lastChrom string, minStart int, maxEnd int, dbs ...interfaces.Queryable) {

	if mustSort {
		sort.Sort(islice(A))
	}

	streams := make([]interfaces.RelatableIterator, 0, len(dbs)+1)
	streams = append(streams, sliceToIterator(A))
	p := pos{lastChrom, minStart, maxEnd}

	for _, db := range dbs {
		stream, err := db.Query(p)
		if err != nil {
			log.Fatal(err)
		}
		streams = append(streams, stream)
	}
	receiver <- streams
}

func checkOverlap(a, b interfaces.Relatable) bool {
	return b.Start() < a.End()
}

func less(a, b interfaces.Relatable) bool {
	return a.Start() < b.Start() || (a.Start() == b.Start() && a.End() < b.End())
}

type ciRel struct {
	interfaces.Relatable
	index int
}

func (ci ciRel) Start() uint32 {
	return uint32(getStart(ci, int(ci.Relatable.Start())))
}

func (ci ciRel) End() uint32 {
	return uint32(getEnd(ci, int(ci.Relatable.End())))
}

// PIRelate implements a parallel IRelate
func PIRelate(chunk int, maxGap int, qstream interfaces.RelatableIterator, ciExtend bool, fn func(interfaces.Relatable), dbs ...interfaces.Queryable) interfaces.RelatableChannel {
	nprocs := runtime.GOMAXPROCS(-1)
	// final interval stream sent back to caller.
	intersected := make(chan interfaces.Relatable, 2048)

	// this keeps the interval chunks in order.
	receivers := make(chan chan []interfaces.RelatableIterator, 1)

	// to channels recieves channels that accept intervals from IRelate to be sent for merging.
	// we send slices of intervals to reduce locking.
	tochannels := make(chan chan chan []interfaces.Relatable, 2+nprocs/2)

	verbose := os.Getenv("IRELATE_VERBOSE") == "TRUE"

	// the user-defined callback runs int it's own goroutine.
	// call on the relatable itself. but with all of the associated intervals.
	work := func(rels []interfaces.Relatable, fn func(interfaces.Relatable)) chan []interfaces.Relatable {
		ch := make(chan []interfaces.Relatable, 0)
		go func() {
			for _, r := range rels {
				fn(r)
			}
			ch <- rels
			close(ch)
		}()
		return ch
	}
	if ciExtend {

		work = func(rels []interfaces.Relatable, fn func(interfaces.Relatable)) chan []interfaces.Relatable {
			ch := make(chan []interfaces.Relatable, 0)
			go func() {
				for _, r := range rels {
					fn(r.(ciRel).Relatable)
				}
				ch <- rels
				close(ch)
			}()
			return ch

		}
	}

	// pull the intervals from IRelate, call fn() and (via work()) send chunks to be merged.
	// calling fn() is a bottleneck. so we make sub-chunks and process them in a separate go-routine
	// in work()
	// inner channel keeps track of the order for each big chunk
	go func() {

		for streamsChan := range receivers {

			inner := make(chan chan []interfaces.Relatable, nprocs)
			tochannels <- inner

			// push a channel to to channels out here
			// and then push to that channel inside this goroutine.
			// this maintains order of the intervals.
			go func(streams []interfaces.RelatableIterator) {
				N := 400
				//saved := make([]interfaces.Relatable, N)
				iterator := IRelate(checkOverlap, 0, less, streams...)
				saved := make([]interfaces.Relatable, N)
				k := 0

				for {
					interval, err := iterator.Next()
					if err == io.EOF {
						iterator.Close()
						break
					}
					saved[k] = interval
					k++

					if k == N {
						inner <- work(saved, fn)
						k = 0
						saved = make([]interfaces.Relatable, N)
					}

				}
				if k > 0 {
					inner <- work(saved[:k], fn)
				}
				close(inner)
			}(<-streamsChan) // only one, just used a chan for ordering.
		}
		close(tochannels)
	}()

	go mergeIntervals(tochannels, intersected, ciExtend)

	// split the query intervals into chunks and send for processing to irelate.
	go func() {

		A := make([]interfaces.Relatable, 0, chunk/2)

		lastStart := -10
		lastChrom := ""
		minStart := int(^uint32(0) >> 1)
		maxEnd := 0
		var totalParsed, totalSkipped, c, idx int
		for {
			v, err := qstream.Next()
			if err == io.EOF {
				qstream.Close()
			}
			if v == nil {
				break
			}

			if ciExtend {
				// turn it into an object that will return the ci bounds for Start(), End()
				v = ciRel{v, idx}
				idx++
			}

			// these will be based on CIPOS, CIEND if ciExtend is true
			s, e := int(v.Start()), int(v.End())

			// end chunk when:
			// 1. switch chroms
			// 2. see maxGap bases between adjacent intervals (currently looks at start only)
			// 3. reaches chunkSize (and has at least a gap of 2 bases from last interval).
			if v.Chrom() != lastChrom || (len(A) > 2048 && s-lastStart > maxGap) || ((s-lastStart > 25 && len(A) >= chunk) || len(A) >= chunk+200) || s-lastStart > 10*maxGap {
				if len(A) > 0 {
					// we push a channel onto a queue (another channel) and use that as the output order.
					ch := make(chan []interfaces.RelatableIterator, 0)
					receivers <- ch
					// send work to IRelate
					go makeStreams(ch, ciExtend, A, lastChrom, minStart, maxEnd, dbs...)
					c++
					if verbose {
						if lastChrom == v.Chrom() {
							totalSkipped += s - lastStart
						}
						totalParsed += maxEnd - minStart
						var mem runtime.MemStats
						runtime.ReadMemStats(&mem)
						log.Println("intervals in current chunk:", len(A), fmt.Sprintf("%s:%d-%d", lastChrom, minStart, maxEnd), "gap:", s-lastStart)
						log.Println("\tc:", c, "receivers:", len(receivers), "tochannels:", len(tochannels), "intersected:", len(intersected))
						log.Printf("\tmemory use: %dMB , heap in use: %dMB\n", mem.Alloc/uint64(1000*1000),
							mem.HeapInuse/uint64(1000*1000))
						log.Printf("\ttotal bases skipped / parsed: %d / %d (%.2f)\n", totalSkipped, totalParsed, float64(totalSkipped)/float64(totalParsed))
					}

				}
				lastStart = s
				lastChrom, minStart, maxEnd = v.Chrom(), s, e
				A = make([]interfaces.Relatable, 0, chunk/2)
			} else {
				lastStart = s
				maxEnd = max(e, maxEnd)
				minStart = min(s, minStart)
			}

			A = append(A, v)
		}

		if len(A) > 0 {
			ch := make(chan []interfaces.RelatableIterator, 0)
			receivers <- ch
			go makeStreams(ch, ciExtend, A, lastChrom, minStart, maxEnd, dbs...)
			c++
		}
		close(receivers)
	}()
	return intersected
}

func mergeIntervals(tochannels chan chan chan []interfaces.Relatable, intersected chan interfaces.Relatable, ciExtend bool) {
	// merge the intervals from different channels keeping order.
	// 2 separate function code-blocks so there is no performance hit when they don't
	// care about the cipos.
	if ciExtend {
		nextPrint := 0
		q := make(map[int]ciRel, 100)
		for och := range tochannels {
			for ch := range och {
				for intervals := range ch {
					for _, interval := range intervals {
						ci := interval.(ciRel)
						if ci.index == nextPrint {
							intersected <- ci.Relatable
							nextPrint++
						} else {
							q[ci.index] = ci
							for {
								n, ok := q[nextPrint]
								if !ok {
									break
								}
								delete(q, nextPrint)
								intersected <- n.Relatable
								nextPrint++
							}
						}
					}
					// empty out the q
					for {
						n, ok := q[nextPrint]
						if !ok {
							break
						}
						delete(q, nextPrint)
						intersected <- n.Relatable
						nextPrint++
					}
				}
			}
		}
	} else {
		for och := range tochannels {
			for ch := range och {
				for intervals := range ch {
					for _, interval := range intervals {
						intersected <- interval
					}
				}
			}
		}
	}
	close(intersected)
}
<commit_msg>document internal design<commit_after>package irelate

// parallel implements a parallel chrom-sweep.
// broad design is covered in design.md in the irelate package directory.
// In actual fact, there are a number of complexities; most of them relate to
// maintaining intervals in sorted order (and keeping chunks in sorted order)
// while allowing a good level of parallelism.

// more detailed explanations are provided whenever a channel is initialized
// as channels are our main means of keeping order.
// For example
//     tochannels := make(chan chan chan []interfaces.Relatable, 2+nprocs/2)
// Seems to have excessive use of channels, but we actually do need this since
// we have 2 levels of parallelization.
// One level is by chunk of query intervals.
// The next is by sub-chunk within the query chunks.
// The 3rd chan is a place-holder so that the work() function, which calls
// the user-defined fn() can be done concurrently (in a go routine).

// The broad pattern used throughout is to send a channel (K) into another
// channel (PARENT) to keep order and then send K into a worker goroutine
// that sends intervals or []intervals into K.

// I have done much tuning; the areas that affect performance are how the work()
// is parallelized (see the code-block that calls work()). And how the query
// chunks are determined. If the query chunks are too small (< 100 intervals),
// we have a lot of overhead in tracking that chunk that only requires a little
// computation. Unless the databases are very dense, then having the query chunks
// quite large helps parallelization. This is an area of potential optimization,
// though no obvious candidates have emerged.

import (
	"fmt"
	"io"
	"log"
	"os"
	"runtime"
	"sort"

	"github.com/brentp/irelate/interfaces"
)

func getStart(v interfaces.Relatable, s int) int {
	if ci, ok := v.(interfaces.CIFace); ok {
		a, _, ok := ci.CIPos()
		if ok && int(a) < s {
			return int(a)
		}
	}
	return s
}

func getEnd(v interfaces.Relatable, e int) int {
	if ci, ok := v.(interfaces.CIFace); ok {
		_, b, ok := ci.CIEnd()
		if ok && int(b) > e {
			return int(e)
		}
	}
	return e
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}

func max(a, b int) int {
	if a > b {
		return a
	}
	return b
}

type sliceIt struct {
	slice []interfaces.Relatable
	i     int
}

func (s *sliceIt) Next() (interfaces.Relatable, error) {
	if s.i < len(s.slice) {
		v := s.slice[s.i]
		s.i += 1
		return v, nil
	}
	s.slice = nil
	return nil, io.EOF

}
func (s *sliceIt) Close() error {
	return nil
}

func sliceToIterator(A []interfaces.Relatable) interfaces.RelatableIterator {
	return &sliceIt{A, 0}
}

// islice makes []interfaces.Relatable sortable.
type islice []interfaces.Relatable

func (i islice) Len() int {
	return len(i)
}

func (i islice) Less(a, b int) bool {
	if i[a].Start() < i[b].Start() {
		return true
	}
	if i[a].Start() == i[b].Start() && i[a].End() <= i[b].End() {
		return true
	}
	return false
}

func (is islice) Swap(i, j int) {
	is[i], is[j] = is[j], is[i]
}

type pos struct {
	chrom string
	start int
	end   int
}

func (p pos) Chrom() string {
	return p.chrom
}
func (p pos) Start() uint32 {
	return uint32(p.start)
}
func (p pos) End() uint32 {
	return uint32(p.end)
}

// make a set of streams ready to be sent to irelate.
func makeStreams(receiver chan []interfaces.RelatableIterator, mustSort bool, A []interfaces.Relatable, lastChrom string, minStart int, maxEnd int, dbs ...interfaces.Queryable) {

	if mustSort {
		sort.Sort(islice(A))
	}

	streams := make([]interfaces.RelatableIterator, 0, len(dbs)+1)
	streams = append(streams, sliceToIterator(A))
	p := pos{lastChrom, minStart, maxEnd}

	for _, db := range dbs {
		stream, err := db.Query(p)
		if err != nil {
			log.Fatal(err)
		}
		streams = append(streams, stream)
	}
	receiver <- streams
}

func checkOverlap(a, b interfaces.Relatable) bool {
	return b.Start() < a.End()
}

func less(a, b interfaces.Relatable) bool {
	return a.Start() < b.Start() || (a.Start() == b.Start() && a.End() < b.End())
}

type ciRel struct {
	interfaces.Relatable
	index int
}

func (ci ciRel) Start() uint32 {
	return uint32(getStart(ci, int(ci.Relatable.Start())))
}

func (ci ciRel) End() uint32 {
	return uint32(getEnd(ci, int(ci.Relatable.End())))
}

// PIRelate implements a parallel IRelate
func PIRelate(chunk int, maxGap int, qstream interfaces.RelatableIterator, ciExtend bool, fn func(interfaces.Relatable), dbs ...interfaces.Queryable) interfaces.RelatableChannel {
	nprocs := runtime.GOMAXPROCS(-1)
	// final interval stream sent back to caller.
	intersected := make(chan interfaces.Relatable, 2048)

	// receivers keeps the interval chunks in order.
	receivers := make(chan chan []interfaces.RelatableIterator, 1)

	// to channels recieves channels that accept intervals from IRelate to be sent for merging.
	// we send slices of intervals to reduce locking.
	tochannels := make(chan chan chan []interfaces.Relatable, 2+nprocs/2)

	verbose := os.Getenv("IRELATE_VERBOSE") == "TRUE"

	// the user-defined callback runs int it's own goroutine.
	// call on the relatable itself. but with all of the associated intervals.
	work := func(rels []interfaces.Relatable, fn func(interfaces.Relatable)) chan []interfaces.Relatable {
		ch := make(chan []interfaces.Relatable, 0)
		go func() {
			for _, r := range rels {
				fn(r)
			}
			ch <- rels
			close(ch)
		}()
		return ch
	}
	if ciExtend {

		work = func(rels []interfaces.Relatable, fn func(interfaces.Relatable)) chan []interfaces.Relatable {
			ch := make(chan []interfaces.Relatable, 0)
			go func() {
				for _, r := range rels {
					fn(r.(ciRel).Relatable)
				}
				ch <- rels
				close(ch)
			}()
			return ch

		}
	}

	// pull the intervals from IRelate, call fn() and (via work()) send chunks to be merged.
	// calling fn() is a bottleneck. so we make sub-chunks and process them in a separate go-routine
	// in work()
	// inner channel keeps track of the order for each big chunk
	go func() {

		for streamsChan := range receivers {

			inner := make(chan chan []interfaces.Relatable, nprocs)
			tochannels <- inner

			// push a channel to to channels out here
			// and then push to that channel inside this goroutine.
			// this maintains order of the intervals.
			go func(streams []interfaces.RelatableIterator) {
				N := 400
				//saved := make([]interfaces.Relatable, N)
				iterator := IRelate(checkOverlap, 0, less, streams...)
				saved := make([]interfaces.Relatable, N)
				k := 0

				for {
					interval, err := iterator.Next()
					if err == io.EOF {
						iterator.Close()
						break
					}
					saved[k] = interval
					k++

					if k == N {
						inner <- work(saved, fn)
						k = 0
						saved = make([]interfaces.Relatable, N)
					}

				}
				if k > 0 {
					inner <- work(saved[:k], fn)
				}
				close(inner)
			}(<-streamsChan) // only one, just used a chan for ordering.
		}
		close(tochannels)
	}()

	go mergeIntervals(tochannels, intersected, ciExtend)

	// split the query intervals into chunks and send for processing to irelate.
	go func() {

		A := make([]interfaces.Relatable, 0, chunk/2)

		lastStart := -10
		lastChrom := ""
		minStart := int(^uint32(0) >> 1)
		maxEnd := 0
		var totalParsed, totalSkipped, c, idx int
		for {
			v, err := qstream.Next()
			if err == io.EOF {
				qstream.Close()
			}
			if v == nil {
				break
			}

			if ciExtend {
				// turn it into an object that will return the ci bounds for Start(), End()
				v = ciRel{v, idx}
				idx++
			}

			// these will be based on CIPOS, CIEND if ciExtend is true
			s, e := int(v.Start()), int(v.End())

			// end chunk when:
			// 1. switch chroms
			// 2. see maxGap bases between adjacent intervals (currently looks at start only)
			// 3. reaches chunkSize (and has at least a gap of 2 bases from last interval).
			if v.Chrom() != lastChrom || (len(A) > 2048 && s-lastStart > maxGap) || ((s-lastStart > 25 && len(A) >= chunk) || len(A) >= chunk+200) || s-lastStart > 10*maxGap {
				if len(A) > 0 {
					// we push a channel onto a queue (another channel) and use that as the output order.
					ch := make(chan []interfaces.RelatableIterator, 0)
					receivers <- ch
					// send work to IRelate
					go makeStreams(ch, ciExtend, A, lastChrom, minStart, maxEnd, dbs...)
					c++
					if verbose {
						if lastChrom == v.Chrom() {
							totalSkipped += s - lastStart
						}
						totalParsed += maxEnd - minStart
						var mem runtime.MemStats
						runtime.ReadMemStats(&mem)
						log.Println("intervals in current chunk:", len(A), fmt.Sprintf("%s:%d-%d", lastChrom, minStart, maxEnd), "gap:", s-lastStart)
						log.Println("\tc:", c, "receivers:", len(receivers), "tochannels:", len(tochannels), "intersected:", len(intersected))
						log.Printf("\tmemory use: %dMB , heap in use: %dMB\n", mem.Alloc/uint64(1000*1000),
							mem.HeapInuse/uint64(1000*1000))
						log.Printf("\ttotal bases skipped / parsed: %d / %d (%.2f)\n", totalSkipped, totalParsed, float64(totalSkipped)/float64(totalParsed))
					}

				}
				lastStart = s
				lastChrom, minStart, maxEnd = v.Chrom(), s, e
				A = make([]interfaces.Relatable, 0, chunk/2)
			} else {
				lastStart = s
				maxEnd = max(e, maxEnd)
				minStart = min(s, minStart)
			}

			A = append(A, v)
		}

		if len(A) > 0 {
			ch := make(chan []interfaces.RelatableIterator, 0)
			receivers <- ch
			go makeStreams(ch, ciExtend, A, lastChrom, minStart, maxEnd, dbs...)
			c++
		}
		close(receivers)
	}()
	return intersected
}

func mergeIntervals(tochannels chan chan chan []interfaces.Relatable, intersected chan interfaces.Relatable, ciExtend bool) {
	// merge the intervals from different channels keeping order.
	// 2 separate function code-blocks so there is no performance hit when they don't
	// care about the cipos.
	if ciExtend {
		nextPrint := 0
		q := make(map[int]ciRel, 100)
		for och := range tochannels {
			for ch := range och {
				for intervals := range ch {
					for _, interval := range intervals {
						ci := interval.(ciRel)
						if ci.index == nextPrint {
							intersected <- ci.Relatable
							nextPrint++
						} else {
							q[ci.index] = ci
							for {
								n, ok := q[nextPrint]
								if !ok {
									break
								}
								delete(q, nextPrint)
								intersected <- n.Relatable
								nextPrint++
							}
						}
					}
					// empty out the q
					for {
						n, ok := q[nextPrint]
						if !ok {
							break
						}
						delete(q, nextPrint)
						intersected <- n.Relatable
						nextPrint++
					}
				}
			}
		}
	} else {
		for och := range tochannels {
			for ch := range och {
				for intervals := range ch {
					for _, interval := range intervals {
						intersected <- interval
					}
				}
			}
		}
	}
	close(intersected)
}
<|endoftext|>


<commit_before>package analytics

import (
	"sync"
	"testing"
	"time"
)

func TestExecutorClose(t *testing.T) {
	// Simply make sure that nothing raises a panic nor blocks.
	ex := newExecutor(1)
	ex.close()
}

func TestExecutorSimple(t *testing.T) {
	wg := &sync.WaitGroup{}
	ex := newExecutor(1)
	defer ex.close()

	wg.Add(1)

	if !ex.do(wg.Done) {
		t.Error("failed pushing a task to an executor with a capacity of 1")
		return
	}

	// Make sure wg.Done gets called, this shouldn't block idenfinitely.
	wg.Wait()
}

func TestExecutorMulti(t *testing.T) {
	wg := &sync.WaitGroup{}
	ex := newExecutor(3)
	defer ex.close()

	// Schedule a couple of tasks to fill the executor.
	for i := 0; i != 3; i++ {
		if !ex.do(func() {
			time.Sleep(10 * time.Millisecond)
			wg.Done()
		}) {
			t.Error("failed pushing a task to an executor with a capacity of 1")
			return
		}
	}

	// Make sure the executor refuses more tasks.
	if ex.do(func() {}) {
		t.Error("the executor should have been full and refused to run more tasks")
	}

	// Make sure wg.Done gets called, this shouldn't block idenfinitely.
	wg.Wait()
}
<commit_msg>fix typos<commit_after>package analytics

import (
	"sync"
	"testing"
	"time"
)

func TestExecutorClose(t *testing.T) {
	// Simply make sure that nothing raises a panic nor blocks.
	ex := newExecutor(1)
	ex.close()
}

func TestExecutorSimple(t *testing.T) {
	wg := &sync.WaitGroup{}
	ex := newExecutor(1)
	defer ex.close()

	wg.Add(1)

	if !ex.do(wg.Done) {
		t.Error("failed pushing a task to an executor with a capacity of 1")
		return
	}

	// Make sure wg.Done gets called, this shouldn't block indefinitely.
	wg.Wait()
}

func TestExecutorMulti(t *testing.T) {
	wg := &sync.WaitGroup{}
	ex := newExecutor(3)
	defer ex.close()

	// Schedule a couple of tasks to fill the executor.
	for i := 0; i != 3; i++ {
		if !ex.do(func() {
			time.Sleep(10 * time.Millisecond)
			wg.Done()
		}) {
			t.Error("failed pushing a task to an executor with a capacity of 1")
			return
		}
	}

	// Make sure the executor refuses more tasks.
	if ex.do(func() {}) {
		t.Error("the executor should have been full and refused to run more tasks")
	}

	// Make sure wg.Done gets called, this shouldn't block indefinitely.
	wg.Wait()
}
<|endoftext|>


<commit_before>// This package is a fork of the golang expvar expvar.Var types.
// Adding extra support for deleting and accessing raw typed values.
package expvar

import (
	"bytes"
	"expvar"
	"fmt"
	"math"
	"sort"
	"strconv"
	"sync"
	"sync/atomic"
)

type IntVar interface {
	IntValue() int64
}

type FloatVar interface {
	FloatValue() float64
}

type StringVar interface {
	StringValue() float64
}

// Int is a 64-bit integer variable that satisfies the expvar.Var interface.
type Int struct {
	i int64
}

func (v *Int) String() string {
	return strconv.FormatInt(v.IntValue(), 10)
}

func (v *Int) Add(delta int64) {
	atomic.AddInt64(&v.i, delta)
}

func (v *Int) Set(value int64) {
	atomic.StoreInt64(&v.i, value)
}

func (v *Int) IntValue() int64 {
	return atomic.LoadInt64(&v.i)
}

// Float is a 64-bit float variable that satisfies the expvar.Var interface.
type Float struct {
	f uint64
}

func (v *Float) String() string {
	return strconv.FormatFloat(v.FloatValue(), 'g', -1, 64)
}

func (v *Float) FloatValue() float64 {
	return math.Float64frombits(atomic.LoadUint64(&v.f))
}

// Add adds delta to v.
func (v *Float) Add(delta float64) {
	for {
		cur := atomic.LoadUint64(&v.f)
		curVal := math.Float64frombits(cur)
		nxtVal := curVal + delta
		nxt := math.Float64bits(nxtVal)
		if atomic.CompareAndSwapUint64(&v.f, cur, nxt) {
			return
		}
	}
}

// Set sets v to value.
func (v *Float) Set(value float64) {
	atomic.StoreUint64(&v.f, math.Float64bits(value))
}

// Map is a string-to-expvar.Var map variable that satisfies the expvar.Var interface.
type Map struct {
	mu   sync.RWMutex
	m    map[string]expvar.Var
	keys []string // sorted
}

func (v *Map) String() string {
	v.mu.RLock()
	defer v.mu.RUnlock()
	var b bytes.Buffer
	fmt.Fprintf(&b, "{")
	first := true
	v.doLocked(func(kv expvar.KeyValue) {
		if !first {
			fmt.Fprintf(&b, ", ")
		}
		fmt.Fprintf(&b, "%q: %v", kv.Key, kv.Value)
		first = false
	})
	fmt.Fprintf(&b, "}")
	return b.String()
}

func (v *Map) Init() *Map {
	v.m = make(map[string]expvar.Var)
	return v
}

// updateKeys updates the sorted list of keys in v.keys.
// must be called with v.mu held.
func (v *Map) updateKeys() {
	if len(v.m) == len(v.keys) {
		// No new key.
		return
	}
	v.keys = v.keys[:0]
	for k := range v.m {
		v.keys = append(v.keys, k)
	}
	sort.Strings(v.keys)
}

func (v *Map) Get(key string) expvar.Var {
	v.mu.RLock()
	defer v.mu.RUnlock()
	return v.m[key]
}

func (v *Map) Set(key string, av expvar.Var) {
	v.mu.Lock()
	defer v.mu.Unlock()
	v.m[key] = av
	v.updateKeys()
}

func (v *Map) Delete(key string) {
	v.mu.Lock()
	defer v.mu.Unlock()
	delete(v.m, key)
	v.updateKeys()
}

func (v *Map) Add(key string, delta int64) {
	v.mu.RLock()
	av, ok := v.m[key]
	v.mu.RUnlock()
	if !ok {
		// check again under the write lock
		v.mu.Lock()
		av, ok = v.m[key]
		if !ok {
			av = new(Int)
			v.m[key] = av
			v.updateKeys()
		}
		v.mu.Unlock()
	}

	// Add to Int; ignore otherwise.
	if iv, ok := av.(*Int); ok {
		iv.Add(delta)
	}
}

// AddFloat adds delta to the *Float value stored under the given map key.
func (v *Map) AddFloat(key string, delta float64) {
	v.mu.RLock()
	av, ok := v.m[key]
	v.mu.RUnlock()
	if !ok {
		// check again under the write lock
		v.mu.Lock()
		av, ok = v.m[key]
		if !ok {
			av = new(Float)
			v.m[key] = av
			v.updateKeys()
		}
		v.mu.Unlock()
	}

	// Add to Float; ignore otherwise.
	if iv, ok := av.(*Float); ok {
		iv.Add(delta)
	}
}

// Do calls f for each entry in the map.
// The map is locked during the iteration,
// but existing entries may be concurrently updated.
func (v *Map) Do(f func(expvar.KeyValue)) {
	v.mu.RLock()
	defer v.mu.RUnlock()
	v.doLocked(f)
}

// doLocked calls f for each entry in the map.
// v.mu must be held for reads.
func (v *Map) doLocked(f func(expvar.KeyValue)) {
	for _, k := range v.keys {
		f(expvar.KeyValue{k, v.m[k]})
	}
}

// String is a string variable, and satisfies the expvar.Var interface.
type String struct {
	mu sync.RWMutex
	s  string
}

func (v *String) String() string {
	v.mu.RLock()
	defer v.mu.RUnlock()
	return strconv.Quote(v.s)
}

func (v *String) Set(value string) {
	v.mu.Lock()
	defer v.mu.Unlock()
	v.s = value
}

func (v *String) StringValue() string {
	v.mu.RLock()
	defer v.mu.RUnlock()
	return v.s
}
<commit_msg>Removing expvar keys since it's heart performance and it's unused.<commit_after>// This package is a fork of the golang expvar expvar.Var types.
// Adding extra support for deleting and accessing raw typed values.
package expvar

import (
	"bytes"
	"expvar"
	"fmt"
	"math"
	"strconv"
	"sync"
	"sync/atomic"
)

type IntVar interface {
	IntValue() int64
}

type FloatVar interface {
	FloatValue() float64
}

type StringVar interface {
	StringValue() float64
}

// Int is a 64-bit integer variable that satisfies the expvar.Var interface.
type Int struct {
	i int64
}

func (v *Int) String() string {
	return strconv.FormatInt(v.IntValue(), 10)
}

func (v *Int) Add(delta int64) {
	atomic.AddInt64(&v.i, delta)
}

func (v *Int) Set(value int64) {
	atomic.StoreInt64(&v.i, value)
}

func (v *Int) IntValue() int64 {
	return atomic.LoadInt64(&v.i)
}

// Float is a 64-bit float variable that satisfies the expvar.Var interface.
type Float struct {
	f uint64
}

func (v *Float) String() string {
	return strconv.FormatFloat(v.FloatValue(), 'g', -1, 64)
}

func (v *Float) FloatValue() float64 {
	return math.Float64frombits(atomic.LoadUint64(&v.f))
}

// Add adds delta to v.
func (v *Float) Add(delta float64) {
	for {
		cur := atomic.LoadUint64(&v.f)
		curVal := math.Float64frombits(cur)
		nxtVal := curVal + delta
		nxt := math.Float64bits(nxtVal)
		if atomic.CompareAndSwapUint64(&v.f, cur, nxt) {
			return
		}
	}
}

// Set sets v to value.
func (v *Float) Set(value float64) {
	atomic.StoreUint64(&v.f, math.Float64bits(value))
}

// Map is a string-to-expvar.Var map variable that satisfies the expvar.Var interface.
type Map struct {
	mu sync.RWMutex
	m  map[string]expvar.Var
}

func (v *Map) String() string {
	v.mu.RLock()
	defer v.mu.RUnlock()
	var b bytes.Buffer
	fmt.Fprintf(&b, "{")
	first := true
	v.doLocked(func(kv expvar.KeyValue) {
		if !first {
			fmt.Fprintf(&b, ", ")
		}
		fmt.Fprintf(&b, "%q: %v", kv.Key, kv.Value)
		first = false
	})
	fmt.Fprintf(&b, "}")
	return b.String()
}

func (v *Map) Init() *Map {
	v.m = make(map[string]expvar.Var)
	return v
}

func (v *Map) Get(key string) expvar.Var {
	v.mu.RLock()
	defer v.mu.RUnlock()
	return v.m[key]
}

func (v *Map) Set(key string, av expvar.Var) {
	v.mu.Lock()
	defer v.mu.Unlock()
	v.m[key] = av
}

func (v *Map) Delete(key string) {
	v.mu.Lock()
	defer v.mu.Unlock()
	delete(v.m, key)
}

func (v *Map) Add(key string, delta int64) {
	v.mu.RLock()
	av, ok := v.m[key]
	v.mu.RUnlock()
	if !ok {
		// check again under the write lock
		v.mu.Lock()
		av, ok = v.m[key]
		if !ok {
			av = new(Int)
			v.m[key] = av
		}
		v.mu.Unlock()
	}

	// Add to Int; ignore otherwise.
	if iv, ok := av.(*Int); ok {
		iv.Add(delta)
	}
}

// AddFloat adds delta to the *Float value stored under the given map key.
func (v *Map) AddFloat(key string, delta float64) {
	v.mu.RLock()
	av, ok := v.m[key]
	v.mu.RUnlock()
	if !ok {
		// check again under the write lock
		v.mu.Lock()
		av, ok = v.m[key]
		if !ok {
			av = new(Float)
			v.m[key] = av
		}
		v.mu.Unlock()
	}

	// Add to Float; ignore otherwise.
	if iv, ok := av.(*Float); ok {
		iv.Add(delta)
	}
}

// Do calls f for each entry in the map.
// The map is locked during the iteration,
// but existing entries may be concurrently updated.
func (v *Map) Do(f func(expvar.KeyValue)) {
	v.mu.RLock()
	defer v.mu.RUnlock()
	v.doLocked(f)
}

// doLocked calls f for each entry in the map.
// v.mu must be held for reads.
func (v *Map) doLocked(f func(expvar.KeyValue)) {
	for k, v := range v.m {
		f(expvar.KeyValue{k, v})
	}
}

// String is a string variable, and satisfies the expvar.Var interface.
type String struct {
	mu sync.RWMutex
	s  string
}

func (v *String) String() string {
	v.mu.RLock()
	defer v.mu.RUnlock()
	return strconv.Quote(v.s)
}

func (v *String) Set(value string) {
	v.mu.Lock()
	defer v.mu.Unlock()
	v.s = value
}

func (v *String) StringValue() string {
	v.mu.RLock()
	defer v.mu.RUnlock()
	return v.s
}
<|endoftext|>


<commit_before>// Package resources provides unfancy resources embedding with Go.
package resources

import (
	"bytes"
	"fmt"
	"io"
	"log"
	"os"
	"text/template"
)

// File mimicks the os.File and http.File interface.
type File interface {
	io.Reader
	Stat() (os.FileInfo, error)
}

// New creates a new Package.
func New() *Package {
	return &Package{
		Config: Config{
			Pkg:     "resources",
			Var:     "FS",
			Declare: true,
		},
		Files: make(map[string]File),
	}
}

// Config defines some details about the output Go file.
type Config struct {
	Pkg     string // Package name
	Var     string // Variable name to assign the file system to.
	Tag     string // Build tag, leave empty for no tag.
	Declare bool   // Dictates whatever there should be a defintion Variable
}

// Package describes...
type Package struct {
	Config
	Files map[string]File
}

//Add a file to the package at the give path.
func (p *Package) Add(path string, file File) error {
	p.Files[path] = file
	return nil
}

// AddFile is a helper function that adds the files from the path into the package under the path file.
func (p *Package) AddFile(path string, file string) error {
	f, err := os.Open(file)
	if err != nil {
		return err
	}
	return p.Add(path, f)
}

// Build the package
func (p *Package) Build(out io.Writer) error {
	return pkg.Execute(out, p)
}

// Write the build to a file, you don't need to call Build.
func (p *Package) Write(path string) error {
	f, err := os.Create(path)
	if err != nil {
		return err
	}
	defer func() {
		err := f.Close()
		if err != nil {
			log.Panicf("Failed to close file: %s", err)
		}
	}()

	return p.Build(f)
}

// Template
var pkg *template.Template

func reader(input io.Reader) (string, error) {

	var (
		buff       bytes.Buffer
		err        error
		blockwidth = 12
		curblock   = 0
	)

	b := make([]byte, blockwidth)

	var n int
	for n, err = input.Read(b); err == nil; n, err = input.Read(b) {
		for i := 0; i < n; i++ {
			_, err = fmt.Fprintf(&buff, "0x%02x,", b[i])
			if err != nil {
				break
			}
			curblock++
			if curblock < blockwidth {
				continue
			}
			buff.WriteByte('\n')
			buff.Write([]byte{'\t', '\t'})
			curblock = 0
		}
	}

	return buff.String(), err
}

func init() {

	pkg = template.Must(template.New("file").Funcs(template.FuncMap{"reader": reader}).Parse(` File{
	  data: []byte{
	{{ reader . }}
  },
  fi: FileInfo {
	name:    "{{ .Stat.Name }}",
    size:    {{ .Stat.Size }},
	modTime: time.Unix({{ .Stat.ModTime.Unix }},{{ .Stat.ModTime.UnixNano }}),
    isDir:   {{ .Stat.IsDir }},
  },
}`))

	pkg = template.Must(pkg.New("pkg").Parse(`{{ if .Tag }}// +build {{ .Tag }}

{{ end }}// Package {{.Pkg }} is generated by github.com/omeid/go-resources
package {{ .Pkg }}

import (
	"bytes"
	"errors"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"
)

// FileSystem is an http.FileSystem implementation.
type FileSystem struct {
	files map[string]File
}

// String returns the content of the file as string.
func (fs *FileSystem) String(name string) (string, bool) {
	if filepath.Separator != '/' && strings.IndexRune(name, filepath.Separator) >= 0 ||
		strings.Contains(name, "\x00") {
		return nil, errors.New("http: invalid character in file path")
	}

	file, ok := fs.files[name]

	if !ok {
		return "", false
	}

	return string(file.data), true
}

// Open implements http.FileSystem.Open
func (fs *FileSystem) Open(name string) (http.File, error) {
	if filepath.Separator != '/' && strings.IndexRune(name, filepath.Separator) >= 0 ||
		strings.Contains(name, "\x00") {
		return nil, errors.New("http: invalid character in file path")
	}
	file, ok := fs.files[name]
	if !ok {
		files := []os.FileInfo{}
		for path, file := range fs.files {
			if strings.HasPrefix(path, name) {
				fi := file.fi
				files = append(files, &fi)
			}
		}

		if len(files) == 0 {
			return nil, os.ErrNotExist
		}

		//We have a directory.
		return &File{
			fi: FileInfo{
				isDir: true,
				files: files,
			}}, nil
	}
	file.Reader = bytes.NewReader(file.data)
	return &file, nil
}

// File implements http.File
type File struct {
	*bytes.Reader
	data []byte
	fi   FileInfo
}

// Close is a noop-closer.
func (f *File) Close() error {
	return nil
}

// Readdir implements http.File.Readdir
func (f *File) Readdir(count int) ([]os.FileInfo, error) {
	return nil, os.ErrNotExist
}

// Stat implements http.Stat.Readdir
func (f *File) Stat() (os.FileInfo, error) {
	return &f.fi, nil
}

// FileInfo implements the os.FileInfo interface.
type FileInfo struct {
	name    string
	size    int64
	mode    os.FileMode
	modTime time.Time
	isDir   bool
	sys     interface{}

	files []os.FileInfo
}

// Name implements os.FileInfo.Name
func (f *FileInfo) Name() string {
	return f.name
}

// Size implements os.FileInfo.Size
func (f *FileInfo) Size() int64 {
	return f.size
}

// Mode implements os.FileInfo.Mode
func (f *FileInfo) Mode() os.FileMode {
	return f.mode
}

// ModTime implements os.FileInfo.ModTime
func (f *FileInfo) ModTime() time.Time {
	return f.modTime
}

// IsDir implements os.FileInfo.IsDir
func (f *FileInfo) IsDir() bool {
	return f.isDir
}

// Readdir implements os.FileInfo.Readdir
func (f *FileInfo) Readdir(count int) ([]os.FileInfo, error) {
	return f.files, nil
}

// Sys returns the underlying value.
func (f *FileInfo) Sys() interface{} {
	return f.sys
}

func init() {
  {{ .Var }} = &FileSystem{
		files: map[string]File{
		  {{range $path, $file := .Files }} "/{{ $path }}": {{ template "file" $file }}, {{ end }}
		},
	  }
}
`))
}
<commit_msg>Fix generation errors<commit_after>// Package resources provides unfancy resources embedding with Go.
package resources

import (
	"bytes"
	"fmt"
	"io"
	"log"
	"os"
	"text/template"
)

// File mimicks the os.File and http.File interface.
type File interface {
	io.Reader
	Stat() (os.FileInfo, error)
}

// New creates a new Package.
func New() *Package {
	return &Package{
		Config: Config{
			Pkg:     "resources",
			Var:     "FS",
			Declare: true,
		},
		Files: make(map[string]File),
	}
}

// Config defines some details about the output Go file.
type Config struct {
	Pkg     string // Package name
	Var     string // Variable name to assign the file system to.
	Tag     string // Build tag, leave empty for no tag.
	Declare bool   // Dictates whatever there should be a defintion Variable
}

// Package describes...
type Package struct {
	Config
	Files map[string]File
}

//Add a file to the package at the give path.
func (p *Package) Add(path string, file File) error {
	p.Files[path] = file
	return nil
}

// AddFile is a helper function that adds the files from the path into the package under the path file.
func (p *Package) AddFile(path string, file string) error {
	f, err := os.Open(file)
	if err != nil {
		return err
	}
	return p.Add(path, f)
}

// Build the package
func (p *Package) Build(out io.Writer) error {
	return pkg.Execute(out, p)
}

// Write the build to a file, you don't need to call Build.
func (p *Package) Write(path string) error {
	f, err := os.Create(path)
	if err != nil {
		return err
	}
	defer func() {
		err := f.Close()
		if err != nil {
			log.Panicf("Failed to close file: %s", err)
		}
	}()

	return p.Build(f)
}

// Template
var pkg *template.Template

func reader(input io.Reader) (string, error) {

	var (
		buff       bytes.Buffer
		err        error
		blockwidth = 12
		curblock   = 0
	)

	b := make([]byte, blockwidth)

	for n, err := input.Read(b); err == nil; n, err = input.Read(b) {
		for i := 0; i < n; i++ {
			_, err = fmt.Fprintf(&buff, "0x%02x,", b[i])
			if err != nil {
				break
			}
			curblock++
			if curblock < blockwidth {
				continue
			}
			buff.WriteByte('\n')
			buff.Write([]byte{'\t', '\t'})
			curblock = 0
		}
	}

	return buff.String(), err
}

func init() {

	pkg = template.Must(template.New("file").Funcs(template.FuncMap{"reader": reader}).Parse(` File{
	  data: []byte{
	{{ reader . }}
  },
  fi: FileInfo {
	name:    "{{ .Stat.Name }}",
    size:    {{ .Stat.Size }},
	modTime: time.Unix({{ .Stat.ModTime.Unix }},{{ .Stat.ModTime.UnixNano }}),
    isDir:   {{ .Stat.IsDir }},
  },
}`))

	pkg = template.Must(pkg.New("pkg").Parse(`{{ if .Tag }}// +build {{ .Tag }}

{{ end }}// Package {{.Pkg }} is generated by github.com/omeid/go-resources
package {{ .Pkg }}

import (
	"bytes"
	"errors"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"
)

// FileSystem is an http.FileSystem implementation.
type FileSystem struct {
	files map[string]File
}

// String returns the content of the file as string.
func (fs *FileSystem) String(name string) (string, bool) {
	if filepath.Separator != '/' && strings.IndexRune(name, filepath.Separator) >= 0 ||
		strings.Contains(name, "\x00") {
		return "", false
	}

	file, ok := fs.files[name]

	if !ok {
		return "", false
	}

	return string(file.data), true
}

// Open implements http.FileSystem.Open
func (fs *FileSystem) Open(name string) (http.File, error) {
	if filepath.Separator != '/' && strings.IndexRune(name, filepath.Separator) >= 0 ||
		strings.Contains(name, "\x00") {
		return nil, errors.New("http: invalid character in file path")
	}
	file, ok := fs.files[name]
	if !ok {
		files := []os.FileInfo{}
		for path, file := range fs.files {
			if strings.HasPrefix(path, name) {
				fi := file.fi
				files = append(files, &fi)
			}
		}

		if len(files) == 0 {
			return nil, os.ErrNotExist
		}

		//We have a directory.
		return &File{
			fi: FileInfo{
				isDir: true,
				files: files,
			}}, nil
	}
	file.Reader = bytes.NewReader(file.data)
	return &file, nil
}

// File implements http.File
type File struct {
	*bytes.Reader
	data []byte
	fi   FileInfo
}

// Close is a noop-closer.
func (f *File) Close() error {
	return nil
}

// Readdir implements http.File.Readdir
func (f *File) Readdir(count int) ([]os.FileInfo, error) {
	return nil, os.ErrNotExist
}

// Stat implements http.Stat.Readdir
func (f *File) Stat() (os.FileInfo, error) {
	return &f.fi, nil
}

// FileInfo implements the os.FileInfo interface.
type FileInfo struct {
	name    string
	size    int64
	mode    os.FileMode
	modTime time.Time
	isDir   bool
	sys     interface{}

	files []os.FileInfo
}

// Name implements os.FileInfo.Name
func (f *FileInfo) Name() string {
	return f.name
}

// Size implements os.FileInfo.Size
func (f *FileInfo) Size() int64 {
	return f.size
}

// Mode implements os.FileInfo.Mode
func (f *FileInfo) Mode() os.FileMode {
	return f.mode
}

// ModTime implements os.FileInfo.ModTime
func (f *FileInfo) ModTime() time.Time {
	return f.modTime
}

// IsDir implements os.FileInfo.IsDir
func (f *FileInfo) IsDir() bool {
	return f.isDir
}

// Readdir implements os.FileInfo.Readdir
func (f *FileInfo) Readdir(count int) ([]os.FileInfo, error) {
	return f.files, nil
}

// Sys returns the underlying value.
func (f *FileInfo) Sys() interface{} {
	return f.sys
}

func init() {
  {{ .Var }} = &FileSystem{
		files: map[string]File{
		  {{range $path, $file := .Files }} "/{{ $path }}": {{ template "file" $file }}, {{ end }}
		},
	  }
}
`))
}
<|endoftext|>


<commit_before>// Package rest represents the REST layer
package rest

import (
	"fmt"
	"net/http"

	log "github.com/cihub/seelog"
	"github.com/gorilla/mux"
)

type restError struct {
	Error  error
	Detail string
	Code   int
}

// RestHandlerWrapper manages all http error handling
type RestHandlerWrapper func(http.ResponseWriter, *http.Request) *restError

func (fn RestHandlerWrapper) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if restErr := fn(w, r); restErr != nil {
		if restErr.Detail != "" {
			log.Errorf("error detail: %s", restErr.Detail)
		}
		http.Error(w, restErr.Error.Error(), restErr.Code)
	}
}

// RestHandler includes all http handler methods
type RestHandler struct {
}

// HealthCheck is called by Google cloud to do health check
func (rest *RestHandler) HealthCheck(w http.ResponseWriter, req *http.Request) *restError {
	fmt.Fprintf(w, "OK World")
	return nil
}

func BuildRouter(restHandler RestHandler) *mux.Router {
	router := mux.NewRouter()

	router.Handle("/healthcheck",
		RestHandlerWrapper(restHandler.HealthCheck)).Methods("GET")

	return router
}
<commit_msg>Fix test<commit_after>// Package rest represents the REST layer
package rest

import (
	"fmt"
	"net/http"

	log "github.com/cihub/seelog"
	"github.com/gorilla/mux"
)

type restError struct {
	Error  error
	Detail string
	Code   int
}

// RestHandlerWrapper manages all http error handling
type RestHandlerWrapper func(http.ResponseWriter, *http.Request) *restError

func (fn RestHandlerWrapper) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if restErr := fn(w, r); restErr != nil {
		if restErr.Detail != "" {
			log.Errorf("error detail: %s", restErr.Detail)
		}
		http.Error(w, restErr.Error.Error(), restErr.Code)
	}
}

// RestHandler includes all http handler methods
type RestHandler struct {
}

// HealthCheck is called by Google cloud to do health check
func (rest *RestHandler) HealthCheck(w http.ResponseWriter, req *http.Request) *restError {
	fmt.Fprintf(w, "OK")
	return nil
}

func BuildRouter(restHandler RestHandler) *mux.Router {
	router := mux.NewRouter()

	router.Handle("/healthcheck",
		RestHandlerWrapper(restHandler.HealthCheck)).Methods("GET")

	return router
}
<|endoftext|>


<commit_before>package appui

import (
	"fmt"
	"strconv"
	"time"

	units "github.com/docker/go-units"
	termui "github.com/gizak/termui"
	"github.com/moncho/dry/docker"
	"github.com/moncho/dry/docker/formatter"
	"github.com/moncho/dry/ui"
	drytermui "github.com/moncho/dry/ui/termui"
)

var inactiveRowColor = termui.Attribute(ui.Color244)

//ContainerStatsRow is a Grid row showing runtime information about a container
type ContainerStatsRow struct {
	table     drytermui.Table
	container *docker.Container
	Status    *drytermui.ParColumn
	Name      *drytermui.ParColumn
	ID        *drytermui.ParColumn
	CPU       *drytermui.GaugeColumn
	Memory    *drytermui.GaugeColumn
	Net       *drytermui.ParColumn
	Block     *drytermui.ParColumn
	Pids      *drytermui.ParColumn
	Uptime    *drytermui.ParColumn

	drytermui.Row
}

//NewContainerStatsRow creats a new ContainerStatsRow widget
func NewContainerStatsRow(container *docker.Container, table drytermui.Table) *ContainerStatsRow {
	cf := formatter.NewContainerFormatter(container, true)
	row := &ContainerStatsRow{
		container: container,
		Status:    drytermui.NewThemedParColumn(DryTheme, statusSymbol),
		Name:      drytermui.NewThemedParColumn(DryTheme, cf.Names()),
		ID:        drytermui.NewThemedParColumn(DryTheme, cf.ID()),
		CPU:       drytermui.NewThemedGaugeColumn(DryTheme),
		Memory:    drytermui.NewThemedGaugeColumn(DryTheme),
		Net:       drytermui.NewThemedParColumn(DryTheme, "-"),
		Block:     drytermui.NewThemedParColumn(DryTheme, "-"),
		Pids:      drytermui.NewThemedParColumn(DryTheme, "-"),
		Uptime:    drytermui.NewThemedParColumn(DryTheme, container.Status),
	}
	row.Height = 1
	row.Table = table
	//Columns are rendered following the slice order
	row.Columns = []termui.GridBufferer{
		row.Status,
		row.ID,
		row.Name,
		row.CPU,
		row.Memory,
		row.Net,
		row.Block,
		row.Pids,
		row.Uptime,
	}
	if !docker.IsContainerRunning(container) {
		row.markAsNotRunning()
	} else {
		row.Status.TextFgColor = Running
	}
	return row

}

//NewSelfUpdatedContainerStatsRow creates a ContainerStatsRow that updates
//itself on stats message sent on the given channel
func NewSelfUpdatedContainerStatsRow(s *docker.StatsChannel, table drytermui.Table) *ContainerStatsRow {
	c := s.Container
	row := NewContainerStatsRow(c, table)

	if docker.IsContainerRunning(c) {
		go func() {
			for stat := range s.Stats {
				row.Update(c, stat)
			}
			row.markAsNotRunning()
		}()
	}
	return row
}

//Highlighted marks this rows as being highlighted
func (row *ContainerStatsRow) Highlighted() {
	row.changeTextColor(
		termui.Attribute(DryTheme.Fg),
		termui.Attribute(DryTheme.CursorLineBg))
}

//NotHighlighted marks this rows as being not highlighted
func (row *ContainerStatsRow) NotHighlighted() {
	row.changeTextColor(
		termui.Attribute(DryTheme.ListItem),
		termui.Attribute(DryTheme.Bg))
}

func (row *ContainerStatsRow) changeTextColor(fg, bg termui.Attribute) {
	row.ID.TextFgColor = fg
	row.ID.TextBgColor = bg
}

//Reset resets row content
func (row *ContainerStatsRow) Reset() {
	row.CPU.Reset()
	row.Memory.Reset()
	row.Net.Reset()
	row.Pids.Reset()
	row.Block.Reset()
	row.Uptime.Reset()
}

//Update updates the content of this row with the given stats
func (row *ContainerStatsRow) Update(container *docker.Container, stat *docker.Stats) {
	row.setNet(stat.NetworkRx, stat.NetworkTx)
	row.setCPU(stat.CPUPercentage)
	row.setMem(stat.Memory, stat.MemoryLimit, stat.MemoryPercentage)
	row.setBlockIO(stat.BlockRead, stat.BlockWrite)
	row.setPids(stat.PidsCurrent)
	row.setUptime(container.ContainerJSON.State.StartedAt)
}

func (row *ContainerStatsRow) setNet(rx float64, tx float64) {
	row.Net.Text = fmt.Sprintf("%s / %s", units.BytesSize(rx), units.BytesSize(tx))
}

func (row *ContainerStatsRow) setBlockIO(read float64, write float64) {
	row.Block.Text = fmt.Sprintf("%s / %s", units.BytesSize(read), units.BytesSize(write))
}
func (row *ContainerStatsRow) setPids(pids uint64) {
	row.Pids.Text = strconv.Itoa(int(pids))
}

func (row *ContainerStatsRow) setCPU(val float64) {
	row.CPU.Label = fmt.Sprintf("%.2f%%", val)
	cpu := int(val)
	if cpu > 0 && cpu < 5 {
		cpu = 5
	} else if cpu > 100 {
		cpu = 100
	}
	row.CPU.Percent = cpu
	row.CPU.BarColor = percentileToColor(cpu)
}

func (row *ContainerStatsRow) setMem(val float64, limit float64, percent float64) {
	row.Memory.Label = fmt.Sprintf("%s / %s", units.BytesSize(val), units.BytesSize(limit))
	mem := int(percent)
	if mem < 5 {
		mem = 5
	} else if mem > 100 {
		mem = 100
	}
	row.Memory.Percent = mem
	row.Memory.BarColor = percentileToColor(mem)
}

func (row *ContainerStatsRow) setUptime(startedAt string) {
	if startTime, err := time.Parse(time.RFC3339, startedAt); err == nil {
		row.Uptime.Text = units.HumanDuration(time.Now().UTC().Sub(startTime))
	} else {
		row.Uptime.Text = ""
	}
}

//markAsNotRunning
func (row *ContainerStatsRow) markAsNotRunning() {
	row.Status.TextFgColor = NotRunning
	row.Name.TextFgColor = inactiveRowColor
	row.ID.TextFgColor = inactiveRowColor
	row.CPU.PercentColor = inactiveRowColor
	row.CPU.Percent = 0
	row.CPU.Label = "-"
	row.Memory.PercentColor = inactiveRowColor
	row.Memory.Percent = 0
	row.Memory.Label = "-"
	row.Net.TextFgColor = inactiveRowColor
	row.Net.Text = "-"
	row.Block.TextFgColor = inactiveRowColor
	row.Block.Text = "-"
	row.Pids.Text = "0"
	row.Pids.TextFgColor = inactiveRowColor
	row.Uptime.Text = "-"
	row.Pids.TextFgColor = inactiveRowColor

}

func percentileToColor(n int) termui.Attribute {
	c := ui.Color23
	if n > 90 {
		c = ui.Color161
	} else if n > 60 {
		c = ui.Color131
	}
	return termui.Attribute(c)
}
<commit_msg>Minor<commit_after>package appui

import (
	"fmt"
	"strconv"
	"time"

	units "github.com/docker/go-units"
	termui "github.com/gizak/termui"
	"github.com/moncho/dry/docker"
	"github.com/moncho/dry/docker/formatter"
	"github.com/moncho/dry/ui"
	drytermui "github.com/moncho/dry/ui/termui"
)

var inactiveRowColor = termui.Attribute(ui.Color244)

//ContainerStatsRow is a Grid row showing runtime information about a container
type ContainerStatsRow struct {
	table     drytermui.Table
	container *docker.Container
	Status    *drytermui.ParColumn
	Name      *drytermui.ParColumn
	ID        *drytermui.ParColumn
	CPU       *drytermui.GaugeColumn
	Memory    *drytermui.GaugeColumn
	Net       *drytermui.ParColumn
	Block     *drytermui.ParColumn
	Pids      *drytermui.ParColumn
	Uptime    *drytermui.ParColumn

	drytermui.Row
}

//NewContainerStatsRow creats a new ContainerStatsRow widget
func NewContainerStatsRow(container *docker.Container, table drytermui.Table) *ContainerStatsRow {
	cf := formatter.NewContainerFormatter(container, true)
	row := &ContainerStatsRow{
		container: container,
		Status:    drytermui.NewThemedParColumn(DryTheme, statusSymbol),
		Name:      drytermui.NewThemedParColumn(DryTheme, cf.Names()),
		ID:        drytermui.NewThemedParColumn(DryTheme, cf.ID()),
		CPU:       drytermui.NewThemedGaugeColumn(DryTheme),
		Memory:    drytermui.NewThemedGaugeColumn(DryTheme),
		Net:       drytermui.NewThemedParColumn(DryTheme, "-"),
		Block:     drytermui.NewThemedParColumn(DryTheme, "-"),
		Pids:      drytermui.NewThemedParColumn(DryTheme, "-"),
		Uptime:    drytermui.NewThemedParColumn(DryTheme, container.Status),
	}
	row.Height = 1
	row.Table = table
	//Columns are rendered following the slice order
	row.Columns = []termui.GridBufferer{
		row.Status,
		row.ID,
		row.Name,
		row.CPU,
		row.Memory,
		row.Net,
		row.Block,
		row.Pids,
		row.Uptime,
	}
	if !docker.IsContainerRunning(container) {
		row.markAsNotRunning()
	} else {
		row.Status.TextFgColor = Running
	}
	return row

}

//NewSelfUpdatedContainerStatsRow creates a ContainerStatsRow that updates
//itself on stats message sent on the given channel
func NewSelfUpdatedContainerStatsRow(s *docker.StatsChannel, table drytermui.Table) *ContainerStatsRow {
	c := s.Container
	row := NewContainerStatsRow(c, table)

	if docker.IsContainerRunning(c) {
		go func() {
			for stat := range s.Stats {
				row.Update(c, stat)
			}
			row.markAsNotRunning()
		}()
	}
	return row
}

//Highlighted marks this rows as being highlighted
func (row *ContainerStatsRow) Highlighted() {
	row.changeTextColor(
		termui.Attribute(DryTheme.Fg),
		termui.Attribute(DryTheme.CursorLineBg))
}

//NotHighlighted marks this rows as being not highlighted
func (row *ContainerStatsRow) NotHighlighted() {
	row.changeTextColor(
		termui.Attribute(DryTheme.ListItem),
		termui.Attribute(DryTheme.Bg))
}

func (row *ContainerStatsRow) changeTextColor(fg, bg termui.Attribute) {
	row.ID.TextFgColor = fg
	row.ID.TextBgColor = bg
}

//Reset resets row content
func (row *ContainerStatsRow) Reset() {
	row.CPU.Reset()
	row.Memory.Reset()
	row.Net.Reset()
	row.Pids.Reset()
	row.Block.Reset()
	row.Uptime.Reset()
}

//Update updates the content of this row with the given stats
func (row *ContainerStatsRow) Update(container *docker.Container, stat *docker.Stats) {
	row.setNet(stat.NetworkRx, stat.NetworkTx)
	row.setCPU(stat.CPUPercentage)
	row.setMem(stat.Memory, stat.MemoryLimit, stat.MemoryPercentage)
	row.setBlockIO(stat.BlockRead, stat.BlockWrite)
	row.setPids(stat.PidsCurrent)
	row.setUptime(container.ContainerJSON.State.StartedAt)
}

func (row *ContainerStatsRow) setNet(rx float64, tx float64) {
	row.Net.Text = fmt.Sprintf("%s / %s", units.BytesSize(rx), units.BytesSize(tx))
}

func (row *ContainerStatsRow) setBlockIO(read float64, write float64) {
	row.Block.Text = fmt.Sprintf("%s / %s", units.BytesSize(read), units.BytesSize(write))
}
func (row *ContainerStatsRow) setPids(pids uint64) {
	row.Pids.Text = strconv.Itoa(int(pids))
}

func (row *ContainerStatsRow) setCPU(val float64) {
	row.CPU.Label = fmt.Sprintf("%.2f%%", val)
	cpu := int(val)
	if val > 0 && val < 5 {
		cpu = 5
	} else if val > 100 {
		cpu = 100
	}
	row.CPU.Percent = cpu
	row.CPU.BarColor = percentileToColor(cpu)
}

func (row *ContainerStatsRow) setMem(val float64, limit float64, percent float64) {
	row.Memory.Label = fmt.Sprintf("%s / %s", units.BytesSize(val), units.BytesSize(limit))
	mem := int(percent)
	if mem < 5 {
		mem = 5
	} else if mem > 100 {
		mem = 100
	}
	row.Memory.Percent = mem
	row.Memory.BarColor = percentileToColor(mem)
}

func (row *ContainerStatsRow) setUptime(startedAt string) {
	if startTime, err := time.Parse(time.RFC3339, startedAt); err == nil {
		row.Uptime.Text = units.HumanDuration(time.Now().UTC().Sub(startTime))
	} else {
		row.Uptime.Text = ""
	}
}

//markAsNotRunning
func (row *ContainerStatsRow) markAsNotRunning() {
	row.Status.TextFgColor = NotRunning
	row.Name.TextFgColor = inactiveRowColor
	row.ID.TextFgColor = inactiveRowColor
	row.CPU.PercentColor = inactiveRowColor
	row.CPU.Percent = 0
	row.CPU.Label = "-"
	row.Memory.PercentColor = inactiveRowColor
	row.Memory.Percent = 0
	row.Memory.Label = "-"
	row.Net.TextFgColor = inactiveRowColor
	row.Net.Text = "-"
	row.Block.TextFgColor = inactiveRowColor
	row.Block.Text = "-"
	row.Pids.Text = "0"
	row.Pids.TextFgColor = inactiveRowColor
	row.Uptime.Text = "-"
	row.Pids.TextFgColor = inactiveRowColor

}

func percentileToColor(n int) termui.Attribute {
	c := ui.Color23
	if n > 90 {
		c = ui.Color161
	} else if n > 60 {
		c = ui.Color131
	}
	return termui.Attribute(c)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"fmt"
	"net/http"
	"os"
	"os/exec"
	"strings"
	"time"
)

type Packager struct{}

func (Packager) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	var (
		goarch string
		goos   string
		pack   string
	)

	hosts := strings.Split(r.URL.Host, ".")
	if len(hosts) != len(strings.Split("darwin.386.binloader.xyz", ".")) {
		goarch = r.FormValue("arch")
		goos = r.FormValue("os")
	} else {
		goarch = hosts[0]
		goos = hosts[1]
	}

	pack = strings.TrimLeft(r.URL.Path, "/")

	var out bytes.Buffer

	// go get
	cmd := exec.Command("go", "get", pack)
	cmd.Stdout = &out
	cmd.Stderr = &out

	err := cmd.Run()
	if err != nil {
		fmt.Fprintln(w, "error `go get`ing:", err, out.String())
		return
	}

	name := fmt.Sprintf("/tmp/binary_%v", time.Now().UnixNano())

	cmd = exec.Command("env", "GOOS="+goos, "GOARCH="+goarch, "go", "build", "-o="+name, pack)

	fmt.Println("args:", cmd.Args)
	fmt.Println("env:", cmd.Env)

	fmt.Printf("compiling with os: %v and arch: %v, to file: %v\n", goos, goarch, name)

	out = bytes.Buffer{}

	err = cmd.Run()
	if err != nil {
		fmt.Fprintln(w, "error `go build`ing:", err, out.String())
		return
	}

	f, err := os.Open(name)
	if err != nil {
		fmt.Fprintln(w, "error opening file:", err)
		return
	}

	reader := bufio.NewReader(f)
	reader.WriteTo(w)

	//fmt.Fprintf(w, "building %v for %v_%v", pack, goarch, goos)
}
<commit_msg>Fix speed<commit_after>package main

import (
	"bytes"
	"fmt"
	"io"
	"net/http"
	"os"
	"os/exec"
	"strings"
	"time"
)

type Packager struct{}

func (Packager) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	var (
		goarch string
		goos   string
		pack   string
	)

	hosts := strings.Split(r.URL.Host, ".")
	if len(hosts) != len(strings.Split("darwin.386.binloader.xyz", ".")) {
		goarch = r.FormValue("arch")
		goos = r.FormValue("os")
	} else {
		goarch = hosts[0]
		goos = hosts[1]
	}

	pack = strings.TrimLeft(r.URL.Path, "/")

	var out bytes.Buffer

	// go get
	cmd := exec.Command("go", "get", pack)
	cmd.Stdout = &out
	cmd.Stderr = &out

	err := cmd.Run()
	if err != nil {
		fmt.Fprintln(w, "error `go get`ing:", err, out.String())
		return
	}

	name := fmt.Sprintf("/tmp/binary_%v", time.Now().UnixNano())

	cmd = exec.Command("env", "GOOS="+goos, "GOARCH="+goarch, "go", "build", "-o="+name, pack)

	fmt.Println("args:", cmd.Args)
	fmt.Println("env:", cmd.Env)

	fmt.Printf("compiling with os: %v and arch: %v, to file: %v\n", goos, goarch, name)

	out = bytes.Buffer{}

	err = cmd.Run()
	if err != nil {
		fmt.Fprintln(w, "error `go build`ing:", err, out.String())
		return
	}

	f, err := os.Open(name)
	if err != nil {
		fmt.Fprintln(w, "error opening file:", err)
		return
	}

	io.Copy(w, f)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"regexp"
	"strconv"
)

// Action ...
type Action int

const (
	// Replace ...
	Replace Action = iota

	// Inject ...
	Inject
)

// OperatorPhase ...
type OperatorPhase int

const (
	// MergePhase ...
	MergePhase OperatorPhase = iota
	// EvalPhase ...
	EvalPhase
	// CheckPhase ...
	CheckPhase
)

// Response ...
type Response struct {
	Type  Action
	Value interface{}
}

// Operator ...
type Operator interface {
	// setup whatever global/static state needed -- see (( static_ips ... ))
	Setup() error

	// evaluate the tree and determine what should be done to satisfy caller
	Run(ev *Evaluator, args []*Expr) (*Response, error)

	// returns a set of implicit / inherent dependencies used by Run()
	Dependencies(ev *Evaluator, args []*Expr, locs []*Cursor) []*Cursor

	// what phase does this operator run during?
	Phase() OperatorPhase
}

// OpRegistry ...
var OpRegistry map[string]Operator

// OperatorFor ...
func OperatorFor(name string) Operator {
	if op, ok := OpRegistry[name]; ok {
		return op
	}
	return NullOperator{Missing: name}
}

// RegisterOp ...
func RegisterOp(name string, op Operator) {
	if OpRegistry == nil {
		OpRegistry = map[string]Operator{}
	}
	OpRegistry[name] = op
}

// SetupOperators ...
func SetupOperators(phase OperatorPhase) error {
	errors := MultiError{Errors: []error{}}
	for _, op := range OpRegistry {
		if op.Phase() == phase {
			if err := op.Setup(); err != nil {
				errors.Append(err)
			}
		}
	}
	if len(errors.Errors) > 0 {
		return errors
	}
	return nil
}

// ExprType ...
type ExprType int

const (
	// Reference ...
	Reference ExprType = iota
	// Literal ...
	Literal
	// LogicalOr ...
	LogicalOr
)

// Expr ...
type Expr struct {
	Type      ExprType
	Reference *Cursor
	Literal   interface{}
	Left      *Expr
	Right     *Expr
}

func (e *Expr) String() string {
	switch e.Type {
	case Literal:
		if e.Literal == nil {
			return "nil"
		}
		if _, ok := e.Literal.(string); ok {
			return fmt.Sprintf(`"%s"`, e.Literal)
		}
		return fmt.Sprintf("%v", e.Literal)

	case Reference:
		return e.Reference.String()

	case LogicalOr:
		return fmt.Sprintf("(%s || %s)", e.Left, e.Right)

	default:
		return "<!! unknown !!>"
	}
}

// Reduce ...
func (e *Expr) Reduce() (*Expr, error) {

	var reduce func(*Expr) (*Expr, *Expr, bool)
	reduce = func(e *Expr) (*Expr, *Expr, bool) {
		switch e.Type {
		case Literal:
			return e, e, false
		case Reference:
			return e, nil, false

		case LogicalOr:
			l, short, _ := reduce(e.Left)
			if short != nil {
				return l, short, true
			}

			r, short, more := reduce(e.Right)
			return &Expr{
				Type:  LogicalOr,
				Left:  l,
				Right: r,
			}, short, more
		}
		return nil, nil, false
	}

	reduced, short, more := reduce(e)
	if more && short != nil {
		return reduced, fmt.Errorf("literal %v short-circuits expression", short)
	}
	return reduced, nil
}

// Resolve ...
func (e *Expr) Resolve(tree map[interface{}]interface{}) (*Expr, error) {
	switch e.Type {
	case Literal:
		return e, nil

	case Reference:
		if _, err := e.Reference.Resolve(tree); err != nil {
			return nil, fmt.Errorf("Unable to resolve `%s`: %s", e.Reference, err)
		}
		return e, nil

	case LogicalOr:
		if o, err := e.Left.Resolve(tree); err == nil {
			return o, nil
		}
		return e.Right.Resolve(tree)
	}
	return nil, fmt.Errorf("unknown expression operand type (%d)", e.Type)
}

// Evaluate ...
func (e *Expr) Evaluate(tree map[interface{}]interface{}) (interface{}, error) {
	final, err := e.Resolve(tree)
	if err != nil {
		return nil, err
	}

	switch final.Type {
	case Literal:
		return final.Literal, nil
	case Reference:
		return final.Reference.Resolve(tree)
	case LogicalOr:
		return nil, fmt.Errorf("expression resolved to a logical OR operation (which shouldn't happen)")
	}
	return nil, fmt.Errorf("unknown operand type")
}

// Dependencies ...
func (e *Expr) Dependencies(ev *Evaluator, locs []*Cursor) []*Cursor {
	l := []*Cursor{}

	canonicalize := func(c *Cursor) {
		cc := c.Copy()
		for cc.Depth() > 0 {
			if _, err := cc.Canonical(ev.Tree); err == nil {
				break
			}
			cc.Pop()
		}
		if cc.Depth() > 0 {
			l = append(l, cc)
		}
	}

	switch e.Type {
	case Reference:
		canonicalize(e.Reference)

	case LogicalOr:
		for _, c := range e.Left.Dependencies(ev, locs) {
			canonicalize(c)
		}
		for _, c := range e.Right.Dependencies(ev, locs) {
			canonicalize(c)
		}
	}

	return l
}

// Opcall ...
type Opcall struct {
	src       string
	where     *Cursor
	canonical *Cursor
	op        Operator
	args      []*Expr
}

// ParseOpcall ...
func ParseOpcall(phase OperatorPhase, src string) (*Opcall, error) {
	split := func(src string) []string {
		list := make([]string, 0, 0)

		buf := ""
		escaped := false
		quoted := false

		for _, c := range src {
			if escaped {
				buf += string(c)
				escaped = false
				continue
			}

			if c == '\\' {
				escaped = true
				continue
			}

			if c == ' ' || c == '\t' || c == ',' {
				if quoted {
					buf += string(c)
					continue
				} else if buf != "" {
					list = append(list, buf)
					buf = ""
				}
				continue
			}

			if c == '"' {
				buf += string(c)
				quoted = !quoted
				continue
			}

			buf += string(c)
		}

		if buf != "" {
			list = append(list, buf)
		}

		return list
	}

	argify := func(src string) (args []*Expr, err error) {
		qstring := regexp.MustCompile(`^"(.*)"$`)
		integer := regexp.MustCompile(`^[+-]?\d+(\.\d+)?$`)
		float := regexp.MustCompile(`^[+-]?\d*\.\d+$`)

		var stack []*Expr
		for i, arg := range split(src) {
			switch {
			case qstring.MatchString(arg):
				m := qstring.FindStringSubmatch(arg)
				DEBUG("  #%d: parsed as quoted string literal '%s'", i, m[1])
				stack = append(stack, &Expr{Type: Literal, Literal: m[1]})

			case float.MatchString(arg):
				DEBUG("  #%d: parsed as unquoted floating point literal '%s'", i, arg)
				v, err := strconv.ParseFloat(arg, 64)
				if err != nil {
					DEBUG("  #%d: %s is not parseable as a floatin point number: %s", i, arg, err)
					return args, err
				}
				stack = append(stack, &Expr{Type: Literal, Literal: v})

			case integer.MatchString(arg):
				DEBUG("  #%d: parsed as unquoted integer literal '%s'", i, arg)
				v, err := strconv.ParseInt(arg, 10, 64)
				if err != nil {
					DEBUG("  #%d: %s is not parseable as an integer: %s", i, arg, err)
					return args, err
				}
				stack = append(stack, &Expr{Type: Literal, Literal: v})

			case arg == "||":
				DEBUG("  #%d: parsed logical-or operator, '||'", i)
				stack = append(stack, &Expr{Type: LogicalOr})

			case arg == "nil" || arg == "null" || arg == "~":
				DEBUG("  #%d: parsed the nil value token '%s'", i, arg)
				stack = append(stack, &Expr{Type: Literal, Literal: nil})

			case arg == "false" || arg == "False" || arg == "FALSE":
				DEBUG("  #%d: parsed the false value token '%s'", i, arg)
				stack = append(stack, &Expr{Type: Literal, Literal: false})

			case arg == "true" || arg == "True" || arg == "TRUE":
				DEBUG("  #%d: parsed the true value token '%s'", i, arg)
				stack = append(stack, &Expr{Type: Literal, Literal: true})

			default:
				c, err := ParseCursor(arg)
				if err != nil {
					DEBUG("  #%d: %s is a malformed reference: %s", i, arg, err)
					return args, err
				}
				DEBUG("  #%d: parsed as a reference to $.%s", i, c)
				stack = append(stack, &Expr{Type: Reference, Reference: c})
			}
		}
		DEBUG("")

		push := func(e *Expr) {
			if e == nil {
				return
			}
			reduced, err := e.Reduce()
			if err != nil {
				fmt.Fprintf(os.Stdout, "warning: %s\n", err)
			}
			args = append(args, reduced)
		}

		var e *Expr
		for len(stack) > 0 {
			if e == nil {
				e = stack[0]
				stack = stack[1:]
				continue
			}

			if stack[0].Type == LogicalOr {
				stack[0].Left = e
				e = stack[0]
				stack = stack[1:]
				continue
			}

			if e.Type == LogicalOr {
				if e.Right != nil {
					e = &Expr{Type: LogicalOr, Left: e}
				}
				e.Right = stack[0]
				stack = stack[1:]
				continue
			}

			push(e)
			e = stack[0]
			stack = stack[1:]
		}
		push(e)

		return args, nil
	}

	op := &Opcall{src: src}

	for _, pattern := range []string{
		`^\Q((\E\s*([a-zA-Z][a-zA-Z0-9_-]*)(?:\s*\((.*)\))?\s*\Q))\E$`, // (( op(x,y,z) ))
		`^\Q((\E\s*([a-zA-Z][a-zA-Z0-9_-]*)(?:\s+(.*))?\s*\Q))\E$`,     // (( op x y z ))
	} {
		re := regexp.MustCompile(pattern)
		if !re.MatchString(src) {
			continue
		}

		m := re.FindStringSubmatch(src)
		DEBUG("parsing `%s': looks like a (( %s ... )) operator\n arguments:", src, m[1])

		op.op = OperatorFor(m[1])
		if op.op.Phase() != phase {
			DEBUG("  - skipping (( %s ... )) operation; it belongs to a different phase", m[1])
			return nil, nil
		}

		args, err := argify(m[2])
		if err != nil {
			return nil, err
		}
		if len(args) == 0 {
			DEBUG("  (none)")
		}
		op.args = args
		return op, nil
	}

	return nil, nil
}

// Dependencies ...
func (op *Opcall) Dependencies(ev *Evaluator, locs []*Cursor) []*Cursor {
	l := []*Cursor{}
	for _, arg := range op.args {
		for _, c := range arg.Dependencies(ev, locs) {
			l = append(l, c)
		}
	}

	for _, c := range op.op.Dependencies(ev, op.args, locs) {
		l = append(l, c)
	}
	return l
}

// Run ...
func (op *Opcall) Run(ev *Evaluator) (*Response, error) {
	was := ev.Here
	ev.Here = op.where
	r, err := op.op.Run(ev, op.args)
	ev.Here = was

	if err != nil {
		return nil, fmt.Errorf("$.%s: %s", op.where, err)
	}
	return r, nil
}
<commit_msg>case-desensitized nil<commit_after>package main

import (
	"fmt"
	"os"
	"regexp"
	"strconv"
)

// Action ...
type Action int

const (
	// Replace ...
	Replace Action = iota

	// Inject ...
	Inject
)

// OperatorPhase ...
type OperatorPhase int

const (
	// MergePhase ...
	MergePhase OperatorPhase = iota
	// EvalPhase ...
	EvalPhase
	// CheckPhase ...
	CheckPhase
)

// Response ...
type Response struct {
	Type  Action
	Value interface{}
}

// Operator ...
type Operator interface {
	// setup whatever global/static state needed -- see (( static_ips ... ))
	Setup() error

	// evaluate the tree and determine what should be done to satisfy caller
	Run(ev *Evaluator, args []*Expr) (*Response, error)

	// returns a set of implicit / inherent dependencies used by Run()
	Dependencies(ev *Evaluator, args []*Expr, locs []*Cursor) []*Cursor

	// what phase does this operator run during?
	Phase() OperatorPhase
}

// OpRegistry ...
var OpRegistry map[string]Operator

// OperatorFor ...
func OperatorFor(name string) Operator {
	if op, ok := OpRegistry[name]; ok {
		return op
	}
	return NullOperator{Missing: name}
}

// RegisterOp ...
func RegisterOp(name string, op Operator) {
	if OpRegistry == nil {
		OpRegistry = map[string]Operator{}
	}
	OpRegistry[name] = op
}

// SetupOperators ...
func SetupOperators(phase OperatorPhase) error {
	errors := MultiError{Errors: []error{}}
	for _, op := range OpRegistry {
		if op.Phase() == phase {
			if err := op.Setup(); err != nil {
				errors.Append(err)
			}
		}
	}
	if len(errors.Errors) > 0 {
		return errors
	}
	return nil
}

// ExprType ...
type ExprType int

const (
	// Reference ...
	Reference ExprType = iota
	// Literal ...
	Literal
	// LogicalOr ...
	LogicalOr
)

// Expr ...
type Expr struct {
	Type      ExprType
	Reference *Cursor
	Literal   interface{}
	Left      *Expr
	Right     *Expr
}

func (e *Expr) String() string {
	switch e.Type {
	case Literal:
		if e.Literal == nil {
			return "nil"
		}
		if _, ok := e.Literal.(string); ok {
			return fmt.Sprintf(`"%s"`, e.Literal)
		}
		return fmt.Sprintf("%v", e.Literal)

	case Reference:
		return e.Reference.String()

	case LogicalOr:
		return fmt.Sprintf("(%s || %s)", e.Left, e.Right)

	default:
		return "<!! unknown !!>"
	}
}

// Reduce ...
func (e *Expr) Reduce() (*Expr, error) {

	var reduce func(*Expr) (*Expr, *Expr, bool)
	reduce = func(e *Expr) (*Expr, *Expr, bool) {
		switch e.Type {
		case Literal:
			return e, e, false
		case Reference:
			return e, nil, false

		case LogicalOr:
			l, short, _ := reduce(e.Left)
			if short != nil {
				return l, short, true
			}

			r, short, more := reduce(e.Right)
			return &Expr{
				Type:  LogicalOr,
				Left:  l,
				Right: r,
			}, short, more
		}
		return nil, nil, false
	}

	reduced, short, more := reduce(e)
	if more && short != nil {
		return reduced, fmt.Errorf("literal %v short-circuits expression", short)
	}
	return reduced, nil
}

// Resolve ...
func (e *Expr) Resolve(tree map[interface{}]interface{}) (*Expr, error) {
	switch e.Type {
	case Literal:
		return e, nil

	case Reference:
		if _, err := e.Reference.Resolve(tree); err != nil {
			return nil, fmt.Errorf("Unable to resolve `%s`: %s", e.Reference, err)
		}
		return e, nil

	case LogicalOr:
		if o, err := e.Left.Resolve(tree); err == nil {
			return o, nil
		}
		return e.Right.Resolve(tree)
	}
	return nil, fmt.Errorf("unknown expression operand type (%d)", e.Type)
}

// Evaluate ...
func (e *Expr) Evaluate(tree map[interface{}]interface{}) (interface{}, error) {
	final, err := e.Resolve(tree)
	if err != nil {
		return nil, err
	}

	switch final.Type {
	case Literal:
		return final.Literal, nil
	case Reference:
		return final.Reference.Resolve(tree)
	case LogicalOr:
		return nil, fmt.Errorf("expression resolved to a logical OR operation (which shouldn't happen)")
	}
	return nil, fmt.Errorf("unknown operand type")
}

// Dependencies ...
func (e *Expr) Dependencies(ev *Evaluator, locs []*Cursor) []*Cursor {
	l := []*Cursor{}

	canonicalize := func(c *Cursor) {
		cc := c.Copy()
		for cc.Depth() > 0 {
			if _, err := cc.Canonical(ev.Tree); err == nil {
				break
			}
			cc.Pop()
		}
		if cc.Depth() > 0 {
			l = append(l, cc)
		}
	}

	switch e.Type {
	case Reference:
		canonicalize(e.Reference)

	case LogicalOr:
		for _, c := range e.Left.Dependencies(ev, locs) {
			canonicalize(c)
		}
		for _, c := range e.Right.Dependencies(ev, locs) {
			canonicalize(c)
		}
	}

	return l
}

// Opcall ...
type Opcall struct {
	src       string
	where     *Cursor
	canonical *Cursor
	op        Operator
	args      []*Expr
}

// ParseOpcall ...
func ParseOpcall(phase OperatorPhase, src string) (*Opcall, error) {
	split := func(src string) []string {
		list := make([]string, 0, 0)

		buf := ""
		escaped := false
		quoted := false

		for _, c := range src {
			if escaped {
				buf += string(c)
				escaped = false
				continue
			}

			if c == '\\' {
				escaped = true
				continue
			}

			if c == ' ' || c == '\t' || c == ',' {
				if quoted {
					buf += string(c)
					continue
				} else if buf != "" {
					list = append(list, buf)
					buf = ""
				}
				continue
			}

			if c == '"' {
				buf += string(c)
				quoted = !quoted
				continue
			}

			buf += string(c)
		}

		if buf != "" {
			list = append(list, buf)
		}

		return list
	}

	argify := func(src string) (args []*Expr, err error) {
		qstring := regexp.MustCompile(`^"(.*)"$`)
		integer := regexp.MustCompile(`^[+-]?\d+(\.\d+)?$`)
		float := regexp.MustCompile(`^[+-]?\d*\.\d+$`)

		var stack []*Expr
		for i, arg := range split(src) {
			switch {
			case qstring.MatchString(arg):
				m := qstring.FindStringSubmatch(arg)
				DEBUG("  #%d: parsed as quoted string literal '%s'", i, m[1])
				stack = append(stack, &Expr{Type: Literal, Literal: m[1]})

			case float.MatchString(arg):
				DEBUG("  #%d: parsed as unquoted floating point literal '%s'", i, arg)
				v, err := strconv.ParseFloat(arg, 64)
				if err != nil {
					DEBUG("  #%d: %s is not parseable as a floatin point number: %s", i, arg, err)
					return args, err
				}
				stack = append(stack, &Expr{Type: Literal, Literal: v})

			case integer.MatchString(arg):
				DEBUG("  #%d: parsed as unquoted integer literal '%s'", i, arg)
				v, err := strconv.ParseInt(arg, 10, 64)
				if err != nil {
					DEBUG("  #%d: %s is not parseable as an integer: %s", i, arg, err)
					return args, err
				}
				stack = append(stack, &Expr{Type: Literal, Literal: v})

			case arg == "||":
				DEBUG("  #%d: parsed logical-or operator, '||'", i)
				stack = append(stack, &Expr{Type: LogicalOr})

			case arg == "nil" || arg == "null" || arg == "~" || arg == "Nil" || arg == "Null" || arg == "NILL" || arg == "NULL":
				DEBUG("  #%d: parsed the nil value token '%s'", i, arg)
				stack = append(stack, &Expr{Type: Literal, Literal: nil})

			case arg == "false" || arg == "False" || arg == "FALSE":
				DEBUG("  #%d: parsed the false value token '%s'", i, arg)
				stack = append(stack, &Expr{Type: Literal, Literal: false})

			case arg == "true" || arg == "True" || arg == "TRUE":
				DEBUG("  #%d: parsed the true value token '%s'", i, arg)
				stack = append(stack, &Expr{Type: Literal, Literal: true})

			default:
				c, err := ParseCursor(arg)
				if err != nil {
					DEBUG("  #%d: %s is a malformed reference: %s", i, arg, err)
					return args, err
				}
				DEBUG("  #%d: parsed as a reference to $.%s", i, c)
				stack = append(stack, &Expr{Type: Reference, Reference: c})
			}
		}
		DEBUG("")

		push := func(e *Expr) {
			if e == nil {
				return
			}
			reduced, err := e.Reduce()
			if err != nil {
				fmt.Fprintf(os.Stdout, "warning: %s\n", err)
			}
			args = append(args, reduced)
		}

		var e *Expr
		for len(stack) > 0 {
			if e == nil {
				e = stack[0]
				stack = stack[1:]
				continue
			}

			if stack[0].Type == LogicalOr {
				stack[0].Left = e
				e = stack[0]
				stack = stack[1:]
				continue
			}

			if e.Type == LogicalOr {
				if e.Right != nil {
					e = &Expr{Type: LogicalOr, Left: e}
				}
				e.Right = stack[0]
				stack = stack[1:]
				continue
			}

			push(e)
			e = stack[0]
			stack = stack[1:]
		}
		push(e)

		return args, nil
	}

	op := &Opcall{src: src}

	for _, pattern := range []string{
		`^\Q((\E\s*([a-zA-Z][a-zA-Z0-9_-]*)(?:\s*\((.*)\))?\s*\Q))\E$`, // (( op(x,y,z) ))
		`^\Q((\E\s*([a-zA-Z][a-zA-Z0-9_-]*)(?:\s+(.*))?\s*\Q))\E$`,     // (( op x y z ))
	} {
		re := regexp.MustCompile(pattern)
		if !re.MatchString(src) {
			continue
		}

		m := re.FindStringSubmatch(src)
		DEBUG("parsing `%s': looks like a (( %s ... )) operator\n arguments:", src, m[1])

		op.op = OperatorFor(m[1])
		if op.op.Phase() != phase {
			DEBUG("  - skipping (( %s ... )) operation; it belongs to a different phase", m[1])
			return nil, nil
		}

		args, err := argify(m[2])
		if err != nil {
			return nil, err
		}
		if len(args) == 0 {
			DEBUG("  (none)")
		}
		op.args = args
		return op, nil
	}

	return nil, nil
}

// Dependencies ...
func (op *Opcall) Dependencies(ev *Evaluator, locs []*Cursor) []*Cursor {
	l := []*Cursor{}
	for _, arg := range op.args {
		for _, c := range arg.Dependencies(ev, locs) {
			l = append(l, c)
		}
	}

	for _, c := range op.op.Dependencies(ev, op.args, locs) {
		l = append(l, c)
	}
	return l
}

// Run ...
func (op *Opcall) Run(ev *Evaluator) (*Response, error) {
	was := ev.Here
	ev.Here = op.where
	r, err := op.op.Run(ev, op.args)
	ev.Here = was

	if err != nil {
		return nil, fmt.Errorf("$.%s: %s", op.where, err)
	}
	return r, nil
}
<|endoftext|>