<commit_before>package dcc

import "sync"

// CommandRepeat specifies how many times a single
// packet is sent.
var CommandRepeat = 30

// CommandMaxQueue specifies how many commands can
// queue before sending a new command blocks
// the sender
var CommandMaxQueue = 3

// Controller represents a DCC Control Station. The
// controller keeps tracks of the DCC Locomotives and
// is in charge of sending DCC packets continuously to
// the tracks.
type Controller struct {
	locomotives map[string]*Locomotive
	mux         sync.Mutex
	driver      Driver

	started    bool
	doneCh     chan bool
	shutdownCh chan bool
	commandCh  chan *Packet
}

// NewController builds a Controller.
func NewController(d Driver) *Controller {
	d.TracksOff()
	return &Controller{
		driver:      d,
		locomotives: make(map[string]*Locomotive),
		doneCh:      make(chan bool),
		shutdownCh:  make(chan bool),
		commandCh:   make(chan *Packet, CommandMaxQueue),
	}
}

// NewControllerWithConfig builds a new Controller using the
// given configuration.
func NewControllerWithConfig(d Driver, cfg *Config) *Controller {
	c := NewController(d)

	for _, loco := range cfg.Locomotives {
		c.AddLoco(&Locomotive{
			Name:      loco.Name,
			Address:   loco.Address,
			Speed:     loco.Speed,
			Direction: loco.Direction,
			Fl:        loco.Fl})
	}
	return c
}

// AddLoco adds a DCC device to the controller. The device
// will start receiving packets if the controller is running.
func (c *Controller) AddLoco(l *Locomotive) {
	c.mux.Lock()
	defer c.mux.Unlock()
	c.locomotives[l.Name] = l
}

// RmLoco removes a DCC device from the controller. There
// will be no longer packets sent to it.
func (c *Controller) RmLoco(l *Locomotive) {
	c.mux.Lock()
	defer c.mux.Unlock()
	delete(c.locomotives, l.Name)
}

// GetLoco retrieves a DCC device by its Name. The boolean is
// true if the Locomotive was found.
func (c *Controller) GetLoco(n string) (*Locomotive, bool) {
	c.mux.Lock()
	defer c.mux.Unlock()
	l, ok := c.locomotives[n]
	return l, ok
}

// Locos returns a list of all registered Locomotives.
func (c *Controller) Locos() []*Locomotive {
	c.mux.Lock()
	defer c.mux.Unlock()
	locos := make([]*Locomotive, 0, len(c.locomotives))
	for _, l := range c.locomotives {
		locos = append(locos, l)
	}
	return locos
}

// Command allows to send a custom Packet to the tracks.
// The packet will be sent CommandRepeat times.
func (c *Controller) Command(p *Packet) {
	c.commandCh <- p
}

// Start starts the controller: powers on the tracks
// and starts sending packets on them.
func (c *Controller) Start() {
	c.driver.TracksOn()
	go c.run()
	c.started = true
}

// Stop shuts down the controller by stopping to send
// packets and removing power from the tracks.
func (c *Controller) Stop() {
	if c.started {
		c.shutdownCh <- true
		<-c.doneCh
		c.started = false
	}
}

func (c *Controller) run() {
	idle := NewBroadcastIdlePacket(c.driver)
	stop := NewBroadcastStopPacket(c.driver, Forward, false, true)
	for {
		select {
		case <-c.shutdownCh:
			for i := 0; i < CommandRepeat; i++ {
				stop.Send()
			}
			c.driver.TracksOff()
			c.doneCh <- true
			return
		case p := <-c.commandCh:
			for i := 0; i < CommandRepeat; i++ {
				p.Send()
			}
		default:
			if len(c.locomotives) == 0 {
				c.commandCh <- idle
				break
			}
			c.mux.Lock()
			for _, loco := range c.locomotives {
				for i := 0; i < CommandRepeat; i++ {
					loco.sendPackets(c.driver)
				}
			}
			c.mux.Unlock()
			idle.PacketPause()
		}
	}
}
<commit_msg>Fix race due to bad locking<commit_after>package dcc

import "sync"

// CommandRepeat specifies how many times a single
// packet is sent.
var CommandRepeat = 30

// CommandMaxQueue specifies how many commands can
// queue before sending a new command blocks
// the sender
var CommandMaxQueue = 3

// Controller represents a DCC Control Station. The
// controller keeps tracks of the DCC Locomotives and
// is in charge of sending DCC packets continuously to
// the tracks.
type Controller struct {
	locomotives map[string]*Locomotive
	mux         sync.RWMutex
	driver      Driver

	started    bool
	doneCh     chan bool
	shutdownCh chan bool
	commandCh  chan *Packet
}

// NewController builds a Controller.
func NewController(d Driver) *Controller {
	d.TracksOff()
	return &Controller{
		driver:      d,
		locomotives: make(map[string]*Locomotive),
		doneCh:      make(chan bool),
		shutdownCh:  make(chan bool),
		commandCh:   make(chan *Packet, CommandMaxQueue),
	}
}

// NewControllerWithConfig builds a new Controller using the
// given configuration.
func NewControllerWithConfig(d Driver, cfg *Config) *Controller {
	c := NewController(d)

	for _, loco := range cfg.Locomotives {
		c.AddLoco(&Locomotive{
			Name:      loco.Name,
			Address:   loco.Address,
			Speed:     loco.Speed,
			Direction: loco.Direction,
			Fl:        loco.Fl})
	}
	return c
}

// AddLoco adds a DCC device to the controller. The device
// will start receiving packets if the controller is running.
func (c *Controller) AddLoco(l *Locomotive) {
	c.mux.Lock()
	defer c.mux.Unlock()
	c.locomotives[l.Name] = l
}

// RmLoco removes a DCC device from the controller. There
// will be no longer packets sent to it.
func (c *Controller) RmLoco(l *Locomotive) {
	c.mux.Lock()
	defer c.mux.Unlock()
	delete(c.locomotives, l.Name)
}

// GetLoco retrieves a DCC device by its Name. The boolean is
// true if the Locomotive was found.
func (c *Controller) GetLoco(n string) (*Locomotive, bool) {
	c.mux.RLock()
	defer c.mux.RUnlock()
	l, ok := c.locomotives[n]
	return l, ok
}

// Locos returns a list of all registered Locomotives.
func (c *Controller) Locos() []*Locomotive {
	c.mux.RLock()
	defer c.mux.RUnlock()
	locos := make([]*Locomotive, 0, len(c.locomotives))
	for _, l := range c.locomotives {
		locos = append(locos, l)
	}
	return locos
}

// Command allows to send a custom Packet to the tracks.
// The packet will be sent CommandRepeat times.
func (c *Controller) Command(p *Packet) {
	c.commandCh <- p
}

// Start starts the controller: powers on the tracks
// and starts sending packets on them.
func (c *Controller) Start() {
	c.driver.TracksOn()
	go c.run()
	c.started = true
}

// Stop shuts down the controller by stopping to send
// packets and removing power from the tracks.
func (c *Controller) Stop() {
	if c.started {
		c.shutdownCh <- true
		<-c.doneCh
		c.started = false
	}
}

func (c *Controller) run() {
	idle := NewBroadcastIdlePacket(c.driver)
	stop := NewBroadcastStopPacket(c.driver, Forward, false, true)
	for {
		select {
		case <-c.shutdownCh:
			for i := 0; i < CommandRepeat; i++ {
				stop.Send()
			}
			c.driver.TracksOff()
			c.doneCh <- true
			return
		case p := <-c.commandCh:
			for i := 0; i < CommandRepeat; i++ {
				p.Send()
			}
		default:
			c.mux.RLock()
			if len(c.locomotives) == 0 {
				c.commandCh <- idle
				c.mux.RUnlock()
				break
			}
			for _, loco := range c.locomotives {
				for i := 0; i < CommandRepeat; i++ {
					loco.sendPackets(c.driver)
				}
			}
			c.mux.RUnlock()
			idle.PacketPause()
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"time"
)

func dgucoproducer(c chan int, max int) {
	for i := 0; i < max; i++ {
		c <- i
	}
	//close(c)
}

func dgucoconsumer(c chan int) {
	ok := true
	value := 0
	for ok {
		fmt.Println("Wait receive")
		if value, ok = <-c; ok {
			fmt.Println(value)
		}
		if ok == false {
			fmt.Println("*******Break********")
		}
	}
}

func main() {
	c := make(chan int)
	defer close(c)
	go dgucoproducer(c, 10)
	go dgucoconsumer(c)
	time.Sleep(time.Second * 5)
	fmt.Println("Done")
}
<commit_msg>Goroutine control method with wg or only channel<commit_after>package main

import (
	"fmt"
	"runtime"
	"sync"
	"time"
)

func main() {
	threadNum := 4
	length := 200
	start := time.Now()
	chChan(threadNum, length)
	sp1 := time.Now()
	fmt.Println("type 1 duration", sp1.Sub(start))
	wgChan(threadNum, length)
	sp2 := time.Now()
	fmt.Println("type 2 duration", sp2.Sub(sp1))
}

func chChan(threadNum int, length int) {
	var maxG int
	done := make(chan bool)
	defer close(done)
	for n := 0; n < length; {
		var i int
		for i = 0; i < threadNum; i++ {
			n++
			if n >= length {
				break
			}
			go func() {
				done <- true
			}()

		}

		// wait until both are done
		for c := 0; c < i; c++ {
			if c == 0 {
				if runtime.NumGoroutine() > maxG {
					maxG = runtime.NumGoroutine()
				}
			}
			<-done
		}
	}
	fmt.Println("max goroutine is", maxG)
}
func wgChan(threadNum int, length int) {
	var maxG int
	threadKey := make(chan int, threadNum)
	var wg sync.WaitGroup
	for n := 0; n < length; n++ {
		wg.Add(1)
		threadKey <- n
		go func(i int) {
			defer func() {
				wg.Done()
				<-threadKey
			}()
			if runtime.NumGoroutine() > maxG {
				maxG = runtime.NumGoroutine()
			}
			//fmt.Println(time.Now(), "Goruntine number", runtime.NumGoroutine())
		}(n)
	}
	wg.Wait()
	close(threadKey)
	fmt.Println("max goroutine is", maxG)
}
<|endoftext|>


<commit_before>package format

import (
	"bytes"
	"fmt"
	"log"
	"sort"
	"strings"

	"github.com/mitchellh/colorstring"

	"github.com/hashicorp/terraform/addrs"
	"github.com/hashicorp/terraform/plans"
	"github.com/hashicorp/terraform/states"
	"github.com/hashicorp/terraform/terraform"
)

// Plan is a representation of a plan optimized for display to
// an end-user, as opposed to terraform.Plan which is for internal use.
//
// DisplayPlan excludes implementation details that may otherwise appear
// in the main plan, such as destroy actions on data sources (which are
// there only to clean up the state).
type Plan struct {
	Resources []*InstanceDiff
}

// InstanceDiff is a representation of an instance diff optimized
// for display, in conjunction with DisplayPlan.
type InstanceDiff struct {
	Addr   *terraform.ResourceAddress
	Action terraform.DiffChangeType

	// Attributes describes changes to the attributes of the instance.
	//
	// For destroy diffs this is always nil.
	Attributes []*AttributeDiff

	Tainted bool
	Deposed bool
}

// AttributeDiff is a representation of an attribute diff optimized
// for display, in conjunction with DisplayInstanceDiff.
type AttributeDiff struct {
	// Path is a dot-delimited traversal through possibly many levels of list and map structure,
	// intended for display purposes only.
	Path string

	Action terraform.DiffChangeType

	OldValue string
	NewValue string

	NewComputed bool
	Sensitive   bool
	ForcesNew   bool
}

// PlanStats gives summary counts for a Plan.
type PlanStats struct {
	ToAdd, ToChange, ToDestroy int
}

// NewPlan produces a display-oriented Plan from a terraform.Plan.
func NewPlan(changes *plans.Changes) *Plan {
	log.Printf("[TRACE] NewPlan for %#v", changes)
	ret := &Plan{}
	if changes == nil {
		// Nothing to do!
		return ret
	}

	for _, rc := range changes.Resources {
		addr := rc.Addr
		log.Printf("[TRACE] NewPlan found %s", addr)
		dataSource := addr.Resource.Resource.Mode == addrs.DataResourceMode

		// We create "delete" actions for data resources so we can clean
		// up their entries in state, but this is an implementation detail
		// that users shouldn't see.
		if dataSource && rc.Action == plans.Delete {
			continue
		}

		// For now we'll shim this to work with our old types.
		// TODO: Update for the new plan types, ideally also switching over to
		// a structural diff renderer instead of a flat renderer.
		did := &InstanceDiff{
			Addr: terraform.NewLegacyResourceInstanceAddress(addr),
		}

		switch rc.Action {
		case plans.NoOp:
			continue
		case plans.Create:
			if dataSource {
				// Use "refresh" as the action for display, but core
				// currently uses Create for this internally.
				// FIXME: Update core to generate plans.Read for this case
				// instead.
				did.Action = terraform.DiffRefresh
			} else {
				did.Action = terraform.DiffCreate
			}
		case plans.Read:
			did.Action = terraform.DiffRefresh
		case plans.Delete:
			did.Action = terraform.DiffDestroy
		case plans.DeleteThenCreate, plans.CreateThenDelete:
			did.Action = terraform.DiffDestroyCreate
		case plans.Update:
			did.Action = terraform.DiffUpdate
		default:
			panic(fmt.Sprintf("unexpected change action %s", rc.Action))
		}

		if rc.DeposedKey != states.NotDeposed {
			did.Deposed = true
		}

		// Since this is just a temporary stub implementation on the way
		// to us replacing this with the structural diff renderer, we currently
		// don't include any attributes here.
		// FIXME: Implement the structural diff renderer to replace this
		// codepath altogether.

		ret.Resources = append(ret.Resources, did)
	}

	// Sort the instance diffs by their addresses for display.
	sort.Slice(ret.Resources, func(i, j int) bool {
		iAddr := ret.Resources[i].Addr
		jAddr := ret.Resources[j].Addr
		return iAddr.Less(jAddr)
	})

	return ret
}

// Format produces and returns a text representation of the receiving plan
// intended for display in a terminal.
//
// If color is not nil, it is used to colorize the output.
func (p *Plan) Format(color *colorstring.Colorize) string {
	if p.Empty() {
		return "This plan does nothing."
	}

	if color == nil {
		color = &colorstring.Colorize{
			Colors: colorstring.DefaultColors,
			Reset:  false,
		}
	}

	// Find the longest path length of all the paths that are changing,
	// so we can align them all.
	keyLen := 0
	for _, r := range p.Resources {
		for _, attr := range r.Attributes {
			key := attr.Path

			if len(key) > keyLen {
				keyLen = len(key)
			}
		}
	}

	buf := new(bytes.Buffer)
	for _, r := range p.Resources {
		formatPlanInstanceDiff(buf, r, keyLen, color)
	}

	return strings.TrimSpace(buf.String())
}

// Stats returns statistics about the plan
func (p *Plan) Stats() PlanStats {
	var ret PlanStats
	for _, r := range p.Resources {
		switch r.Action {
		case terraform.DiffCreate:
			ret.ToAdd++
		case terraform.DiffUpdate:
			ret.ToChange++
		case terraform.DiffDestroyCreate:
			ret.ToAdd++
			ret.ToDestroy++
		case terraform.DiffDestroy:
			ret.ToDestroy++
		}
	}
	return ret
}

// ActionCounts returns the number of diffs for each action type
func (p *Plan) ActionCounts() map[terraform.DiffChangeType]int {
	ret := map[terraform.DiffChangeType]int{}
	for _, r := range p.Resources {
		ret[r.Action]++
	}
	return ret
}

// Empty returns true if there is at least one resource diff in the receiving plan.
func (p *Plan) Empty() bool {
	return len(p.Resources) == 0
}

// DiffActionSymbol returns a string that, once passed through a
// colorstring.Colorize, will produce a result that can be written
// to a terminal to produce a symbol made of three printable
// characters, possibly interspersed with VT100 color codes.
func DiffActionSymbol(action terraform.DiffChangeType) string {
	switch action {
	case terraform.DiffDestroyCreate:
		return "[red]-[reset]/[green]+[reset]"
	case terraform.DiffCreate:
		return "  [green]+[reset]"
	case terraform.DiffDestroy:
		return "  [red]-[reset]"
	case terraform.DiffRefresh:
		return " [cyan]<=[reset]"
	default:
		return "  [yellow]~[reset]"
	}
}

// formatPlanInstanceDiff writes the text representation of the given instance diff
// to the given buffer, using the given colorizer.
func formatPlanInstanceDiff(buf *bytes.Buffer, r *InstanceDiff, keyLen int, colorizer *colorstring.Colorize) {
	addrStr := r.Addr.String()

	// Determine the color for the text (green for adding, yellow
	// for change, red for delete), and symbol, and output the
	// resource header.
	color := "yellow"
	symbol := DiffActionSymbol(r.Action)
	oldValues := true
	switch r.Action {
	case terraform.DiffDestroyCreate:
		color = "yellow"
	case terraform.DiffCreate:
		color = "green"
		oldValues = false
	case terraform.DiffDestroy:
		color = "red"
	case terraform.DiffRefresh:
		color = "cyan"
		oldValues = false
	}

	var extraStr string
	if r.Tainted {
		extraStr = extraStr + " (tainted)"
	}
	if r.Deposed {
		extraStr = extraStr + " (deposed)"
	}
	if r.Action == terraform.DiffDestroyCreate {
		extraStr = extraStr + colorizer.Color(" [red][bold](new resource required)")
	}

	buf.WriteString(
		colorizer.Color(fmt.Sprintf(
			"[%s]%s [%s]%s%s\n",
			color, symbol, color, addrStr, extraStr,
		)),
	)

	for _, attr := range r.Attributes {

		v := attr.NewValue
		var dispV string
		switch {
		case v == "" && attr.NewComputed:
			dispV = "<computed>"
		case attr.Sensitive:
			dispV = "<sensitive>"
		default:
			dispV = fmt.Sprintf("%q", v)
		}

		updateMsg := ""
		switch {
		case attr.ForcesNew && r.Action == terraform.DiffDestroyCreate:
			updateMsg = colorizer.Color(" [red](forces new resource)")
		case attr.Sensitive && oldValues:
			updateMsg = colorizer.Color(" [yellow](attribute changed)")
		}

		if oldValues {
			u := attr.OldValue
			var dispU string
			switch {
			case attr.Sensitive:
				dispU = "<sensitive>"
			default:
				dispU = fmt.Sprintf("%q", u)
			}
			buf.WriteString(fmt.Sprintf(
				"      %s:%s %s => %s%s\n",
				attr.Path,
				strings.Repeat(" ", keyLen-len(attr.Path)),
				dispU, dispV,
				updateMsg,
			))
		} else {
			buf.WriteString(fmt.Sprintf(
				"      %s:%s %s%s\n",
				attr.Path,
				strings.Repeat(" ", keyLen-len(attr.Path)),
				dispV,
				updateMsg,
			))
		}
	}

	// Write the reset color so we don't bleed color into later text
	buf.WriteString(colorizer.Color("[reset]\n"))
}
<commit_msg>backend/local: Render CBD replacement (+/-) correctly (#19642)<commit_after>package format

import (
	"bytes"
	"fmt"
	"log"
	"sort"
	"strings"

	"github.com/mitchellh/colorstring"

	"github.com/hashicorp/terraform/addrs"
	"github.com/hashicorp/terraform/plans"
	"github.com/hashicorp/terraform/states"
	"github.com/hashicorp/terraform/terraform"
)

// Plan is a representation of a plan optimized for display to
// an end-user, as opposed to terraform.Plan which is for internal use.
//
// DisplayPlan excludes implementation details that may otherwise appear
// in the main plan, such as destroy actions on data sources (which are
// there only to clean up the state).
type Plan struct {
	Resources []*InstanceDiff
}

// InstanceDiff is a representation of an instance diff optimized
// for display, in conjunction with DisplayPlan.
type InstanceDiff struct {
	Addr   *terraform.ResourceAddress
	Action plans.Action

	// Attributes describes changes to the attributes of the instance.
	//
	// For destroy diffs this is always nil.
	Attributes []*AttributeDiff

	Tainted bool
	Deposed bool
}

// AttributeDiff is a representation of an attribute diff optimized
// for display, in conjunction with DisplayInstanceDiff.
type AttributeDiff struct {
	// Path is a dot-delimited traversal through possibly many levels of list and map structure,
	// intended for display purposes only.
	Path string

	Action plans.Action

	OldValue string
	NewValue string

	NewComputed bool
	Sensitive   bool
	ForcesNew   bool
}

// PlanStats gives summary counts for a Plan.
type PlanStats struct {
	ToAdd, ToChange, ToDestroy int
}

// NewPlan produces a display-oriented Plan from a terraform.Plan.
func NewPlan(changes *plans.Changes) *Plan {
	log.Printf("[TRACE] NewPlan for %#v", changes)
	ret := &Plan{}
	if changes == nil {
		// Nothing to do!
		return ret
	}

	for _, rc := range changes.Resources {
		addr := rc.Addr
		log.Printf("[TRACE] NewPlan found %s (%s)", addr, rc.Action)
		dataSource := addr.Resource.Resource.Mode == addrs.DataResourceMode

		// We create "delete" actions for data resources so we can clean
		// up their entries in state, but this is an implementation detail
		// that users shouldn't see.
		if dataSource && rc.Action == plans.Delete {
			continue
		}

		// For now we'll shim this to work with our old types.
		// TODO: Update for the new plan types, ideally also switching over to
		// a structural diff renderer instead of a flat renderer.
		did := &InstanceDiff{
			Addr:   terraform.NewLegacyResourceInstanceAddress(addr),
			Action: rc.Action,
		}

		if rc.DeposedKey != states.NotDeposed {
			did.Deposed = true
		}

		// Since this is just a temporary stub implementation on the way
		// to us replacing this with the structural diff renderer, we currently
		// don't include any attributes here.
		// FIXME: Implement the structural diff renderer to replace this
		// codepath altogether.

		ret.Resources = append(ret.Resources, did)
	}

	// Sort the instance diffs by their addresses for display.
	sort.Slice(ret.Resources, func(i, j int) bool {
		iAddr := ret.Resources[i].Addr
		jAddr := ret.Resources[j].Addr
		return iAddr.Less(jAddr)
	})

	return ret
}

// Format produces and returns a text representation of the receiving plan
// intended for display in a terminal.
//
// If color is not nil, it is used to colorize the output.
func (p *Plan) Format(color *colorstring.Colorize) string {
	if p.Empty() {
		return "This plan does nothing."
	}

	if color == nil {
		color = &colorstring.Colorize{
			Colors: colorstring.DefaultColors,
			Reset:  false,
		}
	}

	// Find the longest path length of all the paths that are changing,
	// so we can align them all.
	keyLen := 0
	for _, r := range p.Resources {
		for _, attr := range r.Attributes {
			key := attr.Path

			if len(key) > keyLen {
				keyLen = len(key)
			}
		}
	}

	buf := new(bytes.Buffer)
	for _, r := range p.Resources {
		formatPlanInstanceDiff(buf, r, keyLen, color)
	}

	return strings.TrimSpace(buf.String())
}

// Stats returns statistics about the plan
func (p *Plan) Stats() PlanStats {
	var ret PlanStats
	for _, r := range p.Resources {
		switch r.Action {
		case plans.Create:
			ret.ToAdd++
		case plans.Update:
			ret.ToChange++
		case plans.DeleteThenCreate, plans.CreateThenDelete:
			ret.ToAdd++
			ret.ToDestroy++
		case plans.Delete:
			ret.ToDestroy++
		}
	}
	return ret
}

// ActionCounts returns the number of diffs for each action type
func (p *Plan) ActionCounts() map[plans.Action]int {
	ret := map[plans.Action]int{}
	for _, r := range p.Resources {
		ret[r.Action]++
	}
	return ret
}

// Empty returns true if there is at least one resource diff in the receiving plan.
func (p *Plan) Empty() bool {
	return len(p.Resources) == 0
}

// DiffActionSymbol returns a string that, once passed through a
// colorstring.Colorize, will produce a result that can be written
// to a terminal to produce a symbol made of three printable
// characters, possibly interspersed with VT100 color codes.
func DiffActionSymbol(action plans.Action) string {
	switch action {
	case plans.DeleteThenCreate:
		return "[red]-[reset]/[green]+[reset]"
	case plans.CreateThenDelete:
		return "[green]+[reset]/[red]-[reset]"
	case plans.Create:
		return "  [green]+[reset]"
	case plans.Delete:
		return "  [red]-[reset]"
	case plans.Read:
		return " [cyan]<=[reset]"
	case plans.Update:
		return "  [yellow]~[reset]"
	default:
		return "  ?"
	}
}

// formatPlanInstanceDiff writes the text representation of the given instance diff
// to the given buffer, using the given colorizer.
func formatPlanInstanceDiff(buf *bytes.Buffer, r *InstanceDiff, keyLen int, colorizer *colorstring.Colorize) {
	addrStr := r.Addr.String()

	// Determine the color for the text (green for adding, yellow
	// for change, red for delete), and symbol, and output the
	// resource header.
	color := "yellow"
	symbol := DiffActionSymbol(r.Action)
	oldValues := true
	switch r.Action {
	case plans.DeleteThenCreate, plans.CreateThenDelete:
		color = "yellow"
	case plans.Create:
		color = "green"
		oldValues = false
	case plans.Delete:
		color = "red"
	case plans.Read:
		color = "cyan"
		oldValues = false
	}

	var extraStr string
	if r.Tainted {
		extraStr = extraStr + " (tainted)"
	}
	if r.Deposed {
		extraStr = extraStr + " (deposed)"
	}
	if r.Action.IsReplace() {
		extraStr = extraStr + colorizer.Color(" [red][bold](new resource required)")
	}

	buf.WriteString(
		colorizer.Color(fmt.Sprintf(
			"[%s]%s [%s]%s%s\n",
			color, symbol, color, addrStr, extraStr,
		)),
	)

	for _, attr := range r.Attributes {

		v := attr.NewValue
		var dispV string
		switch {
		case v == "" && attr.NewComputed:
			dispV = "<computed>"
		case attr.Sensitive:
			dispV = "<sensitive>"
		default:
			dispV = fmt.Sprintf("%q", v)
		}

		updateMsg := ""
		switch {
		case attr.ForcesNew && r.Action.IsReplace():
			updateMsg = colorizer.Color(" [red](forces new resource)")
		case attr.Sensitive && oldValues:
			updateMsg = colorizer.Color(" [yellow](attribute changed)")
		}

		if oldValues {
			u := attr.OldValue
			var dispU string
			switch {
			case attr.Sensitive:
				dispU = "<sensitive>"
			default:
				dispU = fmt.Sprintf("%q", u)
			}
			buf.WriteString(fmt.Sprintf(
				"      %s:%s %s => %s%s\n",
				attr.Path,
				strings.Repeat(" ", keyLen-len(attr.Path)),
				dispU, dispV,
				updateMsg,
			))
		} else {
			buf.WriteString(fmt.Sprintf(
				"      %s:%s %s%s\n",
				attr.Path,
				strings.Repeat(" ", keyLen-len(attr.Path)),
				dispV,
				updateMsg,
			))
		}
	}

	// Write the reset color so we don't bleed color into later text
	buf.WriteString(colorizer.Color("[reset]\n"))
}
<|endoftext|>


<commit_before>package beanstalk

import "time"

// JobMethod describes a type of beanstalk job finalizer.
type JobMethod int

// These are the caller ids that are used when calling back to the Consumer.
const (
	BuryJob JobMethod = iota
	DeleteJob
	ReleaseJob
)

// JobFinisher defines an interface which a Job can call to finish up.
type JobFinisher interface {
	FinishJob(*Job, JobMethod, uint32, time.Duration) error
}

// Job contains the data of a reserved job.
type Job struct {
	ID     uint64
	Body   []byte
	TTR    time.Duration
	Finish JobFinisher
}

func (job *Job) finishJob(method JobMethod, priority uint32, delay time.Duration) error {
	ret := job.Finish.FinishJob(job, method, priority, delay)
	job.Finish = nil
	return ret
}

// Bury tells the consumer to bury this job.
func (job *Job) Bury(priority uint32) error {
	return job.finishJob(BuryJob, priority, 0)
}

// Delete tells the consumer to delete this job.
func (job *Job) Delete() error {
	return job.finishJob(DeleteJob, 0, 0)
}

// Release tells the consumer to release this job.
func (job *Job) Release(priority uint32, delay time.Duration) error {
	return job.finishJob(ReleaseJob, priority, delay)
}
<commit_msg>Return an ErrJobFinished error when a Job's finishers are called more than once, instead of a runtime error<commit_after>package beanstalk

import (
	"errors"
	"time"
)

var ErrJobFinished = errors.New("Job was already finished")

// JobMethod describes a type of beanstalk job finalizer.
type JobMethod int

// These are the caller ids that are used when calling back to the Consumer.
const (
	BuryJob JobMethod = iota
	DeleteJob
	ReleaseJob
)

// JobFinisher defines an interface which a Job can call to finish up.
type JobFinisher interface {
	FinishJob(*Job, JobMethod, uint32, time.Duration) error
}

// Job contains the data of a reserved job.
type Job struct {
	ID     uint64
	Body   []byte
	TTR    time.Duration
	Finish JobFinisher
}

func (job *Job) finishJob(method JobMethod, priority uint32, delay time.Duration) error {
	if job.Finish == nil {
		return ErrJobFinished
	}

	ret := job.Finish.FinishJob(job, method, priority, delay)
	job.Finish = nil

	return ret
}

// Bury tells the consumer to bury this job.
func (job *Job) Bury(priority uint32) error {
	return job.finishJob(BuryJob, priority, 0)
}

// Delete tells the consumer to delete this job.
func (job *Job) Delete() error {
	return job.finishJob(DeleteJob, 0, 0)
}

// Release tells the consumer to release this job.
func (job *Job) Release(priority uint32, delay time.Duration) error {
	return job.finishJob(ReleaseJob, priority, delay)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"strings"
	"unicode"
	"unicode/utf8"

	log "github.com/Sirupsen/logrus"
)

const (
	/* TODO: Maybe remove [] and () from this and handle them differently
	since it may break things that shouldn't be translated */
	slashCharacters = "0123456789[]{}()\\/<>abcdefghijklmnopqrstuvxzwyABCDEFGHIJKLMNOPQRSTUVXZWY!|$^."
	rawCharacters   = "\u3000（）・！？。…【】「」『』\n()/\"[]<>：:#*＊_"
)

// next returns the next rune in the input.
func (l *lexer) next() rune {
	if l.pos >= len(l.input) {
		l.width = 0
		return eof
	}

	r, w := utf8.DecodeRuneInString(l.input[l.pos:])

	l.mark = r
	l.width = w

	l.pos += l.width

	return r
}

func (l *lexer) nextItem() item {
	item := <-l.items

	l.lastPos = item.pos

	return item
}

func (l *lexer) backup(pos int) {
	for i := 0; i < pos; i++ {
		l.pos -= l.width
		if l.pos < 0 {
			l.pos = 0
		}

		r, w := utf8.DecodeRuneInString(l.input[l.pos:])
		l.mark = r
		l.width = w
	}
}

func (l *lexer) peek(locs int) rune {
	pos := saveLexerPosition(l)

	var r rune

	x := 0

	for x < locs {
		l.next()

		if x == locs-1 {
			r = l.mark
		}

		x++
	}

	pos.restore(l)

	return r
}

// emit passes an item back to the client.
func (l *lexer) emit(t itemType) {
	l.items <- item{t, l.start, l.input[l.start:l.pos]}

	l.start = l.pos
}

func (l *lexer) emitBefore(t itemType) {
	l.backup(1)

	if l.pos > l.start {
		l.emit(t)

		l.ignore()
	}
}

// ignore skips over the pending input before this point.
func (l *lexer) ignore() {
	l.start = l.pos
}

// accept consumes the next rune if it's from the valid set.
func (l *lexer) accept(valid string) bool {
	if strings.ContainsRune(valid, l.next()) {
		return true
	}

	l.backup(1)

	return false
}

// acceptRun consumes a run of runes from the valid set.
func (l *lexer) acceptRun(valid string) {
	count := 0

	for strings.ContainsRune(valid, l.next()) {
		count++
	}

	l.backup(1)
}

// errorf returns an error token and terminates the scan by passing
// back a nil pointer that will be the next state, terminating l.nextItem.
func (l *lexer) errorf(format string, args ...interface{}) stateFn {
	l.items <- item{itemError, l.start, fmt.Sprintf(format, args...)}

	return nil
}

// lex creates a new scanner for the input string.
func lex(input string) *lexer {
	l := &lexer{
		input: input,
		items: make(chan item),
	}

	go l.run()

	return l
}

// run runs the state machine for the lexer.
func (l *lexer) run() {
	for l.state = lexText; l.state != nil; {
		l.state = l.state(l)
	}

	close(l.items)
}

func lexText(l *lexer) stateFn {
	log.Debugf("lexText %q", l.input[l.pos:])

	l.width = 0

Loop:
	for {
		//switch l.next() {
		switch r := l.next(); {
		case r == eof:
			break Loop
		case r == '%' && l.peek(1) == 's':
			l.emitBefore(itemText)

			l.next()

			if l.accept("s") {
				l.emit(itemRawString)
			}
		case (r == 'i' && strings.HasPrefix(l.input[l.pos:], "f(")) ||
			(r == 'e' && strings.HasPrefix(l.input[l.pos:], "n(")):
			l.emitBefore(itemText)

			return lexScript
		case r == '\\' || r == '@':
			l.emitBefore(itemText)

			return lexScript

		case strings.ContainsRune(rawCharacters, r) || unicode.IsSymbol(r):
			l.emitBefore(itemText)

			l.next()

			l.emit(itemRawString)

			return lexText
		}
	}

	if l.pos > l.start {
		l.emit(itemText)

		l.ignore()
	}

	l.emit(itemEOF)

	return nil

}

func lexScript(l *lexer) stateFn {
	log.Debugf("lexScript %q", l.input[l.pos:])

Loop:
	for {
		switch l.next() {
		case eof:
			log.Warn("Script not terminated properly %q", l.input[l.start:])
			break Loop
		case '(':
			l.emitBefore(itemScript)

			l.next()

			l.emit(itemLeftParen)

			l.parenDepth++

			return lexInsideAction
		case '[':
			l.backup(1)
			l.emit(itemScript)

			return lexLeftDelim
		case '\\':
			l.acceptRun(slashCharacters)

			break Loop
		case '\n':
			l.acceptRun("[0123456789]")

			break Loop
		case '@':
			l.acceptRun("0123456789-")

			break Loop

		default:
			log.Debug(string(l.mark))
		}
	}

	l.emit(itemScript)

	return lexText
}

func lexLeftDelim(l *lexer) stateFn {
	l.next()

	log.Debug("leftDelim: ", string(l.mark))

	l.emit(itemLeftDelim)

	return lexInsideAction
}

func lexRightDelim(l *lexer) stateFn {
	l.next()

	log.Debug("rightDelim: ", string(l.mark))

	l.emit(itemRightDelim)

	log.Debug("Paren depth: ", l.parenDepth)

	if l.parenDepth == 0 {
		return lexText
	}

	return lexInsideAction
}

// lexInsideAction scans the elements inside action delimiters.
func lexInsideAction(l *lexer) stateFn {
	log.Debugf("lexInsideAction %q", l.input[l.pos:])

	switch r := l.next(); {
	case r == eof:
		return l.errorf("unclosed action")
	case r == '(':
		l.emitBefore(itemParameter)

		l.next()

		l.emit(itemLeftParen)

		l.parenDepth++
	case r == ')':
		l.emitBefore(itemParameter)

		l.next()

		l.emit(itemRightParen)

		l.parenDepth--

		if l.parenDepth < 0 {
			return l.errorf("unexpected right paren %#U", r)
		}

		if l.parenDepth == 0 {
			return lexText
		}
	case r == '[':
		l.emitBefore(itemParameter)

		return lexLeftDelim
	case r == ']':
		l.emitBefore(itemParameter)

		return lexRightDelim
	case r == '%' && l.accept("s"):
		l.emit(itemParameter)
	case r == '"':
		l.emit(itemParameter)

		return lexText
	}

	return lexInsideAction
}
<commit_msg>Add <> to raw characters<commit_after>package main

import (
	"fmt"
	"strings"
	"unicode"
	"unicode/utf8"

	log "github.com/Sirupsen/logrus"
)

const (
	/* TODO: Maybe remove [] and () from this and handle them differently
	since it may break things that shouldn't be translated */
	slashCharacters = "0123456789[]{}()\\/<>abcdefghijklmnopqrstuvxzwyABCDEFGHIJKLMNOPQRSTUVXZWY!|$^."
	rawCharacters   = "\u3000（）・！？。…【】「」『』\n()/\"[]<>〈〉：:#*＊_"
)

// next returns the next rune in the input.
func (l *lexer) next() rune {
	if l.pos >= len(l.input) {
		l.width = 0
		return eof
	}

	r, w := utf8.DecodeRuneInString(l.input[l.pos:])

	l.mark = r
	l.width = w

	l.pos += l.width

	return r
}

func (l *lexer) nextItem() item {
	item := <-l.items

	l.lastPos = item.pos

	return item
}

func (l *lexer) backup(pos int) {
	for i := 0; i < pos; i++ {
		l.pos -= l.width
		if l.pos < 0 {
			l.pos = 0
		}

		r, w := utf8.DecodeRuneInString(l.input[l.pos:])
		l.mark = r
		l.width = w
	}
}

func (l *lexer) peek(locs int) rune {
	pos := saveLexerPosition(l)

	var r rune

	x := 0

	for x < locs {
		l.next()

		if x == locs-1 {
			r = l.mark
		}

		x++
	}

	pos.restore(l)

	return r
}

// emit passes an item back to the client.
func (l *lexer) emit(t itemType) {
	l.items <- item{t, l.start, l.input[l.start:l.pos]}

	l.start = l.pos
}

func (l *lexer) emitBefore(t itemType) {
	l.backup(1)

	if l.pos > l.start {
		l.emit(t)

		l.ignore()
	}
}

// ignore skips over the pending input before this point.
func (l *lexer) ignore() {
	l.start = l.pos
}

// accept consumes the next rune if it's from the valid set.
func (l *lexer) accept(valid string) bool {
	if strings.ContainsRune(valid, l.next()) {
		return true
	}

	l.backup(1)

	return false
}

// acceptRun consumes a run of runes from the valid set.
func (l *lexer) acceptRun(valid string) {
	count := 0

	for strings.ContainsRune(valid, l.next()) {
		count++
	}

	l.backup(1)
}

// errorf returns an error token and terminates the scan by passing
// back a nil pointer that will be the next state, terminating l.nextItem.
func (l *lexer) errorf(format string, args ...interface{}) stateFn {
	l.items <- item{itemError, l.start, fmt.Sprintf(format, args...)}

	return nil
}

// lex creates a new scanner for the input string.
func lex(input string) *lexer {
	l := &lexer{
		input: input,
		items: make(chan item),
	}

	go l.run()

	return l
}

// run runs the state machine for the lexer.
func (l *lexer) run() {
	for l.state = lexText; l.state != nil; {
		l.state = l.state(l)
	}

	close(l.items)
}

func lexText(l *lexer) stateFn {
	log.Debugf("lexText %q", l.input[l.pos:])

	l.width = 0

Loop:
	for {
		//switch l.next() {
		switch r := l.next(); {
		case r == eof:
			break Loop
		case r == '%' && l.peek(1) == 's':
			l.emitBefore(itemText)

			l.next()

			if l.accept("s") {
				l.emit(itemRawString)
			}
		case (r == 'i' && strings.HasPrefix(l.input[l.pos:], "f(")) ||
			(r == 'e' && strings.HasPrefix(l.input[l.pos:], "n(")):
			l.emitBefore(itemText)

			return lexScript
		case r == '\\' || r == '@':
			l.emitBefore(itemText)

			return lexScript

		case strings.ContainsRune(rawCharacters, r) || unicode.IsSymbol(r):
			l.emitBefore(itemText)

			l.next()

			l.emit(itemRawString)

			return lexText
		}
	}

	if l.pos > l.start {
		l.emit(itemText)

		l.ignore()
	}

	l.emit(itemEOF)

	return nil

}

func lexScript(l *lexer) stateFn {
	log.Debugf("lexScript %q", l.input[l.pos:])

Loop:
	for {
		switch l.next() {
		case eof:
			log.Warn("Script not terminated properly %q", l.input[l.start:])
			break Loop
		case '(':
			l.emitBefore(itemScript)

			l.next()

			l.emit(itemLeftParen)

			l.parenDepth++

			return lexInsideAction
		case '[':
			l.backup(1)
			l.emit(itemScript)

			return lexLeftDelim
		case '\\':
			l.acceptRun(slashCharacters)

			break Loop
		case '\n':
			l.acceptRun("[0123456789]")

			break Loop
		case '@':
			l.acceptRun("0123456789-")

			break Loop

		default:
			log.Debug(string(l.mark))
		}
	}

	l.emit(itemScript)

	return lexText
}

func lexLeftDelim(l *lexer) stateFn {
	l.next()

	log.Debug("leftDelim: ", string(l.mark))

	l.emit(itemLeftDelim)

	return lexInsideAction
}

func lexRightDelim(l *lexer) stateFn {
	l.next()

	log.Debug("rightDelim: ", string(l.mark))

	l.emit(itemRightDelim)

	log.Debug("Paren depth: ", l.parenDepth)

	if l.parenDepth == 0 {
		return lexText
	}

	return lexInsideAction
}

// lexInsideAction scans the elements inside action delimiters.
func lexInsideAction(l *lexer) stateFn {
	log.Debugf("lexInsideAction %q", l.input[l.pos:])

	switch r := l.next(); {
	case r == eof:
		return l.errorf("unclosed action")
	case r == '(':
		l.emitBefore(itemParameter)

		l.next()

		l.emit(itemLeftParen)

		l.parenDepth++
	case r == ')':
		l.emitBefore(itemParameter)

		l.next()

		l.emit(itemRightParen)

		l.parenDepth--

		if l.parenDepth < 0 {
			return l.errorf("unexpected right paren %#U", r)
		}

		if l.parenDepth == 0 {
			return lexText
		}
	case r == '[':
		l.emitBefore(itemParameter)

		return lexLeftDelim
	case r == ']':
		l.emitBefore(itemParameter)

		return lexRightDelim
	case r == '%' && l.accept("s"):
		l.emit(itemParameter)
	case r == '"':
		l.emit(itemParameter)

		return lexText
	}

	return lexInsideAction
}
<|endoftext|>


<commit_before>package turnpike

import (
	"errors"
	"fmt"
	glog "log"
	"os"
)

var (
	logFlags = glog.Ldate | glog.Ltime | glog.Lshortfile
	log      Logger
)

// Logger is an interface compatible with log.Logger.
type Logger interface {
	Println(v ...interface{})
	Fatal(v ...interface{})
	Printf(format string, v ...interface{})
}

// setup logger for package, writes to /dev/null by default
func init() {
	if devNull, err := os.Create(os.DevNull); err != nil {
		panic("could not create logger: " + err.Error())
	} else if os.Getenv("DEBUG") != "" {
		log = glog.New(os.Stderr, "", logFlags)
	} else {
		log = glog.New(devNull, "", 0)
	}
}

// change log output to stderr
func Debug() {
	log = glog.New(os.Stderr, "", logFlags)
}

func SetLogger(l Logger) {
	log = l
}

func logErr(v ...interface{}) error {
	err := errors.New(fmt.Sprintln(v...))
	log.Println(err)
	return err
}
<commit_msg>Removing Fatal from the interface<commit_after>package turnpike

import (
	"errors"
	"fmt"
	glog "log"
	"os"
)

var (
	logFlags = glog.Ldate | glog.Ltime | glog.Lshortfile
	log      Logger
)

// Logger is an interface compatible with log.Logger.
type Logger interface {
	Println(v ...interface{})
	Printf(format string, v ...interface{})
}

// setup logger for package, writes to /dev/null by default
func init() {
	if devNull, err := os.Create(os.DevNull); err != nil {
		panic("could not create logger: " + err.Error())
	} else if os.Getenv("DEBUG") != "" {
		log = glog.New(os.Stderr, "", logFlags)
	} else {
		log = glog.New(devNull, "", 0)
	}
}

// change log output to stderr
func Debug() {
	log = glog.New(os.Stderr, "", logFlags)
}

func SetLogger(l Logger) {
	log = l
}

func logErr(v ...interface{}) error {
	err := errors.New(fmt.Sprintln(v...))
	log.Println(err)
	return err
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"time"

	"github.com/litl/galaxy/log"
	"github.com/litl/galaxy/registry"
	"github.com/litl/galaxy/runtime"
	"github.com/litl/galaxy/utils"
)

var (
	stopCutoff      = flag.Int64("cutoff", 10, "Seconds to wait before stopping old containers")
	app             = flag.String("app", "", "App to start")
	redisHost       = flag.String("redis", utils.GetEnv("GALAXY_REDIS_HOST", "127.0.0.1:6379"), "redis host")
	env             = flag.String("env", utils.GetEnv("GALAXY_ENV", "dev"), "Environment namespace")
	pool            = flag.String("pool", utils.GetEnv("GALAXY_POOL", "web"), "Pool namespace")
	loop            = flag.Bool("loop", false, "Run continously")
	shuttleHost     = flag.String("shuttleAddr", "", "IP where containers can reach shuttle proxy. Defaults to docker0 IP.")
	serviceConfigs  []*registry.ServiceConfig
	serviceRegistry *registry.ServiceRegistry
	serviceRuntime  *runtime.ServiceRuntime
)

func initOrDie() {

	serviceRegistry = registry.NewServiceRegistry(
		*env,
		*pool,
		"",
		600,
		"",
	)

	serviceRegistry.Connect(*redisHost)
	serviceRuntime = runtime.NewServiceRuntime(*shuttleHost, *env, *pool, *redisHost)
}

func startContainersIfNecessary() error {
	// FIXME: This should list registered services from the service registry
	serviceConfigs, err := serviceRegistry.ListApps()
	if err != nil {
		log.Printf("ERROR: Could not retrieve service configs for /%s/%s: %s\n", *env, *pool, err)
		return err
	}

	if len(serviceConfigs) == 0 {
		log.Printf("No services configured for /%s/%s\n", *env, *pool)
		return err
	}

	for _, serviceConfig := range serviceConfigs {

		if *app != "" && serviceConfig.Name != *app {
			continue
		}

		if serviceConfig.Version() == "" {
			log.Printf("Skipping %s. No version configured.\n", serviceConfig.Name)
			continue
		}

		container, err := serviceRuntime.StartIfNotRunning(&serviceConfig)
		if err != nil {
			log.Printf("ERROR: Could not determine if %s is running: %s\n",
				serviceConfig.Version(), err)
			return err
		}

		log.Printf("%s version %s running as %s\n", serviceConfig.Name, serviceConfig.Version(), container.ID[0:12])

		serviceRuntime.StopAllButLatest(*stopCutoff)
	}
	return nil
}

func restartContainers(changedConfigs chan *registry.ConfigChange) {
	ticker := time.NewTicker(10 * time.Second)

	for {

		var changedConfig *registry.ConfigChange
		select {

		case changedConfig = <-changedConfigs:
			if changedConfig.Error != nil {
				log.Printf("ERROR: Error watching changes: %s\n", changedConfig.Error)
				continue
			}

			if changedConfig.ServiceConfig == nil {
				continue
			}

			if changedConfig.ServiceConfig.Version() == "" {
				continue
			}

			log.Printf("Restarting %s\n", changedConfig.ServiceConfig.Name)
			container, err := serviceRuntime.Start(changedConfig.ServiceConfig)
			if err != nil {
				log.Printf("ERROR: Could not start %s: %s\n",
					changedConfig.ServiceConfig.Version(), err)
				continue
			}
			log.Printf("Restarted %s as: %s\n", changedConfig.ServiceConfig.Version(), container.ID)

			err = serviceRuntime.StopAllButLatest(*stopCutoff)
			if err != nil {
				log.Printf("ERROR: Could not stop containers: %s\n", err)
			}
		case <-ticker.C:
			err := startContainersIfNecessary()
			if err != nil {
				log.Printf("ERROR: Could not start containers: %s\n", err)
			}

			err = serviceRuntime.StopAllButLatest(*stopCutoff)
			if err != nil {
				log.Printf("ERROR: Could not stop containers: %s\n", err)
			}
		}

	}
}

func main() {
	flag.Parse()

	if *env == "" {
		fmt.Println("Need an env")
		flag.PrintDefaults()
		os.Exit(1)
	}

	if *pool == "" {
		fmt.Println("Need a pool")
		flag.PrintDefaults()
		os.Exit(1)
	}

	initOrDie()
	serviceRegistry.CreatePool(*pool)

	err := startContainersIfNecessary()
	if err != nil && !*loop {
		return
	}

	if !*loop {
		return
	}

	restartChan := make(chan *registry.ConfigChange, 10)
	cancelChan := make(chan struct{})
	// do we need to cancel ever?

	serviceRegistry.Watch(restartChan, cancelChan)
	restartContainers(restartChan)
}
<commit_msg>Move StopAllButLatest call out of startContainersIfNecessary<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"time"

	"github.com/litl/galaxy/log"
	"github.com/litl/galaxy/registry"
	"github.com/litl/galaxy/runtime"
	"github.com/litl/galaxy/utils"
)

var (
	stopCutoff      = flag.Int64("cutoff", 10, "Seconds to wait before stopping old containers")
	app             = flag.String("app", "", "App to start")
	redisHost       = flag.String("redis", utils.GetEnv("GALAXY_REDIS_HOST", "127.0.0.1:6379"), "redis host")
	env             = flag.String("env", utils.GetEnv("GALAXY_ENV", "dev"), "Environment namespace")
	pool            = flag.String("pool", utils.GetEnv("GALAXY_POOL", "web"), "Pool namespace")
	loop            = flag.Bool("loop", false, "Run continously")
	shuttleHost     = flag.String("shuttleAddr", "", "IP where containers can reach shuttle proxy. Defaults to docker0 IP.")
	serviceConfigs  []*registry.ServiceConfig
	serviceRegistry *registry.ServiceRegistry
	serviceRuntime  *runtime.ServiceRuntime
)

func initOrDie() {

	serviceRegistry = registry.NewServiceRegistry(
		*env,
		*pool,
		"",
		600,
		"",
	)

	serviceRegistry.Connect(*redisHost)
	serviceRuntime = runtime.NewServiceRuntime(*shuttleHost, *env, *pool, *redisHost)
}

func startContainersIfNecessary() error {
	serviceConfigs, err := serviceRegistry.ListApps()
	if err != nil {
		log.Printf("ERROR: Could not retrieve service configs for /%s/%s: %s\n", *env, *pool, err)
		return err
	}

	if len(serviceConfigs) == 0 {
		log.Printf("No services configured for /%s/%s\n", *env, *pool)
		return err
	}

	for _, serviceConfig := range serviceConfigs {

		if *app != "" && serviceConfig.Name != *app {
			continue
		}

		if serviceConfig.Version() == "" {
			log.Printf("Skipping %s. No version configured.\n", serviceConfig.Name)
			continue
		}

		container, err := serviceRuntime.StartIfNotRunning(&serviceConfig)
		if err != nil {
			log.Printf("ERROR: Could not determine if %s is running: %s\n",
				serviceConfig.Version(), err)
			return err
		}

		log.Printf("%s version %s running as %s\n", serviceConfig.Name, serviceConfig.Version(), container.ID[0:12])

	}
	return nil
}

func restartContainers(changedConfigs chan *registry.ConfigChange) {
	ticker := time.NewTicker(10 * time.Second)

	for {

		var changedConfig *registry.ConfigChange
		select {

		case changedConfig = <-changedConfigs:
			if changedConfig.Error != nil {
				log.Printf("ERROR: Error watching changes: %s\n", changedConfig.Error)
				continue
			}

			if changedConfig.ServiceConfig == nil {
				continue
			}

			if changedConfig.ServiceConfig.Version() == "" {
				continue
			}

			log.Printf("Restarting %s\n", changedConfig.ServiceConfig.Name)
			container, err := serviceRuntime.Start(changedConfig.ServiceConfig)
			if err != nil {
				log.Printf("ERROR: Could not start %s: %s\n",
					changedConfig.ServiceConfig.Version(), err)
				continue
			}
			log.Printf("Restarted %s as: %s\n", changedConfig.ServiceConfig.Version(), container.ID)

			err = serviceRuntime.StopAllButLatest(*stopCutoff)
			if err != nil {
				log.Printf("ERROR: Could not stop containers: %s\n", err)
			}
		case <-ticker.C:
			err := startContainersIfNecessary()
			if err != nil {
				log.Printf("ERROR: Could not start containers: %s\n", err)
			}

			err = serviceRuntime.StopAllButLatest(*stopCutoff)
			if err != nil {
				log.Printf("ERROR: Could not stop containers: %s\n", err)
			}
		}

	}
}

func main() {
	flag.Parse()

	if *env == "" {
		fmt.Println("Need an env")
		flag.PrintDefaults()
		os.Exit(1)
	}

	if *pool == "" {
		fmt.Println("Need a pool")
		flag.PrintDefaults()
		os.Exit(1)
	}

	initOrDie()
	serviceRegistry.CreatePool(*pool)

	err := startContainersIfNecessary()
	if err != nil && !*loop {
		log.Printf("ERROR: Could not start containers: %s\n", err)
		return
	}

	err = serviceRuntime.StopAllButLatest(*stopCutoff)
	if err != nil && !*loop {
		log.Printf("ERROR: Could not start containers: %s\n", err)
		return
	}

	if !*loop {
		return
	}

	restartChan := make(chan *registry.ConfigChange, 10)
	cancelChan := make(chan struct{})
	// do we need to cancel ever?

	serviceRegistry.Watch(restartChan, cancelChan)
	restartContainers(restartChan)
}
<|endoftext|>


<commit_before>package commands

import (
	"log"

	"github.com/nelsam/gxui"
)

type Copy struct {
	driver gxui.Driver
}

func NewCopy(driver gxui.Driver) *Copy {
	return &Copy{
		driver: driver,
	}
}

func (c *Copy) Start(gxui.Control) gxui.Control {
	return nil
}

func (c *Copy) Name() string {
	return "copy-selection"
}

func (c *Copy) Next() gxui.Focusable {
	return nil
}

func (c *Copy) Exec(target interface{}) (executed, consume bool) {
	finder, ok := target.(EditorFinder)
	if !ok {
		return false, false
	}
	editor := finder.CurrentEditor()
	if editor.Controller().SelectionCount() != 1 {
		panic("Trying to copy without exactly 1 selection")
	}
	selection := editor.Controller().SelectionText(0)
	c.driver.SetClipboard(selection)
	return true, true
}

type Cut struct {
	Copy
}

func NewCut(driver gxui.Driver) *Cut {
	copy := NewCopy(driver)
	return &Cut{Copy: *copy}
}

func (c *Cut) Name() string {
	return "cut-selection"
}

func (c *Cut) Exec(target interface{}) (executed, consume bool) {
	executed, consume = c.Copy.Exec(target)
	if !executed {
		return executed, consume
	}
	editor := target.(EditorFinder).CurrentEditor()
	selection := editor.Controller().FirstSelection()
	newRunes, _ := editor.Controller().ReplaceAt(editor.Runes(), selection.Start(), selection.End(), []rune(""))
	editor.SetText(string(newRunes))
	return
}

type Paste struct {
	driver gxui.Driver
}

func NewPaste(driver gxui.Driver) *Paste {
	return &Paste{
		driver: driver,
	}
}

func (p *Paste) Name() string {
	return "paste"
}

func (p *Paste) Start(gxui.Control) gxui.Control {
	log.Printf("Calling paste start")
	return nil
}

func (p *Paste) Next() gxui.Focusable {
	return nil
}

func (p *Paste) Exec(target interface{}) (executed, consume bool) {
	log.Printf("Calling paste execute")
	finder, ok := target.(EditorFinder)
	if !ok {
		return false, false
	}
	contents, err := p.driver.GetClipboard()
	if err != nil {
		panic(err)
	}
	editor := finder.CurrentEditor()
	editor.Controller().Replace(func(sel gxui.TextSelection) string {
		return contents
	})
	return true, true
}
<commit_msg>Remove debug logging from copy/paste<commit_after>package commands

import "github.com/nelsam/gxui"

type Copy struct {
	driver gxui.Driver
}

func NewCopy(driver gxui.Driver) *Copy {
	return &Copy{
		driver: driver,
	}
}

func (c *Copy) Start(gxui.Control) gxui.Control {
	return nil
}

func (c *Copy) Name() string {
	return "copy-selection"
}

func (c *Copy) Next() gxui.Focusable {
	return nil
}

func (c *Copy) Exec(target interface{}) (executed, consume bool) {
	finder, ok := target.(EditorFinder)
	if !ok {
		return false, false
	}
	editor := finder.CurrentEditor()
	if editor.Controller().SelectionCount() != 1 {
		panic("Trying to copy without exactly 1 selection")
	}
	selection := editor.Controller().SelectionText(0)
	c.driver.SetClipboard(selection)
	return true, true
}

type Cut struct {
	Copy
}

func NewCut(driver gxui.Driver) *Cut {
	copy := NewCopy(driver)
	return &Cut{Copy: *copy}
}

func (c *Cut) Name() string {
	return "cut-selection"
}

func (c *Cut) Exec(target interface{}) (executed, consume bool) {
	executed, consume = c.Copy.Exec(target)
	if !executed {
		return executed, consume
	}
	editor := target.(EditorFinder).CurrentEditor()
	selection := editor.Controller().FirstSelection()
	newRunes, _ := editor.Controller().ReplaceAt(editor.Runes(), selection.Start(), selection.End(), []rune(""))
	editor.SetText(string(newRunes))
	return
}

type Paste struct {
	driver gxui.Driver
}

func NewPaste(driver gxui.Driver) *Paste {
	return &Paste{
		driver: driver,
	}
}

func (p *Paste) Name() string {
	return "paste"
}

func (p *Paste) Start(gxui.Control) gxui.Control {
	return nil
}

func (p *Paste) Next() gxui.Focusable {
	return nil
}

func (p *Paste) Exec(target interface{}) (executed, consume bool) {
	finder, ok := target.(EditorFinder)
	if !ok {
		return false, false
	}
	contents, err := p.driver.GetClipboard()
	if err != nil {
		panic(err)
	}
	editor := finder.CurrentEditor()
	editor.Controller().Replace(func(sel gxui.TextSelection) string {
		return contents
	})
	return true, true
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"github.com/olekukonko/tablewriter"
	"io"
	"net/http"
	"os"
	"strconv"
	"strings"
)

var (
	Board   = flag.String("b", "car", "Specific ptt board")
	Page    = flag.Int("p", 2, "Default pages to fetched")
	Feature = flag.String("f", "hot", "Special features")
)

var (
	BaseUrl     = "https://www.ptt.cc/bbs/"
	CarBoardUrl = "https://www.ptt.cc/bbs/car/index.html"
)

func main() {
	flag.Parse()

	if len(os.Args) == 1 {
		fetchMultiPages(*Board, *Page)
	} else {
		fmt.Println(os.Args[2])
		switch os.Args[1] {
		case "-b":
		case "-t":
		default:
			fetchMultiPages(*Board, *Page)
		}
	}
}

func fetchSingle(url string, str chan string) {
	resp := fetch(url)
	defer resp.Body.Close()

	doc := getDocument(resp)
	tmp := ""
	doc.Find("div.r-ent").Each(func(i int, s *goquery.Selection) {
		link, _ := s.Find("a").Attr("href")
		title := s.Find("a").Text()
		push := s.Find("span").Text()
		if push == "" {
			push = "X"
		}

		if len(link) != 0 {
			tmp += push + "\t" + title + "\t" + "https://www.ptt.cc" + link + "\n"
		}
	})
	str <- tmp
}

func fetchPages(url string, ch chan int) {
	resp := fetch(url)
	defer resp.Body.Close()

	doc := getDocument(resp)
	href, _ := doc.Find("div.action-bar").Find("a.btn").Eq(3).Attr("href")
	pages, _ := strconv.Atoi(strings.Trim(strings.Split(strings.Split(href, "/")[3], ".")[0], "index"))
	ch <- pages
}

func fetchMultiPages(board string, pre int) {
	ch := make(chan int)
	url := BaseUrl + board + "/index.html"
	go fetchPages(url, ch)
	p := <-ch

	var output string

	var pagesURL = make([]string, pre+1)
	for i := pre; i >= 0; i-- {
		pagesURL[i] = BaseUrl + board + "/index" + strconv.Itoa(p+1-i) + ".html"
		s := make(chan string)
		go fetchSingle(pagesURL[i], s)
		result := <-s
		output += result
	}

	printOutput(output)
}

func printOutput(output string) {

	table := tablewriter.NewWriter(os.Stdout)
	table.SetHeader([]string{"Push", "Title", "URL"})
	table.SetBorder(false)
	table.SetAlignment(tablewriter.ALIGN_LEFT)
	o := [][]string{}

	rows := strings.Split(output, "\n")
	for i := 0; i < len(rows); i++ {
		row := strings.Split(rows[i], "\t") // row[0], row[1], row[2]
		if len(row) == 3 {
			arr := []string{row[0], row[1], row[2]}
			o = append(o, arr)
		}
	}

	for _, v := range o {
		table.Append(v)
	}

	table.Render() // Send output
}

func getDocument(resp *http.Response) *goquery.Document {
	r := io.Reader(resp.Body)
	doc, _ := goquery.NewDocumentFromReader(r)
	return doc
}

func fetch(url string) *http.Response {
	resp, _ := http.Get(url)
	return resp
}
<commit_msg>Refactor switch case<commit_after>package main

import (
	"flag"
	"github.com/PuerkitoBio/goquery"
	"github.com/olekukonko/tablewriter"
	"io"
	"net/http"
	"os"
	"strconv"
	"strings"
)

var (
	Board   = flag.String("b", "car", "Specific ptt board")
	Page    = flag.Int("p", 2, "Default pages to fetched")
	Feature = flag.String("f", "hot", "Special features")
)

var (
	BaseUrl     = "https://www.ptt.cc/bbs/"
	CarBoardUrl = "https://www.ptt.cc/bbs/car/index.html"
)

func main() {
	flag.Parse()
	fetchMultiPages(*Board, *Page)
}

func fetchSingle(url string, str chan string) {
	resp := fetch(url)
	defer resp.Body.Close()

	doc := getDocument(resp)
	tmp := ""
	doc.Find("div.r-ent").Each(func(i int, s *goquery.Selection) {
		link, _ := s.Find("a").Attr("href")
		title := s.Find("a").Text()
		push := s.Find("span").Text()
		if push == "" {
			push = "X"
		}

		if len(link) != 0 {
			tmp += push + "\t" + title + "\t" + "https://www.ptt.cc" + link + "\n"
		}
	})
	str <- tmp
}

func fetchPages(url string, ch chan int) {
	resp := fetch(url)
	defer resp.Body.Close()

	doc := getDocument(resp)
	href, _ := doc.Find("div.action-bar").Find("a.btn").Eq(3).Attr("href")
	pages, _ := strconv.Atoi(strings.Trim(strings.Split(strings.Split(href, "/")[3], ".")[0], "index"))
	ch <- pages
}

func fetchMultiPages(board string, pre int) {
	ch := make(chan int)
	url := BaseUrl + board + "/index.html"
	go fetchPages(url, ch)
	p := <-ch

	var output string

	var pagesURL = make([]string, pre+1)
	for i := pre; i >= 0; i-- {
		pagesURL[i] = BaseUrl + board + "/index" + strconv.Itoa(p+1-i) + ".html"
		s := make(chan string)
		go fetchSingle(pagesURL[i], s)
		result := <-s
		output += result
	}

	printOutput(output)
}

func  printOutput(output string) {

	table := tablewriter.NewWriter(os.Stdout)
	table.SetHeader([]string{"Push", "Title", "URL"})
	table.SetBorder(false)
	table.SetAlignment(tablewriter.ALIGN_LEFT)
	o := [][]string{}

	rows := strings.Split(output, "\n")
	for i := 0; i < len(rows); i++ {
		row := strings.Split(rows[i], "\t") // row[0], row[1], row[2]
		if len(row) == 3 {
			arr := []string{row[0], row[1], row[2]}
			o = append(o, arr)
		}
	}

	for _, v := range o {
		table.Append(v)
	}

	table.Render() // Send output
}

func getDocument(resp *http.Response) *goquery.Document {
	r := io.Reader(resp.Body)
	doc, _ := goquery.NewDocumentFromReader(r)
	return doc
}

func fetch(url string) *http.Response {
	resp, _ := http.Get(url)
	return resp
}
<|endoftext|>


<commit_before>package tap

import "fmt"
import "testing/quick"

type T struct {
	nextTestNumber int
}

// New creates a new Tap value
func New() *T {
	return &T{
		nextTestNumber: 1,
	}
}

// Header displays a TAP header including version number and expected
// number of tests to run.
func (t *T) Header(testCount int) {
	fmt.Printf("TAP version 13\n")
	fmt.Printf("1..%d\n", testCount)
}

// Ok generates TAP output indicating whether a test passed or failed.
func (t *T) Ok(test bool, description string) {
	// did the test pass or not?
	ok := "ok"
	if !test {
		ok = "not ok"
	}

	fmt.Printf("%s %d - %s\n", ok, t.nextTestNumber, description)
	t.nextTestNumber++
}

// Check runs randomized tests against a function just as "testing/quick.Check"
// does.  Success or failure generate appropriate TAP output.
func (t *T) Check(function interface{}, description string) {
	err := quick.Check(function, nil)
	if err == nil {
		t.Ok(true, description)
		return
	}

	fmt.Printf("# %s\n", err)
	t.Ok(false, description)
}

// Count returns the number of tests completed so far.
func (t *T) Count() int {
	return t.nextTestNumber - 1
}
<commit_msg>Add AutoPlan method<commit_after>package tap

import "fmt"
import "testing/quick"

type T struct {
	nextTestNumber int
}

// New creates a new Tap value
func New() *T {
	return &T{
		nextTestNumber: 1,
	}
}

// Header displays a TAP header including version number and expected
// number of tests to run.
func (t *T) Header(testCount int) {
	fmt.Printf("TAP version 13\n")
	fmt.Printf("1..%d\n", testCount)
}

// Ok generates TAP output indicating whether a test passed or failed.
func (t *T) Ok(test bool, description string) {
	// did the test pass or not?
	ok := "ok"
	if !test {
		ok = "not ok"
	}

	fmt.Printf("%s %d - %s\n", ok, t.nextTestNumber, description)
	t.nextTestNumber++
}

// Check runs randomized tests against a function just as "testing/quick.Check"
// does.  Success or failure generate appropriate TAP output.
func (t *T) Check(function interface{}, description string) {
	err := quick.Check(function, nil)
	if err == nil {
		t.Ok(true, description)
		return
	}

	fmt.Printf("# %s\n", err)
	t.Ok(false, description)
}

// Count returns the number of tests completed so far.
func (t *T) Count() int {
	return t.nextTestNumber - 1
}

// AutoPlan generates a test plan based on the number of tests that were run.
func (t *T) AutoPlan() {
	fmt.Printf("1..%d\n", t.nextTestNumber-1)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"

	"code.cloudfoundry.org/cli/cf/api/appinstances"
	"code.cloudfoundry.org/cli/cf/api/resources"
	"code.cloudfoundry.org/cli/cf/configuration/coreconfig"
	"code.cloudfoundry.org/cli/cf/models"
	"code.cloudfoundry.org/cli/cf/net"
)

// AppRepo provides functionality to query a Cloud Foundry API for applications.
type AppRepo struct {
	config  coreconfig.Reader
	gateway net.Gateway
}

// NewAppRepo returns a new SpaceRepo which internally uses the specified
// coreconfig.Repository and net.Gateway.
func NewAppRepo(config coreconfig.Repository, gateway net.Gateway) (repo AppRepo) {
	repo.config = config
	repo.gateway = gateway
	return
}

// ListApps lists all applicaitons in the organisation. For each application, it
// calls the provided callback function.
func (ar AppRepo) ListApps(callback func(models.Application) bool) error {
	return ar.gateway.ListPaginatedResources(
		ar.config.APIEndpoint(),
		"/v2/apps",
		resources.ApplicationResource{},
		func(resource interface{}) bool {
			return callback(resource.(resources.ApplicationResource).ToModel())
		})
}

// GetAppStats fetches the stats for the specified application. The application
// needs to be started.
func (ar AppRepo) GetAppStats(app models.Application) (statsResponse appinstances.StatsAPIResponse, err error) {
	path := fmt.Sprintf("%s/v2/apps/%s/stats", ar.config.APIEndpoint(), app.GUID)
	statsResponse = appinstances.StatsAPIResponse{}
	err = ar.gateway.GetResource(path, &statsResponse)
	return
}
<commit_msg>Fix typo<commit_after>package main

import (
	"fmt"

	"code.cloudfoundry.org/cli/cf/api/appinstances"
	"code.cloudfoundry.org/cli/cf/api/resources"
	"code.cloudfoundry.org/cli/cf/configuration/coreconfig"
	"code.cloudfoundry.org/cli/cf/models"
	"code.cloudfoundry.org/cli/cf/net"
)

// AppRepo provides functionality to query a Cloud Foundry API for applications.
type AppRepo struct {
	config  coreconfig.Reader
	gateway net.Gateway
}

// NewAppRepo returns a new SpaceRepo which internally uses the specified
// coreconfig.Repository and net.Gateway.
func NewAppRepo(config coreconfig.Repository, gateway net.Gateway) (repo AppRepo) {
	repo.config = config
	repo.gateway = gateway
	return
}

// ListApps lists all applications in the organisation. For each application, it
// calls the provided callback function.
func (ar AppRepo) ListApps(callback func(models.Application) bool) error {
	return ar.gateway.ListPaginatedResources(
		ar.config.APIEndpoint(),
		"/v2/apps",
		resources.ApplicationResource{},
		func(resource interface{}) bool {
			return callback(resource.(resources.ApplicationResource).ToModel())
		})
}

// GetAppStats fetches the stats for the specified application. The application
// needs to be started.
func (ar AppRepo) GetAppStats(app models.Application) (statsResponse appinstances.StatsAPIResponse, err error) {
	path := fmt.Sprintf("%s/v2/apps/%s/stats", ar.config.APIEndpoint(), app.GUID)
	statsResponse = appinstances.StatsAPIResponse{}
	err = ar.gateway.GetResource(path, &statsResponse)
	return
}
<|endoftext|>


<commit_before>package index

import (
	"fmt"
	"github.com/MG-RAST/Shock/shock-server/conf"
	e "github.com/MG-RAST/Shock/shock-server/errors"
	"github.com/MG-RAST/Shock/shock-server/logger"
	"github.com/MG-RAST/Shock/shock-server/node"
	"github.com/MG-RAST/Shock/shock-server/node/file/index"
	"github.com/MG-RAST/Shock/shock-server/request"
	"github.com/MG-RAST/Shock/shock-server/responder"
	"github.com/MG-RAST/Shock/shock-server/user"
	"github.com/MG-RAST/golib/mgo"
	"github.com/MG-RAST/golib/stretchr/goweb/context"
	"net/http"
	"os"
	"strconv"
)

type getRes struct {
	I interface{} `json:"indexes"`
	A interface{} `json:"available_indexers"`
}

type m map[string]string

// GET, PUT, DELETE: /node/{nid}/index/{idxType}
func IndexTypedRequest(ctx context.Context) {
	nid := ctx.PathValue("nid")
	idxType := ctx.PathValue("idxType")

	u, err := request.Authenticate(ctx.HttpRequest())
	if err != nil && err.Error() != e.NoAuth {
		request.AuthError(err, ctx)
		return
	}

	// Fake public user
	if u == nil {
		u = &user.User{Uuid: ""}
	}

	// Load node and handle user unauthorized
	n, err := node.Load(nid, u)
	if err != nil {
		if err.Error() == e.UnAuth {
			responder.RespondWithError(ctx, http.StatusUnauthorized, e.UnAuth)
		} else if err == mgo.ErrNotFound {
			responder.RespondWithError(ctx, http.StatusNotFound, "Node not found.")
		} else {
			// In theory the db connection could be lost between
			// checking user and load but seems unlikely.
			err_msg := "Err@index:LoadNode: " + nid + ":" + err.Error()
			logger.Error(err_msg)
			responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg)
		}
		return
	}

	switch ctx.HttpRequest().Method {
	case "DELETE":
		rights := n.Acl.Check(u.Uuid)
		if !rights["write"] {
			responder.RespondWithError(ctx, http.StatusUnauthorized, e.UnAuth)
			return
		}

		if _, has := n.Indexes[idxType]; has {
			if err := n.DeleteIndex(idxType); err != nil {
				err_msg := err.Error()
				logger.Error(err_msg)
				responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg)
				return
			}
			responder.RespondOK(ctx)
		} else {
			responder.RespondWithError(ctx, http.StatusBadRequest, fmt.Sprintf("Node %s does not have index of type %s.", n.Id, idxType))
		}

	case "GET":
		if v, has := n.Indexes[idxType]; has {
			responder.RespondWithData(ctx, map[string]interface{}{idxType: v})
		} else {
			responder.RespondWithError(ctx, http.StatusBadRequest, fmt.Sprintf("Node %s does not have index of type %s.", n.Id, idxType))
		}

	case "PUT":
		rights := n.Acl.Check(u.Uuid)
		if !rights["write"] {
			responder.RespondWithError(ctx, http.StatusUnauthorized, e.UnAuth)
			return
		}

		// Gather query params
		query := ctx.HttpRequest().URL.Query()
		_, forceRebuild := query["force_rebuild"]

		if _, has := n.Indexes[idxType]; has {
			if idxType == "size" {
				responder.RespondOK(ctx)
			} else if !forceRebuild {
				responder.RespondWithError(ctx, http.StatusBadRequest, "This index already exists, please add the parameter 'force_rebuild=1' to force a rebuild of the existing index.")
			}
			return
		}

		if !n.HasFile() {
			responder.RespondWithError(ctx, http.StatusBadRequest, "Node has no file.")
			return
		} else if idxType == "" {
			responder.RespondWithError(ctx, http.StatusBadRequest, "Index create requires type.")
			return
		}
		if _, ok := index.Indexers[idxType]; !ok && idxType != "bai" && idxType != "subset" && idxType != "column" {
			responder.RespondWithError(ctx, http.StatusBadRequest, fmt.Sprintf("Index type %s unavailable.", idxType))
			return
		}
		if idxType == "size" {
			responder.RespondWithError(ctx, http.StatusBadRequest, fmt.Sprintf("Index type size is a virtual index and does not require index building."))
			return
		}

		if conf.Bool(conf.Conf["perf-log"]) {
			logger.Perf("START indexing: " + nid)
		}

		if idxType == "bai" {
			//bam index is created by the command-line tool samtools
			if n.Type == "subset" {
				responder.RespondWithError(ctx, http.StatusBadRequest, "Shock does not support bam index creation on subset nodes.")
				return
			}

			if ext := n.FileExt(); ext == ".bam" {
				if err := index.CreateBamIndex(n.FilePath()); err != nil {
					responder.RespondWithError(ctx, http.StatusInternalServerError, "Error while creating bam index.")
					return
				}
				responder.RespondOK(ctx)
			} else {
				responder.RespondWithError(ctx, http.StatusBadRequest, "Index type bai requires .bam file.")
			}
			return
		}

		subsetSize := int64(0)
		count := int64(0)
		indexFormat := ""
		subsetName := ""
		if idxType == "subset" {
			// Utilizing the multipart form parser since we need to upload a file.
			params, files, err := request.ParseMultipartForm(ctx.HttpRequest())
			if err != nil {
				responder.RespondWithError(ctx, http.StatusBadRequest, err.Error())
				return
			}

			parentIndex, hasParent := params["parent_index"]
			if !hasParent {
				responder.RespondWithError(ctx, http.StatusBadRequest, "Index type subset requires parent_index param.")
				return
			} else if _, has := n.Indexes[parentIndex]; !has {
				responder.RespondWithError(ctx, http.StatusBadRequest, fmt.Sprintf("Node %s does not have index of type %s.", n.Id, parentIndex))
				return
			}

			newIndex, hasName := params["index_name"]
			if !hasName {
				responder.RespondWithError(ctx, http.StatusBadRequest, "Index type subset requires index_name param.")
				return
			} else if _, reservedName := index.Indexers[newIndex]; reservedName || newIndex == "bai" {
				responder.RespondWithError(ctx, http.StatusBadRequest, fmt.Sprintf("%s is a reserved index name and cannot be used to create a custom subset index.", newIndex))
				return
			}
			subsetName = newIndex

			subsetIndices, hasFile := files["subset_indices"]
			if !hasFile {
				responder.RespondWithError(ctx, http.StatusBadRequest, "Index type subset requires subset_indices file.")
				return
			}

			f, _ := os.Open(subsetIndices.Path)
			defer f.Close()
			idxer := index.NewSubsetIndexer(f)

			// we default to "array" index format for backwards compatibility
			indexFormat = "array"
			if n.Indexes[parentIndex].Format == "array" || n.Indexes[parentIndex].Format == "matrix" {
				indexFormat = n.Indexes[parentIndex].Format
			}
			count, subsetSize, err = index.CreateSubsetIndex(&idxer, n.IndexPath()+"/"+newIndex+".idx", n.IndexPath()+"/"+parentIndex+".idx", indexFormat, n.Indexes[parentIndex].TotalUnits)
			if err != nil {
				logger.Error("err " + err.Error())
				responder.RespondWithError(ctx, http.StatusBadRequest, err.Error())
				return
			}

		} else if idxType == "column" {
			// Gather query params
			query := ctx.HttpRequest().URL.Query()

			if n.Type == "subset" {
				responder.RespondWithError(ctx, http.StatusBadRequest, "Shock does not support column index creation on subset nodes.")
				return
			}

			if _, exists := query["number"]; !exists {
				err_msg := "Index type column requires a number parameter in the url."
				logger.Error(err_msg)
				responder.RespondWithError(ctx, http.StatusBadRequest, err_msg)
				return
			}

			num_str := query.Get("number")
			idxType = idxType + num_str
			num, err := strconv.Atoi(num_str)
			if err != nil || num < 1 {
				err_msg := "Index type column requires a number parameter in the url of an integer greater than zero."
				logger.Error(err_msg)
				responder.RespondWithError(ctx, http.StatusBadRequest, err_msg)
				return
			}

			f, _ := os.Open(n.FilePath())
			defer f.Close()
			idxer := index.NewColumnIndexer(f)
			count, indexFormat, err = index.CreateColumnIndex(&idxer, num, n.IndexPath()+"/"+idxType+".idx")
			if err != nil {
				logger.Error("err " + err.Error())
				responder.RespondWithError(ctx, http.StatusBadRequest, err.Error())
				return
			}
		} else {
			if n.Type == "subset" && (idxType != "chunkrecord" || n.Subset.Parent.IndexName != "record") {
				responder.RespondWithError(ctx, http.StatusBadRequest, "For subset nodes, Shock currently only supports subset and chunkrecord indexes. Also, for a chunkrecord index, the subset node must have been generated from a record index.")
				return
			}

			newIndexer := index.Indexers[idxType]
			f, _ := os.Open(n.FilePath())
			defer f.Close()
			var idxer index.Indexer
			if n.Type == "subset" {
				idxer = newIndexer(f, n.Type, n.Subset.Index.Format, n.IndexPath()+"/"+n.Subset.Parent.IndexName+".idx")
			} else {
				idxer = newIndexer(f, n.Type, "", "")
			}
			count, indexFormat, err = idxer.Create(n.IndexPath() + "/" + idxType + ".idx")
			if err != nil {
				logger.Error("err " + err.Error())
				responder.RespondWithError(ctx, http.StatusBadRequest, err.Error())
				return
			}
		}

		if count == 0 {
			responder.RespondWithError(ctx, http.StatusBadRequest, "Index empty.")
			return
		}

		idxInfo := node.IdxInfo{
			Type:        idxType,
			TotalUnits:  count,
			AvgUnitSize: n.File.Size / count,
			Format:      indexFormat,
		}

		//if idxType == "chunkrecord" {
		//	idxInfo.AvgUnitSize = conf.CHUNK_SIZE
		//}

		if idxType == "subset" {
			idxType = subsetName
			idxInfo.AvgUnitSize = subsetSize / count
		}

		if err := n.SetIndexInfo(idxType, idxInfo); err != nil {
			logger.Error("err@node.SetIndexInfo: " + err.Error())
		}

		if conf.Bool(conf.Conf["perf-log"]) {
			logger.Perf("END indexing: " + nid)
		}

		responder.RespondOK(ctx)

	default:
		responder.RespondWithError(ctx, http.StatusNotImplemented, "This request type is not implemented.")
	}
	return
}

func contains(list []string, s string) bool {
	for _, i := range list {
		if i == s {
			return true
		}
	}
	return false
}

func availIndexers() (indexers []string) {
	for name, _ := range index.Indexers {
		indexers = append(indexers, name)
	}
	return
}
<commit_msg>Adding new ACL schema to index controller.<commit_after>package index

import (
	"fmt"
	"github.com/MG-RAST/Shock/shock-server/conf"
	e "github.com/MG-RAST/Shock/shock-server/errors"
	"github.com/MG-RAST/Shock/shock-server/logger"
	"github.com/MG-RAST/Shock/shock-server/node"
	"github.com/MG-RAST/Shock/shock-server/node/file/index"
	"github.com/MG-RAST/Shock/shock-server/request"
	"github.com/MG-RAST/Shock/shock-server/responder"
	"github.com/MG-RAST/Shock/shock-server/user"
	"github.com/MG-RAST/golib/mgo"
	"github.com/MG-RAST/golib/stretchr/goweb/context"
	"net/http"
	"os"
	"strconv"
)

type getRes struct {
	I interface{} `json:"indexes"`
	A interface{} `json:"available_indexers"`
}

type m map[string]string

// GET, PUT, DELETE: /node/{nid}/index/{idxType}
func IndexTypedRequest(ctx context.Context) {
	nid := ctx.PathValue("nid")
	idxType := ctx.PathValue("idxType")

	u, err := request.Authenticate(ctx.HttpRequest())
	if err != nil {
		if err.Error() != e.NoAuth {
			request.AuthError(err, ctx)
			return
		} else {
			responder.RespondWithError(ctx, http.StatusUnauthorized, e.NoAuth)
			return
		}
	}

	// public user (no auth) can be used in some cases
	if u == nil {
		u = &user.User{Uuid: "public"}
	}

	// Load node by id
	n, err := node.LoadUnauth(nid)
	if err != nil {
		if err == mgo.ErrNotFound {
			responder.RespondWithError(ctx, http.StatusNotFound, "Node not found.")
		} else {
			// In theory the db connection could be lost between
			// checking user and load but seems unlikely.
			err_msg := "Err@index:LoadNode: " + nid + ":" + err.Error()
			logger.Error(err_msg)
			responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg)
		}
		return
	}

	switch ctx.HttpRequest().Method {
	case "DELETE":
		rights := n.Acl.Check(u.Uuid)
		if !rights["write"] {
			responder.RespondWithError(ctx, http.StatusUnauthorized, e.UnAuth)
			return
		}

		if _, has := n.Indexes[idxType]; has {
			if err := n.DeleteIndex(idxType); err != nil {
				err_msg := err.Error()
				logger.Error(err_msg)
				responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg)
				return
			}
			responder.RespondOK(ctx)
		} else {
			responder.RespondWithError(ctx, http.StatusBadRequest, fmt.Sprintf("Node %s does not have index of type %s.", n.Id, idxType))
		}

	case "GET":
		if v, has := n.Indexes[idxType]; has {
			responder.RespondWithData(ctx, map[string]interface{}{idxType: v})
		} else {
			responder.RespondWithError(ctx, http.StatusBadRequest, fmt.Sprintf("Node %s does not have index of type %s.", n.Id, idxType))
		}

	case "PUT":
		rights := n.Acl.Check(u.Uuid)
		if !rights["write"] {
			responder.RespondWithError(ctx, http.StatusUnauthorized, e.UnAuth)
			return
		}

		// Gather query params
		query := ctx.HttpRequest().URL.Query()
		_, forceRebuild := query["force_rebuild"]

		if _, has := n.Indexes[idxType]; has {
			if idxType == "size" {
				responder.RespondOK(ctx)
			} else if !forceRebuild {
				responder.RespondWithError(ctx, http.StatusBadRequest, "This index already exists, please add the parameter 'force_rebuild=1' to force a rebuild of the existing index.")
			}
			return
		}

		if !n.HasFile() {
			responder.RespondWithError(ctx, http.StatusBadRequest, "Node has no file.")
			return
		} else if idxType == "" {
			responder.RespondWithError(ctx, http.StatusBadRequest, "Index create requires type.")
			return
		}
		if _, ok := index.Indexers[idxType]; !ok && idxType != "bai" && idxType != "subset" && idxType != "column" {
			responder.RespondWithError(ctx, http.StatusBadRequest, fmt.Sprintf("Index type %s unavailable.", idxType))
			return
		}
		if idxType == "size" {
			responder.RespondWithError(ctx, http.StatusBadRequest, fmt.Sprintf("Index type size is a virtual index and does not require index building."))
			return
		}

		if conf.Bool(conf.Conf["perf-log"]) {
			logger.Perf("START indexing: " + nid)
		}

		if idxType == "bai" {
			//bam index is created by the command-line tool samtools
			if n.Type == "subset" {
				responder.RespondWithError(ctx, http.StatusBadRequest, "Shock does not support bam index creation on subset nodes.")
				return
			}

			if ext := n.FileExt(); ext == ".bam" {
				if err := index.CreateBamIndex(n.FilePath()); err != nil {
					responder.RespondWithError(ctx, http.StatusInternalServerError, "Error while creating bam index.")
					return
				}
				responder.RespondOK(ctx)
			} else {
				responder.RespondWithError(ctx, http.StatusBadRequest, "Index type bai requires .bam file.")
			}
			return
		}

		subsetSize := int64(0)
		count := int64(0)
		indexFormat := ""
		subsetName := ""
		if idxType == "subset" {
			// Utilizing the multipart form parser since we need to upload a file.
			params, files, err := request.ParseMultipartForm(ctx.HttpRequest())
			if err != nil {
				responder.RespondWithError(ctx, http.StatusBadRequest, err.Error())
				return
			}

			parentIndex, hasParent := params["parent_index"]
			if !hasParent {
				responder.RespondWithError(ctx, http.StatusBadRequest, "Index type subset requires parent_index param.")
				return
			} else if _, has := n.Indexes[parentIndex]; !has {
				responder.RespondWithError(ctx, http.StatusBadRequest, fmt.Sprintf("Node %s does not have index of type %s.", n.Id, parentIndex))
				return
			}

			newIndex, hasName := params["index_name"]
			if !hasName {
				responder.RespondWithError(ctx, http.StatusBadRequest, "Index type subset requires index_name param.")
				return
			} else if _, reservedName := index.Indexers[newIndex]; reservedName || newIndex == "bai" {
				responder.RespondWithError(ctx, http.StatusBadRequest, fmt.Sprintf("%s is a reserved index name and cannot be used to create a custom subset index.", newIndex))
				return
			}
			subsetName = newIndex

			subsetIndices, hasFile := files["subset_indices"]
			if !hasFile {
				responder.RespondWithError(ctx, http.StatusBadRequest, "Index type subset requires subset_indices file.")
				return
			}

			f, _ := os.Open(subsetIndices.Path)
			defer f.Close()
			idxer := index.NewSubsetIndexer(f)

			// we default to "array" index format for backwards compatibility
			indexFormat = "array"
			if n.Indexes[parentIndex].Format == "array" || n.Indexes[parentIndex].Format == "matrix" {
				indexFormat = n.Indexes[parentIndex].Format
			}
			count, subsetSize, err = index.CreateSubsetIndex(&idxer, n.IndexPath()+"/"+newIndex+".idx", n.IndexPath()+"/"+parentIndex+".idx", indexFormat, n.Indexes[parentIndex].TotalUnits)
			if err != nil {
				logger.Error("err " + err.Error())
				responder.RespondWithError(ctx, http.StatusBadRequest, err.Error())
				return
			}

		} else if idxType == "column" {
			// Gather query params
			query := ctx.HttpRequest().URL.Query()

			if n.Type == "subset" {
				responder.RespondWithError(ctx, http.StatusBadRequest, "Shock does not support column index creation on subset nodes.")
				return
			}

			if _, exists := query["number"]; !exists {
				err_msg := "Index type column requires a number parameter in the url."
				logger.Error(err_msg)
				responder.RespondWithError(ctx, http.StatusBadRequest, err_msg)
				return
			}

			num_str := query.Get("number")
			idxType = idxType + num_str
			num, err := strconv.Atoi(num_str)
			if err != nil || num < 1 {
				err_msg := "Index type column requires a number parameter in the url of an integer greater than zero."
				logger.Error(err_msg)
				responder.RespondWithError(ctx, http.StatusBadRequest, err_msg)
				return
			}

			f, _ := os.Open(n.FilePath())
			defer f.Close()
			idxer := index.NewColumnIndexer(f)
			count, indexFormat, err = index.CreateColumnIndex(&idxer, num, n.IndexPath()+"/"+idxType+".idx")
			if err != nil {
				logger.Error("err " + err.Error())
				responder.RespondWithError(ctx, http.StatusBadRequest, err.Error())
				return
			}
		} else {
			if n.Type == "subset" && (idxType != "chunkrecord" || n.Subset.Parent.IndexName != "record") {
				responder.RespondWithError(ctx, http.StatusBadRequest, "For subset nodes, Shock currently only supports subset and chunkrecord indexes. Also, for a chunkrecord index, the subset node must have been generated from a record index.")
				return
			}

			newIndexer := index.Indexers[idxType]
			f, _ := os.Open(n.FilePath())
			defer f.Close()
			var idxer index.Indexer
			if n.Type == "subset" {
				idxer = newIndexer(f, n.Type, n.Subset.Index.Format, n.IndexPath()+"/"+n.Subset.Parent.IndexName+".idx")
			} else {
				idxer = newIndexer(f, n.Type, "", "")
			}
			count, indexFormat, err = idxer.Create(n.IndexPath() + "/" + idxType + ".idx")
			if err != nil {
				logger.Error("err " + err.Error())
				responder.RespondWithError(ctx, http.StatusBadRequest, err.Error())
				return
			}
		}

		if count == 0 {
			responder.RespondWithError(ctx, http.StatusBadRequest, "Index empty.")
			return
		}

		idxInfo := node.IdxInfo{
			Type:        idxType,
			TotalUnits:  count,
			AvgUnitSize: n.File.Size / count,
			Format:      indexFormat,
		}

		//if idxType == "chunkrecord" {
		//	idxInfo.AvgUnitSize = conf.CHUNK_SIZE
		//}

		if idxType == "subset" {
			idxType = subsetName
			idxInfo.AvgUnitSize = subsetSize / count
		}

		if err := n.SetIndexInfo(idxType, idxInfo); err != nil {
			logger.Error("err@node.SetIndexInfo: " + err.Error())
		}

		if conf.Bool(conf.Conf["perf-log"]) {
			logger.Perf("END indexing: " + nid)
		}

		responder.RespondOK(ctx)

	default:
		responder.RespondWithError(ctx, http.StatusNotImplemented, "This request type is not implemented.")
	}
	return
}

func contains(list []string, s string) bool {
	for _, i := range list {
		if i == s {
			return true
		}
	}
	return false
}

func availIndexers() (indexers []string) {
	for name, _ := range index.Indexers {
		indexers = append(indexers, name)
	}
	return
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/getlantern/edgedetect"
	"github.com/getlantern/golog"
)

var (
	log = golog.LoggerFor("edgedetect.cmd")
)

func main() {
	log.Debugf("Is Edge: %v", edgedetect.DefaultBrowserIsEdge())
}
<commit_msg>Added comment to cmd<commit_after>// This simple test program prints out whether or not the default browser is
// Edge, which is useful for testing/debugging edgedetect.
package main

import (
	"github.com/getlantern/edgedetect"
	"github.com/getlantern/golog"
)

var (
	log = golog.LoggerFor("edgedetect.cmd")
)

func main() {
	log.Debugf("Is Edge: %v", edgedetect.DefaultBrowserIsEdge())
}
<|endoftext|>


<commit_before>package fronted

import (
	"crypto/x509"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/http"
	"sync"
	"time"
)

const (
	NumWorkers = 10 // number of worker goroutines for verifying
)

// Masquerade contains the data for a single masquerade host, including
// the domain and the root CA.
type Masquerade struct {
	// Domain: the domain to use for domain fronting
	Domain string

	// IpAddress: pre-resolved ip address to use instead of Domain (if
	// available)
	IpAddress string
}

type MasqueradeSet []*Masquerade

// verifiedMasqueradeSet represents a set of Masquerade configurations.
// verifiedMasqueradeSet verifies each configured Masquerade by attempting to
// proxy using it.
type verifiedMasqueradeSet struct {
	dialer             *dialer
	candidatesCh       chan *Masquerade
	stopCh             chan interface{}
	verifiedChSize     int
	verifiedCh         chan *Masquerade
	verifiedCount      int
	verifiedCountMutex sync.Mutex
	wg                 sync.WaitGroup
}

// nextVerified returns the next available verified *Masquerade, blocking until
// such is available.  The masquerade is immediately written back onto
// verifiedCh, turning verifiedCh into a sort of cyclic queue.
func (vms *verifiedMasqueradeSet) nextVerified() *Masquerade {
	masquerade := <-vms.verifiedCh
	vms.verifiedCh <- masquerade
	return masquerade
}

// verified sets up a new verifiedMasqueradeSet that verifies each of the
// Masquerades in this MasqueradeSet for the given Dialer.
func (d *dialer) verifiedMasquerades() *verifiedMasqueradeSet {
	// Size verifiedChSize to be able to hold the smaller of MaxMasquerades or
	// the number of configured masquerades.
	verifiedChSize := len(d.Masquerades)
	if d.MaxMasquerades < verifiedChSize {
		verifiedChSize = d.MaxMasquerades
	}
	log.Debugf("Verifying up to %d masquerades", verifiedChSize)

	vms := &verifiedMasqueradeSet{
		dialer:         d,
		candidatesCh:   make(chan *Masquerade),
		stopCh:         make(chan interface{}, 1),
		verifiedChSize: verifiedChSize,
		verifiedCh:     make(chan *Masquerade, verifiedChSize),
	}
	vms.wg.Add(NumWorkers)
	// Spawn some worker goroutines to verify masquerades
	for i := 0; i < NumWorkers; i++ {
		go vms.verify()
	}

	// Feed candidates for verification
	go vms.feedCandidates()

	return vms
}

// feedCandidates feeds the candidate masquerades to our worker routines in
// random order
func (vms *verifiedMasqueradeSet) feedCandidates() {
	for _, i := range rand.Perm(len(vms.dialer.Masquerades)) {
		if !vms.feedCandidate(vms.dialer.Masquerades[i]) {
			break
		}
	}
	close(vms.candidatesCh)
}

func (vms *verifiedMasqueradeSet) feedCandidate(candidate *Masquerade) bool {
	select {
	case <-vms.stopCh:
		log.Trace("Received stop, not feeding any further")
		return false
	case vms.candidatesCh <- candidate:
		log.Trace("Fed candidate")
		return true
	}
}

// stop stops the verification process
func (vms *verifiedMasqueradeSet) stop() {
	log.Trace("masquerades stop called")
	vms.stopCh <- nil
	log.Trace("masquerades waiting for workers to finish")
	vms.wg.Wait()
	log.Trace("masquerades stopped")
}

// verify checks masquerades obtained from candidatesCh to see if they work on
// the test server.
func (vms *verifiedMasqueradeSet) verify() {
	for {
		candidate, ok := <-vms.candidatesCh
		if !ok {
			vms.wg.Done()
			return
		}
		if !vms.doVerify(candidate) {
			return
		}
	}
}

// doVerify does the verification and returns a boolean indicating whether or
// not to continue processing more verifications.
func (vms *verifiedMasqueradeSet) doVerify(masquerade *Masquerade) bool {
	errCh := make(chan error, 2)
	go func() {
		// Limit amount of time we'll wait for a response
		time.Sleep(30 * time.Second)
		errCh <- fmt.Errorf("Timed out verifying %s", masquerade.Domain)
	}()
	go func() {
		start := time.Now()
		httpClient := vms.dialer.HttpClientUsing(masquerade)
		req, err := http.NewRequest("HEAD", "http://www.google.com/humans.txt", nil)
		if err != nil {
			errCh <- fmt.Errorf("http.NewRequest Error: %s", err)
		}
		resp, err := httpClient.Do(req)
		if err != nil {
			errmsg := fmt.Sprintf("HTTP error for masquerade %v: %v", masquerade.Domain, err)
			if _, ok := err.(x509.UnknownAuthorityError); ok {
				// This is a temporary fix, see https://github.com/getlantern/lantern/issues/2398
				errmsg = fmt.Sprintf("%s, tlsInfo: %s", errmsg, vms.dialer.tlsInfo(masquerade))
			}

			errCh <- fmt.Errorf(errmsg)
			return
		} else {
			body, err := ioutil.ReadAll(resp.Body)
			defer resp.Body.Close()
			if err != nil {
				errCh <- fmt.Errorf("HTTP Body Error: %s", body)
			} else {
				delta := time.Now().Sub(start)
				log.Tracef("Sucessful check for: %s in %s, %s", masquerade.Domain, delta, body)
				errCh <- nil
			}
		}
	}()
	err := <-errCh
	if err != nil {
		log.Error(err)
		return true
	}
	if vms.incrementVerifiedCount() {
		vms.verifiedCh <- masquerade
		return true
	}
	return false
}

// incrementVerifiedCount keeps track of the number of verified masquerades and
// caps it at MaxMasquerades.
func (vms *verifiedMasqueradeSet) incrementVerifiedCount() bool {
	vms.verifiedCountMutex.Lock()
	defer vms.verifiedCountMutex.Unlock()
	if vms.verifiedCount == vms.verifiedChSize {
		return false
	}
	vms.verifiedCount += 1
	return true
}
<commit_msg>Forcing extended log, this may help us decide if this error is really that bad.<commit_after>package fronted

import (
	//"crypto/x509"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/http"
	"sync"
	"time"
)

const (
	NumWorkers = 10 // number of worker goroutines for verifying
)

// Masquerade contains the data for a single masquerade host, including
// the domain and the root CA.
type Masquerade struct {
	// Domain: the domain to use for domain fronting
	Domain string

	// IpAddress: pre-resolved ip address to use instead of Domain (if
	// available)
	IpAddress string
}

type MasqueradeSet []*Masquerade

// verifiedMasqueradeSet represents a set of Masquerade configurations.
// verifiedMasqueradeSet verifies each configured Masquerade by attempting to
// proxy using it.
type verifiedMasqueradeSet struct {
	dialer             *dialer
	candidatesCh       chan *Masquerade
	stopCh             chan interface{}
	verifiedChSize     int
	verifiedCh         chan *Masquerade
	verifiedCount      int
	verifiedCountMutex sync.Mutex
	wg                 sync.WaitGroup
}

// nextVerified returns the next available verified *Masquerade, blocking until
// such is available.  The masquerade is immediately written back onto
// verifiedCh, turning verifiedCh into a sort of cyclic queue.
func (vms *verifiedMasqueradeSet) nextVerified() *Masquerade {
	masquerade := <-vms.verifiedCh
	vms.verifiedCh <- masquerade
	return masquerade
}

// verified sets up a new verifiedMasqueradeSet that verifies each of the
// Masquerades in this MasqueradeSet for the given Dialer.
func (d *dialer) verifiedMasquerades() *verifiedMasqueradeSet {
	// Size verifiedChSize to be able to hold the smaller of MaxMasquerades or
	// the number of configured masquerades.
	verifiedChSize := len(d.Masquerades)
	if d.MaxMasquerades < verifiedChSize {
		verifiedChSize = d.MaxMasquerades
	}
	log.Debugf("Verifying up to %d masquerades", verifiedChSize)

	vms := &verifiedMasqueradeSet{
		dialer:         d,
		candidatesCh:   make(chan *Masquerade),
		stopCh:         make(chan interface{}, 1),
		verifiedChSize: verifiedChSize,
		verifiedCh:     make(chan *Masquerade, verifiedChSize),
	}
	vms.wg.Add(NumWorkers)
	// Spawn some worker goroutines to verify masquerades
	for i := 0; i < NumWorkers; i++ {
		go vms.verify()
	}

	// Feed candidates for verification
	go vms.feedCandidates()

	return vms
}

// feedCandidates feeds the candidate masquerades to our worker routines in
// random order
func (vms *verifiedMasqueradeSet) feedCandidates() {
	for _, i := range rand.Perm(len(vms.dialer.Masquerades)) {
		if !vms.feedCandidate(vms.dialer.Masquerades[i]) {
			break
		}
	}
	close(vms.candidatesCh)
}

func (vms *verifiedMasqueradeSet) feedCandidate(candidate *Masquerade) bool {
	select {
	case <-vms.stopCh:
		log.Trace("Received stop, not feeding any further")
		return false
	case vms.candidatesCh <- candidate:
		log.Trace("Fed candidate")
		return true
	}
}

// stop stops the verification process
func (vms *verifiedMasqueradeSet) stop() {
	log.Trace("masquerades stop called")
	vms.stopCh <- nil
	log.Trace("masquerades waiting for workers to finish")
	vms.wg.Wait()
	log.Trace("masquerades stopped")
}

// verify checks masquerades obtained from candidatesCh to see if they work on
// the test server.
func (vms *verifiedMasqueradeSet) verify() {
	for {
		candidate, ok := <-vms.candidatesCh
		if !ok {
			vms.wg.Done()
			return
		}
		if !vms.doVerify(candidate) {
			return
		}
	}
}

// doVerify does the verification and returns a boolean indicating whether or
// not to continue processing more verifications.
func (vms *verifiedMasqueradeSet) doVerify(masquerade *Masquerade) bool {
	errCh := make(chan error, 2)
	go func() {
		// Limit amount of time we'll wait for a response
		time.Sleep(30 * time.Second)
		errCh <- fmt.Errorf("Timed out verifying %s", masquerade.Domain)
	}()
	go func() {
		start := time.Now()
		httpClient := vms.dialer.HttpClientUsing(masquerade)
		req, err := http.NewRequest("HEAD", "http://www.google.com/humans.txt", nil)
		if err != nil {
			errCh <- fmt.Errorf("http.NewRequest Error: %s", err)
		}
		resp, err := httpClient.Do(req)
		if err != nil {
			errmsg := fmt.Sprintf("HTTP error for masquerade %v: %v", masquerade.Domain, err)
			// I'm commenting this out because we haven't captured any extended log.
			// See https://github.com/getlantern/lantern/issues/2398
			//if _, ok := err.(x509.UnknownAuthorityError); ok {
			errmsg = fmt.Sprintf("%s, tlsInfo: %s", errmsg, vms.dialer.tlsInfo(masquerade))
			//}

			errCh <- fmt.Errorf(errmsg)
			return
		} else {
			body, err := ioutil.ReadAll(resp.Body)
			defer resp.Body.Close()
			if err != nil {
				errCh <- fmt.Errorf("HTTP Body Error: %s", body)
			} else {
				delta := time.Now().Sub(start)
				log.Tracef("Sucessful check for: %s in %s, %s", masquerade.Domain, delta, body)
				errCh <- nil
			}
		}
	}()
	err := <-errCh
	if err != nil {
		log.Error(err)
		return true
	}
	if vms.incrementVerifiedCount() {
		vms.verifiedCh <- masquerade
		return true
	}
	return false
}

// incrementVerifiedCount keeps track of the number of verified masquerades and
// caps it at MaxMasquerades.
func (vms *verifiedMasqueradeSet) incrementVerifiedCount() bool {
	vms.verifiedCountMutex.Lock()
	defer vms.verifiedCountMutex.Unlock()
	if vms.verifiedCount == vms.verifiedChSize {
		return false
	}
	vms.verifiedCount += 1
	return true
}
<|endoftext|>