<commit_before>package main

import (
	commander "code.google.com/p/go-commander"
	"fmt"
	"github.com/BrianHicks/finch/core"
	"strconv"

	"log"
)

func Selector(tdb *core.TaskDB, args []string) ([]*core.Task, error) {
	tasks, err := tdb.GetPendingTasks()
	if err != nil {
		return tasks, err
	}

	amtTasks := len(tasks)

	if len(args) == 0 {
		// print tasks so the user can see and select them
		for i := 0; i < amtTasks; i++ {
			fmt.Printf("%d: %s\n", i, tasks[i].Description)
		}

		return []*core.Task{}, nil
	} else {
		// select tasks and print status for each one
		selected := []*core.Task{}

		for i := 0; i < len(args); i++ {
			fmt.Printf("selecting \"%s\"... ", args[i])

			taskNum, err := strconv.Atoi(args[i])
			if err != nil {
				fmt.Print("was not digits!\n")
				continue
			}

			if taskNum > amtTasks-1 {
				fmt.Printf("there is no task %d\n", amtTasks)
				continue
			}

			task := tasks[taskNum]
			task.Attrs[core.TagSelected] = true
			selected = append(selected, task)
			fmt.Printf("selected \"%s\"\n", task.Description)
		}

		if len(selected) > 0 {
			err := tdb.PutTasks(selected...)
			if err != nil {
				return selected, err
			} else {
				fmt.Printf("Wrote %d tasks to DB\n", len(selected))
			}
		}

		return selected, nil
	}

	return []*core.Task{}, nil
}

var Select *commander.Command = &commander.Command{
	UsageLine: "select",
	Short:     "select tasks to work on",
	Long: `select tasks to work on, as a part of a new chain.

Selected tasks will be marked as such, then you should complete them in the
*reverse* order you selected them. You can use the "next" command for this.

To select tasks, call "select" once to view a list of tasks. Then you can run
"select" with the numbers of the tasks you want to select them.`,
	Run: func(cmd *commander.Command, args []string) {
		tdb, err := getTaskDB()
		defer tdb.Close()
		if err != nil {
			log.Fatalf("Error opening Task database: %s\n", err)
		}

		_, err = Selector(tdb, args)
		if err != nil {
			log.Fatalf("Error selecting tasks: %s\n", err)
		}
	},
}
<commit_msg>add better templating for selected tasks<commit_after>package main

import (
	commander "code.google.com/p/go-commander"
	"fmt"
	"github.com/BrianHicks/finch/core"
	"os"
	"strconv"
	"text/template"

	"log"
)

var (
	taskTmpl = template.Must(template.New("task").Parse("{{.Count}}: {{.Task.Description}} {{if .Selected}}(selected){{end}}\n"))
)

type IterTask struct {
	Count    int
	Task     *core.Task
	Selected bool
}

func Selector(tdb *core.TaskDB, args []string) ([]*core.Task, error) {
	tasks, err := tdb.GetPendingTasks()
	if err != nil {
		return tasks, err
	}

	amtTasks := len(tasks)

	if len(args) == 0 {
		// print tasks so the user can see and select them
		for i := 0; i < amtTasks; i++ {
			task := tasks[i]
			selected, ok := task.Attrs[core.TagSelected]

			err := taskTmpl.Execute(os.Stdout, IterTask{i, task, selected && ok})
			if err != nil {
				break
				fmt.Printf("Error: %s", err.Error())
			}
		}

		return []*core.Task{}, nil
	} else {
		// select tasks and print status for each one
		selected := []*core.Task{}

		for i := 0; i < len(args); i++ {
			fmt.Printf("selecting \"%s\"... ", args[i])

			taskNum, err := strconv.Atoi(args[i])
			if err != nil {
				fmt.Print("was not digits!\n")
				continue
			}

			if taskNum > amtTasks-1 {
				fmt.Printf("there is no task %d\n", amtTasks)
				continue
			}

			task := tasks[taskNum]
			task.Attrs[core.TagSelected] = true
			selected = append(selected, task)
			fmt.Printf("selected \"%s\"\n", task.Description)
		}

		if len(selected) > 0 {
			err := tdb.PutTasks(selected...)
			if err != nil {
				return selected, err
			} else {
				fmt.Printf("Wrote %d tasks to DB\n", len(selected))
			}
		}

		return selected, nil
	}

	return []*core.Task{}, nil
}

var Select *commander.Command = &commander.Command{
	UsageLine: "select",
	Short:     "select tasks to work on",
	Long: `select tasks to work on, as a part of a new chain.

Selected tasks will be marked as such, then you should complete them in the
*reverse* order you selected them. You can use the "next" command for this.

To select tasks, call "select" once to view a list of tasks. Then you can run
"select" with the numbers of the tasks you want to select them.`,
	Run: func(cmd *commander.Command, args []string) {
		tdb, err := getTaskDB()
		defer tdb.Close()
		if err != nil {
			log.Fatalf("Error opening Task database: %s\n", err)
		}

		_, err = Selector(tdb, args)
		if err != nil {
			log.Fatalf("Error selecting tasks: %s\n", err)
		}
	},
}
<|endoftext|>


<commit_before><commit_msg>Return 429 (not 507) for exceeded cardinality (#351)<commit_after><|endoftext|>


<commit_before>// The idx package provides a metadata index for metrics

package idx

import (
	"encoding/hex"
	"errors"
	"fmt"
	"strconv"
	"strings"
	"time"

	schema "gopkg.in/raintank/schema.v1"
)

var (
	BothBranchAndLeaf  = errors.New("node can't be both branch and leaf")
	BranchUnderLeaf    = errors.New("can't add branch under leaf")
	errInvalidQuery    = errors.New("invalid query")
	errInvalidIdString = errors.New("invalid ID string")
)

//go:generate msgp
type Node struct {
	Path        string
	Leaf        bool
	Defs        []Archive
	HasChildren bool
}

type Archive struct {
	schema.MetricDefinition
	SchemaId uint16 // index in mdata.schemas (not persisted)
	AggId    uint16 // index in mdata.aggregations (not persisted)
	LastSave uint32 // last time the metricDefinition was saved to a backend store (cassandra)
}

type MetricID struct {
	org int
	key [16]byte
}

func NewMetricIDFromString(s string) (MetricID, error) {
	id := MetricID{}
	err := id.FromString(s)
	return id, err
}

func (id *MetricID) FromString(s string) error {
	splits := strings.Split(s, ".")
	if len(splits) != 2 || len(splits[1]) != 32 {
		return errInvalidIdString
	}

	var err error
	id.org, err = strconv.Atoi(splits[0])
	if err != nil {
		return err
	}

	dst := make([]byte, 16)
	n, err := hex.Decode(dst, []byte(splits[1]))
	if n != 16 {
		return errInvalidIdString
	}
	copy(id.key[:], dst)
	return nil
}

func (id *MetricID) String() string {
	return fmt.Sprintf("%d.%032x", id.org, id.key)
}

// used primarily by tests, for convenience
func NewArchiveBare(name string) Archive {
	return Archive{
		MetricDefinition: schema.MetricDefinition{
			Name: name,
		},
	}
}

/*
Currently the index is solely used for supporting Graphite style queries.
So, the index only needs to be able to search by a pattern that matches the
MetricDefinition.Name field. In future we plan to extend the searching
capabilities to include the other fields in the definition.

Note:

* metrictank is a multi-tenant system where different orgs cannot see each
  other's data

* any given metric may appear multiple times, under different organisations

* Each metric path can be mapped to multiple metricDefinitions in the case that
  fields other then the Name vary.  The most common occurrence of this is when
  the Interval at which the metric is being collected has changed.

Interface

* Init()
  This is the initialization step performed at startup. This method should
  block until the index is ready to handle searches.

* Stop():
 This will be called when metrictank is shutting down.

* AddOrUpdate(*schema.MetricData, int32) Archive:
  Every metric received will result in a call to this method to ensure the
  metric has been added to the index. The method is passed the metricData
  payload and the partition id of the metric

* Get(string) (Archive, bool):
  This method should return the MetricDefintion with the passed Id.

* GetPath(string) (Archive, bool) []Archive:
  This method should return the archives under the given path

* List(int) []Archive:
  This method should return all MetricDefinitions for the passed OrgId.  If the
  passed OrgId is "-1", then all metricDefinitions across all organisations
  should be returned.

* Find(int, string, int64) ([]Node, error):
  This method provides searches.  The method is passed an OrgId, a query
  pattern and a unix timestamp. Searches should return all nodes that match for
  the given OrgId and OrgId -1.  The pattern should be handled in the same way
  Graphite would. see https://graphite.readthedocs.io/en/latest/render_api.html#paths-and-wildcards
  And the unix stimestamp is used to ignore series that have been stale since
  the timestamp.

* Delete(int, string) ([]Archive, error):
  This method is used for deleting items from the index. The method is passed
  an OrgId and a query pattern.  If the pattern matches a branch node, then
  all leaf nodes on that branch should also be deleted. So if the pattern is
  "*", all items in the index should be deleted.  A copy of all of the
  metricDefinitions deleted are returned.

* Prune(int, time.Time) ([]Archive, error):
  This method should delete all metrics from the index for the passed org where
  the last time the metric was seen is older then the passed timestamp. If the org
  passed is -1, then the all orgs should be examined for stale metrics to be deleted.
  The method returns a list of the metricDefinitions deleted from the index and any
  error encountered.

* TagList(int) []string:
  This method returns a list of all tag keys associated with the metrics of a given
  organization.

* Tag(int, string, int64) map[string]uint32:
  This method returns a list of all values associated with a given tag key in the
  given org. The occurences of each value is counted and the count is referred to by
  the series ids in the returned map. If the third parameter is > 0 then the metrics
  will be filtered and only those of which the LastUpdate time is >= the from
  timestamp will be considered while the others are being ignored.

* IdsByTagExpression(int, []string, int64) ([]string, error):
  This method takes a list of expressions in the format key=value.
  The allowed operators are: =, !=, =~, !=~.
  It returns a slice of IDs that match the given conditions, the conditions are
  logically AND-ed. If the third argument is > 0 then the results will be filtered
  and only those where the LastUpdate time is >= from will be returned as results.
*/

type MetricIndex interface {
	Init() error
	Stop()
	AddOrUpdate(*schema.MetricData, int32) Archive
	Get(string) (Archive, bool)
	GetPath(int, string) []Archive
	Delete(int, string) ([]Archive, error)
	Find(int, string, int64) ([]Node, error)
	List(int) []Archive
	Prune(int, time.Time) ([]Archive, error)
	TagList(int) []string
	Tag(int, string, int64) map[string]uint32
	IdsByTagExpressions(int, []string, int64) (map[MetricID]struct{}, error)
}
<commit_msg>formatting of metric id string<commit_after>// The idx package provides a metadata index for metrics

package idx

import (
	"encoding/hex"
	"errors"
	"fmt"
	"strconv"
	"strings"
	"time"

	schema "gopkg.in/raintank/schema.v1"
)

var (
	BothBranchAndLeaf  = errors.New("node can't be both branch and leaf")
	BranchUnderLeaf    = errors.New("can't add branch under leaf")
	errInvalidQuery    = errors.New("invalid query")
	errInvalidIdString = errors.New("invalid ID string")
)

//go:generate msgp
type Node struct {
	Path        string
	Leaf        bool
	Defs        []Archive
	HasChildren bool
}

type Archive struct {
	schema.MetricDefinition
	SchemaId uint16 // index in mdata.schemas (not persisted)
	AggId    uint16 // index in mdata.aggregations (not persisted)
	LastSave uint32 // last time the metricDefinition was saved to a backend store (cassandra)
}

type MetricID struct {
	org int
	key [16]byte
}

func NewMetricIDFromString(s string) (MetricID, error) {
	id := MetricID{}
	err := id.FromString(s)
	return id, err
}

func (id *MetricID) FromString(s string) error {
	splits := strings.Split(s, ".")
	if len(splits) != 2 || len(splits[1]) != 32 {
		return errInvalidIdString
	}

	var err error
	id.org, err = strconv.Atoi(splits[0])
	if err != nil {
		return err
	}

	dst := make([]byte, 16)
	n, err := hex.Decode(dst, []byte(splits[1]))
	if n != 16 {
		return errInvalidIdString
	}
	copy(id.key[:], dst)
	return nil
}

func (id *MetricID) String() string {
	return fmt.Sprintf("%d.%x", id.org, id.key)
}

// used primarily by tests, for convenience
func NewArchiveBare(name string) Archive {
	return Archive{
		MetricDefinition: schema.MetricDefinition{
			Name: name,
		},
	}
}

/*
Currently the index is solely used for supporting Graphite style queries.
So, the index only needs to be able to search by a pattern that matches the
MetricDefinition.Name field. In future we plan to extend the searching
capabilities to include the other fields in the definition.

Note:

* metrictank is a multi-tenant system where different orgs cannot see each
  other's data

* any given metric may appear multiple times, under different organisations

* Each metric path can be mapped to multiple metricDefinitions in the case that
  fields other then the Name vary.  The most common occurrence of this is when
  the Interval at which the metric is being collected has changed.

Interface

* Init()
  This is the initialization step performed at startup. This method should
  block until the index is ready to handle searches.

* Stop():
 This will be called when metrictank is shutting down.

* AddOrUpdate(*schema.MetricData, int32) Archive:
  Every metric received will result in a call to this method to ensure the
  metric has been added to the index. The method is passed the metricData
  payload and the partition id of the metric

* Get(string) (Archive, bool):
  This method should return the MetricDefintion with the passed Id.

* GetPath(string) (Archive, bool) []Archive:
  This method should return the archives under the given path

* List(int) []Archive:
  This method should return all MetricDefinitions for the passed OrgId.  If the
  passed OrgId is "-1", then all metricDefinitions across all organisations
  should be returned.

* Find(int, string, int64) ([]Node, error):
  This method provides searches.  The method is passed an OrgId, a query
  pattern and a unix timestamp. Searches should return all nodes that match for
  the given OrgId and OrgId -1.  The pattern should be handled in the same way
  Graphite would. see https://graphite.readthedocs.io/en/latest/render_api.html#paths-and-wildcards
  And the unix stimestamp is used to ignore series that have been stale since
  the timestamp.

* Delete(int, string) ([]Archive, error):
  This method is used for deleting items from the index. The method is passed
  an OrgId and a query pattern.  If the pattern matches a branch node, then
  all leaf nodes on that branch should also be deleted. So if the pattern is
  "*", all items in the index should be deleted.  A copy of all of the
  metricDefinitions deleted are returned.

* Prune(int, time.Time) ([]Archive, error):
  This method should delete all metrics from the index for the passed org where
  the last time the metric was seen is older then the passed timestamp. If the org
  passed is -1, then the all orgs should be examined for stale metrics to be deleted.
  The method returns a list of the metricDefinitions deleted from the index and any
  error encountered.

* TagList(int) []string:
  This method returns a list of all tag keys associated with the metrics of a given
  organization.

* Tag(int, string, int64) map[string]uint32:
  This method returns a list of all values associated with a given tag key in the
  given org. The occurences of each value is counted and the count is referred to by
  the series ids in the returned map. If the third parameter is > 0 then the metrics
  will be filtered and only those of which the LastUpdate time is >= the from
  timestamp will be considered while the others are being ignored.

* IdsByTagExpression(int, []string, int64) ([]string, error):
  This method takes a list of expressions in the format key=value.
  The allowed operators are: =, !=, =~, !=~.
  It returns a slice of IDs that match the given conditions, the conditions are
  logically AND-ed. If the third argument is > 0 then the results will be filtered
  and only those where the LastUpdate time is >= from will be returned as results.
*/

type MetricIndex interface {
	Init() error
	Stop()
	AddOrUpdate(*schema.MetricData, int32) Archive
	Get(string) (Archive, bool)
	GetPath(int, string) []Archive
	Delete(int, string) ([]Archive, error)
	Find(int, string, int64) ([]Node, error)
	List(int) []Archive
	Prune(int, time.Time) ([]Archive, error)
	TagList(int) []string
	Tag(int, string, int64) map[string]uint32
	IdsByTagExpressions(int, []string, int64) (map[MetricID]struct{}, error)
}
<|endoftext|>


<commit_before>package fpdf

import(
	"fmt"

	"github.com/jung-kurt/gofpdf"
	"github.com/skypies/geo"
	fdb "github.com/skypies/flightdb2"
)

var (
	RedRGB   = []int{0xff, 0, 0}
	GreenRGB = []int{0, 0xff, 0}
	BlueRGB  = []int{0, 0, 0xff}
)

type DescentPdf struct {
	Grids        map[string]*BaseGrid

	AltitudeMin  float64
	AltitudeMax  float64
	OriginPoint  geo.Latlong
	OriginLabel  string
	LengthNM     float64
	ColorScheme  // embedded

	LineThickness  float64
	LineOpacity    float64 // 0.0==transparent, 1.0==opaque

	*gofpdf.Fpdf // Embedded pointer

	Caption      string
	Debug        string
	ShowDebug    bool
}

// {{{ dp.Init

func (g *DescentPdf)Init() {
	g.Fpdf = gofpdf.New("P", "mm", "Letter", "")
	g.AddPage()
	g.SetFont("Arial", "", 10)

	if g.LineThickness == 0.0 { g.LineThickness = 0.25 }
	if g.LineOpacity == 0.0   { g.LineOpacity = 1.0 }

	g.Grids = map[string]*BaseGrid{}

	u,v := 22.0,35.0 // The top-left origin, in PDF space; increment as we go down the page

	incompleteGrid := func() *BaseGrid {
		return &BaseGrid{
			Fpdf: g.Fpdf,
			OffsetU: u,
			OffsetV: v,
			W: 170,
			MinX: 0,
			MaxX: g.LengthNM,
			XGridlineEvery: 10,
			Clip: true, // set to false for debugging datasets
			InvertX: true,  // Descend to origin, on the right
		}
	}

	ng := incompleteGrid()
	g.Grids["altitude"] = ng
	ng.LineColor = RedRGB
	ng.H = 100
	ng.MinY = g.AltitudeMin
	ng.MaxY = g.AltitudeMax
	ng.YGridlineEvery = 5000
	ng.XTickFmt = "%.0fNM"
	ng.YTickFmt = "%.0fft"
	ng.XTickOtherSide = true
	ng.YTickOtherSide = true

	v += 110

	ng = incompleteGrid()
	g.Grids["groundspeed"] = ng
	ng.LineColor = RedRGB
	ng.H = 50
	ng.MinY = 0
	ng.MaxY = 400
	ng.YGridlineEvery = 100
	ng.YTickFmt = "%.0f knots"

	// This is overlayed into the same grid as groundspeed
	ng = incompleteGrid()
	g.Grids["groundacceleration"] = ng
	ng.LineColor = BlueRGB
	ng.H = 50
	ng.MinY = -6
	ng.MaxY = 6
	ng.YGridlineEvery = 3
	ng.YTickFmt = "%.0f knots/sec"
	ng.YTickOtherSide = true
	ng.NoGridlines = true

	v += 60

	ng = incompleteGrid()
	g.Grids["verticalspeed"] = ng
	ng.LineColor = RedRGB
	ng.H = 50
	ng.MinY = -2500
	ng.MaxY =  2500
	ng.YGridlineEvery = 1250
	ng.YTickFmt = "%.0f feet/min"

	// This is overlayed into the same grid as verticalspeed
	ng = incompleteGrid()
	g.Grids["verticalacceleration"] = ng
	ng.LineColor = BlueRGB
	ng.H = 50
	ng.MinY = -100
	ng.MaxY =  100
	ng.YGridlineEvery = 50
	ng.YTickFmt = "%.0f fpm/sec"
	ng.YTickOtherSide = true
	ng.NoGridlines = true

/*
	g.AltitudeGrid = BaseGrid{
			Fpdf: g.Fpdf,
			OffsetU: 10,
			OffsetV: 10,
			W: 245,
			H: 100,
			MinX: 0,
			MaxX: g.LengthNM,
			MinY: g.AltitudeMin,
			MaxY: g.AltitudeMax,
			XGridlineEvery: 10,
			YGridlineEvery: 5000,
			XTickFmt: "%.0fNM",
			YTickFmt: "%.0fft",
			InvertX: true,  // Descend to origin, on the right
		}
*/
}

// }}}

// {{{ dp.DrawFrames

func (g DescentPdf)DrawFrames() {
	for _,grid := range g.Grids {
		grid.DrawGridlines()
	}
}

// }}}
// {{{ dp.DrawCaption

func (g DescentPdf)DrawCaption() {
	title := g.Caption

	if g.ShowDebug {
		title += "--DEBUG--\n" + g.Debug
	}

	g.SetTextColor(0x50, 0x70, 0xc0)
	g.MoveTo(10, 10)
	g.MultiCell(0, 4, title, "", "", false)
	g.DrawPath("D")
}

// }}}
// {{{ dp.DrawColorSchemeKeys

func (g DescentPdf)DrawColorSchemeKeys() {
	for _,grid := range g.Grids {
		grid.DrawColorSchemeKey()
	}
}

// }}}

// {{{ dp.DrawTrackWithDistFunc

type DistanceFunc func(tp fdb.Trackpoint) (float64, float64, []int)

func (g *DescentPdf)DrawTrackWithDistFunc(t fdb.Track, f DistanceFunc, colorscheme ColorScheme) {
	g.SetDrawColor(0xff, 0x00, 0x00)
	g.SetAlpha(g.LineOpacity, "")

	if len(t) == 0 { return }

	for i,_ := range t[1:] {
		x1,alt1,_ := f(t[i])
		x2,alt2,rgb := f(t[i+1])

		g.SetLineWidth(g.LineThickness)
		g.SetDrawColor(rgb[0], rgb[1], rgb[2])

		if grid,exists := g.Grids["altitude"]; exists {
			g.SetDrawColor(rgb[0], rgb[1], rgb[2])
			grid.Line(x1,alt1, x2,alt2)
		}

		// We can re-use the dist values (x1,x2), and plot other functions on the trackpoints
		if grid,exists := g.Grids["groundspeed"]; exists {
			grid.Line(x1,t[i].GroundSpeed, x2,t[i+1].GroundSpeed)
		}
		if grid,exists := g.Grids["groundacceleration"]; exists {
			grid.Line(x1,t[i].GroundAccelerationKPS, x2,t[i+1].GroundAccelerationKPS)
		}
		if grid,exists := g.Grids["verticalspeed"]; exists {
			grid.Line(x1,t[i].VerticalSpeedFPM, x2,t[i+1].VerticalSpeedFPM)
		}
		if grid,exists := g.Grids["verticalacceleration"]; exists {
			grid.Line(x1,t[i].VerticalAccelerationFPMPS, x2,t[i+1].VerticalAccelerationFPMPS)
		}

		g.Debug += fmt.Sprintf("%3d: %.1f, %.1f\n", i, t[i].VerticalSpeedFPM,
			t[i].VerticalAccelerationFPMPS)
	}

	g.DrawPath("D")
	g.SetAlpha(1.0, "")
}

// }}}
// {{{ dp.DrawTrackAsDistanceFromOrigin

// Consider distance as being simply the distance from the origin, and plot against altitude.
// The less the aircraft flies in a straight line to the origin, the less useful this will be.
// (E.g. if an aircraft descends in a spiral, it will plot as a zig-zag, getting closer then
// further away as it descends.)

func (g *DescentPdf)DrawTrackAsDistanceFromOrigin(t fdb.Track) {
	trackpointToXY := func(tp fdb.Trackpoint) (float64, float64, []int) {
		rgb := []int{0,0,0}
		if g.ColorScheme == ByPlotKind { rgb = []int{0,250,0} }
		return tp.DistNM(g.OriginPoint), tp.IndicatedAltitude, rgb
	}

	g.Debug += fmt.Sprintf("DrawTrackAsDistanceFromOrigin\n")

	g.DrawTrackWithDistFunc(t, trackpointToXY, g.ColorScheme)
}

// }}}
// {{{ dp.DrawTrackAsDistanceAlongPath

// Consider distance to be distance travelled along the path.
// (E.g. if the aircraft descends in a steady spiral, we'll plot the 'unrolled' version as a
// long steady line.)
func (g *DescentPdf)DrawTrackAsDistanceAlongPath(t fdb.Track) {
	// We want to render this working backwards from the end, so descents can line up together.
	// That means we're interested in each point's distance travelled in relation to the end point.
	g.Debug += fmt.Sprintf("DrawTrackAsDistanceAlongPath\n")
	iClosest := t.ClosestTo(g.OriginPoint)
	if iClosest < 0 { return }
	endpointKM := t[iClosest].DistanceTravelledKM

	g.Debug += fmt.Sprintf("* endKM=%.2f\n", endpointKM)

	trackpointToXY := func(tp fdb.Trackpoint) (float64, float64, []int) {
		distToGoKM := endpointKM - tp.DistanceTravelledKM
		distToGoNM := distToGoKM * geo.KNauticalMilePerKM

		rgb := []int{0,0,0}
		if g.ColorScheme == ByPlotKind { rgb = []int{250,0,0} }

		//g.Debug += fmt.Sprintf("%s\n", tp)

		return distToGoNM, tp.IndicatedAltitude, rgb
	}

	g.DrawTrackWithDistFunc(t, trackpointToXY, g.ColorScheme)
}

// }}}

// {{{ -------------------------={ E N D }=----------------------------------

// Local variables:
// folded-file: t
// end:

// }}}
<commit_msg>Assume linear flight to airport for incomplete tracks, to get better multi-flight PDFs (i.e. they all line up)<commit_after>package fpdf

import(
	"fmt"

	"github.com/jung-kurt/gofpdf"
	"github.com/skypies/geo"
	fdb "github.com/skypies/flightdb2"
)

var (
	RedRGB   = []int{0xff, 0, 0}
	GreenRGB = []int{0, 0xff, 0}
	BlueRGB  = []int{0, 0, 0xff}
)

type DescentPdf struct {
	Grids        map[string]*BaseGrid

	AltitudeMin  float64
	AltitudeMax  float64
	OriginPoint  geo.Latlong
	OriginLabel  string
	LengthNM     float64
	ColorScheme  // embedded

	LineThickness  float64
	LineOpacity    float64 // 0.0==transparent, 1.0==opaque

	*gofpdf.Fpdf // Embedded pointer

	Caption      string
	Debug        string
	ShowDebug    bool
}

// {{{ dp.Init

func (g *DescentPdf)Init() {
	g.Fpdf = gofpdf.New("P", "mm", "Letter", "")
	g.AddPage()
	g.SetFont("Arial", "", 10)

	if g.LineThickness == 0.0 { g.LineThickness = 0.25 }
	if g.LineOpacity == 0.0   { g.LineOpacity = 1.0 }

	g.Grids = map[string]*BaseGrid{}

	u,v := 22.0,35.0 // The top-left origin, in PDF space; increment as we go down the page

	incompleteGrid := func() *BaseGrid {
		return &BaseGrid{
			Fpdf: g.Fpdf,
			OffsetU: u,
			OffsetV: v,
			W: 170,
			MinX: 0,
			MaxX: g.LengthNM,
			XGridlineEvery: 10,
			Clip: true, // set to false for debugging datasets
			InvertX: true,  // Descend to origin, on the right
		}
	}

	ng := incompleteGrid()
	g.Grids["altitude"] = ng
	ng.LineColor = RedRGB
	ng.H = 100
	ng.MinY = g.AltitudeMin
	ng.MaxY = g.AltitudeMax
	ng.YGridlineEvery = 5000
	ng.XTickFmt = "%.0fNM"
	ng.YTickFmt = "%.0fft"
	ng.XTickOtherSide = true
	ng.YTickOtherSide = true

	v += 110

	ng = incompleteGrid()
	g.Grids["groundspeed"] = ng
	ng.LineColor = RedRGB
	ng.H = 50
	ng.MinY = 0
	ng.MaxY = 400
	ng.YGridlineEvery = 100
	ng.YTickFmt = "%.0f knots"

	// This is overlayed into the same grid as groundspeed
	ng = incompleteGrid()
	g.Grids["groundacceleration"] = ng
	ng.LineColor = BlueRGB
	ng.H = 50
	ng.MinY = -6
	ng.MaxY = 6
	ng.YGridlineEvery = 3
	ng.YTickFmt = "%.0f knots/sec"
	ng.YTickOtherSide = true
	ng.NoGridlines = true

	v += 60

	ng = incompleteGrid()
	g.Grids["verticalspeed"] = ng
	ng.LineColor = RedRGB
	ng.H = 50
	ng.MinY = -2500
	ng.MaxY =  2500
	ng.YGridlineEvery = 1250
	ng.YTickFmt = "%.0f feet/min"

	// This is overlayed into the same grid as verticalspeed
	ng = incompleteGrid()
	g.Grids["verticalacceleration"] = ng
	ng.LineColor = BlueRGB
	ng.H = 50
	ng.MinY = -100
	ng.MaxY =  100
	ng.YGridlineEvery = 50
	ng.YTickFmt = "%.0f fpm/sec"
	ng.YTickOtherSide = true
	ng.NoGridlines = true

/*
	g.AltitudeGrid = BaseGrid{
			Fpdf: g.Fpdf,
			OffsetU: 10,
			OffsetV: 10,
			W: 245,
			H: 100,
			MinX: 0,
			MaxX: g.LengthNM,
			MinY: g.AltitudeMin,
			MaxY: g.AltitudeMax,
			XGridlineEvery: 10,
			YGridlineEvery: 5000,
			XTickFmt: "%.0fNM",
			YTickFmt: "%.0fft",
			InvertX: true,  // Descend to origin, on the right
		}
*/
}

// }}}

// {{{ dp.DrawFrames

func (g DescentPdf)DrawFrames() {
	for _,grid := range g.Grids {
		grid.DrawGridlines()
	}
}

// }}}
// {{{ dp.DrawCaption

func (g DescentPdf)DrawCaption() {
	title := g.Caption

	if g.ShowDebug {
		title += "--DEBUG--\n" + g.Debug
	}

	g.SetTextColor(0x50, 0x70, 0xc0)
	g.MoveTo(10, 10)
	g.MultiCell(0, 4, title, "", "", false)
	g.DrawPath("D")
}

// }}}
// {{{ dp.DrawColorSchemeKeys

func (g DescentPdf)DrawColorSchemeKeys() {
	for _,grid := range g.Grids {
		grid.DrawColorSchemeKey()
	}
}

// }}}

// {{{ dp.DrawTrackWithDistFunc

type DistanceFunc func(tp fdb.Trackpoint) (float64, float64, []int)

func (g *DescentPdf)DrawTrackWithDistFunc(t fdb.Track, f DistanceFunc, colorscheme ColorScheme) {
	g.SetDrawColor(0xff, 0x00, 0x00)
	g.SetAlpha(g.LineOpacity, "")

	if len(t) == 0 { return }

	for i,_ := range t[1:] {
		x1,alt1,_ := f(t[i])
		x2,alt2,rgb := f(t[i+1])

		g.SetLineWidth(g.LineThickness)
		g.SetDrawColor(rgb[0], rgb[1], rgb[2])

		if grid,exists := g.Grids["altitude"]; exists {
			g.SetDrawColor(rgb[0], rgb[1], rgb[2])
			grid.Line(x1,alt1, x2,alt2)
		}

		// We can re-use the dist values (x1,x2), and plot other functions on the trackpoints
		if grid,exists := g.Grids["groundspeed"]; exists {
			grid.Line(x1,t[i].GroundSpeed, x2,t[i+1].GroundSpeed)
		}
		if grid,exists := g.Grids["groundacceleration"]; exists {
			grid.Line(x1,t[i].GroundAccelerationKPS, x2,t[i+1].GroundAccelerationKPS)
		}
		if grid,exists := g.Grids["verticalspeed"]; exists {
			grid.Line(x1,t[i].VerticalSpeedFPM, x2,t[i+1].VerticalSpeedFPM)
		}
		if grid,exists := g.Grids["verticalacceleration"]; exists {
			grid.Line(x1,t[i].VerticalAccelerationFPMPS, x2,t[i+1].VerticalAccelerationFPMPS)
		}

		g.Debug += fmt.Sprintf("%3d: %.1f, %.1f\n", i, t[i].VerticalSpeedFPM,
			t[i].VerticalAccelerationFPMPS)
	}

	g.DrawPath("D")
	g.SetAlpha(1.0, "")
}

// }}}
// {{{ dp.DrawTrackAsDistanceFromOrigin

// Consider distance as being simply the distance from the origin, and plot against altitude.
// The less the aircraft flies in a straight line to the origin, the less useful this will be.
// (E.g. if an aircraft descends in a spiral, it will plot as a zig-zag, getting closer then
// further away as it descends.)

func (g *DescentPdf)DrawTrackAsDistanceFromOrigin(t fdb.Track) {
	trackpointToXY := func(tp fdb.Trackpoint) (float64, float64, []int) {
		rgb := []int{0,0,0}
		if g.ColorScheme == ByPlotKind { rgb = []int{0,250,0} }
		return tp.DistNM(g.OriginPoint), tp.IndicatedAltitude, rgb
	}

	g.Debug += fmt.Sprintf("DrawTrackAsDistanceFromOrigin\n")

	g.DrawTrackWithDistFunc(t, trackpointToXY, g.ColorScheme)
}

// }}}
// {{{ dp.DrawTrackAsDistanceAlongPath

// Consider distance to be distance travelled along the path.
// (E.g. if the aircraft descends in a steady spiral, we'll plot the 'unrolled' version as a
// long steady line.)
func (g *DescentPdf)DrawTrackAsDistanceAlongPath(t fdb.Track) {
	// We want to render this working backwards from the end, so descents can line up together.
	// That means we're interested in each point's distance travelled in relation to the end point.
	g.Debug += fmt.Sprintf("DrawTrackAsDistanceAlongPath\n")
	iClosest := t.ClosestTo(g.OriginPoint)
	if iClosest < 0 { return }
	endpointKM := t[iClosest].DistanceTravelledKM

	// If the closest point isn't all that close, assume linear flight from it to the origin
	offsetKM := t[iClosest].DistKM(g.OriginPoint)

	g.Debug += fmt.Sprintf("* endKM=%.2f, offsetKM=%.2f, index=%d\n", endpointKM, offsetKM, iClosest)

	trackpointToXY := func(tp fdb.Trackpoint) (float64, float64, []int) {
		distToGoKM := endpointKM - tp.DistanceTravelledKM + offsetKM
		distToGoNM := distToGoKM * geo.KNauticalMilePerKM

		rgb := []int{0,0,0}
		if g.ColorScheme == ByPlotKind { rgb = []int{250,0,0} }

		//g.Debug += fmt.Sprintf("%s\n", tp)

		return distToGoNM, tp.IndicatedAltitude, rgb
	}

	g.DrawTrackWithDistFunc(t, trackpointToXY, g.ColorScheme)
}

// }}}

// {{{ -------------------------={ E N D }=----------------------------------

// Local variables:
// folded-file: t
// end:

// }}}
<|endoftext|>


<commit_before>package helpers

var (
	// PrestVersionNumber repesemts prest version.
	PrestVersionNumber = "1.0.5"
	// CommitHash for version
	CommitHash string
)

// PrestReleaseVersion is same as pREST Version.
func PrestReleaseVersion() string {
	return PrestVersionNumber
}
<commit_msg>generate pREST version 1.0.6<commit_after>package helpers

var (
	// PrestVersionNumber repesemts prest version.
	PrestVersionNumber = "1.0.6"
	// CommitHash for version
	CommitHash string
)

// PrestReleaseVersion is same as pREST Version.
func PrestReleaseVersion() string {
	return PrestVersionNumber
}
<|endoftext|>


<commit_before>package bot

import (
	"crypto/tls"
	"errors"
	"fmt"
	"io"
	"log"
	"net"
	"strings"

	"github.com/BurntSushi/toml"
	"github.com/nightlyone/lockfile"

	"github.com/belak/irc"
)

type coreConfig struct {
	PidFile string

	Nick string
	User string
	Name string
	Pass string

	Host        string
	TLS         bool
	TLSNoVerify bool

	Cmds   []string
	Prefix string

	Plugins []string

	Quiet bool
}

// A Bot is our wrapper around the irc.Client. It could be used for a
// general client, but the provided convenience functions are designed
// around using this package to write a bot.
type Bot struct {
	// Everything needed for plugins
	BasicMux   *BasicMux
	CommandMux *CommandMux
	MentionMux *MentionMux

	Auth AuthProvider

	Plugins map[string]interface{}

	// Config stuff
	confValues map[string]toml.Primitive
	md         toml.MetaData

	// Internal things
	conn                *irc.Conn
	connected           bool
	currentNick         string
	config              *coreConfig
	err                 error
	initialCapList      map[string]bool // This is a map so we avoid dupes
	initialCapResponses int
	loadingPlugins      map[string]bool
}

// NewBot will return a new Bot given the name of a toml config file.
func NewBot(conf string) (*Bot, error) {
	b := &Bot{
		NewBasicMux(),
		nil,
		NewMentionMux(),
		&nullAuthProvider{},
		make(map[string]interface{}),
		make(map[string]toml.Primitive),
		toml.MetaData{},
		nil,
		false,
		"",
		&coreConfig{},
		nil,
		make(map[string]bool),
		0,
		make(map[string]bool),
	}

	// Decode the file, but leave all the config sections intact
	// so we can decode those later.
	b.md, b.err = toml.DecodeFile(conf, b.confValues)
	if b.err != nil {
		return nil, b.err
	}

	// Load up the core config
	b.err = b.Config("core", b.config)
	if b.err != nil {
		return nil, b.err
	}

	// Load up the command mux
	b.CommandMux = NewCommandMux(b.config.Prefix)

	// Hook up the other muxes
	b.BasicMux.Event("PRIVMSG", b.CommandMux.HandleEvent)
	b.BasicMux.Event("PRIVMSG", b.MentionMux.HandleEvent)

	return b, nil
}

func (b *Bot) CapReq(caps ...string) {
	if b.connected {
		// If we're already connected, we don't care about
		// negotiation. We just want to ask for it.
		b.Send(&irc.Message{
			Prefix:  &irc.Prefix{},
			Command: "CAP",
			Params: []string{
				"REQ",
				strings.Join(caps, " "),
			},
		})
	} else {
		// TODO: This is not technically correct. We should be
		// handling requests which start with a -
		for _, cap := range caps {
			b.initialCapList[cap] = true
		}
	}
}

// CurrentNick returns the current nick of the bot.
func (b *Bot) CurrentNick() string {
	return b.currentNick
}

// Config will decode the config section for the given name into the
// given interface{}
func (b *Bot) Config(name string, c interface{}) error {
	if v, ok := b.confValues[name]; ok {
		return b.md.PrimitiveDecode(v, c)
	}
	return fmt.Errorf("Config section for %q missing", name)
}

// Send is a simple function to send an IRC event
func (b *Bot) Send(m *irc.Message) {
	if b.err != nil {
		return
	}

	b.conn.WriteMessage(m)
}

// Reply to an irc.Message with a convenience wrapper around
// fmt.Sprintf
func (b *Bot) Reply(m *irc.Message, format string, v ...interface{}) error {
	if len(m.Params) < 1 || len(m.Params[0]) < 1 {
		return errors.New("Invalid IRC message")
	}

	target := m.Prefix.Name
	if m.FromChannel() {
		target = m.Params[0]
	}

	b.Send(&irc.Message{
		Prefix:  &irc.Prefix{},
		Command: "PRIVMSG",
		Params: []string{
			target,
			fmt.Sprintf(format, v...),
		},
	})

	return nil
}

// MentionReply acts the same as Bot.Reply but it will prefix it with
// the user's nick if we are in a channel.
func (b *Bot) MentionReply(m *irc.Message, format string, v ...interface{}) error {
	if m.FromChannel() {
		format = "%s: " + format
		v = prepend(v, m.Prefix.Name)
	}

	return b.Reply(m, format, v...)
}

// CTCPReply is a convenience function to respond to CTCP requests.
func (b *Bot) CTCPReply(m *irc.Message, format string, v ...interface{}) error {
	if m.Command != "CTCP" {
		return errors.New("Invalid CTCP message")
	}

	b.Send(&irc.Message{
		Prefix:  &irc.Prefix{},
		Command: "NOTICE",
		Params: []string{
			m.Prefix.Name,
			fmt.Sprintf(format, v...),
		},
	})
	return nil
}

// HasPerm is a convenience function for checking a user's permissions
func (b *Bot) HasPerm(m *irc.Message, perm string) bool {
	user := b.Auth.LookupUser(b, m.Prefix)

	return user.HasPerm(b, perm)
}

func (b *Bot) handshake() {
	b.conn.Writef("CAP END")
	b.conn.Writef("NICK %s", b.config.Nick)
	b.conn.Writef("USER %s 0.0.0.0 0.0.0.0 :%s", b.config.User, b.config.Name)
}

func (b *Bot) mainLoop() error {
	// Send the initial connection info
	if len(b.config.Pass) > 0 {
		b.conn.Writef("PASS %s", b.config.Pass)
	}

	// Set the nick setting as the current nick
	b.currentNick = b.config.Nick

	// If we have any capabilities, we need to send requests for
	// them. Note that we send multiple requests because it makes
	// parsing the ACK and NAK commands a lot simpler.
	if len(b.initialCapList) > 0 {
		var caps []string
		for cap := range b.initialCapList {
			caps = append(caps, cap)
			b.Send(&irc.Message{
				Prefix:  &irc.Prefix{},
				Command: "CAP",
				Params:  []string{"REQ", cap},
			})
		}
	} else {
		// We can only handshake now if we didn't have any
		// caps we wanted responses to.
		b.handshake()
	}

	var m *irc.Message
	for {
		m, b.err = b.conn.ReadMessage()
		if b.err != nil {
			break
		}

		// Internal handlers to make sure we track the
		// currentNick correctly and send PONGs
		if m.Command == "001" {
			log.Println("Connected")
			b.connected = true

			for _, v := range b.config.Cmds {
				b.Write(v)
			}

		} else if m.Command == "CAP" {
			if len(m.Params) > 1 {
				numParams := len(m.Params)
				if m.Params[numParams-2] == "ACK" {
					// Because we send each CAP
					// individually, we shouldn't
					// need to do anything here.

					b.initialCapResponses++
				} else if m.Params[numParams-2] == "NAK" {
					return fmt.Errorf("Got CAP NAK for %s", m.Params[1])
				}
			}

			if b.initialCapResponses >= len(b.initialCapList) {
				// Now that we've got all the
				// responses back that we needed, we
				// can continue the initial dance.
				b.handshake()
			}
		} else if m.Command == "NICK" {
			if m.Prefix.Name == b.currentNick && len(m.Params) > 0 {
				b.currentNick = m.Params[0]
			}
		} else if m.Command == "PING" {
			b.conn.Writef("PONG :%s", m.Trailing())
		} else if m.Command == "001" {
			b.currentNick = m.Params[0]
		} else if m.Command == "437" || m.Command == "433" {
			b.currentNick = b.currentNick + "_"
			b.conn.Writef("NICK %s", b.currentNick)
		}

		b.BasicMux.HandleEvent(b, m)

		// TODO: Make this work better
		if b.err != nil {
			break
		}
	}

	return b.err
}

// Write will write an raw IRC message to the stream
func (b *Bot) Write(line string) {
	b.conn.Write(line)
}

// Writef is a convenience method around fmt.Sprintf and Bot.Write
func (b *Bot) Writef(format string, args ...interface{}) {
	b.conn.Writef(format, args...)
}

// PluginLoaded will return true if a plugin is loaded and false otherwise
func (b *Bot) PluginLoaded(name string) bool {
	_, ok := b.Plugins[name]
	return ok
}

// LoadPlugin will ensure a plugin is loaded. It is designed to be
// usable in other plugins, so they can ensure plugins they depend on
// are loaded before using them.
func (b *Bot) LoadPlugin(name string) error {
	// We don't need to load the plugin if it's already loaded
	if b.PluginLoaded(name) {
		return nil
	}

	// Ensure the plugin exists
	factory, ok := plugins[name]
	if !ok {
		return fmt.Errorf("Plugin %s does not exist", name)
	}

	// If we're trying to load this plugin already, this is a
	// circular load and we should bail.
	if v := b.loadingPlugins[name]; v {
		return fmt.Errorf("Plugin %s getting loaded circularly", name)
	}

	// Set a flag stating that we're loading this plugin
	b.loadingPlugins[name] = true

	// Actually load the plugin
	plugin, err := factory(b)
	if err != nil {
		return fmt.Errorf("Plugin %s failed to load", name)
	}

	// Save the value returned from the plugin factory.
	b.Plugins[name] = plugin

	// Unset the flag saying we're loading this because we're done now.
	delete(b.loadingPlugins, name)

	return nil
}

// Run starts the bot and loops until it dies
func (b *Bot) Run() error {
	var err error
	// Load all the plugins we need. If there were plugins
	// specified in the config, just load those. Otherwise load
	// ALL of them.
	if len(b.config.Plugins) != 0 {
		for _, name := range b.config.Plugins {
			err = b.LoadPlugin(name)
			if err != nil {
				return err
			}
		}
	} else {
		for name := range plugins {
			err = b.LoadPlugin(name)
			if err != nil {
				return err
			}
		}
	}

	// If we have a pidfile configured, create it and write the PID
	if b.config.PidFile != "" {
		l, err := lockfile.New(b.config.PidFile)
		if err != nil {
			return err
		}

		err = l.TryLock()
		if err != nil {
			return err
		}

		defer l.Unlock()
	}

	// The ReadWriteCloser will contain either a *net.Conn or *tls.Conn
	var c io.ReadWriteCloser
	if b.config.TLS {
		conf := &tls.Config{
			InsecureSkipVerify: b.config.TLSNoVerify,
		}

		c, err = tls.Dial("tcp", b.config.Host, conf)
	} else {
		c, err = net.Dial("tcp", b.config.Host)
	}

	if err != nil {
		return err
	}

	// Create a client from the connection we've just opened
	b.conn = irc.NewConn(c)

	b.conn.DebugCallback = func(line string) {
		if !b.config.Quiet {
			log.Println(line)
		}
	}

	// Start the main loop
	return b.mainLoop()
}
<commit_msg>Fix up some comments<commit_after>package bot

import (
	"crypto/tls"
	"errors"
	"fmt"
	"io"
	"log"
	"net"
	"strings"

	"github.com/BurntSushi/toml"
	"github.com/nightlyone/lockfile"

	"github.com/belak/irc"
)

type coreConfig struct {
	PidFile string

	Nick string
	User string
	Name string
	Pass string

	Host        string
	TLS         bool
	TLSNoVerify bool

	Cmds   []string
	Prefix string

	Plugins []string

	Quiet bool
}

// A Bot is our wrapper around the irc.Client. It could be used for a
// general client, but the provided convenience functions are designed
// around using this package to write a bot.
type Bot struct {
	// Everything needed for plugins
	BasicMux   *BasicMux
	CommandMux *CommandMux
	MentionMux *MentionMux

	Auth AuthProvider

	Plugins map[string]interface{}

	// Config stuff
	confValues map[string]toml.Primitive
	md         toml.MetaData

	// Internal things
	conn                *irc.Conn
	connected           bool
	currentNick         string
	config              *coreConfig
	err                 error
	initialCapList      map[string]bool // This is a map so we avoid dupes
	initialCapResponses int
	loadingPlugins      map[string]bool
}

// NewBot will return a new Bot given the name of a toml config file.
func NewBot(conf string) (*Bot, error) {
	b := &Bot{
		NewBasicMux(),
		nil,
		NewMentionMux(),
		&nullAuthProvider{},
		make(map[string]interface{}),
		make(map[string]toml.Primitive),
		toml.MetaData{},
		nil,
		false,
		"",
		&coreConfig{},
		nil,
		make(map[string]bool),
		0,
		make(map[string]bool),
	}

	// Decode the file, but leave all the config sections intact
	// so we can decode those later.
	b.md, b.err = toml.DecodeFile(conf, b.confValues)
	if b.err != nil {
		return nil, b.err
	}

	// Load up the core config
	b.err = b.Config("core", b.config)
	if b.err != nil {
		return nil, b.err
	}

	// Load up the command mux
	b.CommandMux = NewCommandMux(b.config.Prefix)

	// Hook up the other muxes
	b.BasicMux.Event("PRIVMSG", b.CommandMux.HandleEvent)
	b.BasicMux.Event("PRIVMSG", b.MentionMux.HandleEvent)

	return b, nil
}

// CapReq allows for plugins to request capabilities from the server. Note that
// we only support requiring the caps if the request is made before connection.
func (b *Bot) CapReq(caps ...string) {
	if b.connected {
		// If we're already connected, we don't care about
		// negotiation. We just want to ask for it.
		b.Send(&irc.Message{
			Prefix:  &irc.Prefix{},
			Command: "CAP",
			Params: []string{
				"REQ",
				strings.Join(caps, " "),
			},
		})
	} else {
		// TODO: This is not technically correct. We should be
		// handling requests which start with a -
		for _, cap := range caps {
			b.initialCapList[cap] = true
		}
	}
}

// CurrentNick returns the current nick of the bot.
func (b *Bot) CurrentNick() string {
	return b.currentNick
}

// Config will decode the config section for the given name into the
// given interface{}
func (b *Bot) Config(name string, c interface{}) error {
	if v, ok := b.confValues[name]; ok {
		return b.md.PrimitiveDecode(v, c)
	}
	return fmt.Errorf("Config section for %q missing", name)
}

// Send is a simple function to send an IRC event
func (b *Bot) Send(m *irc.Message) {
	if b.err != nil {
		return
	}

	b.conn.WriteMessage(m)
}

// Reply to an irc.Message with a convenience wrapper around
// fmt.Sprintf
func (b *Bot) Reply(m *irc.Message, format string, v ...interface{}) error {
	if len(m.Params) < 1 || len(m.Params[0]) < 1 {
		return errors.New("Invalid IRC message")
	}

	target := m.Prefix.Name
	if m.FromChannel() {
		target = m.Params[0]
	}

	b.Send(&irc.Message{
		Prefix:  &irc.Prefix{},
		Command: "PRIVMSG",
		Params: []string{
			target,
			fmt.Sprintf(format, v...),
		},
	})

	return nil
}

// MentionReply acts the same as Bot.Reply but it will prefix it with
// the user's nick if we are in a channel.
func (b *Bot) MentionReply(m *irc.Message, format string, v ...interface{}) error {
	if m.FromChannel() {
		format = "%s: " + format
		v = prepend(v, m.Prefix.Name)
	}

	return b.Reply(m, format, v...)
}

// CTCPReply is a convenience function to respond to CTCP requests.
func (b *Bot) CTCPReply(m *irc.Message, format string, v ...interface{}) error {
	if m.Command != "CTCP" {
		return errors.New("Invalid CTCP message")
	}

	b.Send(&irc.Message{
		Prefix:  &irc.Prefix{},
		Command: "NOTICE",
		Params: []string{
			m.Prefix.Name,
			fmt.Sprintf(format, v...),
		},
	})
	return nil
}

// HasPerm is a convenience function for checking a user's permissions
func (b *Bot) HasPerm(m *irc.Message, perm string) bool {
	user := b.Auth.LookupUser(b, m.Prefix)

	return user.HasPerm(b, perm)
}

func (b *Bot) handshake() {
	b.conn.Writef("CAP END")
	b.conn.Writef("NICK %s", b.config.Nick)
	b.conn.Writef("USER %s 0.0.0.0 0.0.0.0 :%s", b.config.User, b.config.Name)
}

func (b *Bot) mainLoop() error {
	// Send the initial connection info
	if len(b.config.Pass) > 0 {
		b.conn.Writef("PASS %s", b.config.Pass)
	}

	// Set the nick setting as the current nick
	b.currentNick = b.config.Nick

	// If we have any capabilities, we need to send requests for
	// them. Note that we send multiple requests because it makes
	// parsing the ACK and NAK commands a lot simpler.
	if len(b.initialCapList) > 0 {
		var caps []string
		for cap := range b.initialCapList {
			caps = append(caps, cap)
			b.Send(&irc.Message{
				Prefix:  &irc.Prefix{},
				Command: "CAP",
				Params:  []string{"REQ", cap},
			})
		}
	} else {
		// We can only handshake now if we didn't have any
		// caps we wanted responses to.
		b.handshake()
	}

	var m *irc.Message
	for {
		m, b.err = b.conn.ReadMessage()
		if b.err != nil {
			break
		}

		// Internal handlers to make sure we track the
		// currentNick correctly and send PONGs
		if m.Command == "001" {
			log.Println("Connected")
			b.connected = true

			for _, v := range b.config.Cmds {
				b.Write(v)
			}

		} else if m.Command == "CAP" {
			if len(m.Params) > 1 {
				numParams := len(m.Params)
				if m.Params[numParams-2] == "ACK" {
					// Because we send each CAP
					// individually, we shouldn't
					// need to do anything here.

					b.initialCapResponses++
				} else if m.Params[numParams-2] == "NAK" {
					return fmt.Errorf("Got CAP NAK for %s", m.Params[1])
				}
			}

			if b.initialCapResponses >= len(b.initialCapList) {
				// Now that we've got all the responses back that we needed, we
				// can continue the initial dance.
				b.handshake()
			}
		} else if m.Command == "NICK" {
			if m.Prefix.Name == b.currentNick && len(m.Params) > 0 {
				b.currentNick = m.Params[0]
			}
		} else if m.Command == "PING" {
			b.conn.Writef("PONG :%s", m.Trailing())
		} else if m.Command == "001" {
			b.currentNick = m.Params[0]
		} else if m.Command == "437" || m.Command == "433" {
			b.currentNick = b.currentNick + "_"
			b.conn.Writef("NICK %s", b.currentNick)
		}

		b.BasicMux.HandleEvent(b, m)

		// TODO: Make this work better
		if b.err != nil {
			break
		}
	}

	return b.err
}

// Write will write an raw IRC message to the stream
func (b *Bot) Write(line string) {
	b.conn.Write(line)
}

// Writef is a convenience method around fmt.Sprintf and Bot.Write
func (b *Bot) Writef(format string, args ...interface{}) {
	b.conn.Writef(format, args...)
}

// PluginLoaded will return true if a plugin is loaded and false otherwise
func (b *Bot) PluginLoaded(name string) bool {
	_, ok := b.Plugins[name]
	return ok
}

// LoadPlugin will ensure a plugin is loaded. It is designed to be
// usable in other plugins, so they can ensure plugins they depend on
// are loaded before using them.
func (b *Bot) LoadPlugin(name string) error {
	// We don't need to load the plugin if it's already loaded
	if b.PluginLoaded(name) {
		return nil
	}

	// Ensure the plugin exists
	factory, ok := plugins[name]
	if !ok {
		return fmt.Errorf("Plugin %s does not exist", name)
	}

	// If we're trying to load this plugin already, this is a
	// circular load and we should bail.
	if v := b.loadingPlugins[name]; v {
		return fmt.Errorf("Plugin %s getting loaded circularly", name)
	}

	// Set a flag stating that we're loading this plugin
	b.loadingPlugins[name] = true

	// Actually load the plugin
	plugin, err := factory(b)
	if err != nil {
		return fmt.Errorf("Plugin %s failed to load", name)
	}

	// Save the value returned from the plugin factory.
	b.Plugins[name] = plugin

	// Unset the flag saying we're loading this because we're done now.
	delete(b.loadingPlugins, name)

	return nil
}

// Run starts the bot and loops until it dies
func (b *Bot) Run() error {
	var err error
	// Load all the plugins we need. If there were plugins
	// specified in the config, just load those. Otherwise load
	// ALL of them.
	if len(b.config.Plugins) != 0 {
		for _, name := range b.config.Plugins {
			err = b.LoadPlugin(name)
			if err != nil {
				return err
			}
		}
	} else {
		for name := range plugins {
			err = b.LoadPlugin(name)
			if err != nil {
				return err
			}
		}
	}

	// If we have a pidfile configured, create it and write the PID
	if b.config.PidFile != "" {
		l, err := lockfile.New(b.config.PidFile)
		if err != nil {
			return err
		}

		err = l.TryLock()
		if err != nil {
			return err
		}

		defer l.Unlock()
	}

	// The ReadWriteCloser will contain either a *net.Conn or *tls.Conn
	var c io.ReadWriteCloser
	if b.config.TLS {
		conf := &tls.Config{
			InsecureSkipVerify: b.config.TLSNoVerify,
		}

		c, err = tls.Dial("tcp", b.config.Host, conf)
	} else {
		c, err = net.Dial("tcp", b.config.Host)
	}

	if err != nil {
		return err
	}

	// Create a client from the connection we've just opened
	b.conn = irc.NewConn(c)

	b.conn.DebugCallback = func(line string) {
		if !b.config.Quiet {
			log.Println(line)
		}
	}

	// Start the main loop
	return b.mainLoop()
}
<|endoftext|>


<commit_before>package manta

import (
	"testing"

	"github.com/dotabuff/manta/dota"
	"github.com/stretchr/testify/assert"
)

func TestParseOneMatch(t *testing.T) {
	assert := assert.New(t)
	debugLevel = 0
	testLevel = 0

	buf := mustGetReplayData("1731962898", "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1731962898.dem")
	parser, err := NewParser(buf)
	assert.Nil(err)

	parser.OnPacketEntity(func(pe *PacketEntity, e EntityEventType) error {
		return nil
	})

	err = parser.Start()
	assert.Nil(err)
}

func TestParseOneMatchWithoutPE(t *testing.T) {
	assert := assert.New(t)
	debugLevel = 0
	testLevel = 0

	buf := mustGetReplayData("1731962898", "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1731962898.dem")
	parser, err := NewParser(buf)
	parser.ProcessPacketEntities = false
	assert.Nil(err)
	err = parser.Start()
	assert.Nil(err)
}

// Simply tests that we can read the outer structure of a real match
func TestParseRealMatches(t *testing.T) {
	assert := assert.New(t)

	// Important: make sure to include the most recent test last. These generate fixtures to easily
	// detect diffs in various data structures upon commit, and the latest replay should always be
	// last to provide a most up-to-date baseline.
	scenarios := []struct {
		matchId                string
		replayUrl              string
		expectCombatLogDamage  int32
		expectCombatLogHealing int32
		expectCombatLogDeaths  int32
		expectCombatLogEvents  int32
		expectUnitOrderEvents  int32
	}{
		{
			matchId:                "1734886116",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1734886116.dem",
			expectCombatLogDamage:  1048805,
			expectCombatLogHealing: 25089,
			expectCombatLogDeaths:  1447,
			expectCombatLogEvents:  42307,
			expectUnitOrderEvents:  59775,
		},
		{
			matchId:                "1731962898",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1731962898.dem",
			expectCombatLogDamage:  415560,
			expectCombatLogHealing: 20018,
			expectCombatLogDeaths:  690,
			expectCombatLogEvents:  24296,
			expectUnitOrderEvents:  27525,
		},
		{
			matchId:                "1605340040",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1605340040.dem",
			expectCombatLogDamage:  522367,
			expectCombatLogHealing: 31721,
			expectCombatLogDeaths:  795,
			expectCombatLogEvents:  21116,
			expectUnitOrderEvents:  40669,
		},
		{
			matchId:                "1560289528",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1560289528.dem",
			expectCombatLogDamage:  1180993,
			expectCombatLogHealing: 57511,
			expectCombatLogDeaths:  1449,
			expectCombatLogEvents:  49146,
			expectUnitOrderEvents:  63387,
		},
		{
			matchId:                "1560294294",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1560294294.dem",
			expectCombatLogDamage:  768154,
			expectCombatLogHealing: 11565,
			expectCombatLogDeaths:  954,
			expectCombatLogEvents:  24535,
			expectUnitOrderEvents:  30657,
		},
		{
			matchId:                "1560315800",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1560315800.dem",
			expectCombatLogDamage:  1332418,
			expectCombatLogHealing: 57874,
			expectCombatLogDeaths:  1645,
			expectCombatLogEvents:  51288,
			expectUnitOrderEvents:  63992,
		},
		{
			matchId:                "1582611189",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1582611189.dem",
			expectCombatLogDamage:  599388,
			expectCombatLogHealing: 28576,
			expectCombatLogDeaths:  930,
			expectCombatLogEvents:  23800,
			expectUnitOrderEvents:  40237,
		},
		{
			matchId:                "1648457986",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1648457986.dem",
			expectCombatLogDamage:  224773,
			expectCombatLogHealing: 5914,
			expectCombatLogDeaths:  466,
			expectCombatLogEvents:  10170,
			expectUnitOrderEvents:  17822,
		},
		{
			matchId:                "1712853372",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1712853372.dem",
			expectCombatLogDamage:  671297,
			expectCombatLogHealing: 23467,
			expectCombatLogDeaths:  1099,
			expectCombatLogEvents:  30381,
			expectUnitOrderEvents:  48107,
		},
		{
			matchId:                "1716444111",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1716444111.dem",
			expectCombatLogDamage:  1398735,
			expectCombatLogHealing: 49659,
			expectCombatLogDeaths:  2169,
			expectCombatLogEvents:  76921,
			expectUnitOrderEvents:  48822,
		},
	}

	for _, s := range scenarios {
		buf := mustGetReplayData(s.matchId, s.replayUrl)
		parser, err := NewParser(buf)
		if err != nil {
			t.Errorf("unable to instantiate parser: %s", err)
			continue
		}

		gotCombatLogDamage := int32(0)
		gotCombatLogHealing := int32(0)
		gotCombatLogDeaths := int32(0)
		gotCombatLogEvents := int32(0)
		gotUnitOrderEvents := int32(0)

		parser.Callbacks.OnCDOTAUserMsg_SpectatorPlayerUnitOrders(func(m *dota.CDOTAUserMsg_SpectatorPlayerUnitOrders) error {
			gotUnitOrderEvents += 1
			return nil
		})

		parser.OnGameEvent("dota_combatlog", func(m *GameEvent) error {
			gotCombatLogEvents += 1

			t, err := m.GetInt32("type")
			assert.Nil(err)

			switch dota.DOTA_COMBATLOG_TYPES(t) {
			case dota.DOTA_COMBATLOG_TYPES_DOTA_COMBATLOG_DAMAGE:
				v, err := m.GetInt32("value")
				assert.Nil(err)
				gotCombatLogDamage += v
			case dota.DOTA_COMBATLOG_TYPES_DOTA_COMBATLOG_DEATH:
				gotCombatLogDeaths += 1
			case dota.DOTA_COMBATLOG_TYPES_DOTA_COMBATLOG_HEAL:
				v, err := m.GetInt32("value")
				assert.Nil(err)
				gotCombatLogHealing += v
			}

			return nil
		})

		if fixturesMode {
			// Writes out the source_1_legacy_game_events_list.json fixture so that we can identify changes to schema.
			parser.Callbacks.OnCMsgSource1LegacyGameEventList(func(m *dota.CMsgSource1LegacyGameEventList) error {
				_dump_fixture("legacy_game_events/list_latest.json", _json_marshal(m))
				return nil
			})
		}

		err = parser.Start()
		assert.Nil(err, s.matchId)

		if fixturesMode {
			// Use this to write out instancebaseline fixtures
			t, _ := parser.StringTables.GetTableByName("instancebaseline")
			for _, i := range t.Items {
				classId, _ := atoi32(i.Key)
				className := parser.ClassInfo[classId]
				_dump_fixture(_sprintf("instancebaseline/%s_%s.rawbuf", className, s.matchId), i.Value)
			}
		}

		assert.Equal(s.expectCombatLogDamage, gotCombatLogDamage, s.matchId)
		assert.Equal(s.expectCombatLogHealing, gotCombatLogHealing, s.matchId)
		assert.Equal(s.expectCombatLogDeaths, gotCombatLogDeaths, s.matchId)
		assert.Equal(s.expectCombatLogEvents, gotCombatLogEvents, s.matchId)
		assert.Equal(s.expectUnitOrderEvents, gotUnitOrderEvents, s.matchId)
	}
}

func BenchmarkParseMatch(b *testing.B) {
	assert := assert.New(b)

	buf := mustGetReplayData("1560315800", "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1560315800.dem")

	b.ResetTimer()
	for n := 0; n < b.N; n++ {
		parser, err := NewParser(buf)
		assert.Nil(err)
		err = parser.Start()
		assert.Nil(err)
	}

	b.ReportAllocs()
}
<commit_msg>Add two failing test cases from sep01 patch<commit_after>package manta

import (
	"testing"

	"github.com/dotabuff/manta/dota"
	"github.com/stretchr/testify/assert"
)

func TestSep01Match1763193771(t *testing.T) {
	assert := assert.New(t)
	debugLevel = 0
	testLevel = 0

	buf := mustGetReplayData("1763193771", "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1763193771.dem")
	parser, err := NewParser(buf)
	assert.Nil(err)

	err = parser.Start()
	assert.Nil(err)
}

func TestSep01Match1763177231(t *testing.T) {
	assert := assert.New(t)
	debugLevel = 0
	testLevel = 0

	buf := mustGetReplayData("1763177231", "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1763177231.dem")
	parser, err := NewParser(buf)
	assert.Nil(err)

	err = parser.Start()
	assert.Nil(err)
}

func TestParseOneMatch(t *testing.T) {
	assert := assert.New(t)
	debugLevel = 0
	testLevel = 0

	buf := mustGetReplayData("1731962898", "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1731962898.dem")
	parser, err := NewParser(buf)
	assert.Nil(err)

	parser.OnPacketEntity(func(pe *PacketEntity, e EntityEventType) error {
		return nil
	})

	err = parser.Start()
	assert.Nil(err)
}

func TestParseOneMatchWithoutPE(t *testing.T) {
	assert := assert.New(t)
	debugLevel = 0
	testLevel = 0

	buf := mustGetReplayData("1731962898", "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1731962898.dem")
	parser, err := NewParser(buf)
	parser.ProcessPacketEntities = false
	assert.Nil(err)
	err = parser.Start()
	assert.Nil(err)
}

// Simply tests that we can read the outer structure of a real match
func TestParseRealMatches(t *testing.T) {
	assert := assert.New(t)

	// Important: make sure to include the most recent test last. These generate fixtures to easily
	// detect diffs in various data structures upon commit, and the latest replay should always be
	// last to provide a most up-to-date baseline.
	scenarios := []struct {
		matchId                string
		replayUrl              string
		expectCombatLogDamage  int32
		expectCombatLogHealing int32
		expectCombatLogDeaths  int32
		expectCombatLogEvents  int32
		expectUnitOrderEvents  int32
	}{
		{
			matchId:                "1763193771",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1763193771.dem",
			expectCombatLogDamage:  0,
			expectCombatLogHealing: 0,
			expectCombatLogDeaths:  0,
			expectCombatLogEvents:  0,
			expectUnitOrderEvents:  0,
		},
		{
			matchId:                "1763177231",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1763177231.dem",
			expectCombatLogDamage:  0,
			expectCombatLogHealing: 0,
			expectCombatLogDeaths:  0,
			expectCombatLogEvents:  0,
			expectUnitOrderEvents:  0,
		},
		{
			matchId:                "1734886116",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1734886116.dem",
			expectCombatLogDamage:  1048805,
			expectCombatLogHealing: 25089,
			expectCombatLogDeaths:  1447,
			expectCombatLogEvents:  42307,
			expectUnitOrderEvents:  59775,
		},
		{
			matchId:                "1731962898",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1731962898.dem",
			expectCombatLogDamage:  415560,
			expectCombatLogHealing: 20018,
			expectCombatLogDeaths:  690,
			expectCombatLogEvents:  24296,
			expectUnitOrderEvents:  27525,
		},
		{
			matchId:                "1605340040",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1605340040.dem",
			expectCombatLogDamage:  522367,
			expectCombatLogHealing: 31721,
			expectCombatLogDeaths:  795,
			expectCombatLogEvents:  21116,
			expectUnitOrderEvents:  40669,
		},
		{
			matchId:                "1560289528",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1560289528.dem",
			expectCombatLogDamage:  1180993,
			expectCombatLogHealing: 57511,
			expectCombatLogDeaths:  1449,
			expectCombatLogEvents:  49146,
			expectUnitOrderEvents:  63387,
		},
		{
			matchId:                "1560294294",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1560294294.dem",
			expectCombatLogDamage:  768154,
			expectCombatLogHealing: 11565,
			expectCombatLogDeaths:  954,
			expectCombatLogEvents:  24535,
			expectUnitOrderEvents:  30657,
		},
		{
			matchId:                "1560315800",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1560315800.dem",
			expectCombatLogDamage:  1332418,
			expectCombatLogHealing: 57874,
			expectCombatLogDeaths:  1645,
			expectCombatLogEvents:  51288,
			expectUnitOrderEvents:  63992,
		},
		{
			matchId:                "1582611189",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1582611189.dem",
			expectCombatLogDamage:  599388,
			expectCombatLogHealing: 28576,
			expectCombatLogDeaths:  930,
			expectCombatLogEvents:  23800,
			expectUnitOrderEvents:  40237,
		},
		{
			matchId:                "1648457986",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1648457986.dem",
			expectCombatLogDamage:  224773,
			expectCombatLogHealing: 5914,
			expectCombatLogDeaths:  466,
			expectCombatLogEvents:  10170,
			expectUnitOrderEvents:  17822,
		},
		{
			matchId:                "1712853372",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1712853372.dem",
			expectCombatLogDamage:  671297,
			expectCombatLogHealing: 23467,
			expectCombatLogDeaths:  1099,
			expectCombatLogEvents:  30381,
			expectUnitOrderEvents:  48107,
		},
		{
			matchId:                "1716444111",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1716444111.dem",
			expectCombatLogDamage:  1398735,
			expectCombatLogHealing: 49659,
			expectCombatLogDeaths:  2169,
			expectCombatLogEvents:  76921,
			expectUnitOrderEvents:  48822,
		},
	}

	for _, s := range scenarios {
		buf := mustGetReplayData(s.matchId, s.replayUrl)
		parser, err := NewParser(buf)
		if err != nil {
			t.Errorf("unable to instantiate parser: %s", err)
			continue
		}

		gotCombatLogDamage := int32(0)
		gotCombatLogHealing := int32(0)
		gotCombatLogDeaths := int32(0)
		gotCombatLogEvents := int32(0)
		gotUnitOrderEvents := int32(0)

		parser.Callbacks.OnCDOTAUserMsg_SpectatorPlayerUnitOrders(func(m *dota.CDOTAUserMsg_SpectatorPlayerUnitOrders) error {
			gotUnitOrderEvents += 1
			return nil
		})

		parser.OnGameEvent("dota_combatlog", func(m *GameEvent) error {
			gotCombatLogEvents += 1

			t, err := m.GetInt32("type")
			assert.Nil(err)

			switch dota.DOTA_COMBATLOG_TYPES(t) {
			case dota.DOTA_COMBATLOG_TYPES_DOTA_COMBATLOG_DAMAGE:
				v, err := m.GetInt32("value")
				assert.Nil(err)
				gotCombatLogDamage += v
			case dota.DOTA_COMBATLOG_TYPES_DOTA_COMBATLOG_DEATH:
				gotCombatLogDeaths += 1
			case dota.DOTA_COMBATLOG_TYPES_DOTA_COMBATLOG_HEAL:
				v, err := m.GetInt32("value")
				assert.Nil(err)
				gotCombatLogHealing += v
			}

			return nil
		})

		if fixturesMode {
			// Writes out the source_1_legacy_game_events_list.json fixture so that we can identify changes to schema.
			parser.Callbacks.OnCMsgSource1LegacyGameEventList(func(m *dota.CMsgSource1LegacyGameEventList) error {
				_dump_fixture("legacy_game_events/list_latest.json", _json_marshal(m))
				return nil
			})
		}

		err = parser.Start()
		assert.Nil(err, s.matchId)

		if fixturesMode {
			// Use this to write out instancebaseline fixtures
			t, _ := parser.StringTables.GetTableByName("instancebaseline")
			for _, i := range t.Items {
				classId, _ := atoi32(i.Key)
				className := parser.ClassInfo[classId]
				_dump_fixture(_sprintf("instancebaseline/%s_%s.rawbuf", className, s.matchId), i.Value)
			}
		}

		assert.Equal(s.expectCombatLogDamage, gotCombatLogDamage, s.matchId)
		assert.Equal(s.expectCombatLogHealing, gotCombatLogHealing, s.matchId)
		assert.Equal(s.expectCombatLogDeaths, gotCombatLogDeaths, s.matchId)
		assert.Equal(s.expectCombatLogEvents, gotCombatLogEvents, s.matchId)
		assert.Equal(s.expectUnitOrderEvents, gotUnitOrderEvents, s.matchId)
	}
}

func BenchmarkParseMatch(b *testing.B) {
	assert := assert.New(b)

	buf := mustGetReplayData("1560315800", "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1560315800.dem")

	b.ResetTimer()
	for n := 0; n < b.N; n++ {
		parser, err := NewParser(buf)
		assert.Nil(err)
		err = parser.Start()
		assert.Nil(err)
	}

	b.ReportAllocs()
}
<|endoftext|>


<commit_before>package connector

import (
	"fmt"
	"strings"
	"sync"

	"github.com/bcicen/ctop/connector/collector"
	"github.com/bcicen/ctop/connector/manager"
	"github.com/bcicen/ctop/container"
	api "github.com/fsouza/go-dockerclient"
)

func init() { enabled["docker"] = NewDocker }

type Docker struct {
	client       *api.Client
	containers   map[string]*container.Container
	needsRefresh chan string // container IDs requiring refresh
	lock         sync.RWMutex
}

func NewDocker() Connector {
	// init docker client
	client, err := api.NewClientFromEnv()
	if err != nil {
		panic(err)
	}
	cm := &Docker{
		client:       client,
		containers:   make(map[string]*container.Container),
		needsRefresh: make(chan string, 60),
		lock:         sync.RWMutex{},
	}
	go cm.Loop()
	cm.refreshAll()
	go cm.watchEvents()
	return cm
}

// Docker events watcher
func (cm *Docker) watchEvents() {
	log.Info("docker event listener starting")
	events := make(chan *api.APIEvents)
	cm.client.AddEventListener(events)

	for e := range events {
		if e.Type != "container" {
			continue
		}

		actionName := strings.Split(e.Action, ":")[0]

		switch actionName {
		case "start", "die", "pause", "unpause", "health_status":
			log.Debugf("handling docker event: action=%s id=%s", e.Action, e.ID)
			cm.needsRefresh <- e.ID
		case "destroy":
			log.Debugf("handling docker event: action=%s id=%s", e.Action, e.ID)
			cm.delByID(e.ID)
		}
	}
}

func portsFormat(ports map[api.Port][]api.PortBinding) string {
	var exposed []string
	var published []string

	for k, v := range ports {
		if len(v) == 0 {
			exposed = append(exposed, string(k))
			continue
		}
		for _, binding := range v {
			s := fmt.Sprintf("%s:%s -> %s", binding.HostIP, binding.HostPort, k)
			published = append(published, s)
		}
	}

	return strings.Join(append(exposed, published...), "\n")
}

func ipsFormat(networks map[string]api.ContainerNetwork) string {
	var ips []string

	for k, v := range networks {
		s := fmt.Sprintf("%s:%s", k, v.IPAddress)
		ips = append(ips, s)
	}

	return strings.Join(ips, "\n")
}

func (cm *Docker) refresh(c *container.Container) {
	insp := cm.inspect(c.Id)
	// remove container if no longer exists
	if insp == nil {
		cm.delByID(c.Id)
		return
	}
	c.SetMeta("name", shortName(insp.Name))
	c.SetMeta("image", insp.Config.Image)
	c.SetMeta("IPs", ipsFormat(insp.NetworkSettings.Networks))
	c.SetMeta("ports", portsFormat(insp.NetworkSettings.Ports))
	c.SetMeta("created", insp.Created.Format("Mon Jan 2 15:04:05 2006"))
	c.SetMeta("health", insp.State.Health.Status)
	for _, env := range insp.Config.Env {
		c.SetMeta("[ENV-VAR]", string(env))
	}
	c.SetState(insp.State.Status)
}

func (cm *Docker) inspect(id string) *api.Container {
	c, err := cm.client.InspectContainer(id)
	if err != nil {
		if _, ok := err.(*api.NoSuchContainer); !ok {
			log.Errorf(err.Error())
		}
	}
	return c
}

// Mark all container IDs for refresh
func (cm *Docker) refreshAll() {
	opts := api.ListContainersOptions{All: true}
	allContainers, err := cm.client.ListContainers(opts)
	if err != nil {
		panic(err)
	}

	for _, i := range allContainers {
		c := cm.MustGet(i.ID)
		c.SetMeta("name", shortName(i.Names[0]))
		c.SetState(i.State)
		cm.needsRefresh <- c.Id
	}
}

func (cm *Docker) Loop() {
	for id := range cm.needsRefresh {
		c := cm.MustGet(id)
		cm.refresh(c)
	}
}

// Get a single container, creating one anew if not existing
func (cm *Docker) MustGet(id string) *container.Container {
	c, ok := cm.Get(id)
	// append container struct for new containers
	if !ok {
		// create collector
		collector := collector.NewDocker(cm.client, id)
		// create manager
		manager := manager.NewDocker(cm.client, id)
		// create container
		c = container.New(id, collector, manager)
		cm.lock.Lock()
		cm.containers[id] = c
		cm.lock.Unlock()
	}
	return c
}

// Get a single container, by ID
func (cm *Docker) Get(id string) (*container.Container, bool) {
	cm.lock.Lock()
	c, ok := cm.containers[id]
	cm.lock.Unlock()
	return c, ok
}

// Remove containers by ID
func (cm *Docker) delByID(id string) {
	cm.lock.Lock()
	delete(cm.containers, id)
	cm.lock.Unlock()
	log.Infof("removed dead container: %s", id)
}

// Return array of all containers, sorted by field
func (cm *Docker) All() (containers container.Containers) {
	cm.lock.Lock()
	for _, c := range cm.containers {
		containers = append(containers, c)
	}

	containers.Sort()
	containers.Filter()
	cm.lock.Unlock()
	return containers
}

// use primary container name
func shortName(name string) string {
	return strings.Replace(name, "/", "", 1)
}
<commit_msg>remove unneeded cast<commit_after>package connector

import (
	"fmt"
	"strings"
	"sync"

	"github.com/bcicen/ctop/connector/collector"
	"github.com/bcicen/ctop/connector/manager"
	"github.com/bcicen/ctop/container"
	api "github.com/fsouza/go-dockerclient"
)

func init() { enabled["docker"] = NewDocker }

type Docker struct {
	client       *api.Client
	containers   map[string]*container.Container
	needsRefresh chan string // container IDs requiring refresh
	lock         sync.RWMutex
}

func NewDocker() Connector {
	// init docker client
	client, err := api.NewClientFromEnv()
	if err != nil {
		panic(err)
	}
	cm := &Docker{
		client:       client,
		containers:   make(map[string]*container.Container),
		needsRefresh: make(chan string, 60),
		lock:         sync.RWMutex{},
	}
	go cm.Loop()
	cm.refreshAll()
	go cm.watchEvents()
	return cm
}

// Docker events watcher
func (cm *Docker) watchEvents() {
	log.Info("docker event listener starting")
	events := make(chan *api.APIEvents)
	cm.client.AddEventListener(events)

	for e := range events {
		if e.Type != "container" {
			continue
		}

		actionName := strings.Split(e.Action, ":")[0]

		switch actionName {
		case "start", "die", "pause", "unpause", "health_status":
			log.Debugf("handling docker event: action=%s id=%s", e.Action, e.ID)
			cm.needsRefresh <- e.ID
		case "destroy":
			log.Debugf("handling docker event: action=%s id=%s", e.Action, e.ID)
			cm.delByID(e.ID)
		}
	}
}

func portsFormat(ports map[api.Port][]api.PortBinding) string {
	var exposed []string
	var published []string

	for k, v := range ports {
		if len(v) == 0 {
			exposed = append(exposed, string(k))
			continue
		}
		for _, binding := range v {
			s := fmt.Sprintf("%s:%s -> %s", binding.HostIP, binding.HostPort, k)
			published = append(published, s)
		}
	}

	return strings.Join(append(exposed, published...), "\n")
}

func ipsFormat(networks map[string]api.ContainerNetwork) string {
	var ips []string

	for k, v := range networks {
		s := fmt.Sprintf("%s:%s", k, v.IPAddress)
		ips = append(ips, s)
	}

	return strings.Join(ips, "\n")
}

func (cm *Docker) refresh(c *container.Container) {
	insp := cm.inspect(c.Id)
	// remove container if no longer exists
	if insp == nil {
		cm.delByID(c.Id)
		return
	}
	c.SetMeta("name", shortName(insp.Name))
	c.SetMeta("image", insp.Config.Image)
	c.SetMeta("IPs", ipsFormat(insp.NetworkSettings.Networks))
	c.SetMeta("ports", portsFormat(insp.NetworkSettings.Ports))
	c.SetMeta("created", insp.Created.Format("Mon Jan 2 15:04:05 2006"))
	c.SetMeta("health", insp.State.Health.Status)
	for _, env := range insp.Config.Env {
		c.SetMeta("[ENV-VAR]", env)
	}
	c.SetState(insp.State.Status)
}

func (cm *Docker) inspect(id string) *api.Container {
	c, err := cm.client.InspectContainer(id)
	if err != nil {
		if _, ok := err.(*api.NoSuchContainer); !ok {
			log.Errorf(err.Error())
		}
	}
	return c
}

// Mark all container IDs for refresh
func (cm *Docker) refreshAll() {
	opts := api.ListContainersOptions{All: true}
	allContainers, err := cm.client.ListContainers(opts)
	if err != nil {
		panic(err)
	}

	for _, i := range allContainers {
		c := cm.MustGet(i.ID)
		c.SetMeta("name", shortName(i.Names[0]))
		c.SetState(i.State)
		cm.needsRefresh <- c.Id
	}
}

func (cm *Docker) Loop() {
	for id := range cm.needsRefresh {
		c := cm.MustGet(id)
		cm.refresh(c)
	}
}

// Get a single container, creating one anew if not existing
func (cm *Docker) MustGet(id string) *container.Container {
	c, ok := cm.Get(id)
	// append container struct for new containers
	if !ok {
		// create collector
		collector := collector.NewDocker(cm.client, id)
		// create manager
		manager := manager.NewDocker(cm.client, id)
		// create container
		c = container.New(id, collector, manager)
		cm.lock.Lock()
		cm.containers[id] = c
		cm.lock.Unlock()
	}
	return c
}

// Get a single container, by ID
func (cm *Docker) Get(id string) (*container.Container, bool) {
	cm.lock.Lock()
	c, ok := cm.containers[id]
	cm.lock.Unlock()
	return c, ok
}

// Remove containers by ID
func (cm *Docker) delByID(id string) {
	cm.lock.Lock()
	delete(cm.containers, id)
	cm.lock.Unlock()
	log.Infof("removed dead container: %s", id)
}

// Return array of all containers, sorted by field
func (cm *Docker) All() (containers container.Containers) {
	cm.lock.Lock()
	for _, c := range cm.containers {
		containers = append(containers, c)
	}

	containers.Sort()
	containers.Filter()
	cm.lock.Unlock()
	return containers
}

// use primary container name
func shortName(name string) string {
	return strings.Replace(name, "/", "", 1)
}
<|endoftext|>


<commit_before>package consul

import (
	"context"
	"fmt"
	"strconv"
	"sync"
	"time"

	"github.com/armon/go-metrics"
	"github.com/hashicorp/consul/consul/agent"
	"github.com/hashicorp/consul/consul/structs"
	"github.com/hashicorp/raft"
	"github.com/hashicorp/serf/serf"
)

// AutopilotPolicy is the interface for the Autopilot mechanism
type AutopilotPolicy interface {
	// PromoteNonVoters defines the handling of non-voting servers
	PromoteNonVoters(*structs.AutopilotConfig) error
}

func (s *Server) startAutopilot() {
	s.autopilotShutdownCh = make(chan struct{})
	s.autopilotWaitGroup = sync.WaitGroup{}
	s.autopilotWaitGroup.Add(1)

	go s.autopilotLoop()
}

func (s *Server) stopAutopilot() {
	close(s.autopilotShutdownCh)
	s.autopilotWaitGroup.Wait()
}

// autopilotLoop periodically looks for nonvoting servers to promote and dead servers to remove.
func (s *Server) autopilotLoop() {
	defer s.autopilotWaitGroup.Done()

	// Monitor server health until shutdown
	ticker := time.NewTicker(s.config.AutopilotInterval)
	defer ticker.Stop()

	for {
		select {
		case <-s.autopilotShutdownCh:
			return
		case <-ticker.C:
			state := s.fsm.State()
			_, autopilotConf, err := state.AutopilotConfig()
			if err != nil {
				s.logger.Printf("[ERR] consul: error retrieving autopilot config: %s", err)
				break
			}

			if err := s.autopilotPolicy.PromoteNonVoters(autopilotConf); err != nil {
				s.logger.Printf("[ERR] consul: error checking for non-voters to promote: %s", err)
			}

			if err := s.pruneDeadServers(); err != nil {
				s.logger.Printf("[ERR] consul: error checking for dead servers to remove: %s", err)
			}
		case <-s.autopilotRemoveDeadCh:
			if err := s.pruneDeadServers(); err != nil {
				s.logger.Printf("[ERR] consul: error checking for dead servers to remove: %s", err)
			}
		}
	}
}

// pruneDeadServers removes up to numPeers/2 failed servers
func (s *Server) pruneDeadServers() error {
	state := s.fsm.State()
	_, autopilotConf, err := state.AutopilotConfig()
	if err != nil {
		return err
	}

	// Find any failed servers
	var failed []string
	staleRaftServers := make(map[string]raft.Server)
	if autopilotConf.CleanupDeadServers {
		future := s.raft.GetConfiguration()
		if future.Error() != nil {
			return err
		}

		for _, server := range future.Configuration().Servers {
			staleRaftServers[string(server.Address)] = server
		}

		for _, member := range s.serfLAN.Members() {
			valid, parts := agent.IsConsulServer(member)

			if valid {
				// Remove this server from the stale list; it has a serf entry
				if _, ok := staleRaftServers[parts.Addr.String()]; ok {
					delete(staleRaftServers, parts.Addr.String())
				}

				if member.Status == serf.StatusFailed {
					failed = append(failed, member.Name)
				}
			}
		}
	}

	removalCount := len(failed) + len(staleRaftServers)

	// Nothing to remove, return early
	if removalCount == 0 {
		return nil
	}

	peers, err := s.numPeers()
	if err != nil {
		return err
	}

	// Only do removals if a minority of servers will be affected
	if removalCount < peers/2 {
		for _, server := range failed {
			s.logger.Printf("[INFO] consul: Attempting removal of failed server: %v", server)
			go s.serfLAN.RemoveFailedNode(server)
		}

		minRaftProtocol, err := ServerMinRaftProtocol(s.serfLAN.Members())
		if err != nil {
			return err
		}
		for _, raftServer := range staleRaftServers {
			var future raft.Future
			if minRaftProtocol >= 2 {
				s.logger.Printf("[INFO] consul: Attempting removal of stale raft server : %v", raftServer.ID)
				future = s.raft.RemoveServer(raftServer.ID, 0, 0)
			} else {
				s.logger.Printf("[INFO] consul: Attempting removal of stale raft server : %v", raftServer.ID)
				future = s.raft.RemovePeer(raftServer.Address)
			}
			if err := future.Error(); err != nil {
				return err
			}
		}
	} else {
		s.logger.Printf("[DEBUG] consul: Failed to remove dead servers: too many dead servers: %d/%d", removalCount, peers)
	}

	return nil
}

// BasicAutopilot defines a policy for promoting non-voting servers in a way
// that maintains an odd-numbered voter count.
type BasicAutopilot struct {
	server *Server
}

// PromoteNonVoters promotes eligible non-voting servers to voters.
func (b *BasicAutopilot) PromoteNonVoters(autopilotConf *structs.AutopilotConfig) error {
	minRaftProtocol, err := ServerMinRaftProtocol(b.server.LANMembers())
	if err != nil {
		return fmt.Errorf("error getting server raft protocol versions: %s", err)
	}

	// If we don't meet the minimum version for non-voter features, bail early
	if minRaftProtocol < 3 {
		return nil
	}

	future := b.server.raft.GetConfiguration()
	if err := future.Error(); err != nil {
		return fmt.Errorf("failed to get raft configuration: %v", err)
	}

	// Find any non-voters eligible for promotion
	var promotions []raft.Server
	voterCount := 0
	for _, server := range future.Configuration().Servers {
		// If this server has been stable and passing for long enough, promote it to a voter
		if !isVoter(server.Suffrage) {
			health := b.server.getServerHealth(string(server.ID))
			if health.IsStable(time.Now(), autopilotConf) {
				promotions = append(promotions, server)
			}
		} else {
			voterCount++
		}
	}

	if _, err := b.server.handlePromotions(voterCount, promotions); err != nil {
		return err
	}

	return nil
}

func (s *Server) handlePromotions(voterCount int, promotions []raft.Server) (bool, error) {
	if len(promotions) == 0 {
		return false, nil
	}

	// If there's currently an even number of servers, we can promote the first server in the list
	// to get to an odd-sized quorum
	newServers := false
	if voterCount%2 == 0 {
		addFuture := s.raft.AddVoter(promotions[0].ID, promotions[0].Address, 0, 0)
		if err := addFuture.Error(); err != nil {
			return newServers, fmt.Errorf("failed to add raft peer: %v", err)
		}
		promotions = promotions[1:]
		newServers = true
	}

	// Promote remaining servers in twos to maintain an odd quorum size
	for i := 0; i < len(promotions)-1; i += 2 {
		addFirst := s.raft.AddVoter(promotions[i].ID, promotions[i].Address, 0, 0)
		if err := addFirst.Error(); err != nil {
			return newServers, fmt.Errorf("failed to add raft peer: %v", err)
		}
		addSecond := s.raft.AddVoter(promotions[i+1].ID, promotions[i+1].Address, 0, 0)
		if err := addSecond.Error(); err != nil {
			return newServers, fmt.Errorf("failed to add raft peer: %v", err)
		}
		newServers = true
	}

	// If we added a new server, trigger a check to remove dead servers
	if newServers {
		select {
		case s.autopilotRemoveDeadCh <- struct{}{}:
		default:
		}
	}

	return newServers, nil
}

// serverHealthLoop monitors the health of the servers in the cluster
func (s *Server) serverHealthLoop() {
	// Monitor server health until shutdown
	ticker := time.NewTicker(s.config.ServerHealthInterval)
	defer ticker.Stop()

	for {
		select {
		case <-s.shutdownCh:
			return
		case <-ticker.C:
			if err := s.updateClusterHealth(); err != nil {
				s.logger.Printf("[ERR] consul: error updating cluster health: %s", err)
			}
		}
	}
}

// updateClusterHealth fetches the Raft stats of the other servers and updates
// s.clusterHealth based on the configured Autopilot thresholds
func (s *Server) updateClusterHealth() error {
	// Don't do anything if the min Raft version is too low
	minRaftProtocol, err := ServerMinRaftProtocol(s.LANMembers())
	if err != nil {
		return fmt.Errorf("error getting server raft protocol versions: %s", err)
	}
	if minRaftProtocol < 3 {
		return nil
	}

	state := s.fsm.State()
	_, autopilotConf, err := state.AutopilotConfig()
	if err != nil {
		return fmt.Errorf("error retrieving autopilot config: %s", err)
	}
	// Bail early if autopilot config hasn't been initialized yet
	if autopilotConf == nil {
		return nil
	}

	// Get the the serf members which are Consul servers
	serverMap := make(map[string]*agent.Server)
	for _, member := range s.LANMembers() {
		if member.Status == serf.StatusLeft {
			continue
		}

		valid, parts := agent.IsConsulServer(member)
		if valid {
			serverMap[parts.ID] = parts
		}
	}

	future := s.raft.GetConfiguration()
	if err := future.Error(); err != nil {
		return fmt.Errorf("error getting Raft configuration %s", err)
	}
	servers := future.Configuration().Servers

	// Fetch the health for each of the servers in parallel so we get as
	// consistent of a sample as possible. We capture the leader's index
	// here as well so it roughly lines up with the same point in time.
	targetLastIndex := s.raft.LastIndex()
	var fetchList []*agent.Server
	for _, server := range servers {
		if parts, ok := serverMap[string(server.ID)]; ok {
			fetchList = append(fetchList, parts)
		}
	}
	d := time.Now().Add(s.config.ServerHealthInterval / 2)
	ctx, cancel := context.WithDeadline(context.Background(), d)
	defer cancel()
	fetchedStats := s.statsFetcher.Fetch(ctx, fetchList)

	// Build a current list of server healths
	leader := s.raft.Leader()
	var clusterHealth structs.OperatorHealthReply
	healthyCount := 0
	voterCount := 0
	for _, server := range servers {
		health := structs.ServerHealth{
			ID:          string(server.ID),
			Address:     string(server.Address),
			Leader:      server.Address == leader,
			LastContact: -1,
			Voter:       server.Suffrage == raft.Voter,
		}

		parts, ok := serverMap[string(server.ID)]
		if ok {
			health.Name = parts.Name
			health.SerfStatus = parts.Status
			health.Version = parts.Build.String()
			if stats, ok := fetchedStats[string(server.ID)]; ok {
				if err := s.updateServerHealth(&health, parts, stats, autopilotConf, targetLastIndex); err != nil {
					s.logger.Printf("[WARN] consul: error updating server health: %s", err)
				}
			}
		} else {
			health.SerfStatus = serf.StatusNone
		}

		if health.Healthy {
			healthyCount++
			if health.Voter {
				voterCount++
			}
		}

		clusterHealth.Servers = append(clusterHealth.Servers, health)
	}
	clusterHealth.Healthy = healthyCount == len(servers)

	// If we have extra healthy voters, update FailureTolerance
	requiredQuorum := len(servers)/2 + 1
	if voterCount > requiredQuorum {
		clusterHealth.FailureTolerance = voterCount - requiredQuorum
	}

	// Heartbeat a metric for monitoring if we're the leader
	if s.IsLeader() {
		metrics.SetGauge([]string{"consul", "autopilot", "failure_tolerance"}, float32(clusterHealth.FailureTolerance))
		if clusterHealth.Healthy {
			metrics.SetGauge([]string{"consul", "autopilot", "healthy"}, 1)
		} else {
			metrics.SetGauge([]string{"consul", "autopilot", "healthy"}, 0)
		}
	}

	s.clusterHealthLock.Lock()
	s.clusterHealth = clusterHealth
	s.clusterHealthLock.Unlock()

	return nil
}

// updateServerHealth computes the resulting health of the server based on its
// fetched stats and the state of the leader.
func (s *Server) updateServerHealth(health *structs.ServerHealth,
	server *agent.Server, stats *structs.ServerStats,
	autopilotConf *structs.AutopilotConfig, targetLastIndex uint64) error {

	health.LastTerm = stats.LastTerm
	health.LastIndex = stats.LastIndex

	if stats.LastContact != "never" {
		var err error
		health.LastContact, err = time.ParseDuration(stats.LastContact)
		if err != nil {
			return fmt.Errorf("error parsing last_contact duration: %s", err)
		}
	}

	lastTerm, err := strconv.ParseUint(s.raft.Stats()["last_log_term"], 10, 64)
	if err != nil {
		return fmt.Errorf("error parsing last_log_term: %s", err)
	}
	health.Healthy = health.IsHealthy(lastTerm, targetLastIndex, autopilotConf)

	// If this is a new server or the health changed, reset StableSince
	lastHealth := s.getServerHealth(server.ID)
	if lastHealth == nil || lastHealth.Healthy != health.Healthy {
		health.StableSince = time.Now()
	} else {
		health.StableSince = lastHealth.StableSince
	}

	return nil
}

func (s *Server) getClusterHealth() structs.OperatorHealthReply {
	s.clusterHealthLock.RLock()
	defer s.clusterHealthLock.RUnlock()
	return s.clusterHealth
}

func (s *Server) getServerHealth(id string) *structs.ServerHealth {
	s.clusterHealthLock.RLock()
	defer s.clusterHealthLock.RUnlock()
	for _, health := range s.clusterHealth.Servers {
		if health.ID == id {
			return &health
		}
	}
	return nil
}

func isVoter(suffrage raft.ServerSuffrage) bool {
	switch suffrage {
	case raft.Voter, raft.Staging:
		return true
	default:
		return false
	}
}
<commit_msg>Use voter count instead of server count for required quorum<commit_after>package consul

import (
	"context"
	"fmt"
	"strconv"
	"sync"
	"time"

	"github.com/armon/go-metrics"
	"github.com/hashicorp/consul/consul/agent"
	"github.com/hashicorp/consul/consul/structs"
	"github.com/hashicorp/raft"
	"github.com/hashicorp/serf/serf"
)

// AutopilotPolicy is the interface for the Autopilot mechanism
type AutopilotPolicy interface {
	// PromoteNonVoters defines the handling of non-voting servers
	PromoteNonVoters(*structs.AutopilotConfig) error
}

func (s *Server) startAutopilot() {
	s.autopilotShutdownCh = make(chan struct{})
	s.autopilotWaitGroup = sync.WaitGroup{}
	s.autopilotWaitGroup.Add(1)

	go s.autopilotLoop()
}

func (s *Server) stopAutopilot() {
	close(s.autopilotShutdownCh)
	s.autopilotWaitGroup.Wait()
}

// autopilotLoop periodically looks for nonvoting servers to promote and dead servers to remove.
func (s *Server) autopilotLoop() {
	defer s.autopilotWaitGroup.Done()

	// Monitor server health until shutdown
	ticker := time.NewTicker(s.config.AutopilotInterval)
	defer ticker.Stop()

	for {
		select {
		case <-s.autopilotShutdownCh:
			return
		case <-ticker.C:
			state := s.fsm.State()
			_, autopilotConf, err := state.AutopilotConfig()
			if err != nil {
				s.logger.Printf("[ERR] consul: error retrieving autopilot config: %s", err)
				break
			}

			if err := s.autopilotPolicy.PromoteNonVoters(autopilotConf); err != nil {
				s.logger.Printf("[ERR] consul: error checking for non-voters to promote: %s", err)
			}

			if err := s.pruneDeadServers(); err != nil {
				s.logger.Printf("[ERR] consul: error checking for dead servers to remove: %s", err)
			}
		case <-s.autopilotRemoveDeadCh:
			if err := s.pruneDeadServers(); err != nil {
				s.logger.Printf("[ERR] consul: error checking for dead servers to remove: %s", err)
			}
		}
	}
}

// pruneDeadServers removes up to numPeers/2 failed servers
func (s *Server) pruneDeadServers() error {
	state := s.fsm.State()
	_, autopilotConf, err := state.AutopilotConfig()
	if err != nil {
		return err
	}

	// Find any failed servers
	var failed []string
	staleRaftServers := make(map[string]raft.Server)
	if autopilotConf.CleanupDeadServers {
		future := s.raft.GetConfiguration()
		if future.Error() != nil {
			return err
		}

		for _, server := range future.Configuration().Servers {
			staleRaftServers[string(server.Address)] = server
		}

		for _, member := range s.serfLAN.Members() {
			valid, parts := agent.IsConsulServer(member)

			if valid {
				// Remove this server from the stale list; it has a serf entry
				if _, ok := staleRaftServers[parts.Addr.String()]; ok {
					delete(staleRaftServers, parts.Addr.String())
				}

				if member.Status == serf.StatusFailed {
					failed = append(failed, member.Name)
				}
			}
		}
	}

	removalCount := len(failed) + len(staleRaftServers)

	// Nothing to remove, return early
	if removalCount == 0 {
		return nil
	}

	peers, err := s.numPeers()
	if err != nil {
		return err
	}

	// Only do removals if a minority of servers will be affected
	if removalCount < peers/2 {
		for _, server := range failed {
			s.logger.Printf("[INFO] consul: Attempting removal of failed server: %v", server)
			go s.serfLAN.RemoveFailedNode(server)
		}

		minRaftProtocol, err := ServerMinRaftProtocol(s.serfLAN.Members())
		if err != nil {
			return err
		}
		for _, raftServer := range staleRaftServers {
			var future raft.Future
			if minRaftProtocol >= 2 {
				s.logger.Printf("[INFO] consul: Attempting removal of stale raft server : %v", raftServer.ID)
				future = s.raft.RemoveServer(raftServer.ID, 0, 0)
			} else {
				s.logger.Printf("[INFO] consul: Attempting removal of stale raft server : %v", raftServer.ID)
				future = s.raft.RemovePeer(raftServer.Address)
			}
			if err := future.Error(); err != nil {
				return err
			}
		}
	} else {
		s.logger.Printf("[DEBUG] consul: Failed to remove dead servers: too many dead servers: %d/%d", removalCount, peers)
	}

	return nil
}

// BasicAutopilot defines a policy for promoting non-voting servers in a way
// that maintains an odd-numbered voter count.
type BasicAutopilot struct {
	server *Server
}

// PromoteNonVoters promotes eligible non-voting servers to voters.
func (b *BasicAutopilot) PromoteNonVoters(autopilotConf *structs.AutopilotConfig) error {
	minRaftProtocol, err := ServerMinRaftProtocol(b.server.LANMembers())
	if err != nil {
		return fmt.Errorf("error getting server raft protocol versions: %s", err)
	}

	// If we don't meet the minimum version for non-voter features, bail early
	if minRaftProtocol < 3 {
		return nil
	}

	future := b.server.raft.GetConfiguration()
	if err := future.Error(); err != nil {
		return fmt.Errorf("failed to get raft configuration: %v", err)
	}

	// Find any non-voters eligible for promotion
	var promotions []raft.Server
	voterCount := 0
	for _, server := range future.Configuration().Servers {
		// If this server has been stable and passing for long enough, promote it to a voter
		if !isVoter(server.Suffrage) {
			health := b.server.getServerHealth(string(server.ID))
			if health.IsStable(time.Now(), autopilotConf) {
				promotions = append(promotions, server)
			}
		} else {
			voterCount++
		}
	}

	if _, err := b.server.handlePromotions(voterCount, promotions); err != nil {
		return err
	}

	return nil
}

func (s *Server) handlePromotions(voterCount int, promotions []raft.Server) (bool, error) {
	if len(promotions) == 0 {
		return false, nil
	}

	// If there's currently an even number of servers, we can promote the first server in the list
	// to get to an odd-sized quorum
	newServers := false
	if voterCount%2 == 0 {
		addFuture := s.raft.AddVoter(promotions[0].ID, promotions[0].Address, 0, 0)
		if err := addFuture.Error(); err != nil {
			return newServers, fmt.Errorf("failed to add raft peer: %v", err)
		}
		promotions = promotions[1:]
		newServers = true
	}

	// Promote remaining servers in twos to maintain an odd quorum size
	for i := 0; i < len(promotions)-1; i += 2 {
		addFirst := s.raft.AddVoter(promotions[i].ID, promotions[i].Address, 0, 0)
		if err := addFirst.Error(); err != nil {
			return newServers, fmt.Errorf("failed to add raft peer: %v", err)
		}
		addSecond := s.raft.AddVoter(promotions[i+1].ID, promotions[i+1].Address, 0, 0)
		if err := addSecond.Error(); err != nil {
			return newServers, fmt.Errorf("failed to add raft peer: %v", err)
		}
		newServers = true
	}

	// If we added a new server, trigger a check to remove dead servers
	if newServers {
		select {
		case s.autopilotRemoveDeadCh <- struct{}{}:
		default:
		}
	}

	return newServers, nil
}

// serverHealthLoop monitors the health of the servers in the cluster
func (s *Server) serverHealthLoop() {
	// Monitor server health until shutdown
	ticker := time.NewTicker(s.config.ServerHealthInterval)
	defer ticker.Stop()

	for {
		select {
		case <-s.shutdownCh:
			return
		case <-ticker.C:
			if err := s.updateClusterHealth(); err != nil {
				s.logger.Printf("[ERR] consul: error updating cluster health: %s", err)
			}
		}
	}
}

// updateClusterHealth fetches the Raft stats of the other servers and updates
// s.clusterHealth based on the configured Autopilot thresholds
func (s *Server) updateClusterHealth() error {
	// Don't do anything if the min Raft version is too low
	minRaftProtocol, err := ServerMinRaftProtocol(s.LANMembers())
	if err != nil {
		return fmt.Errorf("error getting server raft protocol versions: %s", err)
	}
	if minRaftProtocol < 3 {
		return nil
	}

	state := s.fsm.State()
	_, autopilotConf, err := state.AutopilotConfig()
	if err != nil {
		return fmt.Errorf("error retrieving autopilot config: %s", err)
	}
	// Bail early if autopilot config hasn't been initialized yet
	if autopilotConf == nil {
		return nil
	}

	// Get the the serf members which are Consul servers
	serverMap := make(map[string]*agent.Server)
	for _, member := range s.LANMembers() {
		if member.Status == serf.StatusLeft {
			continue
		}

		valid, parts := agent.IsConsulServer(member)
		if valid {
			serverMap[parts.ID] = parts
		}
	}

	future := s.raft.GetConfiguration()
	if err := future.Error(); err != nil {
		return fmt.Errorf("error getting Raft configuration %s", err)
	}
	servers := future.Configuration().Servers

	// Fetch the health for each of the servers in parallel so we get as
	// consistent of a sample as possible. We capture the leader's index
	// here as well so it roughly lines up with the same point in time.
	targetLastIndex := s.raft.LastIndex()
	var fetchList []*agent.Server
	for _, server := range servers {
		if parts, ok := serverMap[string(server.ID)]; ok {
			fetchList = append(fetchList, parts)
		}
	}
	d := time.Now().Add(s.config.ServerHealthInterval / 2)
	ctx, cancel := context.WithDeadline(context.Background(), d)
	defer cancel()
	fetchedStats := s.statsFetcher.Fetch(ctx, fetchList)

	// Build a current list of server healths
	leader := s.raft.Leader()
	var clusterHealth structs.OperatorHealthReply
	voterCount := 0
	healthyCount := 0
	healthyVoterCount := 0
	for _, server := range servers {
		health := structs.ServerHealth{
			ID:          string(server.ID),
			Address:     string(server.Address),
			Leader:      server.Address == leader,
			LastContact: -1,
			Voter:       server.Suffrage == raft.Voter,
		}

		parts, ok := serverMap[string(server.ID)]
		if ok {
			health.Name = parts.Name
			health.SerfStatus = parts.Status
			health.Version = parts.Build.String()
			if stats, ok := fetchedStats[string(server.ID)]; ok {
				if err := s.updateServerHealth(&health, parts, stats, autopilotConf, targetLastIndex); err != nil {
					s.logger.Printf("[WARN] consul: error updating server health: %s", err)
				}
			}
		} else {
			health.SerfStatus = serf.StatusNone
		}

		if health.Voter {
			voterCount++
		}
		if health.Healthy {
			healthyCount++
			if health.Voter {
				healthyVoterCount++
			}
		}

		clusterHealth.Servers = append(clusterHealth.Servers, health)
	}
	clusterHealth.Healthy = healthyCount == len(servers)

	// If we have extra healthy voters, update FailureTolerance
	requiredQuorum := voterCount/2 + 1
	if healthyVoterCount > requiredQuorum {
		clusterHealth.FailureTolerance = healthyVoterCount - requiredQuorum
	}

	// Heartbeat a metric for monitoring if we're the leader
	if s.IsLeader() {
		metrics.SetGauge([]string{"consul", "autopilot", "failure_tolerance"}, float32(clusterHealth.FailureTolerance))
		if clusterHealth.Healthy {
			metrics.SetGauge([]string{"consul", "autopilot", "healthy"}, 1)
		} else {
			metrics.SetGauge([]string{"consul", "autopilot", "healthy"}, 0)
		}
	}

	s.clusterHealthLock.Lock()
	s.clusterHealth = clusterHealth
	s.clusterHealthLock.Unlock()

	return nil
}

// updateServerHealth computes the resulting health of the server based on its
// fetched stats and the state of the leader.
func (s *Server) updateServerHealth(health *structs.ServerHealth,
	server *agent.Server, stats *structs.ServerStats,
	autopilotConf *structs.AutopilotConfig, targetLastIndex uint64) error {

	health.LastTerm = stats.LastTerm
	health.LastIndex = stats.LastIndex

	if stats.LastContact != "never" {
		var err error
		health.LastContact, err = time.ParseDuration(stats.LastContact)
		if err != nil {
			return fmt.Errorf("error parsing last_contact duration: %s", err)
		}
	}

	lastTerm, err := strconv.ParseUint(s.raft.Stats()["last_log_term"], 10, 64)
	if err != nil {
		return fmt.Errorf("error parsing last_log_term: %s", err)
	}
	health.Healthy = health.IsHealthy(lastTerm, targetLastIndex, autopilotConf)

	// If this is a new server or the health changed, reset StableSince
	lastHealth := s.getServerHealth(server.ID)
	if lastHealth == nil || lastHealth.Healthy != health.Healthy {
		health.StableSince = time.Now()
	} else {
		health.StableSince = lastHealth.StableSince
	}

	return nil
}

func (s *Server) getClusterHealth() structs.OperatorHealthReply {
	s.clusterHealthLock.RLock()
	defer s.clusterHealthLock.RUnlock()
	return s.clusterHealth
}

func (s *Server) getServerHealth(id string) *structs.ServerHealth {
	s.clusterHealthLock.RLock()
	defer s.clusterHealthLock.RUnlock()
	for _, health := range s.clusterHealth.Servers {
		if health.ID == id {
			return &health
		}
	}
	return nil
}

func isVoter(suffrage raft.ServerSuffrage) bool {
	switch suffrage {
	case raft.Voter, raft.Staging:
		return true
	default:
		return false
	}
}
<|endoftext|>