<commit_before>package manager

import (
	"fmt"
	"time"

	"github.com/prometheus/common/model"
)

var DefaultRouteOpts = RouteOpts{
	GroupWait:     10 * time.Second,
	GroupInterval: 10 * time.Second,
}

type Routes []*Route

func (rs Routes) Match(lset model.LabelSet) []*RouteOpts {
	fakeParent := &Route{
		Routes:    rs,
		RouteOpts: DefaultRouteOpts,
	}
	return fakeParent.Match(lset)
}

// A Route is a node that contains definitions of how to handle alerts.
type Route struct {
	// The configuration parameters for matches of this route.
	RouteOpts RouteOpts

	// Equality or regex matchers an alert has to fulfill to match
	// this route.
	Matchers Matchers

	// If true, an alert matches further routes on the same level.
	Continue bool

	// Children routes of this route.
	Routes Routes
}

// Match does a depth-first left-to-right search through the route tree
// and returns the flattened configuration for the reached node.
func (r *Route) Match(lset model.LabelSet) []*RouteOpts {
	if !r.Matchers.Match(lset) {
		return nil
	}

	var all []*RouteOpts

	for _, cr := range r.Routes {
		matches := cr.Match(lset)

		for _, ro := range matches {
			ro.populateDefault(&r.RouteOpts)
		}

		all = append(all, matches...)

		if matches != nil && !cr.Continue {
			break
		}
	}

	if len(all) == 0 {
		all = append(all, &r.RouteOpts)
	}

	return all
}

// UnmarshalYAML implements the yaml.Unmarshaler interface.
func (r *Route) UnmarshalYAML(unmarshal func(interface{}) error) error {
	type route struct {
		SendTo        string            `yaml:"send_to,omitempty"`
		GroupBy       []model.LabelName `yaml:"group_by,omitempty"`
		GroupWait     *model.Duration   `yaml:"group_wait,omitempty"`
		GroupInterval *model.Duration   `yaml:"group_interval,omitempty"`

		Match    map[string]string `yaml:"match,omitempty"`
		MatchRE  map[string]string `yaml:"match_re,omitempty"`
		Continue bool              `yaml:"continue,omitempty"`
		Routes   []*Route          `yaml:"routes,omitempty"`

		// Catches all undefined fields and must be empty after parsing.
		XXX map[string]interface{} `yaml:",inline"`
	}
	var v route
	if err := unmarshal(&v); err != nil {
		return err
	}

	for k, val := range v.Match {
		if !model.LabelNameRE.MatchString(k) {
			fmt.Errorf("invalid label name %q", k)
		}
		ln := model.LabelName(k)
		r.Matchers = append(r.Matchers, NewMatcher(ln, val))
	}

	for k, val := range v.MatchRE {
		if !model.LabelNameRE.MatchString(k) {
			fmt.Errorf("invalid label name %q", k)
		}
		ln := model.LabelName(k)

		m, err := NewRegexMatcher(ln, val)
		if err != nil {
			return err
		}
		r.Matchers = append(r.Matchers, m)
	}

	r.RouteOpts.GroupBy = make(map[model.LabelName]struct{}, len(v.GroupBy))

	for _, ln := range v.GroupBy {
		if _, ok := r.RouteOpts.GroupBy[ln]; ok {
			return fmt.Errorf("duplicated label %q in group_by", ln)
		}
		r.RouteOpts.GroupBy[ln] = struct{}{}
	}

	if v.GroupWait != nil {
		r.RouteOpts.GroupWait = time.Duration(*v.GroupWait)
		r.RouteOpts.hasWait = true
	}
	if v.GroupInterval != nil {
		r.RouteOpts.GroupInterval = time.Duration(*v.GroupInterval)
		r.RouteOpts.hasInterval = true
	}
	r.RouteOpts.SendTo = v.SendTo

	r.Continue = v.Continue
	r.Routes = v.Routes

	return checkOverflow(v.XXX, "route")
}

type RouteOpts struct {
	// The identifier of the associated notification configuration
	SendTo string

	// What labels to group alerts by for notifications.
	GroupBy map[model.LabelName]struct{}

	// How long to wait to group matching alerts before sending
	// a notificaiton
	GroupWait     time.Duration
	GroupInterval time.Duration

	hasWait, hasInterval bool
}

func (ro *RouteOpts) String() string {
	var labels []model.LabelName
	for ln := range ro.GroupBy {
		labels = append(labels, ln)
	}
	return fmt.Sprintf("<RouteOpts send_to:%q group_by:%q group_wait:%q>", ro.SendTo, labels, ro.GroupWait)
}

func (ro *RouteOpts) populateDefault(parent *RouteOpts) {
	for ln := range parent.GroupBy {
		if _, ok := ro.GroupBy[ln]; !ok {
			ro.GroupBy[ln] = struct{}{}
		}
	}
	if ro.SendTo == "" {
		ro.SendTo = parent.SendTo
	}
	if !ro.hasWait {
		ro.GroupWait = parent.GroupWait
	}
	if !ro.hasInterval {
		ro.GroupInterval = parent.GroupInterval
	}
}
<commit_msg>Rename GroupInterval to RepeatInterval<commit_after>package manager

import (
	"fmt"
	"time"

	"github.com/prometheus/common/model"
)

var DefaultRouteOpts = RouteOpts{
	GroupWait:     10 * time.Second,
	GroupInterval: 10 * time.Second,
}

type Routes []*Route

func (rs Routes) Match(lset model.LabelSet) []*RouteOpts {
	fakeParent := &Route{
		Routes:    rs,
		RouteOpts: DefaultRouteOpts,
	}
	return fakeParent.Match(lset)
}

// A Route is a node that contains definitions of how to handle alerts.
type Route struct {
	// The configuration parameters for matches of this route.
	RouteOpts RouteOpts

	// Equality or regex matchers an alert has to fulfill to match
	// this route.
	Matchers Matchers

	// If true, an alert matches further routes on the same level.
	Continue bool

	// Children routes of this route.
	Routes Routes
}

// Match does a depth-first left-to-right search through the route tree
// and returns the flattened configuration for the reached node.
func (r *Route) Match(lset model.LabelSet) []*RouteOpts {
	if !r.Matchers.Match(lset) {
		return nil
	}

	var all []*RouteOpts

	for _, cr := range r.Routes {
		matches := cr.Match(lset)

		for _, ro := range matches {
			ro.populateDefault(&r.RouteOpts)
		}

		all = append(all, matches...)

		if matches != nil && !cr.Continue {
			break
		}
	}

	if len(all) == 0 {
		all = append(all, &r.RouteOpts)
	}

	return all
}

// UnmarshalYAML implements the yaml.Unmarshaler interface.
func (r *Route) UnmarshalYAML(unmarshal func(interface{}) error) error {
	type route struct {
		SendTo         string            `yaml:"send_to,omitempty"`
		GroupBy        []model.LabelName `yaml:"group_by,omitempty"`
		GroupWait      *model.Duration   `yaml:"group_wait,omitempty"`
		RepeatInterval *model.Duration   `yaml:"repeat_interval,omitempty"`

		Match    map[string]string `yaml:"match,omitempty"`
		MatchRE  map[string]string `yaml:"match_re,omitempty"`
		Continue bool              `yaml:"continue,omitempty"`
		Routes   []*Route          `yaml:"routes,omitempty"`

		// Catches all undefined fields and must be empty after parsing.
		XXX map[string]interface{} `yaml:",inline"`
	}
	var v route
	if err := unmarshal(&v); err != nil {
		return err
	}

	for k, val := range v.Match {
		if !model.LabelNameRE.MatchString(k) {
			fmt.Errorf("invalid label name %q", k)
		}
		ln := model.LabelName(k)
		r.Matchers = append(r.Matchers, NewMatcher(ln, val))
	}

	for k, val := range v.MatchRE {
		if !model.LabelNameRE.MatchString(k) {
			fmt.Errorf("invalid label name %q", k)
		}
		ln := model.LabelName(k)

		m, err := NewRegexMatcher(ln, val)
		if err != nil {
			return err
		}
		r.Matchers = append(r.Matchers, m)
	}

	r.RouteOpts.GroupBy = make(map[model.LabelName]struct{}, len(v.GroupBy))

	for _, ln := range v.GroupBy {
		if _, ok := r.RouteOpts.GroupBy[ln]; ok {
			return fmt.Errorf("duplicated label %q in group_by", ln)
		}
		r.RouteOpts.GroupBy[ln] = struct{}{}
	}

	if v.GroupWait != nil {
		r.RouteOpts.GroupWait = time.Duration(*v.GroupWait)
		r.RouteOpts.hasWait = true
	}
	if v.RepeatInterval != nil {
		r.RouteOpts.RepeatInterval = time.Duration(*v.RepeatInterval)
		r.RouteOpts.hasInterval = true
	}
	r.RouteOpts.SendTo = v.SendTo

	r.Continue = v.Continue
	r.Routes = v.Routes

	return checkOverflow(v.XXX, "route")
}

type RouteOpts struct {
	// The identifier of the associated notification configuration
	SendTo string

	// What labels to group alerts by for notifications.
	GroupBy map[model.LabelName]struct{}

	// How long to wait to group matching alerts before sending
	// a notificaiton
	GroupWait      time.Duration
	RepeatInterval time.Duration

	hasWait, hasInterval bool
}

func (ro *RouteOpts) String() string {
	var labels []model.LabelName
	for ln := range ro.GroupBy {
		labels = append(labels, ln)
	}
	return fmt.Sprintf("<RouteOpts send_to:%q group_by:%q group_wait:%q>", ro.SendTo, labels, ro.GroupWait)
}

func (ro *RouteOpts) populateDefault(parent *RouteOpts) {
	for ln := range parent.GroupBy {
		if _, ok := ro.GroupBy[ln]; !ok {
			ro.GroupBy[ln] = struct{}{}
		}
	}
	if ro.SendTo == "" {
		ro.SendTo = parent.SendTo
	}
	if !ro.hasWait {
		ro.GroupWait = parent.GroupWait
	}
	if !ro.hasInterval {
		ro.RepeatInterval = parent.RepeatInterval
	}
}
<|endoftext|>


<commit_before>package mapnik

// #include "mapnik_c_api.h"
import "C"

import (
	"unsafe"
)

func init() {
	// register default datasources path and fonts path like the python bindings do
	RegisterDatasources(pluginPath)
	RegisterFonts(fontPath)
}

func RegisterDatasources(path string) {
	C.mapnik_register_datasources(C.CString(path))
}

func RegisterFonts(path string) {
	C.mapnik_register_fonts(C.CString(path))
}

// Point in 2D space
type Coord struct {
	X, Y float64
}

// Projection from one reference system to the other
type Projection struct {
	p *C.struct_mapnik_projection_t
}

func (p *Projection) Free() {
	C.mapnik_projection_free(p.p)
	p.p = nil
}

func (p Projection) Forward(coord Coord) Coord {
	c := C.mapnik_coord_t{C.double(coord.X), C.double(coord.Y)}
	c = C.mapnik_projection_forward(p.p, c)
	return Coord{float64(c.x), float64(c.y)}
}

// Map base type
type Map struct {
	m *C.struct_mapnik_map_t
}

func NewMap(width, height uint32) *Map {
	return &Map{C.mapnik_map(C.uint(width), C.uint(height))}
}

func (m *Map) Load(stylesheet string) {
	C.mapnik_map_load(m.m, C.CString(stylesheet))
}

func (m *Map) Resize(width, height uint32) {
	C.mapnik_map_resize(m.m, C.uint(width), C.uint(height))
}

func (m *Map) Free() {
	C.mapnik_map_free(m.m)
	m.m = nil
}

func (m *Map) SRS() string {
	return C.GoString(C.mapnik_map_get_srs(m.m))
}

func (m *Map) SetSRS(srs string) {
	C.mapnik_map_set_srs(m.m, C.CString(srs))
}

func (m *Map) ZoomAll() {
	C.mapnik_map_zoom_all(m.m)
}

func (m *Map) ZoomToMinMax(minx, miny, maxx, maxy float64) {
	bbox := C.mapnik_bbox(C.double(minx), C.double(miny), C.double(maxx), C.double(maxy))
	defer C.mapnik_bbox_free(bbox)
	C.mapnik_map_zoom_to_box(m.m, bbox)
}

func (m *Map) RenderToFile(path string) {
	C.mapnik_map_render_to_file(m.m, C.CString(path))
}

func (m *Map) RenderToMemoryPng() []byte {
	i := C.mapnik_map_render_to_image(m.m)
	defer C.mapnik_image_free(i)
	b := C.mapnik_image_to_png_blob(i)
	defer C.mapnik_image_blob_free(b)
	return C.GoBytes(unsafe.Pointer(b.ptr), C.int(b.len))
}

func (m *Map) Projection() Projection {
	p := Projection{}
	p.p = C.mapnik_map_projection(m.m)
	return p
}

func (m *Map) SetBufferSize(s int) {
	C.mapnik_map_set_buffer_size(m.m, C.int(s))
}
<commit_msg>add error return value for function that can fail<commit_after>package mapnik

// #include "mapnik_c_api.h"
import "C"

import (
	"errors"
	"unsafe"
)

func init() {
	// register default datasources path and fonts path like the python bindings do
	RegisterDatasources(pluginPath)
	RegisterFonts(fontPath)
}

func RegisterDatasources(path string) {
	C.mapnik_register_datasources(C.CString(path))
}

func RegisterFonts(path string) {
	C.mapnik_register_fonts(C.CString(path))
}

// Point in 2D space
type Coord struct {
	X, Y float64
}

// Projection from one reference system to the other
type Projection struct {
	p *C.struct_mapnik_projection_t
}

func (p *Projection) Free() {
	C.mapnik_projection_free(p.p)
	p.p = nil
}

func (p Projection) Forward(coord Coord) Coord {
	c := C.mapnik_coord_t{C.double(coord.X), C.double(coord.Y)}
	c = C.mapnik_projection_forward(p.p, c)
	return Coord{float64(c.x), float64(c.y)}
}

// Map base type
type Map struct {
	m *C.struct_mapnik_map_t
}

func NewMap(width, height uint32) *Map {
	return &Map{C.mapnik_map(C.uint(width), C.uint(height))}
}

func (m *Map) lastError() error {
	return errors.New("mapnik: " + C.GoString(C.mapnik_map_last_error(m.m)))
}

func (m *Map) Load(stylesheet string) error {
	if C.mapnik_map_load(m.m, C.CString(stylesheet)) != 0 {
		return m.lastError()
	}
	return nil
}

func (m *Map) Resize(width, height uint32) {
	C.mapnik_map_resize(m.m, C.uint(width), C.uint(height))
}

func (m *Map) Free() {
	C.mapnik_map_free(m.m)
	m.m = nil
}

func (m *Map) SRS() string {
	return C.GoString(C.mapnik_map_get_srs(m.m))
}

func (m *Map) SetSRS(srs string) {
	C.mapnik_map_set_srs(m.m, C.CString(srs))
}

func (m *Map) ZoomAll() error {
	if C.mapnik_map_zoom_all(m.m) != 0 {
		return m.lastError()
	}
	return nil
}

func (m *Map) ZoomToMinMax(minx, miny, maxx, maxy float64) {
	bbox := C.mapnik_bbox(C.double(minx), C.double(miny), C.double(maxx), C.double(maxy))
	defer C.mapnik_bbox_free(bbox)
	C.mapnik_map_zoom_to_box(m.m, bbox)
}

func (m *Map) RenderToFile(path string) error {
	if C.mapnik_map_render_to_file(m.m, C.CString(path)) != 0 {
		return m.lastError()
	}
	return nil
}

func (m *Map) RenderToMemoryPng() ([]byte, error) {
	i := C.mapnik_map_render_to_image(m.m)
	if i == nil {
		return nil, m.lastError()
	}
	defer C.mapnik_image_free(i)
	b := C.mapnik_image_to_png_blob(i)
	defer C.mapnik_image_blob_free(b)
	return C.GoBytes(unsafe.Pointer(b.ptr), C.int(b.len)), nil
}

func (m *Map) Projection() Projection {
	p := Projection{}
	p.p = C.mapnik_map_projection(m.m)
	return p
}

func (m *Map) SetBufferSize(s int) {
	C.mapnik_map_set_buffer_size(m.m, C.int(s))
}
<|endoftext|>


<commit_before>package main

import (
	. "github.com/SashaCrofter/cjdngo/admin"
	"github.com/SashaCrofter/network-markup/nmparser"
	"math"
)

// Filter iterates through every route in the given table, and
// discards any Routes that are more than maxHops (discounted if <1)
// away from the origin, or that pass through any of the IP addresses
// as identified by doNotGoThrough. Any domain names that are intended
// to be used should be preresolved. It returns the remaining routes.
func Filter(table []*Route, maxHops int, doNotGoThrough []string) (filtered []*Route) {
	useHops := maxHops > 0

	if !useHops && len(doNotGoThrough) == 0 {
		// If we can avoid filtering altogether, by all means do.
		l.Println("No filters applied")
		return table
	}
	// Otherwise, report the arguments.
	l.Println("Filtering by hops:", useHops)
	l.Println("Number of bad IPs:", len(doNotGoThrough))

	// Call it bad practice, but it's convenient, here.
	containsBadHop := func(hops []*Route) bool {
		if len(doNotGoThrough) < 1 {
			return false
		}
		// For every given hop,
		for _, route := range hops {
			// check every string in doNotGoThrough.
			for _, badHop := range doNotGoThrough {
				if route.IP == badHop {
					// Only return true if at least one hop is
					// considered bad.
					l.Println("Discarding route that passes through",
						route.IP)
					return true
				}
			}
		}
		return false
	}

	for _, route := range table {
		hops := getNodesOnPath(table, route.Path)
		if !(useHops && len(hops) > maxHops) && !containsBadHop(hops) {
			// If there were few enough hops, and it did not pass
			// through any blacklisted nodes, then add it to the array
			// to be returned.
			filtered = append(filtered, route)
		}
		// Otherwise, it is discarded.
	}
	return
}

func ToNetwork(routes []*Route) (network nmparser.Network) {
	network = make(nmparser.Network)
	// For every route in the given table, then use FilterRoutes() to
	// get all of the hops on that route. Use the final hop in that
	// result as a peer. Then, filter duplicates.
	for _, route := range routes {
		node, isPresent := network[route.IP]
		if !isPresent {
			// If the node isn't already present, make it.
			node = &nmparser.Node{
				Connected:  make([]string, 0),
				Attributes: make(map[string]interface{}),
			}
		}

		// Get all of the hops in that route
		hops := getNodesOnPath(routes, route.Path)
		// Because of self-routing bugs, it's necessary to look
		// through the list of hops.
		i := len(hops) - 1
		for {
			if i >= 0 {
				// Then, if there were any, (this should always be
				// true, but let's avoid runtime errors), append its
				// IP to the Connected attribute of the current Node.
				lastHop := hops[i].IP
				if route.IP != lastHop {
					l.Println("Node", route.IP, "connected to", lastHop)
					node.Connected = append(node.Connected, lastHop)
					break
				} else {
					l.Println("Got self-connection on", route.IP)
				}
				i--
			} else {
				// If we get a zero-length path, report it, but don't
				// do anything.
				l.Println("Got broken path for", route.IP)
				break
			}
		}
		network[route.IP] = node
	}
	/*
		// Now, we have to filter out duplicate connections.
		for _, node := range network {
			// Copy all Connected entries into a map[string]interface{},
			// then copy them back.
			connectedMap := make(map[string]interface{})
			for _, connection := range node.Connected {
				connectedMap[connection] = nil
			}
			node.Connected = make([]string, 0, len(connectedMap))
			for k := range connectedMap {
				node.Connected = append(node.Connected, k)
			}
		}*/
	return
}

func getNodesOnPath(table []*Route, target uint64) (hops []*Route) {
	for _, route := range table {

		g := 64 - uint64(math.Log2(float64(route.Path)))
		h := uint64(uint64(0xffffffffffffffff) >> g)

		if h&target == h&route.Path {
			hops = append(hops, route)
		}
	}
	return
}
<commit_msg>Fix some endpoint bugs, but it's not flawless<commit_after>package main

import (
	. "github.com/SashaCrofter/cjdngo/admin"
	"github.com/SashaCrofter/network-markup/nmparser"
	"math"
)

// Filter iterates through every route in the given table, and
// discards any Routes that are more than maxHops (discounted if <1)
// away from the origin, or that pass through any of the IP addresses
// as identified by endpoint. Any domain names that are intended to be
// used should be preresolved. It returns the remaining routes.
func Filter(table []*Route, maxHops int, endpoint []string) (filtered []*Route) {
	useHops := maxHops > 0

	if !useHops && len(endpoint) == 0 {
		// If we can avoid filtering altogether, by all means do.
		l.Println("No filters applied")
		return table
	}
	// Otherwise, report the arguments.
	l.Println("Filtering by hops:", useHops)
	l.Println("Number of end IPs:", len(endpoint))

	// Call it bad practice, but it's convenient, here.
	containsBadHop := func(hops []*Route) bool {
		if len(endpoint) < 1 {
			return false
		}
		// For every given hop, except for the last one
		for _, route := range hops[:len(hops)-1] {
			// check every string in endpoint.
			for _, endHop := range endpoint {
				if route.IP == endHop {
					// Only return true if at least one hop is
					// considered bad.
					l.Println("Discarding route that passes through",
						route.IP)
					return true
				}
			}
		}
		return false
	}

	for _, route := range table {
		hops := getNodesOnPath(table, route.Path)
		if !(useHops && len(hops) > maxHops) && !containsBadHop(hops) {
			// If there were few enough hops, and it did not pass
			// through any blacklisted nodes, then add it to the array
			// to be returned.
			filtered = append(filtered, route)
		}
		// Otherwise, it is discarded.
	}
	return
}

func ToNetwork(routes []*Route) (network nmparser.Network) {
	network = make(nmparser.Network)
	// For every route in the given table, then use FilterRoutes() to
	// get all of the hops on that route. Use the final hop in that
	// result as a peer. Then, filter duplicates.
	for _, route := range routes {
		node, isPresent := network[route.IP]
		if !isPresent {
			// If the node isn't already present, make it.
			node = &nmparser.Node{
				Connected:  make([]string, 0),
				Attributes: make(map[string]interface{}),
			}
		}

		// Get all of the hops in that route
		hops := getNodesOnPath(routes, route.Path)
		// Because of self-routing bugs, it's necessary to look
		// through the list of hops.
		i := len(hops) - 1
		for {
			if i >= 0 {
				// Then, if there were any, (this should always be
				// true, but let's avoid runtime errors), append its
				// IP to the Connected attribute of the current Node.
				lastHop := hops[i].IP
				if route.IP != lastHop {
					l.Println("Node", route.IP, "connected to", lastHop)
					node.Connected = append(node.Connected, lastHop)
					break
				} else {
					l.Println("Got self-connection on", route.IP)
				}
				i--
			} else {
				// If we get a zero-length path, report it, but don't
				// do anything.
				l.Println("Got broken path for", route.IP)
				break
			}
		}
		network[route.IP] = node
	}
	/*
		// Now, we have to filter out duplicate connections.
		for _, node := range network {
			// Copy all Connected entries into a map[string]interface{},
			// then copy them back.
			connectedMap := make(map[string]interface{})
			for _, connection := range node.Connected {
				connectedMap[connection] = nil
			}
			node.Connected = make([]string, 0, len(connectedMap))
			for k := range connectedMap {
				node.Connected = append(node.Connected, k)
			}
		}*/
	return
}

func getNodesOnPath(table []*Route, target uint64) (hops []*Route) {
	for _, route := range table {

		g := 64 - uint64(math.Log2(float64(route.Path)))
		h := uint64(uint64(0xffffffffffffffff) >> g)

		if h&target == h&route.Path {
			hops = append(hops, route)
		}
	}
	return
}
<|endoftext|>


<commit_before>package markdown

import (
	"io/ioutil"
	"path"
	"strings"
	"testing"
)

func TestParse(t *testing.T) {
	m := new(Markdown)
	files, _ := ioutil.ReadDir("./data")
	for _, f := range files {
		fileName := f.Name()
		filePath := path.Join("./data/", fileName)
		fileExt := path.Ext(filePath)
		if ".md" != fileExt {
			continue
		}
		markdown, _ := ioutil.ReadFile(filePath)
		input := string(markdown)

		basename := strings.Replace(fileName, fileExt, "", 1)
		htmlPath := strings.Join([]string{"./data/", basename, ".html"}, "")

		html, _ := ioutil.ReadFile(htmlPath)
		expected := string(html)

		result := m.parse(input)
		if result != expected {
			t.Errorf("'%s' expected but was '%s'.", expected, result)
		}

	}
}
<commit_msg>Replace method of struct with function.<commit_after>package markdown

import (
	"io/ioutil"
	"path"
	"strings"
	"testing"
)

func TestParse(t *testing.T) {
	files, _ := ioutil.ReadDir("./data")
	for _, f := range files {
		fileName := f.Name()
		filePath := path.Join("./data/", fileName)
		fileExt := path.Ext(filePath)
		if ".md" != fileExt {
			continue
		}
		markdown, _ := ioutil.ReadFile(filePath)
		input := string(markdown)

		basename := strings.Replace(fileName, fileExt, "", 1)
		htmlPath := strings.Join([]string{"./data/", basename, ".html"}, "")

		html, _ := ioutil.ReadFile(htmlPath)
		expected := string(html)

		result := parse(input)
		if result != expected {
			t.Errorf("'%s' expected but was '%s'.", expected, result)
		}

	}
}
<|endoftext|>


<commit_before>/*
 * portal - marshal
 *
 * a library that implements an algorithm for doing consumer coordination within Kafka, rather
 * than using Zookeeper or another external system.
 *
 */

package marshal

import (
	"math/rand"
	"sync"
	"sync/atomic"
	"time"

	"github.com/optiopay/kafka"
	"github.com/optiopay/kafka/proto"
)

// claim is instantiated for each partition "claim" we have. This type is responsible for
// pulling data from Kafka and managing its cursors, heartbeating as necessary, and health
// checking itself.
type claim struct {
	// These items are read-only. They are never changed after the object is created,
	// so access to these may be done without the lock.
	topic  string
	partID int

	// lock protects all access to the member variables of this struct except for the
	// messages channel, which can be read from or written to without holding the lock.
	lock          sync.RWMutex
	marshal       *Marshaler
	rand          *rand.Rand
	claimed       *int32
	lastHeartbeat int64
	consumer      kafka.Consumer
	messages      chan *proto.Message

	// Number of heartbeat cycles this claim has been lagging, i.e., consumption is going
	// too slowly (defined as being behind by more than 2 heartbeat cycles)
	cyclesBehind int

	// A Kafka partition consists of N messages with offsets. In the basic case, you
	// can think of an offset like an array index. With log compaction and other trickery
	// it acts more like a sparse array, but it's a close enough metaphor.
	//
	// We keep track of four values for offsets:
	//
	//    offsets       1     2     3     7     9    10    11
	//   partition  [ msg1, msg2, msg3, msg4, msg5, msg6, msg7, ... ]
	//                 ^                  ^                      ^
	//                 \- offsetEarliest  |                      |
	//                                    \- offsetCurrent   offsetLatest
	//                                     and startOffset
	//
	// In this example, offsetEarliest is 1 which is the "oldest" offset within the
	// partition. At any given time this offset might become invalid (if a log rolls)
	// so we might update it. startOffset is simply the value of offsetCurrent at
	// the very beginning of consumption (for tracking velocity).
	//
	// offsetCurrent is 7, which is the offset of the NEXT message i.e. this message
	// has not been consumed yet.
	//
	// offsetLatest is 12, which is the offset that Kafka will assign to the message
	// that next gets committed to the partition. This offset does not yet exist,
	// and mit might never.
	offsetCurrent  int64
	offsetEarliest int64
	offsetLatest   int64

	// History arrays used for calculating average veolocity for health checking.
	offsetCurrentHistory [10]int64
	offsetLatestHistory  [10]int64
}

// newClaim returns an internal claim object, used by the consumer to manage the
// claim of a single partition.
func newClaim(topic string, partID int, marshal *Marshaler) *claim {
	// Get all available offset information
	oEarly, oLate, oCur, err := marshal.GetPartitionOffsets(topic, partID)
	if err != nil {
		log.Errorf("%s:%d failed to get offsets: %s", topic, partID, err)
		return nil
	}
	log.Debugf("%s:%d consumer offsets: early = %d, cur = %d, late = %d",
		topic, partID, oEarly, oCur, oLate)

	// Construct object and set it up
	obj := &claim{
		marshal:        marshal,
		topic:          topic,
		partID:         partID,
		claimed:        new(int32),
		offsetEarliest: oEarly,
		offsetLatest:   oLate,
		offsetCurrent:  oCur,
		messages:       make(chan *proto.Message, 100),
		rand:           rand.New(rand.NewSource(time.Now().UnixNano())),
	}
	atomic.StoreInt32(obj.claimed, 1)

	// Now try to actually claim it, this can block a while
	log.Infof("%s:%d consumer attempting to claim", topic, partID)
	if !marshal.ClaimPartition(topic, partID) {
		log.Infof("%s:%d consumer failed to claim", topic, partID)
		return nil
	}

	// If that worked, kick off the main setup loop and return
	obj.setup()
	return obj
}

// updateOffsets polls Kafka periodically to get information about the partition's
// state.
func (c *claim) updateOffsetsLoop() {
	ctr := 0
	time.Sleep(<-c.marshal.jitters)
	for c.Claimed() {
		c.updateOffsets(ctr)
		ctr++
		time.Sleep(<-c.marshal.jitters)
	}
	log.Debugf("%s:%d no longer claimed, offset loop exiting", c.topic, c.partID)
}

// setup is the initial worker that initializes the claim structure. Until this is done,
// our internal state is inconsistent.
func (c *claim) setup() {
	c.lock.Lock()
	defer c.lock.Unlock()

	// Of course, if the current offset is greater than the earliest, we must reset
	// to the earliest known
	if c.offsetCurrent < c.offsetEarliest {
		log.Warningf("%s:%d consumer fast-forwarding from %d to %d",
			c.topic, c.partID, c.offsetCurrent, c.offsetEarliest)
		c.offsetCurrent = c.offsetEarliest
	}

	// Since it's claimed, we now want to heartbeat with the last seen offset
	err := c.marshal.Heartbeat(c.topic, c.partID, c.offsetCurrent)
	if err != nil {
		log.Errorf("%s:%d consumer failed to heartbeat: %s", c.topic, c.partID, err)
		atomic.StoreInt32(c.claimed, 0)
		return
	}
	c.lastHeartbeat = time.Now().Unix()

	// Set up Kafka consumer
	consumerConf := kafka.NewConsumerConf(c.topic, int32(c.partID))
	consumerConf.StartOffset = c.offsetCurrent
	kafkaConsumer, err := c.marshal.kafka.Consumer(consumerConf)
	if err != nil {
		log.Errorf("%s:%d consumer failed to create Kafka Consumer: %s",
			c.topic, c.partID, err)
		// TODO: There is an optimization here where we could release the partition.
		// As it stands, we're not doing anything,
		atomic.StoreInt32(c.claimed, 0)
		return
	}
	c.consumer = kafkaConsumer

	// Start our maintenance goroutines that keep this system healthy
	go c.updateOffsetsLoop()
	go c.healthCheckLoop()
	go c.messagePump()

	// Totally done, let the world know and move on
	log.Infof("%s:%d consumer claimed at offset %d (is %d behind)",
		c.topic, c.partID, c.offsetCurrent, c.offsetLatest-c.offsetCurrent)
}

// Given a message offset (which should be our current offset), return whether or
// not that offset is allowed to be consumed (i.e., we're still claimed)
func (c *claim) Consumed(offset int64) bool {
	if atomic.LoadInt32(c.claimed) != 1 {
		return false
	}

	c.lock.Lock()
	defer c.lock.Unlock()

	c.offsetCurrent = offset + 1
	return true
}

// Claimed returns whether or not this claim structure is alive and well and believes
// that it is still an active claim.
func (c *claim) Claimed() bool {
	return atomic.LoadInt32(c.claimed) == 1
}

// GetCurrentLag returns this partition's cursor lag.
func (c *claim) GetCurrentLag() int64 {
	c.lock.RLock()
	defer c.lock.RUnlock()

	if c.offsetCurrent < c.offsetLatest {
		return c.offsetLatest - c.offsetCurrent
	}
	return 0
}

// Release will invoke our release mechanism if and only if we are still claimed.
func (c *claim) Release() bool {
	if !atomic.CompareAndSwapInt32(c.claimed, 1, 0) {
		return false
	}

	// Holds the lock through a Kafka transaction, but since we're releasing I think this
	// is reasonable. Held because of using offsetCurrent below.
	c.lock.RLock()
	defer c.lock.RUnlock()

	log.Infof("%s:%d releasing partition claim", c.topic, c.partID)
	err := c.marshal.ReleasePartition(c.topic, c.partID, c.offsetCurrent)
	if err != nil {
		log.Errorf("%s:%d failed to release: %s", c.topic, c.partID, err)
		return false
	}
	return true
}

// messagePump continuously pulls message from Kafka for this partition and makes them
// available for consumption.
func (c *claim) messagePump() {
	// This method MUST NOT make changes to the claim structure. Since we might
	// be running while someone else has the lock, and we can't get it ourselves, we are
	// forbidden to touch anything other than the consumer and the message channel.
	for c.Claimed() {
		msg, err := c.consumer.Consume()
		if err == proto.ErrOffsetOutOfRange {
			// Fell out of range, presumably because we're handling this too slow, so
			// let's abandon this consumer
			log.Errorf("%s:%d error consuming: out of range, abandoning partition",
				c.topic, c.partID)
			go c.Release()
			return
		}
		if err != nil {
			log.Errorf("%s:%d error consuming: %s", c.topic, c.partID, err)

			// Often a consumption error is caused by data going away, such as if we're consuming
			// from the head and Kafka has deleted the data. In that case we need to wait for
			// the next offset update, so let's not go crazy
			time.Sleep(1 * time.Second)
			continue
		}

		c.messages <- msg
	}
	log.Debugf("%s:%d no longer claimed, pump exiting", c.topic, c.partID)
}

// heartbeat is the internal "send a heartbeat" function. Calling this will immediately
// send a heartbeat to Kafka. If we fail to send a heartbeat, we will release the
// partition.
func (c *claim) heartbeat() {
	c.lock.Lock()
	defer c.lock.Unlock()

	err := c.marshal.Heartbeat(c.topic, c.partID, c.offsetCurrent)
	if err != nil {
		log.Errorf("%s:%d failed to heartbeat, releasing", c.topic, c.partID)
		go c.Release()
	}
	c.lastHeartbeat = time.Now().Unix()
}

// healthCheck performs a single health check against the claim. If we have failed
// too many times, this will also start a partition release. Returns true if the
// partition is healthy, else false.
func (c *claim) healthCheck() bool {
	// Get velocities; these functions both use the locks so we have to do this before
	// we personally take the lock (to avoid deadlock)
	consumerVelocity := c.ConsumerVelocity()
	partitionVelocity := c.PartitionVelocity()

	c.lock.Lock()
	defer c.lock.Unlock()

	// If our heartbeat is expired, we are definitely unhealthy... don't even bother
	// with checking velocity
	if c.lastHeartbeat < time.Now().Unix()-HeartbeatInterval {
		log.Warningf("%s:%d consumer unhealthy by heartbeat test, releasing",
			c.topic, c.partID)
		go c.Release()
		return false
	}

	// If current has gone forward of the latest (which is possible, but unlikely)
	// then we are by definition caught up
	if c.offsetCurrent >= c.offsetLatest {
		c.cyclesBehind = 0
		return true
	}

	// If velocity is good, reset cycles behind and exit
	if partitionVelocity <= consumerVelocity {
		c.cyclesBehind = 0
		return true
	}

	// Unhealthy, so increase the cycle count so we know when it's been unhealthy for
	// too long and we want to give it up
	c.cyclesBehind++

	// If were behind by too many cycles, then we should try to release the
	// partition. If so, do this in a goroutine since it will involve calling out
	// to Kafka and releasing the partition.
	if c.cyclesBehind >= 3 {
		log.Errorf("%s:%d consumer unhealthy, releasing",
			c.topic, c.partID)
		go c.Release()
		return false
	} else {
		log.Warningf("%s:%d consumer unhealthy: CV %0.2f < PV %0.2f",
			c.topic, c.partID, consumerVelocity, partitionVelocity)
	}

	// Clearly we haven't been behind for long enough, so we're still "healthy"
	return true
}

// healthCheckLoop runs regularly and will perform a health check. Exits when we are no longer
// a claimed partition
func (c *claim) healthCheckLoop() {
	time.Sleep(<-c.marshal.jitters)
	for c.Claimed() {
		if c.healthCheck() {
			go c.heartbeat()
		}
		time.Sleep(<-c.marshal.jitters)
	}
	log.Debugf("%s:%d health check loop exiting", c.topic, c.partID)
}

// average returns the average of a given slice of int64s. It ignores 0s as
// those are "uninitialized" elements.
func average(vals []int64) float64 {
	min, max, ct := int64(0), int64(0), int64(0)
	for _, val := range vals {
		if val <= 0 {
			continue
		}
		if min == 0 || val < min {
			min = val
		}
		if max == 0 || val > max {
			max = val
		}
		ct++
	}
	if ct == 0 {
		return 0
	}
	return float64(max-min) / float64(ct)
}

// ConsumerVelocity returns the average of our consumers' velocity
func (c *claim) ConsumerVelocity() float64 {
	c.lock.RLock()
	defer c.lock.RUnlock()

	return average(c.offsetCurrentHistory[0:])
}

// PartitionVelocity returns the average of the partition's velocity
func (c *claim) PartitionVelocity() float64 {
	c.lock.RLock()
	defer c.lock.RUnlock()

	return average(c.offsetLatestHistory[0:])
}

// updateOffsets will update the offsets of our current partition.
func (c *claim) updateOffsets(ctr int) error {
	// Slow, hits Kafka. Run in a goroutine.
	oEarly, oLate, _, err := c.marshal.GetPartitionOffsets(c.topic, c.partID)
	if err != nil {
		log.Errorf("%s:%d failed to get offsets: %s", c.topic, c.partID, err)
		return err
	}

	c.lock.Lock()
	defer c.lock.Unlock()

	// Update the earliest/latest offsets that are presently available within the
	// partition
	c.offsetEarliest = oEarly
	c.offsetLatest = oLate

	// Do update our "history" values, this is used for calculating moving averages
	// in the health checking function
	c.offsetLatestHistory[ctr%10] = oLate
	c.offsetCurrentHistory[ctr%10] = c.offsetCurrent

	return nil
}
<commit_msg>Don't allow consumption on expired claim<commit_after>/*
 * portal - marshal
 *
 * a library that implements an algorithm for doing consumer coordination within Kafka, rather
 * than using Zookeeper or another external system.
 *
 */

package marshal

import (
	"math/rand"
	"sync"
	"sync/atomic"
	"time"

	"github.com/optiopay/kafka"
	"github.com/optiopay/kafka/proto"
)

// claim is instantiated for each partition "claim" we have. This type is responsible for
// pulling data from Kafka and managing its cursors, heartbeating as necessary, and health
// checking itself.
type claim struct {
	// These items are read-only. They are never changed after the object is created,
	// so access to these may be done without the lock.
	topic  string
	partID int

	// lock protects all access to the member variables of this struct except for the
	// messages channel, which can be read from or written to without holding the lock.
	lock          sync.RWMutex
	marshal       *Marshaler
	rand          *rand.Rand
	claimed       *int32
	lastHeartbeat int64
	consumer      kafka.Consumer
	messages      chan *proto.Message

	// Number of heartbeat cycles this claim has been lagging, i.e., consumption is going
	// too slowly (defined as being behind by more than 2 heartbeat cycles)
	cyclesBehind int

	// A Kafka partition consists of N messages with offsets. In the basic case, you
	// can think of an offset like an array index. With log compaction and other trickery
	// it acts more like a sparse array, but it's a close enough metaphor.
	//
	// We keep track of four values for offsets:
	//
	//    offsets       1     2     3     7     9    10    11
	//   partition  [ msg1, msg2, msg3, msg4, msg5, msg6, msg7, ... ]
	//                 ^                  ^                      ^
	//                 \- offsetEarliest  |                      |
	//                                    \- offsetCurrent   offsetLatest
	//                                     and startOffset
	//
	// In this example, offsetEarliest is 1 which is the "oldest" offset within the
	// partition. At any given time this offset might become invalid (if a log rolls)
	// so we might update it. startOffset is simply the value of offsetCurrent at
	// the very beginning of consumption (for tracking velocity).
	//
	// offsetCurrent is 7, which is the offset of the NEXT message i.e. this message
	// has not been consumed yet.
	//
	// offsetLatest is 12, which is the offset that Kafka will assign to the message
	// that next gets committed to the partition. This offset does not yet exist,
	// and mit might never.
	offsetCurrent  int64
	offsetEarliest int64
	offsetLatest   int64

	// History arrays used for calculating average veolocity for health checking.
	offsetCurrentHistory [10]int64
	offsetLatestHistory  [10]int64
}

// newClaim returns an internal claim object, used by the consumer to manage the
// claim of a single partition.
func newClaim(topic string, partID int, marshal *Marshaler) *claim {
	// Get all available offset information
	oEarly, oLate, oCur, err := marshal.GetPartitionOffsets(topic, partID)
	if err != nil {
		log.Errorf("%s:%d failed to get offsets: %s", topic, partID, err)
		return nil
	}
	log.Debugf("%s:%d consumer offsets: early = %d, cur = %d, late = %d",
		topic, partID, oEarly, oCur, oLate)

	// Construct object and set it up
	obj := &claim{
		marshal:        marshal,
		topic:          topic,
		partID:         partID,
		claimed:        new(int32),
		offsetEarliest: oEarly,
		offsetLatest:   oLate,
		offsetCurrent:  oCur,
		messages:       make(chan *proto.Message, 100),
		rand:           rand.New(rand.NewSource(time.Now().UnixNano())),
	}
	atomic.StoreInt32(obj.claimed, 1)

	// Now try to actually claim it, this can block a while
	log.Infof("%s:%d consumer attempting to claim", topic, partID)
	if !marshal.ClaimPartition(topic, partID) {
		log.Infof("%s:%d consumer failed to claim", topic, partID)
		return nil
	}

	// If that worked, kick off the main setup loop and return
	obj.setup()
	return obj
}

// updateOffsets polls Kafka periodically to get information about the partition's
// state.
func (c *claim) updateOffsetsLoop() {
	ctr := 0
	time.Sleep(<-c.marshal.jitters)
	for c.Claimed() {
		c.updateOffsets(ctr)
		ctr++
		time.Sleep(<-c.marshal.jitters)
	}
	log.Debugf("%s:%d no longer claimed, offset loop exiting", c.topic, c.partID)
}

// setup is the initial worker that initializes the claim structure. Until this is done,
// our internal state is inconsistent.
func (c *claim) setup() {
	c.lock.Lock()
	defer c.lock.Unlock()

	// Of course, if the current offset is greater than the earliest, we must reset
	// to the earliest known
	if c.offsetCurrent < c.offsetEarliest {
		log.Warningf("%s:%d consumer fast-forwarding from %d to %d",
			c.topic, c.partID, c.offsetCurrent, c.offsetEarliest)
		c.offsetCurrent = c.offsetEarliest
	}

	// Since it's claimed, we now want to heartbeat with the last seen offset
	err := c.marshal.Heartbeat(c.topic, c.partID, c.offsetCurrent)
	if err != nil {
		log.Errorf("%s:%d consumer failed to heartbeat: %s", c.topic, c.partID, err)
		atomic.StoreInt32(c.claimed, 0)
		return
	}
	c.lastHeartbeat = time.Now().Unix()

	// Set up Kafka consumer
	consumerConf := kafka.NewConsumerConf(c.topic, int32(c.partID))
	consumerConf.StartOffset = c.offsetCurrent
	kafkaConsumer, err := c.marshal.kafka.Consumer(consumerConf)
	if err != nil {
		log.Errorf("%s:%d consumer failed to create Kafka Consumer: %s",
			c.topic, c.partID, err)
		// TODO: There is an optimization here where we could release the partition.
		// As it stands, we're not doing anything,
		atomic.StoreInt32(c.claimed, 0)
		return
	}
	c.consumer = kafkaConsumer

	// Start our maintenance goroutines that keep this system healthy
	go c.updateOffsetsLoop()
	go c.healthCheckLoop()
	go c.messagePump()

	// Totally done, let the world know and move on
	log.Infof("%s:%d consumer claimed at offset %d (is %d behind)",
		c.topic, c.partID, c.offsetCurrent, c.offsetLatest-c.offsetCurrent)
}

// Given a message offset (which should be our current offset), return whether or
// not that offset is allowed to be consumed (i.e., we're still claimed)
func (c *claim) Consumed(offset int64) bool {
	if atomic.LoadInt32(c.claimed) != 1 {
		return false
	}

	c.lock.Lock()
	defer c.lock.Unlock()

	// If our heartbeat has gone at all stale, don't allow consumption from this
	// partition. In fact, trigger an immediate release without waiting for the
	// healthcheck loop.
	if c.lastHeartbeat < time.Now().Unix()-HeartbeatInterval {
		log.Errorf("%s:%d has gone stale during consumption",
			c.topic, c.partID)
		go c.Release()
		return false
	}

	c.offsetCurrent = offset + 1
	return true
}

// Claimed returns whether or not this claim structure is alive and well and believes
// that it is still an active claim.
func (c *claim) Claimed() bool {
	return atomic.LoadInt32(c.claimed) == 1
}

// GetCurrentLag returns this partition's cursor lag.
func (c *claim) GetCurrentLag() int64 {
	c.lock.RLock()
	defer c.lock.RUnlock()

	if c.offsetCurrent < c.offsetLatest {
		return c.offsetLatest - c.offsetCurrent
	}
	return 0
}

// Release will invoke our release mechanism if and only if we are still claimed.
func (c *claim) Release() bool {
	if !atomic.CompareAndSwapInt32(c.claimed, 1, 0) {
		return false
	}

	// Holds the lock through a Kafka transaction, but since we're releasing I think this
	// is reasonable. Held because of using offsetCurrent below.
	c.lock.RLock()
	defer c.lock.RUnlock()

	log.Infof("%s:%d releasing partition claim", c.topic, c.partID)
	err := c.marshal.ReleasePartition(c.topic, c.partID, c.offsetCurrent)
	if err != nil {
		log.Errorf("%s:%d failed to release: %s", c.topic, c.partID, err)
		return false
	}
	return true
}

// messagePump continuously pulls message from Kafka for this partition and makes them
// available for consumption.
func (c *claim) messagePump() {
	// This method MUST NOT make changes to the claim structure. Since we might
	// be running while someone else has the lock, and we can't get it ourselves, we are
	// forbidden to touch anything other than the consumer and the message channel.
	for c.Claimed() {
		msg, err := c.consumer.Consume()
		if err == proto.ErrOffsetOutOfRange {
			// Fell out of range, presumably because we're handling this too slow, so
			// let's abandon this consumer
			log.Errorf("%s:%d error consuming: out of range, abandoning partition",
				c.topic, c.partID)
			go c.Release()
			return
		}
		if err != nil {
			log.Errorf("%s:%d error consuming: %s", c.topic, c.partID, err)

			// Often a consumption error is caused by data going away, such as if we're consuming
			// from the head and Kafka has deleted the data. In that case we need to wait for
			// the next offset update, so let's not go crazy
			time.Sleep(1 * time.Second)
			continue
		}

		c.messages <- msg
	}
	log.Debugf("%s:%d no longer claimed, pump exiting", c.topic, c.partID)
}

// heartbeat is the internal "send a heartbeat" function. Calling this will immediately
// send a heartbeat to Kafka. If we fail to send a heartbeat, we will release the
// partition.
func (c *claim) heartbeat() {
	// Unclaimed partitions don't heartbeat.
	if atomic.LoadInt32(c.claimed) != 1 {
		return
	}

	c.lock.Lock()
	defer c.lock.Unlock()

	err := c.marshal.Heartbeat(c.topic, c.partID, c.offsetCurrent)
	if err != nil {
		log.Errorf("%s:%d failed to heartbeat, releasing", c.topic, c.partID)
		go c.Release()
	}
	c.lastHeartbeat = time.Now().Unix()
}

// healthCheck performs a single health check against the claim. If we have failed
// too many times, this will also start a partition release. Returns true if the
// partition is healthy, else false.
func (c *claim) healthCheck() bool {
	// Unclaimed partitions aren't healthy.
	if atomic.LoadInt32(c.claimed) != 1 {
		return false
	}

	// Get velocities; these functions both use the locks so we have to do this before
	// we personally take the lock (to avoid deadlock)
	consumerVelocity := c.ConsumerVelocity()
	partitionVelocity := c.PartitionVelocity()

	c.lock.Lock()
	defer c.lock.Unlock()

	// If our heartbeat is expired, we are definitely unhealthy... don't even bother
	// with checking velocity
	if c.lastHeartbeat < time.Now().Unix()-HeartbeatInterval {
		log.Warningf("%s:%d consumer unhealthy by heartbeat test, releasing",
			c.topic, c.partID)
		go c.Release()
		return false
	}

	// If current has gone forward of the latest (which is possible, but unlikely)
	// then we are by definition caught up
	if c.offsetCurrent >= c.offsetLatest {
		c.cyclesBehind = 0
		return true
	}

	// If velocity is good, reset cycles behind and exit
	if partitionVelocity <= consumerVelocity {
		c.cyclesBehind = 0
		return true
	}

	// Unhealthy, so increase the cycle count so we know when it's been unhealthy for
	// too long and we want to give it up
	c.cyclesBehind++

	// If were behind by too many cycles, then we should try to release the
	// partition. If so, do this in a goroutine since it will involve calling out
	// to Kafka and releasing the partition.
	if c.cyclesBehind >= 3 {
		log.Errorf("%s:%d consumer unhealthy, releasing",
			c.topic, c.partID)
		go c.Release()
		return false
	} else {
		log.Warningf("%s:%d consumer unhealthy: CV %0.2f < PV %0.2f",
			c.topic, c.partID, consumerVelocity, partitionVelocity)
	}

	// Clearly we haven't been behind for long enough, so we're still "healthy"
	return true
}

// healthCheckLoop runs regularly and will perform a health check. Exits when we are no longer
// a claimed partition
func (c *claim) healthCheckLoop() {
	time.Sleep(<-c.marshal.jitters)
	for c.Claimed() {
		if c.healthCheck() {
			go c.heartbeat()
		}
		time.Sleep(<-c.marshal.jitters)
	}
	log.Debugf("%s:%d health check loop exiting", c.topic, c.partID)
}

// average returns the average of a given slice of int64s. It ignores 0s as
// those are "uninitialized" elements.
func average(vals []int64) float64 {
	min, max, ct := int64(0), int64(0), int64(0)
	for _, val := range vals {
		if val <= 0 {
			continue
		}
		if min == 0 || val < min {
			min = val
		}
		if max == 0 || val > max {
			max = val
		}
		ct++
	}
	if ct == 0 {
		return 0
	}
	return float64(max-min) / float64(ct)
}

// ConsumerVelocity returns the average of our consumers' velocity
func (c *claim) ConsumerVelocity() float64 {
	c.lock.RLock()
	defer c.lock.RUnlock()

	return average(c.offsetCurrentHistory[0:])
}

// PartitionVelocity returns the average of the partition's velocity
func (c *claim) PartitionVelocity() float64 {
	c.lock.RLock()
	defer c.lock.RUnlock()

	return average(c.offsetLatestHistory[0:])
}

// updateOffsets will update the offsets of our current partition.
func (c *claim) updateOffsets(ctr int) error {
	// Slow, hits Kafka. Run in a goroutine.
	oEarly, oLate, _, err := c.marshal.GetPartitionOffsets(c.topic, c.partID)
	if err != nil {
		log.Errorf("%s:%d failed to get offsets: %s", c.topic, c.partID, err)
		return err
	}

	c.lock.Lock()
	defer c.lock.Unlock()

	// Update the earliest/latest offsets that are presently available within the
	// partition
	c.offsetEarliest = oEarly
	c.offsetLatest = oLate

	// Do update our "history" values, this is used for calculating moving averages
	// in the health checking function
	c.offsetLatestHistory[ctr%10] = oLate
	c.offsetCurrentHistory[ctr%10] = c.offsetCurrent

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Hari haran. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

// +build linux

// memory provides different memory related data.
package memory

import (
	"bufio"
	"os"
	"strings"
)

type Data map[string]interface{}

func CollectData() (Data, error) {

	// allocate memory
	d := make(Data)
	d["swap"] = make(map[string]string)

	// cast interface{} to a map
	swap := d["swap"].(map[string]string)

	f, err := os.Open("/proc/meminfo")
	if err != nil {
		return d, err
	}
	defer f.Close()
	s := bufio.NewScanner(f)
	for s.Scan() {
		l := strings.Split(s.Text(), ":")
		k, v := l[0], strings.TrimSpace(l[1])
		switch k {
		case "MemTotal":
			d["total"] = v
		case "MemFree":
			d["free"] = v
		case "MemAvailable":
			d["available"] = v
		case "Buffers":
			d["buffers"] = v
		case "Cached":
			d["cached"] = v
		case "SwapCached":
			swap["cached"] = v
		case "Active":
			d["active"] = v
		case "Inactive":
			d["inactive"] = v
		case "SwapTotal":
			swap["total"] = v
		case "SwapFree":
			swap["free"] = v
		case "Dirty":
			d["dirty"] = v
		case "Writeback":
			d["writeback"] = v
		case "AnonPages":
			d["anon_pages"] = v
		case "Mapped":
			d["mapped"] = v
		case "Slab":
			d["slab"] = v
		case "SReclaimable":
			d["slab_reclaimable"] = v
		case "Sunreclaim":
			d["slab_unreclaim"] = v
		case "PageTables":
			d["page_tables"] = v
		case "NFS_Unstable":
			d["nfs_unstable"] = v
		case "Bounce":
			d["bounce"] = v
		case "CommitLimit":
			d["commit_limit"] = v
		case "Committed_AS":
			d["committed_as"] = v
		case "VmallocTotal":
			d["vmalloc_total"] = v
		case "VmallocUsed":
			d["vmalloc_used"] = v
		case "VmallocChunk":
			d["vmalloc_chunk"] = v
		}
	}
	if err := s.Err(); err != nil {
		return d, err
	}
	return d, nil
}
<commit_msg>Fix slab unreclaim case<commit_after>// Copyright 2015 Hari haran. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

// +build linux

// memory provides different memory related data.
package memory

import (
	"bufio"
	"os"
	"strings"
)

type Data map[string]interface{}

func CollectData() (Data, error) {

	// allocate memory
	d := make(Data)
	d["swap"] = make(map[string]string)

	// cast interface{} to a map
	swap := d["swap"].(map[string]string)

	f, err := os.Open("/proc/meminfo")
	if err != nil {
		return d, err
	}
	defer f.Close()
	s := bufio.NewScanner(f)
	for s.Scan() {
		l := strings.Split(s.Text(), ":")
		k, v := l[0], strings.TrimSpace(l[1])
		switch k {
		case "MemTotal":
			d["total"] = v
		case "MemFree":
			d["free"] = v
		case "MemAvailable":
			d["available"] = v
		case "Buffers":
			d["buffers"] = v
		case "Cached":
			d["cached"] = v
		case "SwapCached":
			swap["cached"] = v
		case "Active":
			d["active"] = v
		case "Inactive":
			d["inactive"] = v
		case "SwapTotal":
			swap["total"] = v
		case "SwapFree":
			swap["free"] = v
		case "Dirty":
			d["dirty"] = v
		case "Writeback":
			d["writeback"] = v
		case "AnonPages":
			d["anon_pages"] = v
		case "Mapped":
			d["mapped"] = v
		case "Slab":
			d["slab"] = v
		case "SReclaimable":
			d["slab_reclaimable"] = v
		case "SUnreclaim":
			d["slab_unreclaim"] = v
		case "PageTables":
			d["page_tables"] = v
		case "NFS_Unstable":
			d["nfs_unstable"] = v
		case "Bounce":
			d["bounce"] = v
		case "CommitLimit":
			d["commit_limit"] = v
		case "Committed_AS":
			d["committed_as"] = v
		case "VmallocTotal":
			d["vmalloc_total"] = v
		case "VmallocUsed":
			d["vmalloc_used"] = v
		case "VmallocChunk":
			d["vmalloc_chunk"] = v
		}
	}
	if err := s.Err(); err != nil {
		return d, err
	}
	return d, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"flag"
	"fmt"
	"os"
	"path"
	"strings"
	"testing"
	"time"

	"github.com/golang/glog"
	"github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/config"
	"github.com/onsi/ginkgo/reporters"
	"github.com/onsi/gomega"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/client/unversioned/clientcmd"
	"k8s.io/kubernetes/pkg/cloudprovider"
	gcecloud "k8s.io/kubernetes/pkg/cloudprovider/providers/gce"
	"k8s.io/kubernetes/pkg/util"
	"k8s.io/kubernetes/pkg/util/runtime"
)

const (
	// podStartupTimeout is the time to allow all pods in the cluster to become
	// running and ready before any e2e tests run. It includes pulling all of
	// the pods (as of 5/18/15 this is 8 pods).
	podStartupTimeout = 10 * time.Minute
)

var (
	cloudConfig = &testContext.CloudConfig
)

func RegisterFlags() {
	// Turn on verbose by default to get spec names
	config.DefaultReporterConfig.Verbose = true

	// Turn on EmitSpecProgress to get spec progress (especially on interrupt)
	config.GinkgoConfig.EmitSpecProgress = true

	// Randomize specs as well as suites
	config.GinkgoConfig.RandomizeAllSpecs = true

	flag.StringVar(&testContext.KubeConfig, clientcmd.RecommendedConfigPathFlag, "", "Path to kubeconfig containing embedded authinfo.")
	flag.StringVar(&testContext.KubeContext, clientcmd.FlagContext, "", "kubeconfig context to use/override. If unset, will use value from 'current-context'")
	flag.StringVar(&testContext.KubeVolumeDir, "volume-dir", "/var/lib/kubelet", "Path to the directory containing the kubelet volumes.")
	flag.StringVar(&testContext.CertDir, "cert-dir", "", "Path to the directory containing the certs. Default is empty, which doesn't use certs.")
	flag.StringVar(&testContext.Host, "host", "", "The host, or apiserver, to connect to")
	flag.StringVar(&testContext.RepoRoot, "repo-root", "../../", "Root directory of kubernetes repository, for finding test files.")
	flag.StringVar(&testContext.Provider, "provider", "", "The name of the Kubernetes provider (gce, gke, local, vagrant, etc.)")
	flag.StringVar(&testContext.KubectlPath, "kubectl-path", "kubectl", "The kubectl binary to use. For development, you might use 'cluster/kubectl.sh' here.")
	flag.StringVar(&testContext.OutputDir, "e2e-output-dir", "/tmp", "Output directory for interesting/useful test data, like performance data, benchmarks, and other metrics.")
	flag.StringVar(&testContext.ReportDir, "report-dir", "", "Path to the directory where the JUnit XML reports should be saved. Default is empty, which doesn't generate these reports.")
	flag.StringVar(&testContext.prefix, "prefix", "e2e", "A prefix to be added to cloud resources created during testing.")

	// TODO: Flags per provider?  Rename gce-project/gce-zone?
	flag.StringVar(&cloudConfig.MasterName, "kube-master", "", "Name of the kubernetes master. Only required if provider is gce or gke")
	flag.StringVar(&cloudConfig.ProjectID, "gce-project", "", "The GCE project being used, if applicable")
	flag.StringVar(&cloudConfig.Zone, "gce-zone", "", "GCE zone being used, if applicable")
	flag.StringVar(&cloudConfig.ServiceAccount, "gce-service-account", "", "GCE service account to use for GCE API calls, if applicable")
	flag.StringVar(&cloudConfig.Cluster, "gke-cluster", "", "GKE name of cluster being used, if applicable")
	flag.StringVar(&cloudConfig.NodeInstanceGroup, "node-instance-group", "", "Name of the managed instance group for nodes. Valid only for gce, gke or aws")
	flag.IntVar(&cloudConfig.NumNodes, "num-nodes", -1, "Number of nodes in the cluster")

	flag.StringVar(&cloudConfig.ClusterTag, "cluster-tag", "", "Tag used to identify resources.  Only required if provider is aws.")
	flag.IntVar(&testContext.MinStartupPods, "minStartupPods", 0, "The number of pods which we need to see in 'Running' state with a 'Ready' condition of true, before we try running tests. This is useful in any cluster which needs some base pod-based services running before it can be used.")
	flag.StringVar(&testContext.UpgradeTarget, "upgrade-target", "ci/latest", "Version to upgrade to (e.g. 'release/stable', 'release/latest', 'ci/latest', '0.19.1', '0.19.1-669-gabac8c8') if doing an upgrade test.")
	flag.StringVar(&testContext.PrometheusPushGateway, "prom-push-gateway", "", "The URL to prometheus gateway, so that metrics can be pushed during e2es and scraped by prometheus. Typically something like 127.0.0.1:9091.")
	flag.BoolVar(&testContext.VerifyServiceAccount, "e2e-verify-service-account", true, "If true tests will verify the service account before running.")
	flag.BoolVar(&testContext.DeleteNamespace, "delete-namespace", true, "If true tests will delete namespace after completion. It is only designed to make debugging easier, DO NOT turn it off by default.")
	flag.BoolVar(&testContext.CleanStart, "clean-start", false, "If true, purge all namespaces except default and system before running tests. This serves to cleanup test namespaces from failed/interrupted e2e runs in a long-lived cluster.")
	flag.BoolVar(&testContext.GatherKubeSystemResourceUsageData, "gather-resource-usage", false, "If set to true framework will be monitoring resource usage of system add-ons in (some) e2e tests.")
	flag.BoolVar(&testContext.GatherLogsSizes, "gather-logs-sizes", false, "If set to true framework will be monitoring logs sizes on all machines running e2e tests.")
	flag.BoolVar(&testContext.GatherMetricsAfterTest, "gather-metrics-at-teardown", false, "If set to true framwork will gather metrics from all components after each test.")
	flag.StringVar(&testContext.OutputPrintType, "output-print-type", "hr", "Comma separated list: 'hr' for human readable summaries 'json' for JSON ones.")
}

// setupProviderConfig validates and sets up cloudConfig based on testContext.Provider.
func setupProviderConfig() error {
	switch testContext.Provider {
	case "":
		glog.Info("The --provider flag is not set.  Treating as a conformance test.  Some tests may not be run.")

	case "gce", "gke":
		var err error
		Logf("Fetching cloud provider for %q\r\n", testContext.Provider)
		var tokenSource oauth2.TokenSource
		tokenSource = nil
		if cloudConfig.ServiceAccount != "" {
			// Use specified service account for auth
			Logf("Using service account %q as token source.", cloudConfig.ServiceAccount)
			tokenSource = google.ComputeTokenSource(cloudConfig.ServiceAccount)
		}
		zone := testContext.CloudConfig.Zone
		region, err := gcecloud.GetGCERegion(zone)
		if err != nil {
			return fmt.Errorf("error parsing GCE/GKE region from zone %q: %v", zone, err)
		}
		managedZones := []string{zone} // Only single-zone for now
		cloudConfig.Provider, err = gcecloud.CreateGCECloud(testContext.CloudConfig.ProjectID, region, zone, managedZones, "" /* networkUrl */, tokenSource, false /* useMetadataServer */)
		if err != nil {
			return fmt.Errorf("Error building GCE/GKE provider: %v", err)
		}

	case "aws":
		awsConfig := "[Global]\n"
		if cloudConfig.Zone == "" {
			return fmt.Errorf("gce-zone must be specified for AWS")
		}
		awsConfig += fmt.Sprintf("Zone=%s\n", cloudConfig.Zone)

		if cloudConfig.ClusterTag == "" {
			return fmt.Errorf("--cluster-tag must be specified for AWS")
		}
		awsConfig += fmt.Sprintf("KubernetesClusterTag=%s\n", cloudConfig.ClusterTag)

		var err error
		cloudConfig.Provider, err = cloudprovider.GetCloudProvider(testContext.Provider, strings.NewReader(awsConfig))
		if err != nil {
			return fmt.Errorf("Error building AWS provider: %v", err)
		}

	}

	return nil
}

// There are certain operations we only want to run once per overall test invocation
// (such as deleting old namespaces, or verifying that all system pods are running.
// Because of the way Ginkgo runs tests in parallel, we must use SynchronizedBeforeSuite
// to ensure that these operations only run on the first parallel Ginkgo node.
//
// This function takes two parameters: one function which runs on only the first Ginkgo node,
// returning an opaque byte array, and then a second function which runs on all Ginkgo nodes,
// accepting the byte array.
var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
	// Run only on Ginkgo node 1

	// Delete any namespaces except default and kube-system. This ensures no
	// lingering resources are left over from a previous test run.
	if testContext.CleanStart {
		c, err := loadClient()
		if err != nil {
			glog.Fatal("Error loading client: ", err)
		}

		deleted, err := deleteNamespaces(c, nil /* deleteFilter */, []string{api.NamespaceSystem, api.NamespaceDefault})
		if err != nil {
			Failf("Error deleting orphaned namespaces: %v", err)
		}
		glog.Infof("Waiting for deletion of the following namespaces: %v", deleted)
		if err := waitForNamespacesDeleted(c, deleted, namespaceCleanupTimeout); err != nil {
			Failf("Failed to delete orphaned namespaces %v: %v", deleted, err)
		}
	}

	// Ensure all pods are running and ready before starting tests (otherwise,
	// cluster infrastructure pods that are being pulled or started can block
	// test pods from running, and tests that ensure all pods are running and
	// ready will fail).
	if err := waitForPodsRunningReady(api.NamespaceSystem, testContext.MinStartupPods, podStartupTimeout); err != nil {
		Failf("Error waiting for all pods to be running and ready: %v", err)
	}

	return nil

}, func(data []byte) {
	// Run on all Ginkgo nodes

})

// Similar to SynchornizedBeforeSuite, we want to run some operations only once (such as collecting cluster logs).
// Here, the order of functions is reversed; first, the function which runs everywhere,
// and then the function that only runs on the first Ginkgo node.
var _ = ginkgo.SynchronizedAfterSuite(func() {
	// Run on all Ginkgo nodes

}, func() {
	// Run only Ginkgo on node 1
	if testContext.ReportDir != "" {
		CoreDump(testContext.ReportDir)
	}
})

// TestE2E checks configuration parameters (specified through flags) and then runs
// E2E tests using the Ginkgo runner.
// If a "report directory" is specified, one or more JUnit test reports will be
// generated in this directory, and cluster logs will also be saved.
// This function is called on each Ginkgo node in parallel mode.
func RunE2ETests(t *testing.T) {
	runtime.ReallyCrash = true
	util.InitLogs()
	defer util.FlushLogs()

	// We must call setupProviderConfig first since SynchronizedBeforeSuite needs
	// cloudConfig to be set up already.
	if err := setupProviderConfig(); err != nil {
		glog.Fatalf(err.Error())
	}

	gomega.RegisterFailHandler(ginkgo.Fail)
	// Disable skipped tests unless they are explicitly requested.
	if config.GinkgoConfig.FocusString == "" && config.GinkgoConfig.SkipString == "" {
		// TODO(ihmccreery) Remove [Skipped] once all [Skipped] labels have been reclassified.
		config.GinkgoConfig.SkipString = `\[Flaky\]|\[Skipped\]|\[Feature:.+\]`
	}

	// Run tests through the Ginkgo runner with output to console + JUnit for Jenkins
	var r []ginkgo.Reporter
	if testContext.ReportDir != "" {
		// TODO: we should probably only be trying to create this directory once
		// rather than once-per-Ginkgo-node.
		if err := os.MkdirAll(testContext.ReportDir, 0755); err != nil {
			glog.Errorf("Failed creating report directory: %v", err)
		} else {
			r = append(r, reporters.NewJUnitReporter(path.Join(testContext.ReportDir, fmt.Sprintf("junit_%02d.xml", config.GinkgoConfig.ParallelNode))))
		}
	}
	glog.Infof("Starting e2e run %q on Ginkgo node %d", runId, config.GinkgoConfig.ParallelNode)
	ginkgo.RunSpecsWithDefaultAndCustomReporters(t, "Kubernetes e2e suite", r)
}
<commit_msg>Make e2e.test binary know about KUBECONFIG env var<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"flag"
	"fmt"
	"os"
	"path"
	"strings"
	"testing"
	"time"

	"github.com/golang/glog"
	"github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/config"
	"github.com/onsi/ginkgo/reporters"
	"github.com/onsi/gomega"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/client/unversioned/clientcmd"
	"k8s.io/kubernetes/pkg/cloudprovider"
	gcecloud "k8s.io/kubernetes/pkg/cloudprovider/providers/gce"
	"k8s.io/kubernetes/pkg/util"
	"k8s.io/kubernetes/pkg/util/runtime"
)

const (
	// podStartupTimeout is the time to allow all pods in the cluster to become
	// running and ready before any e2e tests run. It includes pulling all of
	// the pods (as of 5/18/15 this is 8 pods).
	podStartupTimeout = 10 * time.Minute
)

var (
	cloudConfig = &testContext.CloudConfig
)

func RegisterFlags() {
	// Turn on verbose by default to get spec names
	config.DefaultReporterConfig.Verbose = true

	// Turn on EmitSpecProgress to get spec progress (especially on interrupt)
	config.GinkgoConfig.EmitSpecProgress = true

	// Randomize specs as well as suites
	config.GinkgoConfig.RandomizeAllSpecs = true

	flag.StringVar(&testContext.KubeConfig, clientcmd.RecommendedConfigPathFlag, os.Getenv(clientcmd.RecommendedConfigPathEnvVar), "Path to kubeconfig containing embedded authinfo.")
	flag.StringVar(&testContext.KubeContext, clientcmd.FlagContext, "", "kubeconfig context to use/override. If unset, will use value from 'current-context'")
	flag.StringVar(&testContext.KubeVolumeDir, "volume-dir", "/var/lib/kubelet", "Path to the directory containing the kubelet volumes.")
	flag.StringVar(&testContext.CertDir, "cert-dir", "", "Path to the directory containing the certs. Default is empty, which doesn't use certs.")
	flag.StringVar(&testContext.Host, "host", "", "The host, or apiserver, to connect to")
	flag.StringVar(&testContext.RepoRoot, "repo-root", "../../", "Root directory of kubernetes repository, for finding test files.")
	flag.StringVar(&testContext.Provider, "provider", "", "The name of the Kubernetes provider (gce, gke, local, vagrant, etc.)")
	flag.StringVar(&testContext.KubectlPath, "kubectl-path", "kubectl", "The kubectl binary to use. For development, you might use 'cluster/kubectl.sh' here.")
	flag.StringVar(&testContext.OutputDir, "e2e-output-dir", "/tmp", "Output directory for interesting/useful test data, like performance data, benchmarks, and other metrics.")
	flag.StringVar(&testContext.ReportDir, "report-dir", "", "Path to the directory where the JUnit XML reports should be saved. Default is empty, which doesn't generate these reports.")
	flag.StringVar(&testContext.prefix, "prefix", "e2e", "A prefix to be added to cloud resources created during testing.")

	// TODO: Flags per provider?  Rename gce-project/gce-zone?
	flag.StringVar(&cloudConfig.MasterName, "kube-master", "", "Name of the kubernetes master. Only required if provider is gce or gke")
	flag.StringVar(&cloudConfig.ProjectID, "gce-project", "", "The GCE project being used, if applicable")
	flag.StringVar(&cloudConfig.Zone, "gce-zone", "", "GCE zone being used, if applicable")
	flag.StringVar(&cloudConfig.ServiceAccount, "gce-service-account", "", "GCE service account to use for GCE API calls, if applicable")
	flag.StringVar(&cloudConfig.Cluster, "gke-cluster", "", "GKE name of cluster being used, if applicable")
	flag.StringVar(&cloudConfig.NodeInstanceGroup, "node-instance-group", "", "Name of the managed instance group for nodes. Valid only for gce, gke or aws")
	flag.IntVar(&cloudConfig.NumNodes, "num-nodes", -1, "Number of nodes in the cluster")

	flag.StringVar(&cloudConfig.ClusterTag, "cluster-tag", "", "Tag used to identify resources.  Only required if provider is aws.")
	flag.IntVar(&testContext.MinStartupPods, "minStartupPods", 0, "The number of pods which we need to see in 'Running' state with a 'Ready' condition of true, before we try running tests. This is useful in any cluster which needs some base pod-based services running before it can be used.")
	flag.StringVar(&testContext.UpgradeTarget, "upgrade-target", "ci/latest", "Version to upgrade to (e.g. 'release/stable', 'release/latest', 'ci/latest', '0.19.1', '0.19.1-669-gabac8c8') if doing an upgrade test.")
	flag.StringVar(&testContext.PrometheusPushGateway, "prom-push-gateway", "", "The URL to prometheus gateway, so that metrics can be pushed during e2es and scraped by prometheus. Typically something like 127.0.0.1:9091.")
	flag.BoolVar(&testContext.VerifyServiceAccount, "e2e-verify-service-account", true, "If true tests will verify the service account before running.")
	flag.BoolVar(&testContext.DeleteNamespace, "delete-namespace", true, "If true tests will delete namespace after completion. It is only designed to make debugging easier, DO NOT turn it off by default.")
	flag.BoolVar(&testContext.CleanStart, "clean-start", false, "If true, purge all namespaces except default and system before running tests. This serves to cleanup test namespaces from failed/interrupted e2e runs in a long-lived cluster.")
	flag.BoolVar(&testContext.GatherKubeSystemResourceUsageData, "gather-resource-usage", false, "If set to true framework will be monitoring resource usage of system add-ons in (some) e2e tests.")
	flag.BoolVar(&testContext.GatherLogsSizes, "gather-logs-sizes", false, "If set to true framework will be monitoring logs sizes on all machines running e2e tests.")
	flag.BoolVar(&testContext.GatherMetricsAfterTest, "gather-metrics-at-teardown", false, "If set to true framwork will gather metrics from all components after each test.")
	flag.StringVar(&testContext.OutputPrintType, "output-print-type", "hr", "Comma separated list: 'hr' for human readable summaries 'json' for JSON ones.")
}

// setupProviderConfig validates and sets up cloudConfig based on testContext.Provider.
func setupProviderConfig() error {
	switch testContext.Provider {
	case "":
		glog.Info("The --provider flag is not set.  Treating as a conformance test.  Some tests may not be run.")

	case "gce", "gke":
		var err error
		Logf("Fetching cloud provider for %q\r\n", testContext.Provider)
		var tokenSource oauth2.TokenSource
		tokenSource = nil
		if cloudConfig.ServiceAccount != "" {
			// Use specified service account for auth
			Logf("Using service account %q as token source.", cloudConfig.ServiceAccount)
			tokenSource = google.ComputeTokenSource(cloudConfig.ServiceAccount)
		}
		zone := testContext.CloudConfig.Zone
		region, err := gcecloud.GetGCERegion(zone)
		if err != nil {
			return fmt.Errorf("error parsing GCE/GKE region from zone %q: %v", zone, err)
		}
		managedZones := []string{zone} // Only single-zone for now
		cloudConfig.Provider, err = gcecloud.CreateGCECloud(testContext.CloudConfig.ProjectID, region, zone, managedZones, "" /* networkUrl */, tokenSource, false /* useMetadataServer */)
		if err != nil {
			return fmt.Errorf("Error building GCE/GKE provider: %v", err)
		}

	case "aws":
		awsConfig := "[Global]\n"
		if cloudConfig.Zone == "" {
			return fmt.Errorf("gce-zone must be specified for AWS")
		}
		awsConfig += fmt.Sprintf("Zone=%s\n", cloudConfig.Zone)

		if cloudConfig.ClusterTag == "" {
			return fmt.Errorf("--cluster-tag must be specified for AWS")
		}
		awsConfig += fmt.Sprintf("KubernetesClusterTag=%s\n", cloudConfig.ClusterTag)

		var err error
		cloudConfig.Provider, err = cloudprovider.GetCloudProvider(testContext.Provider, strings.NewReader(awsConfig))
		if err != nil {
			return fmt.Errorf("Error building AWS provider: %v", err)
		}

	}

	return nil
}

// There are certain operations we only want to run once per overall test invocation
// (such as deleting old namespaces, or verifying that all system pods are running.
// Because of the way Ginkgo runs tests in parallel, we must use SynchronizedBeforeSuite
// to ensure that these operations only run on the first parallel Ginkgo node.
//
// This function takes two parameters: one function which runs on only the first Ginkgo node,
// returning an opaque byte array, and then a second function which runs on all Ginkgo nodes,
// accepting the byte array.
var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
	// Run only on Ginkgo node 1

	// Delete any namespaces except default and kube-system. This ensures no
	// lingering resources are left over from a previous test run.
	if testContext.CleanStart {
		c, err := loadClient()
		if err != nil {
			glog.Fatal("Error loading client: ", err)
		}

		deleted, err := deleteNamespaces(c, nil /* deleteFilter */, []string{api.NamespaceSystem, api.NamespaceDefault})
		if err != nil {
			Failf("Error deleting orphaned namespaces: %v", err)
		}
		glog.Infof("Waiting for deletion of the following namespaces: %v", deleted)
		if err := waitForNamespacesDeleted(c, deleted, namespaceCleanupTimeout); err != nil {
			Failf("Failed to delete orphaned namespaces %v: %v", deleted, err)
		}
	}

	// Ensure all pods are running and ready before starting tests (otherwise,
	// cluster infrastructure pods that are being pulled or started can block
	// test pods from running, and tests that ensure all pods are running and
	// ready will fail).
	if err := waitForPodsRunningReady(api.NamespaceSystem, testContext.MinStartupPods, podStartupTimeout); err != nil {
		Failf("Error waiting for all pods to be running and ready: %v", err)
	}

	return nil

}, func(data []byte) {
	// Run on all Ginkgo nodes

})

// Similar to SynchornizedBeforeSuite, we want to run some operations only once (such as collecting cluster logs).
// Here, the order of functions is reversed; first, the function which runs everywhere,
// and then the function that only runs on the first Ginkgo node.
var _ = ginkgo.SynchronizedAfterSuite(func() {
	// Run on all Ginkgo nodes

}, func() {
	// Run only Ginkgo on node 1
	if testContext.ReportDir != "" {
		CoreDump(testContext.ReportDir)
	}
})

// TestE2E checks configuration parameters (specified through flags) and then runs
// E2E tests using the Ginkgo runner.
// If a "report directory" is specified, one or more JUnit test reports will be
// generated in this directory, and cluster logs will also be saved.
// This function is called on each Ginkgo node in parallel mode.
func RunE2ETests(t *testing.T) {
	runtime.ReallyCrash = true
	util.InitLogs()
	defer util.FlushLogs()

	// We must call setupProviderConfig first since SynchronizedBeforeSuite needs
	// cloudConfig to be set up already.
	if err := setupProviderConfig(); err != nil {
		glog.Fatalf(err.Error())
	}

	gomega.RegisterFailHandler(ginkgo.Fail)
	// Disable skipped tests unless they are explicitly requested.
	if config.GinkgoConfig.FocusString == "" && config.GinkgoConfig.SkipString == "" {
		// TODO(ihmccreery) Remove [Skipped] once all [Skipped] labels have been reclassified.
		config.GinkgoConfig.SkipString = `\[Flaky\]|\[Skipped\]|\[Feature:.+\]`
	}

	// Run tests through the Ginkgo runner with output to console + JUnit for Jenkins
	var r []ginkgo.Reporter
	if testContext.ReportDir != "" {
		// TODO: we should probably only be trying to create this directory once
		// rather than once-per-Ginkgo-node.
		if err := os.MkdirAll(testContext.ReportDir, 0755); err != nil {
			glog.Errorf("Failed creating report directory: %v", err)
		} else {
			r = append(r, reporters.NewJUnitReporter(path.Join(testContext.ReportDir, fmt.Sprintf("junit_%02d.xml", config.GinkgoConfig.ParallelNode))))
		}
	}
	glog.Infof("Starting e2e run %q on Ginkgo node %d", runId, config.GinkgoConfig.ParallelNode)
	ginkgo.RunSpecsWithDefaultAndCustomReporters(t, "Kubernetes e2e suite", r)
}
<|endoftext|>


<commit_before>// Copyright (C) 2019-2021, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package message

import (
	"errors"
	"fmt"
	"math"
	"sync"
	"time"

	"github.com/prometheus/client_golang/prometheus"

	"github.com/ava-labs/avalanchego/ids"
	"github.com/ava-labs/avalanchego/utils/compression"
	"github.com/ava-labs/avalanchego/utils/constants"
	"github.com/ava-labs/avalanchego/utils/metric"
	"github.com/ava-labs/avalanchego/utils/timer/mockable"
	"github.com/ava-labs/avalanchego/utils/wrappers"
)

var (
	errMissingField = errors.New("message missing field")
	errBadOp        = errors.New("input field has invalid operation")

	_ Codec = &codec{}
)

type Packer interface {
	Pack(
		op Op,
		fieldValues map[Field]interface{},
		compress bool,
		bypassThrottling bool,
	) (OutboundMessage, error)
}

type Parser interface {
	SetTime(t time.Time) // useful in UTs

	// Parse reads given bytes as InboundMessage
	// Overrides client specified deadline in a message to maxDeadlineDuration
	Parse(bytes []byte, nodeID ids.NodeID, onFinishedHandling func()) (InboundMessage, error)
}

type Codec interface {
	Packer
	Parser
}

// codec defines the serialization and deserialization of network messages.
// It's safe for multiple goroutines to call Pack and Parse concurrently.
type codec struct {
	// Contains []byte. Used as an optimization.
	// Can be accessed by multiple goroutines concurrently.
	byteSlicePool sync.Pool

	clock mockable.Clock

	compressTimeMetrics   map[Op]metric.Averager
	decompressTimeMetrics map[Op]metric.Averager
	compressor            compression.Compressor
	maxMessageTimeout     time.Duration
}

func NewCodecWithMemoryPool(namespace string, metrics prometheus.Registerer, maxMessageSize int64, maxMessageTimeout time.Duration) (Codec, error) {
	c := &codec{
		byteSlicePool: sync.Pool{
			New: func() interface{} {
				return make([]byte, 0, constants.DefaultByteSliceCap)
			},
		},
		compressTimeMetrics:   make(map[Op]metric.Averager, len(ExternalOps)),
		decompressTimeMetrics: make(map[Op]metric.Averager, len(ExternalOps)),
		compressor:            compression.NewGzipCompressor(maxMessageSize),
		maxMessageTimeout:     maxMessageTimeout,
	}

	errs := wrappers.Errs{}
	for _, op := range ExternalOps {
		if !op.Compressible() {
			continue
		}

		c.compressTimeMetrics[op] = metric.NewAveragerWithErrs(
			namespace,
			fmt.Sprintf("%s_compress_time", op),
			fmt.Sprintf("time (in ns) to compress %s messages", op),
			metrics,
			&errs,
		)
		c.decompressTimeMetrics[op] = metric.NewAveragerWithErrs(
			namespace,
			fmt.Sprintf("%s_decompress_time", op),
			fmt.Sprintf("time (in ns) to decompress %s messages", op),
			metrics,
			&errs,
		)
	}
	return c, errs.Err
}

func (c *codec) SetTime(t time.Time) {
	c.clock.Set(t)
}

// Pack attempts to pack a map of fields into a message.
// The first byte of the message is the opcode of the message.
// Uses [buffer] to hold the message's byte repr.
// [buffer]'s contents may be overwritten by this method.
// [buffer] may be nil.
// If [compress], compress the payload.
// If [bypassThrottling], mark the message to avoid outbound throttling checks.
func (c *codec) Pack(
	op Op,
	fieldValues map[Field]interface{},
	compress bool,
	bypassThrottling bool,
) (OutboundMessage, error) {
	msgFields, ok := messages[op]
	if !ok {
		return nil, errBadOp
	}

	buffer := c.byteSlicePool.Get().([]byte)
	p := wrappers.Packer{
		MaxSize: math.MaxInt32,
		Bytes:   buffer[:0],
	}
	// Pack the op code (message type)
	p.PackByte(byte(op))

	// Optionally, pack whether the payload is compressed
	if op.Compressible() {
		p.PackBool(compress)
	}

	// Pack the uncompressed payload
	for _, field := range msgFields {
		data, ok := fieldValues[field]
		if !ok {
			return nil, errMissingField
		}
		field.Packer()(&p, data)
	}
	if p.Err != nil {
		return nil, p.Err
	}
	msg := &outboundMessage{
		op:               op,
		bytes:            p.Bytes,
		refs:             1,
		c:                c,
		bypassThrottling: bypassThrottling,
	}
	if !compress {
		return msg, nil
	}

	// If [compress], compress the payload (not the op code, not isCompressed).
	// The slice below is guaranteed to be in-bounds because [p.Err] == nil
	// implies that len(msg.bytes) >= 2
	payloadBytes := msg.bytes[wrappers.BoolLen+wrappers.ByteLen:]
	startTime := time.Now()
	compressedPayloadBytes, err := c.compressor.Compress(payloadBytes)
	if err != nil {
		return nil, fmt.Errorf("couldn't compress payload of %s message: %w", op, err)
	}
	c.compressTimeMetrics[op].Observe(float64(time.Since(startTime)))
	msg.bytesSavedCompression = len(payloadBytes) - len(compressedPayloadBytes) // may be negative
	// Remove the uncompressed payload (keep just the message type and isCompressed)
	msg.bytes = msg.bytes[:wrappers.BoolLen+wrappers.ByteLen]
	// Attach the compressed payload
	msg.bytes = append(msg.bytes, compressedPayloadBytes...)
	return msg, nil
}

// Parse attempts to convert bytes into a message.
// The first byte of the message is the opcode of the message.
// Overrides client specified deadline in a message to maxDeadlineDuration
func (c *codec) Parse(bytes []byte, nodeID ids.NodeID, onFinishedHandling func()) (InboundMessage, error) {
	p := wrappers.Packer{Bytes: bytes}

	// Unpack the op code (message type)
	op := Op(p.UnpackByte())

	msgFields, ok := messages[op]
	if !ok { // Unknown message type
		return nil, errBadOp
	}

	// See if messages of this type may be compressed
	compressed := false
	if op.Compressible() {
		compressed = p.UnpackBool()
	}
	if p.Err != nil {
		return nil, p.Err
	}

	bytesSaved := 0

	// If the payload is compressed, decompress it
	if compressed {
		// The slice below is guaranteed to be in-bounds because [p.Err] == nil
		compressedPayloadBytes := p.Bytes[wrappers.ByteLen+wrappers.BoolLen:]
		startTime := time.Now()
		payloadBytes, err := c.compressor.Decompress(compressedPayloadBytes)
		if err != nil {
			return nil, fmt.Errorf("couldn't decompress payload of %s message: %w", op, err)
		}
		c.decompressTimeMetrics[op].Observe(float64(time.Since(startTime)))
		// Replace the compressed payload with the decompressed payload.
		// Remove the compressed payload and isCompressed; keep just the message type
		p.Bytes = p.Bytes[:wrappers.ByteLen]
		// Rewind offset by 1 because we removed the bool flag
		// since the data now is uncompressed
		p.Offset -= wrappers.BoolLen
		// Attach the decompressed payload.
		p.Bytes = append(p.Bytes, payloadBytes...)
		bytesSaved = len(payloadBytes) - len(compressedPayloadBytes)
	}

	// Parse each field of the payload
	fieldValues := make(map[Field]interface{}, len(msgFields))
	for _, field := range msgFields {
		fieldValues[field] = field.Unpacker()(&p)
	}

	if p.Offset != len(p.Bytes) {
		return nil, fmt.Errorf("expected length %d but got %d", p.Offset, len(p.Bytes))
	}

	var expirationTime time.Time
	if deadline, hasDeadline := fieldValues[Deadline]; hasDeadline {
		deadlineDuration := time.Duration(deadline.(uint64))
		if deadlineDuration > c.maxMessageTimeout {
			deadlineDuration = c.maxMessageTimeout
		}
		expirationTime = c.clock.Time().Add(deadlineDuration)
	}

	return &inboundMessage{
		op:                    op,
		fields:                fieldValues,
		bytesSavedCompression: bytesSaved,
		nodeID:                nodeID,
		expirationTime:        expirationTime,
		onFinishedHandling:    onFinishedHandling,
	}, p.Err
}
<commit_msg>Always return the first p2p parsing error (#1502)<commit_after>// Copyright (C) 2019-2021, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package message

import (
	"errors"
	"fmt"
	"math"
	"sync"
	"time"

	"github.com/prometheus/client_golang/prometheus"

	"github.com/ava-labs/avalanchego/ids"
	"github.com/ava-labs/avalanchego/utils/compression"
	"github.com/ava-labs/avalanchego/utils/constants"
	"github.com/ava-labs/avalanchego/utils/metric"
	"github.com/ava-labs/avalanchego/utils/timer/mockable"
	"github.com/ava-labs/avalanchego/utils/wrappers"
)

var (
	errMissingField = errors.New("message missing field")
	errBadOp        = errors.New("input field has invalid operation")

	_ Codec = &codec{}
)

type Packer interface {
	Pack(
		op Op,
		fieldValues map[Field]interface{},
		compress bool,
		bypassThrottling bool,
	) (OutboundMessage, error)
}

type Parser interface {
	SetTime(t time.Time) // useful in UTs

	// Parse reads given bytes as InboundMessage
	// Overrides client specified deadline in a message to maxDeadlineDuration
	Parse(bytes []byte, nodeID ids.NodeID, onFinishedHandling func()) (InboundMessage, error)
}

type Codec interface {
	Packer
	Parser
}

// codec defines the serialization and deserialization of network messages.
// It's safe for multiple goroutines to call Pack and Parse concurrently.
type codec struct {
	// Contains []byte. Used as an optimization.
	// Can be accessed by multiple goroutines concurrently.
	byteSlicePool sync.Pool

	clock mockable.Clock

	compressTimeMetrics   map[Op]metric.Averager
	decompressTimeMetrics map[Op]metric.Averager
	compressor            compression.Compressor
	maxMessageTimeout     time.Duration
}

func NewCodecWithMemoryPool(namespace string, metrics prometheus.Registerer, maxMessageSize int64, maxMessageTimeout time.Duration) (Codec, error) {
	c := &codec{
		byteSlicePool: sync.Pool{
			New: func() interface{} {
				return make([]byte, 0, constants.DefaultByteSliceCap)
			},
		},
		compressTimeMetrics:   make(map[Op]metric.Averager, len(ExternalOps)),
		decompressTimeMetrics: make(map[Op]metric.Averager, len(ExternalOps)),
		compressor:            compression.NewGzipCompressor(maxMessageSize),
		maxMessageTimeout:     maxMessageTimeout,
	}

	errs := wrappers.Errs{}
	for _, op := range ExternalOps {
		if !op.Compressible() {
			continue
		}

		c.compressTimeMetrics[op] = metric.NewAveragerWithErrs(
			namespace,
			fmt.Sprintf("%s_compress_time", op),
			fmt.Sprintf("time (in ns) to compress %s messages", op),
			metrics,
			&errs,
		)
		c.decompressTimeMetrics[op] = metric.NewAveragerWithErrs(
			namespace,
			fmt.Sprintf("%s_decompress_time", op),
			fmt.Sprintf("time (in ns) to decompress %s messages", op),
			metrics,
			&errs,
		)
	}
	return c, errs.Err
}

func (c *codec) SetTime(t time.Time) {
	c.clock.Set(t)
}

// Pack attempts to pack a map of fields into a message.
// The first byte of the message is the opcode of the message.
// Uses [buffer] to hold the message's byte repr.
// [buffer]'s contents may be overwritten by this method.
// [buffer] may be nil.
// If [compress], compress the payload.
// If [bypassThrottling], mark the message to avoid outbound throttling checks.
func (c *codec) Pack(
	op Op,
	fieldValues map[Field]interface{},
	compress bool,
	bypassThrottling bool,
) (OutboundMessage, error) {
	msgFields, ok := messages[op]
	if !ok {
		return nil, errBadOp
	}

	buffer := c.byteSlicePool.Get().([]byte)
	p := wrappers.Packer{
		MaxSize: math.MaxInt32,
		Bytes:   buffer[:0],
	}
	// Pack the op code (message type)
	p.PackByte(byte(op))

	// Optionally, pack whether the payload is compressed
	if op.Compressible() {
		p.PackBool(compress)
	}

	// Pack the uncompressed payload
	for _, field := range msgFields {
		data, ok := fieldValues[field]
		if !ok {
			return nil, errMissingField
		}
		field.Packer()(&p, data)
	}
	if p.Err != nil {
		return nil, p.Err
	}
	msg := &outboundMessage{
		op:               op,
		bytes:            p.Bytes,
		refs:             1,
		c:                c,
		bypassThrottling: bypassThrottling,
	}
	if !compress {
		return msg, nil
	}

	// If [compress], compress the payload (not the op code, not isCompressed).
	// The slice below is guaranteed to be in-bounds because [p.Err] == nil
	// implies that len(msg.bytes) >= 2
	payloadBytes := msg.bytes[wrappers.BoolLen+wrappers.ByteLen:]
	startTime := time.Now()
	compressedPayloadBytes, err := c.compressor.Compress(payloadBytes)
	if err != nil {
		return nil, fmt.Errorf("couldn't compress payload of %s message: %w", op, err)
	}
	c.compressTimeMetrics[op].Observe(float64(time.Since(startTime)))
	msg.bytesSavedCompression = len(payloadBytes) - len(compressedPayloadBytes) // may be negative
	// Remove the uncompressed payload (keep just the message type and isCompressed)
	msg.bytes = msg.bytes[:wrappers.BoolLen+wrappers.ByteLen]
	// Attach the compressed payload
	msg.bytes = append(msg.bytes, compressedPayloadBytes...)
	return msg, nil
}

// Parse attempts to convert bytes into a message.
// The first byte of the message is the opcode of the message.
// Overrides client specified deadline in a message to maxDeadlineDuration
func (c *codec) Parse(bytes []byte, nodeID ids.NodeID, onFinishedHandling func()) (InboundMessage, error) {
	p := wrappers.Packer{Bytes: bytes}

	// Unpack the op code (message type)
	op := Op(p.UnpackByte())

	msgFields, ok := messages[op]
	if !ok { // Unknown message type
		return nil, errBadOp
	}

	// See if messages of this type may be compressed
	compressed := false
	if op.Compressible() {
		compressed = p.UnpackBool()
	}
	if p.Err != nil {
		return nil, p.Err
	}

	bytesSaved := 0

	// If the payload is compressed, decompress it
	if compressed {
		// The slice below is guaranteed to be in-bounds because [p.Err] == nil
		compressedPayloadBytes := p.Bytes[wrappers.ByteLen+wrappers.BoolLen:]
		startTime := time.Now()
		payloadBytes, err := c.compressor.Decompress(compressedPayloadBytes)
		if err != nil {
			return nil, fmt.Errorf("couldn't decompress payload of %s message: %w", op, err)
		}
		c.decompressTimeMetrics[op].Observe(float64(time.Since(startTime)))
		// Replace the compressed payload with the decompressed payload.
		// Remove the compressed payload and isCompressed; keep just the message type
		p.Bytes = p.Bytes[:wrappers.ByteLen]
		// Rewind offset by 1 because we removed the bool flag
		// since the data now is uncompressed
		p.Offset -= wrappers.BoolLen
		// Attach the decompressed payload.
		p.Bytes = append(p.Bytes, payloadBytes...)
		bytesSaved = len(payloadBytes) - len(compressedPayloadBytes)
	}

	// Parse each field of the payload
	fieldValues := make(map[Field]interface{}, len(msgFields))
	for _, field := range msgFields {
		fieldValues[field] = field.Unpacker()(&p)
	}
	if p.Err != nil {
		return nil, p.Err
	}

	if p.Offset != len(p.Bytes) {
		return nil, fmt.Errorf("expected length %d but got %d", p.Offset, len(p.Bytes))
	}

	var expirationTime time.Time
	if deadline, hasDeadline := fieldValues[Deadline]; hasDeadline {
		deadlineDuration := time.Duration(deadline.(uint64))
		if deadlineDuration > c.maxMessageTimeout {
			deadlineDuration = c.maxMessageTimeout
		}
		expirationTime = c.clock.Time().Add(deadlineDuration)
	}

	return &inboundMessage{
		op:                    op,
		fields:                fieldValues,
		bytesSavedCompression: bytesSaved,
		nodeID:                nodeID,
		expirationTime:        expirationTime,
		onFinishedHandling:    onFinishedHandling,
	}, nil
}
<|endoftext|>