<commit_before>package sudoku

import (
	"fmt"
	"math/rand"
)

type swordfishTechnique struct {
	*basicSolveTechnique
}

func (self *swordfishTechnique) humanLikelihood(step *SolveStep) float64 {
	//TODO: reason more carefully about how hard this technique is.
	return self.difficultyHelper(70.0)
}

func (self *swordfishTechnique) Description(step *SolveStep) string {

	if len(step.TargetNums) != 1 {
		return ""
	}

	groupName := "columns"
	otherGroupName := "rows"

	if self.groupType == _GROUP_ROW {
		groupName = "rows"
		otherGroupName = "columns"
	}

	return fmt.Sprintf("%d is only possible in two cells each in three different %s, all of which align onto three %s, which means that %d can't be in any of the other cells in those %s (%s)",
		step.TargetNums[0],
		groupName,
		otherGroupName,
		step.TargetNums[0],
		otherGroupName,
		step.TargetCells.Description(),
	)

}

func (self *swordfishTechnique) Find(grid *Grid, results chan *SolveStep, done chan bool) {

	getter := self.getter(grid)

	//For this technique, the primary access is the first type of group we look at to find
	//cells with only 2 spots for a given number.

	//TODO: Implement the "relaxed" version of this technique, too.

	for _, i := range rand.Perm(DIM) {
		//The candidate we're considering

		//Consider each of the major-axis groups to see if more than three have
		//only two candidates for

		var majorAxisGroupsWithTwoOptionsForCandidate []CellSlice

		for c := 0; c < DIM; c++ {
			majorAxisGroup := getter(c)

			cellsWithCandidatePossibility := majorAxisGroup.FilterByPossible(i)

			if len(cellsWithCandidatePossibility) == 2 {
				//TODO: shouldn't we keep track of the rows where the possibilities were, too?
				majorAxisGroupsWithTwoOptionsForCandidate = append(majorAxisGroupsWithTwoOptionsForCandidate, cellsWithCandidatePossibility)
			}

		}

		//Do we have more than three major axis groups identified?
		if len(majorAxisGroupsWithTwoOptionsForCandidate) < 3 {
			continue
		}

		//Consider every subset of size three
		for _, indexes := range subsetIndexes(len(majorAxisGroupsWithTwoOptionsForCandidate), 3) {
			majorAxisGroups := make([]CellSlice, 3)
			for i, index := range indexes {
				majorAxisGroups[i] = majorAxisGroupsWithTwoOptionsForCandidate[index]
			}

			//OK, now majorAxisGroups has the set of three we're operating on.
			//Do their minorAxis groups line up to a set of three as well?

			var minorGroupIndexSet intSet

			for _, group := range majorAxisGroups {
				if self.groupType == _GROUP_COL {
					minorGroupIndexSet = minorGroupIndexSet.union(group.AllRows().toIntSet())
				} else {
					minorGroupIndexSet = minorGroupIndexSet.union(group.AllCols().toIntSet())
				}
			}

			if len(minorGroupIndexSet) != 3 {
				//Nah, didn't have three rows.
				continue
			}

			//Woot, looks like we found a valid set.

			//Generate the list of cells that will be affected
			var affectedCells CellSlice

			for minorGroupIndex, _ := range minorGroupIndexSet {
				if self.groupType == _GROUP_COL {
					affectedCells = append(affectedCells, grid.Row(minorGroupIndex)...)
				} else {
					affectedCells = append(affectedCells, grid.Col(minorGroupIndex)...)
				}
			}

			//Get rid of cells that are already filled, or where removing 'i'
			//would be a no-op anyway since it's not already a possible there.
			affectedCells = affectedCells.FilterByPossible(i)

			//Gather the list of all cells that are "pointing" to this

			var pointerCells CellSlice

			for _, group := range majorAxisGroups {
				pointerCells = append(pointerCells, group...)
			}

			//Get rid of all pointer cells from the rows of cells that will be
			//affected.
			affectedCells = affectedCells.RemoveCells(pointerCells)

			//Okay, now the set is solid.

			//Okay, we have a candidate step (unchunked). Is it useful?
			step := &SolveStep{self,
				affectedCells,
				IntSlice{i},
				pointerCells,
				nil,
				nil,
			}

			if step.IsUseful(grid) {
				select {
				case results <- step:
				case <-done:
					return
				}
			}

		}

	}

}
<commit_msg>Fixed an off-by-one error in last commit<commit_after>package sudoku

import (
	"fmt"
	"math/rand"
)

type swordfishTechnique struct {
	*basicSolveTechnique
}

func (self *swordfishTechnique) humanLikelihood(step *SolveStep) float64 {
	//TODO: reason more carefully about how hard this technique is.
	return self.difficultyHelper(70.0)
}

func (self *swordfishTechnique) Description(step *SolveStep) string {

	if len(step.TargetNums) != 1 {
		return ""
	}

	groupName := "columns"
	otherGroupName := "rows"

	if self.groupType == _GROUP_ROW {
		groupName = "rows"
		otherGroupName = "columns"
	}

	return fmt.Sprintf("%d is only possible in two cells each in three different %s, all of which align onto three %s, which means that %d can't be in any of the other cells in those %s (%s)",
		step.TargetNums[0],
		groupName,
		otherGroupName,
		step.TargetNums[0],
		otherGroupName,
		step.TargetCells.Description(),
	)

}

func (self *swordfishTechnique) Find(grid *Grid, results chan *SolveStep, done chan bool) {

	getter := self.getter(grid)

	//For this technique, the primary access is the first type of group we look at to find
	//cells with only 2 spots for a given number.

	//TODO: Implement the "relaxed" version of this technique, too.

	for _, i := range rand.Perm(DIM + 1) {

		if i == 0 {
			continue
		}
		//The candidate we're considering

		//Consider each of the major-axis groups to see if more than three have
		//only two candidates for

		var majorAxisGroupsWithTwoOptionsForCandidate []CellSlice

		for c := 0; c < DIM; c++ {
			majorAxisGroup := getter(c)

			cellsWithCandidatePossibility := majorAxisGroup.FilterByPossible(i)

			if len(cellsWithCandidatePossibility) == 2 {
				//TODO: shouldn't we keep track of the rows where the possibilities were, too?
				majorAxisGroupsWithTwoOptionsForCandidate = append(majorAxisGroupsWithTwoOptionsForCandidate, cellsWithCandidatePossibility)
			}

		}

		//Do we have more than three major axis groups identified?
		if len(majorAxisGroupsWithTwoOptionsForCandidate) < 3 {
			continue
		}

		//Consider every subset of size three
		for _, indexes := range subsetIndexes(len(majorAxisGroupsWithTwoOptionsForCandidate), 3) {
			majorAxisGroups := make([]CellSlice, 3)
			for i, index := range indexes {
				majorAxisGroups[i] = majorAxisGroupsWithTwoOptionsForCandidate[index]
			}

			//OK, now majorAxisGroups has the set of three we're operating on.
			//Do their minorAxis groups line up to a set of three as well?

			var minorGroupIndexSet intSet

			for _, group := range majorAxisGroups {
				if self.groupType == _GROUP_COL {
					minorGroupIndexSet = minorGroupIndexSet.union(group.AllRows().toIntSet())
				} else {
					minorGroupIndexSet = minorGroupIndexSet.union(group.AllCols().toIntSet())
				}
			}

			if len(minorGroupIndexSet) != 3 {
				//Nah, didn't have three rows.
				continue
			}

			//Woot, looks like we found a valid set.

			//Generate the list of cells that will be affected
			var affectedCells CellSlice

			for minorGroupIndex, _ := range minorGroupIndexSet {
				if self.groupType == _GROUP_COL {
					affectedCells = append(affectedCells, grid.Row(minorGroupIndex)...)
				} else {
					affectedCells = append(affectedCells, grid.Col(minorGroupIndex)...)
				}
			}

			//Get rid of cells that are already filled, or where removing 'i'
			//would be a no-op anyway since it's not already a possible there.
			affectedCells = affectedCells.FilterByPossible(i)

			//Gather the list of all cells that are "pointing" to this

			var pointerCells CellSlice

			for _, group := range majorAxisGroups {
				pointerCells = append(pointerCells, group...)
			}

			//Get rid of all pointer cells from the rows of cells that will be
			//affected.
			affectedCells = affectedCells.RemoveCells(pointerCells)

			//Okay, now the set is solid.

			//Okay, we have a candidate step (unchunked). Is it useful?
			step := &SolveStep{self,
				affectedCells,
				IntSlice{i},
				pointerCells,
				nil,
				nil,
			}

			if step.IsUseful(grid) {
				select {
				case results <- step:
				case <-done:
					return
				}
			}

		}

	}

}
<|endoftext|>


<commit_before>package persist

import (
	"fmt"
	"io"
	"log"
	"os"
	"sync"
	"path/filepath"

	"github.com/NebulousLabs/Sia/build"
)

// Logger is a wrapper for the standard library logger that enforces logging
// with the Sia-standard settings. It also supports a Close method, which
// attempts to close the underlying io.Writer.
type Logger struct {
	*log.Logger
	w io.Writer
}

// Close logs a shutdown message and closes the Logger's underlying io.Writer,
// if it is also an io.Closer.
func (l *Logger) Close() error {
	l.Output(2, "SHUTDOWN: Logging has terminated.")
	if c, ok := l.w.(io.Closer); ok {
		return c.Close()
	}
	return nil
}

// Critical logs a message with a CRITICAL prefix that guides the user to the
// Sia github tracker. If debug mode is enabled, it will also write the message
// to os.Stderr and panic. Critical should only be called if there has been a
// developer error, otherwise Severe should be called.
func (l *Logger) Critical(v ...interface{}) {
	l.Output(2, "CRITICAL: "+fmt.Sprintln(v...))
	build.Critical(v...)
}

// Debug is equivalent to Logger.Print when build.DEBUG is true. Otherwise it
// is a no-op.
func (l *Logger) Debug(v ...interface{}) {
	if build.DEBUG {
		l.Output(2, fmt.Sprint(v...))
	}
}

// Debugf is equivalent to Logger.Printf when build.DEBUG is true. Otherwise it
// is a no-op.
func (l *Logger) Debugf(format string, v ...interface{}) {
	if build.DEBUG {
		l.Output(2, fmt.Sprintf(format, v...))
	}
}

// Debugln is equivalent to Logger.Println when build.DEBUG is true. Otherwise
// it is a no-op.
func (l *Logger) Debugln(v ...interface{}) {
	if build.DEBUG {
		l.Output(2, "[DEBUG] "+fmt.Sprintln(v...))
	}
}

// Severe logs a message with a SEVERE prefix. If debug mode is enabled, it
// will also write the message to os.Stderr and panic. Severe should be called
// if there is a severe problem with the user's machine or setup that should be
// addressed ASAP but does not necessarily require that the machine crash or
// exit.
func (l *Logger) Severe(v ...interface{}) {
	l.Output(2, "SEVERE: "+fmt.Sprintln(v...))
	build.Severe(v...)
}

// NewLogger returns a logger that can be closed. Calls should not be made to
// the logger after 'Close' has been called.
func NewLogger(w io.Writer) *Logger {
	l := log.New(w, "", log.Ldate|log.Ltime|log.Lmicroseconds|log.Lshortfile|log.LUTC)
	l.Output(3, "STARTUP: Logging has started.") // Call depth is 3 because NewLogger is usually called by NewFileLogger
	return &Logger{l, w}
}

// closeableFile wraps an os.File to perform sanity checks on its Write and
// Close methods. When the checks are enabled, calls to Write or Close will
// panic if they are called after the file has already been closed.
type closeableFile struct {
	*os.File
	closed bool
	mu     sync.RWMutex
}

// Close closes the file and sets the closed flag.
func (cf *closeableFile) Close() error {
	cf.mu.Lock()
	defer cf.mu.Unlock()
	// Sanity check - close should not have been called yet.
	if cf.closed {
		build.Critical("cannot close the file; already closed")
	}

	// Ensure that all data has actually hit the disk.
	if err := cf.Sync(); err != nil {
		return err
	}
	cf.closed = true
	return cf.File.Close()
}

// Write takes the input data and writes it to the file.
func (cf *closeableFile) Write(b []byte) (int, error) {
	cf.mu.RLock()
	defer cf.mu.RUnlock()
	// Sanity check - close should not have been called yet.
	if cf.closed {
		build.Critical("cannot write to the file after it has been closed")
	}
	return cf.File.Write(b)
}

// NewFileLogger returns a logger that logs to logFilename. The file is opened
// in append mode, and created if it does not exist.
func NewFileLogger(logFilename string) (*Logger, error) {
	logFile, err := os.OpenFile(filepath.Join("/var/log/sia", filepath.Base(logFilename)), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0664)
	if err != nil {
		return nil, err
	}
	cf := &closeableFile{File: logFile}
	return NewLogger(cf), nil
}
<commit_msg>undo patch in master for logging<commit_after>package persist

import (
	"fmt"
	"io"
	"log"
	"os"
	"sync"

	"github.com/NebulousLabs/Sia/build"
)

// Logger is a wrapper for the standard library logger that enforces logging
// with the Sia-standard settings. It also supports a Close method, which
// attempts to close the underlying io.Writer.
type Logger struct {
	*log.Logger
	w io.Writer
}

// Close logs a shutdown message and closes the Logger's underlying io.Writer,
// if it is also an io.Closer.
func (l *Logger) Close() error {
	l.Output(2, "SHUTDOWN: Logging has terminated.")
	if c, ok := l.w.(io.Closer); ok {
		return c.Close()
	}
	return nil
}

// Critical logs a message with a CRITICAL prefix that guides the user to the
// Sia github tracker. If debug mode is enabled, it will also write the message
// to os.Stderr and panic. Critical should only be called if there has been a
// developer error, otherwise Severe should be called.
func (l *Logger) Critical(v ...interface{}) {
	l.Output(2, "CRITICAL: "+fmt.Sprintln(v...))
	build.Critical(v...)
}

// Debug is equivalent to Logger.Print when build.DEBUG is true. Otherwise it
// is a no-op.
func (l *Logger) Debug(v ...interface{}) {
	if build.DEBUG {
		l.Output(2, fmt.Sprint(v...))
	}
}

// Debugf is equivalent to Logger.Printf when build.DEBUG is true. Otherwise it
// is a no-op.
func (l *Logger) Debugf(format string, v ...interface{}) {
	if build.DEBUG {
		l.Output(2, fmt.Sprintf(format, v...))
	}
}

// Debugln is equivalent to Logger.Println when build.DEBUG is true. Otherwise
// it is a no-op.
func (l *Logger) Debugln(v ...interface{}) {
	if build.DEBUG {
		l.Output(2, "[DEBUG] "+fmt.Sprintln(v...))
	}
}

// Severe logs a message with a SEVERE prefix. If debug mode is enabled, it
// will also write the message to os.Stderr and panic. Severe should be called
// if there is a severe problem with the user's machine or setup that should be
// addressed ASAP but does not necessarily require that the machine crash or
// exit.
func (l *Logger) Severe(v ...interface{}) {
	l.Output(2, "SEVERE: "+fmt.Sprintln(v...))
	build.Severe(v...)
}

// NewLogger returns a logger that can be closed. Calls should not be made to
// the logger after 'Close' has been called.
func NewLogger(w io.Writer) *Logger {
	l := log.New(w, "", log.Ldate|log.Ltime|log.Lmicroseconds|log.Lshortfile|log.LUTC)
	l.Output(3, "STARTUP: Logging has started.") // Call depth is 3 because NewLogger is usually called by NewFileLogger
	return &Logger{l, w}
}

// closeableFile wraps an os.File to perform sanity checks on its Write and
// Close methods. When the checks are enabled, calls to Write or Close will
// panic if they are called after the file has already been closed.
type closeableFile struct {
	*os.File
	closed bool
	mu     sync.RWMutex
}

// Close closes the file and sets the closed flag.
func (cf *closeableFile) Close() error {
	cf.mu.Lock()
	defer cf.mu.Unlock()
	// Sanity check - close should not have been called yet.
	if cf.closed {
		build.Critical("cannot close the file; already closed")
	}

	// Ensure that all data has actually hit the disk.
	if err := cf.Sync(); err != nil {
		return err
	}
	cf.closed = true
	return cf.File.Close()
}

// Write takes the input data and writes it to the file.
func (cf *closeableFile) Write(b []byte) (int, error) {
	cf.mu.RLock()
	defer cf.mu.RUnlock()
	// Sanity check - close should not have been called yet.
	if cf.closed {
		build.Critical("cannot write to the file after it has been closed")
	}
	return cf.File.Write(b)
}

// NewFileLogger returns a logger that logs to logFilename. The file is opened
// in append mode, and created if it does not exist.
func NewFileLogger(logFilename string) (*Logger, error) {
	logFile, err := os.OpenFile(logFilename, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0660)
	if err != nil {
		return nil, err
	}
	cf := &closeableFile{File: logFile}
	return NewLogger(cf), nil
}
<|endoftext|>


<commit_before>package main

// scanning an HTTP response for phrases

import (
	"http"
	"mahonia.googlecode.com/hg"
)

// phrasesInResponse scans the content of an http.Response for phrases,
// and returns a map of phrases and counts.
func phrasesInResponse(res *http.Response) map[string]int {
	defer res.Body.Close()
	wr := newWordReader(res.Body, mahonia.NewDecoder("UTF-8")) // TODO: support other encodings, HTML entities
	ps := newPhraseScanner()
	ps.scanByte(' ')
	buf := make([]byte, 4096)
	for {
		n, err := wr.Read(buf)
		if err != nil {
			break
		}
		for _, c := range buf[:n] {
			ps.scanByte(c)
		}
	}
	ps.scanByte(' ')

	return ps.tally
}
<commit_msg>Use charset from Content-Type header instead of assuming all pages are UTF-8.<commit_after>package main

// scanning an HTTP response for phrases

import (
	"http"
	"mahonia.googlecode.com/hg"
	"strings"
)

// phrasesInResponse scans the content of an http.Response for phrases,
// and returns a map of phrases and counts.
func phrasesInResponse(res *http.Response) map[string]int {
	defer res.Body.Close()

	contentType := res.Header.Get("Content-Type")

	wr := newWordReader(res.Body, decoderForContentType(contentType))
	ps := newPhraseScanner()
	ps.scanByte(' ')
	buf := make([]byte, 4096)
	for {
		n, err := wr.Read(buf)
		if err != nil {
			break
		}
		for _, c := range buf[:n] {
			ps.scanByte(c)
		}
	}
	ps.scanByte(' ')

	return ps.tally
}

func decoderForContentType(t string) mahonia.Decoder {
	t = strings.ToLower(t)
	var result mahonia.Decoder

	i := strings.Index(t, "charset=")
	if i != -1 {
		charset := t[i+len("charset="):]
		i = strings.Index(charset, ";")
		if i != -1 {
			charset = charset[:i]
		}
		result = mahonia.NewDecoder(charset)
	}

	if result == nil {
		result = mahonia.FallbackDecoder(mahonia.NewDecoder("UTF-8"), mahonia.NewDecoder("windows-1252"))
	}

	if strings.Contains(t, "html") {
		result = mahonia.FallbackDecoder(mahonia.EntityDecoder(), result)
	}

	return result
}
<|endoftext|>


<commit_before>package probstruct

import (
	"errors"
	"fmt"
	"math"
	"math/bits"

	"github.com/markuskont/probstruct/pkg/hasher"
	"github.com/markuskont/probstruct/pkg/util"
)

const (
	bitness = 64
)

// HyperLogLog implements hyperloglog prob counting algorithm
type HyperLogLog struct {
	m uint32
	p uint
	// each bucket will hold max( count_zeroes + 1 ) in 64bit uint with 4..16 bits already derived
	// thus, uint8 cannot overflow
	buckets []uint8
	alpha   float64

	cardinality uint64

	// select hashing method, in line with bloom and cms implementations
	hash hasher.Algorithm
}

// NewHyperLogLog initializes new HLL struct
// hashing algorithm will default to fnv if left as nil
func NewHyperLogLog(precision uint, hash *hasher.Algorithm) (h *HyperLogLog, err error) {
	if precision < 4 || precision > 16 {
		return nil, errors.New("precision must be integer between 4 and 16")
	}
	h = &HyperLogLog{
		p:    precision,
		m:    1 << precision,
		hash: hasher.Fnv,
	}
	if hash != nil {
		h.hash = *hash
	}
	h.buckets = make([]uint8, h.m)
	// Magic numbers for hash collision correction
	switch h.m {
	case 16:
		h.alpha = 0.673
	case 32:
		h.alpha = 0.697
	case 64:
		h.alpha = 0.709
	default:
		h.alpha = 0.7213 / (1 + 1.079/float64(h.m))
	}
	return h, err
}

// Add is a wrapper for adding new items to leading zero counter
func (h *HyperLogLog) Add(items ...[]byte) *HyperLogLog {
	if items == nil || len(items) == 0 {
		return h
	}
	for _, v := range items {
		h.add64(h.hash.GetBaseHash(v).First())
	}
	return h
}

// Add64 calculates leading zeros from 64bit hash value and updates respective buckets
func (h *HyperLogLog) add64(hash uint64) *HyperLogLog {
	diff := bitness - h.p
	index := hash >> diff
	tail := hash << h.p
	count := uint8(bits.LeadingZeros64(tail)) + 1

	if count > h.buckets[index] {
		h.buckets[index] = count
	}
	return h
}

// Count calculates harmonic mean and updates cardinality
func (h *HyperLogLog) Count() *HyperLogLog {
	Z := float64(0)
	for _, c := range h.buckets {
		if c > 0 {
			Z += float64(1 / math.Pow(float64(2), float64(c)))
		}
	}
	Z = 1 / Z
	count := h.alpha * math.Pow(float64(h.m), 2) * Z
	h.cardinality = uint64(math.Floor(count))
	return h
}

// Merge allows multiple hll objects to be merged into a single structure
func Merge(containers ...*HyperLogLog) (*HyperLogLog, error) {
	if containers == nil || len(containers) < 2 {
		return nil, &ErrNoContainers{msg: "At least 2 HLL containers needed for merge"}
	}
	precision := containers[0].p
	hash := containers[0].hash
	for _, v := range containers {
		if v.p != precision {
			return nil, &ErrPrecisionMismatch{p1: v.p, p2: precision}
		}
		if v.hash != hash {
			return nil, &ErrHashFnMismatch{h1: v.hash, h2: hash}
		}
		precision = v.p
		hash = v.hash
	}
	merged, err := NewHyperLogLog(precision, &hash)
	if err != nil {
		return nil, err
	}
	for _, v := range containers {
		for j, v2 := range v.buckets {
			merged.buckets[j] = util.MaxUint8(v2, merged.buckets[j])
		}
	}
	return merged, nil
}

type ErrNoContainers struct{ msg string }

func (e ErrNoContainers) Error() string { return e.msg }

type ErrPrecisionMismatch struct{ p1, p2 uint }

func (e ErrPrecisionMismatch) Error() string {
	return fmt.Sprintf("HLL precision mismatch %d - %d, cannot merge", e.p1, e.p2)
}

type ErrHashFnMismatch struct{ h1, h2 hasher.Algorithm }

func (e ErrHashFnMismatch) Error() string {
	return fmt.Sprintf("HLL hash function mismatch %d - %d, cannot merge", e.h1, e.h2)
}
<commit_msg>cover possible fail case<commit_after>package probstruct

import (
	"errors"
	"fmt"
	"math"
	"math/bits"

	"github.com/markuskont/probstruct/pkg/hasher"
	"github.com/markuskont/probstruct/pkg/util"
)

const (
	bitness = 64
)

// HyperLogLog implements hyperloglog prob counting algorithm
type HyperLogLog struct {
	m uint32
	p uint
	// each bucket will hold max( count_zeroes + 1 ) in 64bit uint with 4..16 bits already derived
	// thus, uint8 cannot overflow
	buckets []uint8
	alpha   float64

	cardinality uint64

	// select hashing method, in line with bloom and cms implementations
	hash hasher.Algorithm
}

// NewHyperLogLog initializes new HLL struct
// hashing algorithm will default to fnv if left as nil
func NewHyperLogLog(precision uint, hash *hasher.Algorithm) (h *HyperLogLog, err error) {
	if precision < 4 || precision > 16 {
		return nil, errors.New("precision must be integer between 4 and 16")
	}
	h = &HyperLogLog{
		p:    precision,
		m:    1 << precision,
		hash: hasher.Fnv,
	}
	if hash != nil {
		h.hash = *hash
	}
	h.buckets = make([]uint8, h.m)
	// Magic numbers for hash collision correction
	switch h.m {
	case 16:
		h.alpha = 0.673
	case 32:
		h.alpha = 0.697
	case 64:
		h.alpha = 0.709
	default:
		h.alpha = 0.7213 / (1 + 1.079/float64(h.m))
	}
	return h, err
}

// Add is a wrapper for adding new items to leading zero counter
func (h *HyperLogLog) Add(items ...[]byte) *HyperLogLog {
	if items == nil || len(items) == 0 {
		return h
	}
	for _, v := range items {
		h.add64(h.hash.GetBaseHash(v).First())
	}
	return h
}

// Add64 calculates leading zeros from 64bit hash value and updates respective buckets
func (h *HyperLogLog) add64(hash uint64) *HyperLogLog {
	diff := bitness - h.p
	index := hash >> diff
	tail := hash << h.p
	count := uint8(bits.LeadingZeros64(tail)) + 1

	if count > h.buckets[index] {
		h.buckets[index] = count
	}
	return h
}

// Count calculates harmonic mean and updates cardinality
func (h *HyperLogLog) Count() *HyperLogLog {
	Z := float64(0)
	for _, c := range h.buckets {
		if c > 0 {
			Z += float64(1 / math.Pow(float64(2), float64(c)))
		}
	}
	Z = 1 / Z
	count := h.alpha * math.Pow(float64(h.m), 2) * Z
	h.cardinality = uint64(math.Floor(count))
	return h
}

// Merge allows multiple hll objects to be merged into a single structure
func Merge(containers ...*HyperLogLog) (*HyperLogLog, error) {
	if containers == nil || len(containers) < 2 {
		return nil, &ErrNoContainers{msg: "At least 2 HLL containers needed for merge"}
	}
	precision := containers[0].p
	hash := containers[0].hash
	for _, v := range containers {
		if v.p != precision {
			return nil, &ErrPrecisionMismatch{p1: v.p, p2: precision}
		}
		if v.hash != hash {
			return nil, &ErrHashFnMismatch{h1: v.hash, h2: hash}
		}
		precision = v.p
		hash = v.hash
	}
	merged, err := NewHyperLogLog(precision, &hash)
	if err != nil {
		return nil, err
	}
	for _, v := range containers {
		if v == nil {
			return nil, fmt.Errorf("Missing hll container in merge list")
		}
		for j, v2 := range v.buckets {
			merged.buckets[j] = util.MaxUint8(v2, merged.buckets[j])
		}
	}
	return merged, nil
}

type ErrNoContainers struct{ msg string }

func (e ErrNoContainers) Error() string { return e.msg }

type ErrPrecisionMismatch struct{ p1, p2 uint }

func (e ErrPrecisionMismatch) Error() string {
	return fmt.Sprintf("HLL precision mismatch %d - %d, cannot merge", e.p1, e.p2)
}

type ErrHashFnMismatch struct{ h1, h2 hasher.Algorithm }

func (e ErrHashFnMismatch) Error() string {
	return fmt.Sprintf("HLL hash function mismatch %d - %d, cannot merge", e.h1, e.h2)
}
<|endoftext|>


<commit_before>package hub

import (
	"golang.org/x/net/context"

	log "github.com/inconshreveable/log15"

	"github.com/tsileo/blobstash/pkg/blob"
	_ "github.com/tsileo/blobstash/pkg/ctxutil"
)

type EventType int

const (
	NewBlob EventType = iota
	ScanBlob
	GarbageCollection
	AppUpdate
)

type AppUpdateData struct {
	Ref  string `json:"ref"`
	Name string `json:"name"`
}

type Hub struct {
	log         log.Logger
	subscribers map[EventType]map[string]func(context.Context, *blob.Blob, interface{}) error
}

func (h *Hub) Subscribe(etype EventType, name string, callback func(context.Context, *blob.Blob, interface{}) error) {
	h.log.Info("new subscription", "type", etype, "name", name)
	h.subscribers[etype][name] = callback
}

func (h *Hub) newEvent(ctx context.Context, etype EventType, blob *blob.Blob, data interface{}) error {
	l := h.log.New("type", etype, "blob", blob, "data", data)
	l.Debug("new event")
	for name, callback := range h.subscribers[etype] {
		h.log.Debug("triggering callback", "name", name)
		if err := callback(ctx, blob, data); err != nil {
			return err
		}
	}
	return nil
}

func (h *Hub) NewAppUpdateEvent(ctx context.Context, blob *blob.Blob, data interface{}) error {
	return h.newEvent(ctx, AppUpdate, blob, data)
}

func (h *Hub) NewBlobEvent(ctx context.Context, blob *blob.Blob, data interface{}) error {
	return h.newEvent(ctx, NewBlob, blob, data)
}

func (h *Hub) ScanBlobEvent(ctx context.Context, blob *blob.Blob, data interface{}) error {
	return h.newEvent(ctx, ScanBlob, blob, data)
}

func New(logger log.Logger) *Hub {
	logger.Debug("init")
	return &Hub{
		log: logger,
		subscribers: map[EventType]map[string]func(context.Context, *blob.Blob, interface{}) error{
			NewBlob:   map[string]func(context.Context, *blob.Blob, interface{}) error{},
			ScanBlob:  map[string]func(context.Context, *blob.Blob, interface{}) error{},
			AppUpdate: map[string]func(context.Context, *blob.Blob, interface{}) error{},
		},
	}
}
<commit_msg>hub: upload app event payload<commit_after>package hub

import (
	"golang.org/x/net/context"

	log "github.com/inconshreveable/log15"

	"github.com/tsileo/blobstash/pkg/blob"
	_ "github.com/tsileo/blobstash/pkg/ctxutil"
)

type EventType int

const (
	NewBlob EventType = iota
	ScanBlob
	GarbageCollection
	AppUpdate
)

type AppUpdateData struct {
	Ref          string
	Name         string
	RawAppConfig []byte
}

type Hub struct {
	log         log.Logger
	subscribers map[EventType]map[string]func(context.Context, *blob.Blob, interface{}) error
}

func (h *Hub) Subscribe(etype EventType, name string, callback func(context.Context, *blob.Blob, interface{}) error) {
	h.log.Info("new subscription", "type", etype, "name", name)
	h.subscribers[etype][name] = callback
}

func (h *Hub) newEvent(ctx context.Context, etype EventType, blob *blob.Blob, data interface{}) error {
	l := h.log.New("type", etype, "blob", blob, "data", data)
	l.Debug("new event")
	for name, callback := range h.subscribers[etype] {
		h.log.Debug("triggering callback", "name", name)
		if err := callback(ctx, blob, data); err != nil {
			return err
		}
	}
	return nil
}

func (h *Hub) NewAppUpdateEvent(ctx context.Context, blob *blob.Blob, data interface{}) error {
	return h.newEvent(ctx, AppUpdate, blob, data)
}

func (h *Hub) NewBlobEvent(ctx context.Context, blob *blob.Blob, data interface{}) error {
	return h.newEvent(ctx, NewBlob, blob, data)
}

func (h *Hub) ScanBlobEvent(ctx context.Context, blob *blob.Blob, data interface{}) error {
	return h.newEvent(ctx, ScanBlob, blob, data)
}

func New(logger log.Logger) *Hub {
	logger.Debug("init")
	return &Hub{
		log: logger,
		subscribers: map[EventType]map[string]func(context.Context, *blob.Blob, interface{}) error{
			NewBlob:   map[string]func(context.Context, *blob.Blob, interface{}) error{},
			ScanBlob:  map[string]func(context.Context, *blob.Blob, interface{}) error{},
			AppUpdate: map[string]func(context.Context, *blob.Blob, interface{}) error{},
		},
	}
}
<|endoftext|>


<commit_before>// Copyright 2012 The Plotinum Authors. All rights reserved.
// Use of this source code is governed by an MIT-style license
// that can be found in the LICENSE file.

package plot

import (
	"code.google.com/p/plotinum/vg"
)

// A Legend gives a description of the meaning of different
// data elements of the plot.
type Legend struct {
	// TextStyle is the style given to the legend
	// entry texts.
	TextStyle

	// Top and Left specify the location of the legend.
	// If Top is true the legend is located along the top
	// edge of the plot, otherwise it is located along
	// the bottom edge.  If Left is true then the legend
	// is located along the left edge of the plot, and the
	// text is positioned after the icons, otherwise it is
	// located along the right edge and the text is
	// positioned before the icons.
	Top, Left bool

	// XOffs and YOffs are added to the legend's
	// final position.
	XOffs, YOffs vg.Length

	// ThumbnailWidth is the width of legend icons.
	ThumbnailWidth vg.Length

	// entries are all of the legendEntries described
	// by this legend.
	entries []legendEntry
}

// A legendEntry represents a single line of a legend, it
// has a name and an icon.
type legendEntry struct {
	// text is the text associated with this entry.
	text string

	// thumbs is a slice of all of the thumbnails styles
	thumbs []Thumbnailer
}

// Thumbnailer wraps the Thumbnail method, which
// draws the small image in a legend representing the
// style of data.
type Thumbnailer interface {
	// Thumbnail draws an thumbnail representing
	// a legend entry.  The thumbnail will usually show
	// a smaller representation of the style used
	// to plot the corresponding data.
	Thumbnail(da *DrawArea)
}

// makeLegend returns a legend with the default
// parameter settings.
func makeLegend() (Legend, error) {
	font, err := vg.MakeFont(defaultFont, vg.Points(12))
	if err != nil {
		return Legend{}, err
	}
	return Legend{
		ThumbnailWidth: vg.Points(20),
		TextStyle: TextStyle{Font: font},
	}, nil
}

// draw draws the legend to the given DrawArea.
func (l *Legend) draw(da *DrawArea) {
	iconx := da.Min.X
	textx := iconx + l.ThumbnailWidth + l.TextStyle.Width(" ")
	xalign := 0.0
	if !l.Left {
		iconx = da.Max().X - l.ThumbnailWidth
		textx = iconx - l.TextStyle.Width(" ")
		xalign = -1
	}
	textx += l.XOffs
	iconx += l.XOffs

	enth := l.entryHeight()
	y := da.Max().Y - enth
	if !l.Top {
		y = da.Min.Y + enth*(vg.Length(len(l.entries))-1)
	}
	y += l.YOffs

	icon := &DrawArea{
		Canvas: da.Canvas,
		Rect:   Rect{Min: Point{iconx, y}, Size: Point{l.ThumbnailWidth, enth}},
	}
	for _, e := range l.entries {
		for _, t := range e.thumbs {
			t.Thumbnail(icon)
		}
		yoffs := (enth - l.TextStyle.Height(e.text)) / 2
		da.FillText(l.TextStyle, textx, icon.Min.Y+yoffs, xalign, 0, e.text)
		icon.Min.Y -= enth
	}
}

// entryHeight returns the height of the tallest legend
// entry text.
func (l *Legend) entryHeight() (height vg.Length) {
	for _, e := range l.entries {
		if h := l.TextStyle.Height(e.text); h > height {
			height = h
		}
	}
	return
}

// Add adds an entry to the legend with the given name.
// The entry's thumbnail is drawn as the composite of all of the
// thumbnails.
func (l *Legend) Add(name string, thumbs ...Thumbnailer) {
	l.entries = append(l.entries, legendEntry{text: name, thumbs: thumbs})
}
<commit_msg>More comment fixes.<commit_after>// Copyright 2012 The Plotinum Authors. All rights reserved.
// Use of this source code is governed by an MIT-style license
// that can be found in the LICENSE file.

package plot

import (
	"code.google.com/p/plotinum/vg"
)

// A Legend gives a description of the meaning of different
// data elements of the plot.  Each legend entry has a name
// and a thumbnail, where the thumbnail shows a small
// sample of the display style of the corresponding data.
type Legend struct {
	// TextStyle is the style given to the legend
	// entry texts.
	TextStyle

	// Top and Left specify the location of the legend.
	// If Top is true the legend is located along the top
	// edge of the plot, otherwise it is located along
	// the bottom edge.  If Left is true then the legend
	// is located along the left edge of the plot, and the
	// text is positioned after the icons, otherwise it is
	// located along the right edge and the text is
	// positioned before the icons.
	Top, Left bool

	// XOffs and YOffs are added to the legend's
	// final position.
	XOffs, YOffs vg.Length

	// ThumbnailWidth is the width of legend thumbnails.
	ThumbnailWidth vg.Length

	// entries are all of the legendEntries described
	// by this legend.
	entries []legendEntry
}

// A legendEntry represents a single line of a legend, it
// has a name and an icon.
type legendEntry struct {
	// text is the text associated with this entry.
	text string

	// thumbs is a slice of all of the thumbnails styles
	thumbs []Thumbnailer
}

// Thumbnailer wraps the Thumbnail method, which
// draws the small image in a legend representing the
// style of data.
type Thumbnailer interface {
	// Thumbnail draws an thumbnail representing
	// a legend entry.  The thumbnail will usually show
	// a smaller representation of the style used
	// to plot the corresponding data.
	Thumbnail(da *DrawArea)
}

// makeLegend returns a legend with the default
// parameter settings.
func makeLegend() (Legend, error) {
	font, err := vg.MakeFont(defaultFont, vg.Points(12))
	if err != nil {
		return Legend{}, err
	}
	return Legend{
		ThumbnailWidth: vg.Points(20),
		TextStyle: TextStyle{Font: font},
	}, nil
}

// draw draws the legend to the given DrawArea.
func (l *Legend) draw(da *DrawArea) {
	iconx := da.Min.X
	textx := iconx + l.ThumbnailWidth + l.TextStyle.Width(" ")
	xalign := 0.0
	if !l.Left {
		iconx = da.Max().X - l.ThumbnailWidth
		textx = iconx - l.TextStyle.Width(" ")
		xalign = -1
	}
	textx += l.XOffs
	iconx += l.XOffs

	enth := l.entryHeight()
	y := da.Max().Y - enth
	if !l.Top {
		y = da.Min.Y + enth*(vg.Length(len(l.entries))-1)
	}
	y += l.YOffs

	icon := &DrawArea{
		Canvas: da.Canvas,
		Rect:   Rect{Min: Point{iconx, y}, Size: Point{l.ThumbnailWidth, enth}},
	}
	for _, e := range l.entries {
		for _, t := range e.thumbs {
			t.Thumbnail(icon)
		}
		yoffs := (enth - l.TextStyle.Height(e.text)) / 2
		da.FillText(l.TextStyle, textx, icon.Min.Y+yoffs, xalign, 0, e.text)
		icon.Min.Y -= enth
	}
}

// entryHeight returns the height of the tallest legend
// entry text.
func (l *Legend) entryHeight() (height vg.Length) {
	for _, e := range l.entries {
		if h := l.TextStyle.Height(e.text); h > height {
			height = h
		}
	}
	return
}

// Add adds an entry to the legend with the given name.
// The entry's thumbnail is drawn as the composite of all of the
// thumbnails.
func (l *Legend) Add(name string, thumbs ...Thumbnailer) {
	l.entries = append(l.entries, legendEntry{text: name, thumbs: thumbs})
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package metrics

import (
	"context"
	"errors"
	"fmt"
	"net/http"
	"strconv"
	"sync"
	"sync/atomic"
	"time"

	"go.opencensus.io/stats"
	"go.opencensus.io/stats/view"
	"go.uber.org/zap"
	"golang.org/x/sync/errgroup"

	pkgmetrics "knative.dev/pkg/metrics"
	av1alpha1 "knative.dev/serving/pkg/apis/autoscaling/v1alpha1"
	"knative.dev/serving/pkg/apis/networking"
	"knative.dev/serving/pkg/apis/serving"
	"knative.dev/serving/pkg/metrics"
	"knative.dev/serving/pkg/resources"
)

const (
	httpClientTimeout = 3 * time.Second

	// scraperPodName is the name used in all stats sent from the scraper to
	// the autoscaler. The actual customer pods are hidden behind the scraper. The
	// autoscaler does need to know how many customer pods are reporting metrics.
	// Instead, the autoscaler knows the stats it receives are either from the
	// scraper or the activator.
	scraperPodName = "service-scraper"

	// scraperMaxRetries are retries to be done to the actual Scrape routine. We want
	// to retry if a Scrape returns an error or if the Scrape goes to a pod we already
	// scraped.
	scraperMaxRetries = 10
)

var (
	// ErrFailedGetEndpoints specifies the error returned by scraper when it fails to
	// get endpoints.
	ErrFailedGetEndpoints = errors.New("failed to get endpoints")

	// ErrDidNotReceiveStat specifies the error returned by scraper when it does not receive
	// stat from an unscraped pod
	ErrDidNotReceiveStat = errors.New("did not receive stat from an unscraped pod")

	// Sentinel error to retrun from pod scraping routine, when we could not
	// scrape even a single pod.
	errNoPodsScraped = errors.New("no pods scraped")
	errPodsExhausted = errors.New("pods exhausted")

	scrapeTimeM = stats.Float64(
		"scrape_time",
		"Time to scrape metrics in milliseconds",
		stats.UnitMilliseconds)
)

func init() {
	if err := view.Register(
		&view.View{
			Description: "The time to scrape metrics in milliseconds",
			Measure:     scrapeTimeM,
			Aggregation: view.Distribution(pkgmetrics.Buckets125(1, 100000)...),
			TagKeys:     metrics.CommonRevisionKeys,
		},
	); err != nil {
		panic(err)
	}
}

// StatsScraper defines the interface for collecting Revision metrics
type StatsScraper interface {
	// Scrape scrapes the Revision queue metric endpoint. The duration is used
	// to cutoff young pods, whose stats might skew lower.
	Scrape(time.Duration) (Stat, error)
}

// scrapeClient defines the interface for collecting Revision metrics for a given
// URL. Internal used only.
type scrapeClient interface {
	// Scrape scrapes the given URL.
	Scrape(url string) (Stat, error)
}

// cacheDisabledClient is a http client with cache disabled. It is shared by
// every goruntime for a revision scraper.
var cacheDisabledClient = &http.Client{
	Transport: &http.Transport{
		// Do not use the cached connection
		DisableKeepAlives: true,
	},
	Timeout: httpClientTimeout,
}

// serviceScraper scrapes Revision metrics via a K8S service by sampling. Which
// pod to be picked up to serve the request is decided by K8S. Please see
// https://kubernetes.io/docs/concepts/services-networking/network-policies/
// for details.
type serviceScraper struct {
	sClient  scrapeClient
	counter  resources.EndpointsCounter
	url      string
	statsCtx context.Context
	logger   *zap.SugaredLogger

	podAccessor     resources.PodAccessor
	podsAddressable bool
}

// NewStatsScraper creates a new StatsScraper for the Revision which
// the given Metric is responsible for.
func NewStatsScraper(metric *av1alpha1.Metric, counter resources.EndpointsCounter,
	podAccessor resources.PodAccessor, logger *zap.SugaredLogger) (StatsScraper, error) {
	sClient, err := newHTTPScrapeClient(cacheDisabledClient)
	if err != nil {
		return nil, err
	}
	return newServiceScraperWithClient(metric, counter, podAccessor, sClient, logger)
}

func newServiceScraperWithClient(
	metric *av1alpha1.Metric,
	counter resources.EndpointsCounter,
	podAccessor resources.PodAccessor,
	sClient scrapeClient,
	logger *zap.SugaredLogger) (*serviceScraper, error) {
	if metric == nil {
		return nil, errors.New("metric must not be nil")
	}
	if counter == nil {
		return nil, errors.New("counter must not be nil")
	}
	if sClient == nil {
		return nil, errors.New("scrape client must not be nil")
	}
	revName := metric.Labels[serving.RevisionLabelKey]
	if revName == "" {
		return nil, errors.New("no Revision label found for Metric " + metric.Name)
	}
	svcName := metric.Labels[serving.ServiceLabelKey]
	cfgName := metric.Labels[serving.ConfigurationLabelKey]

	ctx, err := metrics.RevisionContext(metric.ObjectMeta.Namespace, svcName, cfgName, revName)
	if err != nil {
		return nil, err
	}

	return &serviceScraper{
		sClient:         sClient,
		counter:         counter,
		url:             urlFromTarget(metric.Spec.ScrapeTarget, metric.ObjectMeta.Namespace),
		podAccessor:     podAccessor,
		podsAddressable: true,
		statsCtx:        ctx,
		logger:          logger,
	}, nil
}

var portAndPath = strconv.Itoa(networking.AutoscalingQueueMetricsPort) + "/metrics"

func urlFromTarget(t, ns string) string {
	return fmt.Sprintf("http://%s.%s:", t, ns) + portAndPath
}

// Scrape calls the destination service then sends it
// to the given stats channel.
func (s *serviceScraper) Scrape(window time.Duration) (Stat, error) {
	readyPodsCount, err := s.counter.ReadyCount()
	if err != nil {
		return emptyStat, ErrFailedGetEndpoints
	}

	if readyPodsCount == 0 {
		return emptyStat, nil
	}

	startTime := time.Now()
	defer func() {
		scrapeTime := time.Since(startTime)
		pkgmetrics.RecordBatch(s.statsCtx, scrapeTimeM.M(float64(scrapeTime.Milliseconds())))
	}()

	if s.podsAddressable {
		stat, err := s.scrapePods(readyPodsCount)
		// Some pods were scraped, but not enough.
		if err != errNoPodsScraped {
			return stat, err
		}
		// Else fall back to service scrape.
	}
	stat, err := s.scrapeService(window, readyPodsCount)
	if err == nil {
		s.logger.Info("Direct pod scraping off, service scraping, on")
		// If err == nil, this means that we failed to scrape all pods, but service worked
		// thus it is probably a mesh case.
		s.podsAddressable = false
	}
	return stat, err
}

func (s *serviceScraper) scrapePods(readyPods int) (Stat, error) {
	pods, err := s.podAccessor.PodIPsByAge()
	if err != nil {
		s.logger.Info("Error querying pods by age: ", err)
		return emptyStat, err
	}
	// Race condition when scaling to 0, where the check above
	// for endpoint count worked, but here we had no ready pods.
	if len(pods) == 0 {
		s.logger.Infof("For %s ready pods found 0 pods, are we scaling to 0?", readyPods)
		return emptyStat, nil
	}

	frpc := float64(readyPods)
	sampleSizeF := populationMeanSampleSize(frpc)
	sampleSize := int(sampleSizeF)
	results := make(chan Stat, sampleSize)

	grp := errgroup.Group{}
	idx := int32(-1)
	// Start |sampleSize| threads to scan in parallel.
	for i := 0; i < sampleSize; i++ {
		grp.Go(func() error {
			// If a given pod failed to scrape, we want to continue
			// scanning pods down the line.
			for {
				// Acquire next pod.
				myIdx := int(atomic.AddInt32(&idx, 1))
				// All out?
				if myIdx >= len(pods) {
					return errPodsExhausted
				}

				// Scrape!
				target := "http://" + pods[myIdx] + ":" + portAndPath
				stat, err := s.sClient.Scrape(target)
				if err == nil {
					results <- stat
					return nil
				}
				s.logger.Infof("Pod %s failed scraping: %v", pods[myIdx], err)
			}
		})
	}

	err = grp.Wait()
	close(results)

	// We only get here if one of the scrapers failed to scrape
	// at least one pod.
	if err != nil {
		// Got some successful pods.
		// TODO(vagababov): perhaps separate |pods| == 1 case here as well?
		if len(results) > 0 {
			s.logger.Warn("Too many pods failed scraping for meaningful interpolation")
			return emptyStat, errPodsExhausted
		}
		s.logger.Warn("0 pods were successfully scraped out of ", strconv.Itoa(len(pods)))
		// Didn't scrape a single pod, switch to service scraping.
		return emptyStat, errNoPodsScraped
	}

	return computeAverages(results, sampleSizeF, frpc), nil
}

func computeAverages(results <-chan Stat, sample, total float64) Stat {
	ret := Stat{
		Time:    time.Now(),
		PodName: scraperPodName,
	}

	// Sum the stats from individual pods.
	for stat := range results {
		ret.add(stat)
	}

	ret.average(sample, total)
	return ret
}

// scrapeService scrapes the metrics using service endpoint
// as its target, rather than individual pods.
func (s *serviceScraper) scrapeService(window time.Duration, readyPods int) (Stat, error) {
	frpc := float64(readyPods)

	sampleSizeF := populationMeanSampleSize(frpc)
	sampleSize := int(sampleSizeF)
	oldStatCh := make(chan Stat, sampleSize)
	youngStatCh := make(chan Stat, sampleSize)
	scrapedPods := &sync.Map{}

	grp := errgroup.Group{}
	youngPodCutOffSecs := window.Seconds()
	for i := 0; i < sampleSize; i++ {
		grp.Go(func() error {
			for tries := 1; ; tries++ {
				stat, err := s.tryScrape(scrapedPods)
				if err == nil {
					if stat.ProcessUptime >= youngPodCutOffSecs {
						// We run |sampleSize| goroutines and each of them terminates
						// as soon as it sees stat from an `oldPod`.
						// The channel is allocated to |sampleSize|, thus this will never
						// deadlock.
						oldStatCh <- stat
						return nil
					} else {
						select {
						// This in theory might loop over all the possible pods, thus might
						// fill up the channel.
						case youngStatCh <- stat:
						default:
							// If so, just return.
							return nil
						}
					}
				} else if tries >= scraperMaxRetries {
					// Return the error if we exhausted our retries and
					// we had an error returned (we can end up here if
					// all the pods were young, which is not an error condition).
					return err
				}
			}
		})
	}
	// Now at this point we have two possibilities.
	// 1. We scraped |sampleSize| distinct pods, with the invariant of
	// 		   sampleSize <= len(oldStatCh) + len(youngStatCh) <= sampleSize*2.
	//    Note, that `err` might still be non-nil, especially when the overall
	//    pod population is small.
	//    Consider the following case: sampleSize=3, in theory the first go routine
	//    might scrape 2 pods, the second 1 and the third won't be be able to scrape
	//		any unseen pod, so it will return `ErrDidNotReceiveStat`.
	// 2. We did not: in this case `err` below will be non-nil.

	// Return the inner error, if any.
	if err := grp.Wait(); err != nil {
		// Ignore the error if we have received enough statistics.
		if err != ErrDidNotReceiveStat || len(oldStatCh)+len(youngStatCh) < sampleSize {
			return emptyStat, fmt.Errorf("unsuccessful scrape, sampleSize=%d: %w", sampleSize, err)
		}
	}
	close(oldStatCh)
	close(youngStatCh)

	ret := Stat{
		Time:    time.Now(),
		PodName: scraperPodName,
	}

	// Sum the stats from individual pods.
	oldCnt := len(oldStatCh)
	for stat := range oldStatCh {
		ret.add(stat)
	}
	for i := oldCnt; i < sampleSize; i++ {
		// This will always succeed, see reasoning above.
		ret.add(<-youngStatCh)
	}

	ret.average(sampleSizeF, frpc)
	return ret, nil
}

// tryScrape runs a single scrape and returns stat if this is a pod that has not been
// seen before. An error otherwise or if scraping failed.
func (s *serviceScraper) tryScrape(scrapedPods *sync.Map) (Stat, error) {
	stat, err := s.sClient.Scrape(s.url)
	if err != nil {
		return emptyStat, err
	}

	if _, exists := scrapedPods.LoadOrStore(stat.PodName, struct{}{}); exists {
		return emptyStat, ErrDidNotReceiveStat
	}

	return stat, nil
}
<commit_msg>Refactor stats scraper to make lint happier (#8020)<commit_after>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package metrics

import (
	"context"
	"errors"
	"fmt"
	"net/http"
	"strconv"
	"sync"
	"sync/atomic"
	"time"

	"go.opencensus.io/stats"
	"go.opencensus.io/stats/view"
	"go.uber.org/zap"
	"golang.org/x/sync/errgroup"

	pkgmetrics "knative.dev/pkg/metrics"
	av1alpha1 "knative.dev/serving/pkg/apis/autoscaling/v1alpha1"
	"knative.dev/serving/pkg/apis/networking"
	"knative.dev/serving/pkg/apis/serving"
	"knative.dev/serving/pkg/metrics"
	"knative.dev/serving/pkg/resources"
)

const (
	httpClientTimeout = 3 * time.Second

	// scraperPodName is the name used in all stats sent from the scraper to
	// the autoscaler. The actual customer pods are hidden behind the scraper. The
	// autoscaler does need to know how many customer pods are reporting metrics.
	// Instead, the autoscaler knows the stats it receives are either from the
	// scraper or the activator.
	scraperPodName = "service-scraper"

	// scraperMaxRetries are retries to be done to the actual Scrape routine. We want
	// to retry if a Scrape returns an error or if the Scrape goes to a pod we already
	// scraped.
	scraperMaxRetries = 10
)

var (
	// ErrFailedGetEndpoints specifies the error returned by scraper when it fails to
	// get endpoints.
	ErrFailedGetEndpoints = errors.New("failed to get endpoints")

	// ErrDidNotReceiveStat specifies the error returned by scraper when it does not receive
	// stat from an unscraped pod
	ErrDidNotReceiveStat = errors.New("did not receive stat from an unscraped pod")

	// Sentinel error to retrun from pod scraping routine, when we could not
	// scrape even a single pod.
	errNoPodsScraped = errors.New("no pods scraped")
	errPodsExhausted = errors.New("pods exhausted")

	scrapeTimeM = stats.Float64(
		"scrape_time",
		"Time to scrape metrics in milliseconds",
		stats.UnitMilliseconds)
)

func init() {
	if err := view.Register(
		&view.View{
			Description: "The time to scrape metrics in milliseconds",
			Measure:     scrapeTimeM,
			Aggregation: view.Distribution(pkgmetrics.Buckets125(1, 100000)...),
			TagKeys:     metrics.CommonRevisionKeys,
		},
	); err != nil {
		panic(err)
	}
}

// StatsScraper defines the interface for collecting Revision metrics
type StatsScraper interface {
	// Scrape scrapes the Revision queue metric endpoint. The duration is used
	// to cutoff young pods, whose stats might skew lower.
	Scrape(time.Duration) (Stat, error)
}

// scrapeClient defines the interface for collecting Revision metrics for a given
// URL. Internal used only.
type scrapeClient interface {
	// Scrape scrapes the given URL.
	Scrape(url string) (Stat, error)
}

// cacheDisabledClient is a http client with cache disabled. It is shared by
// every goruntime for a revision scraper.
var cacheDisabledClient = &http.Client{
	Transport: &http.Transport{
		// Do not use the cached connection
		DisableKeepAlives: true,
	},
	Timeout: httpClientTimeout,
}

// serviceScraper scrapes Revision metrics via a K8S service by sampling. Which
// pod to be picked up to serve the request is decided by K8S. Please see
// https://kubernetes.io/docs/concepts/services-networking/network-policies/
// for details.
type serviceScraper struct {
	sClient  scrapeClient
	counter  resources.EndpointsCounter
	url      string
	statsCtx context.Context
	logger   *zap.SugaredLogger

	podAccessor     resources.PodAccessor
	podsAddressable bool
}

// NewStatsScraper creates a new StatsScraper for the Revision which
// the given Metric is responsible for.
func NewStatsScraper(metric *av1alpha1.Metric, counter resources.EndpointsCounter,
	podAccessor resources.PodAccessor, logger *zap.SugaredLogger) (StatsScraper, error) {
	sClient, err := newHTTPScrapeClient(cacheDisabledClient)
	if err != nil {
		return nil, err
	}
	return newServiceScraperWithClient(metric, counter, podAccessor, sClient, logger)
}

func newServiceScraperWithClient(
	metric *av1alpha1.Metric,
	counter resources.EndpointsCounter,
	podAccessor resources.PodAccessor,
	sClient scrapeClient,
	logger *zap.SugaredLogger) (*serviceScraper, error) {
	if metric == nil {
		return nil, errors.New("metric must not be nil")
	}
	if counter == nil {
		return nil, errors.New("counter must not be nil")
	}
	if sClient == nil {
		return nil, errors.New("scrape client must not be nil")
	}
	revName := metric.Labels[serving.RevisionLabelKey]
	if revName == "" {
		return nil, errors.New("no Revision label found for Metric " + metric.Name)
	}
	svcName := metric.Labels[serving.ServiceLabelKey]
	cfgName := metric.Labels[serving.ConfigurationLabelKey]

	ctx, err := metrics.RevisionContext(metric.ObjectMeta.Namespace, svcName, cfgName, revName)
	if err != nil {
		return nil, err
	}

	return &serviceScraper{
		sClient:         sClient,
		counter:         counter,
		url:             urlFromTarget(metric.Spec.ScrapeTarget, metric.ObjectMeta.Namespace),
		podAccessor:     podAccessor,
		podsAddressable: true,
		statsCtx:        ctx,
		logger:          logger,
	}, nil
}

var portAndPath = strconv.Itoa(networking.AutoscalingQueueMetricsPort) + "/metrics"

func urlFromTarget(t, ns string) string {
	return fmt.Sprintf("http://%s.%s:", t, ns) + portAndPath
}

// Scrape calls the destination service then sends it
// to the given stats channel.
func (s *serviceScraper) Scrape(window time.Duration) (Stat, error) {
	readyPodsCount, err := s.counter.ReadyCount()
	if err != nil {
		return emptyStat, ErrFailedGetEndpoints
	}

	if readyPodsCount == 0 {
		return emptyStat, nil
	}

	startTime := time.Now()
	defer func() {
		scrapeTime := time.Since(startTime)
		pkgmetrics.RecordBatch(s.statsCtx, scrapeTimeM.M(float64(scrapeTime.Milliseconds())))
	}()

	if s.podsAddressable {
		stat, err := s.scrapePods(readyPodsCount)
		// Some pods were scraped, but not enough.
		if err != errNoPodsScraped {
			return stat, err
		}
		// Else fall back to service scrape.
	}
	stat, err := s.scrapeService(window, readyPodsCount)
	if err == nil {
		s.logger.Info("Direct pod scraping off, service scraping, on")
		// If err == nil, this means that we failed to scrape all pods, but service worked
		// thus it is probably a mesh case.
		s.podsAddressable = false
	}
	return stat, err
}

func (s *serviceScraper) scrapePods(readyPods int) (Stat, error) {
	pods, err := s.podAccessor.PodIPsByAge()
	if err != nil {
		s.logger.Info("Error querying pods by age: ", err)
		return emptyStat, err
	}
	// Race condition when scaling to 0, where the check above
	// for endpoint count worked, but here we had no ready pods.
	if len(pods) == 0 {
		s.logger.Infof("For %s ready pods found 0 pods, are we scaling to 0?", readyPods)
		return emptyStat, nil
	}

	frpc := float64(readyPods)
	sampleSizeF := populationMeanSampleSize(frpc)
	sampleSize := int(sampleSizeF)
	results := make(chan Stat, sampleSize)

	grp := errgroup.Group{}
	idx := int32(-1)
	// Start |sampleSize| threads to scan in parallel.
	for i := 0; i < sampleSize; i++ {
		grp.Go(func() error {
			// If a given pod failed to scrape, we want to continue
			// scanning pods down the line.
			for {
				// Acquire next pod.
				myIdx := int(atomic.AddInt32(&idx, 1))
				// All out?
				if myIdx >= len(pods) {
					return errPodsExhausted
				}

				// Scrape!
				target := "http://" + pods[myIdx] + ":" + portAndPath
				stat, err := s.sClient.Scrape(target)
				if err == nil {
					results <- stat
					return nil
				}
				s.logger.Infof("Pod %s failed scraping: %v", pods[myIdx], err)
			}
		})
	}

	err = grp.Wait()
	close(results)

	// We only get here if one of the scrapers failed to scrape
	// at least one pod.
	if err != nil {
		// Got some successful pods.
		// TODO(vagababov): perhaps separate |pods| == 1 case here as well?
		if len(results) > 0 {
			s.logger.Warn("Too many pods failed scraping for meaningful interpolation")
			return emptyStat, errPodsExhausted
		}
		s.logger.Warn("0 pods were successfully scraped out of ", strconv.Itoa(len(pods)))
		// Didn't scrape a single pod, switch to service scraping.
		return emptyStat, errNoPodsScraped
	}

	return computeAverages(results, sampleSizeF, frpc), nil
}

func computeAverages(results <-chan Stat, sample, total float64) Stat {
	ret := Stat{
		Time:    time.Now(),
		PodName: scraperPodName,
	}

	// Sum the stats from individual pods.
	for stat := range results {
		ret.add(stat)
	}

	ret.average(sample, total)
	return ret
}

// scrapeService scrapes the metrics using service endpoint
// as its target, rather than individual pods.
func (s *serviceScraper) scrapeService(window time.Duration, readyPods int) (Stat, error) {
	frpc := float64(readyPods)

	sampleSizeF := populationMeanSampleSize(frpc)
	sampleSize := int(sampleSizeF)
	oldStatCh := make(chan Stat, sampleSize)
	youngStatCh := make(chan Stat, sampleSize)
	scrapedPods := &sync.Map{}

	grp := errgroup.Group{}
	youngPodCutOffSecs := window.Seconds()
	for i := 0; i < sampleSize; i++ {
		grp.Go(func() error {
			for tries := 1; ; tries++ {
				stat, err := s.tryScrape(scrapedPods)
				if err != nil {
					// Return the error if we exhausted our retries and
					// we had an error returned (we can end up here if
					// all the pods were young, which is not an error condition).
					if tries >= scraperMaxRetries {
						return err
					}
					continue
				}

				if stat.ProcessUptime >= youngPodCutOffSecs {
					// We run |sampleSize| goroutines and each of them terminates
					// as soon as it sees a stat from an `oldPod`.
					// The channel is allocated to |sampleSize|, thus this will never
					// deadlock.
					oldStatCh <- stat
					return nil
				}

				select {
				// This in theory might loop over all the possible pods, thus might
				// fill up the channel.
				case youngStatCh <- stat:
				default:
					// If so, just return.
					return nil
				}
			}
		})
	}

	// Now at this point we have two possibilities.
	// 1. We scraped |sampleSize| distinct pods, with the invariant of
	// 		   sampleSize <= len(oldStatCh) + len(youngStatCh) <= sampleSize*2.
	//    Note, that `err` might still be non-nil, especially when the overall
	//    pod population is small.
	//    Consider the following case: sampleSize=3, in theory the first go routine
	//    might scrape 2 pods, the second 1 and the third won't be be able to scrape
	//		any unseen pod, so it will return `ErrDidNotReceiveStat`.
	// 2. We did not: in this case `err` below will be non-nil.

	// Return the inner error, if any.
	if err := grp.Wait(); err != nil {
		// Ignore the error if we have received enough statistics.
		if err != ErrDidNotReceiveStat || len(oldStatCh)+len(youngStatCh) < sampleSize {
			return emptyStat, fmt.Errorf("unsuccessful scrape, sampleSize=%d: %w", sampleSize, err)
		}
	}
	close(oldStatCh)
	close(youngStatCh)

	ret := Stat{
		Time:    time.Now(),
		PodName: scraperPodName,
	}

	// Sum the stats from individual pods.
	oldCnt := len(oldStatCh)
	for stat := range oldStatCh {
		ret.add(stat)
	}
	for i := oldCnt; i < sampleSize; i++ {
		// This will always succeed, see reasoning above.
		ret.add(<-youngStatCh)
	}

	ret.average(sampleSizeF, frpc)
	return ret, nil
}

// tryScrape runs a single scrape and returns stat if this is a pod that has not been
// seen before. An error otherwise or if scraping failed.
func (s *serviceScraper) tryScrape(scrapedPods *sync.Map) (Stat, error) {
	stat, err := s.sClient.Scrape(s.url)
	if err != nil {
		return emptyStat, err
	}

	if _, exists := scrapedPods.LoadOrStore(stat.PodName, struct{}{}); exists {
		return emptyStat, ErrDidNotReceiveStat
	}

	return stat, nil
}
<|endoftext|>


<commit_before>package keeper

import (
	"context"

	sdk "github.com/cosmos/cosmos-sdk/types"
	sdkerrors "github.com/cosmos/cosmos-sdk/types/errors"
	"github.com/zigbee-alliance/distributed-compliance-ledger/x/dclupgrade/types"
)

func (k msgServer) RejectUpgrade(goCtx context.Context, msg *types.MsgRejectUpgrade) (*types.MsgRejectUpgradeResponse, error) {
	ctx := sdk.UnwrapSDKContext(goCtx)

	creatorAddr, err := sdk.AccAddressFromBech32(msg.Creator)
	if err != nil {
		return nil, sdkerrors.Wrapf(sdkerrors.ErrInvalidAddress, "Invalid Address: (%s)", err)
	}

	// check if message creator has enough rights to reject upgrade
	if !k.dclauthKeeper.HasRole(ctx, creatorAddr, types.UpgradeApprovalRole) {
		return nil, sdkerrors.Wrapf(sdkerrors.ErrUnauthorized,
			"MsgApproveUpgrade transaction should be signed by an account with the %s role",
			types.UpgradeApprovalRole,
		)
	}

	// check if proposed upgrade exists
	proposedUpgrade, isFound := k.GetProposedUpgrade(ctx, msg.Name)
	if !isFound {
		return nil, types.NewErrProposedUpgradeDoesNotExist(msg.Name)
	}

	// check if proposed upgrade already has reject from message creator
	if proposedUpgrade.HasRejectFrom(creatorAddr) {
		return nil, sdkerrors.Wrapf(sdkerrors.ErrUnauthorized,
			"Proposed upgrade with name=%v already has reject from=%v",
			msg.Name, msg.Creator,
		)
	}

	// check if proposed upgrade already has approval from message creator
	if proposedUpgrade.HasApprovalFrom(creatorAddr) {
		return nil, sdkerrors.Wrapf(sdkerrors.ErrUnauthorized,
			"Proposed upgrade with name=%v already has approval from=%v",
			msg.Name, msg.Creator,
		)
	}

	// append approval
	grant := types.Grant{
		Address: creatorAddr.String(),
		Time:    msg.Time,
		Info:    msg.Info,
	}

	proposedUpgrade.Rejects = append(proposedUpgrade.Rejects, &grant)

	// check if proposed upgrade has enough rejects
	if len(proposedUpgrade.Rejects) == k.UpgradeRejectsCount(ctx) {
		// schedule upgrade
		err = k.upgradeKeeper.ScheduleUpgrade(ctx, proposedUpgrade.Plan)
		if err != nil {
			return nil, err
		}

		// remove proposed upgrade
		k.RemoveProposedUpgrade(ctx, proposedUpgrade.Plan.Name)

		rejectedUpgrade := types.RejectedUpgrade(proposedUpgrade)
		k.SetRejectedUpgrade(ctx, rejectedUpgrade)
	} else {
		// update proposed upgrade
		k.SetProposedUpgrade(ctx, proposedUpgrade)
	}

	return &types.MsgRejectUpgradeResponse{}, nil
}
<commit_msg>When we try to re-propose some upgrade we should remove from the entity rejectedUpgrade if this upgrade exist in rejectedUpgrade<commit_after>package keeper

import (
	"context"

	sdk "github.com/cosmos/cosmos-sdk/types"
	sdkerrors "github.com/cosmos/cosmos-sdk/types/errors"
	"github.com/zigbee-alliance/distributed-compliance-ledger/x/dclupgrade/types"
)

func (k msgServer) RejectUpgrade(goCtx context.Context, msg *types.MsgRejectUpgrade) (*types.MsgRejectUpgradeResponse, error) {
	ctx := sdk.UnwrapSDKContext(goCtx)

	creatorAddr, err := sdk.AccAddressFromBech32(msg.Creator)
	if err != nil {
		return nil, sdkerrors.Wrapf(sdkerrors.ErrInvalidAddress, "Invalid Address: (%s)", err)
	}

	// check if message creator has enough rights to reject upgrade
	if !k.dclauthKeeper.HasRole(ctx, creatorAddr, types.UpgradeApprovalRole) {
		return nil, sdkerrors.Wrapf(sdkerrors.ErrUnauthorized,
			"MsgApproveUpgrade transaction should be signed by an account with the %s role",
			types.UpgradeApprovalRole,
		)
	}

	// check if proposed upgrade exists
	proposedUpgrade, isFound := k.GetProposedUpgrade(ctx, msg.Name)
	if !isFound {
		return nil, types.NewErrProposedUpgradeDoesNotExist(msg.Name)
	}

	// check if proposed upgrade already has reject from message creator
	if proposedUpgrade.HasRejectFrom(creatorAddr) {
		return nil, sdkerrors.Wrapf(sdkerrors.ErrUnauthorized,
			"Proposed upgrade with name=%v already has reject from=%v",
			msg.Name, msg.Creator,
		)
	}

	// check if proposed upgrade already has approval from message creator
	if proposedUpgrade.HasApprovalFrom(creatorAddr) {
		return nil, sdkerrors.Wrapf(sdkerrors.ErrUnauthorized,
			"Proposed upgrade with name=%v already has approval from=%v",
			msg.Name, msg.Creator,
		)
	}

	// append approval
	grant := types.Grant{
		Address: creatorAddr.String(),
		Time:    msg.Time,
		Info:    msg.Info,
	}

	proposedUpgrade.Rejects = append(proposedUpgrade.Rejects, &grant)

	// check if proposed upgrade has enough rejects
	if len(proposedUpgrade.Rejects) == k.UpgradeRejectsCount(ctx) {
		// schedule upgrade
		err = k.upgradeKeeper.ScheduleUpgrade(ctx, proposedUpgrade.Plan)
		if err != nil {
			return nil, err
		}

		// remove proposed upgrade
		k.RemoveProposedUpgrade(ctx, proposedUpgrade.Plan.Name)

		rejectedUpgrade := types.RejectedUpgrade(proposedUpgrade)
		k.SetRejectedUpgrade(ctx, rejectedUpgrade)
	} else {
		// update proposed upgrade
		k.SetProposedUpgrade(ctx, proposedUpgrade)
	}

	_, isFound = k.GetRejectedUpgrade(ctx, msg.Name)
	if isFound {
		k.RemoveRejectedUpgrade(ctx, msg.Name)
	}

	return &types.MsgRejectUpgradeResponse{}, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package metrics

import (
	"strings"
	"sync"
	"time"

	"k8s.io/apimachinery/pkg/util/clock"
	compbasemetrics "k8s.io/component-base/metrics"
	"k8s.io/component-base/metrics/legacyregistry"
	basemetricstestutil "k8s.io/component-base/metrics/testutil"
)

const (
	namespace = "apiserver"
	subsystem = "flowcontrol"
)

const (
	requestKind   = "request_kind"
	priorityLevel = "priority_level"
	flowSchema    = "flow_schema"
	phase         = "phase"
	mark          = "mark"
)

var (
	queueLengthBuckets            = []float64{0, 10, 25, 50, 100, 250, 500, 1000}
	requestDurationSecondsBuckets = []float64{0, 0.005, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5, 10, 30}
)

var registerMetrics sync.Once

// Register all metrics.
func Register() {
	registerMetrics.Do(func() {
		for _, metric := range metrics {
			legacyregistry.MustRegister(metric)
		}
	})
}

type resettable interface {
	Reset()
}

// Reset all metrics to zero
func Reset() {
	for _, metric := range metrics {
		rm := metric.(resettable)
		rm.Reset()
	}
}

// GatherAndCompare the given metrics with the given Prometheus syntax expected value
func GatherAndCompare(expected string, metricNames ...string) error {
	return basemetricstestutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expected), metricNames...)
}

// Registerables is a slice of Registerable
type Registerables []compbasemetrics.Registerable

// Append adds more
func (rs Registerables) Append(more ...compbasemetrics.Registerable) Registerables {
	return append(rs, more...)
}

var (
	apiserverRejectedRequestsTotal = compbasemetrics.NewCounterVec(
		&compbasemetrics.CounterOpts{
			Namespace: namespace,
			Subsystem: subsystem,
			Name:      "rejected_requests_total",
			Help:      "Number of requests rejected by API Priority and Fairness system",
		},
		[]string{priorityLevel, flowSchema, "reason"},
	)
	apiserverDispatchedRequestsTotal = compbasemetrics.NewCounterVec(
		&compbasemetrics.CounterOpts{
			Namespace: namespace,
			Subsystem: subsystem,
			Name:      "dispatched_requests_total",
			Help:      "Number of requests released by API Priority and Fairness system for service",
		},
		[]string{priorityLevel, flowSchema},
	)

	// PriorityLevelConcurrencyObserverPairGenerator creates pairs that observe concurrency for priority levels
	PriorityLevelConcurrencyObserverPairGenerator = NewSampleAndWaterMarkHistogramsPairGenerator(clock.RealClock{}, time.Millisecond,
		&compbasemetrics.HistogramOpts{
			Namespace:      namespace,
			Subsystem:      subsystem,
			Name:           "priority_level_request_count_samples",
			Help:           "Periodic observations of the number of requests",
			Buckets:        []float64{0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1},
			StabilityLevel: compbasemetrics.ALPHA,
		},
		&compbasemetrics.HistogramOpts{
			Namespace:      namespace,
			Subsystem:      subsystem,
			Name:           "priority_level_request_count_watermarks",
			Help:           "Watermarks of the number of requests",
			Buckets:        []float64{0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1},
			StabilityLevel: compbasemetrics.ALPHA,
		},
		[]string{priorityLevel})

	// ReadWriteConcurrencyObserverPairGenerator creates pairs that observe concurrency broken down by mutating vs readonly
	ReadWriteConcurrencyObserverPairGenerator = NewSampleAndWaterMarkHistogramsPairGenerator(clock.RealClock{}, time.Millisecond,
		&compbasemetrics.HistogramOpts{
			Namespace:      namespace,
			Subsystem:      subsystem,
			Name:           "read_vs_write_request_count_samples",
			Help:           "Periodic observations of the number of requests",
			Buckets:        []float64{0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1},
			StabilityLevel: compbasemetrics.ALPHA,
		},
		&compbasemetrics.HistogramOpts{
			Namespace:      namespace,
			Subsystem:      subsystem,
			Name:           "read_vs_write_request_count_watermarks",
			Help:           "Watermarks of the number of requests",
			Buckets:        []float64{0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1},
			StabilityLevel: compbasemetrics.ALPHA,
		},
		[]string{requestKind})

	apiserverCurrentInqueueRequests = compbasemetrics.NewGaugeVec(
		&compbasemetrics.GaugeOpts{
			Namespace: namespace,
			Subsystem: subsystem,
			Name:      "current_inqueue_requests",
			Help:      "Number of requests currently pending in queues of the API Priority and Fairness system",
		},
		[]string{priorityLevel, flowSchema},
	)
	apiserverRequestQueueLength = compbasemetrics.NewHistogramVec(
		&compbasemetrics.HistogramOpts{
			Namespace: namespace,
			Subsystem: subsystem,
			Name:      "request_queue_length_after_enqueue",
			Help:      "Length of queue in the API Priority and Fairness system, as seen by each request after it is enqueued",
			Buckets:   queueLengthBuckets,
		},
		[]string{priorityLevel, flowSchema},
	)
	apiserverRequestConcurrencyLimit = compbasemetrics.NewGaugeVec(
		&compbasemetrics.GaugeOpts{
			Namespace: namespace,
			Subsystem: subsystem,
			Name:      "request_concurrency_limit",
			Help:      "Shared concurrency limit in the API Priority and Fairness system",
		},
		[]string{priorityLevel},
	)
	apiserverCurrentExecutingRequests = compbasemetrics.NewGaugeVec(
		&compbasemetrics.GaugeOpts{
			Namespace: namespace,
			Subsystem: subsystem,
			Name:      "current_executing_requests",
			Help:      "Number of requests currently executing in the API Priority and Fairness system",
		},
		[]string{priorityLevel, flowSchema},
	)
	apiserverRequestWaitingSeconds = compbasemetrics.NewHistogramVec(
		&compbasemetrics.HistogramOpts{
			Namespace: namespace,
			Subsystem: subsystem,
			Name:      "request_wait_duration_seconds",
			Help:      "Length of time a request spent waiting in its queue",
			Buckets:   requestDurationSecondsBuckets,
		},
		[]string{priorityLevel, flowSchema, "execute"},
	)
	apiserverRequestExecutionSeconds = compbasemetrics.NewHistogramVec(
		&compbasemetrics.HistogramOpts{
			Namespace: namespace,
			Subsystem: subsystem,
			Name:      "request_execution_seconds",
			Help:      "Duration of request execution in the API Priority and Fairness system",
			Buckets:   requestDurationSecondsBuckets,
		},
		[]string{priorityLevel, flowSchema},
	)
	metrics = Registerables{
		apiserverRejectedRequestsTotal,
		apiserverDispatchedRequestsTotal,
		apiserverCurrentInqueueRequests,
		apiserverRequestQueueLength,
		apiserverRequestConcurrencyLimit,
		apiserverCurrentExecutingRequests,
		apiserverRequestWaitingSeconds,
		apiserverRequestExecutionSeconds,
	}.
		Append(PriorityLevelConcurrencyObserverPairGenerator.metrics()...).
		Append(ReadWriteConcurrencyObserverPairGenerator.metrics()...)
)

// AddRequestsInQueues adds the given delta to the gauge of the # of requests in the queues of the specified flowSchema and priorityLevel
func AddRequestsInQueues(priorityLevel, flowSchema string, delta int) {
	apiserverCurrentInqueueRequests.WithLabelValues(priorityLevel, flowSchema).Add(float64(delta))
}

// AddRequestsExecuting adds the given delta to the gauge of executing requests of the given flowSchema and priorityLevel
func AddRequestsExecuting(priorityLevel, flowSchema string, delta int) {
	apiserverCurrentExecutingRequests.WithLabelValues(priorityLevel, flowSchema).Add(float64(delta))
}

// UpdateSharedConcurrencyLimit updates the value for the concurrency limit in flow control
func UpdateSharedConcurrencyLimit(priorityLevel string, limit int) {
	apiserverRequestConcurrencyLimit.WithLabelValues(priorityLevel).Set(float64(limit))
}

// AddReject increments the # of rejected requests for flow control
func AddReject(priorityLevel, flowSchema, reason string) {
	apiserverRejectedRequestsTotal.WithLabelValues(priorityLevel, flowSchema, reason).Add(1)
}

// AddDispatch increments the # of dispatched requests for flow control
func AddDispatch(priorityLevel, flowSchema string) {
	apiserverDispatchedRequestsTotal.WithLabelValues(priorityLevel, flowSchema).Add(1)
}

// ObserveQueueLength observes the queue length for flow control
func ObserveQueueLength(priorityLevel, flowSchema string, length int) {
	apiserverRequestQueueLength.WithLabelValues(priorityLevel, flowSchema).Observe(float64(length))
}

// ObserveWaitingDuration observes the queue length for flow control
func ObserveWaitingDuration(priorityLevel, flowSchema, execute string, waitTime time.Duration) {
	apiserverRequestWaitingSeconds.WithLabelValues(priorityLevel, flowSchema, execute).Observe(waitTime.Seconds())
}

// ObserveExecutionDuration observes the execution duration for flow control
func ObserveExecutionDuration(priorityLevel, flowSchema string, executionTime time.Duration) {
	apiserverRequestExecutionSeconds.WithLabelValues(priorityLevel, flowSchema).Observe(executionTime.Seconds())
}
<commit_msg>APF metrics: set StabilityLevel to ALPHA<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package metrics

import (
	"strings"
	"sync"
	"time"

	"k8s.io/apimachinery/pkg/util/clock"
	compbasemetrics "k8s.io/component-base/metrics"
	"k8s.io/component-base/metrics/legacyregistry"
	basemetricstestutil "k8s.io/component-base/metrics/testutil"
)

const (
	namespace = "apiserver"
	subsystem = "flowcontrol"
)

const (
	requestKind   = "request_kind"
	priorityLevel = "priority_level"
	flowSchema    = "flow_schema"
	phase         = "phase"
	mark          = "mark"
)

var (
	queueLengthBuckets            = []float64{0, 10, 25, 50, 100, 250, 500, 1000}
	requestDurationSecondsBuckets = []float64{0, 0.005, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5, 10, 30}
)

var registerMetrics sync.Once

// Register all metrics.
func Register() {
	registerMetrics.Do(func() {
		for _, metric := range metrics {
			legacyregistry.MustRegister(metric)
		}
	})
}

type resettable interface {
	Reset()
}

// Reset all metrics to zero
func Reset() {
	for _, metric := range metrics {
		rm := metric.(resettable)
		rm.Reset()
	}
}

// GatherAndCompare the given metrics with the given Prometheus syntax expected value
func GatherAndCompare(expected string, metricNames ...string) error {
	return basemetricstestutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expected), metricNames...)
}

// Registerables is a slice of Registerable
type Registerables []compbasemetrics.Registerable

// Append adds more
func (rs Registerables) Append(more ...compbasemetrics.Registerable) Registerables {
	return append(rs, more...)
}

var (
	apiserverRejectedRequestsTotal = compbasemetrics.NewCounterVec(
		&compbasemetrics.CounterOpts{
			Namespace:      namespace,
			Subsystem:      subsystem,
			Name:           "rejected_requests_total",
			Help:           "Number of requests rejected by API Priority and Fairness system",
			StabilityLevel: compbasemetrics.ALPHA,
		},
		[]string{priorityLevel, flowSchema, "reason"},
	)
	apiserverDispatchedRequestsTotal = compbasemetrics.NewCounterVec(
		&compbasemetrics.CounterOpts{
			Namespace:      namespace,
			Subsystem:      subsystem,
			Name:           "dispatched_requests_total",
			Help:           "Number of requests released by API Priority and Fairness system for service",
			StabilityLevel: compbasemetrics.ALPHA,
		},
		[]string{priorityLevel, flowSchema},
	)

	// PriorityLevelConcurrencyObserverPairGenerator creates pairs that observe concurrency for priority levels
	PriorityLevelConcurrencyObserverPairGenerator = NewSampleAndWaterMarkHistogramsPairGenerator(clock.RealClock{}, time.Millisecond,
		&compbasemetrics.HistogramOpts{
			Namespace:      namespace,
			Subsystem:      subsystem,
			Name:           "priority_level_request_count_samples",
			Help:           "Periodic observations of the number of requests",
			Buckets:        []float64{0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1},
			StabilityLevel: compbasemetrics.ALPHA,
		},
		&compbasemetrics.HistogramOpts{
			Namespace:      namespace,
			Subsystem:      subsystem,
			Name:           "priority_level_request_count_watermarks",
			Help:           "Watermarks of the number of requests",
			Buckets:        []float64{0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1},
			StabilityLevel: compbasemetrics.ALPHA,
		},
		[]string{priorityLevel})

	// ReadWriteConcurrencyObserverPairGenerator creates pairs that observe concurrency broken down by mutating vs readonly
	ReadWriteConcurrencyObserverPairGenerator = NewSampleAndWaterMarkHistogramsPairGenerator(clock.RealClock{}, time.Millisecond,
		&compbasemetrics.HistogramOpts{
			Namespace:      namespace,
			Subsystem:      subsystem,
			Name:           "read_vs_write_request_count_samples",
			Help:           "Periodic observations of the number of requests",
			Buckets:        []float64{0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1},
			StabilityLevel: compbasemetrics.ALPHA,
		},
		&compbasemetrics.HistogramOpts{
			Namespace:      namespace,
			Subsystem:      subsystem,
			Name:           "read_vs_write_request_count_watermarks",
			Help:           "Watermarks of the number of requests",
			Buckets:        []float64{0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1},
			StabilityLevel: compbasemetrics.ALPHA,
		},
		[]string{requestKind})

	apiserverCurrentInqueueRequests = compbasemetrics.NewGaugeVec(
		&compbasemetrics.GaugeOpts{
			Namespace:      namespace,
			Subsystem:      subsystem,
			Name:           "current_inqueue_requests",
			Help:           "Number of requests currently pending in queues of the API Priority and Fairness system",
			StabilityLevel: compbasemetrics.ALPHA,
		},
		[]string{priorityLevel, flowSchema},
	)
	apiserverRequestQueueLength = compbasemetrics.NewHistogramVec(
		&compbasemetrics.HistogramOpts{
			Namespace:      namespace,
			Subsystem:      subsystem,
			Name:           "request_queue_length_after_enqueue",
			Help:           "Length of queue in the API Priority and Fairness system, as seen by each request after it is enqueued",
			Buckets:        queueLengthBuckets,
			StabilityLevel: compbasemetrics.ALPHA,
		},
		[]string{priorityLevel, flowSchema},
	)
	apiserverRequestConcurrencyLimit = compbasemetrics.NewGaugeVec(
		&compbasemetrics.GaugeOpts{
			Namespace:      namespace,
			Subsystem:      subsystem,
			Name:           "request_concurrency_limit",
			Help:           "Shared concurrency limit in the API Priority and Fairness system",
			StabilityLevel: compbasemetrics.ALPHA,
		},
		[]string{priorityLevel},
	)
	apiserverCurrentExecutingRequests = compbasemetrics.NewGaugeVec(
		&compbasemetrics.GaugeOpts{
			Namespace:      namespace,
			Subsystem:      subsystem,
			Name:           "current_executing_requests",
			Help:           "Number of requests currently executing in the API Priority and Fairness system",
			StabilityLevel: compbasemetrics.ALPHA,
		},
		[]string{priorityLevel, flowSchema},
	)
	apiserverRequestWaitingSeconds = compbasemetrics.NewHistogramVec(
		&compbasemetrics.HistogramOpts{
			Namespace:      namespace,
			Subsystem:      subsystem,
			Name:           "request_wait_duration_seconds",
			Help:           "Length of time a request spent waiting in its queue",
			Buckets:        requestDurationSecondsBuckets,
			StabilityLevel: compbasemetrics.ALPHA,
		},
		[]string{priorityLevel, flowSchema, "execute"},
	)
	apiserverRequestExecutionSeconds = compbasemetrics.NewHistogramVec(
		&compbasemetrics.HistogramOpts{
			Namespace:      namespace,
			Subsystem:      subsystem,
			Name:           "request_execution_seconds",
			Help:           "Duration of request execution in the API Priority and Fairness system",
			Buckets:        requestDurationSecondsBuckets,
			StabilityLevel: compbasemetrics.ALPHA,
		},
		[]string{priorityLevel, flowSchema},
	)
	metrics = Registerables{
		apiserverRejectedRequestsTotal,
		apiserverDispatchedRequestsTotal,
		apiserverCurrentInqueueRequests,
		apiserverRequestQueueLength,
		apiserverRequestConcurrencyLimit,
		apiserverCurrentExecutingRequests,
		apiserverRequestWaitingSeconds,
		apiserverRequestExecutionSeconds,
	}.
		Append(PriorityLevelConcurrencyObserverPairGenerator.metrics()...).
		Append(ReadWriteConcurrencyObserverPairGenerator.metrics()...)
)

// AddRequestsInQueues adds the given delta to the gauge of the # of requests in the queues of the specified flowSchema and priorityLevel
func AddRequestsInQueues(priorityLevel, flowSchema string, delta int) {
	apiserverCurrentInqueueRequests.WithLabelValues(priorityLevel, flowSchema).Add(float64(delta))
}

// AddRequestsExecuting adds the given delta to the gauge of executing requests of the given flowSchema and priorityLevel
func AddRequestsExecuting(priorityLevel, flowSchema string, delta int) {
	apiserverCurrentExecutingRequests.WithLabelValues(priorityLevel, flowSchema).Add(float64(delta))
}

// UpdateSharedConcurrencyLimit updates the value for the concurrency limit in flow control
func UpdateSharedConcurrencyLimit(priorityLevel string, limit int) {
	apiserverRequestConcurrencyLimit.WithLabelValues(priorityLevel).Set(float64(limit))
}

// AddReject increments the # of rejected requests for flow control
func AddReject(priorityLevel, flowSchema, reason string) {
	apiserverRejectedRequestsTotal.WithLabelValues(priorityLevel, flowSchema, reason).Add(1)
}

// AddDispatch increments the # of dispatched requests for flow control
func AddDispatch(priorityLevel, flowSchema string) {
	apiserverDispatchedRequestsTotal.WithLabelValues(priorityLevel, flowSchema).Add(1)
}

// ObserveQueueLength observes the queue length for flow control
func ObserveQueueLength(priorityLevel, flowSchema string, length int) {
	apiserverRequestQueueLength.WithLabelValues(priorityLevel, flowSchema).Observe(float64(length))
}

// ObserveWaitingDuration observes the queue length for flow control
func ObserveWaitingDuration(priorityLevel, flowSchema, execute string, waitTime time.Duration) {
	apiserverRequestWaitingSeconds.WithLabelValues(priorityLevel, flowSchema, execute).Observe(waitTime.Seconds())
}

// ObserveExecutionDuration observes the execution duration for flow control
func ObserveExecutionDuration(priorityLevel, flowSchema string, executionTime time.Duration) {
	apiserverRequestExecutionSeconds.WithLabelValues(priorityLevel, flowSchema).Observe(executionTime.Seconds())
}
<|endoftext|>


<commit_before>package vm

import (
	"../types"
	"sync"
	"sync/atomic"
)

type State uint32

const (
	VALUE State = iota
	LOCKED
	APP
)

type Thunk struct {
	result, function, args types.Object
	state                  State
	blackHole              sync.WaitGroup
}

func NewValueThunk(v types.Object) *Thunk {
	return &Thunk{result: v, state: VALUE}
}

func NewAppThunk(f types.Object, as types.Object) *Thunk {
	return &Thunk{function: f, args: as, state: APP}
}

func (t *Thunk) Wait() {
	t.blackHole.Wait()
}

func (t *Thunk) Lock() bool {
	return t.compareAndSwapState(APP, LOCKED)
}

func (t *Thunk) SaveResult(o types.Object) {
	if t.state != LOCKED {
		panic("Thunk is not locked yet.")
	}

	t.result = o
	t.function = nil
	t.args = nil

	t.storeState(VALUE)
}

func (t *Thunk) compareAndSwapState(old, new State) bool {
	return atomic.CompareAndSwapUint32((*uint32)(&t.state), uint32(old), uint32(new))
}

func (t *Thunk) storeState(new State) {
	atomic.StoreUint32((*uint32)(&t.state), uint32(new))
}
<commit_msg>Fix indentation<commit_after>package vm

import (
	"../types"
	"sync"
	"sync/atomic"
)

type State uint32

const (
	VALUE State = iota
	LOCKED
	APP
)

type Thunk struct {
	result, function, args types.Object
	state                  State
	blackHole              sync.WaitGroup
}

func NewValueThunk(v types.Object) *Thunk {
	return &Thunk{result: v, state: VALUE}
}

func NewAppThunk(f types.Object, as types.Object) *Thunk {
	return &Thunk{function: f, args: as, state: APP}
}

func (t *Thunk) Wait() {
	t.blackHole.Wait()
}

func (t *Thunk) Lock() bool {
	return t.compareAndSwapState(APP, LOCKED)
}

func (t *Thunk) SaveResult(o types.Object) {
	if t.state != LOCKED {
		panic("Thunk is not locked yet.")
	}

	t.result = o
	t.function = nil
	t.args = nil

	t.storeState(VALUE)
}

func (t *Thunk) compareAndSwapState(old, new State) bool {
	return atomic.CompareAndSwapUint32(
		(*uint32)(&t.state),
		uint32(old),
		uint32(new))
}

func (t *Thunk) storeState(new State) {
	atomic.StoreUint32((*uint32)(&t.state), uint32(new))
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"strings"
)

const (
	HEADING_UNDERLINE_LENGTH = 20
	TABLE_LEFT_SPACING       = 5
)

func getRepeatedChars(character string, repeatCount int) string {
	formatted := ""
	for i := 0; i < repeatCount; i++ {
		formatted = fmt.Sprintf("%s%s", formatted, character)
	}
	return formatted
}

func formatSpecHeading(specHeading string) string {
	return formatHeading(specHeading, "=")
}

func formatScenarioHeading(scenarioHeading string) string {
	return fmt.Sprintf("%s", formatHeading(scenarioHeading, "-"))
}

func formatStep(step *step) string {
	text := step.value
	paramCount := strings.Count(text, PARAMETER_PLACEHOLDER)
	for i := 0; i < paramCount; i++ {
		argument := step.args[i]
		formattedArg := ""
		if argument.argType == tableArg {
			formattedTable := formatTable(&argument.table)
			formattedArg = fmt.Sprintf("\n%s", formattedTable)
		} else if argument.argType == dynamic {
			formattedArg = fmt.Sprintf("<%s>", argument.value)
		} else {
			formattedArg = fmt.Sprintf("\"%s\"", argument.value)
		}
		text = strings.Replace(text, PARAMETER_PLACEHOLDER, formattedArg, 1)
	}
	stepText := fmt.Sprintf("* %s\n", text)
	return stepText
}

func formatHeading(heading, headingChar string) string {
	length := len(heading)
	if length > HEADING_UNDERLINE_LENGTH {
		length = HEADING_UNDERLINE_LENGTH
	}

	return fmt.Sprintf("%s\n%s\n", heading, getRepeatedChars(headingChar, length))
}

func formatTable(table *table) string {
	columnToWidthMap := make(map[int]int)
	for i, header := range table.headers {
		//table.get(header) returns a list of cells in that particular column
		cells := table.get(header)
		columnToWidthMap[i] = findLongestCellWidth(cells, len(header))
	}

	var tableStringBuffer bytes.Buffer
	tableStringBuffer.WriteString(fmt.Sprintf("%s|", getRepeatedChars(" ", TABLE_LEFT_SPACING)))
	for i, header := range table.headers {
		width := columnToWidthMap[i]
		tableStringBuffer.WriteString(fmt.Sprintf("%s|", addPaddingToCell(header, width)))
	}

	tableStringBuffer.WriteString("\n")
	tableStringBuffer.WriteString(fmt.Sprintf("%s|", getRepeatedChars(" ", TABLE_LEFT_SPACING)))
	for i, _ := range table.headers {
		width := columnToWidthMap[i]
		cell := getRepeatedChars("-", width)
		tableStringBuffer.WriteString(fmt.Sprintf("%s|", addPaddingToCell(cell, width)))
	}

	tableStringBuffer.WriteString("\n")
	for _, row := range table.getRows() {
		tableStringBuffer.WriteString(fmt.Sprintf("%s|", getRepeatedChars(" ", TABLE_LEFT_SPACING)))
		for i, cell := range row {
			width := columnToWidthMap[i]
			tableStringBuffer.WriteString(fmt.Sprintf("%s|", addPaddingToCell(cell, width)))
		}
		tableStringBuffer.WriteString("\n")
	}

	return string(tableStringBuffer.Bytes())
}

func addPaddingToCell(cellValue string, width int) string {
	padding := getRepeatedChars(" ", width-len(cellValue))
	return fmt.Sprintf("%s%s", cellValue, padding)
}

func findLongestCellWidth(columnCells []tableCell, minValue int) int {
	longestLength := minValue
	for _, cellValue := range columnCells {
		cellValueLen := len(cellValue.value)
		if cellValueLen > longestLength {
			longestLength = cellValueLen
		}
	}
	return longestLength
}

func formatItem(item item) string {
	switch item.kind() {
	case commentKind:
		comment := item.(*comment)
		if comment.value == "\n" {
			return comment.value
		}
		return fmt.Sprintf("%s\n", comment.value)
	case stepKind:
		step := item.(*step)
		return formatStep(step)
	case tableKind:
		table := item.(*table)
		return formatTable(table)
	case scenarioKind:
		scenario := item.(*scenario)
		var b bytes.Buffer
		b.WriteString(formatScenarioHeading(scenario.heading.value))
		b.WriteString(formatItems(scenario.items))
		return string(b.Bytes())
	case tagKind:
		tags := item.(*tags)
		var b bytes.Buffer
		b.WriteString("tags: ")
		for i, tag := range tags.values {
			b.WriteString(tag)
			if (i + 1) != len(tags.values) {
				b.WriteString(", ")
			}
		}
		b.WriteString("\n")
		return string(b.Bytes())
	}
	return ""
}

func formatItems(items []item) string {
	var result bytes.Buffer
	for _, item := range items {
		result.WriteString(formatItem(item))
	}
	return string(result.Bytes())
}

func formatSpecification(specification *specification) string {
	var formattedText bytes.Buffer
	formattedText.WriteString(formatSpecHeading(specification.heading.value))
	formattedText.WriteString(formatItems(specification.items))
	return string(formattedText.Bytes())
}
<commit_msg>Removing unwanted newlines when formating steps with tables<commit_after>package main

import (
	"bytes"
	"fmt"
	"strings"
)

const (
	HEADING_UNDERLINE_LENGTH = 20
	TABLE_LEFT_SPACING       = 5
)

func getRepeatedChars(character string, repeatCount int) string {
	formatted := ""
	for i := 0; i < repeatCount; i++ {
		formatted = fmt.Sprintf("%s%s", formatted, character)
	}
	return formatted
}

func formatSpecHeading(specHeading string) string {
	return formatHeading(specHeading, "=")
}

func formatScenarioHeading(scenarioHeading string) string {
	return fmt.Sprintf("%s", formatHeading(scenarioHeading, "-"))
}

func formatStep(step *step) string {
	text := step.value
	paramCount := strings.Count(text, PARAMETER_PLACEHOLDER)
	for i := 0; i < paramCount; i++ {
		argument := step.args[i]
		formattedArg := ""
		if argument.argType == tableArg {
			formattedTable := formatTable(&argument.table)
			formattedArg = fmt.Sprintf("\n%s", formattedTable)
		} else if argument.argType == dynamic {
			formattedArg = fmt.Sprintf("<%s>", argument.value)
		} else {
			formattedArg = fmt.Sprintf("\"%s\"", argument.value)
		}
		text = strings.Replace(text, PARAMETER_PLACEHOLDER, formattedArg, 1)
	}
	stepText := ""
	if strings.HasSuffix(text, "\n") {
		stepText = fmt.Sprintf("* %s", text)
	} else {
		stepText = fmt.Sprintf("* %s\n", text)
	}
	return stepText
}

func formatHeading(heading, headingChar string) string {
	length := len(heading)
	if length > HEADING_UNDERLINE_LENGTH {
		length = HEADING_UNDERLINE_LENGTH
	}

	return fmt.Sprintf("%s\n%s\n", heading, getRepeatedChars(headingChar, length))
}

func formatTable(table *table) string {
	columnToWidthMap := make(map[int]int)
	for i, header := range table.headers {
		//table.get(header) returns a list of cells in that particular column
		cells := table.get(header)
		columnToWidthMap[i] = findLongestCellWidth(cells, len(header))
	}

	var tableStringBuffer bytes.Buffer
	tableStringBuffer.WriteString(fmt.Sprintf("%s|", getRepeatedChars(" ", TABLE_LEFT_SPACING)))
	for i, header := range table.headers {
		width := columnToWidthMap[i]
		tableStringBuffer.WriteString(fmt.Sprintf("%s|", addPaddingToCell(header, width)))
	}

	tableStringBuffer.WriteString("\n")
	tableStringBuffer.WriteString(fmt.Sprintf("%s|", getRepeatedChars(" ", TABLE_LEFT_SPACING)))
	for i, _ := range table.headers {
		width := columnToWidthMap[i]
		cell := getRepeatedChars("-", width)
		tableStringBuffer.WriteString(fmt.Sprintf("%s|", addPaddingToCell(cell, width)))
	}

	tableStringBuffer.WriteString("\n")
	for _, row := range table.getRows() {
		tableStringBuffer.WriteString(fmt.Sprintf("%s|", getRepeatedChars(" ", TABLE_LEFT_SPACING)))
		for i, cell := range row {
			width := columnToWidthMap[i]
			tableStringBuffer.WriteString(fmt.Sprintf("%s|", addPaddingToCell(cell, width)))
		}
		tableStringBuffer.WriteString("\n")
	}

	return string(tableStringBuffer.Bytes())
}

func addPaddingToCell(cellValue string, width int) string {
	padding := getRepeatedChars(" ", width-len(cellValue))
	return fmt.Sprintf("%s%s", cellValue, padding)
}

func findLongestCellWidth(columnCells []tableCell, minValue int) int {
	longestLength := minValue
	for _, cellValue := range columnCells {
		cellValueLen := len(cellValue.value)
		if cellValueLen > longestLength {
			longestLength = cellValueLen
		}
	}
	return longestLength
}

func formatItem(item item) string {
	switch item.kind() {
	case commentKind:
		comment := item.(*comment)
		if comment.value == "\n" {
			return comment.value
		}
		return fmt.Sprintf("%s\n", comment.value)
	case stepKind:
		step := item.(*step)
		return formatStep(step)
	case tableKind:
		table := item.(*table)
		return formatTable(table)
	case scenarioKind:
		scenario := item.(*scenario)
		var b bytes.Buffer
		b.WriteString(formatScenarioHeading(scenario.heading.value))
		b.WriteString(formatItems(scenario.items))
		return string(b.Bytes())
	case tagKind:
		tags := item.(*tags)
		var b bytes.Buffer
		b.WriteString("tags: ")
		for i, tag := range tags.values {
			b.WriteString(tag)
			if (i + 1) != len(tags.values) {
				b.WriteString(", ")
			}
		}
		b.WriteString("\n")
		return string(b.Bytes())
	}
	return ""
}

func formatItems(items []item) string {
	var result bytes.Buffer
	for _, item := range items {
		result.WriteString(formatItem(item))
	}
	return string(result.Bytes())
}

func formatSpecification(specification *specification) string {
	var formattedText bytes.Buffer
	formattedText.WriteString(formatSpecHeading(specification.heading.value))
	formattedText.WriteString(formatItems(specification.items))
	return string(formattedText.Bytes())
}
<|endoftext|>


<commit_before>package fullerene

import (
	"time"
)

type Fullerene struct {
	t time.Time
}

func Now() Fullerene {
	return Fullerene{
		t: time.Now(),
	}
}

func (fr Fullerene) Date() (year int, month time.Month, day int) {
	return fr.t.Date()
}

func (fr Fullerene) After(u Fullerene) bool {
	return fr.t.After(u.t)
}

func (fr Fullerene) Before(u Fullerene) bool {
	return fr.t.Before(u.t)
}

func (fr Fullerene) Equal(u Fullerene) bool {
	return fr.t.Equal(u.t)
}

func (fr Fullerene) IsZero() bool {
	return fr.t.IsZero()
}

func Date(year int, month time.Month, day, hour, min, sec, nsec int, loc *time.Location) Fullerene {
	return Fullerene{t: time.Date(year, month, day, hour, min, sec, nsec, loc)}
}

func (fr Fullerene) IsLeapYear() bool {
	y, _, _ := fr.Date()
	if y%4 == 0 {
		if y%100 == 0 && y%400 != 0 {
			return false
		}
		return true
	}
	return false
}

func (fr Fullerene) IsLeapDay() bool {
	_, m, d := fr.Date()
	if m == 2 && d == 29 {
		return true
	}
	return false
}

func (fr Fullerene) IsBirthday(targetTime Fullerene, beforeDayIfLeap bool) bool {
	_, m, d := fr.Date()
	_, mm, dd := targetTime.Date()
	if m == mm && d == dd {
		return true
	}
	// consider leap year.
	isLeapYear := Now().IsLeapYear()
	if !isLeapYear {
		return false
	}

	// there are countries which a person get old at the day before leap day, and the day after in a leap year.
	if beforeDayIfLeap {
		if mm == 2 && dd == 28 {
			return true
		}
	}
	if mm == 3 && dd == 1 {
		return true
	}
	return false
}
<commit_msg>Refactoring.<commit_after>package fullerene

import (
	"time"
)

type Fullerene struct {
	t time.Time
}

func Now() Fullerene {
	return Fullerene{
		t: time.Now(),
	}
}

func (fr Fullerene) Date() (year int, month time.Month, day int) {
	return fr.t.Date()
}

func (fr Fullerene) After(u Fullerene) bool {
	return fr.t.After(u.t)
}

func (fr Fullerene) Before(u Fullerene) bool {
	return fr.t.Before(u.t)
}

func (fr Fullerene) Equal(u Fullerene) bool {
	return fr.t.Equal(u.t)
}

func (fr Fullerene) IsZero() bool {
	return fr.t.IsZero()
}

func Date(year int, month time.Month, day, hour, min, sec, nsec int, loc *time.Location) Fullerene {
	return Fullerene{t: time.Date(year, month, day, hour, min, sec, nsec, loc)}
}

func (fr Fullerene) IsLeapYear() bool {
	y, _, _ := fr.Date()
	if y%4 != 0 {
		return false
	}
	if y%100 == 0 && y%400 != 0 {
		return false
	}
	return true
}

func (fr Fullerene) IsLeapDay() bool {
	_, m, d := fr.Date()
	if m == 2 && d == 29 {
		return true
	}
	return false
}

func (fr Fullerene) IsBirthday(targetTime Fullerene, beforeDayIfLeap bool) bool {
	_, m, d := fr.Date()
	_, mm, dd := targetTime.Date()
	if m == mm && d == dd {
		return true
	}
	// consider leap year.
	isLeapYear := Now().IsLeapYear()
	if !isLeapYear {
		return false
	}

	// there are countries which a person get old at the day before leap day, and the day after in a leap year.
	if beforeDayIfLeap {
		if mm == 2 && dd == 28 {
			return true
		}
	}
	if mm == 3 && dd == 1 {
		return true
	}
	return false
}
<|endoftext|>