<commit_before>package peco

import (
	"io"
	"sync"
	"time"

	"github.com/google/btree"
	"github.com/nsf/termbox-go"
	"github.com/peco/peco/internal/keyseq"
)

type idGen struct {
	genCh chan uint64
}

// Line represents each of the line that peco uses to display
// and match against queries.
type Line interface {
	btree.Item

	ID() uint64

	// Buffer returns the raw buffer
	Buffer() string

	// DisplayString returns the string to be displayed. This means if you have
	// a null separator, the contents after the separator are not included
	// in this string
	DisplayString() string

	// Indices return the matched portion(s) of a string after filtering.
	// Note that while Indices may return nil, that just means that there are
	// no substrings to be highlighted. It doesn't mean there were no matches
	Indices() [][]int

	// Output returns the string to be display as peco finishes up doing its
	// thing. This means if you have null separator, the contents before the
	// separator are not included in this string
	Output() string

	// IsDirty returns true if this line should be forcefully redrawn
	IsDirty() bool

	// SetDirty sets the dirty flag on or off
	SetDirty(bool)
}

// RawLine is the input line as sent to peco, before filtering and what not.
type RawLine struct {
	id            uint64
	buf           string
	sepLoc        int
	displayString string
	dirty         bool
}

// MatchedLine contains the indices to the matches
type MatchedLine struct {
	Line
	indices [][]int
}

// BufferReader reads from either stdin or a file. In case of stdin,
// it also handles possible infinite source.
type BufferReader struct {
	*Ctx
	input        io.ReadCloser
	inputReadyCh chan struct{}
}

type Keyseq interface {
	Add(keyseq.KeyList, interface{})
	AcceptKey(keyseq.Key) (interface{}, error)
	CancelChain()
	Clear()
	Compile() error
	InMiddleOfChain() bool
}

// PagingRequest can be sent to move the selection cursor
type PagingRequestType int

type PagingRequest interface {
	Type() PagingRequestType
}

type JumpToLineRequest uint

// Selection stores the line ids that were selected by the user.
// The contents of the Selection is always sorted from smallest to
// largest line ID
type Selection struct{ *btree.BTree }

// StatusMsgRequest specifies the string to be drawn
// on the status message bar and an optional delay that tells
// the view to clear that message
type StatusMsgRequest struct {
	message    string
	clearDelay time.Duration
}

// Screen hides termbox from the consuming code so that
// it can be swapped out for testing
type Screen interface {
	Init() error
	Close() error
	Flush() error
	PollEvent() chan termbox.Event
	SetCell(int, int, rune, termbox.Attribute, termbox.Attribute)
	Size() (int, int)
	SendEvent(termbox.Event)
}

// Termbox just hands out the processing to the termbox library
type Termbox struct{}

// View handles the drawing/updating the screen
type View struct {
	*Ctx
	mutex  sync.Locker
	layout Layout
}

// PageCrop filters out a new LineBuffer based on entries
// per page and the page number
type PageCrop struct {
	perPage     int
	currentPage int
}

// LayoutType describes the types of layout that peco can take
type LayoutType string

// VerticalAnchor describes the direction to which elements in the
// layout are anchored to
type VerticalAnchor int

// Layout represents the component that controls where elements are placed on screen
type Layout interface {
	PrintStatus(string, time.Duration)
	DrawPrompt()
	DrawScreen(bool)
	MovePage(PagingRequest) (moved bool)
	PurgeDisplayCache()
}

// AnchorSettings groups items that are required to control
// where an anchored item is actually placed
type AnchorSettings struct {
	anchor       VerticalAnchor // AnchorTop or AnchorBottom
	anchorOffset int            // offset this many lines from the anchor
}

// UserPrompt draws the prompt line
type UserPrompt struct {
	*Ctx
	*AnchorSettings
	prefix     string
	prefixLen  int
	basicStyle Style
	queryStyle Style
}

// StatusBar draws the status message bar
type StatusBar struct {
	*Ctx
	*AnchorSettings
	clearTimer *time.Timer
	timerMutex sync.Locker
	basicStyle Style
}

// ListArea represents the area where the actual line buffer is
// displayed in the screen
type ListArea struct {
	*Ctx
	*AnchorSettings
	sortTopDown         bool
	displayCache        []Line
	dirty               bool
	basicStyle          Style
	queryStyle          Style
	matchedStyle        Style
	selectedStyle       Style
	savedSelectionStyle Style
}

// BasicLayout is... the basic layout :) At this point this is the
// only struct for layouts, which means that while the position
// of components may be configurable, the actual types of components
// that are used are set and static
type BasicLayout struct {
	*Ctx
	*StatusBar
	prompt *UserPrompt
	list   *ListArea
}

// Keymap holds all the key sequence to action map
type Keymap struct {
	Config map[string]string
	Action map[string][]string // custom actions
	seq    Keyseq
}

// internal stuff
type regexpFlags interface {
	flags(string) []string
}
type regexpFlagList []string

type regexpFlagFunc func(string) []string

// Filter is responsible for the actual "grep" part of peco
type Filter struct {
	*Ctx
}

// Action describes an action that can be executed upon receiving user
// input. It's an interface so you can create any kind of Action you need,
// but most everything is implemented in terms of ActionFunc, which is
// callback based Action
type Action interface {
	Register(string, ...termbox.Key)
	RegisterKeySequence(string, keyseq.KeyList)
	Execute(*Input, termbox.Event)
}

// ActionFunc is a type of Action that is basically just a callback.
type ActionFunc func(*Input, termbox.Event)

type Pipeliner interface {
	Pipeline() (chan struct{}, chan Line)
}

type pipelineCtx struct {
	onIncomingLine func(Line) (Line, error)
	onEnd          func()
}

type simplePipeline struct {
	// Close this channel if you want to cancel the entire pipeline.
	// InputReader is the generator for the pipeline, so this is the
	// only object that can create the cancelCh
	cancelCh chan struct{}
	// Consumers of this generator read from this channel.
	outputCh chan Line
}

// LineBuffer represents a set of lines. This could be the
// raw data read in, or filtered data, such as result of
// running a match, or applying a selection by the user
//
// Buffers should be immutable.
type LineBuffer interface {
	Pipeliner

	LineAt(int) (Line, error)
	Size() int

	// Register registers another LineBuffer that is dependent on
	// this buffer.
	Register(LineBuffer)
	Unregister(LineBuffer)

	// InvalidateUpTo is called when a source buffer invalidates
	// some lines. The argument is the largest line number that
	// should be invalidated (so anything up to that line is no
	// longer valid in the source)
	InvalidateUpTo(int)
}

type dependentBuffers []LineBuffer

// RawLineBuffer holds the raw set of lines as read into peco.
type RawLineBuffer struct {
	simplePipeline
	buffers  dependentBuffers
	lines    []Line
	capacity int // max number of lines. 0 means unlimited
	onEnd    func()
}

// FilteredLineBuffer holds a "filtered" buffer. It holds a reference to
// the source buffer (note: should be immutable) and a list of indices
// into the source buffer
type FilteredLineBuffer struct {
	simplePipeline
	buffers dependentBuffers
	src     LineBuffer
	// maps from our index to src's index
	selection []int
}

// Input handles input events from termbox.
type Input struct {
	*Ctx
	mutex         sync.Locker // Currently only used for protecting Alt/Esc workaround
	mod           *time.Timer
	keymap        Keymap
	currentKeySeq []string
}

// Hub acts as the messaging hub between components -- that is,
// it controls how the communication that goes through channels
// are handled.
type Hub struct {
	isSync      bool
	mutex       sync.Locker
	loopCh      chan struct{}
	queryCh     chan HubReq
	drawCh      chan HubReq
	statusMsgCh chan HubReq
	pagingCh    chan HubReq
}

// HubReq is a wrapper around the actual request value that needs
// to be passed. It contains an optional channel field which can
// be filled to force synchronous communication between the
// sender and receiver
type HubReq struct {
	data    interface{}
	replyCh chan struct{}
}

// Config holds all the data that can be configured in the
// external configuran file
type Config struct {
	Action map[string][]string `json:"Action"`
	// Keymap used to be directly responsible for dispatching
	// events against user input, but since then this has changed
	// into something that just records the user's config input
	Keymap              map[string]string `json:"Keymap"`
	Matcher             string            `json:"Matcher"`        // Deprecated.
	InitialMatcher      string            `json:"InitialMatcher"` // Use this instead of Matcher
	InitialFilter       string            `json:"InitialFilter"`
	Style               *StyleSet         `json:"Style"`
	Prompt              string            `json:"Prompt"`
	Layout              string            `json:"Layout"`
	CustomMatcher       map[string][]string
	CustomFilter        map[string]CustomFilterConfig
	QueryExecutionDelay int
	StickySelection     bool

	// If this is true, then the prefix for single key jump mode
	// is displayed by default.
	SingleKeyJump SingleKeyJumpConfig `json:"SingleKeyJump"`
}

type SingleKeyJumpConfig struct {
	ShowPrefix bool          `json:"ShowPrefix"`
	PrefixList []rune        `json:"-"`
	PrefixMap  map[rune]uint `json:"-"`
}

// CustomFilterConfig is used to specify configuration parameters
// to CustomFilters
type CustomFilterConfig struct {
	// Cmd is the name of the command to invoke
	Cmd string

	// TODO: need to check if how we use this is correct
	Args []string

	// BufferThreshold defines how many lines peco buffers before
	// invoking the external command. If this value is big, we
	// will execute the external command fewer times, but the
	// results will not be generated for longer periods of time.
	// If this value is small, we will execute the external command
	// more often, but you pay the penalty of invoking that command
	// more times.
	BufferThreshold int
}

// StyleSet holds styles for various sections
type StyleSet struct {
	Basic          Style `json:"Basic"`
	SavedSelection Style `json:"SavedSelection"`
	Selected       Style `json:"Selected"`
	Query          Style `json:"Query"`
	Matched        Style `json:"Matched"`
}

// Style describes termbox styles
type Style struct {
	fg termbox.Attribute
	bg termbox.Attribute
}

// CtxOptions is the interface that defines that options can be
// passed in from the command line
type CtxOptions interface {
	// EnableNullSep should return if the null separator is
	// enabled (--null)
	EnableNullSep() bool

	// BufferSize should return the buffer size. By default (i.e.
	// when it returns 0), the buffer size is unlimited.
	// (--buffer-size)
	BufferSize() int

	// InitialIndex is the line number to put the cursor on
	// when peco starts
	InitialIndex() int

	// LayoutType returns the name of the layout to use
	LayoutType() string
}

type PageInfo struct {
	page    int
	offset  int
	perPage int
	total   int
	maxPage int
}

type FilterQuery struct {
	query      []rune
	savedQuery []rune
	mutex      sync.Locker
}

// Ctx contains all the important data. while you can easily access
// data in this struct from anywhere, only do so via channels
type Ctx struct {
	*Hub
	*FilterQuery
	filters             FilterSet
	caretPosition       int
	enableSep           bool
	resultCh            chan Line
	mutex               sync.Locker
	currentLine         int
	currentCol          int
	currentPage         *PageInfo
	selection           *Selection
	activeLineBuffer    LineBuffer
	rawLineBuffer       *RawLineBuffer
	lines               []Line
	linesMutex          sync.Locker
	current             []Line
	currentMutex        sync.Locker
	bufferSize          int
	config              *Config
	selectionRangeStart int
	layoutType          string
	singleKeyJumpMode   bool
	select1             bool

	wait *sync.WaitGroup
	err  error
}

type CLIOptions struct {
	OptHelp           bool   `short:"h" long:"help" description:"show this help message and exit"`
	OptTTY            string `long:"tty" description:"path to the TTY (usually, the value of $TTY)"`
	OptQuery          string `long:"query" description:"initial value for query"`
	OptRcfile         string `long:"rcfile" description:"path to the settings file"`
	OptVersion        bool   `long:"version" description:"print the version and exit"`
	OptBufferSize     int    `long:"buffer-size" short:"b" description:"number of lines to keep in search buffer"`
	OptEnableNullSep  bool   `long:"null" description:"expect NUL (\\0) as separator for target/output"`
	OptInitialIndex   int    `long:"initial-index" description:"position of the initial index of the selection (0 base)"`
	OptInitialMatcher string `long:"initial-matcher" description:"specify the default matcher (deprecated)"`
	OptInitialFilter  string `long:"initial-filter" description:"specify the default filter"`
	OptPrompt         string `long:"prompt" description:"specify the prompt string"`
	OptLayout         string `long:"layout" description:"layout to be used 'top-down' (default) or 'bottom-up'" default:"top-down"`
	OptSelect1        bool   `long:"select-1" description:"select first item if given 1 item"`
}

type CLI struct {
}
<commit_msg>Update help message<commit_after>package peco

import (
	"io"
	"sync"
	"time"

	"github.com/google/btree"
	"github.com/nsf/termbox-go"
	"github.com/peco/peco/internal/keyseq"
)

type idGen struct {
	genCh chan uint64
}

// Line represents each of the line that peco uses to display
// and match against queries.
type Line interface {
	btree.Item

	ID() uint64

	// Buffer returns the raw buffer
	Buffer() string

	// DisplayString returns the string to be displayed. This means if you have
	// a null separator, the contents after the separator are not included
	// in this string
	DisplayString() string

	// Indices return the matched portion(s) of a string after filtering.
	// Note that while Indices may return nil, that just means that there are
	// no substrings to be highlighted. It doesn't mean there were no matches
	Indices() [][]int

	// Output returns the string to be display as peco finishes up doing its
	// thing. This means if you have null separator, the contents before the
	// separator are not included in this string
	Output() string

	// IsDirty returns true if this line should be forcefully redrawn
	IsDirty() bool

	// SetDirty sets the dirty flag on or off
	SetDirty(bool)
}

// RawLine is the input line as sent to peco, before filtering and what not.
type RawLine struct {
	id            uint64
	buf           string
	sepLoc        int
	displayString string
	dirty         bool
}

// MatchedLine contains the indices to the matches
type MatchedLine struct {
	Line
	indices [][]int
}

// BufferReader reads from either stdin or a file. In case of stdin,
// it also handles possible infinite source.
type BufferReader struct {
	*Ctx
	input        io.ReadCloser
	inputReadyCh chan struct{}
}

type Keyseq interface {
	Add(keyseq.KeyList, interface{})
	AcceptKey(keyseq.Key) (interface{}, error)
	CancelChain()
	Clear()
	Compile() error
	InMiddleOfChain() bool
}

// PagingRequest can be sent to move the selection cursor
type PagingRequestType int

type PagingRequest interface {
	Type() PagingRequestType
}

type JumpToLineRequest uint

// Selection stores the line ids that were selected by the user.
// The contents of the Selection is always sorted from smallest to
// largest line ID
type Selection struct{ *btree.BTree }

// StatusMsgRequest specifies the string to be drawn
// on the status message bar and an optional delay that tells
// the view to clear that message
type StatusMsgRequest struct {
	message    string
	clearDelay time.Duration
}

// Screen hides termbox from the consuming code so that
// it can be swapped out for testing
type Screen interface {
	Init() error
	Close() error
	Flush() error
	PollEvent() chan termbox.Event
	SetCell(int, int, rune, termbox.Attribute, termbox.Attribute)
	Size() (int, int)
	SendEvent(termbox.Event)
}

// Termbox just hands out the processing to the termbox library
type Termbox struct{}

// View handles the drawing/updating the screen
type View struct {
	*Ctx
	mutex  sync.Locker
	layout Layout
}

// PageCrop filters out a new LineBuffer based on entries
// per page and the page number
type PageCrop struct {
	perPage     int
	currentPage int
}

// LayoutType describes the types of layout that peco can take
type LayoutType string

// VerticalAnchor describes the direction to which elements in the
// layout are anchored to
type VerticalAnchor int

// Layout represents the component that controls where elements are placed on screen
type Layout interface {
	PrintStatus(string, time.Duration)
	DrawPrompt()
	DrawScreen(bool)
	MovePage(PagingRequest) (moved bool)
	PurgeDisplayCache()
}

// AnchorSettings groups items that are required to control
// where an anchored item is actually placed
type AnchorSettings struct {
	anchor       VerticalAnchor // AnchorTop or AnchorBottom
	anchorOffset int            // offset this many lines from the anchor
}

// UserPrompt draws the prompt line
type UserPrompt struct {
	*Ctx
	*AnchorSettings
	prefix     string
	prefixLen  int
	basicStyle Style
	queryStyle Style
}

// StatusBar draws the status message bar
type StatusBar struct {
	*Ctx
	*AnchorSettings
	clearTimer *time.Timer
	timerMutex sync.Locker
	basicStyle Style
}

// ListArea represents the area where the actual line buffer is
// displayed in the screen
type ListArea struct {
	*Ctx
	*AnchorSettings
	sortTopDown         bool
	displayCache        []Line
	dirty               bool
	basicStyle          Style
	queryStyle          Style
	matchedStyle        Style
	selectedStyle       Style
	savedSelectionStyle Style
}

// BasicLayout is... the basic layout :) At this point this is the
// only struct for layouts, which means that while the position
// of components may be configurable, the actual types of components
// that are used are set and static
type BasicLayout struct {
	*Ctx
	*StatusBar
	prompt *UserPrompt
	list   *ListArea
}

// Keymap holds all the key sequence to action map
type Keymap struct {
	Config map[string]string
	Action map[string][]string // custom actions
	seq    Keyseq
}

// internal stuff
type regexpFlags interface {
	flags(string) []string
}
type regexpFlagList []string

type regexpFlagFunc func(string) []string

// Filter is responsible for the actual "grep" part of peco
type Filter struct {
	*Ctx
}

// Action describes an action that can be executed upon receiving user
// input. It's an interface so you can create any kind of Action you need,
// but most everything is implemented in terms of ActionFunc, which is
// callback based Action
type Action interface {
	Register(string, ...termbox.Key)
	RegisterKeySequence(string, keyseq.KeyList)
	Execute(*Input, termbox.Event)
}

// ActionFunc is a type of Action that is basically just a callback.
type ActionFunc func(*Input, termbox.Event)

type Pipeliner interface {
	Pipeline() (chan struct{}, chan Line)
}

type pipelineCtx struct {
	onIncomingLine func(Line) (Line, error)
	onEnd          func()
}

type simplePipeline struct {
	// Close this channel if you want to cancel the entire pipeline.
	// InputReader is the generator for the pipeline, so this is the
	// only object that can create the cancelCh
	cancelCh chan struct{}
	// Consumers of this generator read from this channel.
	outputCh chan Line
}

// LineBuffer represents a set of lines. This could be the
// raw data read in, or filtered data, such as result of
// running a match, or applying a selection by the user
//
// Buffers should be immutable.
type LineBuffer interface {
	Pipeliner

	LineAt(int) (Line, error)
	Size() int

	// Register registers another LineBuffer that is dependent on
	// this buffer.
	Register(LineBuffer)
	Unregister(LineBuffer)

	// InvalidateUpTo is called when a source buffer invalidates
	// some lines. The argument is the largest line number that
	// should be invalidated (so anything up to that line is no
	// longer valid in the source)
	InvalidateUpTo(int)
}

type dependentBuffers []LineBuffer

// RawLineBuffer holds the raw set of lines as read into peco.
type RawLineBuffer struct {
	simplePipeline
	buffers  dependentBuffers
	lines    []Line
	capacity int // max number of lines. 0 means unlimited
	onEnd    func()
}

// FilteredLineBuffer holds a "filtered" buffer. It holds a reference to
// the source buffer (note: should be immutable) and a list of indices
// into the source buffer
type FilteredLineBuffer struct {
	simplePipeline
	buffers dependentBuffers
	src     LineBuffer
	// maps from our index to src's index
	selection []int
}

// Input handles input events from termbox.
type Input struct {
	*Ctx
	mutex         sync.Locker // Currently only used for protecting Alt/Esc workaround
	mod           *time.Timer
	keymap        Keymap
	currentKeySeq []string
}

// Hub acts as the messaging hub between components -- that is,
// it controls how the communication that goes through channels
// are handled.
type Hub struct {
	isSync      bool
	mutex       sync.Locker
	loopCh      chan struct{}
	queryCh     chan HubReq
	drawCh      chan HubReq
	statusMsgCh chan HubReq
	pagingCh    chan HubReq
}

// HubReq is a wrapper around the actual request value that needs
// to be passed. It contains an optional channel field which can
// be filled to force synchronous communication between the
// sender and receiver
type HubReq struct {
	data    interface{}
	replyCh chan struct{}
}

// Config holds all the data that can be configured in the
// external configuran file
type Config struct {
	Action map[string][]string `json:"Action"`
	// Keymap used to be directly responsible for dispatching
	// events against user input, but since then this has changed
	// into something that just records the user's config input
	Keymap              map[string]string `json:"Keymap"`
	Matcher             string            `json:"Matcher"`        // Deprecated.
	InitialMatcher      string            `json:"InitialMatcher"` // Use this instead of Matcher
	InitialFilter       string            `json:"InitialFilter"`
	Style               *StyleSet         `json:"Style"`
	Prompt              string            `json:"Prompt"`
	Layout              string            `json:"Layout"`
	CustomMatcher       map[string][]string
	CustomFilter        map[string]CustomFilterConfig
	QueryExecutionDelay int
	StickySelection     bool

	// If this is true, then the prefix for single key jump mode
	// is displayed by default.
	SingleKeyJump SingleKeyJumpConfig `json:"SingleKeyJump"`
}

type SingleKeyJumpConfig struct {
	ShowPrefix bool          `json:"ShowPrefix"`
	PrefixList []rune        `json:"-"`
	PrefixMap  map[rune]uint `json:"-"`
}

// CustomFilterConfig is used to specify configuration parameters
// to CustomFilters
type CustomFilterConfig struct {
	// Cmd is the name of the command to invoke
	Cmd string

	// TODO: need to check if how we use this is correct
	Args []string

	// BufferThreshold defines how many lines peco buffers before
	// invoking the external command. If this value is big, we
	// will execute the external command fewer times, but the
	// results will not be generated for longer periods of time.
	// If this value is small, we will execute the external command
	// more often, but you pay the penalty of invoking that command
	// more times.
	BufferThreshold int
}

// StyleSet holds styles for various sections
type StyleSet struct {
	Basic          Style `json:"Basic"`
	SavedSelection Style `json:"SavedSelection"`
	Selected       Style `json:"Selected"`
	Query          Style `json:"Query"`
	Matched        Style `json:"Matched"`
}

// Style describes termbox styles
type Style struct {
	fg termbox.Attribute
	bg termbox.Attribute
}

// CtxOptions is the interface that defines that options can be
// passed in from the command line
type CtxOptions interface {
	// EnableNullSep should return if the null separator is
	// enabled (--null)
	EnableNullSep() bool

	// BufferSize should return the buffer size. By default (i.e.
	// when it returns 0), the buffer size is unlimited.
	// (--buffer-size)
	BufferSize() int

	// InitialIndex is the line number to put the cursor on
	// when peco starts
	InitialIndex() int

	// LayoutType returns the name of the layout to use
	LayoutType() string
}

type PageInfo struct {
	page    int
	offset  int
	perPage int
	total   int
	maxPage int
}

type FilterQuery struct {
	query      []rune
	savedQuery []rune
	mutex      sync.Locker
}

// Ctx contains all the important data. while you can easily access
// data in this struct from anywhere, only do so via channels
type Ctx struct {
	*Hub
	*FilterQuery
	filters             FilterSet
	caretPosition       int
	enableSep           bool
	resultCh            chan Line
	mutex               sync.Locker
	currentLine         int
	currentCol          int
	currentPage         *PageInfo
	selection           *Selection
	activeLineBuffer    LineBuffer
	rawLineBuffer       *RawLineBuffer
	lines               []Line
	linesMutex          sync.Locker
	current             []Line
	currentMutex        sync.Locker
	bufferSize          int
	config              *Config
	selectionRangeStart int
	layoutType          string
	singleKeyJumpMode   bool
	select1             bool

	wait *sync.WaitGroup
	err  error
}

type CLIOptions struct {
	OptHelp           bool   `short:"h" long:"help" description:"show this help message and exit"`
	OptTTY            string `long:"tty" description:"path to the TTY (usually, the value of $TTY)"`
	OptQuery          string `long:"query" description:"initial value for query"`
	OptRcfile         string `long:"rcfile" description:"path to the settings file"`
	OptVersion        bool   `long:"version" description:"print the version and exit"`
	OptBufferSize     int    `long:"buffer-size" short:"b" description:"number of lines to keep in search buffer"`
	OptEnableNullSep  bool   `long:"null" description:"expect NUL (\\0) as separator for target/output"`
	OptInitialIndex   int    `long:"initial-index" description:"position of the initial index of the selection (0 base)"`
	OptInitialMatcher string `long:"initial-matcher" description:"specify the default matcher (deprecated)"`
	OptInitialFilter  string `long:"initial-filter" description:"specify the default filter"`
	OptPrompt         string `long:"prompt" description:"specify the prompt string"`
	OptLayout         string `long:"layout" description:"layout to be used 'top-down' (default) or 'bottom-up'" default:"top-down"`
	OptSelect1        bool   `long:"select-1" description:"select first item and immediately exit if the input contains only 1 item"`
}

type CLI struct {
}
<|endoftext|>


<commit_before>package nativebase

import (
	"math"
	"functorama.com/demo/base"
)

// Basis for all native numerics
type NativeBaseNumerics struct {
	base.BaseNumerics

	RealMin float64
	RealMax float64
	ImagMin float64
	ImagMax float64

	Runit float64
	Iunit float64

	SqrtDivergeLimit float64
}

func CreateNativeBaseNumerics(app RenderApplication) NativeBaseNumerics {
	planeMin, planeMax := app.NativeUserCoords()
	planeWidth := real(planeMax) - real(planeMin)
	planeHeight := imag(planeMax) - imag(planeMin)
	planeAspect := planeWidth / planeHeight
	pictureWidth, pictureHeight := app.PictureDimensions()
	pictureAspect := base.PictureAspectRatio(pictureWidth, pictureHeight)
	config := app.BaseConfig()

	if config.FixAspect {
		// If the plane aspect is greater than image aspect
		// Then the plane is too short, so must be made taller
		if planeAspect > pictureAspect {
			taller := planeWidth / pictureAspect
			bottom := imag(planeMax) - taller
			planeMin = complex(real(planeMin), bottom)
			planeHeight = imag(planeMax) - bottom
		} else if planeAspect < pictureAspect {
			// If the plane aspect is less than the image aspect
			// Then the plane is too thin, and must be made fatter
			fatter := planeHeight * pictureAspect
			right := real(planeMin) + fatter
			planeMax = complex(right, imag(planeMax))
			planeWidth = right - real(planeMin)
		}
	}

	uq := UnitQuery{pictureWidth, pictureHeight, planeWidth, planeHeight}
	rUnit, iUnit := uq.PixelUnits()

	return NativeBaseNumerics{
		BaseNumerics: base.CreateBaseNumerics(app),
		RealMin:      real(planeMin),
		RealMax:      real(planeMax),
		ImagMin:      imag(planeMin),
		ImagMax:      imag(planeMax),

		SqrtDivergeLimit: math.Sqrt(config.DivergeLimit),

		Runit: rUnit,
		Iunit: iUnit,
	}
}

func (nbn *NativeBaseNumerics) CreateMandelbrot(c complex128) NativeMandelbrotMember {
	return NativeMandelbrotMember{
		C: c,
		SqrtDivergeLimit: nbn.SqrtDivergeLimit,
	}
}

// Size on the plane of 1px
func (nbn *NativeBaseNumerics) PixelSize() (float64, float64) {
	return nbn.Runit, nbn.Iunit
}

func (nbn *NativeBaseNumerics) PlaneToPixel(c complex128) (rx int, ry int) {
	rUnit, iUnit := nbn.PixelSize()
	// Translate x
	tx := real(c) - nbn.RealMin

	// Translate y
	ty := nbn.ImagMax - imag(c)

	// Scale x
	sx := tx / rUnit
	// Scale y
	sy := ty / iUnit

	rx = int(math.Floor(sx))
	// Remember that we draw downwards
	ry = int(math.Ceil(sy))

	return
}

type UnitQuery struct {
	pictureW uint
	pictureH uint
	planeW float64
	planeH float64
}

func (uq UnitQuery) PixelUnits() (float64, float64) {
	rUnit := uq.planeW / float64(uq.pictureW)
	iUnit := uq.planeH / float64(uq.pictureH)
	return rUnit, iUnit
}<commit_msg>UnitQuery exposes its fields<commit_after>package nativebase

import (
	"math"
	"functorama.com/demo/base"
)

// Basis for all native numerics
type NativeBaseNumerics struct {
	base.BaseNumerics

	RealMin float64
	RealMax float64
	ImagMin float64
	ImagMax float64

	Runit float64
	Iunit float64

	SqrtDivergeLimit float64
}

func CreateNativeBaseNumerics(app RenderApplication) NativeBaseNumerics {
	planeMin, planeMax := app.NativeUserCoords()
	planeWidth := real(planeMax) - real(planeMin)
	planeHeight := imag(planeMax) - imag(planeMin)
	planeAspect := planeWidth / planeHeight
	pictureWidth, pictureHeight := app.PictureDimensions()
	pictureAspect := base.PictureAspectRatio(pictureWidth, pictureHeight)
	config := app.BaseConfig()

	if config.FixAspect {
		// If the plane aspect is greater than image aspect
		// Then the plane is too short, so must be made taller
		if planeAspect > pictureAspect {
			taller := planeWidth / pictureAspect
			bottom := imag(planeMax) - taller
			planeMin = complex(real(planeMin), bottom)
			planeHeight = imag(planeMax) - bottom
		} else if planeAspect < pictureAspect {
			// If the plane aspect is less than the image aspect
			// Then the plane is too thin, and must be made fatter
			fatter := planeHeight * pictureAspect
			right := real(planeMin) + fatter
			planeMax = complex(right, imag(planeMax))
			planeWidth = right - real(planeMin)
		}
	}

	uq := UnitQuery{pictureWidth, pictureHeight, planeWidth, planeHeight}
	rUnit, iUnit := uq.PixelUnits()

	return NativeBaseNumerics{
		BaseNumerics: base.CreateBaseNumerics(app),
		RealMin:      real(planeMin),
		RealMax:      real(planeMax),
		ImagMin:      imag(planeMin),
		ImagMax:      imag(planeMax),

		SqrtDivergeLimit: math.Sqrt(config.DivergeLimit),

		Runit: rUnit,
		Iunit: iUnit,
	}
}

func (nbn *NativeBaseNumerics) CreateMandelbrot(c complex128) NativeMandelbrotMember {
	return NativeMandelbrotMember{
		C: c,
		SqrtDivergeLimit: nbn.SqrtDivergeLimit,
	}
}

// Size on the plane of 1px
func (nbn *NativeBaseNumerics) PixelSize() (float64, float64) {
	return nbn.Runit, nbn.Iunit
}

func (nbn *NativeBaseNumerics) PlaneToPixel(c complex128) (rx int, ry int) {
	rUnit, iUnit := nbn.PixelSize()
	// Translate x
	tx := real(c) - nbn.RealMin

	// Translate y
	ty := nbn.ImagMax - imag(c)

	// Scale x
	sx := tx / rUnit
	// Scale y
	sy := ty / iUnit

	rx = int(math.Floor(sx))
	// Remember that we draw downwards
	ry = int(math.Ceil(sy))

	return
}

type UnitQuery struct {
	PictureW uint
	PictureH uint
	PlaneW float64
	PlaneH float64
}

func (uq UnitQuery) PixelUnits() (float64, float64) {
	rUnit := uq.PlaneW / float64(uq.PictureW)
	iUnit := uq.PlaneH / float64(uq.PictureH)
	return rUnit, iUnit
}<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd.

// Package lintapp implements the go-lint.appspot.com server.
package lintapp

import (
	"bytes"
	"encoding/gob"
	"fmt"
	"html/template"
	"net/http"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"appengine"
	"appengine/datastore"
	"appengine/urlfetch"

	"github.com/golang/gddo/gosrc"
	"github.com/golang/lint"
)

func init() {
	http.Handle("/", handlerFunc(serveRoot))
	http.Handle("/-/bot", handlerFunc(serveBot))
	http.Handle("/-/refresh", handlerFunc(serveRefresh))
}

var (
	contactEmail    = "info@godoc.org"
	homeTemplate    = parseTemplate("common.html", "index.html")
	packageTemplate = parseTemplate("common.html", "package.html")
	errorTemplate   = parseTemplate("common.html", "error.html")
	templateFuncs   = template.FuncMap{
		"timeago":      timeagoFn,
		"contactEmail": contactEmailFn,
	}
	gitHubCredentials = ""
)

func parseTemplate(fnames ...string) *template.Template {
	paths := make([]string, len(fnames))
	for i := range fnames {
		paths[i] = filepath.Join("assets/templates", fnames[i])
	}
	t, err := template.New("").Funcs(templateFuncs).ParseFiles(paths...)
	if err != nil {
		panic(err)
	}
	t = t.Lookup("ROOT")
	if t == nil {
		panic(fmt.Sprintf("ROOT template not found in %v", fnames))
	}
	return t
}

func contactEmailFn() string {
	return contactEmail
}

func timeagoFn(t time.Time) string {
	d := time.Since(t)
	switch {
	case d < time.Second:
		return "just now"
	case d < 2*time.Second:
		return "one second ago"
	case d < time.Minute:
		return fmt.Sprintf("%d seconds ago", d/time.Second)
	case d < 2*time.Minute:
		return "one minute ago"
	case d < time.Hour:
		return fmt.Sprintf("%d minutes ago", d/time.Minute)
	case d < 2*time.Hour:
		return "one hour ago"
	case d < 48*time.Hour:
		return fmt.Sprintf("%d hours ago", d/time.Hour)
	default:
		return fmt.Sprintf("%d days ago", d/(time.Hour*24))
	}
}

func writeResponse(w http.ResponseWriter, status int, t *template.Template, v interface{}) error {
	var buf bytes.Buffer
	if err := t.Execute(&buf, v); err != nil {
		return err
	}
	w.Header().Set("Content-Type", "text/html; charset=utf-8")
	w.Header().Set("Content-Length", strconv.Itoa(buf.Len()))
	w.WriteHeader(status)
	_, err := w.Write(buf.Bytes())
	return err
}

func writeErrorResponse(w http.ResponseWriter, status int) error {
	return writeResponse(w, status, errorTemplate, http.StatusText(status))
}

type transport struct {
	rt http.RoundTripper
	ua string
}

func (t transport) RoundTrip(r *http.Request) (*http.Response, error) {
	r.Header.Set("User-Agent", t.ua)
	if r.URL.Host == "api.github.com" && gitHubCredentials != "" {
		if r.URL.RawQuery == "" {
			r.URL.RawQuery = gitHubCredentials
		} else {
			r.URL.RawQuery += "&" + gitHubCredentials
		}
	}
	return t.rt.RoundTrip(r)
}

func httpClient(r *http.Request) *http.Client {
	c := appengine.NewContext(r)
	return &http.Client{
		Transport: &transport{
			rt: &urlfetch.Transport{Context: c, Deadline: 10 * time.Second},
			ua: fmt.Sprintf("%s (+http://%s/-/bot)", appengine.AppID(c), r.Host),
		},
	}
}

const version = 1

type storePackage struct {
	Data    []byte
	Version int
}

type lintPackage struct {
	Files   []*lintFile
	Path    string
	Updated time.Time
	LineFmt string
	URL     string
}

type lintFile struct {
	Name     string
	Problems []*lintProblem
	URL      string
}

type lintProblem struct {
	Line       int
	Text       string
	LineText   string
	Confidence float64
	Link       string
}

func putPackage(c appengine.Context, importPath string, pkg *lintPackage) error {
	var buf bytes.Buffer
	if err := gob.NewEncoder(&buf).Encode(pkg); err != nil {
		return err
	}
	_, err := datastore.Put(c,
		datastore.NewKey(c, "Package", importPath, 0, nil),
		&storePackage{Data: buf.Bytes(), Version: version})
	return err
}

func getPackage(c appengine.Context, importPath string) (*lintPackage, error) {
	var spkg storePackage
	if err := datastore.Get(c, datastore.NewKey(c, "Package", importPath, 0, nil), &spkg); err != nil {
		if err == datastore.ErrNoSuchEntity {
			err = nil
		}
		return nil, err
	}
	if spkg.Version != version {
		return nil, nil
	}
	var pkg lintPackage
	if err := gob.NewDecoder(bytes.NewReader(spkg.Data)).Decode(&pkg); err != nil {
		return nil, err
	}
	return &pkg, nil
}

func runLint(r *http.Request, importPath string) (*lintPackage, error) {
	dir, err := gosrc.Get(httpClient(r), importPath, "")
	if err != nil {
		return nil, err
	}

	pkg := lintPackage{
		Path:    importPath,
		Updated: time.Now(),
		LineFmt: dir.LineFmt,
		URL:     dir.BrowseURL,
	}
	linter := lint.Linter{}
	for _, f := range dir.Files {
		if !strings.HasSuffix(f.Name, ".go") {
			continue
		}
		problems, err := linter.Lint(f.Name, f.Data)
		if err == nil && len(problems) == 0 {
			continue
		}
		file := lintFile{Name: f.Name, URL: f.BrowseURL}
		if err != nil {
			file.Problems = []*lintProblem{{Text: err.Error()}}
		} else {
			for _, p := range problems {
				file.Problems = append(file.Problems, &lintProblem{
					Line:       p.Position.Line,
					Text:       p.Text,
					LineText:   p.LineText,
					Confidence: p.Confidence,
					Link:       p.Link,
				})
			}
		}
		if len(file.Problems) > 0 {
			pkg.Files = append(pkg.Files, &file)
		}
	}

	if err := putPackage(appengine.NewContext(r), importPath, &pkg); err != nil {
		return nil, err
	}

	return &pkg, nil
}

func filterByConfidence(r *http.Request, pkg *lintPackage) {
	minConfidence, err := strconv.ParseFloat(r.FormValue("minConfidence"), 64)
	if err != nil {
		minConfidence = 0.8
	}
	for _, f := range pkg.Files {
		j := 0
		for i := range f.Problems {
			if f.Problems[i].Confidence >= minConfidence {
				f.Problems[j] = f.Problems[i]
				j += 1
			}
		}
		f.Problems = f.Problems[:j]
	}
}

type handlerFunc func(http.ResponseWriter, *http.Request) error

func (f handlerFunc) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	err := f(w, r)
	if err == nil {
		return
	} else if gosrc.IsNotFound(err) {
		writeErrorResponse(w, 404)
	} else if e, ok := err.(*gosrc.RemoteError); ok {
		c.Infof("Remote error %s: %v", e.Host, e)
		writeResponse(w, 500, errorTemplate, fmt.Sprintf("Error accessing %s.", e.Host))
	} else if err != nil {
		c.Errorf("Internal error %v", err)
		writeErrorResponse(w, 500)
	}
}

func serveRoot(w http.ResponseWriter, r *http.Request) error {
	switch {
	case r.Method != "GET" && r.Method != "HEAD":
		return writeErrorResponse(w, 405)
	case r.URL.Path == "/":
		return writeResponse(w, 200, homeTemplate, nil)
	default:
		importPath := r.URL.Path[1:]
		if !gosrc.IsValidPath(importPath) {
			return gosrc.NotFoundError{Message: "bad path"}
		}
		c := appengine.NewContext(r)
		pkg, err := getPackage(c, importPath)
		if pkg == nil && err == nil {
			pkg, err = runLint(r, importPath)
		}
		if err != nil {
			return err
		}
		filterByConfidence(r, pkg)
		return writeResponse(w, 200, packageTemplate, pkg)
	}
}

func serveRefresh(w http.ResponseWriter, r *http.Request) error {
	if r.Method != "POST" {
		return writeErrorResponse(w, 405)
	}
	importPath := r.FormValue("importPath")
	pkg, err := runLint(r, importPath)
	if err != nil {
		return err
	}
	http.Redirect(w, r, "/"+pkg.Path, 301)
	return nil
}

func serveBot(w http.ResponseWriter, r *http.Request) error {
	c := appengine.NewContext(r)
	_, err := fmt.Fprintf(w, "Contact %s for help with the %s bot.", contactEmail, appengine.AppID(c))
	return err
}
<commit_msg>fix contact email<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd.

// Package lintapp implements the go-lint.appspot.com server.
package lintapp

import (
	"bytes"
	"encoding/gob"
	"fmt"
	"html/template"
	"net/http"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"appengine"
	"appengine/datastore"
	"appengine/urlfetch"

	"github.com/golang/gddo/gosrc"
	"github.com/golang/lint"
)

func init() {
	http.Handle("/", handlerFunc(serveRoot))
	http.Handle("/-/bot", handlerFunc(serveBot))
	http.Handle("/-/refresh", handlerFunc(serveRefresh))
}

var (
	contactEmail    = "golang-dev@googlegroups.com"
	homeTemplate    = parseTemplate("common.html", "index.html")
	packageTemplate = parseTemplate("common.html", "package.html")
	errorTemplate   = parseTemplate("common.html", "error.html")
	templateFuncs   = template.FuncMap{
		"timeago":      timeagoFn,
		"contactEmail": contactEmailFn,
	}
	gitHubCredentials = ""
)

func parseTemplate(fnames ...string) *template.Template {
	paths := make([]string, len(fnames))
	for i := range fnames {
		paths[i] = filepath.Join("assets/templates", fnames[i])
	}
	t, err := template.New("").Funcs(templateFuncs).ParseFiles(paths...)
	if err != nil {
		panic(err)
	}
	t = t.Lookup("ROOT")
	if t == nil {
		panic(fmt.Sprintf("ROOT template not found in %v", fnames))
	}
	return t
}

func contactEmailFn() string {
	return contactEmail
}

func timeagoFn(t time.Time) string {
	d := time.Since(t)
	switch {
	case d < time.Second:
		return "just now"
	case d < 2*time.Second:
		return "one second ago"
	case d < time.Minute:
		return fmt.Sprintf("%d seconds ago", d/time.Second)
	case d < 2*time.Minute:
		return "one minute ago"
	case d < time.Hour:
		return fmt.Sprintf("%d minutes ago", d/time.Minute)
	case d < 2*time.Hour:
		return "one hour ago"
	case d < 48*time.Hour:
		return fmt.Sprintf("%d hours ago", d/time.Hour)
	default:
		return fmt.Sprintf("%d days ago", d/(time.Hour*24))
	}
}

func writeResponse(w http.ResponseWriter, status int, t *template.Template, v interface{}) error {
	var buf bytes.Buffer
	if err := t.Execute(&buf, v); err != nil {
		return err
	}
	w.Header().Set("Content-Type", "text/html; charset=utf-8")
	w.Header().Set("Content-Length", strconv.Itoa(buf.Len()))
	w.WriteHeader(status)
	_, err := w.Write(buf.Bytes())
	return err
}

func writeErrorResponse(w http.ResponseWriter, status int) error {
	return writeResponse(w, status, errorTemplate, http.StatusText(status))
}

type transport struct {
	rt http.RoundTripper
	ua string
}

func (t transport) RoundTrip(r *http.Request) (*http.Response, error) {
	r.Header.Set("User-Agent", t.ua)
	if r.URL.Host == "api.github.com" && gitHubCredentials != "" {
		if r.URL.RawQuery == "" {
			r.URL.RawQuery = gitHubCredentials
		} else {
			r.URL.RawQuery += "&" + gitHubCredentials
		}
	}
	return t.rt.RoundTrip(r)
}

func httpClient(r *http.Request) *http.Client {
	c := appengine.NewContext(r)
	return &http.Client{
		Transport: &transport{
			rt: &urlfetch.Transport{Context: c, Deadline: 10 * time.Second},
			ua: fmt.Sprintf("%s (+http://%s/-/bot)", appengine.AppID(c), r.Host),
		},
	}
}

const version = 1

type storePackage struct {
	Data    []byte
	Version int
}

type lintPackage struct {
	Files   []*lintFile
	Path    string
	Updated time.Time
	LineFmt string
	URL     string
}

type lintFile struct {
	Name     string
	Problems []*lintProblem
	URL      string
}

type lintProblem struct {
	Line       int
	Text       string
	LineText   string
	Confidence float64
	Link       string
}

func putPackage(c appengine.Context, importPath string, pkg *lintPackage) error {
	var buf bytes.Buffer
	if err := gob.NewEncoder(&buf).Encode(pkg); err != nil {
		return err
	}
	_, err := datastore.Put(c,
		datastore.NewKey(c, "Package", importPath, 0, nil),
		&storePackage{Data: buf.Bytes(), Version: version})
	return err
}

func getPackage(c appengine.Context, importPath string) (*lintPackage, error) {
	var spkg storePackage
	if err := datastore.Get(c, datastore.NewKey(c, "Package", importPath, 0, nil), &spkg); err != nil {
		if err == datastore.ErrNoSuchEntity {
			err = nil
		}
		return nil, err
	}
	if spkg.Version != version {
		return nil, nil
	}
	var pkg lintPackage
	if err := gob.NewDecoder(bytes.NewReader(spkg.Data)).Decode(&pkg); err != nil {
		return nil, err
	}
	return &pkg, nil
}

func runLint(r *http.Request, importPath string) (*lintPackage, error) {
	dir, err := gosrc.Get(httpClient(r), importPath, "")
	if err != nil {
		return nil, err
	}

	pkg := lintPackage{
		Path:    importPath,
		Updated: time.Now(),
		LineFmt: dir.LineFmt,
		URL:     dir.BrowseURL,
	}
	linter := lint.Linter{}
	for _, f := range dir.Files {
		if !strings.HasSuffix(f.Name, ".go") {
			continue
		}
		problems, err := linter.Lint(f.Name, f.Data)
		if err == nil && len(problems) == 0 {
			continue
		}
		file := lintFile{Name: f.Name, URL: f.BrowseURL}
		if err != nil {
			file.Problems = []*lintProblem{{Text: err.Error()}}
		} else {
			for _, p := range problems {
				file.Problems = append(file.Problems, &lintProblem{
					Line:       p.Position.Line,
					Text:       p.Text,
					LineText:   p.LineText,
					Confidence: p.Confidence,
					Link:       p.Link,
				})
			}
		}
		if len(file.Problems) > 0 {
			pkg.Files = append(pkg.Files, &file)
		}
	}

	if err := putPackage(appengine.NewContext(r), importPath, &pkg); err != nil {
		return nil, err
	}

	return &pkg, nil
}

func filterByConfidence(r *http.Request, pkg *lintPackage) {
	minConfidence, err := strconv.ParseFloat(r.FormValue("minConfidence"), 64)
	if err != nil {
		minConfidence = 0.8
	}
	for _, f := range pkg.Files {
		j := 0
		for i := range f.Problems {
			if f.Problems[i].Confidence >= minConfidence {
				f.Problems[j] = f.Problems[i]
				j += 1
			}
		}
		f.Problems = f.Problems[:j]
	}
}

type handlerFunc func(http.ResponseWriter, *http.Request) error

func (f handlerFunc) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	err := f(w, r)
	if err == nil {
		return
	} else if gosrc.IsNotFound(err) {
		writeErrorResponse(w, 404)
	} else if e, ok := err.(*gosrc.RemoteError); ok {
		c.Infof("Remote error %s: %v", e.Host, e)
		writeResponse(w, 500, errorTemplate, fmt.Sprintf("Error accessing %s.", e.Host))
	} else if err != nil {
		c.Errorf("Internal error %v", err)
		writeErrorResponse(w, 500)
	}
}

func serveRoot(w http.ResponseWriter, r *http.Request) error {
	switch {
	case r.Method != "GET" && r.Method != "HEAD":
		return writeErrorResponse(w, 405)
	case r.URL.Path == "/":
		return writeResponse(w, 200, homeTemplate, nil)
	default:
		importPath := r.URL.Path[1:]
		if !gosrc.IsValidPath(importPath) {
			return gosrc.NotFoundError{Message: "bad path"}
		}
		c := appengine.NewContext(r)
		pkg, err := getPackage(c, importPath)
		if pkg == nil && err == nil {
			pkg, err = runLint(r, importPath)
		}
		if err != nil {
			return err
		}
		filterByConfidence(r, pkg)
		return writeResponse(w, 200, packageTemplate, pkg)
	}
}

func serveRefresh(w http.ResponseWriter, r *http.Request) error {
	if r.Method != "POST" {
		return writeErrorResponse(w, 405)
	}
	importPath := r.FormValue("importPath")
	pkg, err := runLint(r, importPath)
	if err != nil {
		return err
	}
	http.Redirect(w, r, "/"+pkg.Path, 301)
	return nil
}

func serveBot(w http.ResponseWriter, r *http.Request) error {
	c := appengine.NewContext(r)
	_, err := fmt.Fprintf(w, "Contact %s for help with the %s bot.", contactEmail, appengine.AppID(c))
	return err
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/fsouza/go-dockerclient"

	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
)

type Image struct {
	Id          string
	ParentId    string   `json:",omitempty"`
	RepoTags    []string `json:",omitempty"`
	VirtualSize int64
	Size        int64
	Created     int64
}

type ImagesCommand struct {
	Dot        bool `short:"d" long:"dot" description:"Show image information as Graphviz dot."`
	Tree       bool `short:"t" long:"tree" description:"Show image information as tree."`
	Short      bool `short:"s" long:"short" description:"Show short summary of images (repo name and list of tags)."`
	NoTruncate bool `short:"n" long:"no-trunc" description:"Don't truncate the image IDs."`
}

var imagesCommand ImagesCommand

func (x *ImagesCommand) Execute(args []string) error {

	var images *[]Image

	stat, err := os.Stdin.Stat()
	if err != nil {
		return fmt.Errorf("error reading stdin stat", err)
	}

	if (stat.Mode() & os.ModeCharDevice) == 0 {
		// read in stdin
		stdin, err := ioutil.ReadAll(os.Stdin)
		if err != nil {
			return fmt.Errorf("error reading all input", err)
		}

		images, err = parseImagesJSON(stdin)
		if err != nil {
			return err
		}

	} else {

		client, err := connect()
		if err != nil {
			return err
		}

		clientImages, err := client.ListImages(docker.ListImagesOptions{All: true})
		if err != nil {
			return err
		}

		var ims []Image
		for _, image := range clientImages {
			// fmt.Println(image)
			ims = append(ims, Image{
				image.ID,
				image.ParentID,
				image.RepoTags,
				image.VirtualSize,
				image.Size,
				image.Created,
			})
		}

		images = &ims
	}

	if imagesCommand.Dot {
		fmt.Printf(jsonToDot(images))
	} else if imagesCommand.Tree {

		var startImageArg = ""
		if len(args) > 0 {
			startImageArg = args[0]
		}

		fmt.Printf(jsonToTree(images, startImageArg, imagesCommand.NoTruncate))
	} else if imagesCommand.Short {
		fmt.Printf(jsonToShort(images))
	} else {
		return fmt.Errorf("Please specify either --dot, --tree, or --short")
	}

	return nil
}

func jsonToTree(images *[]Image, startImageArg string, noTrunc bool) string {
	var buffer bytes.Buffer

	var startImage Image

	var roots []Image
	var byParent = make(map[string][]Image)
	for _, image := range *images {
		if image.ParentId == "" {
			roots = append(roots, image)
		} else {
			if children, exists := byParent[image.ParentId]; exists {
				byParent[image.ParentId] = append(children, image)
			} else {
				byParent[image.ParentId] = []Image{image}
			}
		}

		if startImageArg != "" {
			if startImageArg == image.Id || startImageArg == truncate(image.Id) {
				startImage = image
			}

			for _, repotag := range image.RepoTags {
				if repotag == startImageArg {
					startImage = image
				}
			}
		}
	}

	if startImageArg != "" {
		WalkTree(&buffer, noTrunc, []Image{startImage}, byParent, "")
	} else {
		WalkTree(&buffer, noTrunc, roots, byParent, "")
	}

	return buffer.String()
}

func WalkTree(buffer *bytes.Buffer, noTrunc bool, images []Image, byParent map[string][]Image, prefix string) {
	if len(images) > 1 {
		length := len(images)
		for index, image := range images {
			if index+1 == length {
				PrintTreeNode(buffer, noTrunc, image, prefix+"└─")
				if subimages, exists := byParent[image.Id]; exists {
					WalkTree(buffer, noTrunc, subimages, byParent, prefix+"  ")
				}
			} else {
				PrintTreeNode(buffer, noTrunc, image, prefix+"├─")
				if subimages, exists := byParent[image.Id]; exists {
					WalkTree(buffer, noTrunc, subimages, byParent, prefix+"│ ")
				}
			}
		}
	} else {
		for _, image := range images {
			PrintTreeNode(buffer, noTrunc, image, prefix+"└─")
			if subimages, exists := byParent[image.Id]; exists {
				WalkTree(buffer, noTrunc, subimages, byParent, prefix+"  ")
			}
		}
	}
}

func PrintTreeNode(buffer *bytes.Buffer, noTrunc bool, image Image, prefix string) {
	var imageID string
	if noTrunc {
		imageID = image.Id
	} else {
		imageID = truncate(image.Id)
	}

	buffer.WriteString(fmt.Sprintf("%s%s Virtual Size: %s", prefix, imageID, humanSize(image.VirtualSize)))
	if image.RepoTags[0] != "<none>:<none>" {
		buffer.WriteString(fmt.Sprintf(" Tags: %s\n", strings.Join(image.RepoTags, ", ")))
	} else {
		buffer.WriteString(fmt.Sprintf("\n"))
	}
}

func humanSize(raw int64) string {
	sizes := []string{"B", "KB", "MB", "GB", "TB"}

	rawFloat := float64(raw)
	ind := 0

	for {
		if rawFloat < 1000 {
			break
		} else {
			rawFloat = rawFloat / 1000
			ind = ind + 1
		}
	}

	return fmt.Sprintf("%.01f %s", rawFloat, sizes[ind])
}

func truncate(id string) string {
	return id[0:12]
}

func parseImagesJSON(rawJSON []byte) (*[]Image, error) {

	var images []Image
	err := json.Unmarshal(rawJSON, &images)

	if err != nil {
		return nil, fmt.Errorf("Error reading JSON: ", err)
	}

	return &images, nil
}

func jsonToDot(images *[]Image) string {

	var buffer bytes.Buffer
	buffer.WriteString("digraph docker {\n")

	for _, image := range *images {
		if image.ParentId == "" {
			buffer.WriteString(fmt.Sprintf(" base -> \"%s\" [style=invis]\n", truncate(image.Id)))
		} else {
			buffer.WriteString(fmt.Sprintf(" \"%s\" -> \"%s\"\n", truncate(image.ParentId), truncate(image.Id)))
		}
		if image.RepoTags[0] != "<none>:<none>" {
			buffer.WriteString(fmt.Sprintf(" \"%s\" [label=\"%s\\n%s\",shape=box,fillcolor=\"paleturquoise\",style=\"filled,rounded\"];\n", truncate(image.Id), truncate(image.Id), strings.Join(image.RepoTags, "\\n")))
		}
	}

	buffer.WriteString(" base [style=invisible]\n}\n")

	return buffer.String()
}

func jsonToShort(images *[]Image) string {
	var buffer bytes.Buffer

	var byRepo = make(map[string][]string)

	for _, image := range *images {
		for _, repotag := range image.RepoTags {
			if repotag != "<none>:<none>" {

				// parse the repo name and tag name out
				// tag is after the last colon
				lastColonIndex := strings.LastIndex(repotag, ":")
				tagname := repotag[lastColonIndex+1:]
				reponame := repotag[0:lastColonIndex]

				if tags, exists := byRepo[reponame]; exists {
					byRepo[reponame] = append(tags, tagname)
				} else {
					byRepo[reponame] = []string{tagname}
				}
			}
		}
	}

	for repo, tags := range byRepo {
		buffer.WriteString(fmt.Sprintf("%s: %s\n", repo, strings.Join(tags, ", ")))
	}

	return buffer.String()
}

func init() {
	parser.AddCommand("images",
		"Visualize docker images.",
		"",
		&imagesCommand)
}
<commit_msg>support passing repo name as starting point for -t<commit_after>package main

import (
	"github.com/fsouza/go-dockerclient"

	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
)

type Image struct {
	Id          string
	ParentId    string   `json:",omitempty"`
	RepoTags    []string `json:",omitempty"`
	VirtualSize int64
	Size        int64
	Created     int64
}

type ImagesCommand struct {
	Dot        bool `short:"d" long:"dot" description:"Show image information as Graphviz dot."`
	Tree       bool `short:"t" long:"tree" description:"Show image information as tree."`
	Short      bool `short:"s" long:"short" description:"Show short summary of images (repo name and list of tags)."`
	NoTruncate bool `short:"n" long:"no-trunc" description:"Don't truncate the image IDs."`
}

var imagesCommand ImagesCommand

func (x *ImagesCommand) Execute(args []string) error {

	var images *[]Image

	stat, err := os.Stdin.Stat()
	if err != nil {
		return fmt.Errorf("error reading stdin stat", err)
	}

	if (stat.Mode() & os.ModeCharDevice) == 0 {
		// read in stdin
		stdin, err := ioutil.ReadAll(os.Stdin)
		if err != nil {
			return fmt.Errorf("error reading all input", err)
		}

		images, err = parseImagesJSON(stdin)
		if err != nil {
			return err
		}

	} else {

		client, err := connect()
		if err != nil {
			return err
		}

		clientImages, err := client.ListImages(docker.ListImagesOptions{All: true})
		if err != nil {
			return err
		}

		var ims []Image
		for _, image := range clientImages {
			// fmt.Println(image)
			ims = append(ims, Image{
				image.ID,
				image.ParentID,
				image.RepoTags,
				image.VirtualSize,
				image.Size,
				image.Created,
			})
		}

		images = &ims
	}

	if imagesCommand.Dot {
		fmt.Printf(jsonToDot(images))
	} else if imagesCommand.Tree {

		var startImage = ""
		if len(args) > 0 {

			// attempt to find the start image, which can be specified as an
			// image ID or a repository name

			startImageArg := args[0]
			startImageRepo := args[0]

			// in case a repo name was specified, append ":latest" if it isn't
			// already there
			if !strings.HasSuffix(startImageRepo, ":latest") {
				startImageRepo = fmt.Sprintf("%s:latest", startImageRepo)
			}

		IMAGES:
			for _, image := range *images {
				// check if the start image arg matches an image id
				if strings.Index(image.Id, startImageArg) == 0 {
					startImage = startImageArg
					break IMAGES
				}

				// check if the start image arg matches an repository name
				if image.RepoTags[0] != "<none>:<none>" {
					for _, repotag := range image.RepoTags {
						if repotag == startImageRepo {
							startImage = image.Id
							break IMAGES
						}
					}
				}
			}

			if startImage == "" {
				return fmt.Errorf("Unable to find image %s.", startImageArg)
			}
		}

		fmt.Printf(jsonToTree(images, startImage, imagesCommand.NoTruncate))
	} else if imagesCommand.Short {
		fmt.Printf(jsonToShort(images))
	} else {
		return fmt.Errorf("Please specify either --dot, --tree, or --short")
	}

	return nil
}

func jsonToTree(images *[]Image, startImageArg string, noTrunc bool) string {
	var buffer bytes.Buffer

	var startImage Image

	var roots []Image
	var byParent = make(map[string][]Image)
	for _, image := range *images {
		if image.ParentId == "" {
			roots = append(roots, image)
		} else {
			if children, exists := byParent[image.ParentId]; exists {
				byParent[image.ParentId] = append(children, image)
			} else {
				byParent[image.ParentId] = []Image{image}
			}
		}

		if startImageArg != "" {
			if startImageArg == image.Id || startImageArg == truncate(image.Id) {
				startImage = image
			}

			for _, repotag := range image.RepoTags {
				if repotag == startImageArg {
					startImage = image
				}
			}
		}
	}

	if startImageArg != "" {
		WalkTree(&buffer, noTrunc, []Image{startImage}, byParent, "")
	} else {
		WalkTree(&buffer, noTrunc, roots, byParent, "")
	}

	return buffer.String()
}

func WalkTree(buffer *bytes.Buffer, noTrunc bool, images []Image, byParent map[string][]Image, prefix string) {
	if len(images) > 1 {
		length := len(images)
		for index, image := range images {
			if index+1 == length {
				PrintTreeNode(buffer, noTrunc, image, prefix+"└─")
				if subimages, exists := byParent[image.Id]; exists {
					WalkTree(buffer, noTrunc, subimages, byParent, prefix+"  ")
				}
			} else {
				PrintTreeNode(buffer, noTrunc, image, prefix+"├─")
				if subimages, exists := byParent[image.Id]; exists {
					WalkTree(buffer, noTrunc, subimages, byParent, prefix+"│ ")
				}
			}
		}
	} else {
		for _, image := range images {
			PrintTreeNode(buffer, noTrunc, image, prefix+"└─")
			if subimages, exists := byParent[image.Id]; exists {
				WalkTree(buffer, noTrunc, subimages, byParent, prefix+"  ")
			}
		}
	}
}

func PrintTreeNode(buffer *bytes.Buffer, noTrunc bool, image Image, prefix string) {
	var imageID string
	if noTrunc {
		imageID = image.Id
	} else {
		imageID = truncate(image.Id)
	}

	buffer.WriteString(fmt.Sprintf("%s%s Virtual Size: %s", prefix, imageID, humanSize(image.VirtualSize)))
	if image.RepoTags[0] != "<none>:<none>" {
		buffer.WriteString(fmt.Sprintf(" Tags: %s\n", strings.Join(image.RepoTags, ", ")))
	} else {
		buffer.WriteString(fmt.Sprintf("\n"))
	}
}

func humanSize(raw int64) string {
	sizes := []string{"B", "KB", "MB", "GB", "TB"}

	rawFloat := float64(raw)
	ind := 0

	for {
		if rawFloat < 1000 {
			break
		} else {
			rawFloat = rawFloat / 1000
			ind = ind + 1
		}
	}

	return fmt.Sprintf("%.01f %s", rawFloat, sizes[ind])
}

func truncate(id string) string {
	return id[0:12]
}

func parseImagesJSON(rawJSON []byte) (*[]Image, error) {

	var images []Image
	err := json.Unmarshal(rawJSON, &images)

	if err != nil {
		return nil, fmt.Errorf("Error reading JSON: ", err)
	}

	return &images, nil
}

func jsonToDot(images *[]Image) string {

	var buffer bytes.Buffer
	buffer.WriteString("digraph docker {\n")

	for _, image := range *images {
		if image.ParentId == "" {
			buffer.WriteString(fmt.Sprintf(" base -> \"%s\" [style=invis]\n", truncate(image.Id)))
		} else {
			buffer.WriteString(fmt.Sprintf(" \"%s\" -> \"%s\"\n", truncate(image.ParentId), truncate(image.Id)))
		}
		if image.RepoTags[0] != "<none>:<none>" {
			buffer.WriteString(fmt.Sprintf(" \"%s\" [label=\"%s\\n%s\",shape=box,fillcolor=\"paleturquoise\",style=\"filled,rounded\"];\n", truncate(image.Id), truncate(image.Id), strings.Join(image.RepoTags, "\\n")))
		}
	}

	buffer.WriteString(" base [style=invisible]\n}\n")

	return buffer.String()
}

func jsonToShort(images *[]Image) string {
	var buffer bytes.Buffer

	var byRepo = make(map[string][]string)

	for _, image := range *images {
		for _, repotag := range image.RepoTags {
			if repotag != "<none>:<none>" {

				// parse the repo name and tag name out
				// tag is after the last colon
				lastColonIndex := strings.LastIndex(repotag, ":")
				tagname := repotag[lastColonIndex+1:]
				reponame := repotag[0:lastColonIndex]

				if tags, exists := byRepo[reponame]; exists {
					byRepo[reponame] = append(tags, tagname)
				} else {
					byRepo[reponame] = []string{tagname}
				}
			}
		}
	}

	for repo, tags := range byRepo {
		buffer.WriteString(fmt.Sprintf("%s: %s\n", repo, strings.Join(tags, ", ")))
	}

	return buffer.String()
}

func init() {
	parser.AddCommand("images",
		"Visualize docker images.",
		"",
		&imagesCommand)
}
<|endoftext|>


<commit_before>package backend

import (
	"encoding/xml"
	"fmt"
	"regexp"
	"time"
)

/*
<graphite_rollup>
 	<pattern>
 		<regexp>click_cost</regexp>
 		<function>any</function>
 		<retention>
 			<age>0</age>
 			<precision>3600</precision>
 		</retention>
 		<retention>
 			<age>86400</age>
 			<precision>60</precision>
 		</retention>
 	</pattern>
 	<default>
 		<function>max</function>
 		<retention>
 			<age>0</age>
 			<precision>60</precision>
 		</retention>
 		<retention>
 			<age>3600</age>
 			<precision>300</precision>
 		</retention>
 		<retention>
 			<age>86400</age>
 			<precision>3600</precision>
 		</retention>
 	</default>
</graphite_rollup>
*/

type RollupRetention struct {
	Age       int32 `xml:"age"`
	Precision int32 `xml:"precision"`
}

type RollupRule struct {
	Regexp    string                `xml:"regexp"`
	Function  string                `xml:"function"`
	Retention []*RollupRetention    `xml:"retention"`
	aggr      func([]Point) float64 `xml:"-"`
	re        *regexp.Regexp        `xml:"-"`
}

type Rollup struct {
	Pattern []*RollupRule `xml:"pattern"`
	Default *RollupRule   `xml:"default"`
}

func (rr *RollupRule) compile(hasRegexp bool) error {
	var err error
	if hasRegexp {
		rr.re, err = regexp.Compile(rr.Regexp)
		if err != nil {
			return err
		}
	}

	aggrMap := map[string](func([]Point) float64){
		"avg": aggrAvg,
		"max": aggrMax,
		"min": aggrMin,
		"sum": aggrSum,
		"any": aggrAny,
		"anyLast": aggrAnyLast,
	}

	var exists bool
	rr.aggr, exists = aggrMap[rr.Function]

	if !exists {
		return fmt.Errorf("unknown function %#v", rr.Function)
	}

	return nil
}

func (r *Rollup) compile() error {
	if r.Pattern == nil {
		r.Pattern = make([]*RollupRule, 0)
	}

	if r.Default == nil {
		return fmt.Errorf("default rollup rule not set")
	}

	if err := r.Default.compile(false); err != nil {
		return err
	}

	for _, rr := range r.Pattern {
		if err := rr.compile(true); err != nil {
			return err
		}
	}

	return nil
}

func ParseRollupXML(body []byte) (*Rollup, error) {
	r := &Rollup{}
	err := xml.Unmarshal(body, r)
	if err != nil {
		return nil, err
	}

	err = r.compile()
	if err != nil {
		return nil, err
	}

	return r, nil
}

// PointsCleanup removes points with empty metric
// for run after Deduplicate, Merge, etc for result cleanup
func PointsCleanup(points []Point) []Point {
	l := len(points)
	squashed := 0

	for i := 0; i < l; i++ {
		if points[i].Metric == "" {
			squashed++
			continue
		}
		if squashed > 0 {
			points[i-squashed] = points[i]
		}
	}

	return points[:l-squashed]
}

// PointsUniq removes points with equal metric and time
func PointsUniq(points []Point) []Point {
	l := len(points)
	var i, n int
	// i - current position of iterator
	// n - position on first record with current key (metric + time)

	for i = 1; i < l; i++ {
		if points[i].Metric != points[n].Metric ||
			points[i].Time != points[n].Time {
			n = i
			continue
		}

		if points[i].Timestamp > points[n].Timestamp {
			points[n] = points[i]
		}

		points[i].Metric = "" // mark for remove
	}

	return PointsCleanup(points)
}

// Match returns rollup rules for metric
func (r *Rollup) Match(metric string) *RollupRule {
	for _, rr := range r.Pattern {
		if rr.re.MatchString(metric) {
			return rr
		}
	}

	return r.Default
}

func aggrSum(points []Point) (r float64) {
	for _, p := range points {
		r += p.Value
	}
	return
}

func aggrMax(points []Point) (r float64) {
	if len(points) > 0 {
		r = points[0].Value
	}
	for _, p := range points {
		if p.Value > r {
			r = p.Value
		}
	}
	return
}

func aggrMin(points []Point) (r float64) {
	if len(points) > 0 {
		r = points[0].Value
	}
	for _, p := range points {
		if p.Value < r {
			r = p.Value
		}
	}
	return
}

func aggrAvg(points []Point) (r float64) {
	if len(points) == 0 {
		return
	}
	r = aggrSum(points) / float64(len(points))
	return
}

func aggrAny(points []Point) (r float64) {
	if len(points) > 0 {
		r = points[0].Value
	}
	return
}

func aggrAnyLast(points []Point) (r float64) {
	if len(points) > 0 {
		r = points[len(points)].Value
	}
	return
}

func doMetricPrecision(points []Point, precision int32, aggr func([]Point) float64) []Point {
	l := len(points)
	var i, n int
	// i - current position of iterator
	// n - position of the first record with time rounded to precision

	if l == 0 {
		return points
	}

	// set first point time
	t := points[0].Time
	t = t - (t % precision)
	points[0].Time = t

	for i = 1; i < l; i++ {
		t = points[i].Time
		t = t - (t % precision)
		points[i].Time = t

		if points[n].Time == t {
			points[i].Metric = ""
		} else {
			if i > n+1 {
				points[n].Value = aggr(points[n:i])
			}
			n = i
		}
	}
	if i > n+1 {
		points[n].Value = aggr(points[n:i])
	}

	return PointsCleanup(points)
}

// RollupMetric rolling up list of points of ONE metric sorted by key "time"
// returns (new points slice, precision)
func (r *Rollup) RollupMetric(points []Point) ([]Point, int32) {
	// pp.Println(points)

	l := len(points)
	if l == 0 {
		return points, 1
	}

	now := int32(time.Now().Unix())
	rule := r.Match(points[0].Metric)
	precision := int32(1)

	for _, retention := range rule.Retention {
		if points[0].Time > now-retention.Age && retention.Age != 0 {
			break
		}

		points = doMetricPrecision(points, retention.Precision, rule.aggr)
		precision = retention.Precision
	}

	// pp.Println(points)
	return points, precision
}
<commit_msg>Fix typo in anyLast<commit_after>package backend

import (
	"encoding/xml"
	"fmt"
	"regexp"
	"time"
)

/*
<graphite_rollup>
 	<pattern>
 		<regexp>click_cost</regexp>
 		<function>any</function>
 		<retention>
 			<age>0</age>
 			<precision>3600</precision>
 		</retention>
 		<retention>
 			<age>86400</age>
 			<precision>60</precision>
 		</retention>
 	</pattern>
 	<default>
 		<function>max</function>
 		<retention>
 			<age>0</age>
 			<precision>60</precision>
 		</retention>
 		<retention>
 			<age>3600</age>
 			<precision>300</precision>
 		</retention>
 		<retention>
 			<age>86400</age>
 			<precision>3600</precision>
 		</retention>
 	</default>
</graphite_rollup>
*/

type RollupRetention struct {
	Age       int32 `xml:"age"`
	Precision int32 `xml:"precision"`
}

type RollupRule struct {
	Regexp    string                `xml:"regexp"`
	Function  string                `xml:"function"`
	Retention []*RollupRetention    `xml:"retention"`
	aggr      func([]Point) float64 `xml:"-"`
	re        *regexp.Regexp        `xml:"-"`
}

type Rollup struct {
	Pattern []*RollupRule `xml:"pattern"`
	Default *RollupRule   `xml:"default"`
}

func (rr *RollupRule) compile(hasRegexp bool) error {
	var err error
	if hasRegexp {
		rr.re, err = regexp.Compile(rr.Regexp)
		if err != nil {
			return err
		}
	}

	aggrMap := map[string](func([]Point) float64){
		"avg": aggrAvg,
		"max": aggrMax,
		"min": aggrMin,
		"sum": aggrSum,
		"any": aggrAny,
		"anyLast": aggrAnyLast,
	}

	var exists bool
	rr.aggr, exists = aggrMap[rr.Function]

	if !exists {
		return fmt.Errorf("unknown function %#v", rr.Function)
	}

	return nil
}

func (r *Rollup) compile() error {
	if r.Pattern == nil {
		r.Pattern = make([]*RollupRule, 0)
	}

	if r.Default == nil {
		return fmt.Errorf("default rollup rule not set")
	}

	if err := r.Default.compile(false); err != nil {
		return err
	}

	for _, rr := range r.Pattern {
		if err := rr.compile(true); err != nil {
			return err
		}
	}

	return nil
}

func ParseRollupXML(body []byte) (*Rollup, error) {
	r := &Rollup{}
	err := xml.Unmarshal(body, r)
	if err != nil {
		return nil, err
	}

	err = r.compile()
	if err != nil {
		return nil, err
	}

	return r, nil
}

// PointsCleanup removes points with empty metric
// for run after Deduplicate, Merge, etc for result cleanup
func PointsCleanup(points []Point) []Point {
	l := len(points)
	squashed := 0

	for i := 0; i < l; i++ {
		if points[i].Metric == "" {
			squashed++
			continue
		}
		if squashed > 0 {
			points[i-squashed] = points[i]
		}
	}

	return points[:l-squashed]
}

// PointsUniq removes points with equal metric and time
func PointsUniq(points []Point) []Point {
	l := len(points)
	var i, n int
	// i - current position of iterator
	// n - position on first record with current key (metric + time)

	for i = 1; i < l; i++ {
		if points[i].Metric != points[n].Metric ||
			points[i].Time != points[n].Time {
			n = i
			continue
		}

		if points[i].Timestamp > points[n].Timestamp {
			points[n] = points[i]
		}

		points[i].Metric = "" // mark for remove
	}

	return PointsCleanup(points)
}

// Match returns rollup rules for metric
func (r *Rollup) Match(metric string) *RollupRule {
	for _, rr := range r.Pattern {
		if rr.re.MatchString(metric) {
			return rr
		}
	}

	return r.Default
}

func aggrSum(points []Point) (r float64) {
	for _, p := range points {
		r += p.Value
	}
	return
}

func aggrMax(points []Point) (r float64) {
	if len(points) > 0 {
		r = points[0].Value
	}
	for _, p := range points {
		if p.Value > r {
			r = p.Value
		}
	}
	return
}

func aggrMin(points []Point) (r float64) {
	if len(points) > 0 {
		r = points[0].Value
	}
	for _, p := range points {
		if p.Value < r {
			r = p.Value
		}
	}
	return
}

func aggrAvg(points []Point) (r float64) {
	if len(points) == 0 {
		return
	}
	r = aggrSum(points) / float64(len(points))
	return
}

func aggrAny(points []Point) (r float64) {
	if len(points) > 0 {
		r = points[0].Value
	}
	return
}

func aggrAnyLast(points []Point) (r float64) {
	if len(points) > 0 {
		r = points[len(points)-1].Value
	}
	return
}

func doMetricPrecision(points []Point, precision int32, aggr func([]Point) float64) []Point {
	l := len(points)
	var i, n int
	// i - current position of iterator
	// n - position of the first record with time rounded to precision

	if l == 0 {
		return points
	}

	// set first point time
	t := points[0].Time
	t = t - (t % precision)
	points[0].Time = t

	for i = 1; i < l; i++ {
		t = points[i].Time
		t = t - (t % precision)
		points[i].Time = t

		if points[n].Time == t {
			points[i].Metric = ""
		} else {
			if i > n+1 {
				points[n].Value = aggr(points[n:i])
			}
			n = i
		}
	}
	if i > n+1 {
		points[n].Value = aggr(points[n:i])
	}

	return PointsCleanup(points)
}

// RollupMetric rolling up list of points of ONE metric sorted by key "time"
// returns (new points slice, precision)
func (r *Rollup) RollupMetric(points []Point) ([]Point, int32) {
	// pp.Println(points)

	l := len(points)
	if l == 0 {
		return points, 1
	}

	now := int32(time.Now().Unix())
	rule := r.Match(points[0].Metric)
	precision := int32(1)

	for _, retention := range rule.Retention {
		if points[0].Time > now-retention.Age && retention.Age != 0 {
			break
		}

		points = doMetricPrecision(points, retention.Precision, rule.aggr)
		precision = retention.Precision
	}

	// pp.Println(points)
	return points, precision
}
<|endoftext|>


<commit_before>package secio

import (
	"bytes"
	"crypto/rand"
	"errors"
	"fmt"
	"io"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
	msgio "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-msgio"

	ci "github.com/jbenet/go-ipfs/p2p/crypto"
	pb "github.com/jbenet/go-ipfs/p2p/crypto/secio/internal/pb"
	peer "github.com/jbenet/go-ipfs/p2p/peer"
	eventlog "github.com/jbenet/go-ipfs/thirdparty/eventlog"
	u "github.com/jbenet/go-ipfs/util"
)

var log = eventlog.Logger("secio")

// ErrUnsupportedKeyType is returned when a private key cast/type switch fails.
var ErrUnsupportedKeyType = errors.New("unsupported key type")

// ErrClosed signals the closing of a connection.
var ErrClosed = errors.New("connection closed")

// nonceSize is the size of our nonces (in bytes)
const nonceSize = 16

// secureSession encapsulates all the parameters needed for encrypting
// and decrypting traffic from an insecure channel.
type secureSession struct {
	secure msgio.ReadWriteCloser

	insecure  io.ReadWriter
	insecureM msgio.ReadWriter

	localKey   ci.PrivKey
	localPeer  peer.ID
	remotePeer peer.ID

	local  encParams
	remote encParams

	sharedSecret []byte
}

func newSecureSession(local peer.ID, key ci.PrivKey) (*secureSession, error) {
	s := &secureSession{localPeer: local, localKey: key}

	switch {
	case s.localPeer == "":
		return nil, errors.New("no local id provided")
	case s.localKey == nil:
		return nil, errors.New("no local private key provided")
	case !s.localPeer.MatchesPrivateKey(s.localKey):
		return nil, fmt.Errorf("peer.ID does not match PrivateKey")
	}

	return s, nil
}

// handsahke performs initial communication over insecure channel to share
// keys, IDs, and initiate communication, assigning all necessary params.
// requires the duplex channel to be a msgio.ReadWriter (for framed messaging)
func (s *secureSession) handshake(ctx context.Context, insecure io.ReadWriter) error {

	s.insecure = insecure
	s.insecureM = msgio.NewReadWriter(insecure)

	// =============================================================================
	// step 1. Propose -- propose cipher suite + send pubkeys + nonce

	// Generate and send Hello packet.
	// Hello = (rand, PublicKey, Supported)
	nonceOut := make([]byte, nonceSize)
	_, err := rand.Read(nonceOut)
	if err != nil {
		return err
	}

	defer log.EventBegin(ctx, "secureHandshake", s.localPeer).Done()

	s.local.permanentPubKey = s.localKey.GetPublic()
	myPubKeyBytes, err := s.local.permanentPubKey.Bytes()
	if err != nil {
		return err
	}

	proposeOut := new(pb.Propose)
	proposeOut.Rand = nonceOut
	proposeOut.Pubkey = myPubKeyBytes
	proposeOut.Exchanges = &SupportedExchanges
	proposeOut.Ciphers = &SupportedCiphers
	proposeOut.Hashes = &SupportedHashes

	// log.Debugf("1.0 Propose: nonce:%s exchanges:%s ciphers:%s hashes:%s",
	// 	nonceOut, SupportedExchanges, SupportedCiphers, SupportedHashes)

	// Send Propose packet (respects ctx)
	proposeOutBytes, err := writeMsgCtx(ctx, s.insecureM, proposeOut)
	if err != nil {
		return err
	}

	// Receive + Parse their Propose packet and generate an Exchange packet.
	proposeIn := new(pb.Propose)
	proposeInBytes, err := readMsgCtx(ctx, s.insecureM, proposeIn)
	if err != nil {
		return err
	}

	// log.Debugf("1.0.1 Propose recv: nonce:%s exchanges:%s ciphers:%s hashes:%s",
	// 	proposeIn.GetRand(), proposeIn.GetExchanges(), proposeIn.GetCiphers(), proposeIn.GetHashes())

	// =============================================================================
	// step 1.1 Identify -- get identity from their key

	// get remote identity
	s.remote.permanentPubKey, err = ci.UnmarshalPublicKey(proposeIn.GetPubkey())
	if err != nil {
		return err
	}

	// get peer id
	s.remotePeer, err = peer.IDFromPublicKey(s.remote.permanentPubKey)
	if err != nil {
		return err
	}

	log.Debugf("1.1 Identify: %s Remote Peer Identified as %s", s.localPeer, s.remotePeer)

	// =============================================================================
	// step 1.2 Selection -- select/agree on best encryption parameters

	// to determine order, use cmp(H(lr||rpk), H(rr||lpk)).
	oh1 := u.Hash(append(proposeIn.GetPubkey(), nonceOut...))
	oh2 := u.Hash(append(myPubKeyBytes, proposeIn.GetRand()...))
	order := bytes.Compare(oh1, oh2)
	s.local.curveT, err = selectBest(order, SupportedExchanges, proposeIn.GetExchanges())
	if err != nil {
		return err
	}

	s.local.cipherT, err = selectBest(order, SupportedCiphers, proposeIn.GetCiphers())
	if err != nil {
		return err
	}

	s.local.hashT, err = selectBest(order, SupportedHashes, proposeIn.GetHashes())
	if err != nil {
		return err
	}

	// we use the same params for both directions (must choose same curve)
	// WARNING: if they dont SelectBest the same way, this won't work...
	s.remote.curveT = s.local.curveT
	s.remote.cipherT = s.local.cipherT
	s.remote.hashT = s.local.hashT

	// log.Debugf("1.2 selection: exchange:%s cipher:%s hash:%s",
	// 	s.local.curveT, s.local.cipherT, s.local.hashT)

	// =============================================================================
	// step 2. Exchange -- exchange (signed) ephemeral keys. verify signatures.

	// Generate EphemeralPubKey
	var genSharedKey ci.GenSharedKey
	s.local.ephemeralPubKey, genSharedKey, err = ci.GenerateEKeyPair(s.local.curveT)

	// Gather corpus to sign.
	var selectionOut bytes.Buffer
	selectionOut.Write(proposeOutBytes)
	selectionOut.Write(proposeInBytes)
	selectionOut.Write(s.local.ephemeralPubKey)
	selectionOutBytes := selectionOut.Bytes()

	// log.Debugf("2.0 exchange: %v", selectionOutBytes)
	exchangeOut := new(pb.Exchange)
	exchangeOut.Epubkey = s.local.ephemeralPubKey
	exchangeOut.Signature, err = s.localKey.Sign(selectionOutBytes)
	if err != nil {
		return err
	}

	// Send Propose packet (respects ctx)
	if _, err := writeMsgCtx(ctx, s.insecureM, exchangeOut); err != nil {
		return err
	}

	// Receive + Parse their Propose packet and generate an Exchange packet.
	exchangeIn := new(pb.Exchange)
	if _, err := readMsgCtx(ctx, s.insecureM, exchangeIn); err != nil {
		return err
	}

	// =============================================================================
	// step 2.1. Verify -- verify their exchange packet is good.

	// get their ephemeral pub key
	s.remote.ephemeralPubKey = exchangeIn.GetEpubkey()

	var selectionIn bytes.Buffer
	selectionIn.Write(proposeInBytes)
	selectionIn.Write(proposeOutBytes)
	selectionIn.Write(s.remote.ephemeralPubKey)
	selectionInBytes := selectionIn.Bytes()
	// log.Debugf("2.0.1 exchange recv: %v", selectionInBytes)

	// u.POut("Remote Peer Identified as %s\n", s.remote)
	sigOK, err := s.remote.permanentPubKey.Verify(selectionInBytes, exchangeIn.GetSignature())
	if err != nil {
		// log.Error("2.1 Verify: failed: %s", err)
		return err
	}

	if !sigOK {
		err := errors.New("Bad signature!")
		// log.Error("2.1 Verify: failed: %s", err)
		return err
	}
	// log.Debugf("2.1 Verify: signature verified.")

	// =============================================================================
	// step 2.2. Keys -- generate keys for mac + encryption

	// OK! seems like we're good to go.
	s.sharedSecret, err = genSharedKey(exchangeIn.GetEpubkey())
	if err != nil {
		return err
	}

	// generate two sets of keys (stretching)
	k1, k2 := ci.KeyStretcher(s.local.cipherT, s.local.hashT, s.sharedSecret)

	// use random nonces to decide order.
	switch order {
	case 1:
	case -1:
		k1, k2 = k2, k1 // swap
	default:
		log.Error("WOAH: same keys (AND same nonce: 1/(2^128) chance!).")
		// this shouldn't happen. must determine order another way.
		// use the same keys but, make sure to copy underlying data!
		copy(k2.IV, k1.IV)
		copy(k2.MacKey, k1.MacKey)
		copy(k2.CipherKey, k1.CipherKey)
	}
	s.local.keys = k1
	s.remote.keys = k2

	// log.Debug("2.2 keys:\n\tshared: %v\n\tk1: %v\n\tk2: %v",
	// 	s.sharedSecret, s.local.keys, s.remote.keys)

	// =============================================================================
	// step 2.3. MAC + Cipher -- prepare MAC + cipher

	if err := s.local.makeMacAndCipher(); err != nil {
		return err
	}

	if err := s.remote.makeMacAndCipher(); err != nil {
		return err
	}

	// log.Debug("2.3 mac + cipher.")

	// =============================================================================
	// step 3. Finish -- send expected message (the nonces), verify encryption works

	// setup ETM ReadWriter
	w := NewETMWriter(s.insecure, s.local.cipher, s.local.mac)
	r := NewETMReader(s.insecure, s.remote.cipher, s.remote.mac)
	s.secure = msgio.Combine(w, r).(msgio.ReadWriteCloser)

	// log.Debug("3.0 finish. sending: %v", proposeIn.GetRand())
	// send their Nonce.
	if _, err := s.secure.Write(proposeIn.GetRand()); err != nil {
		return fmt.Errorf("Failed to write Finish nonce: %s", err)
	}

	// read our Nonce
	nonceOut2 := make([]byte, len(nonceOut))
	if _, err := io.ReadFull(s.secure, nonceOut2); err != nil {
		return fmt.Errorf("Failed to read Finish nonce: %s", err)
	}

	// log.Debug("3.0 finish.\n\texpect: %v\n\tactual: %v", nonceOut, nonceOut2)
	if !bytes.Equal(nonceOut, nonceOut2) {
		return fmt.Errorf("Failed to read our encrypted nonce: %s != %s", nonceOut2, nonceOut)
	}

	// Whew! ok, that's all folks.
	log.Event(ctx, "secureHandshakeFinish", s.localPeer, s.remotePeer)
	return nil
}
<commit_msg>log(secio): session loggable for event<commit_after>package secio

import (
	"bytes"
	"crypto/rand"
	"errors"
	"fmt"
	"io"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
	msgio "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-msgio"

	ci "github.com/jbenet/go-ipfs/p2p/crypto"
	pb "github.com/jbenet/go-ipfs/p2p/crypto/secio/internal/pb"
	peer "github.com/jbenet/go-ipfs/p2p/peer"
	eventlog "github.com/jbenet/go-ipfs/thirdparty/eventlog"
	u "github.com/jbenet/go-ipfs/util"
)

var log = eventlog.Logger("secio")

// ErrUnsupportedKeyType is returned when a private key cast/type switch fails.
var ErrUnsupportedKeyType = errors.New("unsupported key type")

// ErrClosed signals the closing of a connection.
var ErrClosed = errors.New("connection closed")

// nonceSize is the size of our nonces (in bytes)
const nonceSize = 16

// secureSession encapsulates all the parameters needed for encrypting
// and decrypting traffic from an insecure channel.
type secureSession struct {
	secure msgio.ReadWriteCloser

	insecure  io.ReadWriter
	insecureM msgio.ReadWriter

	localKey   ci.PrivKey
	localPeer  peer.ID
	remotePeer peer.ID

	local  encParams
	remote encParams

	sharedSecret []byte
}

func (s *secureSession) Loggable() map[string]interface{} {
	m := make(map[string]interface{})
	m["localPeer"] = s.localPeer.Pretty()
	m["remotePeer"] = s.remotePeer.Pretty()
	m["established"] = (s.secure != nil)
	return m
}

func newSecureSession(local peer.ID, key ci.PrivKey) (*secureSession, error) {
	s := &secureSession{localPeer: local, localKey: key}

	switch {
	case s.localPeer == "":
		return nil, errors.New("no local id provided")
	case s.localKey == nil:
		return nil, errors.New("no local private key provided")
	case !s.localPeer.MatchesPrivateKey(s.localKey):
		return nil, fmt.Errorf("peer.ID does not match PrivateKey")
	}

	return s, nil
}

// handsahke performs initial communication over insecure channel to share
// keys, IDs, and initiate communication, assigning all necessary params.
// requires the duplex channel to be a msgio.ReadWriter (for framed messaging)
func (s *secureSession) handshake(ctx context.Context, insecure io.ReadWriter) error {

	s.insecure = insecure
	s.insecureM = msgio.NewReadWriter(insecure)

	// =============================================================================
	// step 1. Propose -- propose cipher suite + send pubkeys + nonce

	// Generate and send Hello packet.
	// Hello = (rand, PublicKey, Supported)
	nonceOut := make([]byte, nonceSize)
	_, err := rand.Read(nonceOut)
	if err != nil {
		return err
	}

	defer log.EventBegin(ctx, "secureHandshake", s).Done()

	s.local.permanentPubKey = s.localKey.GetPublic()
	myPubKeyBytes, err := s.local.permanentPubKey.Bytes()
	if err != nil {
		return err
	}

	proposeOut := new(pb.Propose)
	proposeOut.Rand = nonceOut
	proposeOut.Pubkey = myPubKeyBytes
	proposeOut.Exchanges = &SupportedExchanges
	proposeOut.Ciphers = &SupportedCiphers
	proposeOut.Hashes = &SupportedHashes

	// log.Debugf("1.0 Propose: nonce:%s exchanges:%s ciphers:%s hashes:%s",
	// 	nonceOut, SupportedExchanges, SupportedCiphers, SupportedHashes)

	// Send Propose packet (respects ctx)
	proposeOutBytes, err := writeMsgCtx(ctx, s.insecureM, proposeOut)
	if err != nil {
		return err
	}

	// Receive + Parse their Propose packet and generate an Exchange packet.
	proposeIn := new(pb.Propose)
	proposeInBytes, err := readMsgCtx(ctx, s.insecureM, proposeIn)
	if err != nil {
		return err
	}

	// log.Debugf("1.0.1 Propose recv: nonce:%s exchanges:%s ciphers:%s hashes:%s",
	// 	proposeIn.GetRand(), proposeIn.GetExchanges(), proposeIn.GetCiphers(), proposeIn.GetHashes())

	// =============================================================================
	// step 1.1 Identify -- get identity from their key

	// get remote identity
	s.remote.permanentPubKey, err = ci.UnmarshalPublicKey(proposeIn.GetPubkey())
	if err != nil {
		return err
	}

	// get peer id
	s.remotePeer, err = peer.IDFromPublicKey(s.remote.permanentPubKey)
	if err != nil {
		return err
	}

	log.Debugf("1.1 Identify: %s Remote Peer Identified as %s", s.localPeer, s.remotePeer)

	// =============================================================================
	// step 1.2 Selection -- select/agree on best encryption parameters

	// to determine order, use cmp(H(lr||rpk), H(rr||lpk)).
	oh1 := u.Hash(append(proposeIn.GetPubkey(), nonceOut...))
	oh2 := u.Hash(append(myPubKeyBytes, proposeIn.GetRand()...))
	order := bytes.Compare(oh1, oh2)
	s.local.curveT, err = selectBest(order, SupportedExchanges, proposeIn.GetExchanges())
	if err != nil {
		return err
	}

	s.local.cipherT, err = selectBest(order, SupportedCiphers, proposeIn.GetCiphers())
	if err != nil {
		return err
	}

	s.local.hashT, err = selectBest(order, SupportedHashes, proposeIn.GetHashes())
	if err != nil {
		return err
	}

	// we use the same params for both directions (must choose same curve)
	// WARNING: if they dont SelectBest the same way, this won't work...
	s.remote.curveT = s.local.curveT
	s.remote.cipherT = s.local.cipherT
	s.remote.hashT = s.local.hashT

	// log.Debugf("1.2 selection: exchange:%s cipher:%s hash:%s",
	// 	s.local.curveT, s.local.cipherT, s.local.hashT)

	// =============================================================================
	// step 2. Exchange -- exchange (signed) ephemeral keys. verify signatures.

	// Generate EphemeralPubKey
	var genSharedKey ci.GenSharedKey
	s.local.ephemeralPubKey, genSharedKey, err = ci.GenerateEKeyPair(s.local.curveT)

	// Gather corpus to sign.
	var selectionOut bytes.Buffer
	selectionOut.Write(proposeOutBytes)
	selectionOut.Write(proposeInBytes)
	selectionOut.Write(s.local.ephemeralPubKey)
	selectionOutBytes := selectionOut.Bytes()

	// log.Debugf("2.0 exchange: %v", selectionOutBytes)
	exchangeOut := new(pb.Exchange)
	exchangeOut.Epubkey = s.local.ephemeralPubKey
	exchangeOut.Signature, err = s.localKey.Sign(selectionOutBytes)
	if err != nil {
		return err
	}

	// Send Propose packet (respects ctx)
	if _, err := writeMsgCtx(ctx, s.insecureM, exchangeOut); err != nil {
		return err
	}

	// Receive + Parse their Propose packet and generate an Exchange packet.
	exchangeIn := new(pb.Exchange)
	if _, err := readMsgCtx(ctx, s.insecureM, exchangeIn); err != nil {
		return err
	}

	// =============================================================================
	// step 2.1. Verify -- verify their exchange packet is good.

	// get their ephemeral pub key
	s.remote.ephemeralPubKey = exchangeIn.GetEpubkey()

	var selectionIn bytes.Buffer
	selectionIn.Write(proposeInBytes)
	selectionIn.Write(proposeOutBytes)
	selectionIn.Write(s.remote.ephemeralPubKey)
	selectionInBytes := selectionIn.Bytes()
	// log.Debugf("2.0.1 exchange recv: %v", selectionInBytes)

	// u.POut("Remote Peer Identified as %s\n", s.remote)
	sigOK, err := s.remote.permanentPubKey.Verify(selectionInBytes, exchangeIn.GetSignature())
	if err != nil {
		// log.Error("2.1 Verify: failed: %s", err)
		return err
	}

	if !sigOK {
		err := errors.New("Bad signature!")
		// log.Error("2.1 Verify: failed: %s", err)
		return err
	}
	// log.Debugf("2.1 Verify: signature verified.")

	// =============================================================================
	// step 2.2. Keys -- generate keys for mac + encryption

	// OK! seems like we're good to go.
	s.sharedSecret, err = genSharedKey(exchangeIn.GetEpubkey())
	if err != nil {
		return err
	}

	// generate two sets of keys (stretching)
	k1, k2 := ci.KeyStretcher(s.local.cipherT, s.local.hashT, s.sharedSecret)

	// use random nonces to decide order.
	switch order {
	case 1:
	case -1:
		k1, k2 = k2, k1 // swap
	default:
		log.Error("WOAH: same keys (AND same nonce: 1/(2^128) chance!).")
		// this shouldn't happen. must determine order another way.
		// use the same keys but, make sure to copy underlying data!
		copy(k2.IV, k1.IV)
		copy(k2.MacKey, k1.MacKey)
		copy(k2.CipherKey, k1.CipherKey)
	}
	s.local.keys = k1
	s.remote.keys = k2

	// log.Debug("2.2 keys:\n\tshared: %v\n\tk1: %v\n\tk2: %v",
	// 	s.sharedSecret, s.local.keys, s.remote.keys)

	// =============================================================================
	// step 2.3. MAC + Cipher -- prepare MAC + cipher

	if err := s.local.makeMacAndCipher(); err != nil {
		return err
	}

	if err := s.remote.makeMacAndCipher(); err != nil {
		return err
	}

	// log.Debug("2.3 mac + cipher.")

	// =============================================================================
	// step 3. Finish -- send expected message (the nonces), verify encryption works

	// setup ETM ReadWriter
	w := NewETMWriter(s.insecure, s.local.cipher, s.local.mac)
	r := NewETMReader(s.insecure, s.remote.cipher, s.remote.mac)
	s.secure = msgio.Combine(w, r).(msgio.ReadWriteCloser)

	// log.Debug("3.0 finish. sending: %v", proposeIn.GetRand())
	// send their Nonce.
	if _, err := s.secure.Write(proposeIn.GetRand()); err != nil {
		return fmt.Errorf("Failed to write Finish nonce: %s", err)
	}

	// read our Nonce
	nonceOut2 := make([]byte, len(nonceOut))
	if _, err := io.ReadFull(s.secure, nonceOut2); err != nil {
		return fmt.Errorf("Failed to read Finish nonce: %s", err)
	}

	// log.Debug("3.0 finish.\n\texpect: %v\n\tactual: %v", nonceOut, nonceOut2)
	if !bytes.Equal(nonceOut, nonceOut2) {
		return fmt.Errorf("Failed to read our encrypted nonce: %s != %s", nonceOut2, nonceOut)
	}

	// Whew! ok, that's all folks.
	return nil
}
<|endoftext|>