<commit_before>package clui

import (
	term "github.com/nsf/termbox-go"
)

// ThumbPosition returns a scrollbar thumb position depending
// on currently active item(itemNo), total number of items
// (itemCount), and length/height of the scrollbar(length)
// including arrows. Returns position in interval of (1..lenght-2)
// or -1 if the thumb is not visible
func ThumbPosition(itemNo, itemCount, length int) int {
	length -= 2
	if itemNo < 0 {
		return -1
	}
	if itemNo >= itemCount-1 {
		return length - 1
	}

	if length < 4 {
		return 0
	}

	ydiff := int(float32(itemNo) / float32(itemCount-1.0) * float32(length-1))
	return ydiff
}

// ItemByThumbPosition calculates item number by scrollbar
// thumb position. Position - thumb position inside scrollbar,
// itemCount - total number of items, lenght - lenght or heigth
// of scrollbar. Return -1 if it is not possible to calculate:
// e.g, itemCount equals zero
func ItemByThumbPosition(position, itemCount, length int) int {
	length -= 2
	if position < 1 {
		return -1
	}
	if itemCount < 1 {
		return -1
	}
	if itemCount == 1 {
		return 0
	}

	newPos := int(float32(itemCount-1)*float32(position-1)/float32(length-1) + 0.9)

	if newPos < 0 {
		newPos = 0
	} else if newPos >= itemCount {
		newPos = itemCount - 1
	}

	return newPos
}

// ChildAt returns the children of parent control that is at absolute
// coordinates x, y. Returns nil if x, y are outside parent control and
// returns parent if no child is at x, y
func ChildAt(parent Control, x, y int) Control {
	px, py := parent.Pos()
	pw, ph := parent.Size()
	if px > x || py > y || px+pw <= x || py+ph <= y {
		return nil
	}

	if len(parent.Children()) == 0 {
		return parent
	}

	var ctrl Control
	ctrl = parent
	for _, child := range parent.Children() {
		check := ChildAt(child, x, y)
		if check != nil {
			ctrl = check
			break
		}
	}

	return ctrl
}

// DeactivateControls makes all children of parent inactive
func DeactivateControls(parent Control) {
	for _, ctrl := range parent.Children() {
		if ctrl.Active() {
			ctrl.SetActive(false)
			ctrl.ProcessEvent(Event{Type: EventActivate, X: 0})
		}

		DeactivateControls(ctrl)
	}
}

// ActivateControl makes control active and disables all other children of
// the parent. Returns true if control was found and activated
func ActivateControl(parent, control Control) bool {
	DeactivateControls(parent)
	res := false
	ctrl := FindChild(parent, control)
	if ctrl != nil {
		res = true
		if !ctrl.Active() {
			ctrl.ProcessEvent(Event{Type: EventActivate, X: 1})
			ctrl.SetActive(true)
		}
	}

	return res
}

// FindChild returns control if it is a child of the parent and nil otherwise
func FindChild(parent, control Control) Control {
	var res Control

	if parent == control {
		return parent
	}

	for _, ctrl := range parent.Children() {
		if ctrl == control {
			res = ctrl
			break
		}

		res = FindChild(ctrl, control)
		if res != nil {
			break
		}
	}

	return res
}

// IsMouseClickEvent returns if a user action can be treated as mouse click.
func IsMouseClickEvent(ev Event) bool {
	if ev.Type == EventClick {
		return true
	}
	if ev.Type == EventMouse && ev.Key == term.MouseLeft {
		return true
	}

	return false
}

// FindFirstControl returns the first child for that fn returns true.
// The function is used to find active or tab-stop control
func FindFirstControl(parent Control, fn func(Control) bool) Control {
	linear := getLinearControlList(parent, fn)
	if len(linear) == 0 {
		return nil
	}

	return linear[0]
}

// FindLastControl returns the first child for that fn returns true.
// The function is used by TAB processing method if a user goes backwards
// with TAB key - not supported now
func FindLastControl(parent Control, fn func(Control) bool) Control {
	linear := getLinearControlList(parent, fn)

	if len(linear) == 0 {
		return nil
	}

	return linear[len(linear)-1]
}

// ActiveControl returns the active child of the parent or nil if no child is
// active
func ActiveControl(parent Control) Control {
	fnActive := func(c Control) bool {
		return c.Active()
	}
	return FindFirstControl(parent, fnActive)
}

func getLinearControlList(parent Control, fn func(Control) bool) []Control {
	result := []Control{}

	for _, curr := range parent.Children() {
		if fn(curr) {
			result = append(result, curr)
		}

		if len(curr.Children()) == 0 {
			continue
		}

		ch := getLinearControlList(curr, fn)
		if len(ch) != 0 {
			result = append(result, ch...)
		}
	}

	return result
}

// NextControl returns the next or previous child (depends on next parameter)
// that has tab-stop feature on. Used by library when processing TAB key
func NextControl(parent Control, curr Control, next bool) Control {
	fnTab := func(c Control) bool {
		return c.TabStop() && c.Visible()
	}

	linear := getLinearControlList(parent, fnTab)

	var pIndex, nIndex int

	for i, ch := range linear {
		if ch != curr {
			continue
		}

		pIndex = i - 1
		nIndex = i + 1
		break
	}

	if nIndex > len(linear)-1 {
		nIndex = 0
	}

	if pIndex < 0 {
		pIndex = len(linear) - 1
	}

	if next {
		return linear[nIndex]
	} else {
		return linear[pIndex]
	}
}

// SendEventToChild tries to find a child control that should recieve the evetn
// For mouse click events it looks for a control at coordinates of event,
// makes it active, and then sends the event to it.
// If it is not mouse click event then it looks for the first active child and
// sends the event to it if it is not nil
func SendEventToChild(parent Control, ev Event) bool {
	var child Control
	if IsMouseClickEvent(ev) {
		child = ChildAt(parent, ev.X, ev.Y)
		if child != nil && !child.Active() {
			ActivateControl(parent, child)
		}
	} else {
		child = ActiveControl(parent)
	}

	if child != nil && child != parent {
		return child.ProcessEvent(ev)
	}

	return false
}
<commit_msg>clui: NextControl() also filter out disabled controls<commit_after>package clui

import (
	term "github.com/nsf/termbox-go"
)

// ThumbPosition returns a scrollbar thumb position depending
// on currently active item(itemNo), total number of items
// (itemCount), and length/height of the scrollbar(length)
// including arrows. Returns position in interval of (1..lenght-2)
// or -1 if the thumb is not visible
func ThumbPosition(itemNo, itemCount, length int) int {
	length -= 2
	if itemNo < 0 {
		return -1
	}
	if itemNo >= itemCount-1 {
		return length - 1
	}

	if length < 4 {
		return 0
	}

	ydiff := int(float32(itemNo) / float32(itemCount-1.0) * float32(length-1))
	return ydiff
}

// ItemByThumbPosition calculates item number by scrollbar
// thumb position. Position - thumb position inside scrollbar,
// itemCount - total number of items, lenght - lenght or heigth
// of scrollbar. Return -1 if it is not possible to calculate:
// e.g, itemCount equals zero
func ItemByThumbPosition(position, itemCount, length int) int {
	length -= 2
	if position < 1 {
		return -1
	}
	if itemCount < 1 {
		return -1
	}
	if itemCount == 1 {
		return 0
	}

	newPos := int(float32(itemCount-1)*float32(position-1)/float32(length-1) + 0.9)

	if newPos < 0 {
		newPos = 0
	} else if newPos >= itemCount {
		newPos = itemCount - 1
	}

	return newPos
}

// ChildAt returns the children of parent control that is at absolute
// coordinates x, y. Returns nil if x, y are outside parent control and
// returns parent if no child is at x, y
func ChildAt(parent Control, x, y int) Control {
	px, py := parent.Pos()
	pw, ph := parent.Size()
	if px > x || py > y || px+pw <= x || py+ph <= y {
		return nil
	}

	if len(parent.Children()) == 0 {
		return parent
	}

	var ctrl Control
	ctrl = parent
	for _, child := range parent.Children() {
		check := ChildAt(child, x, y)
		if check != nil {
			ctrl = check
			break
		}
	}

	return ctrl
}

// DeactivateControls makes all children of parent inactive
func DeactivateControls(parent Control) {
	for _, ctrl := range parent.Children() {
		if ctrl.Active() {
			ctrl.SetActive(false)
			ctrl.ProcessEvent(Event{Type: EventActivate, X: 0})
		}

		DeactivateControls(ctrl)
	}
}

// ActivateControl makes control active and disables all other children of
// the parent. Returns true if control was found and activated
func ActivateControl(parent, control Control) bool {
	DeactivateControls(parent)
	res := false
	ctrl := FindChild(parent, control)
	if ctrl != nil {
		res = true
		if !ctrl.Active() {
			ctrl.ProcessEvent(Event{Type: EventActivate, X: 1})
			ctrl.SetActive(true)
		}
	}

	return res
}

// FindChild returns control if it is a child of the parent and nil otherwise
func FindChild(parent, control Control) Control {
	var res Control

	if parent == control {
		return parent
	}

	for _, ctrl := range parent.Children() {
		if ctrl == control {
			res = ctrl
			break
		}

		res = FindChild(ctrl, control)
		if res != nil {
			break
		}
	}

	return res
}

// IsMouseClickEvent returns if a user action can be treated as mouse click.
func IsMouseClickEvent(ev Event) bool {
	if ev.Type == EventClick {
		return true
	}
	if ev.Type == EventMouse && ev.Key == term.MouseLeft {
		return true
	}

	return false
}

// FindFirstControl returns the first child for that fn returns true.
// The function is used to find active or tab-stop control
func FindFirstControl(parent Control, fn func(Control) bool) Control {
	linear := getLinearControlList(parent, fn)
	if len(linear) == 0 {
		return nil
	}

	return linear[0]
}

// FindLastControl returns the first child for that fn returns true.
// The function is used by TAB processing method if a user goes backwards
// with TAB key - not supported now
func FindLastControl(parent Control, fn func(Control) bool) Control {
	linear := getLinearControlList(parent, fn)

	if len(linear) == 0 {
		return nil
	}

	return linear[len(linear)-1]
}

// ActiveControl returns the active child of the parent or nil if no child is
// active
func ActiveControl(parent Control) Control {
	fnActive := func(c Control) bool {
		return c.Active()
	}
	return FindFirstControl(parent, fnActive)
}

func getLinearControlList(parent Control, fn func(Control) bool) []Control {
	result := []Control{}

	for _, curr := range parent.Children() {
		if fn(curr) {
			result = append(result, curr)
		}

		if len(curr.Children()) == 0 {
			continue
		}

		ch := getLinearControlList(curr, fn)
		if len(ch) != 0 {
			result = append(result, ch...)
		}
	}

	return result
}

// NextControl returns the next or previous child (depends on next parameter)
// that has tab-stop feature on. Used by library when processing TAB key
func NextControl(parent Control, curr Control, next bool) Control {
	fnTab := func(c Control) bool {
		return c.TabStop() && c.Visible() && c.Enabled()
	}

	linear := getLinearControlList(parent, fnTab)

	var pIndex, nIndex int

	for i, ch := range linear {
		if ch != curr {
			continue
		}

		pIndex = i - 1
		nIndex = i + 1
		break
	}

	if nIndex > len(linear)-1 {
		nIndex = 0
	}

	if pIndex < 0 {
		pIndex = len(linear) - 1
	}

	if next {
		return linear[nIndex]
	} else {
		return linear[pIndex]
	}
}

// SendEventToChild tries to find a child control that should recieve the evetn
// For mouse click events it looks for a control at coordinates of event,
// makes it active, and then sends the event to it.
// If it is not mouse click event then it looks for the first active child and
// sends the event to it if it is not nil
func SendEventToChild(parent Control, ev Event) bool {
	var child Control
	if IsMouseClickEvent(ev) {
		child = ChildAt(parent, ev.X, ev.Y)
		if child != nil && !child.Active() {
			ActivateControl(parent, child)
		}
	} else {
		child = ActiveControl(parent)
	}

	if child != nil && child != parent {
		return child.ProcessEvent(ev)
	}

	return false
}
<|endoftext|>


<commit_before>package ctx

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/user"
	"path/filepath"
	"strings"

	jsconf "github.com/funkygao/jsconf"
)

func LoadConfig(fn string) {
	cf, err := jsconf.Load(fn)
	if err != nil {
		panic(err)
	}

	conf = new(config)
	conf.hostname, _ = os.Hostname()
	conf.kafkaHome = cf.String("kafka_home", "")
	conf.logLevel = cf.String("loglevel", "info")
	conf.influxdbHost = cf.String("influxdb_host", "")
	conf.zones = make(map[string]string)
	conf.consulBootstrap = cf.String("consul_bootstrap", "")
	conf.zkDefaultZone = cf.String("zk_default_zone", "")
	conf.upgradeCenter = cf.String("upgrade_center", "")
	if !strings.HasPrefix(conf.upgradeCenter, "http://") {
		panic("invalid upgrade_center")
	}
	conf.tunnels = make(map[string]string)
	conf.aliases = make(map[string]string)
	for i := 0; i < len(cf.List("aliases", nil)); i++ {
		section, err := cf.Section(fmt.Sprintf("aliases[%d]", i))
		if err != nil {
			panic(err)
		}

		conf.aliases[section.String("cmd", "")] = section.String("alias", "")
	}

	for i := 0; i < len(cf.List("zones", nil)); i++ {
		section, err := cf.Section(fmt.Sprintf("zones[%d]", i))
		if err != nil {
			panic(err)
		}

		z := new(zone)
		z.loadConfig(section)
		conf.zones[z.name] = z.zk
		conf.tunnels[z.name] = z.tunnel
	}

	conf.reverseDns = make(map[string][]string)
	for _, entry := range cf.StringList("reverse_dns", nil) {
		if entry != "" {
			// entry e,g. k11000b.sit.mycorp.kfk.com:10.10.1.1
			parts := strings.SplitN(entry, ":", 2)
			if len(parts) != 2 {
				panic("invalid reverse_dns record")
			}

			ip, host := strings.TrimSpace(parts[1]), strings.TrimSpace(parts[0])
			if _, present := conf.reverseDns[ip]; !present {
				conf.reverseDns[ip] = make([]string, 0)
			}

			conf.reverseDns[ip] = append(conf.reverseDns[ip], host)
		}
	}

}

func LoadFromHome() {
	var configFile string
	if usr, err := user.Current(); err == nil {
		configFile = filepath.Join(usr.HomeDir, ".gafka.cf")
	} else {
		panic(err)
	}

	_, err := os.Stat(configFile)
	if err != nil {
		if os.IsNotExist(err) {
			// create the config file on the fly
			if e := ioutil.WriteFile(configFile,
				[]byte(strings.TrimSpace(DefaultConfig)), 0644); e != nil {
				panic(e)
			}
		} else {
			panic(err)
		}
	}

	LoadConfig(configFile)
}
<commit_msg>no panic<commit_after>package ctx

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/user"
	"path/filepath"
	"strings"

	jsconf "github.com/funkygao/jsconf"
)

func LoadConfig(fn string) {
	cf, err := jsconf.Load(fn)
	if err != nil {
		panic(err)
	}

	conf = new(config)
	conf.hostname, _ = os.Hostname()
	conf.kafkaHome = cf.String("kafka_home", "")
	conf.logLevel = cf.String("loglevel", "info")
	conf.influxdbHost = cf.String("influxdb_host", "")
	conf.zones = make(map[string]string)
	conf.consulBootstrap = cf.String("consul_bootstrap", "")
	conf.zkDefaultZone = cf.String("zk_default_zone", "")
	conf.upgradeCenter = cf.String("upgrade_center", "")
	conf.tunnels = make(map[string]string)
	conf.aliases = make(map[string]string)
	for i := 0; i < len(cf.List("aliases", nil)); i++ {
		section, err := cf.Section(fmt.Sprintf("aliases[%d]", i))
		if err != nil {
			panic(err)
		}

		conf.aliases[section.String("cmd", "")] = section.String("alias", "")
	}

	for i := 0; i < len(cf.List("zones", nil)); i++ {
		section, err := cf.Section(fmt.Sprintf("zones[%d]", i))
		if err != nil {
			panic(err)
		}

		z := new(zone)
		z.loadConfig(section)
		conf.zones[z.name] = z.zk
		conf.tunnels[z.name] = z.tunnel
	}

	conf.reverseDns = make(map[string][]string)
	for _, entry := range cf.StringList("reverse_dns", nil) {
		if entry != "" {
			// entry e,g. k11000b.sit.mycorp.kfk.com:10.10.1.1
			parts := strings.SplitN(entry, ":", 2)
			if len(parts) != 2 {
				panic("invalid reverse_dns record")
			}

			ip, host := strings.TrimSpace(parts[1]), strings.TrimSpace(parts[0])
			if _, present := conf.reverseDns[ip]; !present {
				conf.reverseDns[ip] = make([]string, 0)
			}

			conf.reverseDns[ip] = append(conf.reverseDns[ip], host)
		}
	}

}

func LoadFromHome() {
	var configFile string
	if usr, err := user.Current(); err == nil {
		configFile = filepath.Join(usr.HomeDir, ".gafka.cf")
	} else {
		panic(err)
	}

	_, err := os.Stat(configFile)
	if err != nil {
		if os.IsNotExist(err) {
			// create the config file on the fly
			if e := ioutil.WriteFile(configFile,
				[]byte(strings.TrimSpace(DefaultConfig)), 0644); e != nil {
				panic(e)
			}
		} else {
			panic(err)
		}
	}

	LoadConfig(configFile)
}
<|endoftext|>


<commit_before>package stripe

import (
	"encoding/json"
)

// CustomerTaxExempt is the type of tax exemption associated with a customer.
type CustomerTaxExempt string

// List of values that CustomerTaxInfoType can take.
const (
	CustomerTaxExemptExempt  CustomerTaxExempt = "exempt"
	CustomerTaxExemptNone    CustomerTaxExempt = "none"
	CustomerTaxExemptReverse CustomerTaxExempt = "reverse"
)

// CustomerTaxInfoType is the type of tax info associated with a customer.
type CustomerTaxInfoType string

// List of values that CustomerTaxInfoType can take.
const (
	CustomerTaxInfoTypeVAT CustomerTaxInfoType = "vat"
)

// CustomerTaxInfoVerificationStatus is the type of tax info associated with a customer.
type CustomerTaxInfoVerificationStatus string

// List of values that CustomerTaxInfoType can take.
const (
	CustomerTaxInfoVerificationStatusPending    CustomerTaxInfoVerificationStatus = "pending"
	CustomerTaxInfoVerificationStatusUnverified CustomerTaxInfoVerificationStatus = "unverified"
	CustomerTaxInfoVerificationStatusVerified   CustomerTaxInfoVerificationStatus = "verified"
)

// CustomerParams is the set of parameters that can be used when creating or updating a customer.
// For more details see https://stripe.com/docs/api#create_customer and https://stripe.com/docs/api#update_customer.
type CustomerParams struct {
	Params           `form:"*"`
	AccountBalance   *int64                         `form:"account_balance"`
	Address          *AddressParams                 `form:"address"`
	Coupon           *string                        `form:"coupon"`
	DefaultSource    *string                        `form:"default_source"`
	Description      *string                        `form:"description"`
	Email            *string                        `form:"email"`
	InvoicePrefix    *string                        `form:"invoice_prefix"`
	InvoiceSettings  *CustomerInvoiceSettingsParams `form:"invoice_settings"`
	Name             *string                        `form:"name"`
	PaymentMethod    *string                        `form:"payment_method"`
	Phone            *string                        `form:"phone"`
	PreferredLocales []*string                      `form:"preferred_locales"`
	Shipping         *CustomerShippingDetailsParams `form:"shipping"`
	Source           *SourceParams                  `form:"*"` // SourceParams has custom encoding so brought to top level with "*"
	TaxExempt        *string                        `form:"tax_exempt"`
	TaxIDData        []*CustomerTaxIDDataParams     `form:"tax_id_data"`
	Token            *string                        `form:"-"` // This doesn't seem to be used?

	// The following parameter is deprecated. Use TaxIDData instead.
	TaxInfo *CustomerTaxInfoParams `form:"tax_info"`

	// The parameters below are considered deprecated. Consider creating a Subscription separately instead.
	Plan       *string  `form:"plan"`
	Quantity   *int64   `form:"quantity"`
	TaxPercent *float64 `form:"tax_percent"`
	TrialEnd   *int64   `form:"trial_end"`
}

// CustomerInvoiceCustomFieldParams represents the parameters associated with one custom field on
// the customer's invoices.
type CustomerInvoiceCustomFieldParams struct {
	Name  *string `form:"name"`
	Value *string `form:"value"`
}

// CustomerInvoiceSettingsParams is the structure containing the default settings for invoices
// associated with this customer.
type CustomerInvoiceSettingsParams struct {
	CustomFields         []*CustomerInvoiceCustomFieldParams `form:"custom_fields"`
	DefaultPaymentMethod *string                             `form:"default_payment_method"`
	Footer               *string                             `form:"footer"`
}

// CustomerShippingDetailsParams is the structure containing shipping information.
type CustomerShippingDetailsParams struct {
	Address *AddressParams `form:"address"`
	Name    *string        `form:"name"`
	Phone   *string        `form:"phone"`
}

// CustomerTaxIDDataParams lets you pass the tax id details associated with a Customer.
type CustomerTaxIDDataParams struct {
	Type  *string `form:"type"`
	Value *string `form:"value"`
}

// CustomerTaxInfoParams is the structure containing tax information for the customer.
type CustomerTaxInfoParams struct {
	TaxID *string `form:"tax_id"`
	Type  *string `form:"type"`
}

// SetSource adds valid sources to a CustomerParams object,
// returning an error for unsupported sources.
func (cp *CustomerParams) SetSource(sp interface{}) error {
	source, err := SourceParamsFor(sp)
	cp.Source = source
	return err
}

// CustomerListParams is the set of parameters that can be used when listing customers.
// For more details see https://stripe.com/docs/api#list_customers.
type CustomerListParams struct {
	ListParams   `form:"*"`
	Created      *int64            `form:"created"`
	CreatedRange *RangeQueryParams `form:"created"`
	Email        *string           `form:"email"`
}

// Customer is the resource representing a Stripe customer.
// For more details see https://stripe.com/docs/api#customers.
type Customer struct {
	AccountBalance   int64                    `json:"account_balance"`
	Address          Address                  `json:"address"`
	Created          int64                    `json:"created"`
	Currency         Currency                 `json:"currency"`
	DefaultSource    *PaymentSource           `json:"default_source"`
	Deleted          bool                     `json:"deleted"`
	Delinquent       bool                     `json:"delinquent"`
	Description      string                   `json:"description"`
	Discount         *Discount                `json:"discount"`
	Email            string                   `json:"email"`
	ID               string                   `json:"id"`
	InvoicePrefix    string                   `json:"invoice_prefix"`
	Livemode         bool                     `json:"livemode"`
	Metadata         map[string]string        `json:"metadata"`
	Name             string                   `json:"name"`
	Phone            string                   `json:"phone"`
	PreferredLocales []string                 `json:"preferred_locales"`
	Shipping         *CustomerShippingDetails `json:"shipping"`
	Sources          *SourceList              `json:"sources"`
	Subscriptions    *SubscriptionList        `json:"subscriptions"`
	TaxExempt        CustomerTaxExempt        `json:"tax_exempt"`
	TaxIDs           *TaxIDList               `json:"tax_ids"`

	// The following properties are deprecated. Use TaxIds instead.
	TaxInfo             *CustomerTaxInfo             `json:"tax_info"`
	TaxInfoVerification *CustomerTaxInfoVerification `json:"tax_info_verification"`
}

// CustomerInvoiceCustomField represents a custom field associated with the customer's invoices.
type CustomerInvoiceCustomField struct {
	Name  *string `form:"name"`
	Value *string `form:"value"`
}

// CustomerInvoiceSettings is the structure containing the default settings for invoices associated
// with this customer.
type CustomerInvoiceSettings struct {
	CustomFields         []*CustomerInvoiceCustomField `json:"custom_fields"`
	DefaultPaymentMethod *PaymentMethod                `json:"default_payment_method"`
	Footer               string                        `json:"footer"`
}

// CustomerList is a list of customers as retrieved from a list endpoint.
type CustomerList struct {
	ListMeta
	Data []*Customer `json:"data"`
}

// CustomerShippingDetails is the structure containing shipping information.
type CustomerShippingDetails struct {
	Address Address `json:"address"`
	Name    string  `json:"name"`
	Phone   string  `json:"phone"`
}

// CustomerTaxInfo is the structure containing tax information for the customer.
type CustomerTaxInfo struct {
	TaxID string              `json:"tax_id"`
	Type  CustomerTaxInfoType `json:"type"`
}

// CustomerTaxInfoVerification is the structure containing tax verification for the customer.
type CustomerTaxInfoVerification struct {
	Status       CustomerTaxInfoVerificationStatus `json:"status"`
	VerifiedName string                            `json:"verified_name"`
}

// UnmarshalJSON handles deserialization of a Customer.
// This custom unmarshaling is needed because the resulting
// property may be an id or the full struct if it was expanded.
func (c *Customer) UnmarshalJSON(data []byte) error {
	if id, ok := ParseID(data); ok {
		c.ID = id
		return nil
	}

	type customer Customer
	var v customer
	if err := json.Unmarshal(data, &v); err != nil {
		return err
	}

	*c = Customer(v)
	return nil
}
<commit_msg>Add InvoiceSettings to Customer<commit_after>package stripe

import (
	"encoding/json"
)

// CustomerTaxExempt is the type of tax exemption associated with a customer.
type CustomerTaxExempt string

// List of values that CustomerTaxInfoType can take.
const (
	CustomerTaxExemptExempt  CustomerTaxExempt = "exempt"
	CustomerTaxExemptNone    CustomerTaxExempt = "none"
	CustomerTaxExemptReverse CustomerTaxExempt = "reverse"
)

// CustomerTaxInfoType is the type of tax info associated with a customer.
type CustomerTaxInfoType string

// List of values that CustomerTaxInfoType can take.
const (
	CustomerTaxInfoTypeVAT CustomerTaxInfoType = "vat"
)

// CustomerTaxInfoVerificationStatus is the type of tax info associated with a customer.
type CustomerTaxInfoVerificationStatus string

// List of values that CustomerTaxInfoType can take.
const (
	CustomerTaxInfoVerificationStatusPending    CustomerTaxInfoVerificationStatus = "pending"
	CustomerTaxInfoVerificationStatusUnverified CustomerTaxInfoVerificationStatus = "unverified"
	CustomerTaxInfoVerificationStatusVerified   CustomerTaxInfoVerificationStatus = "verified"
)

// CustomerParams is the set of parameters that can be used when creating or updating a customer.
// For more details see https://stripe.com/docs/api#create_customer and https://stripe.com/docs/api#update_customer.
type CustomerParams struct {
	Params           `form:"*"`
	AccountBalance   *int64                         `form:"account_balance"`
	Address          *AddressParams                 `form:"address"`
	Coupon           *string                        `form:"coupon"`
	DefaultSource    *string                        `form:"default_source"`
	Description      *string                        `form:"description"`
	Email            *string                        `form:"email"`
	InvoicePrefix    *string                        `form:"invoice_prefix"`
	InvoiceSettings  *CustomerInvoiceSettingsParams `form:"invoice_settings"`
	Name             *string                        `form:"name"`
	PaymentMethod    *string                        `form:"payment_method"`
	Phone            *string                        `form:"phone"`
	PreferredLocales []*string                      `form:"preferred_locales"`
	Shipping         *CustomerShippingDetailsParams `form:"shipping"`
	Source           *SourceParams                  `form:"*"` // SourceParams has custom encoding so brought to top level with "*"
	TaxExempt        *string                        `form:"tax_exempt"`
	TaxIDData        []*CustomerTaxIDDataParams     `form:"tax_id_data"`
	Token            *string                        `form:"-"` // This doesn't seem to be used?

	// The following parameter is deprecated. Use TaxIDData instead.
	TaxInfo *CustomerTaxInfoParams `form:"tax_info"`

	// The parameters below are considered deprecated. Consider creating a Subscription separately instead.
	Plan       *string  `form:"plan"`
	Quantity   *int64   `form:"quantity"`
	TaxPercent *float64 `form:"tax_percent"`
	TrialEnd   *int64   `form:"trial_end"`
}

// CustomerInvoiceCustomFieldParams represents the parameters associated with one custom field on
// the customer's invoices.
type CustomerInvoiceCustomFieldParams struct {
	Name  *string `form:"name"`
	Value *string `form:"value"`
}

// CustomerInvoiceSettingsParams is the structure containing the default settings for invoices
// associated with this customer.
type CustomerInvoiceSettingsParams struct {
	CustomFields         []*CustomerInvoiceCustomFieldParams `form:"custom_fields"`
	DefaultPaymentMethod *string                             `form:"default_payment_method"`
	Footer               *string                             `form:"footer"`
}

// CustomerShippingDetailsParams is the structure containing shipping information.
type CustomerShippingDetailsParams struct {
	Address *AddressParams `form:"address"`
	Name    *string        `form:"name"`
	Phone   *string        `form:"phone"`
}

// CustomerTaxIDDataParams lets you pass the tax id details associated with a Customer.
type CustomerTaxIDDataParams struct {
	Type  *string `form:"type"`
	Value *string `form:"value"`
}

// CustomerTaxInfoParams is the structure containing tax information for the customer.
type CustomerTaxInfoParams struct {
	TaxID *string `form:"tax_id"`
	Type  *string `form:"type"`
}

// SetSource adds valid sources to a CustomerParams object,
// returning an error for unsupported sources.
func (cp *CustomerParams) SetSource(sp interface{}) error {
	source, err := SourceParamsFor(sp)
	cp.Source = source
	return err
}

// CustomerListParams is the set of parameters that can be used when listing customers.
// For more details see https://stripe.com/docs/api#list_customers.
type CustomerListParams struct {
	ListParams   `form:"*"`
	Created      *int64            `form:"created"`
	CreatedRange *RangeQueryParams `form:"created"`
	Email        *string           `form:"email"`
}

// Customer is the resource representing a Stripe customer.
// For more details see https://stripe.com/docs/api#customers.
type Customer struct {
	AccountBalance   int64                    `json:"account_balance"`
	Address          Address                  `json:"address"`
	Created          int64                    `json:"created"`
	Currency         Currency                 `json:"currency"`
	DefaultSource    *PaymentSource           `json:"default_source"`
	Deleted          bool                     `json:"deleted"`
	Delinquent       bool                     `json:"delinquent"`
	Description      string                   `json:"description"`
	Discount         *Discount                `json:"discount"`
	Email            string                   `json:"email"`
	ID               string                   `json:"id"`
	InvoicePrefix    string                   `json:"invoice_prefix"`
	InvoiceSettings  *CustomerInvoiceSettings `json:"invoice_settings"`
	Livemode         bool                     `json:"livemode"`
	Metadata         map[string]string        `json:"metadata"`
	Name             string                   `json:"name"`
	Phone            string                   `json:"phone"`
	PreferredLocales []string                 `json:"preferred_locales"`
	Shipping         *CustomerShippingDetails `json:"shipping"`
	Sources          *SourceList              `json:"sources"`
	Subscriptions    *SubscriptionList        `json:"subscriptions"`
	TaxExempt        CustomerTaxExempt        `json:"tax_exempt"`
	TaxIDs           *TaxIDList               `json:"tax_ids"`

	// The following properties are deprecated. Use TaxIds instead.
	TaxInfo             *CustomerTaxInfo             `json:"tax_info"`
	TaxInfoVerification *CustomerTaxInfoVerification `json:"tax_info_verification"`
}

// CustomerInvoiceCustomField represents a custom field associated with the customer's invoices.
type CustomerInvoiceCustomField struct {
	Name  *string `form:"name"`
	Value *string `form:"value"`
}

// CustomerInvoiceSettings is the structure containing the default settings for invoices associated
// with this customer.
type CustomerInvoiceSettings struct {
	CustomFields         []*CustomerInvoiceCustomField `json:"custom_fields"`
	DefaultPaymentMethod *PaymentMethod                `json:"default_payment_method"`
	Footer               string                        `json:"footer"`
}

// CustomerList is a list of customers as retrieved from a list endpoint.
type CustomerList struct {
	ListMeta
	Data []*Customer `json:"data"`
}

// CustomerShippingDetails is the structure containing shipping information.
type CustomerShippingDetails struct {
	Address Address `json:"address"`
	Name    string  `json:"name"`
	Phone   string  `json:"phone"`
}

// CustomerTaxInfo is the structure containing tax information for the customer.
type CustomerTaxInfo struct {
	TaxID string              `json:"tax_id"`
	Type  CustomerTaxInfoType `json:"type"`
}

// CustomerTaxInfoVerification is the structure containing tax verification for the customer.
type CustomerTaxInfoVerification struct {
	Status       CustomerTaxInfoVerificationStatus `json:"status"`
	VerifiedName string                            `json:"verified_name"`
}

// UnmarshalJSON handles deserialization of a Customer.
// This custom unmarshaling is needed because the resulting
// property may be an id or the full struct if it was expanded.
func (c *Customer) UnmarshalJSON(data []byte) error {
	if id, ok := ParseID(data); ok {
		c.ID = id
		return nil
	}

	type customer Customer
	var v customer
	if err := json.Unmarshal(data, &v); err != nil {
		return err
	}

	*c = Customer(v)
	return nil
}
<|endoftext|>


<commit_before>package data

import (
	"sync"
)

//TagContainer can be set to a object, then check if contains a tag
type TagContainer interface {
	Contains(names ...string) bool
	Add(names ...string)
	Remove(names ...string)
}

type TagContainerImpl struct {
	sync.Mutex
	data map[string]bool
}

func (t TagContainerImpl) Contains(names ...string) bool {
	t.Lock()
	defer t.Unlock()

	for _, name := range names {
		_, ok := t.data[name]
		if !ok {
			return false
		}
	}

	return true
}

func (t TagContainerImpl) Add(names ...string) {
	t.Lock()
	defer t.Unlock()

	for _, name := range names {
		t.data[name] = true
	}
}

func (t TagContainerImpl) Remove(names ...string) {
	t.Lock()
	defer t.Unlock()

	for _, name := range names {
		delete(t.data, name)
	}
}
<commit_msg>make Tags visible, that can be encoded<commit_after>package data

import (
	"sync"
)

//TagContainer can be set to a object, then check if contains a tag
type TagContainer interface {
	Contains(names ...string) bool
	Add(names ...string)
	Remove(names ...string)
}

type TagContainerImpl struct {
	sync.Mutex
	Tags map[string]bool
}

func (t TagContainerImpl) Contains(names ...string) bool {
	t.Lock()
	defer t.Unlock()

	for _, name := range names {
		_, ok := t.Tags[name]
		if !ok {
			return false
		}
	}

	return true
}

func (t TagContainerImpl) Add(names ...string) {
	t.Lock()
	defer t.Unlock()

	for _, name := range names {
		t.Tags[name] = true
	}
}

func (t TagContainerImpl) Remove(names ...string) {
	t.Lock()
	defer t.Unlock()

	for _, name := range names {
		delete(t.Tags, name)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"net/http"

	// Alias this because we call our connection couchbase
	cb "github.com/couchbaselabs/go-couchbase"
)

var couchbase *cb.Bucket

func dbConnect() (*cb.Bucket, error) {

	cb.HttpClient = &http.Client{
		Transport: TimeoutTransport(*viewTimeout),
	}

	log.Printf("Connecting to couchbase bucket %v at %v",
		*couchbaseBucket, *couchbaseServer)
	rv, err := cb.GetBucket(*couchbaseServer, "default", *couchbaseBucket)
	if err != nil {
		return nil, err
	}
	return rv, nil
}
<commit_msg>Install/update views on server startup.<commit_after>package main

import (
	"encoding/json"
	"log"
	"net/http"
	"time"

	// Alias this because we call our connection couchbase
	cb "github.com/couchbaselabs/go-couchbase"
)

var couchbase *cb.Bucket

type viewMarker struct {
	Version   int       `json:"version"`
	Node      string    `json:"node"`
	Timestamp time.Time `json:"timestamp"`
	Type      string    `json:"type"`
}

const ddocKey = "/@ddocVersion"
const ddocVersion = 1
const designDoc = `
{
    "spatialInfos": [],
    "viewInfos": [
        {
            "map": "function (doc, meta) {\n  if (doc.type === \"file\") {\n    var toEmit = {};\n    toEmit[doc.oid] = meta.id;\n    if (doc.older) {\n      for (var i = 0; i < doc.older.length; i++) {\n        toEmit[doc.older[i].oid] = meta.id;\n      }\n    }\n    for (var k in toEmit) {\n      emit([k, \"file\", meta.id], null);\n    }\n  } else if (doc.type === \"blob\") {\n    for (var node in doc.nodes) {\n      emit([doc.oid, \"blob\", node], null);\n    }\n  }\n}",
            "name": "file_blobs",
            "removeLink": "#removeView=cbfs%2F_design%252Fdev_cbfs%2F_view%2Ffile_blobs",
            "viewLink": "#showView=cbfs%2F_design%252Fdev_cbfs%2F_view%2Ffile_blobs"
        },
        {
            "map": "function (doc, meta) {\n  if(doc.type == \"file\") {  \n    var idarr = meta.id.split(\"/\");\n    emit(idarr, doc.length);\n  }\n}",
            "name": "file_browse",
            "reduce": "_stats",
            "removeLink": "#removeView=cbfs%2F_design%252Fdev_cbfs%2F_view%2Ffile_browse",
            "viewLink": "#showView=cbfs%2F_design%252Fdev_cbfs%2F_view%2Ffile_browse"
        },
        {
            "map": "function (doc, meta) {\n  if (doc.type === \"blob\") {\n    for (var n in doc.nodes) {\n      emit(n, null);\n    }\n  }\n}",
            "name": "node_blobs",
            "reduce": "_count",
            "removeLink": "#removeView=cbfs%2F_design%252Fdev_cbfs%2F_view%2Fnode_blobs",
            "viewLink": "#showView=cbfs%2F_design%252Fdev_cbfs%2F_view%2Fnode_blobs"
        },
        {
            "map": "function (doc, meta) {\n  if (doc.type === \"node\") {\n    emit(meta.id.substring(1), 0);\n  } else if (doc.type === \"blob\") {\n    for (var n in doc.nodes) {\n      emit(n, doc.length);\n    }\n  }\n}",
            "name": "node_size",
            "reduce": "_sum",
            "removeLink": "#removeView=cbfs%2F_design%252Fdev_cbfs%2F_view%2Fnode_size",
            "viewLink": "#showView=cbfs%2F_design%252Fdev_cbfs%2F_view%2Fnode_size"
        },
        {
            "map": "function (doc, meta) {\n  if (doc.type === \"blob\") {\n    var nreps = 0;\n    for (var x in doc.nodes) {\n      nreps++;\n    }\n    emit(nreps, null);\n  }\n}",
            "name": "repcounts",
            "reduce": "_count",
            "removeLink": "#removeView=cbfs%2F_design%252Fdev_cbfs%2F_view%2Frepcounts",
            "viewLink": "#showView=cbfs%2F_design%252Fdev_cbfs%2F_view%2Frepcounts"
        }
    ],
    "views": {
        "file_blobs": {
            "map": "function (doc, meta) {\n  if (doc.type === \"file\") {\n    var toEmit = {};\n    toEmit[doc.oid] = meta.id;\n    if (doc.older) {\n      for (var i = 0; i < doc.older.length; i++) {\n        toEmit[doc.older[i].oid] = meta.id;\n      }\n    }\n    for (var k in toEmit) {\n      emit([k, \"file\", meta.id], null);\n    }\n  } else if (doc.type === \"blob\") {\n    var replicas=0;\n    for (var node in doc.nodes) {\n      replicas++;\n      emit([doc.oid, \"blob\", node], null);\n    }\n    if (replicas === 0) {\n      emit([doc.oid, \"blob\", \"\"], null);\n    }\n  }\n}"
        },
        "file_browse": {
            "map": "function (doc, meta) {\n  if(doc.type == \"file\") {  \n    var idarr = meta.id.split(\"/\");\n    emit(idarr, doc.length);\n  }\n}",
            "reduce": "_stats"
        },
        "node_blobs": {
            "map": "function (doc, meta) {\n  if (doc.type === \"blob\") {\n    for (var n in doc.nodes) {\n      emit(n, null);\n    }\n  }\n}",
            "reduce": "_count"
        },
        "node_size": {
            "map": "function (doc, meta) {\n  if (doc.type === \"node\") {\n    emit(meta.id.substring(1), 0);\n  } else if (doc.type === \"blob\") {\n    for (var n in doc.nodes) {\n      emit(n, doc.length);\n    }\n  }\n}",
            "reduce": "_sum"
        },
        "repcounts": {
            "map": "function (doc, meta) {\n  if (doc.type === \"blob\") {\n    var nreps = 0;\n    for (var x in doc.nodes) {\n      nreps++;\n    }\n    emit(nreps, null);\n  }\n}",
            "reduce": "_count"
        }
    }
}
`

func dbConnect() (*cb.Bucket, error) {

	cb.HttpClient = &http.Client{
		Transport: TimeoutTransport(*viewTimeout),
	}

	log.Printf("Connecting to couchbase bucket %v at %v",
		*couchbaseBucket, *couchbaseServer)
	rv, err := cb.GetBucket(*couchbaseServer, "default", *couchbaseBucket)
	if err != nil {
		return nil, err
	}

	marker := viewMarker{}
	err = rv.Get(ddocKey, &marker)
	if err != nil {
		log.Printf("Error checking view version: %v", err)
	}
	if marker.Version < ddocVersion {
		log.Printf("Installing new version of views (old version=%v)",
			marker.Version)
		doc := json.RawMessage([]byte(designDoc))
		err = rv.PutDDoc("cbfs", &doc)
		if err != nil {
			return nil, err
		}
		marker.Version = ddocVersion
		marker.Node = serverId
		marker.Timestamp = time.Now().UTC()
		marker.Type = "ddocmarker"

		rv.Set(ddocKey, 0, &marker)
	}

	return rv, nil
}
<|endoftext|>


<commit_before>package sqlbuilder

import "strconv"

// DBMS represents a DBMS.
type DBMS int

const (
	MySQL    DBMS = iota // MySQL
	Postgres             // Postgres
)

// Placeholder returns the placeholder string for the given index.
func (dbms DBMS) Placeholder(idx int) string {
	switch dbms {
	case MySQL:
		return "?"
	case Postgres:
		return "$" + strconv.Itoa(idx+1)
	default:
		panic("unknown DBMS")
	}
}

// Select returns a SELECT statement.
func (dbms DBMS) Select(table string) *SelectStatement {
	return &SelectStatement{
		dbms:  dbms,
		table: table,
	}
}

// Insert returns an INSERT statement.
func (dbms DBMS) Insert(table string) *InsertStatement {
	return &InsertStatement{
		dbms:  dbms,
		table: table,
	}
}

// Update returns an UPDATE statement.
func (dbms DBMS) Update(table string) *UpdateStatement {
	return &UpdateStatement{
		dbms:  dbms,
		table: table,
	}
}

// DefaultDBMS is the DBMS used by the package-level Select, Insert and Update functions.
var DefaultDatabase = MySQL

// Select returns a SELECT statement using the default Database.
func Select(table string) *SelectStatement {
	return DefaultDatabase.Select(table)
}

// Insert returns an INSERT statement using the default Database.
func Insert(table string) *InsertStatement {
	return DefaultDatabase.Insert(table)
}

// Update returns an UPDATE statement using the default Database.
func Update(table string) *UpdateStatement {
	return DefaultDatabase.Update(table)
}
<commit_msg>Use DefaultDBMS name as documented<commit_after>package sqlbuilder

import "strconv"

// DBMS represents a DBMS.
type DBMS int

const (
	MySQL    DBMS = iota // MySQL
	Postgres             // Postgres
)

// Placeholder returns the placeholder string for the given index.
func (dbms DBMS) Placeholder(idx int) string {
	switch dbms {
	case MySQL:
		return "?"
	case Postgres:
		return "$" + strconv.Itoa(idx+1)
	default:
		panic("unknown DBMS")
	}
}

// Select returns a SELECT statement.
func (dbms DBMS) Select(table string) *SelectStatement {
	return &SelectStatement{
		dbms:  dbms,
		table: table,
	}
}

// Insert returns an INSERT statement.
func (dbms DBMS) Insert(table string) *InsertStatement {
	return &InsertStatement{
		dbms:  dbms,
		table: table,
	}
}

// Update returns an UPDATE statement.
func (dbms DBMS) Update(table string) *UpdateStatement {
	return &UpdateStatement{
		dbms:  dbms,
		table: table,
	}
}

// DefaultDBMS is the DBMS used by the package-level Select, Insert and Update functions.
var DefaultDBMS = MySQL

// Select returns a SELECT statement using the default Database.
func Select(table string) *SelectStatement {
	return DefaultDBMS.Select(table)
}

// Insert returns an INSERT statement using the default Database.
func Insert(table string) *InsertStatement {
	return DefaultDBMS.Insert(table)
}

// Update returns an UPDATE statement using the default Database.
func Update(table string) *UpdateStatement {
	return DefaultDBMS.Update(table)
}
<|endoftext|>


<commit_before>package logmetrics

import (
	"fmt"
	"log"
	"strconv"
	"strings"
	"syseng/go-timemetrics"
	"time"
)

type dataPoint struct {
	name        string
	value       int64
	metric_type string
}

type dataPointTime struct {
	name string
	time int64
}

type tsdPoint struct {
	data             interface{}
	lastPush         time.Time
	lastCrunchedPush time.Time
}

func (lg *LogGroup) extractTags(data []string) []string {
	tags := make([]string, lg.getNbTags())

	i := 0

	//General tags
	for tagname, position := range lg.tags {
		tags[i] = fmt.Sprintf("%s=%s", tagname, data[position])
		i++
	}

	return tags
}

func (lg *LogGroup) getKeys(data []string) ([]dataPoint, time.Time) {
	y := time.Now().Year()

	tags := lg.extractTags(data)

	nbKeys := len(lg.metrics)
	dataPoints := make([]dataPoint, nbKeys)

	//Time
	var t time.Time
	if data[lg.date_position] == lg.last_date_str {
		t = lg.last_date
	} else {
		var err error
		t, err = time.Parse(lg.date_format, data[lg.date_position])
		if err != nil {
			log.Print(err)
			var nt time.Time
			return nil, nt
		}
	}

	//Keep time around to only parse new dates
	lg.last_date_str = data[lg.date_position]
	lg.last_date = t

	//Patch in year if missing - rfc3164
	if t.Year() == 0 {
		t = time.Date(y, t.Month(), t.Day(), t.Hour(), t.Minute(),
			t.Second(), t.Nanosecond(), t.Location())
	}

	//Make a first pass extracting the data, applying float->int conversion on multiplier
	values := make([]int64, lg.expected_matches+1)
	for position, keyType := range lg.metrics {
		if position == 0 {
			values[position] = 1
		} else {
			var val int64
			var err error
			if keyType.format == "float" {
				var val_float float64
				if val_float, err = strconv.ParseFloat(data[position], 64); err == nil {
					val = int64(val_float * float64(keyType.multiply))
				}
			} else {
				if val, err = strconv.ParseInt(data[position], 10, 64); err == nil {
					val = val * int64(keyType.multiply)
				}
			}

			if err != nil {
				log.Printf("Unable to extract data from value match, %s: %s", err, data[position])
				var nt time.Time
				return nil, nt
			} else {
				values[position] = val
			}
		}
	}

	//Second pass applies operation and create datapoints
	var i = 0
	for position, val := range values {
		//Is the value a metric?
		if keyType, ok := lg.metrics[position]; ok {
			//Key name
			key := fmt.Sprintf("%s.%s.%s %s %s", lg.key_prefix, keyType.key_suffix, "%s %d %s", strings.Join(tags, " "), keyType.tag)

			//Do we need to do any operation on this val?
			for op, opvalues := range keyType.operations {
				for _, op_position := range opvalues {
					//log.Printf("%s %d on pos %d, current val: %d", op, op_position, position, val)
					if op_position != 0 {
						switch op {
						case "add":
							val += values[op_position]

						case "sub":
							val -= values[op_position]
						}
					}
				}
			}

			if val < 0 && lg.fail_operation_warn {
				log.Printf("Values cannot be negative after applying operation. Offending line: %s", data[0])
				var nt time.Time
				return nil, nt
			}

			dataPoints[i] = dataPoint{name: key, value: val, metric_type: keyType.metric_type}
			i++
		}
	}

	return dataPoints, t
}

func getCounterKeys(name string, c timemetrics.Counter) []string {
	t := int(c.GetMaxTime().Unix())

	keys := make([]string, 1)
	keys[0] = fmt.Sprintf(name, "count", t, fmt.Sprintf("%d", c.Count()))

	return keys
}

func getMeterKeyCount(name string, m timemetrics.Meter) []string {
	t := int(m.GetMaxTime().Unix())

	keys := make([]string, 1)
	keys[0] = fmt.Sprintf(name, "count", t, fmt.Sprintf("%d", m.Count()))

	return keys
}

func getMeterKeyRates(name string, m timemetrics.Meter) []string {
	t := int(m.GetMaxEWMATime().Unix())

	keys := make([]string, 3)
	keys[0] = fmt.Sprintf(name, "rate._1min", t, fmt.Sprintf("%.4f", m.Rate1()))
	keys[1] = fmt.Sprintf(name, "rate._5min", t, fmt.Sprintf("%.4f", m.Rate5()))
	keys[2] = fmt.Sprintf(name, "rate._15min", t, fmt.Sprintf("%.4f", m.Rate15()))

	return keys
}

func getHistogramKeys(name string, h timemetrics.Histogram) []string {
	t := int(h.GetMaxTime().Unix())
	ps := h.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999})

	keys := make([]string, 10)

	keys[0] = fmt.Sprintf(name, "min", t, fmt.Sprintf("%d", h.Min()))
	keys[1] = fmt.Sprintf(name, "max", t, fmt.Sprintf("%d", h.Max()))
	keys[2] = fmt.Sprintf(name, "mean", t, fmt.Sprintf("%.4f", h.Mean()))
	keys[3] = fmt.Sprintf(name, "std-dev", t, fmt.Sprintf("%.4f", h.StdDev()))
	keys[4] = fmt.Sprintf(name, "p50", t, fmt.Sprintf("%d", int64(ps[0])))
	keys[5] = fmt.Sprintf(name, "p75", t, fmt.Sprintf("%d", int64(ps[1])))
	keys[6] = fmt.Sprintf(name, "p95", t, fmt.Sprintf("%d", int64(ps[2])))
	keys[7] = fmt.Sprintf(name, "p99", t, fmt.Sprintf("%d", int64(ps[3])))
	keys[8] = fmt.Sprintf(name, "p999", t, fmt.Sprintf("%d", int64(ps[4])))
	keys[9] = fmt.Sprintf(name, "sample_size", t, fmt.Sprintf("%d", h.Sample().Size()))

	return keys
}

func (lg LogGroup) dataPoolHandler(channel_number int, tsd_pushers []chan []string, tsd_channel_number int) error {
	dataPool := make(map[string]*tsdPoint)
	tsd_push := tsd_pushers[tsd_channel_number]

	log.Printf("Datapool[%s:%d] started. Pushing keys to TsdPusher[%d]", lg.name, channel_number, tsd_channel_number)

	//Start the handler
	go func() {

		//Failsafe if anything goes really wrong
		//defer func() {
		//	if r := recover(); r != nil {
		//		log.Printf("Recovered error in %s: %s", lg.name, r)
		//	}
		//}()

		pushStats := make(chan bool)

		var lastTimePushed *time.Time
		lastNbKeys := 0
		for {
			select {
			case data := <-lg.tail_data[channel_number]:
				data_points, point_time := lg.getKeys(data)

				//To start things off
				if lastTimePushed == nil {
					lastTimePushed = &point_time
				}

				for _, data_point := range data_points {
					//New metrics, add
					if _, ok := dataPool[data_point.name]; !ok {
						switch data_point.metric_type {
						case "histogram":
							s := timemetrics.NewExpDecaySample(point_time, lg.histogram_size, lg.histogram_alpha_decay, lg.histogram_rescale_threshold_min)
							dataPool[data_point.name] = &tsdPoint{data: timemetrics.NewHistogram(s),
								lastPush: point_time}
						case "counter":
							dataPool[data_point.name] = &tsdPoint{data: timemetrics.NewCounter(point_time),
								lastPush: point_time}
						case "meter":
							dataPool[data_point.name] = &tsdPoint{data: timemetrics.NewMeter(point_time, lg.interval),
								lastPush: point_time, lastCrunchedPush: point_time}
						default:
							log.Fatalf("Unexpected metric type %s!", data_point.metric_type)
						}
					}

					//Make sure data is ordered or we risk sending duplicate data
					if dataPool[data_point.name].lastPush.Unix() > point_time.Unix() {
						log.Printf("Non-ordered data detected in log file. Its key already had a update at %s in the future. Offending line: %s",
							dataPool[data_point.name].lastPush, data[0])
					}

					switch d := dataPool[data_point.name].data.(type) {
					case timemetrics.Histogram:
						d.Update(point_time, data_point.value)
					case timemetrics.Counter:
						d.Inc(point_time, data_point.value)
					case timemetrics.Meter:
						d.Mark(point_time, data_point.value)
					}
				}

				//Support for log playback - Push when <interval> has pass in the logs, not real time
				if point_time.Sub(*lastTimePushed) > (time.Duration(lg.interval) * time.Second) {
					interval := int(point_time.Sub(*lastTimePushed).Seconds())

					//Update EWMAs
					for _, tsdPoint := range dataPool {
						switch v := tsdPoint.data.(type) {
						case timemetrics.Meter:
							sec_since_last_value := point_time.Sub(v.GetMaxTime()).Seconds()
							sec_since_last_ewma_crunch := point_time.Sub(v.GetMaxEWMATime()).Seconds()

							if sec_since_last_value > float64(interval) || sec_since_last_ewma_crunch > float64(lg.ewmaInterval) {
								v.CrunchEWMA(point_time, interval)
							}
						}
					}

					lastTimePushed = &point_time

					go func() { pushStats <- true }()
				}

			case <-pushStats:
				nbKeys := 0
				for tsd_key, tsdPoint := range dataPool {
					switch v := tsdPoint.data.(type) {
					case timemetrics.Histogram:
						snap := v.Snapshot()

						if snap.GetMaxTime().Unix() > tsdPoint.lastPush.Unix() { //Only push updated metrics
							tsdPoint.lastPush = snap.GetMaxTime()
							keys := getHistogramKeys(tsd_key, snap)
							tsd_push <- keys
						}

						nbKeys += 10
					case timemetrics.Counter:
						snap := v.Snapshot()
						if snap.GetMaxTime().Unix() > tsdPoint.lastPush.Unix() {
							tsd_push <- getCounterKeys(tsd_key, snap)
						}
						nbKeys += 1
					case timemetrics.Meter:
						snap := v.Snapshot()

						if snap.GetMaxTime().Unix() > tsdPoint.lastPush.Unix() {
							tsdPoint.lastPush = snap.GetMaxTime()
							tsd_push <- getMeterKeyCount(tsd_key, snap)
						}

						if snap.GetMaxEWMATime().Unix() > tsdPoint.lastCrunchedPush.Unix() {
							tsdPoint.lastCrunchedPush = snap.GetMaxEWMATime()
							tsd_push <- getMeterKeyRates(tsd_key, snap)
						}

						nbKeys += 4
					}
				}

				if lastNbKeys != nbKeys {
					log.Printf("Datapool[%s:%d] currently tracking %d keys", lg.name, channel_number, nbKeys)
				}

				lastNbKeys = nbKeys
			}
		}
	}()

	return nil
}

func StartDataPools(config *Config, tsd_pushers []chan []string) {
	//Start a queryHandler by log group
	nb_tsd_push := 0
	for _, lg := range config.logGroups {
		for i := 0; i < lg.goroutines; i++ {
			lg.dataPoolHandler(i, tsd_pushers, nb_tsd_push)
			nb_tsd_push = (nb_tsd_push + 1) % config.GetPusherNumber()
		}
	}
}
<commit_msg>Bugfix in the EWMA refresh logic.<commit_after>package logmetrics

import (
	"fmt"
	"log"
	"strconv"
	"strings"
	"syseng/go-timemetrics"
	"time"
)

type dataPoint struct {
	name        string
	value       int64
	metric_type string
}

type dataPointTime struct {
	name string
	time int64
}

type tsdPoint struct {
	data             interface{}
	lastPush         time.Time
	lastCrunchedPush time.Time
}

func (lg *LogGroup) extractTags(data []string) []string {
	tags := make([]string, lg.getNbTags())

	i := 0

	//General tags
	for tagname, position := range lg.tags {
		tags[i] = fmt.Sprintf("%s=%s", tagname, data[position])
		i++
	}

	return tags
}

func (lg *LogGroup) getKeys(data []string) ([]dataPoint, time.Time) {
	y := time.Now().Year()

	tags := lg.extractTags(data)

	nbKeys := len(lg.metrics)
	dataPoints := make([]dataPoint, nbKeys)

	//Time
	var t time.Time
	if data[lg.date_position] == lg.last_date_str {
		t = lg.last_date
	} else {
		var err error
		t, err = time.Parse(lg.date_format, data[lg.date_position])
		if err != nil {
			log.Print(err)
			var nt time.Time
			return nil, nt
		}
	}

	//Keep time around to only parse new dates
	lg.last_date_str = data[lg.date_position]
	lg.last_date = t

	//Patch in year if missing - rfc3164
	if t.Year() == 0 {
		t = time.Date(y, t.Month(), t.Day(), t.Hour(), t.Minute(),
			t.Second(), t.Nanosecond(), t.Location())
	}

	//Make a first pass extracting the data, applying float->int conversion on multiplier
	values := make([]int64, lg.expected_matches+1)
	for position, keyType := range lg.metrics {
		if position == 0 {
			values[position] = 1
		} else {
			var val int64
			var err error
			if keyType.format == "float" {
				var val_float float64
				if val_float, err = strconv.ParseFloat(data[position], 64); err == nil {
					val = int64(val_float * float64(keyType.multiply))
				}
			} else {
				if val, err = strconv.ParseInt(data[position], 10, 64); err == nil {
					val = val * int64(keyType.multiply)
				}
			}

			if err != nil {
				log.Printf("Unable to extract data from value match, %s: %s", err, data[position])
				var nt time.Time
				return nil, nt
			} else {
				values[position] = val
			}
		}
	}

	//Second pass applies operation and create datapoints
	var i = 0
	for position, val := range values {
		//Is the value a metric?
		if keyType, ok := lg.metrics[position]; ok {
			//Key name
			key := fmt.Sprintf("%s.%s.%s %s %s", lg.key_prefix, keyType.key_suffix, "%s %d %s", strings.Join(tags, " "), keyType.tag)

			//Do we need to do any operation on this val?
			for op, opvalues := range keyType.operations {
				for _, op_position := range opvalues {
					//log.Printf("%s %d on pos %d, current val: %d", op, op_position, position, val)
					if op_position != 0 {
						switch op {
						case "add":
							val += values[op_position]

						case "sub":
							val -= values[op_position]
						}
					}
				}
			}

			if val < 0 && lg.fail_operation_warn {
				log.Printf("Values cannot be negative after applying operation. Offending line: %s", data[0])
				var nt time.Time
				return nil, nt
			}

			dataPoints[i] = dataPoint{name: key, value: val, metric_type: keyType.metric_type}
			i++
		}
	}

	return dataPoints, t
}

func getCounterKeys(name string, c timemetrics.Counter) []string {
	t := int(c.GetMaxTime().Unix())

	keys := make([]string, 1)
	keys[0] = fmt.Sprintf(name, "count", t, fmt.Sprintf("%d", c.Count()))

	return keys
}

func getMeterKeyCount(name string, m timemetrics.Meter) []string {
	t := int(m.GetMaxTime().Unix())

	keys := make([]string, 1)
	keys[0] = fmt.Sprintf(name, "count", t, fmt.Sprintf("%d", m.Count()))

	return keys
}

func getMeterKeyRates(name string, m timemetrics.Meter) []string {
	t := int(m.GetMaxEWMATime().Unix())

	keys := make([]string, 3)
	keys[0] = fmt.Sprintf(name, "rate._1min", t, fmt.Sprintf("%.4f", m.Rate1()))
	keys[1] = fmt.Sprintf(name, "rate._5min", t, fmt.Sprintf("%.4f", m.Rate5()))
	keys[2] = fmt.Sprintf(name, "rate._15min", t, fmt.Sprintf("%.4f", m.Rate15()))

	return keys
}

func getHistogramKeys(name string, h timemetrics.Histogram) []string {
	t := int(h.GetMaxTime().Unix())
	ps := h.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999})

	keys := make([]string, 10)

	keys[0] = fmt.Sprintf(name, "min", t, fmt.Sprintf("%d", h.Min()))
	keys[1] = fmt.Sprintf(name, "max", t, fmt.Sprintf("%d", h.Max()))
	keys[2] = fmt.Sprintf(name, "mean", t, fmt.Sprintf("%.4f", h.Mean()))
	keys[3] = fmt.Sprintf(name, "std-dev", t, fmt.Sprintf("%.4f", h.StdDev()))
	keys[4] = fmt.Sprintf(name, "p50", t, fmt.Sprintf("%d", int64(ps[0])))
	keys[5] = fmt.Sprintf(name, "p75", t, fmt.Sprintf("%d", int64(ps[1])))
	keys[6] = fmt.Sprintf(name, "p95", t, fmt.Sprintf("%d", int64(ps[2])))
	keys[7] = fmt.Sprintf(name, "p99", t, fmt.Sprintf("%d", int64(ps[3])))
	keys[8] = fmt.Sprintf(name, "p999", t, fmt.Sprintf("%d", int64(ps[4])))
	keys[9] = fmt.Sprintf(name, "sample_size", t, fmt.Sprintf("%d", h.Sample().Size()))

	return keys
}

func (lg LogGroup) dataPoolHandler(channel_number int, tsd_pushers []chan []string, tsd_channel_number int) error {
	dataPool := make(map[string]*tsdPoint)
	tsd_push := tsd_pushers[tsd_channel_number]

	log.Printf("Datapool[%s:%d] started. Pushing keys to TsdPusher[%d]", lg.name, channel_number, tsd_channel_number)

	//Start the handler
	go func() {

		//Failsafe if anything goes really wrong
		//defer func() {
		//	if r := recover(); r != nil {
		//		log.Printf("Recovered error in %s: %s", lg.name, r)
		//	}
		//}()

		pushStats := make(chan bool)

		var lastTimePushed *time.Time
		lastNbKeys := 0
		for {
			select {
			case data := <-lg.tail_data[channel_number]:
				data_points, point_time := lg.getKeys(data)

				//To start things off
				if lastTimePushed == nil {
					lastTimePushed = &point_time
				}

				for _, data_point := range data_points {
					//New metrics, add
					if _, ok := dataPool[data_point.name]; !ok {
						switch data_point.metric_type {
						case "histogram":
							s := timemetrics.NewExpDecaySample(point_time, lg.histogram_size, lg.histogram_alpha_decay, lg.histogram_rescale_threshold_min)
							dataPool[data_point.name] = &tsdPoint{data: timemetrics.NewHistogram(s),
								lastPush: point_time}
						case "counter":
							dataPool[data_point.name] = &tsdPoint{data: timemetrics.NewCounter(point_time),
								lastPush: point_time}
						case "meter":
							dataPool[data_point.name] = &tsdPoint{data: timemetrics.NewMeter(point_time, lg.interval),
								lastPush: point_time, lastCrunchedPush: point_time}
						default:
							log.Fatalf("Unexpected metric type %s!", data_point.metric_type)
						}
					}

					//Make sure data is ordered or we risk sending duplicate data
					if dataPool[data_point.name].lastPush.Unix() > point_time.Unix() {
						log.Printf("Non-ordered data detected in log file. Its key already had a update at %s in the future. Offending line: %s",
							dataPool[data_point.name].lastPush, data[0])
					}

					switch d := dataPool[data_point.name].data.(type) {
					case timemetrics.Histogram:
						d.Update(point_time, data_point.value)
					case timemetrics.Counter:
						d.Inc(point_time, data_point.value)
					case timemetrics.Meter:
						d.Mark(point_time, data_point.value)
					}
				}

				//Support for log playback - Push when <interval> has pass in the logs, not real time
				if point_time.Sub(*lastTimePushed) > (time.Duration(lg.interval) * time.Second) {
					interval := int(point_time.Sub(*lastTimePushed).Seconds())

					//Update EWMAs
					for _, tsdPoint := range dataPool {
						switch v := tsdPoint.data.(type) {
						case timemetrics.Meter:
							sec_since_last_value := point_time.Sub(v.GetMaxTime()).Unix()
							sec_since_last_ewma_crunch := point_time.Sub(v.GetMaxEWMATime()).Unix()

							if sec_since_last_value > float64(interval) || sec_since_last_ewma_crunch > float64(lg.ewmaInterval) {
								v.CrunchEWMA(point_time, interval)
							}
						}
					}

					lastTimePushed = &point_time

					go func() { pushStats <- true }()
				}

			case <-pushStats:
				nbKeys := 0
				for tsd_key, tsdPoint := range dataPool {
					switch v := tsdPoint.data.(type) {
					case timemetrics.Histogram:
						snap := v.Snapshot()

						if snap.GetMaxTime().Unix() > tsdPoint.lastPush.Unix() { //Only push updated metrics
							tsdPoint.lastPush = snap.GetMaxTime()
							keys := getHistogramKeys(tsd_key, snap)
							tsd_push <- keys
						}

						nbKeys += 10
					case timemetrics.Counter:
						snap := v.Snapshot()
						if snap.GetMaxTime().Unix() > tsdPoint.lastPush.Unix() {
							tsd_push <- getCounterKeys(tsd_key, snap)
						}
						nbKeys += 1
					case timemetrics.Meter:
						snap := v.Snapshot()

						if snap.GetMaxTime().Unix() > tsdPoint.lastPush.Unix() {
							tsdPoint.lastPush = snap.GetMaxTime()
							tsd_push <- getMeterKeyCount(tsd_key, snap)
						}

						if snap.GetMaxEWMATime().Unix() > tsdPoint.lastCrunchedPush.Unix() {
							tsdPoint.lastCrunchedPush = snap.GetMaxEWMATime()
							tsd_push <- getMeterKeyRates(tsd_key, snap)
						}

						nbKeys += 4
					}
				}

				if lastNbKeys != nbKeys {
					log.Printf("Datapool[%s:%d] currently tracking %d keys", lg.name, channel_number, nbKeys)
				}

				lastNbKeys = nbKeys
			}
		}
	}()

	return nil
}

func StartDataPools(config *Config, tsd_pushers []chan []string) {
	//Start a queryHandler by log group
	nb_tsd_push := 0
	for _, lg := range config.logGroups {
		for i := 0; i < lg.goroutines; i++ {
			lg.dataPoolHandler(i, tsd_pushers, nb_tsd_push)
			nb_tsd_push = (nb_tsd_push + 1) % config.GetPusherNumber()
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"
)

func infof(w io.Writer, format string, a ...interface{}) {
	fmt.Fprintf(w, format, a...)
}

func warnf(w io.Writer, format string, a ...interface{}) {
	fmt.Fprintf(w, format, a...)
}

func readManDir() (string, []os.FileInfo) {
	rootDirs := []string{
		"..",
		"/tmp/docker_run/git-lfs",
	}

	var err error
	for _, rootDir := range rootDirs {
		fs, err := ioutil.ReadDir(filepath.Join(rootDir, "docs", "man"))
		if err == nil {
			return rootDir, fs
		}
	}

	warnf(os.Stderr, "Failed to open man dir: %v\n", err)
	os.Exit(2)
	return "", nil
}

var (
	verbose = flag.Bool("verbose", false, "Show verbose output.")
)

// Reads all .ronn files & and converts them to string literals
// triggered by "go generate" comment
// Literals are inserted into a map using an init function, this means
// that there are no compilation errors if 'go generate' hasn't been run, just
// blank man files.
func main() {
	flag.Parse()

	infof(os.Stderr, "Converting man pages into code...\n")
	rootDir, fs := readManDir()
	manDir := filepath.Join(rootDir, "docs", "man")
	out, err := os.Create(filepath.Join(rootDir, "commands", "mancontent_gen.go"))
	if err != nil {
		warnf(os.Stderr, "Failed to create go file: %v\n", err)
		os.Exit(2)
	}
	out.WriteString("package commands\n\nfunc init() {\n")
	out.WriteString("// THIS FILE IS GENERATED, DO NOT EDIT\n")
	out.WriteString("// Use 'go generate ./commands' to update\n")
	fileregex := regexp.MustCompile(`git-lfs(?:-([A-Za-z\-]+))?.\d.ronn`)
	headerregex := regexp.MustCompile(`^###?\s+([A-Za-z0-9 ]+)`)
	// only pick up caps in links to avoid matching optional args
	linkregex := regexp.MustCompile(`\[([A-Z\- ]+)\]`)
	// man links
	manlinkregex := regexp.MustCompile(`(git)(?:-(lfs))?-([a-z\-]+)\(\d\)`)
	count := 0
	for _, f := range fs {
		if match := fileregex.FindStringSubmatch(f.Name()); match != nil {
			infof(os.Stderr, "%v\n", f.Name())
			cmd := match[1]
			if len(cmd) == 0 {
				// This is git-lfs.1.ronn
				cmd = "git-lfs"
			}
			out.WriteString("ManPages[\"" + cmd + "\"] = `")
			contentf, err := os.Open(filepath.Join(manDir, f.Name()))
			if err != nil {
				warnf(os.Stderr, "Failed to open %v: %v\n", f.Name(), err)
				os.Exit(2)
			}
			// Process the ronn to make it nicer as help text
			scanner := bufio.NewScanner(contentf)
			firstHeaderDone := false
			skipNextLineIfBlank := false
			lastLineWasBullet := false
		scanloop:
			for scanner.Scan() {
				line := scanner.Text()
				trimmedline := strings.TrimSpace(line)
				if skipNextLineIfBlank && len(trimmedline) == 0 {
					skipNextLineIfBlank = false
					lastLineWasBullet = false
					continue
				}

				// Special case headers
				if hmatch := headerregex.FindStringSubmatch(line); hmatch != nil {
					header := strings.ToLower(hmatch[1])
					switch header {
					case "synopsis":
						// Ignore this, just go direct to command

					case "description":
						// Just skip the header & newline
						skipNextLineIfBlank = true
					case "options":
						out.WriteString("Options:" + "\n")
					case "see also":
						// don't include any content after this
						break scanloop
					default:
						out.WriteString(strings.ToUpper(header[:1]) + header[1:] + "\n")
						out.WriteString(strings.Repeat("-", len(header)) + "\n")
					}
					firstHeaderDone = true
					lastLineWasBullet = false
					continue
				}

				if lmatches := linkregex.FindAllStringSubmatch(line, -1); lmatches != nil {
					for _, lmatch := range lmatches {
						linktext := strings.ToLower(lmatch[1])
						line = strings.Replace(line, lmatch[0], `"`+strings.ToUpper(linktext[:1])+linktext[1:]+`"`, 1)
					}
				}
				if manmatches := manlinkregex.FindAllStringSubmatch(line, -1); manmatches != nil {
					for _, manmatch := range manmatches {
						line = strings.Replace(line, manmatch[0], strings.Join(manmatch[1:], " "), 1)
					}
				}

				// Skip content until after first header
				if !firstHeaderDone {
					continue
				}
				// OK, content here

				// remove characters that markdown would render invisible in a text env.
				for _, invis := range []string{"`", "
"} {
					line = strings.Replace(line, invis, "", -1)
				}

				// indent bullets
				if strings.HasPrefix(line, "*") {
					lastLineWasBullet = true
				} else if lastLineWasBullet && !strings.HasPrefix(line, " ") {
					// indent paragraphs under bullets if not already done
					line = "  " + line
				}

				out.WriteString(line + "\n")
			}
			out.WriteString("`\n")
			contentf.Close()
			count++
		}
	}
	out.WriteString("}\n")
	infof(os.Stderr, "Successfully processed %d man pages.\n", count)

}
<commit_msg>docs/man/mangen.go: don't show output without --verbose<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"
)

func infof(w io.Writer, format string, a ...interface{}) {
	if !verbose {
		return
	}
	fmt.Fprintf(w, format, a...)
}

func warnf(w io.Writer, format string, a ...interface{}) {
	fmt.Fprintf(w, format, a...)
}

func readManDir() (string, []os.FileInfo) {
	rootDirs := []string{
		"..",
		"/tmp/docker_run/git-lfs",
	}

	var err error
	for _, rootDir := range rootDirs {
		fs, err := ioutil.ReadDir(filepath.Join(rootDir, "docs", "man"))
		if err == nil {
			return rootDir, fs
		}
	}

	warnf(os.Stderr, "Failed to open man dir: %v\n", err)
	os.Exit(2)
	return "", nil
}

var (
	verbose = flag.Bool("verbose", false, "Show verbose output.")
)

// Reads all .ronn files & and converts them to string literals
// triggered by "go generate" comment
// Literals are inserted into a map using an init function, this means
// that there are no compilation errors if 'go generate' hasn't been run, just
// blank man files.
func main() {
	flag.Parse()

	infof(os.Stderr, "Converting man pages into code...\n")
	rootDir, fs := readManDir()
	manDir := filepath.Join(rootDir, "docs", "man")
	out, err := os.Create(filepath.Join(rootDir, "commands", "mancontent_gen.go"))
	if err != nil {
		warnf(os.Stderr, "Failed to create go file: %v\n", err)
		os.Exit(2)
	}
	out.WriteString("package commands\n\nfunc init() {\n")
	out.WriteString("// THIS FILE IS GENERATED, DO NOT EDIT\n")
	out.WriteString("// Use 'go generate ./commands' to update\n")
	fileregex := regexp.MustCompile(`git-lfs(?:-([A-Za-z\-]+))?.\d.ronn`)
	headerregex := regexp.MustCompile(`^###?\s+([A-Za-z0-9 ]+)`)
	// only pick up caps in links to avoid matching optional args
	linkregex := regexp.MustCompile(`\[([A-Z\- ]+)\]`)
	// man links
	manlinkregex := regexp.MustCompile(`(git)(?:-(lfs))?-([a-z\-]+)\(\d\)`)
	count := 0
	for _, f := range fs {
		if match := fileregex.FindStringSubmatch(f.Name()); match != nil {
			infof(os.Stderr, "%v\n", f.Name())
			cmd := match[1]
			if len(cmd) == 0 {
				// This is git-lfs.1.ronn
				cmd = "git-lfs"
			}
			out.WriteString("ManPages[\"" + cmd + "\"] = `")
			contentf, err := os.Open(filepath.Join(manDir, f.Name()))
			if err != nil {
				warnf(os.Stderr, "Failed to open %v: %v\n", f.Name(), err)
				os.Exit(2)
			}
			// Process the ronn to make it nicer as help text
			scanner := bufio.NewScanner(contentf)
			firstHeaderDone := false
			skipNextLineIfBlank := false
			lastLineWasBullet := false
		scanloop:
			for scanner.Scan() {
				line := scanner.Text()
				trimmedline := strings.TrimSpace(line)
				if skipNextLineIfBlank && len(trimmedline) == 0 {
					skipNextLineIfBlank = false
					lastLineWasBullet = false
					continue
				}

				// Special case headers
				if hmatch := headerregex.FindStringSubmatch(line); hmatch != nil {
					header := strings.ToLower(hmatch[1])
					switch header {
					case "synopsis":
						// Ignore this, just go direct to command

					case "description":
						// Just skip the header & newline
						skipNextLineIfBlank = true
					case "options":
						out.WriteString("Options:" + "\n")
					case "see also":
						// don't include any content after this
						break scanloop
					default:
						out.WriteString(strings.ToUpper(header[:1]) + header[1:] + "\n")
						out.WriteString(strings.Repeat("-", len(header)) + "\n")
					}
					firstHeaderDone = true
					lastLineWasBullet = false
					continue
				}

				if lmatches := linkregex.FindAllStringSubmatch(line, -1); lmatches != nil {
					for _, lmatch := range lmatches {
						linktext := strings.ToLower(lmatch[1])
						line = strings.Replace(line, lmatch[0], `"`+strings.ToUpper(linktext[:1])+linktext[1:]+`"`, 1)
					}
				}
				if manmatches := manlinkregex.FindAllStringSubmatch(line, -1); manmatches != nil {
					for _, manmatch := range manmatches {
						line = strings.Replace(line, manmatch[0], strings.Join(manmatch[1:], " "), 1)
					}
				}

				// Skip content until after first header
				if !firstHeaderDone {
					continue
				}
				// OK, content here

				// remove characters that markdown would render invisible in a text env.
				for _, invis := range []string{"`", "
"} {
					line = strings.Replace(line, invis, "", -1)
				}

				// indent bullets
				if strings.HasPrefix(line, "*") {
					lastLineWasBullet = true
				} else if lastLineWasBullet && !strings.HasPrefix(line, " ") {
					// indent paragraphs under bullets if not already done
					line = "  " + line
				}

				out.WriteString(line + "\n")
			}
			out.WriteString("`\n")
			contentf.Close()
			count++
		}
	}
	out.WriteString("}\n")
	infof(os.Stderr, "Successfully processed %d man pages.\n", count)

}
<|endoftext|>


<commit_before>package dynago

import (
	"encoding/json"
	"time"
)

const (
	HASH_KEY_TYPE  = "HASH"
	RANGE_KEY_TYPE = "RANGE"

	TABLE_STATUS_CREATING = "CREATING"
	TABLE_STATUS_DELETING = "DELETING"
	TABLE_STATUS_UPDATING = "UPDATING"
	TABLE_STATUS_ACTIVE   = "ACTIVE"

	STREAM_VIEW_NEW  = "NEW_IMAGE"
	STREAM_VIEW_OLD  = "OLD_IMAGE"
	STREAM_VIEW_ALL  = "NEW_AND_OLD_IMAGES"
	STREAM_VIEW_KEYS = "KEYS_ONLY"

	STREAM_VIEW_DISABLED = "NO" // this is NOT a real value, it tells the API to disable streams for the table
)

// EpochTime is like Time, but unmarshal from a number (seconds since Unix epoch) instead of a formatted string
// (this is what AWS returns)

type EpochTime struct {
	time.Time
}

// Unmarshal from number to time.Time

func (t *EpochTime) UnmarshalJSON(data []byte) (err error) {
	var v float64
	if err = json.Unmarshal(data, &v); err != nil {
		return
	}

	*t = EpochTime{time.Unix(int64(v), 0)} // need to convert the fractional part in nanoseconds
	return nil
}

// Table definition

type AttributeDefinition struct {
	AttributeName string
	AttributeType string
}

type KeySchemaElement struct {
	AttributeName string
	KeyType       string
}

type ProjectionDescription struct {
	NonKeyAttributes []string
	ProjectionType   string
}

type LocalSecondaryIndexDescription struct {
	IndexName      string
	IndexSizeBytes int64
	ItemCount      int64

	KeySchema  []KeySchemaElement
	Projection ProjectionDescription
}

type ProvisionedThroughputDescription struct {
	LastDecreaseDateTime   EpochTime
	LastIncreaseDateTime   EpochTime
	NumberOfDecreasesToday int
	ReadCapacityUnits      int
	WriteCapacityUnits     int
}

type TableDescription struct {
	AttributeDefinitions []AttributeDefinition

	CreationDateTime EpochTime
	ItemCount        int64

	KeySchema             []KeySchemaElement
	LocalSecondaryIndexes []LocalSecondaryIndexDescription
	ProvisionedThroughput ProvisionedThroughputDescription

	TableName      string
	TableSizeBytes int64
	TableStatus    string

	StreamSpecification StreamSpecification
}

type StreamSpecification struct {
	StreamEnabled  bool
	StreamViewType string `json:",omitempty"`
}

//////////////////////////////////////////////////////////////////////////////
//
// ListTables
//

type ListTablesResult struct {
	TableNames []string
}

func (db *DBClient) ListTables() ([]string, error) {
	var listRes ListTablesResult
	if err := db.Query("ListTables", nil).Decode(&listRes); err != nil {
		return nil, err
	} else {
		return listRes.TableNames, nil
	}
}

//////////////////////////////////////////////////////////////////////////////
//
// DescribeTable
//

type DescribeTableRequest struct {
	TableName string
}

type DescribeTableResult struct {
	Table TableDescription
}

func (db *DBClient) DescribeTable(tableName string) (*TableDescription, error) {
	var descRes DescribeTableResult

	if err := db.Query("DescribeTable", DescribeTableRequest{tableName}).Decode(&descRes); err != nil {
		return nil, err
	}

	return &descRes.Table, nil
}

//////////////////////////////////////////////////////////////////////////////
//
// CreateTable
//

type ProvisionedThroughputRequest struct {
	ReadCapacityUnits  int
	WriteCapacityUnits int
}

type LocalSecondaryIndexRequest struct {
	IndexName  string
	KeySchema  []KeySchemaElement
	Projection ProjectionDescription
}

type CreateTableRequest struct {
	TableName             string
	ProvisionedThroughput ProvisionedThroughputRequest
	AttributeDefinitions  []AttributeDefinition
	KeySchema             []KeySchemaElement
	LocalSecondaryIndexes []LocalSecondaryIndexRequest
	StreamSpecification   StreamSpecification
}

type CreateTableResult struct {
	TableDescription TableDescription
}

func (db *DBClient) CreateTable(tableName string, hashKey AttributeDefinition, rangeKey AttributeDefinition, rc, wc int, streamView string) (*TableDescription, error) {
	createReq := CreateTableRequest{
		TableName:             tableName,
		ProvisionedThroughput: ProvisionedThroughputRequest{rc, wc},
	}

	attrs := []AttributeDefinition{hashKey}
	schema := []KeySchemaElement{KeySchemaElement{hashKey.AttributeName, HASH_KEY_TYPE}}
	if len(rangeKey.AttributeName) > 0 {
		attrs = append(attrs, rangeKey)
		schema = append(schema, KeySchemaElement{rangeKey.AttributeName, RANGE_KEY_TYPE})
	}

	createReq.AttributeDefinitions = attrs
	createReq.KeySchema = schema

	if streamView == STREAM_VIEW_DISABLED {
		createReq.StreamSpecification.StreamEnabled = false
	} else if len(streamView) > 0 {
		createReq.StreamSpecification.StreamEnabled = true
		createReq.StreamSpecification.StreamViewType = streamView
	}

	var createRes CreateTableResult

	if err := db.Query("CreateTable", createReq).Decode(&createRes); err != nil {
		return nil, err
	}

	return &createRes.TableDescription, nil
}

//////////////////////////////////////////////////////////////////////////////
//
// UpdateTable
//

type UpdateTableRequest struct {
	TableName string

	// TODO: add secondary indices

	ProvisionedThroughput *ProvisionedThroughputRequest `json:",omitempty"`
	StreamSpecification   *StreamSpecification          `json:",omitempty"`
}

type UpdateTableResult struct {
	TableDescription TableDescription
}

func (db *DBClient) UpdateTable(tableName string, rc, wc int, streamView string) (*TableDescription, error) {
	updReq := UpdateTableRequest{
		TableName: tableName,
	}

	if rc > 0 && wc > 0 {
		updReq.ProvisionedThroughput = &ProvisionedThroughputRequest{rc, wc}
	}

	if streamView == STREAM_VIEW_DISABLED {
		updReq.StreamSpecification = &StreamSpecification{StreamEnabled: false}
	} else if len(streamView) > 0 {
		updReq.StreamSpecification = &StreamSpecification{StreamEnabled: true, StreamViewType: streamView}
	}

	var updRes UpdateTableResult

	if err := db.Query("UpdateTable", updReq).Decode(&updRes); err != nil {
		return nil, err
	}

	return &updRes.TableDescription, nil
}

//////////////////////////////////////////////////////////////////////////////
//
// DeleteTable
//

type DeleteTableRequest struct {
	TableName string
}

type DeleteTableResult struct {
	Table TableDescription
}

func (db *DBClient) DeleteTable(tableName string) (*TableDescription, error) {
	var delRes DeleteTableResult

	if err := db.Query("DeleteTable", DeleteTableRequest{tableName}).Decode(&delRes); err != nil {
		return nil, err
	}

	return &delRes.Table, nil
}

//////////////////////////////////////////////////////////////////////////////
//
// TableInstance
//

type TableInstance struct {
	DB   *DBClient
	Name string
	Keys map[string]*AttributeDefinition
}

func (db *DBClient) GetTable(tableName string) (*TableInstance, error) {

	desc, err := db.DescribeTable(tableName)
	if err != nil {
		return nil, err
	}

	table := TableInstance{DB: db, Name: desc.TableName, Keys: map[string]*AttributeDefinition{}}

	for _, ks := range desc.KeySchema {
		table.Keys[ks.KeyType] = desc.getAttribute(ks.AttributeName)

	}

	return &table, nil
}

func (table *TableDescription) getAttribute(name string) *AttributeDefinition {
	for _, a := range table.AttributeDefinitions {
		if a.AttributeName == name {
			return &a
		}
	}

	return nil
}

//
// HashRange returns true if this table requires hash and range key
func (table *TableInstance) HashRange() bool {
	return table.Keys[RANGE_KEY_TYPE] != nil
}

func (table *TableInstance) GetItem(hashKey interface{}, rangeKey interface{}, attributes []string, consistent bool, consumed bool) (map[string]interface{}, float32, error) {
	hkey := &KeyValue{*table.Keys[HASH_KEY_TYPE], hashKey}

	var rkey *KeyValue
	if table.Keys[RANGE_KEY_TYPE] != nil {
		rkey = &KeyValue{*table.Keys[RANGE_KEY_TYPE], rangeKey}
	}

	return table.DB.GetItem(table.Name, hkey, rkey, attributes, consistent, consumed)
}

func (table *TableInstance) PutItem(item Item, options ...ItemOption) (*Item, float32, error) {
	return table.DB.PutItem(table.Name, item, options...)
}

func (table *TableInstance) UpdateItem(hashKey interface{}, rangeKey interface{}, updates string, options ...ItemOption) (*Item, float32, error) {
	hkey := &KeyValue{*table.Keys[HASH_KEY_TYPE], hashKey}

	var rkey *KeyValue
	if table.Keys[RANGE_KEY_TYPE] != nil {
		rkey = &KeyValue{*table.Keys[RANGE_KEY_TYPE], rangeKey}
	}

	return table.DB.UpdateItem(table.Name, hkey, rkey, updates, options...)
}

func (table *TableInstance) DeleteItem(hashKey interface{}, rangeKey interface{}, options ...ItemOption) (*Item, float32, error) {
	hkey := &KeyValue{*table.Keys[HASH_KEY_TYPE], hashKey}

	var rkey *KeyValue
	if table.Keys[RANGE_KEY_TYPE] != nil {
		rkey = &KeyValue{*table.Keys[RANGE_KEY_TYPE], rangeKey}
	}

	return table.DB.DeleteItem(table.Name, hkey, rkey, options...)
}

func (table *TableInstance) Query(hashKey interface{}) *QueryRequest {
	query := QueryTable(table)
	hkey := *table.HashKey()
	query = query.SetCondition(hkey.AttributeName, EQ(EncodeAttributeValue(hkey, hashKey)))

	return query
}

func (table *TableInstance) RangeKey() *AttributeDefinition {
	return table.Keys[RANGE_KEY_TYPE]
}

func (table *TableInstance) HashKey() *AttributeDefinition {
	return table.Keys[HASH_KEY_TYPE]
}
<commit_msg>changed CreateTable to support multiple attribute definitions (more than just hash and range key)<commit_after>package dynago

import (
	"encoding/json"
	"errors"
	"time"
)

const (
	HASH_KEY_TYPE  = "HASH"
	RANGE_KEY_TYPE = "RANGE"

	TABLE_STATUS_CREATING = "CREATING"
	TABLE_STATUS_DELETING = "DELETING"
	TABLE_STATUS_UPDATING = "UPDATING"
	TABLE_STATUS_ACTIVE   = "ACTIVE"

	STREAM_VIEW_NEW  = "NEW_IMAGE"
	STREAM_VIEW_OLD  = "OLD_IMAGE"
	STREAM_VIEW_ALL  = "NEW_AND_OLD_IMAGES"
	STREAM_VIEW_KEYS = "KEYS_ONLY"

	STREAM_VIEW_DISABLED = "NO" // this is NOT a real value, it tells the API to disable streams for the table
)

var (
	ERR_MISSING_KEY   = errors.New("hash-key required")
	ERR_TOO_MANY_KEYS = errors.New("too many keys")
)

// EpochTime is like Time, but unmarshal from a number (seconds since Unix epoch) instead of a formatted string
// (this is what AWS returns)

type EpochTime struct {
	time.Time
}

// Unmarshal from number to time.Time

func (t *EpochTime) UnmarshalJSON(data []byte) (err error) {
	var v float64
	if err = json.Unmarshal(data, &v); err != nil {
		return
	}

	*t = EpochTime{time.Unix(int64(v), 0)} // need to convert the fractional part in nanoseconds
	return nil
}

// Table definition

type AttributeDefinition struct {
	AttributeName string
	AttributeType string
}

type KeySchemaElement struct {
	AttributeName string
	KeyType       string
}

type ProjectionDescription struct {
	NonKeyAttributes []string
	ProjectionType   string
}

type LocalSecondaryIndexDescription struct {
	IndexName      string
	IndexSizeBytes int64
	ItemCount      int64

	KeySchema  []KeySchemaElement
	Projection ProjectionDescription
}

type ProvisionedThroughputDescription struct {
	LastDecreaseDateTime   EpochTime
	LastIncreaseDateTime   EpochTime
	NumberOfDecreasesToday int
	ReadCapacityUnits      int
	WriteCapacityUnits     int
}

type TableDescription struct {
	AttributeDefinitions []AttributeDefinition

	CreationDateTime EpochTime
	ItemCount        int64

	KeySchema             []KeySchemaElement
	LocalSecondaryIndexes []LocalSecondaryIndexDescription
	ProvisionedThroughput ProvisionedThroughputDescription

	TableName      string
	TableSizeBytes int64
	TableStatus    string

	StreamSpecification StreamSpecification
}

type StreamSpecification struct {
	StreamEnabled  bool
	StreamViewType string `json:",omitempty"`
}

//////////////////////////////////////////////////////////////////////////////
//
// ListTables
//

type ListTablesResult struct {
	TableNames []string
}

func (db *DBClient) ListTables() ([]string, error) {
	var listRes ListTablesResult
	if err := db.Query("ListTables", nil).Decode(&listRes); err != nil {
		return nil, err
	} else {
		return listRes.TableNames, nil
	}
}

//////////////////////////////////////////////////////////////////////////////
//
// DescribeTable
//

type DescribeTableRequest struct {
	TableName string
}

type DescribeTableResult struct {
	Table TableDescription
}

func (db *DBClient) DescribeTable(tableName string) (*TableDescription, error) {
	var descRes DescribeTableResult

	if err := db.Query("DescribeTable", DescribeTableRequest{tableName}).Decode(&descRes); err != nil {
		return nil, err
	}

	return &descRes.Table, nil
}

//////////////////////////////////////////////////////////////////////////////
//
// CreateTable
//

type ProvisionedThroughputRequest struct {
	ReadCapacityUnits  int
	WriteCapacityUnits int
}

type LocalSecondaryIndexRequest struct {
	IndexName  string
	KeySchema  []KeySchemaElement
	Projection ProjectionDescription
}

type CreateTableRequest struct {
	TableName             string
	ProvisionedThroughput ProvisionedThroughputRequest
	AttributeDefinitions  []AttributeDefinition
	KeySchema             []KeySchemaElement
	LocalSecondaryIndexes []LocalSecondaryIndexRequest
	StreamSpecification   StreamSpecification
}

type CreateTableResult struct {
	TableDescription TableDescription
}

func (db *DBClient) CreateTable(tableName string, attributes []AttributeDefinition, keys []string, rc, wc int, streamView string) (*TableDescription, error) {
	createReq := CreateTableRequest{
		TableName:             tableName,
		ProvisionedThroughput: ProvisionedThroughputRequest{rc, wc},
	}

	if len(keys) < 1 {
		return nil, ERR_MISSING_KEY
	}
	if len(keys) > 2 {
		return nil, ERR_TOO_MANY_KEYS
	}

	schema := []KeySchemaElement{KeySchemaElement{keys[0], HASH_KEY_TYPE}}
	if len(keys) > 1 {
		schema = append(schema, KeySchemaElement{keys[1], RANGE_KEY_TYPE})
	}

	createReq.AttributeDefinitions = attributes
	createReq.KeySchema = schema

	if streamView == STREAM_VIEW_DISABLED {
		createReq.StreamSpecification.StreamEnabled = false
	} else if len(streamView) > 0 {
		createReq.StreamSpecification.StreamEnabled = true
		createReq.StreamSpecification.StreamViewType = streamView
	}

	var createRes CreateTableResult

	if err := db.Query("CreateTable", createReq).Decode(&createRes); err != nil {
		return nil, err
	}

	return &createRes.TableDescription, nil
}

//////////////////////////////////////////////////////////////////////////////
//
// UpdateTable
//

type UpdateTableRequest struct {
	TableName string

	// TODO: add secondary indices

	ProvisionedThroughput *ProvisionedThroughputRequest `json:",omitempty"`
	StreamSpecification   *StreamSpecification          `json:",omitempty"`
}

type UpdateTableResult struct {
	TableDescription TableDescription
}

func (db *DBClient) UpdateTable(tableName string, rc, wc int, streamView string) (*TableDescription, error) {
	updReq := UpdateTableRequest{
		TableName: tableName,
	}

	if rc > 0 && wc > 0 {
		updReq.ProvisionedThroughput = &ProvisionedThroughputRequest{rc, wc}
	}

	if streamView == STREAM_VIEW_DISABLED {
		updReq.StreamSpecification = &StreamSpecification{StreamEnabled: false}
	} else if len(streamView) > 0 {
		updReq.StreamSpecification = &StreamSpecification{StreamEnabled: true, StreamViewType: streamView}
	}

	var updRes UpdateTableResult

	if err := db.Query("UpdateTable", updReq).Decode(&updRes); err != nil {
		return nil, err
	}

	return &updRes.TableDescription, nil
}

//////////////////////////////////////////////////////////////////////////////
//
// DeleteTable
//

type DeleteTableRequest struct {
	TableName string
}

type DeleteTableResult struct {
	Table TableDescription
}

func (db *DBClient) DeleteTable(tableName string) (*TableDescription, error) {
	var delRes DeleteTableResult

	if err := db.Query("DeleteTable", DeleteTableRequest{tableName}).Decode(&delRes); err != nil {
		return nil, err
	}

	return &delRes.Table, nil
}

//////////////////////////////////////////////////////////////////////////////
//
// TableInstance
//

type TableInstance struct {
	DB   *DBClient
	Name string
	Keys map[string]*AttributeDefinition
}

func (db *DBClient) GetTable(tableName string) (*TableInstance, error) {

	desc, err := db.DescribeTable(tableName)
	if err != nil {
		return nil, err
	}

	table := TableInstance{DB: db, Name: desc.TableName, Keys: map[string]*AttributeDefinition{}}

	for _, ks := range desc.KeySchema {
		table.Keys[ks.KeyType] = desc.getAttribute(ks.AttributeName)

	}

	return &table, nil
}

func (table *TableDescription) getAttribute(name string) *AttributeDefinition {
	for _, a := range table.AttributeDefinitions {
		if a.AttributeName == name {
			return &a
		}
	}

	return nil
}

//
// HashRange returns true if this table requires hash and range key
func (table *TableInstance) HashRange() bool {
	return table.Keys[RANGE_KEY_TYPE] != nil
}

func (table *TableInstance) GetItem(hashKey interface{}, rangeKey interface{}, attributes []string, consistent bool, consumed bool) (map[string]interface{}, float32, error) {
	hkey := &KeyValue{*table.Keys[HASH_KEY_TYPE], hashKey}

	var rkey *KeyValue
	if table.Keys[RANGE_KEY_TYPE] != nil {
		rkey = &KeyValue{*table.Keys[RANGE_KEY_TYPE], rangeKey}
	}

	return table.DB.GetItem(table.Name, hkey, rkey, attributes, consistent, consumed)
}

func (table *TableInstance) PutItem(item Item, options ...ItemOption) (*Item, float32, error) {
	return table.DB.PutItem(table.Name, item, options...)
}

func (table *TableInstance) UpdateItem(hashKey interface{}, rangeKey interface{}, updates string, options ...ItemOption) (*Item, float32, error) {
	hkey := &KeyValue{*table.Keys[HASH_KEY_TYPE], hashKey}

	var rkey *KeyValue
	if table.Keys[RANGE_KEY_TYPE] != nil {
		rkey = &KeyValue{*table.Keys[RANGE_KEY_TYPE], rangeKey}
	}

	return table.DB.UpdateItem(table.Name, hkey, rkey, updates, options...)
}

func (table *TableInstance) DeleteItem(hashKey interface{}, rangeKey interface{}, options ...ItemOption) (*Item, float32, error) {
	hkey := &KeyValue{*table.Keys[HASH_KEY_TYPE], hashKey}

	var rkey *KeyValue
	if table.Keys[RANGE_KEY_TYPE] != nil {
		rkey = &KeyValue{*table.Keys[RANGE_KEY_TYPE], rangeKey}
	}

	return table.DB.DeleteItem(table.Name, hkey, rkey, options...)
}

func (table *TableInstance) Query(hashKey interface{}) *QueryRequest {
	query := QueryTable(table)
	hkey := *table.HashKey()
	query = query.SetCondition(hkey.AttributeName, EQ(EncodeAttributeValue(hkey, hashKey)))

	return query
}

func (table *TableInstance) RangeKey() *AttributeDefinition {
	return table.Keys[RANGE_KEY_TYPE]
}

func (table *TableInstance) HashKey() *AttributeDefinition {
	return table.Keys[HASH_KEY_TYPE]
}
<|endoftext|>