<commit_before>package decor

import (
	"fmt"
	"math"
	"time"
	"unicode/utf8"

	"github.com/VividCortex/ewma"
)

const (
	// DidentRight bit specifies identation direction.
	// |foo   |b     | With DidentRight
	// |   foo|     b| Without DidentRight
	DidentRight = 1 << iota

	// DextraSpace bit adds extra space, makes sense with DSyncWidth only.
	// When DidentRight bit set, the space will be added to the right,
	// otherwise to the left.
	DextraSpace

	// DSyncWidth bit enables same column width synchronization.
	// Effective with multiple bars only.
	DSyncWidth

	// DSyncWidthR is shortcut for DSyncWidth|DidentRight
	DSyncWidthR = DSyncWidth | DidentRight

	// DSyncSpace is shortcut for DSyncWidth|DextraSpace
	DSyncSpace = DSyncWidth | DextraSpace

	// DSyncSpaceR is shortcut for DSyncWidth|DextraSpace|DidentRight
	DSyncSpaceR = DSyncWidth | DextraSpace | DidentRight
)

const (
	ET_STYLE_GO = iota
	ET_STYLE_HHMMSS
	ET_STYLE_HHMM
	ET_STYLE_MMSS
)

// Statistics contains values useful for implementing a DecoratorFunc.
type Statistics struct {
	ID          int
	Completed   bool
	Total       int64
	Current     int64
	StartTime   time.Time
	TimeElapsed time.Duration
}

// Decorator is an interface with one method:
// Decor(st *Statistics, widthAccumulator chan<- int, widthDistributor <-chan int) string
// All decorators in this package implement this interface.
type Decorator interface {
	Decor(*Statistics, chan<- int, <-chan int) string
}

type CompleteMessenger interface {
	OnComplete(string, ...WC)
}

// DecoratorFunc is an adapter for Decorator interface
type DecoratorFunc func(*Statistics, chan<- int, <-chan int) string

func (f DecoratorFunc) Decor(s *Statistics, widthAccumulator chan<- int, widthDistributor <-chan int) string {
	return f(s, widthAccumulator, widthDistributor)
}

// WC is a struct with two public fields W and C, both of int type.
// W represents width and C represents bit set of width related config.
type WC struct {
	W      int
	C      int
	format string
}

func (wc WC) formatMsg(msg string, widthAccumulator chan<- int, widthDistributor <-chan int) string {
	format := wc.buildFormat()
	if (wc.C & DSyncWidth) != 0 {
		widthAccumulator <- utf8.RuneCountInString(msg)
		max := <-widthDistributor
		if max == 0 {
			max = wc.W
		}
		if (wc.C & DextraSpace) != 0 {
			max++
		}
		return fmt.Sprintf(fmt.Sprintf(format, max), msg)
	}
	return fmt.Sprintf(fmt.Sprintf(format, wc.W), msg)
}

func (wc *WC) buildFormat() string {
	if wc.format != "" {
		return wc.format
	}
	wc.format = "%%"
	if (wc.C & DidentRight) != 0 {
		wc.format += "-"
	}
	wc.format += "%ds"
	return wc.format
}

// Global convenience shortcuts
var (
	WCSyncWidth  = WC{C: DSyncWidth}
	WCSyncWidthR = WC{C: DSyncWidthR}
	WCSyncSpace  = WC{C: DSyncSpace}
	WCSyncSpaceR = WC{C: DSyncSpaceR}
)

// OnComplete returns decorator, which wraps provided decorator, with sole
// purpose to display provided message on complete event.
//
//	`decorator` Decorator to wrap
//
//	`message` message to display on complete event
//
//	`wc` optional WC config
func OnComplete(decorator Decorator, message string, wc ...WC) Decorator {
	if cm, ok := decorator.(CompleteMessenger); ok {
		cm.OnComplete(message, wc...)
		return decorator
	}
	msgDecorator := Name(message, wc...)
	return DecoratorFunc(func(s *Statistics, widthAccumulator chan<- int, widthDistributor <-chan int) string {
		if s.Completed {
			return msgDecorator.Decor(s, widthAccumulator, widthDistributor)
		}
		return decorator.Decor(s, widthAccumulator, widthDistributor)
	})
}

// StaticName returns name decorator.
//
//	`name` string to display
//
//	`wc` optional WC config
func StaticName(name string, wc ...WC) Decorator {
	return Name(name, wc...)
}

// Name returns name decorator.
//
//	`name` string to display
//
//	`wc` optional WC config
func Name(name string, wc ...WC) Decorator {
	var wc0 WC
	if len(wc) > 0 {
		wc0 = wc[0]
	}
	return DecoratorFunc(func(s *Statistics, widthAccumulator chan<- int, widthDistributor <-chan int) string {
		return wc0.formatMsg(name, widthAccumulator, widthDistributor)
	})
}

// CountersNoUnit returns raw counters decorator
//
//	`pairFormat` printf compatible verbs for current and total, like "%f" or "%d"
//
//	`wc` optional WC config
func CountersNoUnit(pairFormat string, wc ...WC) Decorator {
	return counters(pairFormat, 0, wc...)
}

// CountersKibiByte returns human friendly byte counters decorator, where counters unit is multiple by 1024.
//
//	`pairFormat` printf compatible verbs for current and total, like "%f" or "%d"
//
//	`wc` optional WC config
//
// pairFormat example:
//
//	"%.1f / %.1f" = "1.0MiB / 12.0MiB" or "% .1f / % .1f" = "1.0 MiB / 12.0 MiB"
func CountersKibiByte(pairFormat string, wc ...WC) Decorator {
	return counters(pairFormat, unitKiB, wc...)
}

// CountersKiloByte returns human friendly byte counters decorator, where counters unit is multiple by 1000.
//
//	`pairFormat` printf compatible verbs for current and total, like "%f" or "%d"
//
//	`wc` optional WC config
//
// pairFormat example:
//
//	"%.1f / %.1f" = "1.0MB / 12.0MB" or "% .1f / % .1f" = "1.0 MB / 12.0 MB"
func CountersKiloByte(pairFormat string, wc ...WC) Decorator {
	return counters(pairFormat, unitKB, wc...)
}

func counters(pairFormat string, unit int, wc ...WC) Decorator {
	var wc0 WC
	if len(wc) > 0 {
		wc0 = wc[0]
	}
	return DecoratorFunc(func(s *Statistics, widthAccumulator chan<- int, widthDistributor <-chan int) string {
		var str string
		switch unit {
		case unitKiB:
			str = fmt.Sprintf(pairFormat, CounterKiB(s.Current), CounterKiB(s.Total))
		case unitKB:
			str = fmt.Sprintf(pairFormat, CounterKB(s.Current), CounterKB(s.Total))
		default:
			str = fmt.Sprintf(pairFormat, s.Current, s.Total)
		}
		return wc0.formatMsg(str, widthAccumulator, widthDistributor)
	})
}

// ETA returns exponential-weighted-moving-average ETA decorator.
//
//	`style` one of [ET_STYLE_GO|ET_STYLE_HHMMSS|ET_STYLE_HHMM|ET_STYLE_MMSS]
//
//	`age` is a decay factor alpha for underlying ewma.
//	 General rule of thumb, for the best value:
//	 expected progress time in seconds divided by two.
//	 For example expected progress duration is one hour.
//	 age = 3600 / 2
//
//	`startBlock` is channel, user suppose to send time.Now() on each iteration of block start.
//
//	`wc` optional WC config
func ETA(style int, age float64, startBlock chan time.Time, wc ...WC) Decorator {
	var wc0 WC
	if len(wc) > 0 {
		wc0 = wc[0]
	}
	if age == .0 {
		age = ewma.AVG_METRIC_AGE
	}
	return &EwmaETA{
		MovingAverage: ewma.NewMovingAverage(age),
		StartBlockCh:  startBlock,
		style:         style,
		wc:            wc0,
	}
}

// EwmaETA is a struct, which implements ewma based ETA decorator.
// Should not be used directly, use ETA(int, float64, chan time.Time, ...WC)
// func helper instead.
type EwmaETA struct {
	ewma.MovingAverage
	StartBlockCh chan time.Time
	style        int
	wc           WC
	onComplete   *struct {
		msg string
		wc  WC
	}
}

func (s *EwmaETA) Decor(st *Statistics, widthAccumulator chan<- int, widthDistributor <-chan int) string {
	if st.Completed && s.onComplete != nil {
		return s.onComplete.wc.formatMsg(s.onComplete.msg, widthAccumulator, widthDistributor)
	}

	var str string
	timeRemaining := time.Duration(float64(st.Total-st.Current) * s.Value())
	hours := int64((timeRemaining / time.Hour) % 60)
	minutes := int64((timeRemaining / time.Minute) % 60)
	seconds := int64((timeRemaining / time.Second) % 60)

	switch s.style {
	case ET_STYLE_GO:
		str = fmt.Sprint(time.Duration(timeRemaining.Seconds()) * time.Second)
	case ET_STYLE_HHMMSS:
		str = fmt.Sprintf("%02d:%02d:%02d", hours, minutes, seconds)
	case ET_STYLE_HHMM:
		str = fmt.Sprintf("%02d:%02d", hours, minutes)
	case ET_STYLE_MMSS:
		str = fmt.Sprintf("%02d:%02d", minutes, seconds)
	}

	return s.wc.formatMsg(str, widthAccumulator, widthDistributor)
}

func (s *EwmaETA) OnComplete(msg string, wc ...WC) {
	var wc0 WC
	if len(wc) > 0 {
		wc0 = wc[0]
	}
	s.onComplete = &struct {
		msg string
		wc  WC
	}{msg, wc0}
}

// Elapsed returns elapsed time decorator.
//
//	`style` one of [ET_STYLE_GO|ET_STYLE_HHMMSS|ET_STYLE_HHMM|ET_STYLE_MMSS]
//
//	`wc` optional WC config
func Elapsed(style int, wc ...WC) Decorator {
	var wc0 WC
	if len(wc) > 0 {
		wc0 = wc[0]
	}
	return DecoratorFunc(func(s *Statistics, widthAccumulator chan<- int, widthDistributor <-chan int) string {
		var str string
		hours := int64((s.TimeElapsed / time.Hour) % 60)
		minutes := int64((s.TimeElapsed / time.Minute) % 60)
		seconds := int64((s.TimeElapsed / time.Second) % 60)

		switch style {
		case ET_STYLE_GO:
			str = fmt.Sprint(time.Duration(s.TimeElapsed.Seconds()) * time.Second)
		case ET_STYLE_HHMMSS:
			str = fmt.Sprintf("%02d:%02d:%02d", hours, minutes, seconds)
		case ET_STYLE_HHMM:
			str = fmt.Sprintf("%02d:%02d", hours, minutes)
		case ET_STYLE_MMSS:
			str = fmt.Sprintf("%02d:%02d", minutes, seconds)
		}
		return wc0.formatMsg(str, widthAccumulator, widthDistributor)
	})
}

// Percentage returns percentage decorator.
//
//	`wc` optional WC config
func Percentage(wc ...WC) Decorator {
	var wc0 WC
	if len(wc) > 0 {
		wc0 = wc[0]
	}
	return DecoratorFunc(func(s *Statistics, widthAccumulator chan<- int, widthDistributor <-chan int) string {
		str := fmt.Sprintf("%d %%", CalcPercentage(s.Total, s.Current, 100))
		return wc0.formatMsg(str, widthAccumulator, widthDistributor)
	})
}

// CalcPercentage is a helper function, to calculate percentage.
func CalcPercentage(total, current, width int64) (perc int64) {
	if total <= 0 {
		return 0
	}
	if current > total {
		current = total
	}

	num := float64(width) * float64(current) / float64(total)
	ceil := math.Ceil(num)
	diff := ceil - num
	// num = 2.34 will return 2
	// num = 2.44 will return 3
	if math.Max(diff, 0.6) == diff {
		return int64(num)
	}
	return int64(ceil)
}

// SpeedNoUnit returns raw I/O operation speed decorator.
//
//	`unitFormat` printf compatible verb for value, like "%f" or "%d"
//
//	`wc` optional WC config
//
// unitFormat example:
//
//	"%.1f" = "1.0" or "% .1f" = "1.0"
func SpeedNoUnit(unitFormat string, wc ...WC) Decorator {
	return speed(unitFormat, 0, wc...)
}

// SpeedKibiByte returns human friendly I/O operation speed decorator,
//
//	`unitFormat` printf compatible verb for value, like "%f" or "%d"
//
//	`wc` optional WC config
//
// unitFormat example:
//
//	"%.1f" = "1.0MiB/s" or "% .1f" = "1.0 MiB/s"
func SpeedKibiByte(unitFormat string, wc ...WC) Decorator {
	return speed(unitFormat, unitKiB, wc...)
}

// SpeedKiloByte returns human friendly I/O operation speed decorator,
//
//	`unitFormat` printf compatible verb for value, like "%f" or "%d"
//
//	`wc` optional WC config
//
// unitFormat example:
//
//	"%.1f" = "1.0MB/s" or "% .1f" = "1.0 MB/s"
func SpeedKiloByte(unitFormat string, wc ...WC) Decorator {
	return speed(unitFormat, unitKB, wc...)
}

func speed(unitFormat string, unit int, wc ...WC) Decorator {
	var wc0 WC
	if len(wc) > 0 {
		wc0 = wc[0]
	}
	return DecoratorFunc(func(s *Statistics, widthAccumulator chan<- int, widthDistributor <-chan int) string {
		var str string
		speed := float64(s.Current) / s.TimeElapsed.Seconds()
		if math.IsNaN(speed) || math.IsInf(speed, 0) {
			speed = .0
		}

		switch unit {
		case unitKiB:
			str = fmt.Sprintf(unitFormat, SpeedKiB(speed))
		case unitKB:
			str = fmt.Sprintf(unitFormat, SpeedKB(speed))
		default:
			str = fmt.Sprintf(unitFormat, speed)
		}
		return wc0.formatMsg(str, widthAccumulator, widthDistributor)
	})
}
<commit_msg>better godoc<commit_after>package decor

import (
	"fmt"
	"math"
	"time"
	"unicode/utf8"

	"github.com/VividCortex/ewma"
)

const (
	// DidentRight bit specifies identation direction.
	// |foo   |b     | With DidentRight
	// |   foo|     b| Without DidentRight
	DidentRight = 1 << iota

	// DextraSpace bit adds extra space, makes sense with DSyncWidth only.
	// When DidentRight bit set, the space will be added to the right,
	// otherwise to the left.
	DextraSpace

	// DSyncWidth bit enables same column width synchronization.
	// Effective with multiple bars only.
	DSyncWidth

	// DSyncWidthR is shortcut for DSyncWidth|DidentRight
	DSyncWidthR = DSyncWidth | DidentRight

	// DSyncSpace is shortcut for DSyncWidth|DextraSpace
	DSyncSpace = DSyncWidth | DextraSpace

	// DSyncSpaceR is shortcut for DSyncWidth|DextraSpace|DidentRight
	DSyncSpaceR = DSyncWidth | DextraSpace | DidentRight
)

const (
	ET_STYLE_GO = iota
	ET_STYLE_HHMMSS
	ET_STYLE_HHMM
	ET_STYLE_MMSS
)

// Statistics contains values useful for implementing a DecoratorFunc.
type Statistics struct {
	ID          int
	Completed   bool
	Total       int64
	Current     int64
	StartTime   time.Time
	TimeElapsed time.Duration
}

// Decorator is an interface with one method:
//
//	Decor(st *Statistics, widthAccumulator chan<- int, widthDistributor <-chan int) string
//
// All decorators in this package implement this interface.
type Decorator interface {
	Decor(*Statistics, chan<- int, <-chan int) string
}

type CompleteMessenger interface {
	OnComplete(string, ...WC)
}

// DecoratorFunc is an adapter for Decorator interface
type DecoratorFunc func(*Statistics, chan<- int, <-chan int) string

func (f DecoratorFunc) Decor(s *Statistics, widthAccumulator chan<- int, widthDistributor <-chan int) string {
	return f(s, widthAccumulator, widthDistributor)
}

// WC is a struct with two public fields W and C, both of int type.
// W represents width and C represents bit set of width related config.
type WC struct {
	W      int
	C      int
	format string
}

func (wc WC) formatMsg(msg string, widthAccumulator chan<- int, widthDistributor <-chan int) string {
	format := wc.buildFormat()
	if (wc.C & DSyncWidth) != 0 {
		widthAccumulator <- utf8.RuneCountInString(msg)
		max := <-widthDistributor
		if max == 0 {
			max = wc.W
		}
		if (wc.C & DextraSpace) != 0 {
			max++
		}
		return fmt.Sprintf(fmt.Sprintf(format, max), msg)
	}
	return fmt.Sprintf(fmt.Sprintf(format, wc.W), msg)
}

func (wc *WC) buildFormat() string {
	if wc.format != "" {
		return wc.format
	}
	wc.format = "%%"
	if (wc.C & DidentRight) != 0 {
		wc.format += "-"
	}
	wc.format += "%ds"
	return wc.format
}

// Global convenience shortcuts
var (
	WCSyncWidth  = WC{C: DSyncWidth}
	WCSyncWidthR = WC{C: DSyncWidthR}
	WCSyncSpace  = WC{C: DSyncSpace}
	WCSyncSpaceR = WC{C: DSyncSpaceR}
)

// OnComplete returns decorator, which wraps provided decorator, with sole
// purpose to display provided message on complete event.
//
//	`decorator` Decorator to wrap
//
//	`message` message to display on complete event
//
//	`wc` optional WC config
func OnComplete(decorator Decorator, message string, wc ...WC) Decorator {
	if cm, ok := decorator.(CompleteMessenger); ok {
		cm.OnComplete(message, wc...)
		return decorator
	}
	msgDecorator := Name(message, wc...)
	return DecoratorFunc(func(s *Statistics, widthAccumulator chan<- int, widthDistributor <-chan int) string {
		if s.Completed {
			return msgDecorator.Decor(s, widthAccumulator, widthDistributor)
		}
		return decorator.Decor(s, widthAccumulator, widthDistributor)
	})
}

// StaticName returns name decorator.
//
//	`name` string to display
//
//	`wc` optional WC config
func StaticName(name string, wc ...WC) Decorator {
	return Name(name, wc...)
}

// Name returns name decorator.
//
//	`name` string to display
//
//	`wc` optional WC config
func Name(name string, wc ...WC) Decorator {
	var wc0 WC
	if len(wc) > 0 {
		wc0 = wc[0]
	}
	return DecoratorFunc(func(s *Statistics, widthAccumulator chan<- int, widthDistributor <-chan int) string {
		return wc0.formatMsg(name, widthAccumulator, widthDistributor)
	})
}

// CountersNoUnit returns raw counters decorator
//
//	`pairFormat` printf compatible verbs for current and total, like "%f" or "%d"
//
//	`wc` optional WC config
func CountersNoUnit(pairFormat string, wc ...WC) Decorator {
	return counters(pairFormat, 0, wc...)
}

// CountersKibiByte returns human friendly byte counters decorator, where counters unit is multiple by 1024.
//
//	`pairFormat` printf compatible verbs for current and total, like "%f" or "%d"
//
//	`wc` optional WC config
//
// pairFormat example:
//
//	"%.1f / %.1f" = "1.0MiB / 12.0MiB" or "% .1f / % .1f" = "1.0 MiB / 12.0 MiB"
func CountersKibiByte(pairFormat string, wc ...WC) Decorator {
	return counters(pairFormat, unitKiB, wc...)
}

// CountersKiloByte returns human friendly byte counters decorator, where counters unit is multiple by 1000.
//
//	`pairFormat` printf compatible verbs for current and total, like "%f" or "%d"
//
//	`wc` optional WC config
//
// pairFormat example:
//
//	"%.1f / %.1f" = "1.0MB / 12.0MB" or "% .1f / % .1f" = "1.0 MB / 12.0 MB"
func CountersKiloByte(pairFormat string, wc ...WC) Decorator {
	return counters(pairFormat, unitKB, wc...)
}

func counters(pairFormat string, unit int, wc ...WC) Decorator {
	var wc0 WC
	if len(wc) > 0 {
		wc0 = wc[0]
	}
	return DecoratorFunc(func(s *Statistics, widthAccumulator chan<- int, widthDistributor <-chan int) string {
		var str string
		switch unit {
		case unitKiB:
			str = fmt.Sprintf(pairFormat, CounterKiB(s.Current), CounterKiB(s.Total))
		case unitKB:
			str = fmt.Sprintf(pairFormat, CounterKB(s.Current), CounterKB(s.Total))
		default:
			str = fmt.Sprintf(pairFormat, s.Current, s.Total)
		}
		return wc0.formatMsg(str, widthAccumulator, widthDistributor)
	})
}

// ETA returns exponential-weighted-moving-average ETA decorator.
//
//	`style` one of [ET_STYLE_GO|ET_STYLE_HHMMSS|ET_STYLE_HHMM|ET_STYLE_MMSS]
//
//	`age` is a decay factor alpha for underlying ewma.
//	 General rule of thumb, for the best value:
//	 expected progress time in seconds divided by two.
//	 For example expected progress duration is one hour.
//	 age = 3600 / 2
//
//	`startBlock` is channel, user suppose to send time.Now() on each iteration of block start.
//
//	`wc` optional WC config
func ETA(style int, age float64, startBlock chan time.Time, wc ...WC) Decorator {
	var wc0 WC
	if len(wc) > 0 {
		wc0 = wc[0]
	}
	if age == .0 {
		age = ewma.AVG_METRIC_AGE
	}
	return &EwmaETA{
		MovingAverage: ewma.NewMovingAverage(age),
		StartBlockCh:  startBlock,
		style:         style,
		wc:            wc0,
	}
}

// EwmaETA is a struct, which implements ewma based ETA decorator.
// Normally should not be used directly, use helper func instead:
//
//	decor.ETA(int, float64, chan time.Time, ...decor.WC)
type EwmaETA struct {
	ewma.MovingAverage
	StartBlockCh chan time.Time
	style        int
	wc           WC
	onComplete   *struct {
		msg string
		wc  WC
	}
}

func (s *EwmaETA) Decor(st *Statistics, widthAccumulator chan<- int, widthDistributor <-chan int) string {
	if st.Completed && s.onComplete != nil {
		return s.onComplete.wc.formatMsg(s.onComplete.msg, widthAccumulator, widthDistributor)
	}

	var str string
	timeRemaining := time.Duration(float64(st.Total-st.Current) * s.Value())
	hours := int64((timeRemaining / time.Hour) % 60)
	minutes := int64((timeRemaining / time.Minute) % 60)
	seconds := int64((timeRemaining / time.Second) % 60)

	switch s.style {
	case ET_STYLE_GO:
		str = fmt.Sprint(time.Duration(timeRemaining.Seconds()) * time.Second)
	case ET_STYLE_HHMMSS:
		str = fmt.Sprintf("%02d:%02d:%02d", hours, minutes, seconds)
	case ET_STYLE_HHMM:
		str = fmt.Sprintf("%02d:%02d", hours, minutes)
	case ET_STYLE_MMSS:
		str = fmt.Sprintf("%02d:%02d", minutes, seconds)
	}

	return s.wc.formatMsg(str, widthAccumulator, widthDistributor)
}

func (s *EwmaETA) OnComplete(msg string, wc ...WC) {
	var wc0 WC
	if len(wc) > 0 {
		wc0 = wc[0]
	}
	s.onComplete = &struct {
		msg string
		wc  WC
	}{msg, wc0}
}

// Elapsed returns elapsed time decorator.
//
//	`style` one of [ET_STYLE_GO|ET_STYLE_HHMMSS|ET_STYLE_HHMM|ET_STYLE_MMSS]
//
//	`wc` optional WC config
func Elapsed(style int, wc ...WC) Decorator {
	var wc0 WC
	if len(wc) > 0 {
		wc0 = wc[0]
	}
	return DecoratorFunc(func(s *Statistics, widthAccumulator chan<- int, widthDistributor <-chan int) string {
		var str string
		hours := int64((s.TimeElapsed / time.Hour) % 60)
		minutes := int64((s.TimeElapsed / time.Minute) % 60)
		seconds := int64((s.TimeElapsed / time.Second) % 60)

		switch style {
		case ET_STYLE_GO:
			str = fmt.Sprint(time.Duration(s.TimeElapsed.Seconds()) * time.Second)
		case ET_STYLE_HHMMSS:
			str = fmt.Sprintf("%02d:%02d:%02d", hours, minutes, seconds)
		case ET_STYLE_HHMM:
			str = fmt.Sprintf("%02d:%02d", hours, minutes)
		case ET_STYLE_MMSS:
			str = fmt.Sprintf("%02d:%02d", minutes, seconds)
		}
		return wc0.formatMsg(str, widthAccumulator, widthDistributor)
	})
}

// Percentage returns percentage decorator.
//
//	`wc` optional WC config
func Percentage(wc ...WC) Decorator {
	var wc0 WC
	if len(wc) > 0 {
		wc0 = wc[0]
	}
	return DecoratorFunc(func(s *Statistics, widthAccumulator chan<- int, widthDistributor <-chan int) string {
		str := fmt.Sprintf("%d %%", CalcPercentage(s.Total, s.Current, 100))
		return wc0.formatMsg(str, widthAccumulator, widthDistributor)
	})
}

// CalcPercentage is a helper function, to calculate percentage.
func CalcPercentage(total, current, width int64) (perc int64) {
	if total <= 0 {
		return 0
	}
	if current > total {
		current = total
	}

	num := float64(width) * float64(current) / float64(total)
	ceil := math.Ceil(num)
	diff := ceil - num
	// num = 2.34 will return 2
	// num = 2.44 will return 3
	if math.Max(diff, 0.6) == diff {
		return int64(num)
	}
	return int64(ceil)
}

// SpeedNoUnit returns raw I/O operation speed decorator.
//
//	`unitFormat` printf compatible verb for value, like "%f" or "%d"
//
//	`wc` optional WC config
//
// unitFormat example:
//
//	"%.1f" = "1.0" or "% .1f" = "1.0"
func SpeedNoUnit(unitFormat string, wc ...WC) Decorator {
	return speed(unitFormat, 0, wc...)
}

// SpeedKibiByte returns human friendly I/O operation speed decorator,
//
//	`unitFormat` printf compatible verb for value, like "%f" or "%d"
//
//	`wc` optional WC config
//
// unitFormat example:
//
//	"%.1f" = "1.0MiB/s" or "% .1f" = "1.0 MiB/s"
func SpeedKibiByte(unitFormat string, wc ...WC) Decorator {
	return speed(unitFormat, unitKiB, wc...)
}

// SpeedKiloByte returns human friendly I/O operation speed decorator,
//
//	`unitFormat` printf compatible verb for value, like "%f" or "%d"
//
//	`wc` optional WC config
//
// unitFormat example:
//
//	"%.1f" = "1.0MB/s" or "% .1f" = "1.0 MB/s"
func SpeedKiloByte(unitFormat string, wc ...WC) Decorator {
	return speed(unitFormat, unitKB, wc...)
}

func speed(unitFormat string, unit int, wc ...WC) Decorator {
	var wc0 WC
	if len(wc) > 0 {
		wc0 = wc[0]
	}
	return DecoratorFunc(func(s *Statistics, widthAccumulator chan<- int, widthDistributor <-chan int) string {
		var str string
		speed := float64(s.Current) / s.TimeElapsed.Seconds()
		if math.IsNaN(speed) || math.IsInf(speed, 0) {
			speed = .0
		}

		switch unit {
		case unitKiB:
			str = fmt.Sprintf(unitFormat, SpeedKiB(speed))
		case unitKB:
			str = fmt.Sprintf(unitFormat, SpeedKB(speed))
		default:
			str = fmt.Sprintf(unitFormat, speed)
		}
		return wc0.formatMsg(str, widthAccumulator, widthDistributor)
	})
}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"fmt"
	"log"
	"os"
	"os/exec"
	"strings"
	"time"
)

type commitInfo struct {
	hash         string
	commitDate   time.Time
	binPath      string
	gover        bool
	logPath      string
	count, fails int
	buildFailed  bool
}

// getCommits returns the commit info for all of the revisions in the
// given git revision range, where the revision range is spelled as
// documented in gitrevisions(7). Commits are returned in reverse
// chronological order, most recent commit first (the same as
// git-rev-list(1)).
func getCommits(revRange []string) []*commitInfo {
	// Get commit sequence.
	hashes := lines(git("rev-list", revRange...))

	// Get commit dates.
	args := append([]string{"-s", "--format=format:%cI"}, hashes...)
	dates := lines(git("show", args...))

	// Get gover-cached builds. It's okay if this fails.
	cachedHashes := make(map[string]bool)
	x, _ := exec.Command("gover", "list").CombinedOutput()
	for _, cached := range lines(string(x)) {
		fs := strings.SplitN(cached, " ", 2)
		cachedHashes[fs[0]] = true
	}

	// Load current benchmark state.
	var commits []*commitInfo
	for i, hash := range hashes {
		logPath := fmt.Sprintf("log.%s", hash[:7])
		count, fails, buildFailed := countRuns(logPath)
		commitDate, err := time.Parse(time.RFC3339, dates[i])
		if err != nil {
			log.Fatalf("cannot parse commit date: %v", err)
		}
		// TODO: This assumes the 7 character hash is
		// unambiguous.
		commit := &commitInfo{
			hash:        hash,
			commitDate:  commitDate,
			binPath:     fmt.Sprintf("bench.%s", hash[:7]),
			gover:       cachedHashes[hash[:7]],
			logPath:     logPath,
			count:       count,
			fails:       fails,
			buildFailed: buildFailed,
		}
		commits = append(commits, commit)
	}

	return commits
}

// countRuns parses the log at path and returns the number of
// successful runs, the number of failed runs, and whether the build
// failed.
func countRuns(path string) (count, fails int, buildFailed bool) {
	f, err := os.Open(path)
	if os.IsNotExist(err) {
		return
	} else if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		t := scanner.Text()
		if t == "PASS" {
			count++
		} else if t == "FAILED:" {
			fails++
		} else if t == "BUILD FAILED:" {
			buildFailed = true
		}
	}
	if err := scanner.Err(); err != nil {
		fmt.Fprintf(os.Stderr, "reading log %s: %v", path, err)
		os.Exit(1)
	}
	return
}

// runnable returns whether commit c needs to be benchmarked at least
// one more time.
func (c *commitInfo) runnable() bool {
	return !c.buildFailed && c.fails < maxFails && c.count < iterations
}

// partial returns true if this commit is both runnable and already
// has some runs.
func (c *commitInfo) partial() bool {
	return c.count > 0 && c.runnable()
}

// writeLog appends msg to c's log file.
func (c *commitInfo) writeLog(msg []byte) {
	logFile, err := os.OpenFile(c.logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
	if err != nil {
		log.Fatal(err)
	}
	if _, err := logFile.Write(msg); err != nil {
		log.Fatal(err)
	}
	if err := logFile.Close(); err != nil {
		log.Fatal(err)
	}
}
<commit_msg>benchmany: fix run counting for x/benchmarks output<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"fmt"
	"log"
	"os"
	"os/exec"
	"strings"
	"time"
)

type commitInfo struct {
	hash         string
	commitDate   time.Time
	binPath      string
	gover        bool
	logPath      string
	count, fails int
	buildFailed  bool
}

// getCommits returns the commit info for all of the revisions in the
// given git revision range, where the revision range is spelled as
// documented in gitrevisions(7). Commits are returned in reverse
// chronological order, most recent commit first (the same as
// git-rev-list(1)).
func getCommits(revRange []string) []*commitInfo {
	// Get commit sequence.
	hashes := lines(git("rev-list", revRange...))

	// Get commit dates.
	args := append([]string{"-s", "--format=format:%cI"}, hashes...)
	dates := lines(git("show", args...))

	// Get gover-cached builds. It's okay if this fails.
	cachedHashes := make(map[string]bool)
	x, _ := exec.Command("gover", "list").CombinedOutput()
	for _, cached := range lines(string(x)) {
		fs := strings.SplitN(cached, " ", 2)
		cachedHashes[fs[0]] = true
	}

	// Load current benchmark state.
	var commits []*commitInfo
	for i, hash := range hashes {
		logPath := fmt.Sprintf("log.%s", hash[:7])
		count, fails, buildFailed := countRuns(logPath)
		commitDate, err := time.Parse(time.RFC3339, dates[i])
		if err != nil {
			log.Fatalf("cannot parse commit date: %v", err)
		}
		// TODO: This assumes the 7 character hash is
		// unambiguous.
		commit := &commitInfo{
			hash:        hash,
			commitDate:  commitDate,
			binPath:     fmt.Sprintf("bench.%s", hash[:7]),
			gover:       cachedHashes[hash[:7]],
			logPath:     logPath,
			count:       count,
			fails:       fails,
			buildFailed: buildFailed,
		}
		commits = append(commits, commit)
	}

	return commits
}

// countRuns parses the log at path and returns the number of
// successful runs, the number of failed runs, and whether the build
// failed.
func countRuns(path string) (count, fails int, buildFailed bool) {
	f, err := os.Open(path)
	if os.IsNotExist(err) {
		return
	} else if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	scanner := bufio.NewScanner(f)
	inXMetrics := false
	for scanner.Scan() {
		t := scanner.Text()
		if t == "PASS" {
			count++
		} else if t == "FAILED:" {
			fails++
		} else if t == "BUILD FAILED:" {
			buildFailed = true
		}
		if strings.HasPrefix(t, "GOPERF-METRIC:") {
			if !inXMetrics {
				count++
			}
			inXMetrics = true
		} else {
			inXMetrics = false
		}
	}
	if err := scanner.Err(); err != nil {
		fmt.Fprintf(os.Stderr, "reading log %s: %v", path, err)
		os.Exit(1)
	}
	return
}

// runnable returns whether commit c needs to be benchmarked at least
// one more time.
func (c *commitInfo) runnable() bool {
	return !c.buildFailed && c.fails < maxFails && c.count < iterations
}

// partial returns true if this commit is both runnable and already
// has some runs.
func (c *commitInfo) partial() bool {
	return c.count > 0 && c.runnable()
}

// writeLog appends msg to c's log file.
func (c *commitInfo) writeLog(msg []byte) {
	logFile, err := os.OpenFile(c.logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
	if err != nil {
		log.Fatal(err)
	}
	if _, err := logFile.Write(msg); err != nil {
		log.Fatal(err)
	}
	if err := logFile.Close(); err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build !go1.8

package spanner

import (
	"golang.org/x/net/context"
	"google.golang.org/api/option"
)

// OpenCensus only supports go 1.8 and higher.

func openCensusOptions() []option.ClientOption { return nil }

func traceStartSpan(ctx context.Context, _ string) context.Context {
	return ctx
}

func traceEndSpan(context.Context, error) {
}

func tracePrintf(context.Context, map[string]interface{}, string, ...interface{}) {
}
<commit_msg>bigtable: fix erroneous package name<commit_after>// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build !go1.8

package bigtable

import (
	"golang.org/x/net/context"
	"google.golang.org/api/option"
)

// OpenCensus only supports go 1.8 and higher.

func openCensusOptions() []option.ClientOption { return nil }

func traceStartSpan(ctx context.Context, _ string) context.Context {
	return ctx
}

func traceEndSpan(context.Context, error) {
}

func tracePrintf(context.Context, map[string]interface{}, string, ...interface{}) {
}
<|endoftext|>


<commit_before>package starbound

import (
	"os"
	"testing"
)

const (
	WORLD = "../test.world"
)

type logger interface {
	Fatalf(format string, args ...interface{})
}

func getDB(log logger) *BTreeDB5 {
	file, err := os.Open(WORLD)
	if err != nil {
		log.Fatalf("failed to open world file: %v", err)
	}
	db, err := NewBTreeDB5(file)
	if err != nil {
		log.Fatalf("failed to read world: %v", err)
	}
	return db
}

func TestHeader(t *testing.T) {
	db := getDB(t)
	if db.Name != "World4" {
		t.Errorf("incorrect database name: %v", db.Name)
	}
}

func TestInvalidKeyLength(t *testing.T) {
	db := getDB(t)
	_, err := db.Get([]byte("\x00\x00\x00\x00"))
	if err != ErrInvalidKeyLength {
		t.Errorf("expected invalid key length, got: %v", err)
	}
}

func TestMissingKey(t *testing.T) {
	db := getDB(t)
	data, err := db.Get([]byte("\x00\x00\x00\x00\x01"))
	if data != nil {
		t.Error("data should be <nil>")
	}
	if err != ErrKeyNotFound {
		t.Errorf("expected key error, got: %v", err)
	}
}

func BenchmarkHeader(b *testing.B) {
	file, err := os.Open(WORLD)
	if err != nil {
		b.Fatalf("failed to open world file: %v", err)
	}
	for i := 0; i < b.N; i++ {
		NewBTreeDB5(file)
	}
}

func BenchmarkLookup(b *testing.B) {
	db := getDB(b)
	for i := 0; i < b.N; i++ {
		db.Get([]byte("\x00\x00\x00\x00\x00"))
	}
}
<commit_msg>Success/Fail benchmarks<commit_after>package starbound

import (
	"os"
	"testing"
)

const (
	WORLD = "../test.world"
)

type logger interface {
	Fatalf(format string, args ...interface{})
}

func getDB(log logger) *BTreeDB5 {
	file, err := os.Open(WORLD)
	if err != nil {
		log.Fatalf("failed to open world file: %v", err)
	}
	db, err := NewBTreeDB5(file)
	if err != nil {
		log.Fatalf("failed to read world: %v", err)
	}
	return db
}

func TestHeader(t *testing.T) {
	db := getDB(t)
	if db.Name != "World4" {
		t.Errorf("incorrect database name: %v", db.Name)
	}
}

func TestInvalidKeyLength(t *testing.T) {
	db := getDB(t)
	_, err := db.Get([]byte("\x00\x00\x00\x00"))
	if err != ErrInvalidKeyLength {
		t.Errorf("expected invalid key length, got: %v", err)
	}
}

func TestMissingKey(t *testing.T) {
	db := getDB(t)
	data, err := db.Get([]byte("\x00\x00\x00\x00\x01"))
	if data != nil {
		t.Error("data should be <nil>")
	}
	if err != ErrKeyNotFound {
		t.Errorf("expected key error, got: %v", err)
	}
}

func BenchmarkHeader(b *testing.B) {
	file, err := os.Open(WORLD)
	if err != nil {
		b.Fatalf("failed to open world file: %v", err)
	}
	for i := 0; i < b.N; i++ {
		NewBTreeDB5(file)
	}
}

func BenchmarkLookupFail(b *testing.B) {
	db := getDB(b)
	for i := 0; i < b.N; i++ {
		db.Get([]byte("\x04\x03\x02\x01\x00"))
	}
}

func BenchmarkLookupSuccess(b *testing.B) {
	db := getDB(b)
	for i := 0; i < b.N; i++ {
		db.Get([]byte("\x00\x00\x00\x00\x00"))
	}
}
<|endoftext|>


<commit_before>package ast

import (
	"fmt"
	"github.com/gtfierro/hod/lang/token"
	"github.com/gtfierro/hod/turtle"
	"github.com/kr/pretty"
	"strings"
)

type QueryType uint

const (
	SELECT_QUERY QueryType = 1 << iota
	INSERT_QUERY
	DELETE_QUERY
)

var debug = false

func SetDebug() {
	debug = true
}
func ClearDebug() {
	debug = false
}

type Query struct {
	Select    SelectClause
	From      FromClause
	Count     bool
	Insert    InsertClause
	Where     WhereClause
	Variables []string
	Type      QueryType
}

func (q Query) IsInsert() bool {
	return (q.Type & INSERT_QUERY) == INSERT_QUERY
}

func (q Query) IsSelect() bool {
	return (q.Type & SELECT_QUERY) == SELECT_QUERY
}

func (q Query) CopyWithNewTerms(terms []Triple) Query {
	newq := Query{
		Select:    q.Select,
		From:      q.From,
		Variables: q.Variables,
		Type:      q.Type,
	}
	newq.Where.Terms = make([]Triple, len(terms))
	copy(newq.Where.Terms, terms)

	newq.Insert.Terms = make([]Triple, len(terms))
	copy(newq.Insert.Terms, terms)
	return newq
}

func (q Query) Copy() *Query {
	return &Query{
		Select:    q.Select,
		From:      q.From,
		Variables: q.Variables,
		Where:     q.Where,
		Insert:    q.Insert,
		Count:     q.Count,
		Type:      q.Type,
	}
}

func NewQuery(selectclause, whereclause interface{}, count bool) (Query, error) {
	if debug {
		fmt.Printf("%# v", pretty.Formatter(whereclause.(WhereClause)))
	}
	q := Query{
		Where:  whereclause.(WhereClause),
		Select: selectclause.(SelectClause),
		Count:  count,
		Type:   SELECT_QUERY,
	}
	if q.From.Empty() {
		q.From.AllDBs = true
	}
	q.PopulateVars()
	if q.Select.AllVars {
		q.Select.Vars = q.Variables
	}
	return q, nil
}

func NewInsertQuery(insertclause, whereclause interface{}, count bool) (Query, error) {
	if debug {
		fmt.Printf("%# v", pretty.Formatter(whereclause.(WhereClause)))
		fmt.Printf("%# v", pretty.Formatter(insertclause.(InsertClause)))
	}
	q := Query{
		Where:  whereclause.(WhereClause),
		Select: SelectClause{AllVars: true},
		Insert: insertclause.(InsertClause),
		Count:  count,
		Type:   INSERT_QUERY,
	}
	if q.From.Empty() {
		q.From.AllDBs = true
	}
	q.PopulateVars()
	if q.Select.AllVars {
		q.Select.Vars = q.Variables
	}
	return q, nil
}

func NewQueryMulti(selectclause, fromclause, whereclause interface{}, count bool) (Query, error) {
	if debug {
		fmt.Printf("%# v", pretty.Formatter(whereclause.(WhereClause)))
	}
	q := Query{
		Where:  whereclause.(WhereClause),
		From:   fromclause.(FromClause),
		Select: selectclause.(SelectClause),
		Count:  count,
	}
	if q.From.Empty() {
		q.From.AllDBs = true
	}
	q.PopulateVars()
	if q.Select.AllVars {
		q.Select.Vars = q.Variables
	}
	return q, nil
}

func NewInsertQueryMulti(insertclause, fromclause, whereclause interface{}, count bool) (Query, error) {
	if debug {
		fmt.Printf("%# v", pretty.Formatter(whereclause.(WhereClause)))
		fmt.Printf("%# v", pretty.Formatter(insertclause.(InsertClause)))
	}
	q := Query{
		Where:  whereclause.(WhereClause),
		Select: SelectClause{AllVars: true},
		From:   fromclause.(FromClause),
		Insert: insertclause.(InsertClause),
		Count:  count,
		Type:   INSERT_QUERY,
	}
	if q.From.Empty() {
		q.From.AllDBs = true
	}
	q.PopulateVars()
	if q.Select.AllVars {
		q.Select.Vars = q.Variables
	}
	return q, nil
}

func (q *Query) PopulateVars() {
	vars := make(map[string]int)
	// get all variables
	for _, triple := range q.Where.Terms {
		AddIfVar(triple.Subject, vars)
		AddIfVar(triple.Object, vars)
		for _, path := range triple.Predicates {
			AddIfVar(path.Predicate, vars)
		}
	}
	for _, triple := range q.Insert.Terms {
		AddIfVar(triple.Subject, vars)
		AddIfVar(triple.Object, vars)
		for _, path := range triple.Predicates {
			AddIfVar(path.Predicate, vars)
		}
	}
	if q.Where.GraphGroup != nil {
		VarsFromGroup(*q.Where.GraphGroup, vars)
	}
	q.Variables = []string{} // clear
	for varname := range vars {
		q.Variables = append(q.Variables, varname)
	}
}

func (q Query) IterTriples(f func(t Triple) Triple) {
	for idx, triple := range q.Where.Terms {
		q.Where.Terms[idx] = f(triple)
	}
	for idx, triple := range q.Insert.Terms {
		q.Insert.Terms[idx] = f(triple)
	}
	if q.Where.GraphGroup != nil {
		q.Where.GraphGroup.IterTriples(f)
	}
}

func AddIfVar(uri turtle.URI, m map[string]int) {
	if uri.IsVariable() {
		m[uri.String()] = 1
	}
}

func VarsFromGroup(group GraphGroup, m map[string]int) {
	for _, triple := range group.Terms {
		AddIfVar(triple.Subject, m)
		AddIfVar(triple.Object, m)
		for _, path := range triple.Predicates {
			AddIfVar(path.Predicate, m)
		}
	}
	for _, union := range group.Unions {
		VarsFromGroup(union, m)
	}
}

func (grp GraphGroup) Expand() [][]Triple {
	var terms = make([]Triple, len(grp.Terms))
	copy(terms, grp.Terms)
	var groups [][]Triple

	if len(grp.Unions) > 0 {
		for _, union := range grp.Unions {
			for _, subgroup := range union.Expand() {
				groups = append(groups, append(terms, subgroup...))
			}
		}
	} else {
		groups = append(groups, terms)
	}
	return groups
}

func (grp GraphGroup) Iter(f func(t turtle.URI)) {
	for _, triple := range grp.Terms {
		f(triple.Subject)
		f(triple.Object)
		for _, path := range triple.Predicates {
			f(path.Predicate)
		}
	}
	for _, union := range grp.Unions {
		union.Iter(f)
	}
}

func (grp *GraphGroup) IterTriples(f func(t Triple) Triple) {
	for idx, triple := range grp.Terms {
		grp.Terms[idx] = f(triple)
	}
	for _, union := range grp.Unions {
		union.IterTriples(f)
	}
}

type SelectClause struct {
	Vars    []string
	AllVars bool
}

func NewAllSelectClause() (SelectClause, error) {
	return SelectClause{AllVars: true}, nil
}

func NewSelectClause(varlist interface{}) (SelectClause, error) {
	return SelectClause{Vars: varlist.([]string)}, nil
}

type InsertClause struct {
	Terms []Triple
}

func NewInsertClause(triples interface{}) (InsertClause, error) {
	return InsertClause{
		Terms: triples.([]Triple),
	}, nil
}

type FromClause struct {
	Databases []string
	AllDBs    bool
}

func (f FromClause) String() string {
	if f.AllDBs {
		return "*"
	}
	return strings.Join(f.Databases, " ")
}

func NewAllFromClause() (FromClause, error) {
	return FromClause{AllDBs: true}, nil
}

func NewFromClause(dblist interface{}) (FromClause, error) {
	return FromClause{Databases: dblist.([]string)}, nil
}

func (from FromClause) Empty() bool {
	return len(from.Databases) == 0 && !from.AllDBs
}

type WhereClause struct {
	Terms      []Triple
	GraphGroup *GraphGroup
}

func NewWhereClause(triples interface{}) (WhereClause, error) {
	return WhereClause{
		Terms: triples.([]Triple),
	}, nil
}

func NewWhereClauseWithGraphGroup(triples, group interface{}) (WhereClause, error) {
	g := group.(GraphGroup)
	return WhereClause{
		Terms:      triples.([]Triple),
		GraphGroup: &g,
	}, nil
}

func NewWhereClauseGraphGroup(group interface{}) (WhereClause, error) {
	g := group.(GraphGroup)
	return WhereClause{
		GraphGroup: &g,
	}, nil
}

type GraphGroup struct {
	Terms  []Triple
	Unions []GraphGroup
}

func GraphGroupFromTriples(triples interface{}) (GraphGroup, error) {
	return GraphGroup{
		Terms: triples.([]Triple),
	}, nil
}

func GraphGroupUnion(left, right interface{}) (GraphGroup, error) {
	return GraphGroup{
		Unions: []GraphGroup{left.(GraphGroup), right.(GraphGroup)},
	}, nil
}

func AddTriplesToGraphGroup(left, triples interface{}) (GraphGroup, error) {
	return GraphGroup{
		Terms:  append(left.(GraphGroup).Terms, triples.([]Triple)...),
		Unions: left.(GraphGroup).Unions,
	}, nil
}

func MergeGraphGroups(left, right interface{}) (GraphGroup, error) {
	return GraphGroup{
		Terms:  append(left.(GraphGroup).Terms, right.(GraphGroup).Terms...),
		Unions: append(left.(GraphGroup).Unions, right.(GraphGroup).Unions...),
	}, nil
}

type Triple struct {
	Subject    turtle.URI
	Predicates []PathPattern
	Object     turtle.URI
}

func (t Triple) String() string {
	s := "<" + t.Subject.String() + "|"
	for _, pp := range t.Predicates {
		s += " " + pp.String()
	}
	return s + " | " + t.Object.String() + ">"
}

func NewTriple(subject, predicates, object interface{}) (Triple, error) {
	return Triple{
		Subject:    subject.(turtle.URI),
		Predicates: predicates.([]PathPattern),
		Object:     object.(turtle.URI),
	}, nil
}

func NewTripleBlock(triple interface{}) ([]Triple, error) {
	return []Triple{triple.(Triple)}, nil
}

func AppendTripleBlock(block, triple interface{}) ([]Triple, error) {
	return append(block.([]Triple), triple.(Triple)), nil
}

func NewURI(value interface{}) (turtle.URI, error) {
	return turtle.ParseURI(value.(string)), nil
}

func NewVarList(_var interface{}) ([]string, error) {
	return []string{_var.(string)}, nil
}

func AppendVar(varlist, _var interface{}) ([]string, error) {
	return append(varlist.([]string), _var.(string)), nil
}

func NewStringList(_str interface{}) ([]string, error) {
	return []string{_str.(string)}, nil
}

func AppendString(strlist, _str interface{}) ([]string, error) {
	return append(strlist.([]string), _str.(string)), nil
}

func ParseString(_var interface{}) (string, error) {
	return string(_var.(*token.Token).Lit), nil
}

func ParseQuotedString(_var interface{}) (string, error) {
	return string(_var.(*token.Token).Lit), nil
}

func NewPathSequence(_pred interface{}) ([]PathPattern, error) {
	return []PathPattern{_pred.(PathPattern)}, nil
}

func AppendPathSequence(_seq, _pred interface{}) ([]PathPattern, error) {
	return append(_seq.([]PathPattern), _pred.(PathPattern)), nil
}

func NewPathPattern(_pred interface{}) (PathPattern, error) {
	pred, _ := ParseString(_pred)
	if pred == "a" {
		pred = "rdf:type"
	}
	return PathPattern{
		Predicate: turtle.ParseURI(pred),
		Pattern:   PATTERN_SINGLE,
	}, nil
}

func AddPathMod(_pred, _mod interface{}) (PathPattern, error) {
	return PathPattern{
		Predicate: _pred.(PathPattern).Predicate,
		Pattern:   _mod.(Pattern),
	}, nil
}

type PathPattern struct {
	Predicate turtle.URI
	Pattern   Pattern
}

func PathFromVar(_var interface{}) ([]PathPattern, error) {
	return []PathPattern{
		PathPattern{
			Predicate: turtle.ParseURI(_var.(string)),
			Pattern:   PATTERN_SINGLE,
		},
	}, nil
}

func (pp PathPattern) String() string {
	return pp.Predicate.String() + pp.Pattern.String()
}

type Pattern uint

const (
	PATTERN_SINGLE = iota + 1
	PATTERN_ZERO_ONE
	PATTERN_ONE_PLUS
	PATTERN_ZERO_PLUS
)

func (p Pattern) String() string {
	switch p {
	case PATTERN_SINGLE:
		return ""
	case PATTERN_ZERO_ONE:
		return "?"
	case PATTERN_ONE_PLUS:
		return "+"
	case PATTERN_ZERO_PLUS:
		return "*"
	}
	return "unknown"
}
<commit_msg>add triple copy()<commit_after>package ast

import (
	"fmt"
	"github.com/gtfierro/hod/lang/token"
	"github.com/gtfierro/hod/turtle"
	"github.com/kr/pretty"
	"strings"
)

type QueryType uint

const (
	SELECT_QUERY QueryType = 1 << iota
	INSERT_QUERY
	DELETE_QUERY
)

var debug = false

func SetDebug() {
	debug = true
}
func ClearDebug() {
	debug = false
}

type Query struct {
	Select    SelectClause
	From      FromClause
	Count     bool
	Insert    InsertClause
	Where     WhereClause
	Variables []string
	Type      QueryType
}

func (q Query) IsInsert() bool {
	return (q.Type & INSERT_QUERY) == INSERT_QUERY
}

func (q Query) IsSelect() bool {
	return (q.Type & SELECT_QUERY) == SELECT_QUERY
}

func (q Query) CopyWithNewTerms(terms []Triple) Query {
	newq := Query{
		Select:    q.Select,
		From:      q.From,
		Variables: q.Variables,
		Type:      q.Type,
	}
	newq.Where.Terms = make([]Triple, len(terms))
	copy(newq.Where.Terms, terms)

	newq.Insert.Terms = make([]Triple, len(terms))
	copy(newq.Insert.Terms, terms)
	return newq
}

func (q Query) Copy() *Query {
	return &Query{
		Select:    q.Select,
		From:      q.From,
		Variables: q.Variables,
		Where:     q.Where,
		Insert:    q.Insert,
		Count:     q.Count,
		Type:      q.Type,
	}
}

func NewQuery(selectclause, whereclause interface{}, count bool) (Query, error) {
	if debug {
		fmt.Printf("%# v", pretty.Formatter(whereclause.(WhereClause)))
	}
	q := Query{
		Where:  whereclause.(WhereClause),
		Select: selectclause.(SelectClause),
		Count:  count,
		Type:   SELECT_QUERY,
	}
	if q.From.Empty() {
		q.From.AllDBs = true
	}
	q.PopulateVars()
	if q.Select.AllVars {
		q.Select.Vars = q.Variables
	}
	return q, nil
}

func NewInsertQuery(insertclause, whereclause interface{}, count bool) (Query, error) {
	if debug {
		fmt.Printf("%# v", pretty.Formatter(whereclause.(WhereClause)))
		fmt.Printf("%# v", pretty.Formatter(insertclause.(InsertClause)))
	}
	q := Query{
		Where:  whereclause.(WhereClause),
		Select: SelectClause{AllVars: true},
		Insert: insertclause.(InsertClause),
		Count:  count,
		Type:   INSERT_QUERY,
	}
	if q.From.Empty() {
		q.From.AllDBs = true
	}
	q.PopulateVars()
	if q.Select.AllVars {
		q.Select.Vars = q.Variables
	}
	return q, nil
}

func NewQueryMulti(selectclause, fromclause, whereclause interface{}, count bool) (Query, error) {
	if debug {
		fmt.Printf("%# v", pretty.Formatter(whereclause.(WhereClause)))
	}
	q := Query{
		Where:  whereclause.(WhereClause),
		From:   fromclause.(FromClause),
		Select: selectclause.(SelectClause),
		Count:  count,
	}
	if q.From.Empty() {
		q.From.AllDBs = true
	}
	q.PopulateVars()
	if q.Select.AllVars {
		q.Select.Vars = q.Variables
	}
	return q, nil
}

func NewInsertQueryMulti(insertclause, fromclause, whereclause interface{}, count bool) (Query, error) {
	if debug {
		fmt.Printf("%# v", pretty.Formatter(whereclause.(WhereClause)))
		fmt.Printf("%# v", pretty.Formatter(insertclause.(InsertClause)))
	}
	q := Query{
		Where:  whereclause.(WhereClause),
		Select: SelectClause{AllVars: true},
		From:   fromclause.(FromClause),
		Insert: insertclause.(InsertClause),
		Count:  count,
		Type:   INSERT_QUERY,
	}
	if q.From.Empty() {
		q.From.AllDBs = true
	}
	q.PopulateVars()
	if q.Select.AllVars {
		q.Select.Vars = q.Variables
	}
	return q, nil
}

func (q *Query) PopulateVars() {
	vars := make(map[string]int)
	// get all variables
	for _, triple := range q.Where.Terms {
		AddIfVar(triple.Subject, vars)
		AddIfVar(triple.Object, vars)
		for _, path := range triple.Predicates {
			AddIfVar(path.Predicate, vars)
		}
	}
	for _, triple := range q.Insert.Terms {
		AddIfVar(triple.Subject, vars)
		AddIfVar(triple.Object, vars)
		for _, path := range triple.Predicates {
			AddIfVar(path.Predicate, vars)
		}
	}
	if q.Where.GraphGroup != nil {
		VarsFromGroup(*q.Where.GraphGroup, vars)
	}
	q.Variables = []string{} // clear
	for varname := range vars {
		q.Variables = append(q.Variables, varname)
	}
}

func (q Query) IterTriples(f func(t Triple) Triple) {
	for idx, triple := range q.Where.Terms {
		q.Where.Terms[idx] = f(triple)
	}
	for idx, triple := range q.Insert.Terms {
		q.Insert.Terms[idx] = f(triple)
	}
	if q.Where.GraphGroup != nil {
		q.Where.GraphGroup.IterTriples(f)
	}
}

func AddIfVar(uri turtle.URI, m map[string]int) {
	if uri.IsVariable() {
		m[uri.String()] = 1
	}
}

func VarsFromGroup(group GraphGroup, m map[string]int) {
	for _, triple := range group.Terms {
		AddIfVar(triple.Subject, m)
		AddIfVar(triple.Object, m)
		for _, path := range triple.Predicates {
			AddIfVar(path.Predicate, m)
		}
	}
	for _, union := range group.Unions {
		VarsFromGroup(union, m)
	}
}

func (grp GraphGroup) Expand() [][]Triple {
	var terms = make([]Triple, len(grp.Terms))
	copy(terms, grp.Terms)
	var groups [][]Triple

	if len(grp.Unions) > 0 {
		for _, union := range grp.Unions {
			for _, subgroup := range union.Expand() {
				groups = append(groups, append(terms, subgroup...))
			}
		}
	} else {
		groups = append(groups, terms)
	}
	return groups
}

func (grp GraphGroup) Iter(f func(t turtle.URI)) {
	for _, triple := range grp.Terms {
		f(triple.Subject)
		f(triple.Object)
		for _, path := range triple.Predicates {
			f(path.Predicate)
		}
	}
	for _, union := range grp.Unions {
		union.Iter(f)
	}
}

func (grp *GraphGroup) IterTriples(f func(t Triple) Triple) {
	for idx, triple := range grp.Terms {
		grp.Terms[idx] = f(triple)
	}
	for _, union := range grp.Unions {
		union.IterTriples(f)
	}
}

type SelectClause struct {
	Vars    []string
	AllVars bool
}

func NewAllSelectClause() (SelectClause, error) {
	return SelectClause{AllVars: true}, nil
}

func NewSelectClause(varlist interface{}) (SelectClause, error) {
	return SelectClause{Vars: varlist.([]string)}, nil
}

type InsertClause struct {
	Terms []Triple
}

func NewInsertClause(triples interface{}) (InsertClause, error) {
	return InsertClause{
		Terms: triples.([]Triple),
	}, nil
}

type FromClause struct {
	Databases []string
	AllDBs    bool
}

func (f FromClause) String() string {
	if f.AllDBs {
		return "*"
	}
	return strings.Join(f.Databases, " ")
}

func NewAllFromClause() (FromClause, error) {
	return FromClause{AllDBs: true}, nil
}

func NewFromClause(dblist interface{}) (FromClause, error) {
	return FromClause{Databases: dblist.([]string)}, nil
}

func (from FromClause) Empty() bool {
	return len(from.Databases) == 0 && !from.AllDBs
}

type WhereClause struct {
	Terms      []Triple
	GraphGroup *GraphGroup
}

func NewWhereClause(triples interface{}) (WhereClause, error) {
	return WhereClause{
		Terms: triples.([]Triple),
	}, nil
}

func NewWhereClauseWithGraphGroup(triples, group interface{}) (WhereClause, error) {
	g := group.(GraphGroup)
	return WhereClause{
		Terms:      triples.([]Triple),
		GraphGroup: &g,
	}, nil
}

func NewWhereClauseGraphGroup(group interface{}) (WhereClause, error) {
	g := group.(GraphGroup)
	return WhereClause{
		GraphGroup: &g,
	}, nil
}

type GraphGroup struct {
	Terms  []Triple
	Unions []GraphGroup
}

func GraphGroupFromTriples(triples interface{}) (GraphGroup, error) {
	return GraphGroup{
		Terms: triples.([]Triple),
	}, nil
}

func GraphGroupUnion(left, right interface{}) (GraphGroup, error) {
	return GraphGroup{
		Unions: []GraphGroup{left.(GraphGroup), right.(GraphGroup)},
	}, nil
}

func AddTriplesToGraphGroup(left, triples interface{}) (GraphGroup, error) {
	return GraphGroup{
		Terms:  append(left.(GraphGroup).Terms, triples.([]Triple)...),
		Unions: left.(GraphGroup).Unions,
	}, nil
}

func MergeGraphGroups(left, right interface{}) (GraphGroup, error) {
	return GraphGroup{
		Terms:  append(left.(GraphGroup).Terms, right.(GraphGroup).Terms...),
		Unions: append(left.(GraphGroup).Unions, right.(GraphGroup).Unions...),
	}, nil
}

type Triple struct {
	Subject    turtle.URI
	Predicates []PathPattern
	Object     turtle.URI
}

func (t Triple) String() string {
	s := "<" + t.Subject.String() + "|"
	for _, pp := range t.Predicates {
		s += " " + pp.String()
	}
	return s + " | " + t.Object.String() + ">"
}

func (t Triple) Copy() Triple {
	var p = make([]PathPattern, len(t.Predicates))
	copy(p, t.Predicates)
	return Triple{
		Subject:    t.Subject,
		Object:     t.Object,
		Predicates: p,
	}
}

func NewTriple(subject, predicates, object interface{}) (Triple, error) {
	return Triple{
		Subject:    subject.(turtle.URI),
		Predicates: predicates.([]PathPattern),
		Object:     object.(turtle.URI),
	}, nil
}

func NewTripleBlock(triple interface{}) ([]Triple, error) {
	return []Triple{triple.(Triple)}, nil
}

func AppendTripleBlock(block, triple interface{}) ([]Triple, error) {
	return append(block.([]Triple), triple.(Triple)), nil
}

func NewURI(value interface{}) (turtle.URI, error) {
	return turtle.ParseURI(value.(string)), nil
}

func NewVarList(_var interface{}) ([]string, error) {
	return []string{_var.(string)}, nil
}

func AppendVar(varlist, _var interface{}) ([]string, error) {
	return append(varlist.([]string), _var.(string)), nil
}

func NewStringList(_str interface{}) ([]string, error) {
	return []string{_str.(string)}, nil
}

func AppendString(strlist, _str interface{}) ([]string, error) {
	return append(strlist.([]string), _str.(string)), nil
}

func ParseString(_var interface{}) (string, error) {
	return string(_var.(*token.Token).Lit), nil
}

func ParseQuotedString(_var interface{}) (string, error) {
	return string(_var.(*token.Token).Lit), nil
}

func NewPathSequence(_pred interface{}) ([]PathPattern, error) {
	return []PathPattern{_pred.(PathPattern)}, nil
}

func AppendPathSequence(_seq, _pred interface{}) ([]PathPattern, error) {
	return append(_seq.([]PathPattern), _pred.(PathPattern)), nil
}

func NewPathPattern(_pred interface{}) (PathPattern, error) {
	pred, _ := ParseString(_pred)
	if pred == "a" {
		pred = "rdf:type"
	}
	return PathPattern{
		Predicate: turtle.ParseURI(pred),
		Pattern:   PATTERN_SINGLE,
	}, nil
}

func AddPathMod(_pred, _mod interface{}) (PathPattern, error) {
	return PathPattern{
		Predicate: _pred.(PathPattern).Predicate,
		Pattern:   _mod.(Pattern),
	}, nil
}

type PathPattern struct {
	Predicate turtle.URI
	Pattern   Pattern
}

func PathFromVar(_var interface{}) ([]PathPattern, error) {
	return []PathPattern{
		PathPattern{
			Predicate: turtle.ParseURI(_var.(string)),
			Pattern:   PATTERN_SINGLE,
		},
	}, nil
}

func (pp PathPattern) String() string {
	return pp.Predicate.String() + pp.Pattern.String()
}

type Pattern uint

const (
	PATTERN_SINGLE = iota + 1
	PATTERN_ZERO_ONE
	PATTERN_ONE_PLUS
	PATTERN_ZERO_PLUS
)

func (p Pattern) String() string {
	switch p {
	case PATTERN_SINGLE:
		return ""
	case PATTERN_ZERO_ONE:
		return "?"
	case PATTERN_ONE_PLUS:
		return "+"
	case PATTERN_ZERO_PLUS:
		return "*"
	}
	return "unknown"
}
<|endoftext|>


<commit_before>package main

import "fmt"

func main() {

	m := map[string]int{
		"a": 1,
		"b": 2,
		"c": 3,
	}

	for k, v := range m {
		fmt.Println(k, v)
	}

	// When iterating over map keys, key order is not guaranteed

	// One Possible Output:
	// b 2
	// c 3
	// a 1

	// Another Possible Output:
	// c 3
	// a 1
	// b 2

	// Another Possible Output:
	// a 1
	// b 2
	// c 3
}
<commit_msg>Update map.go show adding values after map is initialized<commit_after>package main

import "fmt"

func main() {

	team := map[string]bool{}
	team["jane"] = true
	team["john"] = true
	team["mel"] = true

	delete(team, "jane")

	fmt.Println(team["bob"], team["jane"], team["mel"])
	// OUTPUT: false true true

	m := map[string]int{
		"a": 1,
		"b": 2,
		"c": 3,
	}

	for k, v := range m {
		fmt.Println(k, v)
	}

	// When iterating over map keys, key order is not guaranteed

	// One Possible Output:
	// b 2
	// c 3
	// a 1

	// Another Possible Output:
	// c 3
	// a 1
	// b 2

	// Another Possible Output:
	// a 1
	// b 2
	// c 3
}
<|endoftext|>


<commit_before>package account

import (
	"crypto/rand"
	"io"

	"github.com/melange-app/nmcd/btcutil"
	"github.com/melange-app/nmcd/txscript"
	"github.com/melange-app/nmcd/wire"
)

const (
	// Name network fee is currently 10 mNMC
	nameNetworkFee         = 10e5
	nameTransactionVersion = 28928
)

// ... << OP_DUP << OP_HASH160 << hash160 << OP_EQUALVERIFY << OP_CHECKSIG
func (a *Account) finishNameTransaction(sb *txscript.ScriptBuilder) (*Transaction, error) {
	pk := a.Keys.PubKey().SerializeCompressed()

	// Add the normal pk2pkh script at the end.
	sb.AddOp(txscript.OP_DUP).AddOp(txscript.OP_HASH160).
		AddData(btcutil.Hash160(pk)).
		AddOp(txscript.OP_EQUALVERIFY).AddOp(txscript.OP_CHECKSIG)

	script, err := sb.Script()
	if err != nil {
		return nil, err
	}
	txOutput := []*wire.TxOut{
		wire.NewTxOut(nameNetworkFee, script),
	}

	// Build the transaction
	finalTx, err := a.buildTransactionVersion(txOutput, nameTransactionVersion)
	if err != nil {
		return nil, err
	}

	return finalTx, nil
}

// CreateNameNew will make a new transaction that preregisters a
// namecoin name in the blockchain.
//
// OP_NAME_NEW (1) << hash << OP_2DROP << ...
func (a *Account) CreateNameNew(name string) (*Transaction, []byte, error) {
	sb := txscript.NewScriptBuilder()

	// read a 64-bit random number to use as the salt
	buffer := make([]byte, 8)
	_, err := io.ReadFull(rand.Reader, buffer)
	if err != nil {
		return nil, nil, err
	}

	// Hash the name with the salt.
	nameBy := []byte(name)
	hash := btcutil.Hash160(append(nameBy, buffer...))

	// Build the name script
	sb.AddOp(txscript.OP_1).AddData(hash).AddOp(txscript.OP_2DROP)

	tx, err := a.finishNameTransaction(sb)
	if err != nil {
		return nil, nil, err
	}

	return tx, buffer, err
}

// CreateNameFirstUpdate will reveal a name and initial value to the
// blockchain.
//
// OP_NAME_FIRSTUPDATE (2) << vchName << vchRand << vchValue << OP_2DROP << OP_2DROP << ...
func (a *Account) CreateNameFirstUpdate(rand []byte, name, value string) (*Transaction, error) {
	sb := txscript.NewScriptBuilder()

	// Build the name script
	sb.AddOp(txscript.OP_2).
		AddData([]byte(name)).AddData(rand).AddData([]byte(value)).
		AddOp(txscript.OP_2DROP).AddOp(txscript.OP_2DROP)

	return a.finishNameTransaction(sb)
}

// CreateNameUpdate will make a new transation that updates a name to
// a new value.
//
// OP_NAME_UPDATE (3) << vchName << vchValue << OP_2DROP << OP_DROP << ...
func (a *Account) CreateNameUpdate(name, value, address string) (*Transaction, error) {
	sb := txscript.NewScriptBuilder()

	// Build the name script
	sb.AddOp(txscript.OP_3).
		AddData([]byte(name)).AddData([]byte(value)).
		AddOp(txscript.OP_2DROP).AddOp(txscript.OP_DROP)

	return a.finishNameTransaction(sb)
}
<commit_msg>zooko: fix bug in OP_NAME_NEW registration<commit_after>package account

import (
	"crypto/rand"
	"io"

	"github.com/melange-app/nmcd/btcutil"
	"github.com/melange-app/nmcd/txscript"
	"github.com/melange-app/nmcd/wire"
)

const (
	// Name network fee is currently 10 mNMC
	nameNetworkFee         = 10e5
	nameTransactionVersion = 28928
)

// ... << OP_DUP << OP_HASH160 << hash160 << OP_EQUALVERIFY << OP_CHECKSIG
func (a *Account) finishNameTransaction(sb *txscript.ScriptBuilder) (*Transaction, error) {
	pk := a.Keys.PubKey().SerializeCompressed()

	// Add the normal pk2pkh script at the end.
	sb.AddOp(txscript.OP_DUP).AddOp(txscript.OP_HASH160).
		AddData(btcutil.Hash160(pk)).
		AddOp(txscript.OP_EQUALVERIFY).AddOp(txscript.OP_CHECKSIG)

	script, err := sb.Script()
	if err != nil {
		return nil, err
	}
	txOutput := []*wire.TxOut{
		wire.NewTxOut(nameNetworkFee, script),
	}

	// Build the transaction
	finalTx, err := a.buildTransactionVersion(txOutput, nameTransactionVersion)
	if err != nil {
		return nil, err
	}

	return finalTx, nil
}

// CreateNameNew will make a new transaction that preregisters a
// namecoin name in the blockchain.
//
// OP_NAME_NEW (1) << hash << OP_2DROP << ...
func (a *Account) CreateNameNew(name string) (*Transaction, []byte, error) {
	sb := txscript.NewScriptBuilder()

	// read a 64-bit random number to use as the salt
	buffer := make([]byte, 8)
	_, err := io.ReadFull(rand.Reader, buffer)
	if err != nil {
		return nil, nil, err
	}

	// Hash the name with the salt.
	nameBy := []byte(name)
	hash := btcutil.Hash160(append(buffer, nameBy...))

	// Build the name script
	sb.AddOp(txscript.OP_1).AddData(hash).AddOp(txscript.OP_2DROP)

	tx, err := a.finishNameTransaction(sb)
	if err != nil {
		return nil, nil, err
	}

	return tx, buffer, err
}

// CreateNameFirstUpdate will reveal a name and initial value to the
// blockchain.
//
// OP_NAME_FIRSTUPDATE (2) << vchName << vchRand << vchValue << OP_2DROP << OP_2DROP << ...
func (a *Account) CreateNameFirstUpdate(rand []byte, name, value string) (*Transaction, error) {
	sb := txscript.NewScriptBuilder()

	// Build the name script
	sb.AddOp(txscript.OP_2).
		AddData([]byte(name)).AddData(rand).AddData([]byte(value)).
		AddOp(txscript.OP_2DROP).AddOp(txscript.OP_2DROP)

	return a.finishNameTransaction(sb)
}

// CreateNameUpdate will make a new transation that updates a name to
// a new value.
//
// OP_NAME_UPDATE (3) << vchName << vchValue << OP_2DROP << OP_DROP << ...
func (a *Account) CreateNameUpdate(name, value, address string) (*Transaction, error) {
	sb := txscript.NewScriptBuilder()

	// Build the name script
	sb.AddOp(txscript.OP_3).
		AddData([]byte(name)).AddData([]byte(value)).
		AddOp(txscript.OP_2DROP).AddOp(txscript.OP_DROP)

	return a.finishNameTransaction(sb)
}
<|endoftext|>


<commit_before>/******************************************************************************
pageviewsync Source Code
Copyright (C) 2013 Lumen LLC.

This file is part of the pageviewsync Source Code.

pageviewsync is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

pageviewsync is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with pageviewsync.  If not, see <http://www.gnu.org/licenses/>.
*****************************************************************************/

package worker

import (
	"database/sql"
	"errors"
	"fmt"
	_ "github.com/go-sql-driver/mysql"
	"lumenlearning.com/pageviews/canvas"
	"net/http"
	"sync"
)

type Worker struct {
	UserID string
	DBInfo DBConnectInfo
	APIInfo APIConnectInfo

	WorkGrp chan bool
	WaitGrp *sync.WaitGroup
	Logger chan interface{}
}

type DBConnectInfo struct {
	User string
	Pass string
	Schema string
	Table string
}

func (d *DBConnectInfo) String() string {
	return "User:"+d.User+",Pass:"+d.Pass+",Schema:"+d.Schema+",Table:"+d.Table
}

type APIConnectInfo struct {
	Host string
	Auth string
	Client *http.Client
}

func (a *APIConnectInfo) String() string {
	return "Host:"+a.Host+",Auth:"+a.Auth+",Client:"+fmt.Sprint(a.Client)
}

// Indicate that this goroutine is finished.
// Free up space for another worker to start.
func (w *Worker) Done() {
	w.WaitGrp.Done()
	<- w.WorkGrp
}

func (w *Worker) Log(format string, args... interface{}) {
	w.Logger <- fmt.Sprintf("%v: %v", w.UserID, fmt.Sprintf(format, args...))
}

func (w *Worker) RunPageviewUpdate() error {
	// Find out what we need to do to get this user up-to-date.
	// We'll find out what the last timestamp we have recorded is,
	// if any.
	lastRequestID, lastTimestamp, err := GetUpdateReqs(w.UserID, w.DBInfo)
	if err != nil {
		w.Done()
		w.Log("Terminating. GetUpdateReqs says: %v", err.Error())
		return err
	}
	var fullUpdate bool = (lastTimestamp == "" || lastRequestID == "")

	w.Log("lastRequestID=%v, lastRequestID=%v, fullUpdate=%v", lastRequestID, lastTimestamp, fullUpdate)

	// Get the information needed to update the local DB
	url := "https://"+w.APIInfo.Host+"/api/v1/users/"+w.UserID+"/page_views?per_page=1000"
	w.Log("First URL is %v", url)

	// Go get as many pages as we need
	var pvs []canvas.Pageview
	keepGoing := true
	for keepGoing {
		w.Log("Fetching %v", url)
		nextLink, results, err := ParsePage(url, w.APIInfo)
		if err != nil {
			w.Done()
			w.Log("Terminating. ParsePage says: %v", err.Error())
			return err
		}

		// Update url to be the next link
		w.Log("Next page: %v", nextLink)
		url = nextLink

		if !fullUpdate {
			// Find out if we need to keep going
			for _, r := range results {
				ts, _ := r["created_at"]
				id, _ := r["request_id"]
				w.Log("Pageview at %v (%v)", ts, id)

				tISO, err := canvas.TimeFromISO8601(ts.(string))
				if err != nil {
					w.Done()
					w.Log("Terminated. TimeFromISO8601 says: %v", err)
					return err
				}
				tUnix, err := canvas.UnixFromTime(tISO)
				if err != nil {
					w.Done()
					w.Log("Termianted. UnixFromTime says: %v", err)
					return err
				}

				if id.(string) == lastRequestID || tUnix < lastTimestamp {
					keepGoing = false
					break
				} else {
					pvs = append(pvs, r)
				}
			}
		} else {
			// Add everything from this page
			pvs = append(pvs, results...)
		}
		// Is there anything left to grab?
		if nextLink == "" {
			keepGoing = false
			break
		}
	}

	insertCount, errorFunc, err := UpdateDB(&w.DBInfo, &pvs)
	if err != nil {
		w.Done()
		w.Log("Terminated. UpdateDB/%v says: %v", errorFunc, err.Error())
		return err
	}

	// If we got to this point, no errors!
	w.Done()
	w.Log("Updated with %v rows.", insertCount)
	return nil
}

func UpdateDB(dbInfo *DBConnectInfo, pvs *[]canvas.Pageview) (int64, string, error) {
	//// send the results to the local database ////
	// make the connection
	con, err := sql.Open("mysql", dbInfo.User+":"+dbInfo.Pass+"@unix(/var/run/mysqld/mysqld.sock)"+"/"+dbInfo.Schema)
	defer con.Close()
	if err != nil {
		return 0, "sql.Open", err
	}

	// start a transaction
	tx, err := con.Begin()
	if err != nil {
		return 0, "sql.Begin", err
	}

	// Keep track of total pageview records updated
	var insertCount int64

	// convert each canvas.Pageview to an insert statement
	for _, pv := range *pvs {
		ins := "INSERT INTO "+dbInfo.Table+" ("
		val := " VALUES ("
		i := 0
		for k, v := range pv {
			// Get the string representation of the value
			strVal := fmt.Sprint(v)
			if strVal == "<nil>" {
				strVal = "NULL"
			}

			// build insert and value strings for this item
			insI, valI := BuildInsertAndValues(k, strVal, i)
			ins = ins + insI
			val = val + valI
			i += 1

			// If it's the "created_at" field, we also need to make a copy
			//   in YYYY-mm-dd HH:MM:SS format.  It's NOT "Unix" time format,
			//   but that's what I called it for lack of a better term.
			// Also for the "updated_at" field.
			if k == "created_at" {
				insI, valI, err := GetDateTimeValue("created_at_datetime", strVal, i)
				if err != nil {
					return 0, "GetDateTimeValue", err
				}
				ins = ins + insI
				val = val + valI
				i += 1
			}
			if k == "updated_at" {
				insI, valI, err := GetDateTimeValue("updated_at_datetime", strVal, i)
				if err != nil {
					return 0, "GetDateTimeValue", err
				}
				ins = ins + insI
				val = val + valI
				i += 1
			}

		}

		// Finish up the query bits
		ins = ins + ")"
		val = val + ")"

		// add each insert to the transaction
		qu := ins + " " + val
		res, err := tx.Exec(qu)
		if err != nil {
			err1 := err
			err := tx.Rollback()
			return 0, "tx.Exec", errors.New(err1.Error()+"\n"+err.Error())
		}

		// Record number of rows inserted
		rowsAffected, err := res.(sql.Result).RowsAffected()
		if err != nil {
			return 0, "res.RowsAffected", err
		}
		insertCount += rowsAffected
	}

	// commit the transaction
	err = tx.Commit()
	if err != nil {
		return 0, "tx.Commit", err
	}

	return insertCount, "", nil
}

func GetDateTimeValue(k, strVal string, i int) (string, string, error) {
	t, err := canvas.TimeFromISO8601(strVal)
	if err != nil {
		return "", "", err
	}

	tu, err := canvas.UnixFromTime(t)
	if err != nil {
		return "", "", err
	}

	insI, valI := BuildInsertAndValues(k, tu, i)
	return insI, valI, nil
}

func BuildInsertAndValues(k, strVal string, i int) (string, string) {
	ins := ""
	val := ""

	// Change the quote character to handle NULLs, which shouldn't be in quotes.
	qc := "\""
	if strVal == "NULL" {
		qc = ""
	}

	if i > 0 {
		ins = fmt.Sprintf(",`%v`", k)
		val  = fmt.Sprintf(",%v%v%v", qc, strVal, qc)
	} else {
		ins = fmt.Sprintf("`%v`", k)
		val = fmt.Sprintf("%v%v%v", qc, strVal, qc)
	}

	return ins, val
}

// Call the API and parse the results
// Return  the "next" link
// Return a []canvas.Pageview
func ParsePage(url string, apiInfo APIConnectInfo) (string, []canvas.Pageview, error) {
	// Call the API and get the response
	resp, body, err := GetResponse(url, apiInfo)
	if err != nil {
		return "", nil, err
	}

	// Get the Link header, check for rel="next"
	nextLink, err := canvas.GetNextLink(resp)
	if err != nil {
		return "", nil, err
	}

	// Get a map representation of the data returned
	obj, err := canvas.GetObjFromJSON(body)
	if err != nil {
		return "", nil, err
	}

	var results []interface{}

	switch vt := (*obj).(type) {
	case []interface{}:
		results = (*obj).([]interface{})
	default:
		return "", nil, errors.New(fmt.Sprint("Expecting an array (received ", vt, ")"))
	}

	var pvs []canvas.Pageview

	for _, v := range results {
		switch v.(type) {
		case map[string]interface{}:
			pageviewResult := v.(map[string]interface{})
			var pv canvas.Pageview = make(canvas.Pageview)
			for k, v := range pageviewResult {
				if k == "created_at" {

				}
				pv[k] = v
			}
			pvs = append(pvs, pv)
		}
	}

	return nextLink, pvs, nil
}

func GetResponse(url string, apiInfo APIConnectInfo) (*http.Response, *[]byte, error) {
	resp, _, err := canvas.AuthorizedCall(url, apiInfo.Auth, apiInfo.Client)
	if err != nil {
		return nil, nil, err
	}

	body, _, err := canvas.ReadResponse(resp)
	if err != nil {
		return nil, nil, err
	}

	return resp, body, nil
}

func GetUpdateReqs(userID string, dbInfo DBConnectInfo) (string, string, error) {
	// Connect to the DB and the appropriate schema
	con, err := sql.Open("mysql", dbInfo.User+":"+dbInfo.Pass+"@unix(/var/run/mysqld/mysqld.sock)"+"/"+dbInfo.Schema)
	defer con.Close()
	if err != nil {
		return "", "", err
	}

	// Build the query
	query := "SELECT request_id, created_at FROM pageviews WHERE user_id = '"+userID+"' ORDER BY created_at DESC LIMIT 1"

	// Find out the last pageview timestamp for this userID
	row := con.QueryRow(query)
	var request_id string
	var created_at string
	err = row.Scan(&request_id, &created_at)

	if err != nil {
		if err == sql.ErrNoRows {
			request_id = ""
			created_at = ""
		} else {
			return "", "", err
		}
	}

	return request_id, created_at, nil
}
<commit_msg>Combine error messages to provide info like a stack trace.<commit_after>/******************************************************************************
pageviewsync Source Code
Copyright (C) 2013 Lumen LLC.

This file is part of the pageviewsync Source Code.

pageviewsync is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

pageviewsync is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with pageviewsync.  If not, see <http://www.gnu.org/licenses/>.
*****************************************************************************/

package worker

import (
	"database/sql"
	"errors"
	"fmt"
	_ "github.com/go-sql-driver/mysql"
	"lumenlearning.com/pageviews/canvas"
	"net/http"
	"sync"
)

type Worker struct {
	UserID string
	DBInfo DBConnectInfo
	APIInfo APIConnectInfo

	WorkGrp chan bool
	WaitGrp *sync.WaitGroup
	Logger chan interface{}
}

type DBConnectInfo struct {
	User string
	Pass string
	Schema string
	Table string
}

func (d *DBConnectInfo) String() string {
	return "User:"+d.User+",Pass:"+d.Pass+",Schema:"+d.Schema+",Table:"+d.Table
}

type APIConnectInfo struct {
	Host string
	Auth string
	Client *http.Client
}

func (a *APIConnectInfo) String() string {
	return "Host:"+a.Host+",Auth:"+a.Auth+",Client:"+fmt.Sprint(a.Client)
}

// Indicate that this goroutine is finished.
// Free up space for another worker to start.
func (w *Worker) Done() {
	w.WaitGrp.Done()
	<- w.WorkGrp
}

func (w *Worker) Log(format string, args... interface{}) {
	w.Logger <- fmt.Sprintf("%v: %v", w.UserID, fmt.Sprintf(format, args...))
}

func (w *Worker) RunPageviewUpdate() error {
	// Find out what we need to do to get this user up-to-date.
	// We'll find out what the last timestamp we have recorded is,
	// if any.
	lastRequestID, lastTimestamp, err := GetUpdateReqs(w.UserID, w.DBInfo)
	if err != nil {
		w.Done()
		w.Log("Terminating. GetUpdateReqs says: %v", err.Error())
		return errors.New("worker.GetUpdateReqs => " + err.Error())
	}
	var fullUpdate bool = (lastTimestamp == "" || lastRequestID == "")

	w.Log("lastRequestID=%v, lastRequestID=%v, fullUpdate=%v", lastRequestID, lastTimestamp, fullUpdate)

	// Get the information needed to update the local DB
	url := "https://"+w.APIInfo.Host+"/api/v1/users/"+w.UserID+"/page_views?per_page=1000"

	// Go get as many pages as we need
	var pvs []canvas.Pageview
	keepGoing := true
	for keepGoing {
		w.Log("Fetching %v", url)
		nextLink, results, err := ParsePage(url, w.APIInfo)
		if err != nil {
			w.Done()
			w.Log("Terminating. ParsePage says: %v", err.Error())
			return errors.New("worker.ParsePage => " + err.Error())
		}

		// Update url to be the next link
		url = nextLink

		if !fullUpdate {
			// Find out if we need to keep going
			for _, r := range results {
				ts, _ := r["created_at"]
				id, _ := r["request_id"]

				tISO, err := canvas.TimeFromISO8601(ts.(string))
				if err != nil {
					w.Done()
					w.Log("Terminated. TimeFromISO8601 says: %v", err)
					return errors.New("canvas.TimeFromISO8601 => " + err.Error())
				}
				tUnix, err := canvas.UnixFromTime(tISO)
				if err != nil {
					w.Done()
					w.Log("Terminated. UnixFromTime says: %v", err)
					return errors.New("canvas.UnixFromTime => " + err.Error())
				}

				if id.(string) == lastRequestID || tUnix < lastTimestamp {
					keepGoing = false
					break
				} else {
					pvs = append(pvs, r)
				}
			}
		} else {
			// Add everything from this page
			pvs = append(pvs, results...)
		}
		// Is there anything left to grab?
		if nextLink == "" {
			keepGoing = false
			break
		}
	}

	insertCount, err := UpdateDB(&w.DBInfo, &pvs)
	if err != nil {
		w.Done()
		w.Log("Terminated. UpdateDB says: %v", err.Error())
		return errors.New("worker.UpdateDB => " + err.Error())
	}

	// If we got to this point, no errors!
	w.Done()
	w.Log("Updated with %v rows.", insertCount)
	return nil
}

func UpdateDB(dbInfo *DBConnectInfo, pvs *[]canvas.Pageview) (int64, error) {
	//// send the results to the local database ////
	// make the connection
	con, err := sql.Open("mysql", dbInfo.User+":"+dbInfo.Pass+"@unix(/var/run/mysqld/mysqld.sock)"+"/"+dbInfo.Schema)
	defer con.Close()
	if err != nil {
		return 0, errors.New("sql.Open => " + err.Error())
	}

	// start a transaction
	tx, err := con.Begin()
	if err != nil {
		return 0, errors.New("con.Begin => " + err.Error())
	}

	// Keep track of total pageview records updated
	var insertCount int64

	// convert each canvas.Pageview to an insert statement
	for _, pv := range *pvs {
		ins := "INSERT INTO "+dbInfo.Table+" ("
		val := " VALUES ("
		i := 0
		for k, v := range pv {
			// Get the string representation of the value
			strVal := fmt.Sprint(v)
			if strVal == "<nil>" {
				strVal = "NULL"
			}

			// build insert and value strings for this item
			insI, valI := BuildInsertAndValues(k, strVal, i)
			ins = ins + insI
			val = val + valI
			i += 1

			// If it's the "created_at" field, we also need to make a copy
			//   in YYYY-mm-dd HH:MM:SS format.  It's NOT "Unix" time format,
			//   but that's what I called it for lack of a better term.
			// Also for the "updated_at" field.
			if k == "created_at" {
				insI, valI, err := GetDateTimeValue("created_at_datetime", strVal, i)
				if err != nil {
					return 0, errors.New("worker.GetDateTimeValue => " + err.Error())
				}
				ins = ins + insI
				val = val + valI
				i += 1
			}
			if k == "updated_at" {
				insI, valI, err := GetDateTimeValue("updated_at_datetime", strVal, i)
				if err != nil {
					return 0, errors.New("worker.GetDateTimeValue => " + err.Error())
				}
				ins = ins + insI
				val = val + valI
				i += 1
			}

		}

		// Finish up the query bits
		ins = ins + ")"
		val = val + ")"

		// add each insert to the transaction
		qu := ins + " " + val
		res, err := tx.Exec(qu)
		if err != nil {
			err1 := err
			err := tx.Rollback()
			return 0, errors.New("tx.Exec => "+err1.Error()+"\n"+err.Error())
		}

		// Record number of rows inserted
		rowsAffected, err := res.(sql.Result).RowsAffected()
		if err != nil {
			return 0, errors.New("res.RowsAffected => " + err.Error())
		}
		insertCount += rowsAffected
	}

	// commit the transaction
	err = tx.Commit()
	if err != nil {
		return 0, errors.New("tx.Commit => " + err.Error())
	}

	return insertCount, nil
}

func GetDateTimeValue(k, strVal string, i int) (string, string, error) {
	t, err := canvas.TimeFromISO8601(strVal)
	if err != nil {
		return "", "", errors.New("canvas.TimeFromISO8601 => " + err.Error())
	}

	tu, err := canvas.UnixFromTime(t)
	if err != nil {
		return "", "", errors.New("canvas.UnixFromTime => " + err.Error())
	}

	insI, valI := BuildInsertAndValues(k, tu, i)
	return insI, valI, nil
}

func BuildInsertAndValues(k, strVal string, i int) (string, string) {
	ins := ""
	val := ""

	// Change the quote character to handle NULLs, which shouldn't be in quotes.
	qc := "\""
	if strVal == "NULL" {
		qc = ""
	}

	if i > 0 {
		ins = fmt.Sprintf(",`%v`", k)
		val  = fmt.Sprintf(",%v%v%v", qc, strVal, qc)
	} else {
		ins = fmt.Sprintf("`%v`", k)
		val = fmt.Sprintf("%v%v%v", qc, strVal, qc)
	}

	return ins, val
}

// Call the API and parse the results
// Return  the "next" link
// Return a []canvas.Pageview
func ParsePage(url string, apiInfo APIConnectInfo) (string, []canvas.Pageview, error) {
	// Call the API and get the response
	resp, body, err := GetResponse(url, apiInfo)
	if err != nil {
		return "", nil, errors.New("worker.GetResponse" + err.Error())
	}

	// Get the Link header, check for rel="next"
	nextLink, err := canvas.GetNextLink(resp)
	if err != nil {
		return "", nil, errors.New("canvas.GetNextLink => " + err.Error())
	}

	// Get a map representation of the data returned
	obj, err := canvas.GetObjFromJSON(body)
	if err != nil {
		return "", nil, errors.New("canvas.GetObjFromJSON => " + err.Error())
	}

	var results []interface{}

	switch vt := (*obj).(type) {
	case []interface{}:
		results = (*obj).([]interface{})
	default:
		return "", nil, errors.New(fmt.Sprintf("Expecting an array, received %v", vt))
	}

	var pvs []canvas.Pageview

	for _, v := range results {
		switch v.(type) {
		case map[string]interface{}:
			pageviewResult := v.(map[string]interface{})
			var pv canvas.Pageview = make(canvas.Pageview)
			for k, v := range pageviewResult {
				pv[k] = v
			}
			pvs = append(pvs, pv)
		}
	}

	return nextLink, pvs, nil
}

func GetResponse(url string, apiInfo APIConnectInfo) (*http.Response, *[]byte, error) {
	resp, _, err := canvas.AuthorizedCall(url, apiInfo.Auth, apiInfo.Client)
	if err != nil {
		return nil, nil, errors.New("canvas.AuthorizedCall => " + err.Error())
	}

	body, _, err := canvas.ReadResponse(resp)
	if err != nil {
		return nil, nil, errors.New("canvas.ReadResponse => " + err.Error())
	}

	return resp, body, nil
}

func GetUpdateReqs(userID string, dbInfo DBConnectInfo) (string, string, error) {
	// Connect to the DB and the appropriate schema
	con, err := sql.Open("mysql", dbInfo.User+":"+dbInfo.Pass+"@unix(/var/run/mysqld/mysqld.sock)"+"/"+dbInfo.Schema)
	defer con.Close()
	if err != nil {
		return "", "", errors.New("sql.Open => " + err.Error())
	}

	// Build the query
	query := "SELECT request_id, created_at FROM pageviews WHERE user_id = '"+userID+"' ORDER BY created_at DESC LIMIT 1"

	// Find out the last pageview timestamp for this userID
	row := con.QueryRow(query)
	var request_id string
	var created_at string
	err = row.Scan(&request_id, &created_at)

	if err != nil {
		if err == sql.ErrNoRows {
			request_id = ""
			created_at = ""
		} else {
			return "", "", errors.New("row.Scan => " + err.Error())
		}
	}

	return request_id, created_at, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"regexp"
	"sync"
)

func main() {
	results := make(chan string)
	errs := make(chan error)

	go producer(results, errs)

	for {
		select {
		case r, ok := <-results:
			if !ok {
				fmt.Printf("Channel is closed\n")
				return
			}
			fmt.Printf("result = %v\n", r)
		case err := <-errs:
			fmt.Printf("err = %v\n", err)
		}
	}
}

func producer(results chan string, errs chan error) {
	urls := []string{
		"https://oinume.hatenablog.com/entry/paid-mac-apps",
		"https://oinume.hatenablog.com/entry/generating-an-unpredictable-random-value-in-go",
		"https://oinume.hatenablog.com/entry/review-2018-04",
	}
	var wg sync.WaitGroup
	for _, url := range urls {
		wg.Add(1)
		go func(url string) {
			defer wg.Done()
			resp, err := http.DefaultClient.Get(url)
			//fmt.Printf("GET %v\n", url)
			if err != nil {
				errs <- err
				return
			}
			b, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				errs <- err
				return
			}
			defer resp.Body.Close()
			title := extractTitle(string(b))
			results <- title
		}(url)
	}
	wg.Wait()
	close(results)
	//close(errs)
}

func extractTitle(s string) string {
	title := regexp.MustCompile("<title>(.*)</title>")
	group := title.FindStringSubmatch(s)
	return group[1]
}
<commit_msg>Use global variable<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"regexp"
	"sync"
)

func main() {
	results := make(chan string)
	errs := make(chan error)

	go producer(results, errs)

	for {
		select {
		case r, ok := <-results:
			if !ok {
				fmt.Printf("Channel is closed\n")
				return
			}
			fmt.Printf("result = %v\n", r)
		case err := <-errs:
			fmt.Printf("err = %v\n", err)
		}
	}
}

func producer(results chan string, errs chan error) {
	urls := []string{
		"https://oinume.hatenablog.com/entry/paid-mac-apps",
		"https://oinume.hatenablog.com/entry/generating-an-unpredictable-random-value-in-go",
		"https://oinume.hatenablog.com/entry/review-2018-04",
	}
	var wg sync.WaitGroup
	for _, url := range urls {
		wg.Add(1)
		go func(url string) {
			defer wg.Done()
			resp, err := http.DefaultClient.Get(url)
			//fmt.Printf("GET %v\n", url)
			if err != nil {
				errs <- err
				return
			}
			b, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				errs <- err
				return
			}
			defer resp.Body.Close()
			title := extractTitle(string(b))
			results <- title
		}(url)
	}
	wg.Wait()
	close(results)
	//close(errs)
}

var titleRegexp = regexp.MustCompile("<title>(.*)</title>")

func extractTitle(s string) string {
	group := titleRegexp.FindStringSubmatch(s)
	return group[1]
}
<|endoftext|>


<commit_before>package validate

import (
	"reflect"
	"regexp"
	"strings"

	"github.com/tonyhb/govalidate/rules"
	_ "github.com/tonyhb/govalidate/rules/alpha"
	_ "github.com/tonyhb/govalidate/rules/alphanumeric"
	_ "github.com/tonyhb/govalidate/rules/email"
	_ "github.com/tonyhb/govalidate/rules/greaterthan"
	_ "github.com/tonyhb/govalidate/rules/length"
	_ "github.com/tonyhb/govalidate/rules/lessthan"
	_ "github.com/tonyhb/govalidate/rules/maxlength"
	_ "github.com/tonyhb/govalidate/rules/minlength"
	_ "github.com/tonyhb/govalidate/rules/notempty"
	_ "github.com/tonyhb/govalidate/rules/notzero"
	_ "github.com/tonyhb/govalidate/rules/notzerotime"
	_ "github.com/tonyhb/govalidate/rules/regexp"
	_ "github.com/tonyhb/govalidate/rules/url"
	_ "github.com/tonyhb/govalidate/rules/uuid"
)

// Takes a struct, loops through all fields and calls check on any fields that
// have a validate tag.
func Run(object interface{}, fieldsSlice ...string) error {
	pass := true // We'll override this if checking returns false
	err := ValidationError{}

	// If we have been passed a slice of fields to valiate - to check only a
	// subset of fields - change the slice into a map for O(1) lookups instead
	// of O(n).
	fields := map[string]struct{}{}
	for _, field := range fieldsSlice {
		fields[field] = struct{}{}
	}

	// If we're passed a pointer to a struct we need to dereference the pointer before
	// inspecting its tags
	value := reflect.ValueOf(object)
	if value.Kind() == reflect.Ptr {
		value = value.Elem()
	}

	// Iterate through each field of the struct and validate
	typ := value.Type() // A Type's Field method returns StructFields
	for i := 0; i < value.NumField(); i++ {
		var validateTag string
		var validateError error

		// Is this an anonymous struct? If so, we also need to validate on this.
		if typ.Field(i).Anonymous == true {
			if anonErr := Run(value.Field(i).Interface(), fieldsSlice...); anonErr != nil {
				// The validation failed: set pass to false and merge the anonymous struct's
				// validation errors with our current validation error above to give a complete
				// error message.
				pass = false

				// A non validation error occurred: return this immediately
				if _, ok := anonErr.(ValidationError); !ok {
					return anonErr
				}

				err.Merge(anonErr.(ValidationError))
			}
		}

		if len(fields) > 0 {
			// We're only checking for a subset of fields; if this field isn't
			// included in the subset of fields to validate we can skip.
			if _, ok := fields[typ.Field(i).Name]; !ok {
				continue
			}
		}

		if validateTag = typ.Field(i).Tag.Get("validate"); validateTag == "" {
			continue
		}

		// Validate this particular field against the options in our tag
		if validateError = validateField(value.Field(i).Interface(), typ.Field(i).Name, validateTag); validateError == nil {
			continue
		}

		// If there was no validation rule defined for the given tag return
		// that error immediately.
		if _, ok := validateError.(rules.ErrNoValidationMethod); ok {
			return validateError
		}

		pass = false
		err.addFailure(typ.Field(i).Name, validateError.Error())
	}

	if pass {
		return nil
	}

	return err
}

var rxRegexp = regexp.MustCompile(`Regexp:\/.+/`)

// Takes a field's value and the validation tag and applies each check
// until either a test fails or all tests pass.
func validateField(data interface{}, fieldName, tag string) (err error) {
	// A tag can specify multiple validation rules which are delimited via ','.
	// However, because we allow regular expressions we can't split the tag field
	// via all commas to find our validation rules: we need to extract the regular expression
	// first (in case it specifies a comma), and *then* run through our validation rules.
	if match := rxRegexp.FindString(tag); match != "" {
		// If we fail validating the regexp we can break here
		if err := validateRule(data, fieldName, match); err != nil {
			return err
		}
		// Now we need to replace our regular expression from the tag list to continue
		// normally.
		tag = rxRegexp.ReplaceAllString(tag, "")
	}

	for tag != "" {
		var next string

		i := strings.Index(tag, ",")
		if i >= 0 {
			tag, next = tag[:i], tag[i+1:]
		}

		if err := validateRule(data, fieldName, tag); err != nil {
			return err
		}

		// Continue with the next tag
		tag = next
	}

	return nil
}

// Given a validation rule from a tag, run the associated validation methods and return
// the result.
func validateRule(data interface{}, fieldName, rule string) error {
	var args []string

	// Remove any preceeding spaces from comma separated tags
	rule = strings.TrimLeft(rule, " ")

	// If the rule is empty we don't need to process anything. This only happens
	// if we have a regex followed by another rule:
	//   `validate:"Regexp:/.+/, NotEmpty"`
	// Becomes:
	//   `validate:", NotEmpty"`
	// After processing in validateField()
	if rule == "" {
		return nil
	}

	// rule is the method we want to call. If it has a colon we need to further
	// process the rule to extract arguments to our validation method.
	i := strings.Index(rule, ":")
	if i > 0 {
		var arg string
		rule, arg = rule[:i], rule[i+1:]
		args = append(args, arg)
	}

	// Attempt to validate the data using methods registered with the rules
	// sub package
	if method, err := rules.Get(rule); err != nil {
		return err
	} else {
		var data = rules.ValidationData{
			Field: fieldName,
			Value: data,
			Args:  args,
		}
		return method(data)
	}
}
<commit_msg>comments<commit_after>package validate

import (
	"reflect"
	"regexp"
	"strings"

	"github.com/tonyhb/govalidate/rules"
	_ "github.com/tonyhb/govalidate/rules/alpha"
	_ "github.com/tonyhb/govalidate/rules/alphanumeric"
	_ "github.com/tonyhb/govalidate/rules/email"
	_ "github.com/tonyhb/govalidate/rules/greaterthan"
	_ "github.com/tonyhb/govalidate/rules/length"
	_ "github.com/tonyhb/govalidate/rules/lessthan"
	_ "github.com/tonyhb/govalidate/rules/maxlength"
	_ "github.com/tonyhb/govalidate/rules/minlength"
	_ "github.com/tonyhb/govalidate/rules/notempty"
	_ "github.com/tonyhb/govalidate/rules/notzero"
	_ "github.com/tonyhb/govalidate/rules/notzerotime"
	_ "github.com/tonyhb/govalidate/rules/regexp"
	_ "github.com/tonyhb/govalidate/rules/url"
	_ "github.com/tonyhb/govalidate/rules/uuid"
)

// Takes a struct, loops through all fields and calls check on any fields that
// have a validate tag. If the field is an anonymous struct recursively run
// validation on it.
func Run(object interface{}, fieldsSlice ...string) error {
	pass := true // We'll override this if checking returns false
	err := ValidationError{}

	// If we have been passed a slice of fields to valiate - to check only a
	// subset of fields - change the slice into a map for O(1) lookups instead
	// of O(n).
	fields := map[string]struct{}{}
	for _, field := range fieldsSlice {
		fields[field] = struct{}{}
	}

	// If we're passed a pointer to a struct we need to dereference the pointer before
	// inspecting its tags
	value := reflect.ValueOf(object)
	if value.Kind() == reflect.Ptr {
		value = value.Elem()
	}

	// Iterate through each field of the struct and validate
	typ := value.Type() // A Type's Field method returns StructFields
	for i := 0; i < value.NumField(); i++ {
		var validateTag string
		var validateError error

		// Is this an anonymous struct? If so, we also need to validate on this.
		if typ.Field(i).Anonymous == true {
			if anonErr := Run(value.Field(i).Interface(), fieldsSlice...); anonErr != nil {
				// The validation failed: set pass to false and merge the anonymous struct's
				// validation errors with our current validation error above to give a complete
				// error message.
				pass = false

				// A non validation error occurred: return this immediately
				if _, ok := anonErr.(ValidationError); !ok {
					return anonErr
				}

				err.Merge(anonErr.(ValidationError))
			}
		}

		if len(fields) > 0 {
			// We're only checking for a subset of fields; if this field isn't
			// included in the subset of fields to validate we can skip.
			if _, ok := fields[typ.Field(i).Name]; !ok {
				continue
			}
		}

		if validateTag = typ.Field(i).Tag.Get("validate"); validateTag == "" {
			continue
		}

		// Validate this particular field against the options in our tag
		if validateError = validateField(value.Field(i).Interface(), typ.Field(i).Name, validateTag); validateError == nil {
			continue
		}

		// If there was no validation rule defined for the given tag return
		// that error immediately.
		if _, ok := validateError.(rules.ErrNoValidationMethod); ok {
			return validateError
		}

		pass = false
		err.addFailure(typ.Field(i).Name, validateError.Error())
	}

	if pass {
		return nil
	}

	return err
}

var rxRegexp = regexp.MustCompile(`Regexp:\/.+/`)

// Takes a field's value and the validation tag and applies each check
// until either a test fails or all tests pass.
func validateField(data interface{}, fieldName, tag string) (err error) {
	// A tag can specify multiple validation rules which are delimited via ','.
	// However, because we allow regular expressions we can't split the tag field
	// via all commas to find our validation rules: we need to extract the regular expression
	// first (in case it specifies a comma), and *then* run through our validation rules.
	if match := rxRegexp.FindString(tag); match != "" {
		// If we fail validating the regexp we can break here
		if err := validateRule(data, fieldName, match); err != nil {
			return err
		}
		// Now we need to replace our regular expression from the tag list to continue
		// normally.
		tag = rxRegexp.ReplaceAllString(tag, "")
	}

	for tag != "" {
		var next string

		i := strings.Index(tag, ",")
		if i >= 0 {
			tag, next = tag[:i], tag[i+1:]
		}

		if err := validateRule(data, fieldName, tag); err != nil {
			return err
		}

		// Continue with the next tag
		tag = next
	}

	return nil
}

// Given a validation rule from a tag, run the associated validation methods and return
// the result.
func validateRule(data interface{}, fieldName, rule string) error {
	var args []string

	// Remove any preceeding spaces from comma separated tags
	rule = strings.TrimLeft(rule, " ")

	// If the rule is empty we don't need to process anything. This only happens
	// if we have a regex followed by another rule:
	//   `validate:"Regexp:/.+/, NotEmpty"`
	// Becomes:
	//   `validate:", NotEmpty"`
	// After processing in validateField()
	if rule == "" {
		return nil
	}

	// rule is the method we want to call. If it has a colon we need to further
	// process the rule to extract arguments to our validation method.
	i := strings.Index(rule, ":")
	if i > 0 {
		var arg string
		rule, arg = rule[:i], rule[i+1:]
		args = append(args, arg)
	}

	// Attempt to validate the data using methods registered with the rules
	// sub package
	if method, err := rules.Get(rule); err != nil {
		return err
	} else {
		var data = rules.ValidationData{
			Field: fieldName,
			Value: data,
			Args:  args,
		}
		return method(data)
	}
}
<|endoftext|>