<commit_before>package config

import (
	"github.com/coreos/kube-aws/coreos/userdatavalidation"
	"github.com/coreos/kube-aws/filereader/jsontemplate"
	"fmt"
	"net/url"
)

type StackConfig struct {
	*ComputedConfig
	UserDataWorker string
	StackTemplateOptions
}

type CompressedStackConfig struct {
	*StackConfig
}

func (c *StackConfig) UserDataWorkerS3Path() (string, error) {
	s3uri, err := url.Parse(c.S3URI)
	if err != nil {
		return "", fmt.Errorf("Error in UserDataWorkerS3Path : %v", err)
	}
	return fmt.Sprintf("%s%s/%s/userdata-worker", s3uri.Host, s3uri.Path, c.StackName()), nil
}

func (c *StackConfig) ValidateUserData() error {
	err := userdatavalidation.Execute([]userdatavalidation.Entry{
		{Name: "UserDataWorker", Content: c.UserDataWorker},
	})

	return err
}

func (c *StackConfig) Compress() (*CompressedStackConfig, error) {
	//var err error
	//var compressedWorkerUserData string
	//
	//if compressedWorkerUserData, err = gzipcompressor.CompressString(c.UserDataWorker); err != nil {
	//	return nil, err
	//}

	var stackConfig CompressedStackConfig
	stackConfig.StackConfig = &(*c)
	//stackConfig.UserDataWorker = compressedWorkerUserData
	stackConfig.UserDataWorker = c.UserDataWorker

	return &stackConfig, nil
}

func (c *CompressedStackConfig) RenderStackTemplateAsBytes() ([]byte, error) {
	bytes, err := jsontemplate.GetBytes(c.StackTemplateTmplFile, *c, c.PrettyPrint)
	if err != nil {
		return []byte{}, fmt.Errorf("failed to render : %v", err)
	}

	return bytes, nil
}

func (c *CompressedStackConfig) RenderStackTemplateAsString() (string, error) {
	bytes, err := c.RenderStackTemplateAsBytes()
	if err != nil {
		return "", fmt.Errorf("failed to render to str : %v", err)
	}
	return string(bytes), nil
}
<commit_msg>format code<commit_after>package config

import (
	"fmt"
	"github.com/coreos/kube-aws/coreos/userdatavalidation"
	"github.com/coreos/kube-aws/filereader/jsontemplate"
	"net/url"
)

type StackConfig struct {
	*ComputedConfig
	UserDataWorker string
	StackTemplateOptions
}

type CompressedStackConfig struct {
	*StackConfig
}

func (c *StackConfig) UserDataWorkerS3Path() (string, error) {
	s3uri, err := url.Parse(c.S3URI)
	if err != nil {
		return "", fmt.Errorf("Error in UserDataWorkerS3Path : %v", err)
	}
	return fmt.Sprintf("%s%s/%s/userdata-worker", s3uri.Host, s3uri.Path, c.StackName()), nil
}

func (c *StackConfig) ValidateUserData() error {
	err := userdatavalidation.Execute([]userdatavalidation.Entry{
		{Name: "UserDataWorker", Content: c.UserDataWorker},
	})

	return err
}

func (c *StackConfig) Compress() (*CompressedStackConfig, error) {
	//var err error
	//var compressedWorkerUserData string
	//
	//if compressedWorkerUserData, err = gzipcompressor.CompressString(c.UserDataWorker); err != nil {
	//	return nil, err
	//}

	var stackConfig CompressedStackConfig
	stackConfig.StackConfig = &(*c)
	//stackConfig.UserDataWorker = compressedWorkerUserData
	stackConfig.UserDataWorker = c.UserDataWorker

	return &stackConfig, nil
}

func (c *CompressedStackConfig) RenderStackTemplateAsBytes() ([]byte, error) {
	bytes, err := jsontemplate.GetBytes(c.StackTemplateTmplFile, *c, c.PrettyPrint)
	if err != nil {
		return []byte{}, fmt.Errorf("failed to render : %v", err)
	}

	return bytes, nil
}

func (c *CompressedStackConfig) RenderStackTemplateAsString() (string, error) {
	bytes, err := c.RenderStackTemplateAsBytes()
	if err != nil {
		return "", fmt.Errorf("failed to render to str : %v", err)
	}
	return string(bytes), nil
}
<|endoftext|>


<commit_before>package guard

import (
	"gopkg.in/workanator/go-floc.v2"
	"gopkg.in/workanator/go-floc.v2/errors"
)

const locPanic = "Panic"

// PanicTrigger is triggered when the goroutine state is recovered after
// panic.
type PanicTrigger func(ctx floc.Context, ctrl floc.Control, v interface{})

// Panic protects the job from falling into panic. On panic the flow will
// be canceled with the ErrPanic result. Guarding the job from falling into
// panic is effective only if the job runs in the current goroutine.
func Panic(job floc.Job) floc.Job {
	return OnPanic(job, nil)
}

// IgnorePanic protects the job from falling into panic. On panic the panic
// will be ignored. Guarding the job from falling into
// panic is effective only if the job runs in the current goroutine.
func IgnorePanic(job floc.Job) floc.Job {
	return OnPanic(job, func(ctx floc.Context, ctrl floc.Control, v interface{}) {})
}

// OnPanic protects the job from falling into panic. In addition it
// takes PanicTrigger func which is called in case of panic. Guarding the job
// from falling into panic is effective only if the job runs in the current
// goroutine.
func OnPanic(job floc.Job, panicTrigger PanicTrigger) floc.Job {
	return func(ctx floc.Context, ctrl floc.Control) error {
		defer func() {
			if r := recover(); r != nil {
				if panicTrigger != nil {
					panicTrigger(ctx, ctrl, r)
				} else {
					ctrl.Cancel(errors.NewErrPanic(r))
				}
			}
		}()

		// Do the job
		return job(ctx, ctrl)
	}
}
<commit_msg>Fail flow on panic<commit_after>package guard

import (
	"gopkg.in/workanator/go-floc.v2"
	"gopkg.in/workanator/go-floc.v2/errors"
)

const locPanic = "Panic"

// PanicTrigger is triggered when the goroutine state is recovered after
// panic.
type PanicTrigger func(ctx floc.Context, ctrl floc.Control, v interface{})

// Panic protects the job from falling into panic. On panic the flow will
// be canceled with the ErrPanic result. Guarding the job from falling into
// panic is effective only if the job runs in the current goroutine.
func Panic(job floc.Job) floc.Job {
	return OnPanic(job, nil)
}

// IgnorePanic protects the job from falling into panic. On panic the panic
// will be ignored. Guarding the job from falling into
// panic is effective only if the job runs in the current goroutine.
func IgnorePanic(job floc.Job) floc.Job {
	return OnPanic(job, func(ctx floc.Context, ctrl floc.Control, v interface{}) {})
}

// OnPanic protects the job from falling into panic. In addition it
// takes PanicTrigger func which is called in case of panic. Guarding the job
// from falling into panic is effective only if the job runs in the current
// goroutine.
func OnPanic(job floc.Job, panicTrigger PanicTrigger) floc.Job {
	return func(ctx floc.Context, ctrl floc.Control) error {
		defer func() {
			if r := recover(); r != nil {
				if panicTrigger != nil {
					panicTrigger(ctx, ctrl, r)
				} else {
					ctrl.Fail(r, errors.NewErrPanic(r))
				}
			}
		}()

		// Do the job
		return job(ctx, ctrl)
	}
}
<|endoftext|>


<commit_before>/*
Package hamt64 is the package that implements two Hamt structures for both
functional and transient implementations. The first structure is HamtFunctional,
and the second is HamtTransient. Each of these datastructures implemnents the
hamt64.Hamt interface.
*/
package hamt64

import (
	"unsafe"
)

// HashSize is the size of HashVal in bits.
const HashSize uint = uint(unsafe.Sizeof(HashVal(0))) * 8

// IndexBits is the fundemental setting along with HashSize for the Key
// constants. 2..HashSize/2 step 1
const IndexBits uint = 6

// DepthLimit is the maximum number of levels of the Hamt. It is calculated as
// DepthLimit = floor(HashSize / IndexBits) or a strict integer division.
const DepthLimit = HashSize / IndexBits
const remainder = HashSize - (DepthLimit * IndexBits)

// IndexLimit is the maximum number of entries in the Hamt interior nodes.
// IndexLimit = 1 << IndexBits
const IndexLimit = 1 << IndexBits

// MaxDepth is the maximum value of a depth variable. MaxDepth = DepthLimit - 1
const MaxDepth = DepthLimit - 1

// MaxIndex is the maximum value of a index variable. MaxIndex = IndexLimie - 1
const MaxIndex = IndexLimit - 1

// DowngradeThreshold is the constant that sets the threshold for the size of a
// table, that when a table decreases to the threshold size, the table is
// converted from a FixedTable to a SparseTable.
//
// This conversion only happens if the Hamt structure has be constructed with
// the HybridTables option.
const DowngradeThreshold uint = IndexLimit / 3 // 21

// UpgradeThreshold is the constant that sets the threshold for the size of a
// table, that when a table increases to the threshold size, the table is
// converted from a SparseTable to a FixedTable.
//
// This conversion only happens if the Hamt structure has be constructed with
// the HybridTables option.
const UpgradeThreshold uint = IndexLimit * 2 / 3 // 42

// Configuration contants to be passed to `hamt64.New(int) *Hamt`.
const (
	// FixedTableOnly indicates the structure should use fixedTables ONLY.
	// This was intended to be for speed, as compressed tables use a software
	// bitCount function to access individual cells.
	FixedTablesOnly = iota
	// SparseTablesOnly indicates the structure should use sparseTables ONLY.
	// This was intended just save space, but also seems to be faster; CPU cache
	// locality maybe?
	SparseTablesOnly
	// HybridTables indicates the structure should use sparseTable
	// initially, then upgrade to fixedTable when appropriate.
	HybridTables
)

// TableOptionName is a lookup table to map the integer value of FixedTablesOnly,
// SparseTablesOnly, and HybridTables to a string representing that option.
//     var option = hamt64.FixedTablesOnly
//     hamt64.TableOptionName[option] == "FixedTablesOnly"
var TableOptionName [3]string

// Could have used...
//var TableOptionName = [3]string{
//	"FixedTablesOnly",
//	"SparseTablesOnly",
//	"HybridTables",
//}

func init() {
	TableOptionName[FixedTablesOnly] = "FixedTablesOnly"
	TableOptionName[SparseTablesOnly] = "SparseTablesOnly"
	TableOptionName[HybridTables] = "HybridTables"
}

// Hamt defines the interface that both the HamtFunctional and HamtTransient
// datastructures must (and do) implement.
type Hamt interface {
	IsEmpty() bool
	Nentries() uint
	ToFunctional() Hamt
	ToTransient() Hamt
	DeepCopy() Hamt
	Get(Key) (interface{}, bool)
	Put(Key, interface{}) (Hamt, bool)
	Del(Key) (Hamt, interface{}, bool)
	String() string
	LongString(string) string
}

// New() constructs a datastucture that implements the Hamt interface. When the
// functional argument is true it implements a HamtFunctional datastructure.
// When the functional argument is false it implements a HamtTransient
// datastructure. In either case the opt argument is handed to the to the
// contructore for either NewFunctional(opt) or NewTransient(opt).
func New(functional bool, opt int) Hamt {
	if functional {
		return NewFunctional(opt)
	}
	return NewTransient(opt)
}
<commit_msg>added most pertanent package doc; that HashVal is a uint64<commit_after>/*
Package hamt64 is the package that implements two Hamt structures for both
functional and transient implementations. The first structure is HamtFunctional,
and the second is HamtTransient. Each of these datastructures implemnents the
hamt64.Hamt interface.

Fundementally HashVal is set to a uint64.
*/
package hamt64

import (
	"unsafe"
)

// HashSize is the size of HashVal in bits.
const HashSize uint = uint(unsafe.Sizeof(HashVal(0))) * 8

// IndexBits is the fundemental setting along with HashSize for the Key
// constants. 2..HashSize/2 step 1
const IndexBits uint = 6

// DepthLimit is the maximum number of levels of the Hamt. It is calculated as
// DepthLimit = floor(HashSize / IndexBits) or a strict integer division.
const DepthLimit = HashSize / IndexBits
const remainder = HashSize - (DepthLimit * IndexBits)

// IndexLimit is the maximum number of entries in the Hamt interior nodes.
// IndexLimit = 1 << IndexBits
const IndexLimit = 1 << IndexBits

// MaxDepth is the maximum value of a depth variable. MaxDepth = DepthLimit - 1
const MaxDepth = DepthLimit - 1

// MaxIndex is the maximum value of a index variable. MaxIndex = IndexLimie - 1
const MaxIndex = IndexLimit - 1

// DowngradeThreshold is the constant that sets the threshold for the size of a
// table, that when a table decreases to the threshold size, the table is
// converted from a FixedTable to a SparseTable.
//
// This conversion only happens if the Hamt structure has be constructed with
// the HybridTables option.
const DowngradeThreshold uint = IndexLimit / 3 // 21

// UpgradeThreshold is the constant that sets the threshold for the size of a
// table, that when a table increases to the threshold size, the table is
// converted from a SparseTable to a FixedTable.
//
// This conversion only happens if the Hamt structure has be constructed with
// the HybridTables option.
const UpgradeThreshold uint = IndexLimit * 2 / 3 // 42

// Configuration contants to be passed to `hamt64.New(int) *Hamt`.
const (
	// FixedTableOnly indicates the structure should use fixedTables ONLY.
	// This was intended to be for speed, as compressed tables use a software
	// bitCount function to access individual cells.
	FixedTablesOnly = iota
	// SparseTablesOnly indicates the structure should use sparseTables ONLY.
	// This was intended just save space, but also seems to be faster; CPU cache
	// locality maybe?
	SparseTablesOnly
	// HybridTables indicates the structure should use sparseTable
	// initially, then upgrade to fixedTable when appropriate.
	HybridTables
)

// TableOptionName is a lookup table to map the integer value of FixedTablesOnly,
// SparseTablesOnly, and HybridTables to a string representing that option.
//     var option = hamt64.FixedTablesOnly
//     hamt64.TableOptionName[option] == "FixedTablesOnly"
var TableOptionName [3]string

// Could have used...
//var TableOptionName = [3]string{
//	"FixedTablesOnly",
//	"SparseTablesOnly",
//	"HybridTables",
//}

func init() {
	TableOptionName[FixedTablesOnly] = "FixedTablesOnly"
	TableOptionName[SparseTablesOnly] = "SparseTablesOnly"
	TableOptionName[HybridTables] = "HybridTables"
}

// Hamt defines the interface that both the HamtFunctional and HamtTransient
// datastructures must (and do) implement.
type Hamt interface {
	IsEmpty() bool
	Nentries() uint
	ToFunctional() Hamt
	ToTransient() Hamt
	DeepCopy() Hamt
	Get(Key) (interface{}, bool)
	Put(Key, interface{}) (Hamt, bool)
	Del(Key) (Hamt, interface{}, bool)
	String() string
	LongString(string) string
}

// New() constructs a datastucture that implements the Hamt interface. When the
// functional argument is true it implements a HamtFunctional datastructure.
// When the functional argument is false it implements a HamtTransient
// datastructure. In either case the opt argument is handed to the to the
// contructore for either NewFunctional(opt) or NewTransient(opt).
func New(functional bool, opt int) Hamt {
	if functional {
		return NewFunctional(opt)
	}
	return NewTransient(opt)
}
<|endoftext|>


<commit_before>package hdc

import (
	"io"
	"time"
)

// Bitbang handles communication with HD44780 using some controller that
// implements clocked output of received data onto the D4-D7, E, RS pins (eg.
// FTDI USB chips in bitbang mode).
//
// It writes 3 bytes for one nibble to the provided io.Writer:
// - first:  with E bit unset, need >= 40 ns
// - second: with E bit set,   need >= 230 ns
// - thrid:  with E bit unset, need >= 10 ns
// Full E cycle need >= 500 ns.
//
// When you call its Write method with buffer that contains multiple commands,
// Bitbang can work in two modes:
//
// 1. Conservative mode
//
// Writes only 6 bytes (two nibbles = one command) at a time. Waits desired time
// betwen subsequent commands.
//
// 2. Fast mode (default)
//
// Tries to write up to 80 commands at a time. Every 6-byte command sequence is
// extended using waitTicks zero bytes. This is true for any sequence of "fast
// commands". Any "slow command" (ClearScreen or ReturnHome) breaks this
// fast path and Bitbang need to wait before write next commands.
//
// Additionaly Bitbang provides methods to controll AUX and R/W bits (if you
// want to use R/W bit as second AUX, the real R/W pin should be connected to
// the VSS).
type Bitbang struct {
	w          io.Writer
	e, rw, aux byte
	a          byte

	bpc int
	buf []byte

	fastMode bool

	t time.Time
}

// NewBitbang re
func NewBitbang(w io.Writer, waitTicks int) *Bitbang {
	if waitTicks < 0 {
		panic("waitTicks < 0")
	}
	bpc := 6 + waitTicks
	return &Bitbang{
		w:        w,
		e:        1 << 4,
		rw:       1 << 5,
		aux:      1 << 7,
		bpc:      bpc,
		buf:      make([]byte, 80*bpc),
		fastMode: true,
	}
}

func (o *Bitbang) FastMode(b bool) {
	o.fastMode = b
}

func (o *Bitbang) SetWriter(w io.Writer) {
	o.w = w
}

func (o *Bitbang) SetMapping(e, rw, aux byte) {
	o.e = e
	o.rw = rw
	o.aux = aux
}

func (o *Bitbang) SetRW(b bool) error {
	if b {
		o.a |= o.rw
	} else {
		o.a &^= o.rw
	}
	o.buf[0] = o.a
	_, err := o.w.Write(o.buf[:1])
	return err
}

func (o *Bitbang) SetAUX(b bool) error {
	if b {
		o.a |= o.aux
	} else {
		o.a &^= o.aux
	}
	o.buf[0] = o.a
	_, err := o.w.Write(o.buf[:1])
	return err
}

func (o *Bitbang) wait() {
	if !o.t.IsZero() {
		d := o.t.Sub(time.Now())
		if d > 0 {
			time.Sleep(d)
		}
		o.t = time.Time{}
	}
}

func (o *Bitbang) setWait(d time.Duration) {
	o.t = time.Now().Add(d)
}

func (o *Bitbang) Write(data []byte) (int, error) {
	if len(data) == 0 {
		return 0, nil
	}

	if len(data) == 1 {
		// One nibble: initialisation command
		o.wait()
		b := data[0] | o.a
		o.buf[0] = b
		o.buf[1] = b | o.e
		o.buf[2] = b
		_, err := o.w.Write(o.buf[:3])
		if err != nil {
			return 0, err
		}
		o.setWait(5 * time.Millisecond)
		return 1, nil
	}

	if len(data)%2 != 0 {
		panic("data length must be 1 or an even number")
	}

	if o.fastMode {
		return o.writeFast(data)
	}
	return o.write(data)
}

func (o *Bitbang) write(data []byte) (int, error) {
	buf := o.buf[:6]
	for n := 0; n < len(data); n += 2 {
		// Multiple nibbles: regural commands
		o.wait()
		b0 := data[n]
		b1 := data[n+1]
		b := b0<<4 | b1&0x0f
		b0 |= o.a
		b1 |= o.a
		o.buf[0] = b0
		o.buf[1] = b0 | o.e
		o.buf[2] = b0
		o.buf[3] = b1
		o.buf[4] = b1 | o.e
		o.buf[5] = b1
		_, err := o.w.Write(buf)
		if err != nil {
			return n, err
		}
		if b < 4 {
			// "Clear display" or "Return home"
			o.setWait(16 * time.Millisecond)
		} else {
			// Other command
			o.setWait(40 * time.Microsecond)
		}
	}
	return len(data), nil
}

func (o *Bitbang) writeFast(data []byte) (int, error) {
	dlen := len(data)
	for len(data) > 0 {
		n := len(data) / 2 * o.bpc
		if n > len(o.buf) {
			n = len(o.buf)
		}
		k := 0
		i := 0
		wait := false
		for k < n {
			b0 := data[i]
			b1 := data[i+1]
			b := b0<<4 | b1&0x0f
			b0 |= o.a
			b1 |= o.a
			o.buf[k] = b0
			o.buf[k+1] = b0 | o.e
			o.buf[k+2] = b0
			o.buf[k+3] = b1
			o.buf[k+4] = b1 | o.e
			o.buf[k+5] = b1
			// Next bytes (up to bpc) are always zero
			i += 2
			k += o.bpc
			if b < 4 {
				// Slow command (ClearDisplay or ReturnHome)
				wait = true
				break
			}
		}
		o.wait()
		k, err := o.w.Write(o.buf[:k])
		if err != nil {
			return dlen - len(data) + k*2/o.bpc, err
		}
		if wait {
			o.setWait(16 * time.Millisecond)
		}
		data = data[i:]
	}
	return dlen, nil
}
<commit_msg>hdc: Remove some comments<commit_after>package hdc

import (
	"io"
	"time"
)

// Bitbang handles communication with HD44780 using some controller that
// implements clocked output of received data onto the D4-D7, E, RS pins (eg.
// FTDI USB chips in bitbang mode).
//
// It writes 3 bytes for one nibble to the provided io.Writer:
// - first:  with E bit unset, need >= 40 ns
// - second: with E bit set,   need >= 230 ns
// - thrid:  with E bit unset, need >= 10 ns
// Full E cycle need >= 500 ns.
//
// Bitbang doesn't control proper nible timing - provided io.Writer is
// responsible for this.
//
// When you call its Write method with buffer that contains multiple commands,
// Bitbang can work in two modes:
//
// 1. Conservative mode
//
// Writes only 6 bytes (two nibbles = one command) at a time. Waits desired time
// betwen subsequent commands.
//
// 2. Fast mode (default)
//
// Tries to write up to 80 commands at a time. Every 6-byte command sequence is
// extended using waitTicks zero bytes. This is true for any sequence of "fast
// commands". Any "slow command" (ClearScreen or ReturnHome) breaks this
// fast path and Bitbang need to wait before write next commands.
//
// Additionaly Bitbang provides methods to controll AUX and R/W bits (if you
// want to use R/W bit as second AUX, the real R/W pin should be connected to
// the VSS).
type Bitbang struct {
	w          io.Writer
	e, rw, aux byte
	a          byte

	bpc int
	buf []byte

	fastMode bool

	t time.Time
}

// NewBitbang re
func NewBitbang(w io.Writer, waitTicks int) *Bitbang {
	if waitTicks < 0 {
		panic("waitTicks < 0")
	}
	bpc := 6 + waitTicks
	return &Bitbang{
		w:        w,
		e:        1 << 4,
		rw:       1 << 5,
		aux:      1 << 7,
		bpc:      bpc,
		buf:      make([]byte, 80*bpc),
		fastMode: true,
	}
}

func (o *Bitbang) FastMode(b bool) {
	o.fastMode = b
}

func (o *Bitbang) SetWriter(w io.Writer) {
	o.w = w
}

func (o *Bitbang) SetMapping(e, rw, aux byte) {
	o.e = e
	o.rw = rw
	o.aux = aux
}

func (o *Bitbang) SetRW(b bool) error {
	if b {
		o.a |= o.rw
	} else {
		o.a &^= o.rw
	}
	o.buf[0] = o.a
	_, err := o.w.Write(o.buf[:1])
	return err
}

func (o *Bitbang) SetAUX(b bool) error {
	if b {
		o.a |= o.aux
	} else {
		o.a &^= o.aux
	}
	o.buf[0] = o.a
	_, err := o.w.Write(o.buf[:1])
	return err
}

func (o *Bitbang) wait() {
	if !o.t.IsZero() {
		d := o.t.Sub(time.Now())
		if d > 0 {
			time.Sleep(d)
		}
		o.t = time.Time{}
	}
}

func (o *Bitbang) setWait(d time.Duration) {
	o.t = time.Now().Add(d)
}

func (o *Bitbang) Write(data []byte) (int, error) {
	if len(data) == 0 {
		return 0, nil
	}

	if len(data) == 1 {
		// One nibble: initialisation command
		o.wait()
		b := data[0] | o.a
		o.buf[0] = b
		o.buf[1] = b | o.e
		o.buf[2] = b
		_, err := o.w.Write(o.buf[:3])
		if err != nil {
			return 0, err
		}
		o.setWait(5 * time.Millisecond)
		return 1, nil
	}

	if len(data)%2 != 0 {
		panic("data length must be 1 or an even number")
	}

	if o.fastMode {
		return o.writeFast(data)
	}
	return o.write(data)
}

func (o *Bitbang) write(data []byte) (int, error) {
	buf := o.buf[:6]
	for n := 0; n < len(data); n += 2 {
		o.wait()
		b0 := data[n]
		b1 := data[n+1]
		b := b0<<4 | b1&0x0f
		b0 |= o.a
		b1 |= o.a
		o.buf[0] = b0
		o.buf[1] = b0 | o.e
		o.buf[2] = b0
		o.buf[3] = b1
		o.buf[4] = b1 | o.e
		o.buf[5] = b1
		_, err := o.w.Write(buf)
		if err != nil {
			return n, err
		}
		if b < 4 {
			// "Clear display" or "Return home".
			o.setWait(16 * time.Millisecond)
		} else {
			// Other command.
			o.setWait(40 * time.Microsecond)
		}
	}
	return len(data), nil
}

func (o *Bitbang) writeFast(data []byte) (int, error) {
	dlen := len(data)
	for len(data) > 0 {
		n := len(data) / 2 * o.bpc
		if n > len(o.buf) {
			n = len(o.buf)
		}
		k := 0
		i := 0
		wait := false
		for k < n {
			b0 := data[i]
			b1 := data[i+1]
			b := b0<<4 | b1&0x0f
			b0 |= o.a
			b1 |= o.a
			o.buf[k] = b0
			o.buf[k+1] = b0 | o.e
			o.buf[k+2] = b0
			o.buf[k+3] = b1
			o.buf[k+4] = b1 | o.e
			o.buf[k+5] = b1
			// Next bytes (up to bpc) are always zero
			i += 2
			k += o.bpc
			if b < 4 {
				// Slow command (ClearDisplay or ReturnHome)
				wait = true
				break
			}
		}
		o.wait()
		k, err := o.w.Write(o.buf[:k])
		if err != nil {
			return dlen - len(data) + k*2/o.bpc, err
		}
		if wait {
			o.setWait(16 * time.Millisecond)
		}
		data = data[i:]
	}
	return dlen, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strconv"

	chaincfg "github.com/btcsuite/btcd/chaincfg"
	btcutil "github.com/btcsuite/btcutil"
	keychain "github.com/btcsuite/btcutil/hdkeychain"
	ethcrypto "github.com/ethereum/go-ethereum/crypto"
	cli "github.com/urfave/cli"
)

func main() {
	app := cli.NewApp()
	app.Usage = "A command line utility for manipulating HD wallet keys"
	app.Version = "0.0.1"
	app.Commands = []cli.Command{
		privKeyCmd,
		pubKeyCmd,
		genKeyCmd,
	}
	if err := app.Run(os.Args); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

var genKeyCmd = cli.Command{
	Name:            "gen",
	Usage:           "generate an HD wallet key",
	SkipFlagParsing: true,
	Action: func(c *cli.Context) error {
		f, err := exec.LookPath("hdkeygen")
		if err != nil {
			return fmt.Errorf("could not find 'hdkeygen' binary: %s", err)
		}

		cmd := exec.Command(f, c.Args()...)
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		cmd.Stdin = os.Stdin

		return cmd.Run()
	},
}

var privKeyCmd = cli.Command{
	Name:  "priv",
	Usage: "utilities for working with hd private keys",
	Subcommands: []cli.Command{
		getMasterPubCmd,
		getChildPrivKeyCmd,
	},
}

var getMasterPubCmd = cli.Command{
	Name:  "getmasterpub",
	Usage: "derive the master public key from the given master private key",
	Action: func(c *cli.Context) error {
		if !c.Args().Present() {
			return fmt.Errorf("must pass in private key file")
		}

		data, err := ioutil.ReadFile(c.Args().First())
		if err != nil {
			return err
		}

		key, err := keychain.NewKeyFromString(string(data))
		if err != nil {
			return err
		}

		if !key.IsPrivate() {
			return fmt.Errorf("given key was not a private key")
		}

		pubk, err := key.Neuter()
		if err != nil {
			return err
		}

		fmt.Print(pubk.String())
		return nil
	},
}

var getChildPrivKeyCmd = cli.Command{
	Name:  "child",
	Usage: "derive a child private key",
	Description: `Derive a child private key from the given heirarchically deterministic
   private key and print it out.

   By default, it outputs in wallet import format for use by bitcoin and zcash.
   Optionally, you may pass the --format flag with a parameter of 'eth' to
   signal that it should output a raw ecdsa key for use by ethereum.`,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "format",
			Usage: "output format for key (wif or eth)",
			Value: "wif",
		},
	},
	Action: func(c *cli.Context) error {
		format := c.String("format")

		if len(c.Args()) != 2 {
			return fmt.Errorf("must pass in private key and index")
		}

		i, err := strconv.Atoi(c.Args()[1])
		if err != nil {
			return err
		}

		data, err := ioutil.ReadFile(c.Args().First())
		if err != nil {
			return err
		}

		key, err := keychain.NewKeyFromString(string(data))
		if err != nil {
			return err
		}

		if !key.IsPrivate() {
			return fmt.Errorf("given key was not a private key")
		}

		childpriv, err := key.Child(uint32(i))
		if err != nil {
			return err
		}

		privk, err := childpriv.ECPrivKey()
		if err != nil {
			return err
		}

		switch format {
		case "wif":
			wif, err := btcutil.NewWIF(privk, &chaincfg.MainNetParams, false)
			if err != nil {
				return err
			}

			fmt.Println(wif.String())
		case "eth":
			fmt.Printf("%x\n", privk.Serialize())
		}
		return nil
	},
}

var pubKeyCmd = cli.Command{
	Name:  "pub",
	Usage: "tools for working with hd public keys",
	Subcommands: []cli.Command{
		getChildPubKeyCmd,
	},
}

var getChildPubKeyCmd = cli.Command{
	Name:  "child",
	Usage: "derive a child public key",
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "format",
			Usage: "output format for key (btc, zec, or eth)",
			Value: "btc",
		},
	},
	Action: func(c *cli.Context) error {
		enc := c.String("format")

		if len(c.Args()) != 2 {
			return fmt.Errorf("must pass in public key and index")
		}

		i, err := strconv.Atoi(c.Args()[1])
		if err != nil {
			return err
		}

		data, err := ioutil.ReadFile(c.Args().First())
		if err != nil {
			return err
		}

		key, err := keychain.NewKeyFromString(string(data))
		if err != nil {
			return err
		}

		if key.IsPrivate() {
			return fmt.Errorf("given key was a private key, not public")
		}

		childpub, err := key.Child(uint32(i))
		if err != nil {
			return err
		}

		addr, err := childpub.Address(&chaincfg.MainNetParams)
		if err != nil {
			return err
		}

		switch enc {
		case "btc":
			fmt.Println(addr.EncodeAddress())
		case "zec":
			fmt.Println("t" + addr.EncodeAddress())
		case "eth":
			ecpubkey, err := childpub.ECPubKey()
			if err != nil {
				return err
			}

			addr := ethcrypto.PubkeyToAddress(*ecpubkey.ToECDSA())
			fmt.Println(addr.Hex())
		default:
			return fmt.Errorf("unrecognized output format: %s", enc)
		}
		return nil
	},
}
<commit_msg>more correct address generation for zcash<commit_after>package main

import (
	"crypto/sha256"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strconv"

	chaincfg "github.com/btcsuite/btcd/chaincfg"
	btcutil "github.com/btcsuite/btcutil"
	b58 "github.com/btcsuite/btcutil/base58"
	keychain "github.com/btcsuite/btcutil/hdkeychain"
	ethcrypto "github.com/ethereum/go-ethereum/crypto"
	cli "github.com/urfave/cli"
	"golang.org/x/crypto/ripemd160"
)

func main() {
	app := cli.NewApp()
	app.Usage = "A command line utility for manipulating HD wallet keys"
	app.Version = "0.0.1"
	app.Commands = []cli.Command{
		privKeyCmd,
		pubKeyCmd,
		genKeyCmd,
	}
	if err := app.Run(os.Args); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

var genKeyCmd = cli.Command{
	Name:            "gen",
	Usage:           "generate an HD wallet key",
	SkipFlagParsing: true,
	Action: func(c *cli.Context) error {
		f, err := exec.LookPath("hdkeygen")
		if err != nil {
			return fmt.Errorf("could not find 'hdkeygen' binary: %s", err)
		}

		cmd := exec.Command(f, c.Args()...)
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		cmd.Stdin = os.Stdin

		return cmd.Run()
	},
}

var privKeyCmd = cli.Command{
	Name:  "priv",
	Usage: "utilities for working with hd private keys",
	Subcommands: []cli.Command{
		getMasterPubCmd,
		getChildPrivKeyCmd,
	},
}

var getMasterPubCmd = cli.Command{
	Name:  "getmasterpub",
	Usage: "derive the master public key from the given master private key",
	Action: func(c *cli.Context) error {
		if !c.Args().Present() {
			return fmt.Errorf("must pass in private key file")
		}

		data, err := ioutil.ReadFile(c.Args().First())
		if err != nil {
			return err
		}

		key, err := keychain.NewKeyFromString(string(data))
		if err != nil {
			return err
		}

		if !key.IsPrivate() {
			return fmt.Errorf("given key was not a private key")
		}

		pubk, err := key.Neuter()
		if err != nil {
			return err
		}

		fmt.Print(pubk.String())
		return nil
	},
}

var getChildPrivKeyCmd = cli.Command{
	Name:  "child",
	Usage: "derive a child private key",
	Description: `Derive a child private key from the given heirarchically deterministic
   private key and print it out.

   By default, it outputs in wallet import format for use by bitcoin and zcash.
   Optionally, you may pass the --format flag with a parameter of 'eth' to
   signal that it should output a raw ecdsa key for use by ethereum.`,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "format",
			Usage: "output format for key (wif or eth)",
			Value: "wif",
		},
	},
	Action: func(c *cli.Context) error {
		format := c.String("format")

		if len(c.Args()) != 2 {
			return fmt.Errorf("must pass in private key and index")
		}

		i, err := strconv.Atoi(c.Args()[1])
		if err != nil {
			return err
		}

		data, err := ioutil.ReadFile(c.Args().First())
		if err != nil {
			return err
		}

		key, err := keychain.NewKeyFromString(string(data))
		if err != nil {
			return err
		}

		if !key.IsPrivate() {
			return fmt.Errorf("given key was not a private key")
		}

		childpriv, err := key.Child(uint32(i))
		if err != nil {
			return err
		}

		privk, err := childpriv.ECPrivKey()
		if err != nil {
			return err
		}

		switch format {
		case "wif":
			wif, err := btcutil.NewWIF(privk, &chaincfg.MainNetParams, false)
			if err != nil {
				return err
			}

			fmt.Println(wif.String())
		case "eth":
			fmt.Printf("%x\n", privk.Serialize())
		}
		return nil
	},
}

var pubKeyCmd = cli.Command{
	Name:  "pub",
	Usage: "tools for working with hd public keys",
	Subcommands: []cli.Command{
		getChildPubKeyCmd,
	},
}

var getChildPubKeyCmd = cli.Command{
	Name:  "child",
	Usage: "derive a child public key",
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "format",
			Usage: "output format for key (btc, zec, or eth)",
			Value: "btc",
		},
	},
	Action: func(c *cli.Context) error {
		enc := c.String("format")

		if len(c.Args()) != 2 {
			return fmt.Errorf("must pass in public key and index")
		}

		i, err := strconv.Atoi(c.Args()[1])
		if err != nil {
			return err
		}

		data, err := ioutil.ReadFile(c.Args().First())
		if err != nil {
			return err
		}

		key, err := keychain.NewKeyFromString(string(data))
		if err != nil {
			return err
		}

		if key.IsPrivate() {
			return fmt.Errorf("given key was a private key, not public")
		}

		childpub, err := key.Child(uint32(i))
		if err != nil {
			return err
		}

		addr, err := childpub.Address(&chaincfg.MainNetParams)
		if err != nil {
			return err
		}

		switch enc {
		case "btc":
			fmt.Println(addr.EncodeAddress())
		case "zec":
			ecpk, err := childpub.ECPubKey()
			if err != nil {
				return err
			}
			uncomp := ecpk.SerializeUncompressed()
			shad := sha256.Sum256(uncomp)
			h := ripemd160.New()
			h.Write(shad[:])
			ripemd := h.Sum(nil)
			d := append([]byte{0x1c, 0xb8}, ripemd...)
			fmt.Println(b58.CheckEncode(d, 0))
		case "eth":
			ecpubkey, err := childpub.ECPubKey()
			if err != nil {
				return err
			}

			addr := ethcrypto.PubkeyToAddress(*ecpubkey.ToECDSA())
			fmt.Println(addr.Hex())
		default:
			return fmt.Errorf("unrecognized output format: %s", enc)
		}
		return nil
	},
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strconv"

	chaincfg "github.com/btcsuite/btcd/chaincfg"
	btcutil "github.com/btcsuite/btcutil"
	keychain "github.com/btcsuite/btcutil/hdkeychain"
	cli "github.com/urfave/cli"
	addrs "github.com/whyrusleeping/hdkeyutils/addrs"
)

func main() {
	app := cli.NewApp()
	app.Usage = "A command line utility for manipulating HD wallet keys"
	app.Version = "0.0.1"
	app.Commands = []cli.Command{
		privKeyCmd,
		pubKeyCmd,
		genKeyCmd,
		msigCmd,
	}
	if err := app.Run(os.Args); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

var genKeyCmd = cli.Command{
	Name:            "gen",
	Usage:           "generate an HD wallet key",
	SkipFlagParsing: true,
	Action: func(c *cli.Context) error {
		f, err := exec.LookPath("hdkeygen")
		if err != nil {
			return fmt.Errorf("could not find 'hdkeygen' binary: %s", err)
		}

		cmd := exec.Command(f, c.Args()...)
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		cmd.Stdin = os.Stdin

		return cmd.Run()
	},
}

var privKeyCmd = cli.Command{
	Name:  "priv",
	Usage: "utilities for working with hd private keys",
	Subcommands: []cli.Command{
		getMasterPubCmd,
		getChildPrivKeyCmd,
	},
}

var getMasterPubCmd = cli.Command{
	Name:  "getmasterpub",
	Usage: "derive the master public key from the given master private key",
	Action: func(c *cli.Context) error {
		if !c.Args().Present() {
			return fmt.Errorf("must pass in private key file")
		}

		key, err := loadPrivKey(c.Args().First())
		if err != nil {
			return err
		}

		pubk, err := key.Neuter()
		if err != nil {
			return err
		}

		fmt.Print(pubk.String())
		return nil
	},
}

func loadPrivKey(fi string) (*keychain.ExtendedKey, error) {
	data, err := ioutil.ReadFile(fi)
	if err != nil {
		return nil, err
	}

	key, err := keychain.NewKeyFromString(string(data))
	if err != nil {
		return nil, err
	}

	if !key.IsPrivate() {
		return nil, fmt.Errorf("given key was not a private key")
	}

	return key, nil
}

var getChildPrivKeyCmd = cli.Command{
	Name:  "child",
	Usage: "derive a child private key",
	Description: `Derive a child private key from the given heirarchically deterministic
   private key and print it out.

   By default, it outputs in wallet import format for use by bitcoin and zcash.
   Optionally, you may pass the --format flag with a parameter of 'eth' to
   signal that it should output a raw ecdsa key for use by ethereum.`,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "format",
			Usage: "output format for key (wif or eth)",
			Value: "wif",
		},
		cli.BoolFlag{
			Name: "testnet",
		},
		cli.BoolFlag{
			Name: "harden",
		},
	},
	Action: func(c *cli.Context) error {
		format := c.String("format")

		if len(c.Args()) != 2 {
			return fmt.Errorf("must pass in private key and index")
		}

		i, err := strconv.Atoi(c.Args()[1])
		if err != nil {
			return err
		}

		if c.Bool("harden") {
			i += keychain.HardenedKeyStart
		}

		data, err := ioutil.ReadFile(c.Args().First())
		if err != nil {
			return err
		}

		key, err := keychain.NewKeyFromString(string(data))
		if err != nil {
			return err
		}

		if !key.IsPrivate() {
			return fmt.Errorf("given key was not a private key")
		}

		childpriv, err := key.Child(uint32(i))
		if err != nil {
			return err
		}

		privk, err := childpriv.ECPrivKey()
		if err != nil {
			return err
		}

		switch format {
		case "wif":
			params := &chaincfg.MainNetParams
			if c.Bool("testnet") {
				params = &chaincfg.TestNet3Params
			}
			wif, err := btcutil.NewWIF(privk, params, true)
			if err != nil {
				return err
			}

			fmt.Println(wif.String())
		case "eth":
			fmt.Printf("%x\n", privk.Serialize())
		}
		return nil
	},
}

var pubKeyCmd = cli.Command{
	Name:  "pub",
	Usage: "tools for working with hd public keys",
	Subcommands: []cli.Command{
		getChildPubKeyCmd,
	},
}

var getChildPubKeyCmd = cli.Command{
	Name:  "child",
	Usage: "derive a child public key",
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "format",
			Usage: "output format for key (btc, zec, or eth)",
			Value: "btc",
		},
		cli.BoolFlag{
			Name:  "testnet",
			Usage: "print testnet addrs",
		},
		cli.BoolFlag{
			Name: "harden",
		},
	},
	Action: func(c *cli.Context) error {
		enc := c.String("format")

		if len(c.Args()) != 2 {
			return fmt.Errorf("must pass in public key and index")
		}

		i, err := strconv.Atoi(c.Args()[1])
		if err != nil {
			return err
		}

		if c.Bool("harden") {
			i += keychain.HardenedKeyStart
		}

		data, err := ioutil.ReadFile(c.Args().First())
		if err != nil {
			return err
		}

		key, err := keychain.NewKeyFromString(string(data))
		if err != nil {
			return err
		}

		if key.IsPrivate() {
			return fmt.Errorf("given key was a private key, not public")
		}

		childpub, err := key.Child(uint32(i))
		if err != nil {
			return err
		}

		ecpub, err := childpub.ECPubKey()
		if err != nil {
			return err
		}

		if c.Bool("testnet") {
			addrs.BitcoinPrefix = addrs.BitcoinTestnetPrefix
			addrs.ZcashPrefix = addrs.ZcashTestnetPrefix
		}

		switch enc {
		case "btc":
			fmt.Println(addrs.EncodeBitcoinPubkey(ecpub))
		case "zec":
			fmt.Println(addrs.EncodeZcashPubkey(ecpub))
		case "eth":
			fmt.Println(addrs.EncodeEthereumPubkey(ecpub))
		default:
			return fmt.Errorf("unrecognized output format: %s", enc)
		}
		return nil
	},
}
<commit_msg>care less about trailing newlines<commit_after>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strconv"

	chaincfg "github.com/btcsuite/btcd/chaincfg"
	btcutil "github.com/btcsuite/btcutil"
	keychain "github.com/btcsuite/btcutil/hdkeychain"
	cli "github.com/urfave/cli"
	addrs "github.com/whyrusleeping/hdkeyutils/addrs"
)

func main() {
	app := cli.NewApp()
	app.Usage = "A command line utility for manipulating HD wallet keys"
	app.Version = "0.0.1"
	app.Commands = []cli.Command{
		privKeyCmd,
		pubKeyCmd,
		genKeyCmd,
		msigCmd,
	}
	if err := app.Run(os.Args); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

var genKeyCmd = cli.Command{
	Name:            "gen",
	Usage:           "generate an HD wallet key",
	SkipFlagParsing: true,
	Action: func(c *cli.Context) error {
		f, err := exec.LookPath("hdkeygen")
		if err != nil {
			return fmt.Errorf("could not find 'hdkeygen' binary: %s", err)
		}

		cmd := exec.Command(f, c.Args()...)
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		cmd.Stdin = os.Stdin

		return cmd.Run()
	},
}

var privKeyCmd = cli.Command{
	Name:  "priv",
	Usage: "utilities for working with hd private keys",
	Subcommands: []cli.Command{
		getMasterPubCmd,
		getChildPrivKeyCmd,
	},
}

var getMasterPubCmd = cli.Command{
	Name:  "getmasterpub",
	Usage: "derive the master public key from the given master private key",
	Action: func(c *cli.Context) error {
		if !c.Args().Present() {
			return fmt.Errorf("must pass in private key file")
		}

		key, err := loadPrivKey(c.Args().First())
		if err != nil {
			return err
		}

		pubk, err := key.Neuter()
		if err != nil {
			return err
		}

		fmt.Print(pubk.String())
		return nil
	},
}

func loadPrivKey(fi string) (*keychain.ExtendedKey, error) {
	data, err := ioutil.ReadFile(fi)
	if err != nil {
		return nil, err
	}

	data = bytes.TrimSpace(data)

	key, err := keychain.NewKeyFromString(string(data))
	if err != nil {
		return nil, err
	}

	if !key.IsPrivate() {
		return nil, fmt.Errorf("given key was not a private key")
	}

	return key, nil
}

var getChildPrivKeyCmd = cli.Command{
	Name:  "child",
	Usage: "derive a child private key",
	Description: `Derive a child private key from the given heirarchically deterministic
   private key and print it out.

   By default, it outputs in wallet import format for use by bitcoin and zcash.
   Optionally, you may pass the --format flag with a parameter of 'eth' to
   signal that it should output a raw ecdsa key for use by ethereum.`,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "format",
			Usage: "output format for key (wif or eth)",
			Value: "wif",
		},
		cli.BoolFlag{
			Name: "testnet",
		},
		cli.BoolFlag{
			Name: "harden",
		},
	},
	Action: func(c *cli.Context) error {
		format := c.String("format")

		if len(c.Args()) != 2 {
			return fmt.Errorf("must pass in private key and index")
		}

		i, err := strconv.Atoi(c.Args()[1])
		if err != nil {
			return err
		}

		if c.Bool("harden") {
			i += keychain.HardenedKeyStart
		}

		key, err := loadPrivKey(c.Args().First())
		if err != nil {
			return err
		}

		childpriv, err := key.Child(uint32(i))
		if err != nil {
			return err
		}

		if c.Bool("harden") {
			// just print out the hdkey
			fmt.Println(childpriv.String())
			return nil
		}

		privk, err := childpriv.ECPrivKey()
		if err != nil {
			return err
		}

		switch format {
		case "wif":
			params := &chaincfg.MainNetParams
			if c.Bool("testnet") {
				params = &chaincfg.TestNet3Params
			}
			wif, err := btcutil.NewWIF(privk, params, true)
			if err != nil {
				return err
			}

			fmt.Println(wif.String())
		case "eth":
			fmt.Printf("%x\n", privk.Serialize())
		}
		return nil
	},
}

var pubKeyCmd = cli.Command{
	Name:  "pub",
	Usage: "tools for working with hd public keys",
	Subcommands: []cli.Command{
		getChildPubKeyCmd,
	},
}

var getChildPubKeyCmd = cli.Command{
	Name:  "child",
	Usage: "derive a child public key",
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "format",
			Usage: "output format for key (btc, zec, or eth)",
			Value: "btc",
		},
		cli.BoolFlag{
			Name:  "testnet",
			Usage: "print testnet addrs",
		},
	},
	Action: func(c *cli.Context) error {
		enc := c.String("format")

		if len(c.Args()) != 2 {
			return fmt.Errorf("must pass in public key and index")
		}

		i, err := strconv.Atoi(c.Args()[1])
		if err != nil {
			return err
		}

		data, err := ioutil.ReadFile(c.Args().First())
		if err != nil {
			return err
		}

		key, err := keychain.NewKeyFromString(string(bytes.TrimSpace(data)))
		if err != nil {
			return err
		}

		if key.IsPrivate() {
			return fmt.Errorf("given key was a private key, not public")
		}

		childpub, err := key.Child(uint32(i))
		if err != nil {
			return err
		}

		ecpub, err := childpub.ECPubKey()
		if err != nil {
			return err
		}

		if c.Bool("testnet") {
			addrs.BitcoinPrefix = addrs.BitcoinTestnetPrefix
			addrs.ZcashPrefix = addrs.ZcashTestnetPrefix
		}

		switch enc {
		case "btc":
			fmt.Println(addrs.EncodeBitcoinPubkey(ecpub))
		case "zec":
			fmt.Println(addrs.EncodeZcashPubkey(ecpub))
		case "eth":
			fmt.Println(addrs.EncodeEthereumPubkey(ecpub))
		default:
			return fmt.Errorf("unrecognized output format: %s", enc)
		}
		return nil
	},
}
<|endoftext|>


<commit_before>// fsync benchmark
package main

import (
	"flag"
	"io/ioutil"
	"log"
	"os"
	"syscall"
	"time"
)

var pN = flag.Int("n", 100, "Iterations to test - default 100")

func main() {
	flag.Parse()
	out, err := ioutil.TempFile("", "fsyncbench")
	if err != nil {
		log.Fatal(err)
	}
	err = os.Remove(out.Name())
	if err != nil {
		log.Fatal(err)
	}
	fd := int(out.Fd())

	buf := []byte{'A'}
	N := *pN
	duration := time.Duration(0)
	for i := 0; i < N; i++ {
		_, err = out.Write(buf)
		if err != nil {
			log.Fatal(err)
		}
		start := time.Now()
		err = syscall.Fsync(fd)
		end := time.Now()
		duration += end.Sub(start)
		if err != nil {
			log.Fatal(err)
		}
	}

	log.Printf("That took %s for %d fsyncs", duration, N)
	log.Printf("That took %s per fsync", duration/time.Duration(N))

	err = out.Close()
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Make work on windows<commit_after>// fsync benchmark
package main

import (
	"flag"
	"io/ioutil"
	"log"
	"os"
	"time"
)

var pN = flag.Int("n", 100, "Iterations to test - default 100")

func main() {
	flag.Parse()
	out, err := ioutil.TempFile("", "fsyncbench")
	if err != nil {
		log.Fatal(err)
	}
	err = os.Remove(out.Name())
	if err != nil {
		log.Fatal(err)
	}

	buf := []byte{'A'}
	N := *pN
	duration := time.Duration(0)
	for i := 0; i < N; i++ {
		_, err = out.Write(buf)
		if err != nil {
			log.Fatal(err)
		}
		start := time.Now()
		err = out.Sync()
		end := time.Now()
		duration += end.Sub(start)
		if err != nil {
			log.Fatal(err)
		}
	}

	log.Printf("That took %s for %d fsyncs", duration, N)
	log.Printf("That took %s per fsync", duration/time.Duration(N))

	err = out.Close()
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2020, 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Binary wrap is a test helper program for //elisp:binary_test, which see.
package main

import (
	"encoding/json"
	"flag"
	"io/ioutil"
	"log"
	"os"
	"reflect"

	"github.com/google/go-cmp/cmp"
	"github.com/phst/runfiles"
)

func main() {
	log.Println("Args:", os.Args)
	log.Println("Environment:", os.Environ())
	var manifestFile string
	flag.StringVar(&manifestFile, "manifest", "", "")
	flag.Parse()
	if manifestFile == "" {
		log.Fatal("--manifest is empty")
	}
	runfilesLib, err := runfiles.Path("phst_rules_elisp/elisp/runfiles/runfiles.elc")
	if err != nil {
		log.Fatal(err)
	}
	// The load path setup depends on whether we use manifest-based or
	// directory-based runfiles.
	var loadPathArgs []string
	if dir, err := runfiles.Path("phst_rules_elisp"); err == nil {
		// Directory-based runfiles.
		loadPathArgs = []string{"--directory=" + dir}
	} else {
		// Manifest-based runfiles.
		loadPathArgs = []string{
			"--load=" + runfilesLib,
			"--funcall=elisp/runfiles/install-handler",
			"--directory=/bazel-runfile:phst_rules_elisp",
		}
	}
	inputFile, err := runfiles.Path("phst_rules_elisp/elisp/binary.cc")
	if err != nil {
		log.Fatal(err)
	}
	var outputFile string
	if os.PathSeparator == '/' {
		outputFile = "/tmp/output.dat"
	} else {
		outputFile = `C:\Temp\output.dat`
	}
	gotArgs := flag.Args()
	wantArgs := append(
		append([]string{"--quick", "--batch"}, loadPathArgs...),
		"--option",
		inputFile,
		" \t\n\r\f äα𝐴🐈'\\\"",
		"/:"+outputFile,
	)
	if diff := cmp.Diff(gotArgs, wantArgs); diff != "" {
		log.Fatalf("positional arguments: -got +want:\n%s", diff)
	}
	jsonData, err := ioutil.ReadFile(manifestFile)
	if err != nil {
		log.Fatalf("can’t read manifest: %s", err)
	}
	var gotManifest map[string]interface{}
	if err := json.Unmarshal(jsonData, &gotManifest); err != nil {
		log.Fatalf("can’t decode manifest: %s", err)
	}
	wantManifest := map[string]interface{}{
		"root":        "RUNFILES_ROOT",
		"tags":        []interface{}{"local", "mytag"},
		"loadPath":    []interface{}{"phst_rules_elisp"},
		"inputFiles":  []interface{}{"phst_rules_elisp/elisp/binary.cc", "phst_rules_elisp/elisp/binary.h"},
		"outputFiles": []interface{}{outputFile},
	}
	if diff := cmp.Diff(
		gotManifest, wantManifest,
		cmp.FilterPath(isInputFile, cmp.Transformer("", resolveRunfile)),
	); diff != "" {
		log.Fatalf("manifest: -got +want:\n%s", diff)
	}
}

func isInputFile(p cmp.Path) bool {
	if len(p) < 2 {
		return false
	}
	m, ok := p[1].(cmp.MapIndex)
	if !ok {
		return false
	}
	k := m.Key()
	return k.Kind() == reflect.String && k.String() == "inputFiles"
}

func resolveRunfile(s string) string {
	r, err := runfiles.Path(s)
	if err != nil {
		log.Fatal(err)
	}
	return r
}
<commit_msg>Improve error message<commit_after>// Copyright 2020, 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Binary wrap is a test helper program for //elisp:binary_test, which see.
package main

import (
	"encoding/json"
	"flag"
	"io/ioutil"
	"log"
	"os"
	"reflect"

	"github.com/google/go-cmp/cmp"
	"github.com/phst/runfiles"
)

func main() {
	log.Println("Args:", os.Args)
	log.Println("Environment:", os.Environ())
	var manifestFile string
	flag.StringVar(&manifestFile, "manifest", "", "")
	flag.Parse()
	if manifestFile == "" {
		log.Fatal("--manifest is empty")
	}
	runfilesLib, err := runfiles.Path("phst_rules_elisp/elisp/runfiles/runfiles.elc")
	if err != nil {
		log.Fatal(err)
	}
	// The load path setup depends on whether we use manifest-based or
	// directory-based runfiles.
	var loadPathArgs []string
	if dir, err := runfiles.Path("phst_rules_elisp"); err == nil {
		// Directory-based runfiles.
		loadPathArgs = []string{"--directory=" + dir}
	} else {
		// Manifest-based runfiles.
		loadPathArgs = []string{
			"--load=" + runfilesLib,
			"--funcall=elisp/runfiles/install-handler",
			"--directory=/bazel-runfile:phst_rules_elisp",
		}
	}
	inputFile, err := runfiles.Path("phst_rules_elisp/elisp/binary.cc")
	if err != nil {
		log.Fatal(err)
	}
	var outputFile string
	if os.PathSeparator == '/' {
		outputFile = "/tmp/output.dat"
	} else {
		outputFile = `C:\Temp\output.dat`
	}
	gotArgs := flag.Args()
	wantArgs := append(
		append([]string{"--quick", "--batch"}, loadPathArgs...),
		"--option",
		inputFile,
		" \t\n\r\f äα𝐴🐈'\\\"",
		"/:"+outputFile,
	)
	if diff := cmp.Diff(gotArgs, wantArgs); diff != "" {
		log.Fatalf("positional arguments: -got +want:\n%s", diff)
	}
	jsonData, err := ioutil.ReadFile(manifestFile)
	if err != nil {
		log.Fatalf("can’t read manifest: %s", err)
	}
	var gotManifest map[string]interface{}
	if err := json.Unmarshal(jsonData, &gotManifest); err != nil {
		log.Fatalf("can’t decode manifest: %s", err)
	}
	wantManifest := map[string]interface{}{
		"root":        "RUNFILES_ROOT",
		"tags":        []interface{}{"local", "mytag"},
		"loadPath":    []interface{}{"phst_rules_elisp"},
		"inputFiles":  []interface{}{"phst_rules_elisp/elisp/binary.cc", "phst_rules_elisp/elisp/binary.h"},
		"outputFiles": []interface{}{outputFile},
	}
	if diff := cmp.Diff(
		gotManifest, wantManifest,
		cmp.FilterPath(isInputFile, cmp.Transformer("", resolveRunfile)),
	); diff != "" {
		log.Fatalf("manifest: -got +want:\n%s", diff)
	}
}

func isInputFile(p cmp.Path) bool {
	if len(p) < 2 {
		return false
	}
	m, ok := p[1].(cmp.MapIndex)
	if !ok {
		return false
	}
	k := m.Key()
	return k.Kind() == reflect.String && k.String() == "inputFiles"
}

func resolveRunfile(s string) string {
	r, err := runfiles.Path(s)
	if err != nil {
		log.Fatalf("error resolving runfile for comparison: %s", err)
	}
	return r
}
<|endoftext|>


<commit_before>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

package main

import (
	"os"
	"testing"

	"github.com/go-test/deep"
	"github.com/google/mtail/metrics"
	"github.com/google/mtail/mtail"
	"github.com/google/mtail/testdata"
	"github.com/google/mtail/watcher"
)

var exampleProgramTests = []struct {
	programfile string // Example program file.
	logfile     string // Sample log input.
	goldenfile  string // Expected metrics after processing.
}{
	{
		"examples/rsyncd.mtail",
		"testdata/rsyncd.log",
		"testdata/rsyncd.golden",
	},
	{
		"examples/sftp.mtail",
		"testdata/sftp_chroot.log",
		"testdata/sftp_chroot.golden",
	},
	{
		"examples/dhcpd.mtail",
		"testdata/anonymised_dhcpd_log",
		"testdata/anonymised_dhcpd_log.golden",
	},
	// {
	// 	"examples/ntpd.mtail",
	// 	"testdata/ntp4",
	// 	"testdata/ntp4.golden",
	// },
	// {
	// 	"examples/ntpd.mtail",
	// 	"testdata/xntp3_peerstats",
	// 	"testdata/xntp3_peerstats.golden",
	// },
	{
		"examples/otherwise.mtail",
		"testdata/otherwise.log",
		"testdata/otherwise.golden",
	},
	{
		"examples/else.mtail",
		"testdata/else.log",
		"testdata/else.golden",
	},
}

func TestExamplePrograms(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping test in short mode")
	}
	for _, tc := range exampleProgramTests {
		w := watcher.NewFakeWatcher()
		store := metrics.NewStore()
		o := mtail.Options{Progs: tc.programfile, W: w, Store: store}
		mtail, err := mtail.New(o)
		if err != nil {
			t.Fatalf("create mtail failed: %s", err)
		}

		if _, err := mtail.OneShot(tc.logfile, false); err != nil {
			t.Errorf("Oneshot failed for %s: %s", tc.logfile, err)
			continue
		}

		g, err := os.Open(tc.goldenfile)
		if err != nil {
			t.Fatalf("%s: could not open golden file: %s", tc.goldenfile, err)
		}
		defer g.Close()

		golden_store := metrics.NewStore()
		testdata.ReadTestData(g, tc.programfile, golden_store)

		mtail.Close()

		diff := deep.Equal(golden_store, store)

		if diff != nil {
			t.Errorf("%s: metrics don't match:\n%s", tc.programfile, diff)

			t.Errorf("Store metrics: %#v", store.Metrics)
		}
	}
}
<commit_msg>Fix output of ex_test errors and also allow it to emit AST and type information.<commit_after>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

package main

import (
	"os"
	"testing"

	"github.com/go-test/deep"
	"github.com/google/mtail/metrics"
	"github.com/google/mtail/mtail"
	"github.com/google/mtail/testdata"
	"github.com/google/mtail/watcher"
)

var exampleProgramTests = []struct {
	programfile string // Example program file.
	logfile     string // Sample log input.
	goldenfile  string // Expected metrics after processing.
}{
	{
		"examples/rsyncd.mtail",
		"testdata/rsyncd.log",
		"testdata/rsyncd.golden",
	},
	{
		"examples/sftp.mtail",
		"testdata/sftp_chroot.log",
		"testdata/sftp_chroot.golden",
	},
	{
		"examples/dhcpd.mtail",
		"testdata/anonymised_dhcpd_log",
		"testdata/anonymised_dhcpd_log.golden",
	},
	// {
	// 	"examples/ntpd.mtail",
	// 	"testdata/ntp4",
	// 	"testdata/ntp4.golden",
	// },
	// {
	// 	"examples/ntpd.mtail",
	// 	"testdata/xntp3_peerstats",
	// 	"testdata/xntp3_peerstats.golden",
	// },
	{
		"examples/otherwise.mtail",
		"testdata/otherwise.log",
		"testdata/otherwise.golden",
	},
	{
		"examples/else.mtail",
		"testdata/else.log",
		"testdata/else.golden",
	},
}

func TestExamplePrograms(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping test in short mode")
	}
	for _, tc := range exampleProgramTests {
		w := watcher.NewFakeWatcher()
		store := metrics.NewStore()
		o := mtail.Options{Progs: tc.programfile, W: w, Store: store}
		o.DumpAst = true
		o.DumpTypes = true
		o.DumpBytecode = true
		mtail, err := mtail.New(o)
		if err != nil {
			t.Fatalf("create mtail failed: %s", err)
		}

		if _, err := mtail.OneShot(tc.logfile, false); err != nil {
			t.Errorf("Oneshot failed for %s: %s", tc.logfile, err)
			continue
		}

		g, err := os.Open(tc.goldenfile)
		if err != nil {
			t.Fatalf("%s: could not open golden file: %s", tc.goldenfile, err)
		}
		defer g.Close()

		golden_store := metrics.NewStore()
		testdata.ReadTestData(g, tc.programfile, golden_store)

		mtail.Close()

		diff := deep.Equal(golden_store, store)

		if diff != nil {
			t.Errorf("%s: metrics don't match:\n%v", tc.programfile, diff)

			t.Errorf("Store metrics: %s", store.Metrics)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2018 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// The claat command generates one or more codelabs from "source" documents,
// specified as either Google Doc IDs or local markdown files.
// The command also allows one to preview generated codelabs from local drive
// using "claat serve".
// See more details at https://github.com/googlecodelabs/tools.
package cmd

import (
	"encoding/json"
	"log"
	"sync"

	// allow parsers to register themselves
	_ "github.com/googlecodelabs/tools/claat/parser/gdoc"
	_ "github.com/googlecodelabs/tools/claat/parser/md"
)

const (
	// imgDirname is where a codelab images are stored,
	// relative to the codelab dir.
	imgDirname = "img"
	// metaFilename is codelab metadata file.
	metaFilename = "codelab.json"
	// stdout is a special value for -o cli arg to identify stdout writer.
	stdout = "-"

	// log report formats
	reportErr = "err\t%s %v"
	reportOk  = "ok\t%s"
)

var (
	Exit      int               // program exit code
	exitMu    sync.Mutex        // guards exit
	ExtraVars map[string]string // Extra template variables passed on the command line.
)

// isStdout reports whether filename is stdout.
func isStdout(filename string) bool {
	return filename == stdout
}

// errorf calls log.Printf with fmt and args, and sets non-zero exit code.
func errorf(format string, args ...interface{}) {
	log.Printf(format, args...)
	exitMu.Lock()
	Exit = 1
	exitMu.Unlock()
}

// ParseExtraVars parses extra template variables from command line.
// extra is any additional arguments to pass to format templates. Should be formatted as JSON objects of string:string KV pairs.
func ParseExtraVars(extra string) map[string]string {
	vars := make(map[string]string)
	if extra == "" {
		return vars
	}
	b := []byte(extra)
	err := json.Unmarshal(b, &vars)
	if err != nil {
		errorf("Error parsing additional template data: %v", err)
	}
	return vars
}
<commit_msg>Remove custom errorf from util.go.<commit_after>// Copyright 2018 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// The claat command generates one or more codelabs from "source" documents,
// specified as either Google Doc IDs or local markdown files.
// The command also allows one to preview generated codelabs from local drive
// using "claat serve".
// See more details at https://github.com/googlecodelabs/tools.
package cmd

import (
	"encoding/json"
	"sync"

	// allow parsers to register themselves
	_ "github.com/googlecodelabs/tools/claat/parser/gdoc"
	_ "github.com/googlecodelabs/tools/claat/parser/md"
)

const (
	// imgDirname is where a codelab images are stored,
	// relative to the codelab dir.
	imgDirname = "img"
	// metaFilename is codelab metadata file.
	metaFilename = "codelab.json"
	// stdout is a special value for -o cli arg to identify stdout writer.
	stdout = "-"

	// log report formats
	reportErr = "err\t%s %v"
	reportOk  = "ok\t%s"
)

var (
	Exit      int               // program exit code
	exitMu    sync.Mutex        // guards exit
	ExtraVars map[string]string // Extra template variables passed on the command line.
)

// isStdout reports whether filename is stdout.
func isStdout(filename string) bool {
	return filename == stdout
}

// ParseExtraVars parses extra template variables from command line.
// extra is any additional arguments to pass to format templates. Should be formatted as JSON objects of string:string KV pairs.
func ParseExtraVars(extra string) map[string]string {
	vars := make(map[string]string)
	if extra == "" {
		return vars
	}
	b := []byte(extra)
	err := json.Unmarshal(b, &vars)
	if err != nil {
		errorf("Error parsing additional template data: %v", err)
	}
	return vars
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/nightlyone/go-stackdriver/stackdriver"
	"log"
	"time"
)

func main() {
	now := time.Now().Unix()
	query := &stackdriver.Metrics{
		Timestamp:    now,
		ProtoVersion: 1,
		Data: []stackdriver.Datapoint{
			stackdriver.Datapoint{
				Name:        "one",
				Value:       2,
				CollectedAt: now,
			},
		},
	}

	if err := stackdriver.Submit(query); err != nil {
		log.Fatalln(err)
	}
	log.Println("sent", query, "to apiendpoint")
	deploy := &stackdriver.Deploy{
		RevisionId: "87230611cdc7e5ff7723a91e715367c553ad1115",
		DeployedBy: "John Doe",
		DeployedTo: "production",
		Repository: "example_repo",
	}
	if err := stackdriver.Submit(deploy); err != nil {
		log.Fatalln(err)
	}
	log.Println("sent", deploy, "to apiendpoint")
	annotate := &stackdriver.Annotation{
		Message:     "Started moving more services to the cloud",
		AnnotatedBy: "devops",
		Level:       "INFO",
	}
	if err := stackdriver.Submit(annotate); err != nil {
		log.Fatalln(err)
	}
	log.Println("sent", annotate, "to apiendpoint")
}
<commit_msg>Use new API<commit_after>package main

import (
	"github.com/nightlyone/go-stackdriver/stackdriver"
	"log"
	"time"
)

func main() {
	now := time.Now().Unix()
	query := &stackdriver.Metrics{
		Timestamp:    now,
		ProtoVersion: 1,
		Data: []stackdriver.Datapoint{
			stackdriver.Datapoint{
				Name:        "one",
				Value:       2,
				CollectedAt: now,
			},
		},
	}

	if err := query.Submit(); err != nil {
		log.Fatalln(err)
	}
	log.Println("sent", query, "to apiendpoint")
	deploy := &stackdriver.Deploy{
		RevisionId: "87230611cdc7e5ff7723a91e715367c553ad1115",
		DeployedBy: "John Doe",
		DeployedTo: "production",
		Repository: "example_repo",
	}
	if err := deploy.Submit(); err != nil {
		log.Fatalln(err)
	}
	log.Println("sent", deploy, "to apiendpoint")
	annotate := &stackdriver.Annotation{
		Message:     "Started moving more services to the cloud",
		AnnotatedBy: "devops",
		Level:       "INFO",
	}
	if err := annotate.Submit(); err != nil {
		log.Fatalln(err)
	}
	log.Println("sent", annotate, "to apiendpoint")
}
<|endoftext|>


<commit_before>package dalga

// TODO unexport Dalga.Config
// TODO rename id to job and change limit to 65535
// TODO put a lock on table while working on it
// TODO update readme
// TODO implement raft consensus

import (
	"database/sql"
	"flag"
	"log"
	"net"
	"strconv"
	"time"

	"github.com/cenkalti/dalga/vendor/github.com/go-sql-driver/mysql"
	"github.com/cenkalti/dalga/vendor/github.com/streadway/amqp"
)

var debugging = flag.Bool("debug", false, "turn on debug messages")

func debug(args ...interface{}) {
	if *debugging {
		log.Println(args...)
	}
}

type Dalga struct {
	Config     Config
	db         *sql.DB
	table      *table
	connection *amqp.Connection
	channel    *amqp.Channel
	listener   net.Listener
	// to wake up publisher when a new job is scheduled or cancelled
	notify chan struct{}
	// will be closed when dalga is ready to accept requests
	ready chan struct{}
	// will be closed by Shutdown method
	shutdown chan struct{}
	// to stop publisher goroutine
	stopPublisher chan struct{}
	// will be closed when publisher goroutine is stopped
	publisherStopped chan struct{}
}

func New(config Config) *Dalga {
	return &Dalga{
		Config:           config,
		notify:           make(chan struct{}, 1),
		ready:            make(chan struct{}),
		shutdown:         make(chan struct{}),
		stopPublisher:    make(chan struct{}),
		publisherStopped: make(chan struct{}),
	}
}

// Run the dalga and waits until Shutdown() is called.
func (d *Dalga) Run() error {
	if err := d.connectDB(); err != nil {
		return err
	}
	defer d.db.Close()

	if err := d.connectMQ(); err != nil {
		return err
	}
	defer d.channel.Close()
	defer d.connection.Close()

	var err error
	d.listener, err = net.Listen("tcp", d.Config.HTTP.Addr())
	if err != nil {
		return err
	}

	close(d.ready)

	go d.publisher()
	defer func() {
		close(d.stopPublisher)
		<-d.publisherStopped
	}()

	if err = d.serveHTTP(); err != nil {
		select {
		case _, ok := <-d.shutdown:
			if !ok {
				// shutdown in progress, do not return error
				return nil
			}
		default:
		}
	}
	return err
}

func (d *Dalga) Shutdown() error {
	close(d.shutdown)
	return d.listener.Close()
}

func (d *Dalga) NotifyReady() <-chan struct{} {
	return d.ready
}

func (d *Dalga) connectDB() error {
	var err error
	d.db, err = sql.Open("mysql", d.Config.MySQL.DSN())
	if err != nil {
		return err
	}
	if err = d.db.Ping(); err != nil {
		return err
	}
	log.Print("Connected to MySQL")
	d.table = &table{d.db, d.Config.MySQL.Table}
	return nil
}

func (d *Dalga) connectMQ() error {
	var err error
	d.connection, err = amqp.Dial(d.Config.RabbitMQ.URL())
	if err != nil {
		return err
	}
	log.Print("Connected to RabbitMQ")

	// Exit program when AMQP connection is closed.
	connClosed := make(chan *amqp.Error)
	d.connection.NotifyClose(connClosed)
	go func() {
		if err, ok := <-connClosed; ok {
			log.Fatal(err)
		}
	}()

	d.channel, err = d.connection.Channel()
	if err != nil {
		return err
	}

	// Log undelivered messages.
	returns := make(chan amqp.Return)
	d.channel.NotifyReturn(returns)
	go func() {
		for r := range returns {
			log.Printf("%d: %s exchange=%q routing-key=%q", r.ReplyCode, r.ReplyText, r.Exchange, r.RoutingKey)
		}
	}()

	return nil
}

func (d *Dalga) CreateTable() error {
	db, err := sql.Open("mysql", d.Config.MySQL.DSN())
	if err != nil {
		return err
	}
	defer db.Close()
	t := &table{db, d.Config.MySQL.Table}
	return t.Create()
}

func (d *Dalga) Schedule(id, routingKey string, interval uint32) error {
	job := NewJob(id, routingKey, interval)

	if err := d.table.Insert(job); err != nil {
		return err
	}

	// Wake up the publisher.
	//
	// publisher() may be sleeping for the next job on the queue
	// at the time we schedule a new Job. Let it wake up so it can
	// re-fetch the new Job from the front of the queue.
	select {
	case d.notify <- struct{}{}:
		debug("Sent new job signal")
	default:
	}

	debug("Job is scheduled:", job)
	return nil
}

func (d *Dalga) Cancel(id, routingKey string) error {
	if err := d.table.Delete(id, routingKey); err != nil {
		return err
	}

	select {
	case d.notify <- struct{}{}:
		debug("Sent cancel signal")
	default:
	}

	debug("Job is cancelled:", Job{primaryKey: primaryKey{id, routingKey}})
	return nil
}

// publish sends a message to exchange defined in the config and
// updates the Job's next run time on the database.
func (d *Dalga) publish(j *Job) error {
	debug("publish", *j)

	// Send a message to RabbitMQ
	err := d.channel.Publish(d.Config.RabbitMQ.Exchange, j.RoutingKey, true, false, amqp.Publishing{
		Body:         []byte(j.ID),
		DeliveryMode: amqp.Persistent,
		Expiration:   strconv.FormatFloat(j.Interval.Seconds(), 'f', 0, 64) + "000",
	})
	if err != nil {
		return err
	}

	return d.table.UpdateNextRun(j)
}

// publisher runs a loop that reads the next Job from the queue and publishes it.
func (d *Dalga) publisher() {
	defer close(d.publisherStopped)

	for {
		debug("---")

		var after <-chan time.Time

		job, err := d.table.Front()
		if err != nil {
			if err == sql.ErrNoRows {
				debug("No scheduled jobs in the table")
			} else if myErr, ok := err.(*mysql.MySQLError); ok && myErr.Number == 1146 {
				// Table doesn't exist
				log.Fatal(myErr)
			} else {
				log.Print(err)
				time.Sleep(time.Second)
				continue
			}
		} else {
			remaining := job.Remaining()
			after = time.After(remaining)

			debug("Next job:", job, "Remaining:", remaining)
		}

		// Sleep until the next job's run time or the webserver's wakes us up.
		select {
		case <-after:
			debug("Job sleep time finished")
			if err = d.publish(job); err != nil {
				log.Print(err)
				time.Sleep(time.Second)
			}
		case <-d.notify:
			debug("Woken up from sleep by notification")
			continue
		case <-d.stopPublisher:
			debug("Came quit message")
			return
		}
	}
}
<commit_msg>unexport dalga.Config<commit_after>package dalga

// TODO rename id to job and change limit to 65535
// TODO put a lock on table while working on it
// TODO update readme
// TODO implement raft consensus

import (
	"database/sql"
	"flag"
	"log"
	"net"
	"strconv"
	"time"

	"github.com/cenkalti/dalga/vendor/github.com/go-sql-driver/mysql"
	"github.com/cenkalti/dalga/vendor/github.com/streadway/amqp"
)

var debugging = flag.Bool("debug", false, "turn on debug messages")

func debug(args ...interface{}) {
	if *debugging {
		log.Println(args...)
	}
}

type Dalga struct {
	config     Config
	db         *sql.DB
	table      *table
	connection *amqp.Connection
	channel    *amqp.Channel
	listener   net.Listener
	// to wake up publisher when a new job is scheduled or cancelled
	notify chan struct{}
	// will be closed when dalga is ready to accept requests
	ready chan struct{}
	// will be closed by Shutdown method
	shutdown chan struct{}
	// to stop publisher goroutine
	stopPublisher chan struct{}
	// will be closed when publisher goroutine is stopped
	publisherStopped chan struct{}
}

func New(config Config) *Dalga {
	return &Dalga{
		config:           config,
		notify:           make(chan struct{}, 1),
		ready:            make(chan struct{}),
		shutdown:         make(chan struct{}),
		stopPublisher:    make(chan struct{}),
		publisherStopped: make(chan struct{}),
	}
}

// Run the dalga and waits until Shutdown() is called.
func (d *Dalga) Run() error {
	if err := d.connectDB(); err != nil {
		return err
	}
	defer d.db.Close()

	if err := d.connectMQ(); err != nil {
		return err
	}
	defer d.channel.Close()
	defer d.connection.Close()

	var err error
	d.listener, err = net.Listen("tcp", d.config.HTTP.Addr())
	if err != nil {
		return err
	}

	close(d.ready)

	go d.publisher()
	defer func() {
		close(d.stopPublisher)
		<-d.publisherStopped
	}()

	if err = d.serveHTTP(); err != nil {
		select {
		case _, ok := <-d.shutdown:
			if !ok {
				// shutdown in progress, do not return error
				return nil
			}
		default:
		}
	}
	return err
}

func (d *Dalga) Shutdown() error {
	close(d.shutdown)
	return d.listener.Close()
}

func (d *Dalga) NotifyReady() <-chan struct{} {
	return d.ready
}

func (d *Dalga) connectDB() error {
	var err error
	d.db, err = sql.Open("mysql", d.config.MySQL.DSN())
	if err != nil {
		return err
	}
	if err = d.db.Ping(); err != nil {
		return err
	}
	log.Print("Connected to MySQL")
	d.table = &table{d.db, d.config.MySQL.Table}
	return nil
}

func (d *Dalga) connectMQ() error {
	var err error
	d.connection, err = amqp.Dial(d.config.RabbitMQ.URL())
	if err != nil {
		return err
	}
	log.Print("Connected to RabbitMQ")

	// Exit program when AMQP connection is closed.
	connClosed := make(chan *amqp.Error)
	d.connection.NotifyClose(connClosed)
	go func() {
		if err, ok := <-connClosed; ok {
			log.Fatal(err)
		}
	}()

	d.channel, err = d.connection.Channel()
	if err != nil {
		return err
	}

	// Log undelivered messages.
	returns := make(chan amqp.Return)
	d.channel.NotifyReturn(returns)
	go func() {
		for r := range returns {
			log.Printf("%d: %s exchange=%q routing-key=%q", r.ReplyCode, r.ReplyText, r.Exchange, r.RoutingKey)
		}
	}()

	return nil
}

func (d *Dalga) CreateTable() error {
	db, err := sql.Open("mysql", d.config.MySQL.DSN())
	if err != nil {
		return err
	}
	defer db.Close()
	t := &table{db, d.config.MySQL.Table}
	return t.Create()
}

func (d *Dalga) Schedule(id, routingKey string, interval uint32) error {
	job := NewJob(id, routingKey, interval)

	if err := d.table.Insert(job); err != nil {
		return err
	}

	// Wake up the publisher.
	//
	// publisher() may be sleeping for the next job on the queue
	// at the time we schedule a new Job. Let it wake up so it can
	// re-fetch the new Job from the front of the queue.
	select {
	case d.notify <- struct{}{}:
		debug("Sent new job signal")
	default:
	}

	debug("Job is scheduled:", job)
	return nil
}

func (d *Dalga) Cancel(id, routingKey string) error {
	if err := d.table.Delete(id, routingKey); err != nil {
		return err
	}

	select {
	case d.notify <- struct{}{}:
		debug("Sent cancel signal")
	default:
	}

	debug("Job is cancelled:", Job{primaryKey: primaryKey{id, routingKey}})
	return nil
}

// publish sends a message to exchange defined in the config and
// updates the Job's next run time on the database.
func (d *Dalga) publish(j *Job) error {
	debug("publish", *j)

	// Send a message to RabbitMQ
	err := d.channel.Publish(d.config.RabbitMQ.Exchange, j.RoutingKey, true, false, amqp.Publishing{
		Body:         []byte(j.ID),
		DeliveryMode: amqp.Persistent,
		Expiration:   strconv.FormatFloat(j.Interval.Seconds(), 'f', 0, 64) + "000",
	})
	if err != nil {
		return err
	}

	return d.table.UpdateNextRun(j)
}

// publisher runs a loop that reads the next Job from the queue and publishes it.
func (d *Dalga) publisher() {
	defer close(d.publisherStopped)

	for {
		debug("---")

		var after <-chan time.Time

		job, err := d.table.Front()
		if err != nil {
			if err == sql.ErrNoRows {
				debug("No scheduled jobs in the table")
			} else if myErr, ok := err.(*mysql.MySQLError); ok && myErr.Number == 1146 {
				// Table doesn't exist
				log.Fatal(myErr)
			} else {
				log.Print(err)
				time.Sleep(time.Second)
				continue
			}
		} else {
			remaining := job.Remaining()
			after = time.After(remaining)

			debug("Next job:", job, "Remaining:", remaining)
		}

		// Sleep until the next job's run time or the webserver's wakes us up.
		select {
		case <-after:
			debug("Job sleep time finished")
			if err = d.publish(job); err != nil {
				log.Print(err)
				time.Sleep(time.Second)
			}
		case <-d.notify:
			debug("Woken up from sleep by notification")
			continue
		case <-d.stopPublisher:
			debug("Came quit message")
			return
		}
	}
}
<|endoftext|>


<commit_before>package dalga

import (
	"database/sql"
	"errors"
	"flag"
	"fmt"
	"log"
	"net"
	"os"
	"sync"
	"time"

	"github.com/cenkalti/dalga/dalga/Godeps/_workspace/src/github.com/fzzy/radix/redis"
)

var debugging = flag.Bool("debug", false, "turn on debug messages")

const (
	redisLockKey        = "dalga-lock"
	redisLockExpiry     = 30 * time.Second
	redisLockRenewAfter = 20 * time.Second
)

func debug(args ...interface{}) {
	if *debugging {
		log.Println(args...)
	}
}

type Dalga struct {
	config    Config
	redis     *redis.Client
	db        *sql.DB
	listener  net.Listener
	Jobs      *JobManager
	scheduler *scheduler
	// will be closed when dalga is ready to accept requests
	ready chan struct{}
	// will be closed by Shutdown method
	shutdown     chan struct{}
	onceShutdown sync.Once
}

func New(config Config) (*Dalga, error) {
	db, err := sql.Open("mysql", config.MySQL.DSN())
	if err != nil {
		return nil, err
	}
	t := &table{db, config.MySQL.Table}
	s := newScheduler(t, config.Endpoint.BaseURL, time.Duration(config.Endpoint.Timeout)*time.Second)
	m := newJobManager(t, s)
	return &Dalga{
		config:    config,
		db:        db,
		Jobs:      m,
		scheduler: s,
		ready:     make(chan struct{}),
		shutdown:  make(chan struct{}),
	}, nil
}

// Run Dalga. This function is blocking. Returns nil after Shutdown is called.
func (d *Dalga) Run() error {
	var err error
	d.listener, err = net.Listen("tcp", d.config.Listen.Addr())
	if err != nil {
		return err
	}
	log.Println("listening", d.listener.Addr())

	if !d.config.Redis.Zero() {
		d.redis, err = redis.Dial("tcp", d.config.Redis.Addr())
		if err != nil {
			return err
		}
		log.Print("connected to redis")
		if err = d.holdRedisLock(); err != nil {
			return err
		}
	}

	d.db, err = sql.Open("mysql", d.config.MySQL.DSN())
	if err != nil {
		return err
	}
	defer d.db.Close()

	if err = d.db.Ping(); err != nil {
		return err
	}
	log.Print("connected to mysql")

	log.Print("dalga is ready")
	close(d.ready)

	go d.scheduler.Run()
	defer func() {
		d.scheduler.Stop()
		<-d.scheduler.NotifyDone()
	}()

	if err = d.serveHTTP(); err != nil {
		select {
		case _, ok := <-d.shutdown:
			if !ok {
				// shutdown in progress, do not return error
				return nil
			}
		default:
		}
	}
	return err
}

func (d *Dalga) holdRedisLock() error {
	log.Print("acquiring redis lock")
	hostname, err := os.Hostname()
	if err != nil {
		return err
	}
	value := fmt.Sprintf("%s:%d", hostname, d.listener.Addr().(*net.TCPAddr).Port)
	reply := d.redis.Cmd("SET", redisLockKey, value, "NX", "PX", int(redisLockExpiry/time.Millisecond))
	if reply.Err != nil {
		return reply.Err
	}
	status, err := reply.Str()
	if status != "OK" {
		return errors.New("cannot acquire redis lock")
	}
	log.Print("acquired redis lock")
	go d.renewRedisLock(value)
	return nil
}

func (d *Dalga) renewRedisLock(value string) {
	for {
		select {
		case <-time.After(redisLockRenewAfter):
			debug("renewing redis lock")
			reply := d.redis.Cmd("EVAL", `
				if redis.call("GET", KEYS[1]) == ARGV[1] then
					return redis.call("SET", KEYS[1], ARGV[1], "PX", ARGV[2])
				else
					return 0
				end
				`, 1, redisLockKey, value, int(redisLockExpiry/time.Millisecond))
			if reply.Err != nil {
				log.Print(reply.Err)
				d.Shutdown()
				return
			}
			debug("lock renewed")
		case <-d.scheduler.NotifyDone():
			d.redis.Cmd("DEL", redisLockKey)
			return
		}
	}
}

// Shutdown running Dalga gracefully.
func (d *Dalga) Shutdown() {
	d.onceShutdown.Do(func() {
		close(d.shutdown)
		if err := d.listener.Close(); err != nil {
			log.Print(err)
		}
	})
}

// NotifyReady returns a channel that will be closed when Dalga is ready to accept HTTP requests.
func (d *Dalga) NotifyReady() <-chan struct{} {
	return d.ready
}

// CreateTable creates the table for storing jobs on database.
func (d *Dalga) CreateTable() error {
	db, err := sql.Open("mysql", d.config.MySQL.DSN())
	if err != nil {
		return err
	}
	defer db.Close()
	t := &table{db, d.config.MySQL.Table}
	return t.Create()
}
<commit_msg>check redis query status<commit_after>package dalga

import (
	"database/sql"
	"errors"
	"flag"
	"fmt"
	"log"
	"net"
	"os"
	"sync"
	"time"

	"github.com/cenkalti/dalga/dalga/Godeps/_workspace/src/github.com/fzzy/radix/redis"
)

var debugging = flag.Bool("debug", false, "turn on debug messages")

const (
	redisLockKey        = "dalga-lock"
	redisLockExpiry     = 30 * time.Second
	redisLockRenewAfter = 20 * time.Second
)

func debug(args ...interface{}) {
	if *debugging {
		log.Println(args...)
	}
}

type Dalga struct {
	config    Config
	redis     *redis.Client
	db        *sql.DB
	listener  net.Listener
	Jobs      *JobManager
	scheduler *scheduler
	// will be closed when dalga is ready to accept requests
	ready chan struct{}
	// will be closed by Shutdown method
	shutdown     chan struct{}
	onceShutdown sync.Once
}

func New(config Config) (*Dalga, error) {
	db, err := sql.Open("mysql", config.MySQL.DSN())
	if err != nil {
		return nil, err
	}
	t := &table{db, config.MySQL.Table}
	s := newScheduler(t, config.Endpoint.BaseURL, time.Duration(config.Endpoint.Timeout)*time.Second)
	m := newJobManager(t, s)
	return &Dalga{
		config:    config,
		db:        db,
		Jobs:      m,
		scheduler: s,
		ready:     make(chan struct{}),
		shutdown:  make(chan struct{}),
	}, nil
}

// Run Dalga. This function is blocking. Returns nil after Shutdown is called.
func (d *Dalga) Run() error {
	var err error
	d.listener, err = net.Listen("tcp", d.config.Listen.Addr())
	if err != nil {
		return err
	}
	log.Println("listening", d.listener.Addr())

	if !d.config.Redis.Zero() {
		d.redis, err = redis.Dial("tcp", d.config.Redis.Addr())
		if err != nil {
			return err
		}
		log.Print("connected to redis")
		if err = d.holdRedisLock(); err != nil {
			return err
		}
	}

	d.db, err = sql.Open("mysql", d.config.MySQL.DSN())
	if err != nil {
		return err
	}
	defer d.db.Close()

	if err = d.db.Ping(); err != nil {
		return err
	}
	log.Print("connected to mysql")

	log.Print("dalga is ready")
	close(d.ready)

	go d.scheduler.Run()
	defer func() {
		d.scheduler.Stop()
		<-d.scheduler.NotifyDone()
	}()

	if err = d.serveHTTP(); err != nil {
		select {
		case _, ok := <-d.shutdown:
			if !ok {
				// shutdown in progress, do not return error
				return nil
			}
		default:
		}
	}
	return err
}

func (d *Dalga) holdRedisLock() error {
	log.Print("acquiring redis lock")
	hostname, err := os.Hostname()
	if err != nil {
		return err
	}
	value := fmt.Sprintf("%s:%d", hostname, d.listener.Addr().(*net.TCPAddr).Port)
	reply := d.redis.Cmd("SET", redisLockKey, value, "NX", "PX", int(redisLockExpiry/time.Millisecond))
	if reply.Err != nil {
		return reply.Err
	}
	status, _ := reply.Str()
	if status != "OK" {
		return errors.New("cannot acquire redis lock")
	}
	log.Print("acquired redis lock")
	go d.renewRedisLock(value)
	return nil
}

func (d *Dalga) renewRedisLock(value string) {
	for {
		select {
		case <-time.After(redisLockRenewAfter):
			debug("renewing redis lock")
			reply := d.redis.Cmd("EVAL", `
				if redis.call("GET", KEYS[1]) == ARGV[1] then
					return redis.call("SET", KEYS[1], ARGV[1], "PX", ARGV[2])
				else
					return 0
				end
				`, 1, redisLockKey, value, int(redisLockExpiry/time.Millisecond))
			if reply.Err != nil {
				log.Print(reply.Err)
				d.Shutdown()
				return
			}
			status, _ := reply.Str()
			if status != "OK" {
				log.Print("cannot renew redis lock")
				d.Shutdown()
				return
			}
			debug("lock renewed")
		case <-d.scheduler.NotifyDone():
			d.redis.Cmd("DEL", redisLockKey)
			return
		}
	}
}

// Shutdown running Dalga gracefully.
func (d *Dalga) Shutdown() {
	d.onceShutdown.Do(func() {
		close(d.shutdown)
		if err := d.listener.Close(); err != nil {
			log.Print(err)
		}
	})
}

// NotifyReady returns a channel that will be closed when Dalga is ready to accept HTTP requests.
func (d *Dalga) NotifyReady() <-chan struct{} {
	return d.ready
}

// CreateTable creates the table for storing jobs on database.
func (d *Dalga) CreateTable() error {
	db, err := sql.Open("mysql", d.config.MySQL.DSN())
	if err != nil {
		return err
	}
	defer db.Close()
	t := &table{db, d.config.MySQL.Table}
	return t.Create()
}
<|endoftext|>