<commit_before>package zeroformatter

import (
	"errors"
	"fmt"
	"math"
	"reflect"
	"unicode/utf16"
	"unsafe"
)

const (
	byte1 uint32 = 1 << iota
	byte2
	byte4
	byte8
)

type serializer struct {
	create []byte

	// todo ここにサイズ計算時に突っ込んで、あとからpopする？
	queueMapKey   [][]reflect.Value
	queueMapValue [][]reflect.Value
}

func createSerializer() *serializer {
	return &serializer{}
}

func Serialize(holder interface{}) ([]byte, error) {
	d := createSerializer()

	t := reflect.ValueOf(holder)
	if t.Kind() == reflect.Ptr {
		t = t.Elem()
		if t.Kind() == reflect.Ptr {
			t = t.Elem()
		}
	}

	var err error
	if t.Kind() == reflect.Struct && !isDateTime(t) && !isDateTimeOffset(t) {
		startOffset := uint32(2+t.NumField()) * byte4

		dataPartSize, _ := d.calcSize(t)
		size := startOffset + dataPartSize
		d.create = make([]byte, size)

		err = d.serializeStruct(t, startOffset, size)
	} else {
		size, _ := d.calcSize(t)
		d.create = make([]byte, size)
		_, err = d.serialize(t, 0)
	}

	return d.create, err
}

func (d *serializer) serializeStruct(rv reflect.Value, offset uint32, size uint32) error {
	nf := rv.NumField()
	index := 2 * byte4
	for i := 0; i < nf; i++ {
		s, err := d.serialize(rv.Field(i), offset)
		if err != nil {
			return err
		}

		d.create[index], d.create[index+1], d.create[index+2], d.create[index+3] = byte(offset), byte(offset>>8), byte(offset>>16), byte(offset>>24)
		index += byte4
		offset += s
	}
	// size
	d.create[0], d.create[1], d.create[2], d.create[3] = byte(size), byte(size>>8), byte(size>>16), byte(size>>24)
	// last index
	li := nf - 1
	d.create[4], d.create[5], d.create[6], d.create[7] = byte(li), byte(li>>8), byte(li>>16), byte(li>>24)
	return nil
}

func (d *serializer) isFixedSize(rv reflect.Value) bool {
	ret := false
	switch rv.Kind() {
	case
		reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int,
		reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint,
		reflect.Float32, reflect.Float64,
		reflect.Bool:
		ret = true

	case
		reflect.Struct:
		if isDateTimeOffset(rv) || isDateTime(rv) {
			ret = true
		}

	default:
	}
	return ret
}

func (d *serializer) calcSize(rv reflect.Value) (uint32, error) {
	ret := uint32(0)

	switch rv.Kind() {
	case reflect.Int8:
		ret = byte1

	case reflect.Int16:
		ret = byte2

	case reflect.Int32:
		if isChar(rv) {
			ret = byte2
		} else {
			ret = byte4
		}

	case reflect.Int:
		ret = byte4

	case reflect.Int64:
		if isDuration(rv) {
			ret = byte4 + byte8
		} else {
			ret = byte8
		}

	case reflect.Uint8:
		ret = byte1

	case reflect.Uint16:
		ret = byte2

	case reflect.Uint32, reflect.Uint:
		ret = byte4

	case reflect.Uint64:
		ret = byte8

	case reflect.Float32:
		ret = byte4

	case reflect.Float64:
		ret = byte8

	case reflect.Bool:
		ret = byte1

	case reflect.String:
		l := uint32(rv.Len())
		ret = l + byte4

	case reflect.Array, reflect.Slice:
		l := rv.Len()
		if l > 0 {
			ret += byte4
			isTypeFixed := d.isFixedSize(rv.Index(0))
			if isTypeFixed {
				s, err := d.calcSize(rv.Index(0))
				if err != nil {
					return 0, err
				}
				ret += s * uint32(l)
			} else {
				for i := 0; i < l; i++ {
					s, err := d.calcSize(rv.Index(i))
					if err != nil {
						return 0, err
					}
					ret += s
				}
			}
		} else {
			// only length info
			ret = byte4
		}

	case reflect.Struct:
		if isDateTimeOffset(rv) {
			ret = byte4 + byte8 + byte2
		} else if isDateTime(rv) {
			ret = byte4 + byte8
		} else {
			for i := 0; i < rv.NumField(); i++ {
				s, err := d.calcSize(rv.Field(i))
				if err != nil {
					return 0, err
				}
				ret += s
			}
		}

	case reflect.Map:
		// length
		ret += byte4
		l := uint32(rv.Len())

		if l < 1 {
			return ret, nil
		}
		// check fixed type
		keys := rv.MapKeys()
		isFixedKey := d.isFixedSize(keys[0])
		isFixedVal := d.isFixedSize(rv.MapIndex(keys[0]))

		if isFixedKey && isFixedVal {
			sizeK, err := d.calcSize(keys[0])
			if err != nil {
				return 0, err
			}
			sizeV, err := d.calcSize(rv.MapIndex(keys[0]))
			if err != nil {
				return 0, err
			}
			ret += (sizeK + sizeV) * l
		} else if isFixedKey && !isFixedVal {
			sizeK, err := d.calcSize(keys[0])
			if err != nil {
				return 0, err
			}

			for _, k := range keys {
				sizeV, err := d.calcSize(rv.MapIndex(k))
				if err != nil {
					return 0, err
				}
				ret += sizeK + sizeV
			}

		} else if !isFixedKey && isFixedVal {
			sizeV, err := d.calcSize(rv.MapIndex(keys[0]))
			if err != nil {
				return 0, err
			}
			for _, k := range keys {
				sizeK, err := d.calcSize(k)
				if err != nil {
					return 0, err
				}
				ret += sizeK + sizeV
			}
		} else {
			for _, k := range keys {
				sizeK, err := d.calcSize(k)
				if err != nil {
					return 0, err
				}
				sizeV, err := d.calcSize(rv.MapIndex(k))
				if err != nil {
					return 0, err
				}
				ret += sizeK + sizeV
			}
		}

	case reflect.Ptr:
		if rv.IsNil() {
			return 0, errors.New(fmt.Sprint("pointer is null : ", rv.Type()))
		}
		s, err := d.calcSize(rv.Elem())
		if err != nil {
			return 0, err
		}
		ret = s

	default:
		return 0, errors.New(fmt.Sprint("this type is not supported : ", rv.Type()))
	}

	return ret, nil
}

func (d *serializer) serialize(rv reflect.Value, offset uint32) (uint32, error) {
	size := uint32(0)

	switch rv.Kind() {
	case reflect.Int8:
		d.write_s1_i64(rv.Int(), offset)
		size += byte1

	case reflect.Int16:
		d.write_s2_i64(rv.Int(), offset)
		size += byte2

	case reflect.Int32:
		if isChar(rv) {

			// rune [ushort(2)]
			enc := utf16.Encode([]rune{int32(rv.Int())})
			v := enc[0]
			d.create[offset+0] = byte(v)
			d.create[offset+1] = byte(v >> 8)
			size += byte2
		} else {
			d.write_s4_i64(rv.Int(), offset)
			size += byte4
		}

	case reflect.Int:
		d.write_s4_i64(rv.Int(), offset)
		size += byte4

	case reflect.Int64:
		if isDuration(rv) {
			// seconds
			ns := rv.MethodByName("Nanoseconds").Call([]reflect.Value{})[0]
			nanoseconds := ns.Int()
			sec, nsec := nanoseconds/(1000*1000), int64(nanoseconds%(1000*1000))
			d.write_s8_i64(sec, offset)
			size += byte8
			offset += byte8

			// nanos
			d.write_s4_i64(nsec, offset)
			size += byte4
		} else {
			d.write_s8_i64(rv.Int(), offset)
			size += byte8
		}

	case reflect.Uint8:
		d.write_s1_u64(rv.Uint(), offset)
		size += byte1

	case reflect.Uint16:
		d.write_s2_u64(rv.Uint(), offset)
		size += byte2

	case reflect.Uint32, reflect.Uint:
		d.write_s4_u64(rv.Uint(), offset)
		size += byte4

	case reflect.Uint64:
		d.write_s8_u64(rv.Uint(), offset)
		size += byte8

	case reflect.Float32:
		v := math.Float32bits(float32(rv.Float()))
		d.write_s4_u32(v, offset)
		size += byte4

	case reflect.Float64:
		v := math.Float64bits(rv.Float())
		d.write_s8_u64(v, offset)
		size += byte8

	case reflect.Bool:

		if rv.Bool() {
			d.write_s1_u64(0x01, offset)
		} else {
			d.write_s1_u64(0x00, offset)
		}
		size += byte1

	case reflect.String:
		str := rv.String()
		l := uint32(len(str))
		d.write_s4_u32(l, offset)
		size += byte4
		offset += byte4

		// NOTE : unsafe
		strBytes := *(*[]byte)(unsafe.Pointer(&str))
		for i := uint32(0); i < l; i++ {
			d.create[offset+i] = strBytes[i]
		}
		size += l

	case reflect.Array, reflect.Slice:
		l := rv.Len()
		if l > 0 {
			d.write_s4_i(l, offset)
			size += byte4
			offset += byte4

			for i := 0; i < l; i++ {
				s, err := d.serialize(rv.Index(i), offset)
				if err != nil {
					return 0, err
				}
				offset += s
				size += s
			}
		} else {
			// only make length info
			d.write_s4_i(0, offset)
			size += byte4
		}

	case reflect.Struct:
		if isDateTimeOffset(rv) {

			// offset
			rets := rv.MethodByName("Zone").Call([]reflect.Value{})
			_, offSec := rets[0] /*name*/, rets[1].Int() /*offset*/
			offMin := offSec / 60

			// seconds
			rets = rv.MethodByName("Unix").Call([]reflect.Value{})
			seconds := rets[0].Int() + offSec

			// nanos
			nanos := rv.FieldByName("nsec").Int()

			// seconds to byte
			d.write_s8_i64(seconds, offset)
			size += byte8
			offset += byte8

			// nanos to byte
			d.write_s4_i64(nanos, offset)
			size += byte4
			offset += byte4

			// offset to byte
			d.write_s2_i64(offMin, offset)
			size += byte2
		} else if isDateTime(rv) {
			// seconds
			unixTime := rv.MethodByName("Unix").Call([]reflect.Value{})
			sec := unixTime[0].Int()
			d.write_s8_i64(sec, offset)
			size += byte8
			offset += byte8

			// nanos
			nsec := rv.FieldByName("nsec").Int()
			d.write_s4_i64(nsec, offset)
			size += byte4
		} else {
			for i := 0; i < rv.NumField(); i++ {
				s, err := d.serialize(rv.Field(i), offset)
				if err != nil {
					return 0, err
				}
				offset += s
				size += s
			}
		}

	case reflect.Map:
		// length
		d.write_s4_i(rv.Len(), offset)
		size += byte4
		offset += byte4

		// todo : check fixed type

		for _, k := range rv.MapKeys() {
			addOffByK, err := d.serialize(k, offset)
			if err != nil {
				return 0, err
			}
			addOffByV, err := d.serialize(rv.MapIndex(k), offset+addOffByK)
			if err != nil {
				return 0, err
			}
			offset += addOffByK + addOffByV
			size += addOffByK + addOffByV
		}

	case reflect.Ptr:
		if rv.IsNil() {
			return 0, errors.New(fmt.Sprint("pointer is null : ", rv.Type()))
		}
		s, err := d.serialize(rv.Elem(), offset)
		if err != nil {
			return 0, err
		}
		size += s

	default:
		return 0, errors.New(fmt.Sprint("this type is not supported : ", rv.Type()))
	}

	return size, nil
}
<commit_msg>improve map serialize<commit_after>package zeroformatter

import (
	"errors"
	"fmt"
	"math"
	"reflect"
	"unicode/utf16"
	"unsafe"
)

const (
	byte1 uint32 = 1 << iota
	byte2
	byte4
	byte8
)

type serializer struct {
	create []byte

	queueMapKey   [][]reflect.Value
	queueMapValue []reflect.Value
}

func createSerializer() *serializer {
	return &serializer{
		queueMapKey:   [][]reflect.Value{},
		queueMapValue: []reflect.Value{},
	}
}

func Serialize(holder interface{}) ([]byte, error) {
	d := createSerializer()

	t := reflect.ValueOf(holder)
	if t.Kind() == reflect.Ptr {
		t = t.Elem()
		if t.Kind() == reflect.Ptr {
			t = t.Elem()
		}
	}

	var err error
	if t.Kind() == reflect.Struct && !isDateTime(t) && !isDateTimeOffset(t) {
		startOffset := uint32(2+t.NumField()) * byte4

		dataPartSize, _ := d.calcSize(t)
		size := startOffset + dataPartSize
		d.create = make([]byte, size)

		err = d.serializeStruct(t, startOffset, size)
	} else {
		size, _ := d.calcSize(t)
		d.create = make([]byte, size)
		_, err = d.serialize(t, 0)
	}

	return d.create, err
}

func (d *serializer) serializeStruct(rv reflect.Value, offset uint32, size uint32) error {
	nf := rv.NumField()
	index := 2 * byte4
	for i := 0; i < nf; i++ {
		s, err := d.serialize(rv.Field(i), offset)
		if err != nil {
			return err
		}

		d.create[index], d.create[index+1], d.create[index+2], d.create[index+3] = byte(offset), byte(offset>>8), byte(offset>>16), byte(offset>>24)
		index += byte4
		offset += s
	}
	// size
	d.create[0], d.create[1], d.create[2], d.create[3] = byte(size), byte(size>>8), byte(size>>16), byte(size>>24)
	// last index
	li := nf - 1
	d.create[4], d.create[5], d.create[6], d.create[7] = byte(li), byte(li>>8), byte(li>>16), byte(li>>24)
	return nil
}

func (d *serializer) isFixedSize(rv reflect.Value) bool {
	ret := false
	switch rv.Kind() {
	case
		reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int,
		reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint,
		reflect.Float32, reflect.Float64,
		reflect.Bool:
		ret = true

	case
		reflect.Struct:
		if isDateTimeOffset(rv) || isDateTime(rv) {
			ret = true
		}

	default:
	}
	return ret
}

func (d *serializer) calcSize(rv reflect.Value) (uint32, error) {
	ret := uint32(0)

	switch rv.Kind() {
	case reflect.Int8:
		ret = byte1

	case reflect.Int16:
		ret = byte2

	case reflect.Int32:
		if isChar(rv) {
			ret = byte2
		} else {
			ret = byte4
		}

	case reflect.Int:
		ret = byte4

	case reflect.Int64:
		if isDuration(rv) {
			ret = byte4 + byte8
		} else {
			ret = byte8
		}

	case reflect.Uint8:
		ret = byte1

	case reflect.Uint16:
		ret = byte2

	case reflect.Uint32, reflect.Uint:
		ret = byte4

	case reflect.Uint64:
		ret = byte8

	case reflect.Float32:
		ret = byte4

	case reflect.Float64:
		ret = byte8

	case reflect.Bool:
		ret = byte1

	case reflect.String:
		l := uint32(rv.Len())
		ret = l + byte4

	case reflect.Array, reflect.Slice:
		l := rv.Len()
		if l > 0 {
			ret += byte4
			isTypeFixed := d.isFixedSize(rv.Index(0))
			if isTypeFixed {
				s, err := d.calcSize(rv.Index(0))
				if err != nil {
					return 0, err
				}
				ret += s * uint32(l)
			} else {
				for i := 0; i < l; i++ {
					s, err := d.calcSize(rv.Index(i))
					if err != nil {
						return 0, err
					}
					ret += s
				}
			}
		} else {
			// only length info
			ret = byte4
		}

	case reflect.Struct:
		if isDateTimeOffset(rv) {
			ret = byte4 + byte8 + byte2
		} else if isDateTime(rv) {
			ret = byte4 + byte8
		} else {
			for i := 0; i < rv.NumField(); i++ {
				s, err := d.calcSize(rv.Field(i))
				if err != nil {
					return 0, err
				}
				ret += s
			}
		}

	case reflect.Map:
		// length
		ret += byte4
		l := uint32(rv.Len())

		if l < 1 {
			return ret, nil
		}
		// check fixed type
		keys := rv.MapKeys()

		d.queueMapKey = append(d.queueMapKey, keys)

		isFixedKey := d.isFixedSize(keys[0])
		if isFixedKey {
			sizeK, err := d.calcSize(keys[0])
			if err != nil {
				return 0, err
			}
			startI := len(d.queueMapValue)
			add := make([]reflect.Value, l)
			d.queueMapValue = append(d.queueMapValue, add...)
			for i, k := range keys {
				value := rv.MapIndex(k)
				d.queueMapValue[startI+i] = value
				sizeV, err := d.calcSize(value)
				if err != nil {
					return 0, err
				}
				ret += sizeK + sizeV
			}

		} else {
			startI := len(d.queueMapValue)
			add := make([]reflect.Value, l)
			d.queueMapValue = append(d.queueMapValue, add...)
			for i, k := range keys {
				sizeK, err := d.calcSize(k)
				if err != nil {
					return 0, err
				}
				value := rv.MapIndex(k)
				d.queueMapValue[startI+i] = value
				sizeV, err := d.calcSize(value)
				if err != nil {
					return 0, err
				}
				ret += sizeK + sizeV
			}
		}

	case reflect.Ptr:
		if rv.IsNil() {
			return 0, errors.New(fmt.Sprint("pointer is null : ", rv.Type()))
		}
		s, err := d.calcSize(rv.Elem())
		if err != nil {
			return 0, err
		}
		ret = s

	default:
		return 0, errors.New(fmt.Sprint("this type is not supported : ", rv.Type()))
	}

	return ret, nil
}

func (d *serializer) serialize(rv reflect.Value, offset uint32) (uint32, error) {
	size := uint32(0)

	switch rv.Kind() {
	case reflect.Int8:
		d.write_s1_i64(rv.Int(), offset)
		size += byte1

	case reflect.Int16:
		d.write_s2_i64(rv.Int(), offset)
		size += byte2

	case reflect.Int32:
		if isChar(rv) {

			// rune [ushort(2)]
			enc := utf16.Encode([]rune{int32(rv.Int())})
			v := enc[0]
			d.create[offset+0] = byte(v)
			d.create[offset+1] = byte(v >> 8)
			size += byte2
		} else {
			d.write_s4_i64(rv.Int(), offset)
			size += byte4
		}

	case reflect.Int:
		d.write_s4_i64(rv.Int(), offset)
		size += byte4

	case reflect.Int64:
		if isDuration(rv) {
			// seconds
			ns := rv.MethodByName("Nanoseconds").Call([]reflect.Value{})[0]
			nanoseconds := ns.Int()
			sec, nsec := nanoseconds/(1000*1000), int64(nanoseconds%(1000*1000))
			d.write_s8_i64(sec, offset)
			size += byte8
			offset += byte8

			// nanos
			d.write_s4_i64(nsec, offset)
			size += byte4
		} else {
			d.write_s8_i64(rv.Int(), offset)
			size += byte8
		}

	case reflect.Uint8:
		d.write_s1_u64(rv.Uint(), offset)
		size += byte1

	case reflect.Uint16:
		d.write_s2_u64(rv.Uint(), offset)
		size += byte2

	case reflect.Uint32, reflect.Uint:
		d.write_s4_u64(rv.Uint(), offset)
		size += byte4

	case reflect.Uint64:
		d.write_s8_u64(rv.Uint(), offset)
		size += byte8

	case reflect.Float32:
		v := math.Float32bits(float32(rv.Float()))
		d.write_s4_u32(v, offset)
		size += byte4

	case reflect.Float64:
		v := math.Float64bits(rv.Float())
		d.write_s8_u64(v, offset)
		size += byte8

	case reflect.Bool:

		if rv.Bool() {
			d.write_s1_u64(0x01, offset)
		} else {
			d.write_s1_u64(0x00, offset)
		}
		size += byte1

	case reflect.String:
		str := rv.String()
		l := uint32(len(str))
		d.write_s4_u32(l, offset)
		size += byte4
		offset += byte4

		// NOTE : unsafe
		strBytes := *(*[]byte)(unsafe.Pointer(&str))
		for i := uint32(0); i < l; i++ {
			d.create[offset+i] = strBytes[i]
		}
		size += l

	case reflect.Array, reflect.Slice:
		l := rv.Len()
		if l > 0 {
			d.write_s4_i(l, offset)
			size += byte4
			offset += byte4

			for i := 0; i < l; i++ {
				s, err := d.serialize(rv.Index(i), offset)
				if err != nil {
					return 0, err
				}
				offset += s
				size += s
			}
		} else {
			// only make length info
			d.write_s4_i(0, offset)
			size += byte4
		}

	case reflect.Struct:
		if isDateTimeOffset(rv) {

			// offset
			rets := rv.MethodByName("Zone").Call([]reflect.Value{})
			_, offSec := rets[0] /*name*/, rets[1].Int() /*offset*/
			offMin := offSec / 60

			// seconds
			rets = rv.MethodByName("Unix").Call([]reflect.Value{})
			seconds := rets[0].Int() + offSec

			// nanos
			nanos := rv.FieldByName("nsec").Int()

			// seconds to byte
			d.write_s8_i64(seconds, offset)
			size += byte8
			offset += byte8

			// nanos to byte
			d.write_s4_i64(nanos, offset)
			size += byte4
			offset += byte4

			// offset to byte
			d.write_s2_i64(offMin, offset)
			size += byte2
		} else if isDateTime(rv) {
			// seconds
			unixTime := rv.MethodByName("Unix").Call([]reflect.Value{})
			sec := unixTime[0].Int()
			d.write_s8_i64(sec, offset)
			size += byte8
			offset += byte8

			// nanos
			nsec := rv.FieldByName("nsec").Int()
			d.write_s4_i64(nsec, offset)
			size += byte4
		} else {
			for i := 0; i < rv.NumField(); i++ {
				s, err := d.serialize(rv.Field(i), offset)
				if err != nil {
					return 0, err
				}
				offset += s
				size += s
			}
		}

	case reflect.Map:
		// length
		d.write_s4_i(rv.Len(), offset)
		size += byte4
		offset += byte4

		keys := d.queueMapKey[0]
		keysLen := len(keys)
		values := d.queueMapValue[:keysLen]

		for i, k := range keys {
			addOffByK, err := d.serialize(k, offset)
			if err != nil {
				return 0, err
			}
			addOffByV, err := d.serialize(values[i], offset+addOffByK)
			if err != nil {
				return 0, err
			}
			offset += addOffByK + addOffByV
			size += addOffByK + addOffByV
		}

		// update queue
		if len(d.queueMapKey) > 0 {
			d.queueMapKey = d.queueMapKey[1:]
			d.queueMapValue = d.queueMapValue[keysLen:]
		} else {
			d.queueMapKey = d.queueMapKey[:0]
			d.queueMapValue = d.queueMapValue[:0]
		}

	case reflect.Ptr:
		if rv.IsNil() {
			return 0, errors.New(fmt.Sprint("pointer is null : ", rv.Type()))
		}
		s, err := d.serialize(rv.Elem(), offset)
		if err != nil {
			return 0, err
		}
		size += s

	default:
		return 0, errors.New(fmt.Sprint("this type is not supported : ", rv.Type()))
	}

	return size, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/termie/go-shutil"
	"os"
	"path/filepath"
	"strings"
	"text/template"
)

type upstartTemplateVars struct {
	PackageName string
	TargetDir   string
}

func buildDebianPackage(pkg *NinjaPackage, ctx *buildContext, arch string) {
	dockerCurr := filepath.Join(ctx.stagingDocker, "debian-"+arch)
	stagingCurr := filepath.Join(ctx.stagingHost, "debian-"+arch)
	os.MkdirAll(stagingCurr, 0755)

	targetDir := "apps"
	if strings.HasPrefix(pkg.ShortName(), "driver-") {
		targetDir = "drivers"
	}

	targetPath := filepath.Join(stagingCurr, "opt", "ninjablocks", targetDir, pkg.ShortName())
	os.MkdirAll(targetPath, 0755)

	// binary itself
	binCurr := filepath.Join(targetPath, pkg.ShortName())
	shutil.Copy(ctx.archBinaries[arch], binCurr, false)

	// package.json
	srcFile := filepath.Join(pkg.BasePath, "package.json")
	pkgCurr := filepath.Join(stagingCurr, "package.json")
	shutil.Copy(srcFile, pkgCurr, true) // don't copy symlink itself, just the real file

	tplData := &upstartTemplateVars{
		PackageName: pkg.ShortName(),
		TargetDir:   targetDir,
	}

	// prepare /etc/init
	initPath := filepath.Join(stagingCurr, "etc", "init")
	os.MkdirAll(initPath, 0755)

	// upstart file
	initConfPath := filepath.Join(initPath, pkg.ShortName()+".conf")
	file, err := os.Create(initConfPath)
	if err != nil {
		panic(err)
	}
	template.Must(template.New("ninjaDebUpstart").Parse(ninjaDebUpstart)).Execute(file, tplData)
	file.Close()

	// post install
	postInstallPath := filepath.Join(ctx.stagingHost, "postinstall-deb-"+arch)
	file, err = os.Create(postInstallPath)
	if err != nil {
		panic(err)
	}
	template.Must(template.New("ninjaDebPostInstall").Parse(ninjaDebPostInstall)).Execute(file, tplData)
	file.Close()

	// prepare the build command
	fpm := []string{
		"fpm", "-s", "dir", "-t", "deb",
		"--deb-compression", "xz",
		"--deb-user", "root",
		"--deb-group", "root",
		"--category", "web",
		"-m", "Ninja CLI Builder <builder@ninjablocks.com>",
		"--url", "http://ninjablocks.com/",
		"-n", pkg.ShortName(),
		"-v", pkg.Version(),
		"--description", pkg.Name(),
		"-C", dockerCurr,
		"-a", arch,
		"-d", "ninjasphere-minimal",
		"-p", filepath.Join(ctx.outputDocker, fmt.Sprintf("%s_%s_%s.deb", pkg.ShortName(), pkg.Version(), arch)),
		"--after-install", filepath.Join(ctx.stagingDocker, "postinstall-deb-"+arch),
		".",
	}

	runNativeDockerCommand(ctx.dockerVolumeArgs, fpm...)
}
<commit_msg>Fix install path for package.json.<commit_after>package main

import (
	"fmt"
	"github.com/termie/go-shutil"
	"os"
	"path/filepath"
	"strings"
	"text/template"
)

type upstartTemplateVars struct {
	PackageName string
	TargetDir   string
}

func buildDebianPackage(pkg *NinjaPackage, ctx *buildContext, arch string) {
	dockerCurr := filepath.Join(ctx.stagingDocker, "debian-"+arch)
	stagingCurr := filepath.Join(ctx.stagingHost, "debian-"+arch)
	os.MkdirAll(stagingCurr, 0755)

	targetDir := "apps"
	if strings.HasPrefix(pkg.ShortName(), "driver-") {
		targetDir = "drivers"
	}

	targetPath := filepath.Join(stagingCurr, "opt", "ninjablocks", targetDir, pkg.ShortName())
	os.MkdirAll(targetPath, 0755)

	// binary itself
	binCurr := filepath.Join(targetPath, pkg.ShortName())
	shutil.Copy(ctx.archBinaries[arch], binCurr, false)

	// package.json
	srcFile := filepath.Join(pkg.BasePath, "package.json")
	pkgCurr := filepath.Join(targetPath, "package.json")
	shutil.Copy(srcFile, pkgCurr, true) // don't copy symlink itself, just the real file

	tplData := &upstartTemplateVars{
		PackageName: pkg.ShortName(),
		TargetDir:   targetDir,
	}

	// prepare /etc/init
	initPath := filepath.Join(stagingCurr, "etc", "init")
	os.MkdirAll(initPath, 0755)

	// upstart file
	initConfPath := filepath.Join(initPath, pkg.ShortName()+".conf")
	file, err := os.Create(initConfPath)
	if err != nil {
		panic(err)
	}
	template.Must(template.New("ninjaDebUpstart").Parse(ninjaDebUpstart)).Execute(file, tplData)
	file.Close()

	// post install
	postInstallPath := filepath.Join(ctx.stagingHost, "postinstall-deb-"+arch)
	file, err = os.Create(postInstallPath)
	if err != nil {
		panic(err)
	}
	template.Must(template.New("ninjaDebPostInstall").Parse(ninjaDebPostInstall)).Execute(file, tplData)
	file.Close()

	// prepare the build command
	fpm := []string{
		"fpm", "-s", "dir", "-t", "deb",
		"--deb-compression", "xz",
		"--deb-user", "root",
		"--deb-group", "root",
		"--category", "web",
		"-m", "Ninja CLI Builder <builder@ninjablocks.com>",
		"--url", "http://ninjablocks.com/",
		"-n", pkg.ShortName(),
		"-v", pkg.Version(),
		"--description", pkg.Name(),
		"-C", dockerCurr,
		"-a", arch,
		"-d", "ninjasphere-minimal",
		"-p", filepath.Join(ctx.outputDocker, fmt.Sprintf("%s_%s_%s.deb", pkg.ShortName(), pkg.Version(), arch)),
		"--after-install", filepath.Join(ctx.stagingDocker, "postinstall-deb-"+arch),
		".",
	}

	runNativeDockerCommand(ctx.dockerVolumeArgs, fpm...)
}
<|endoftext|>


<commit_before>package filter

import (
	"context"
	"fmt"
	"testing"

	"github.com/peco/peco/line"
	"github.com/stretchr/testify/assert"
)

type indexer interface {
	Indices() [][]int
}

// TestFuzzy tests a fuzzy filter against various inputs
func TestFuzzy(t *testing.T) {
	ctx := context.Background()

	testValues := []struct {
		input    string
		query    string
		selected bool
	}{
		{"this is a test to test the fuzzy Filter", "tf", true},  // normal selection
		{"this is a test to test the fuzzy Filter", "wp", false}, // incorrect selection
		{"THIS IS A TEST TO TEST THE FUZZY FILTER", "tu", true},  // case insensitivity
		{"this is a Test to test the fuzzy filter", "Tu", true},  // case sensitivity
		{"this is a Test to test the fUzzy filter", "TU", true},  // case sensitivity
		{"this is a test to test the fuzzy filter", "Tu", false}, // case sensitivity
		{"this is a test to Test the fuzzy filter", "TU", false}, // case sensitivity
		{"日本語は難しいです", "難", true},                                 // kanji
		{"あ、日本語は難しいですよ", "あい", true},                             // hiragana
		{"パソコンは遅いですネ", "ソネ", true},                               // katakana
		{"🚴🏻 abcd efgh", "🚴🏻e", true},                            // unicode
	}
	filter := NewFuzzy()
	for i, v := range testValues {
		t.Run(fmt.Sprintf(`"%s" against "%s", expect "%t"`, v.input, v.query, v.selected), func(t *testing.T) {
			ctx = NewContext(ctx, v.query)
			l := line.NewRaw(uint64(i), v.input, false)
			res, err := filter.Apply(ctx, l)

			if !v.selected {
				if !assert.Error(t, err, "filter should fail") {
					return
				}
				if !assert.Nil(t, res, "return value should be nil") {
					return
				}
				return
			}

			if !assert.NoError(t, err, "filtering failed") {
				return
			}

			if !assert.NotNil(t, res, "return value should NOT be nil") {
				return
			}

			if !assert.Implements(t, (*indexer)(nil), res, "can call Indices()") {
				return
			}
			t.Logf("%#v", res.(indexer).Indices())
		})
	}
}
<commit_msg>what mattn wishes is my command<commit_after>package filter

import (
	"context"
	"fmt"
	"testing"

	"github.com/peco/peco/line"
	"github.com/stretchr/testify/assert"
)

type indexer interface {
	Indices() [][]int
}

// TestFuzzy tests a fuzzy filter against various inputs
func TestFuzzy(t *testing.T) {
	ctx := context.Background()

	testValues := []struct {
		input    string
		query    string
		selected bool
	}{
		{"this is a test to test the fuzzy Filter", "tf", true},  // normal selection
		{"this is a test to test the fuzzy Filter", "wp", false}, // incorrect selection
		{"THIS IS A TEST TO TEST THE FUZZY FILTER", "tu", true},  // case insensitivity
		{"this is a Test to test the fuzzy filter", "Tu", true},  // case sensitivity
		{"this is a Test to test the fUzzy filter", "TU", true},  // case sensitivity
		{"this is a test to test the fuzzy filter", "Tu", false}, // case sensitivity
		{"this is a test to Test the fuzzy filter", "TU", false}, // case sensitivity
		{"日本語は難しいです", "難", true},                                 // kanji
		{"あ、日本語は難しいですよ", "あい", true},                             // hiragana
		{"パソコンは遅いですネ", "ソネ", true},                               // katakana
		{"🚴🏻 abcd efgh", "🚴🏻e", true},                            // unicode
		{"This is a test to Test the fuzzy filteR", "TTR", true},
	}
	filter := NewFuzzy()
	for i, v := range testValues {
		t.Run(fmt.Sprintf(`"%s" against "%s", expect "%t"`, v.input, v.query, v.selected), func(t *testing.T) {
			ctx = NewContext(ctx, v.query)
			l := line.NewRaw(uint64(i), v.input, false)
			res, err := filter.Apply(ctx, l)

			if !v.selected {
				if !assert.Error(t, err, "filter should fail") {
					return
				}
				if !assert.Nil(t, res, "return value should be nil") {
					return
				}
				return
			}

			if !assert.NoError(t, err, "filtering failed") {
				return
			}

			if !assert.NotNil(t, res, "return value should NOT be nil") {
				return
			}

			if !assert.Implements(t, (*indexer)(nil), res, "can call Indices()") {
				return
			}
			t.Logf("%#v", res.(indexer).Indices())
		})
	}
}
<|endoftext|>


<commit_before>package finder

import (
	"fmt"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestTaggedWhere(t *testing.T) {
	assert := assert.New(t)

	table := []struct {
		query    string
		where    string
		prewhere string
		isErr    bool
	}{
		// info about _tag "directory"
		{"seriesByTag('key=value')", "Tag1='key=value'", "", false},
		{"seriesByTag('name=rps')", "Tag1='__name__=rps'", "", false},
		{"seriesByTag('name=~cpu.usage')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=cpu%' AND match(Tag1, '^__name__=.*cpu.usage')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=cpu%' AND match(Tag1, '^__name__=.*cpu.usage')", false},
		{"seriesByTag('name=~cpu|mem')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=.*cpu|mem')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=.*cpu|mem')", false},
		{"seriesByTag('name=~cpu|mem$')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=.*cpu|mem$')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=.*cpu|mem$')", false},
		{"seriesByTag('name=~^cpu|mem')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=cpu|mem')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=cpu|mem')", false},
		{"seriesByTag('name=~^cpu|mem$')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=cpu|mem$')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=cpu|mem$')", false},
		{"seriesByTag('name=rps', 'key=~value')", "(Tag1='__name__=rps') AND (arrayExists((x) -> x='key=value', Tags))", "", false},
		{"seriesByTag('name=rps', 'key=~hello.world')", "(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=hello%' AND match(x, '^key=.*hello.world'), Tags))", "", false},
		{`seriesByTag('cpu=cpu-total','host=~Vladimirs-MacBook-Pro\.local')`, `(Tag1='cpu=cpu-total') AND (arrayExists((x) -> x LIKE 'host=Vladimirs-MacBook-Pro%' AND match(x, '^host=.*Vladimirs-MacBook-Pro\\.local'), Tags))`, "", false},
		// grafana multi-value variable produce this
		{"seriesByTag('name=value','what=*')", "(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%', Tags))", "", false},        // If All masked to value with *
		{"seriesByTag('name=value','what=*x')", "(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%x', Tags))", "", false},      // If All masked to value with *
		{"seriesByTag('name=value','what!=*x')", "(Tag1='__name__=value') AND (NOT arrayExists((x) -> x LIKE 'what=%x', Tags))", "", false}, // If All masked to value with *
		{"seriesByTag('name={avg,max}')", "Tag1 IN ('__name__=avg','__name__=max')", "", false},
		{"seriesByTag('name=m{in,ax}')", "Tag1 IN ('__name__=min','__name__=max')", "", false},
		{"seriesByTag('name=m{in,ax')", "Tag1='__name__=m{in,ax'", "", true},
		{"seriesByTag('name=value','what={avg,max}')", "(Tag1='__name__=value') AND (arrayExists((x) -> x IN ('what=avg','what=max'), Tags))", "", false},
		{"seriesByTag('name=value','what!={avg,max}')", "(Tag1='__name__=value') AND (NOT arrayExists((x) -> x IN ('what=avg','what=max'), Tags))", "", false},
	}

	for _, test := range table {
		testName := fmt.Sprintf("query: %#v", test.query)

		terms, err := ParseSeriesByTag(test.query)

		if !test.isErr {
			assert.NoError(err, testName+", err")
		}

		w, pw, err := TaggedWhere(terms)

		if test.isErr {
			assert.Error(err, testName+", err")
			continue
		} else {
			assert.NoError(err, testName+", err")
		}

		assert.Equal(test.where, w.String(), testName+", where")
		assert.Equal(test.prewhere, pw.String(), testName+", prewhere")
	}
}

func TestParseSeriesByTag(t *testing.T) {
	assert := assert.New(t)

	ok := func(query string, expected []TaggedTerm) {
		p, err := ParseSeriesByTag(query)
		assert.NoError(err)
		assert.Equal(expected, p)
	}

	ok(`seriesByTag('key=value')`, []TaggedTerm{
		TaggedTerm{Op: TaggedTermEq, Key: "key", Value: "value"},
	})

	ok(`seriesByTag('name=rps')`, []TaggedTerm{
		TaggedTerm{Op: TaggedTermEq, Key: "__name__", Value: "rps"},
	})

	ok(`seriesByTag('name=~cpu.usage')`, []TaggedTerm{
		TaggedTerm{Op: TaggedTermMatch, Key: "__name__", Value: "cpu.usage"},
	})

	ok(`seriesByTag('name!=cpu.usage')`, []TaggedTerm{
		TaggedTerm{Op: TaggedTermNe, Key: "__name__", Value: "cpu.usage"},
	})

	ok(`seriesByTag('name!=~cpu.usage')`, []TaggedTerm{
		TaggedTerm{Op: TaggedTermNotMatch, Key: "__name__", Value: "cpu.usage"},
	})

	ok(`seriesByTag('cpu=cpu-total','host=~Vladimirs-MacBook-Pro\.local')`, []TaggedTerm{
		TaggedTerm{Op: TaggedTermEq, Key: "cpu", Value: "cpu-total"},
		TaggedTerm{Op: TaggedTermMatch, Key: "host", Value: `Vladimirs-MacBook-Pro\.local`},
	})

}
<commit_msg>test for tagged match (one item glob)<commit_after>package finder

import (
	"fmt"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestTaggedWhere(t *testing.T) {
	assert := assert.New(t)

	table := []struct {
		query    string
		where    string
		prewhere string
		isErr    bool
	}{
		// info about _tag "directory"
		{"seriesByTag('key=value')", "Tag1='key=value'", "", false},
		{"seriesByTag('name=rps')", "Tag1='__name__=rps'", "", false},
		{"seriesByTag('name=~cpu.usage')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=cpu%' AND match(Tag1, '^__name__=.*cpu.usage')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=cpu%' AND match(Tag1, '^__name__=.*cpu.usage')", false},
		{"seriesByTag('name=~cpu|mem')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=.*cpu|mem')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=.*cpu|mem')", false},
		{"seriesByTag('name=~cpu|mem$')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=.*cpu|mem$')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=.*cpu|mem$')", false},
		{"seriesByTag('name=~^cpu|mem')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=cpu|mem')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=cpu|mem')", false},
		{"seriesByTag('name=~^cpu|mem$')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=cpu|mem$')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=cpu|mem$')", false},
		{"seriesByTag('name=rps', 'key=~value')", "(Tag1='__name__=rps') AND (arrayExists((x) -> x='key=value', Tags))", "", false},
		{"seriesByTag('name=rps', 'key=~hello.world')", "(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=hello%' AND match(x, '^key=.*hello.world'), Tags))", "", false},
		{`seriesByTag('cpu=cpu-total','host=~Vladimirs-MacBook-Pro\.local')`, `(Tag1='cpu=cpu-total') AND (arrayExists((x) -> x LIKE 'host=Vladimirs-MacBook-Pro%' AND match(x, '^host=.*Vladimirs-MacBook-Pro\\.local'), Tags))`, "", false},
		// grafana multi-value variable produce this
		{"seriesByTag('name=value','what=*')", "(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%', Tags))", "", false},        // If All masked to value with *
		{"seriesByTag('name=value','what=*x')", "(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%x', Tags))", "", false},      // If All masked to value with *
		{"seriesByTag('name=value','what!=*x')", "(Tag1='__name__=value') AND (NOT arrayExists((x) -> x LIKE 'what=%x', Tags))", "", false}, // If All masked to value with *
		{"seriesByTag('name={avg,max}')", "Tag1 IN ('__name__=avg','__name__=max')", "", false},
		{"seriesByTag('name=m{in}')", "Tag1='__name__=min'", "", false},
		{"seriesByTag('name=m{in,ax}')", "Tag1 IN ('__name__=min','__name__=max')", "", false},
		{"seriesByTag('name=m{in,ax')", "Tag1='__name__=m{in,ax'", "", true},
		{"seriesByTag('name=value','what={avg,max}')", "(Tag1='__name__=value') AND (arrayExists((x) -> x IN ('what=avg','what=max'), Tags))", "", false},
		{"seriesByTag('name=value','what!={avg,max}')", "(Tag1='__name__=value') AND (NOT arrayExists((x) -> x IN ('what=avg','what=max'), Tags))", "", false},
	}

	for _, test := range table {
		testName := fmt.Sprintf("query: %#v", test.query)

		terms, err := ParseSeriesByTag(test.query)

		if !test.isErr {
			assert.NoError(err, testName+", err")
		}

		w, pw, err := TaggedWhere(terms)

		if test.isErr {
			assert.Error(err, testName+", err")
			continue
		} else {
			assert.NoError(err, testName+", err")
		}

		assert.Equal(test.where, w.String(), testName+", where")
		assert.Equal(test.prewhere, pw.String(), testName+", prewhere")
	}
}

func TestParseSeriesByTag(t *testing.T) {
	assert := assert.New(t)

	ok := func(query string, expected []TaggedTerm) {
		p, err := ParseSeriesByTag(query)
		assert.NoError(err)
		assert.Equal(expected, p)
	}

	ok(`seriesByTag('key=value')`, []TaggedTerm{
		TaggedTerm{Op: TaggedTermEq, Key: "key", Value: "value"},
	})

	ok(`seriesByTag('name=rps')`, []TaggedTerm{
		TaggedTerm{Op: TaggedTermEq, Key: "__name__", Value: "rps"},
	})

	ok(`seriesByTag('name=~cpu.usage')`, []TaggedTerm{
		TaggedTerm{Op: TaggedTermMatch, Key: "__name__", Value: "cpu.usage"},
	})

	ok(`seriesByTag('name!=cpu.usage')`, []TaggedTerm{
		TaggedTerm{Op: TaggedTermNe, Key: "__name__", Value: "cpu.usage"},
	})

	ok(`seriesByTag('name!=~cpu.usage')`, []TaggedTerm{
		TaggedTerm{Op: TaggedTermNotMatch, Key: "__name__", Value: "cpu.usage"},
	})

	ok(`seriesByTag('cpu=cpu-total','host=~Vladimirs-MacBook-Pro\.local')`, []TaggedTerm{
		TaggedTerm{Op: TaggedTermEq, Key: "cpu", Value: "cpu-total"},
		TaggedTerm{Op: TaggedTermMatch, Key: "host", Value: `Vladimirs-MacBook-Pro\.local`},
	})

}
<|endoftext|>


<commit_before>// Copyright 2012 Jesse van den Kieboom. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package configure provides a very simple gnu configure/make style configure
// script generating a simple Makefile and go file containing all the configured
// variables.
package configure

import (
	"bytes"
	"fmt"
	"github.com/jessevdk/go-flags"
	"io"
	"os"
	"path"
	"reflect"
	"regexp"
	"sort"
	"strings"
)

// Options contains all the standard configure options to specify various
// directories. Use NewOptions to create an instance of this type with the
// common default values for each variable.
type Options struct {
	Prefix        string `long:"prefix" description:"install architecture-independent files in PREFIX"`
	ExecPrefix    string `long:"execprefix" description:"install architecture-dependent files in EPREFIX"`
	BinDir        string `long:"bindir" description:"user executables"`
	LibExecDir    string `long:"libexecdir" description:"program executables"`
	SysConfDir    string `long:"sysconfdir" description:"read-only single-machine data"`
	LibDir        string `long:"libdir" description:"program executables"`
	DataRootDir   string `long:"datarootdir" description:"read-only arch.-independent data root"`
	DataDir       string `long:"datadir" description:"read-only arc.-independent data"`
	ManDir        string `long:"mandir" description:"man documentation"`
}

// NewOptions creates a new Options with common default values.
func NewOptions() *Options {
	return &Options{
		Prefix:        "/usr/local",
		ExecPrefix:    "${prefix}",
		BinDir:        "${execprefix}/bin",
		LibExecDir:    "${execprefix}/libexec",
		LibDir:        "${execprefix}/lib",
		SysConfDir:    "${prefix}/etc",
		DataRootDir:   "${prefix}/share",
		DataDir:       "${datarootdir}",
		ManDir:        "${datarootdir}/man",
	}
}

// Package is the package name in which the GoConfig file will be written
var Package = "main"

// Makefile is the filename of the makefile that will be generated
var Makefile = "go.make"

// GoConfig is the filename of the go file that will be generated containing
// all the variable values.
var GoConfig = "appconfig"

// GoConfigVariable is the name of the variable inside the GoConfig file
// containing all the variable values.
var GoConfigVariable = "AppConfig"

// Target is the executable name to build. If left empty, the name is deduced
// from the directory (similar to what go does)
var Target = ""

// Version is the application version
var Version []int = []int{0, 1}

type expandStringPart struct {
	Value      string
	IsVariable bool
}

func (x *expandStringPart) expand(m map[string]*expandString) (string, []string) {
	if x.IsVariable {
		s, ok := m[x.Value]

		if !ok {
			return "", nil
		} else {
			ret := s.expand(m)
			rets := make([]string, len(s.dependencies), len(s.dependencies)+1)

			copy(rets, s.dependencies)

			return ret, append(rets, x.Value)
		}
	}

	return x.Value, nil
}

type expandString struct {
	Name  string
	Parts []expandStringPart

	dependencies []string
	value        string
	hasExpanded  bool
}

func (x *expandString) dependsOn(name string) bool {
	i := sort.SearchStrings(x.dependencies, name)

	return i < len(x.dependencies) && x.dependencies[i] == name
}

func (x *expandString) expand(m map[string]*expandString) string {
	if !x.hasExpanded {
		// Prevent infinite loop by circular dependencies
		x.hasExpanded = true
		buf := bytes.Buffer{}

		for _, v := range x.Parts {
			s, deps := v.expand(m)
			buf.WriteString(s)

			x.dependencies = append(x.dependencies, deps...)
		}

		sort.Strings(x.dependencies)
		x.value = buf.String()
	}

	return x.value
}

// Config represents the current configuration. See Configure for more
// information.
type Config struct {
	*flags.Parser

	values   map[string]interface{}
	expanded map[string]*expandString
}

func (x *Config) extract() map[string]interface{} {
	ret := make(map[string]interface{})

	for _, grp := range x.Parser.Groups {
		for longname, option := range grp.LongNames {
			ret[longname] = option.Value.Interface()
		}
	}

	return ret
}

func (x *Config) expand() map[string]*expandString {
	ret := make(map[string]*expandString)

	r, _ := regexp.Compile(`\$\{[^}]*\}`)

	for name, val := range x.values {
		es := expandString{
			Name: name,
		}

		// Find all variable references
		s := fmt.Sprintf("%v", val)

		matches := r.FindAllStringIndex(s, -1)

		for i, match := range matches {
			var prefix string

			if i == 0 {
				prefix = s[0:match[0]]
			} else {
				prefix = s[matches[i-1][1]:match[0]]
			}

			if len(prefix) != 0 {
				es.Parts = append(es.Parts, expandStringPart{Value: prefix, IsVariable: false})
			}

			varname := s[match[0]+2 : match[1]-1]
			es.Parts = append(es.Parts, expandStringPart{Value: varname, IsVariable: true})
		}

		if len(matches) == 0 {
			es.Parts = append(es.Parts, expandStringPart{Value: s, IsVariable: false})
		} else {
			last := matches[len(matches)-1]
			suffix := s[last[1]:]

			if len(suffix) != 0 {
				es.Parts = append(es.Parts, expandStringPart{Value: suffix, IsVariable: false})
			}
		}

		ret[name] = &es
	}

	for _, val := range ret {
		val.expand(ret)
	}

	return ret
}

// Configure runs the configure process with options as provided by the given
// data variable. If data is nil, the default options will be used
// (see NewOptions). Note that the data provided is simply passed to go-flags.
// For more information on flags parsing, see the documentation of go-flags.
// If GoConfig is not empty, then the go configuration will be written to the
// GoConfig file. Similarly, if Makefile is not empty, the Makefile will be
// written.
func Configure(data interface{}) (*Config, error) {
	if data == nil {
		data = NewOptions()
	}

	parser := flags.NewParser(data, flags.PrintErrors)

	if _, err := parser.Parse(); err != nil {
		return nil, err
	}

	ret := &Config{
		Parser: parser,
	}

	ret.values = ret.extract()
	ret.expanded = ret.expand()

	if len(GoConfig) != 0 {
		filename := GoConfig

		if !strings.HasSuffix(filename, ".go") {
			filename += ".go"
		}

		f, err := os.Create(ret.absPath(filename))

		if err != nil {
			return nil, err
		}

		ret.WriteGoConfig(f)
		f.Close()
	}

	if len(Makefile) != 0 {
		f, err := os.Create(ret.absPath(Makefile))

		if err != nil {
			return nil, err
		}

		ret.WriteMakefile(f)
		f.Close()

		os.Chmod(Makefile, 0755)

		f, err = os.OpenFile(path.Join(path.Dir(Makefile), "Makefile"),
		                     os.O_CREATE | os.O_EXCL | os.O_WRONLY,
		                     0644)

		if err == nil {
			fmt.Fprintf(f, "include %s\n", path.Base(Makefile))
			f.Close()
		}
	}

	return ret, nil
}

// Expand expands the variable value indicated by name
func (x *Config) Expand(name string) string {
	return x.expanded[name].expand(x.expanded)
}

func (x *Config) absPath(filename string) string {
	if path.IsAbs(filename) {
		return filename
	}

	wd, _ := os.Getwd()

	return path.Clean(path.Join(wd, path.Dir(os.Args[0]), filename))
}

// WriteGoConfig writes the go configuration file containing all the variable
// values to the given writer. Note that it will write a package line if
// the Package variable is not empty. The GoConfigVariable name will
// be used as the variable name for the configuration.
func (x *Config) WriteGoConfig(writer io.Writer) {
	if len(Package) > 0 {
		fmt.Fprintf(writer, "package %v\n\n", Package)
	}

	fmt.Fprintf(writer, "var %s = struct {\n", GoConfigVariable)
	values := make([]string, 0)

	variables := make([]string, 0, len(x.values))
	optionmap := make(map[string]*flags.Option)

	// Write all options
	for _, grp := range x.Parser.Groups {
		for _, option := range grp.LongNames {
			name := option.Field.Name

			variables = append(variables, name)
			optionmap[name] = option
		}
	}

	sort.Strings(variables)

	for i, name := range variables {
		if i != 0 {
			io.WriteString(writer, "\n")
		}

		option := optionmap[name]
		val := option.Value.Interface()

		fmt.Fprintf(writer, "\t// %s\n", option.Description)
		fmt.Fprintf(writer, "\t%v %T\n", name, val)

		var value string

		if option.Value.Type().Kind() == reflect.String {
			value = fmt.Sprintf("%#v", x.Expand(option.LongName))
		} else {
			value = fmt.Sprintf("%#v", val)
		}

		values = append(values, value)
	}

	if len(variables) > 0 {
		io.WriteString(writer, "\n")
	}

	io.WriteString(writer, "\t// Application version\n")
	io.WriteString(writer, "\tVersion []int\n")
	fmt.Fprintln(writer, "}{")

	for _, v := range values {
		fmt.Fprintf(writer, "\t%v,\n", v)
	}

	for i, v := range Version {
		if i != 0 {
			io.WriteString(writer, ", ")
		} else {
			io.WriteString(writer, "\t[]int{")
		}

		fmt.Fprintf(writer, "%v", v)
	}

	fmt.Fprintln(writer, "},")
	fmt.Fprintln(writer, "}")
}

// WriteMakefile writes a Makefile for the given parser to the given writer.
// The Makefile contains the common build, clean, distclean, install and
// uninstall rules.
func (x *Config) WriteMakefile(writer io.Writer) {
	// Write a very basic makefile
	io.WriteString(writer, "#!/usr/bin/make -f\n\n")

	vars := make([]*expandString, 0, len(x.expanded))

	for name, v := range x.expanded {
		inserted := false

		// Insert into vars based on dependencies
		for i, vv := range vars {
			if vv.dependsOn(name) {
				tail := vars[i:]

				if i == 0 {
					vars = append([]*expandString{v}, vars...)
				} else {
					vars = append(append(vars[0:i-1], v), tail...)
				}

				inserted = true
				break
			}
		}

		if !inserted {
			vars = append(vars, v)
		}
	}

	if len(vars) > 0 {
		io.WriteString(writer, "# Variables\n")

		for _, v := range vars {
			fmt.Fprintf(writer, "%s = ", v.Name)

			for _, part := range v.Parts {
				if part.IsVariable {
					fmt.Fprintf(writer, "$(%s)", part.Value)
				} else {
					fmt.Fprintf(writer, "%s", part.Value)
				}
			}

			io.WriteString(writer, "\n")
		}

		io.WriteString(writer, "\n")
	}

	target := Target

	if len(target) == 0 {
		arg := os.Args[0]
		var rpath string

		if path.IsAbs(arg) {
			rpath = arg
		} else {
			wd, _ := os.Getwd()
			rpath = path.Clean(path.Join(wd, os.Args[0]))
		}

		target = path.Base(path.Dir(rpath))
	}

	fmt.Fprintf(writer, "TARGET = %s\n", target)
	fmt.Fprintf(writer, "\nSOURCES = $(wildcard *.go)\n\n")

	io.WriteString(writer, "# Rules\n")
	io.WriteString(writer, "$(TARGET): $(SOURCES)\n")
	io.WriteString(writer, "\tgo build -o $(TARGET) $(SOURCES)\n\n")

	io.WriteString(writer, "clean:\n")
	io.WriteString(writer, "\trm -f $(TARGET)\n\n")

	io.WriteString(writer, "distclean: clean\n\n")

	io.WriteString(writer, "install: $(TARGET)\n")
	io.WriteString(writer, "\tmkdir -p $(DESTDIR)$(bindir) && cp $(TARGET) $(DESTDIR)$(bindir)/$(TARGET)\n\n")

	io.WriteString(writer, "uninstall:\n")
	io.WriteString(writer, "\trm -f $(DESTDIR)$(bindir)/$(TARGET)\n\n")

	io.WriteString(writer, ".PHONY: install uninstall distclean clean")
}
<commit_msg>Use relative path for generating makefile and go config<commit_after>// Copyright 2012 Jesse van den Kieboom. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package configure provides a very simple gnu configure/make style configure
// script generating a simple Makefile and go file containing all the configured
// variables.
package configure

import (
	"bytes"
	"fmt"
	"github.com/jessevdk/go-flags"
	"io"
	"os"
	"path"
	"reflect"
	"regexp"
	"sort"
	"strings"
)

// Options contains all the standard configure options to specify various
// directories. Use NewOptions to create an instance of this type with the
// common default values for each variable.
type Options struct {
	Prefix        string `long:"prefix" description:"install architecture-independent files in PREFIX"`
	ExecPrefix    string `long:"execprefix" description:"install architecture-dependent files in EPREFIX"`
	BinDir        string `long:"bindir" description:"user executables"`
	LibExecDir    string `long:"libexecdir" description:"program executables"`
	SysConfDir    string `long:"sysconfdir" description:"read-only single-machine data"`
	LibDir        string `long:"libdir" description:"program executables"`
	DataRootDir   string `long:"datarootdir" description:"read-only arch.-independent data root"`
	DataDir       string `long:"datadir" description:"read-only arc.-independent data"`
	ManDir        string `long:"mandir" description:"man documentation"`
}

// NewOptions creates a new Options with common default values.
func NewOptions() *Options {
	return &Options{
		Prefix:        "/usr/local",
		ExecPrefix:    "${prefix}",
		BinDir:        "${execprefix}/bin",
		LibExecDir:    "${execprefix}/libexec",
		LibDir:        "${execprefix}/lib",
		SysConfDir:    "${prefix}/etc",
		DataRootDir:   "${prefix}/share",
		DataDir:       "${datarootdir}",
		ManDir:        "${datarootdir}/man",
	}
}

// Package is the package name in which the GoConfig file will be written
var Package = "main"

// Makefile is the filename of the makefile that will be generated
var Makefile = "go.make"

// GoConfig is the filename of the go file that will be generated containing
// all the variable values.
var GoConfig = "appconfig"

// GoConfigVariable is the name of the variable inside the GoConfig file
// containing all the variable values.
var GoConfigVariable = "AppConfig"

// Target is the executable name to build. If left empty, the name is deduced
// from the directory (similar to what go does)
var Target = ""

// Version is the application version
var Version []int = []int{0, 1}

type expandStringPart struct {
	Value      string
	IsVariable bool
}

func (x *expandStringPart) expand(m map[string]*expandString) (string, []string) {
	if x.IsVariable {
		s, ok := m[x.Value]

		if !ok {
			return "", nil
		} else {
			ret := s.expand(m)
			rets := make([]string, len(s.dependencies), len(s.dependencies)+1)

			copy(rets, s.dependencies)

			return ret, append(rets, x.Value)
		}
	}

	return x.Value, nil
}

type expandString struct {
	Name  string
	Parts []expandStringPart

	dependencies []string
	value        string
	hasExpanded  bool
}

func (x *expandString) dependsOn(name string) bool {
	i := sort.SearchStrings(x.dependencies, name)

	return i < len(x.dependencies) && x.dependencies[i] == name
}

func (x *expandString) expand(m map[string]*expandString) string {
	if !x.hasExpanded {
		// Prevent infinite loop by circular dependencies
		x.hasExpanded = true
		buf := bytes.Buffer{}

		for _, v := range x.Parts {
			s, deps := v.expand(m)
			buf.WriteString(s)

			x.dependencies = append(x.dependencies, deps...)
		}

		sort.Strings(x.dependencies)
		x.value = buf.String()
	}

	return x.value
}

// Config represents the current configuration. See Configure for more
// information.
type Config struct {
	*flags.Parser

	values   map[string]interface{}
	expanded map[string]*expandString
}

func (x *Config) extract() map[string]interface{} {
	ret := make(map[string]interface{})

	for _, grp := range x.Parser.Groups {
		for longname, option := range grp.LongNames {
			ret[longname] = option.Value.Interface()
		}
	}

	return ret
}

func (x *Config) expand() map[string]*expandString {
	ret := make(map[string]*expandString)

	r, _ := regexp.Compile(`\$\{[^}]*\}`)

	for name, val := range x.values {
		es := expandString{
			Name: name,
		}

		// Find all variable references
		s := fmt.Sprintf("%v", val)

		matches := r.FindAllStringIndex(s, -1)

		for i, match := range matches {
			var prefix string

			if i == 0 {
				prefix = s[0:match[0]]
			} else {
				prefix = s[matches[i-1][1]:match[0]]
			}

			if len(prefix) != 0 {
				es.Parts = append(es.Parts, expandStringPart{Value: prefix, IsVariable: false})
			}

			varname := s[match[0]+2 : match[1]-1]
			es.Parts = append(es.Parts, expandStringPart{Value: varname, IsVariable: true})
		}

		if len(matches) == 0 {
			es.Parts = append(es.Parts, expandStringPart{Value: s, IsVariable: false})
		} else {
			last := matches[len(matches)-1]
			suffix := s[last[1]:]

			if len(suffix) != 0 {
				es.Parts = append(es.Parts, expandStringPart{Value: suffix, IsVariable: false})
			}
		}

		ret[name] = &es
	}

	for _, val := range ret {
		val.expand(ret)
	}

	return ret
}

// Configure runs the configure process with options as provided by the given
// data variable. If data is nil, the default options will be used
// (see NewOptions). Note that the data provided is simply passed to go-flags.
// For more information on flags parsing, see the documentation of go-flags.
// If GoConfig is not empty, then the go configuration will be written to the
// GoConfig file. Similarly, if Makefile is not empty, the Makefile will be
// written.
func Configure(data interface{}) (*Config, error) {
	if data == nil {
		data = NewOptions()
	}

	parser := flags.NewParser(data, flags.PrintErrors)

	if _, err := parser.Parse(); err != nil {
		return nil, err
	}

	ret := &Config{
		Parser: parser,
	}

	ret.values = ret.extract()
	ret.expanded = ret.expand()

	if len(GoConfig) != 0 {
		filename := GoConfig

		if !strings.HasSuffix(filename, ".go") {
			filename += ".go"
		}

		f, err := os.Create(filename)

		if err != nil {
			return nil, err
		}

		ret.WriteGoConfig(f)
		f.Close()
	}

	if len(Makefile) != 0 {
		f, err := os.Create(Makefile)

		if err != nil {
			return nil, err
		}

		ret.WriteMakefile(f)
		f.Close()

		os.Chmod(Makefile, 0755)

		f, err = os.OpenFile(path.Join(path.Dir(Makefile), "Makefile"),
		                     os.O_CREATE | os.O_EXCL | os.O_WRONLY,
		                     0644)

		if err == nil {
			fmt.Fprintf(f, "include %s\n", path.Base(Makefile))
			f.Close()
		}
	}

	return ret, nil
}

// Expand expands the variable value indicated by name
func (x *Config) Expand(name string) string {
	return x.expanded[name].expand(x.expanded)
}

// WriteGoConfig writes the go configuration file containing all the variable
// values to the given writer. Note that it will write a package line if
// the Package variable is not empty. The GoConfigVariable name will
// be used as the variable name for the configuration.
func (x *Config) WriteGoConfig(writer io.Writer) {
	if len(Package) > 0 {
		fmt.Fprintf(writer, "package %v\n\n", Package)
	}

	fmt.Fprintf(writer, "var %s = struct {\n", GoConfigVariable)
	values := make([]string, 0)

	variables := make([]string, 0, len(x.values))
	optionmap := make(map[string]*flags.Option)

	// Write all options
	for _, grp := range x.Parser.Groups {
		for _, option := range grp.LongNames {
			name := option.Field.Name

			variables = append(variables, name)
			optionmap[name] = option
		}
	}

	sort.Strings(variables)

	for i, name := range variables {
		if i != 0 {
			io.WriteString(writer, "\n")
		}

		option := optionmap[name]
		val := option.Value.Interface()

		fmt.Fprintf(writer, "\t// %s\n", option.Description)
		fmt.Fprintf(writer, "\t%v %T\n", name, val)

		var value string

		if option.Value.Type().Kind() == reflect.String {
			value = fmt.Sprintf("%#v", x.Expand(option.LongName))
		} else {
			value = fmt.Sprintf("%#v", val)
		}

		values = append(values, value)
	}

	if len(variables) > 0 {
		io.WriteString(writer, "\n")
	}

	io.WriteString(writer, "\t// Application version\n")
	io.WriteString(writer, "\tVersion []int\n")
	fmt.Fprintln(writer, "}{")

	for _, v := range values {
		fmt.Fprintf(writer, "\t%v,\n", v)
	}

	for i, v := range Version {
		if i != 0 {
			io.WriteString(writer, ", ")
		} else {
			io.WriteString(writer, "\t[]int{")
		}

		fmt.Fprintf(writer, "%v", v)
	}

	fmt.Fprintln(writer, "},")
	fmt.Fprintln(writer, "}")
}

// WriteMakefile writes a Makefile for the given parser to the given writer.
// The Makefile contains the common build, clean, distclean, install and
// uninstall rules.
func (x *Config) WriteMakefile(writer io.Writer) {
	// Write a very basic makefile
	io.WriteString(writer, "#!/usr/bin/make -f\n\n")

	vars := make([]*expandString, 0, len(x.expanded))

	for name, v := range x.expanded {
		inserted := false

		// Insert into vars based on dependencies
		for i, vv := range vars {
			if vv.dependsOn(name) {
				tail := vars[i:]

				if i == 0 {
					vars = append([]*expandString{v}, vars...)
				} else {
					vars = append(append(vars[0:i-1], v), tail...)
				}

				inserted = true
				break
			}
		}

		if !inserted {
			vars = append(vars, v)
		}
	}

	if len(vars) > 0 {
		io.WriteString(writer, "# Variables\n")

		for _, v := range vars {
			fmt.Fprintf(writer, "%s = ", v.Name)

			for _, part := range v.Parts {
				if part.IsVariable {
					fmt.Fprintf(writer, "$(%s)", part.Value)
				} else {
					fmt.Fprintf(writer, "%s", part.Value)
				}
			}

			io.WriteString(writer, "\n")
		}

		io.WriteString(writer, "\n")
	}

	target := Target

	if len(target) == 0 {
		arg := os.Args[0]
		var rpath string

		if path.IsAbs(arg) {
			rpath = arg
		} else {
			wd, _ := os.Getwd()
			rpath = path.Clean(path.Join(wd, os.Args[0]))
		}

		target = path.Base(path.Dir(rpath))
	}

	fmt.Fprintf(writer, "TARGET = %s\n", target)
	fmt.Fprintf(writer, "\nSOURCES = $(wildcard *.go)\n\n")

	io.WriteString(writer, "# Rules\n")
	io.WriteString(writer, "$(TARGET): $(SOURCES)\n")
	io.WriteString(writer, "\tgo build -o $(TARGET) $(SOURCES)\n\n")

	io.WriteString(writer, "clean:\n")
	io.WriteString(writer, "\trm -f $(TARGET)\n\n")

	io.WriteString(writer, "distclean: clean\n\n")

	io.WriteString(writer, "install: $(TARGET)\n")
	io.WriteString(writer, "\tmkdir -p $(DESTDIR)$(bindir) && cp $(TARGET) $(DESTDIR)$(bindir)/$(TARGET)\n\n")

	io.WriteString(writer, "uninstall:\n")
	io.WriteString(writer, "\trm -f $(DESTDIR)$(bindir)/$(TARGET)\n\n")

	io.WriteString(writer, ".PHONY: install uninstall distclean clean")
}
<|endoftext|>


<commit_before>package lnwallet

import (
	"sync"

	"github.com/btcsuite/btcd/btcec"
	"github.com/btcsuite/btcd/wire"
	"github.com/btcsuite/btcutil"
)

// ChannelReservation...
type ChannelReservation struct {
	sync.RWMutex // All fields below owned by the lnwallet.

	//for CLTV it is nLockTime, for CSV it's nSequence, for segwit it's not needed
	fundingLockTime int64
	fundingAmount   btcutil.Amount

	//Current state of the channel, progesses through until complete
	//Makes sure we can't go backwards and only accept messages once
	channelState uint8

	theirInputs []*wire.TxIn
	ourInputs   []*wire.TxIn

	// NOTE(j): FundRequest assumes there is only one change (see ChangePkScript)
	theirChange []*wire.TxOut
	ourChange   []*wire.TxOut

	theirMultiSigKey *btcec.PublicKey

	// In order of sorted inputs. Sorting is done in accordance
	// to BIP-69: https://github.com/bitcoin/bips/blob/master/bip-0069.mediawiki.
	ourFundingSigs   [][]byte
	theirFundingSigs [][]byte

	ourRevokeHash    [wire.HashSize]byte
	ourCommitmentSig []byte

	partialState *OpenChannelState

	reservationID uint64
	wallet        *LightningWallet

	chanOpen chan *LightningChannel
}

// newChannelReservation...
func newChannelReservation(t FundingType, fundingAmt btcutil.Amount,
	minFeeRate btcutil.Amount, wallet *LightningWallet, id uint64) *ChannelReservation {
	return &ChannelReservation{
		fundingAmount: fundingAmt,
		// TODO(roasbeef): assumes balanced symmetric channels.
		partialState: &OpenChannelState{
			capacity:    fundingAmt * 2,
			fundingType: t,
		},
		wallet:        wallet,
		reservationID: id,
	}
}

/*//FundRequest serialize
//(reading from ChannelReservation directly to reduce the amount of copies)
//We can move this stuff to another file too if it's too big...
func (r *ChannelReservation) SerializeFundRequest() ([]byte, error) {
	var err error

	//Buffer to dump in the serialized data
	b := new(bytes.Buffer)

	//Fund Request
	err = b.WriteByte(0x30)
	if err != nil {
		return nil, err
	}

	//ChannelType (1)
	//Default to current type
	err = b.WriteByte(uint8(0))
	if err != nil {
		return nil, err
	}

	//RequesterFundingAmount - The amount we are going to fund (8)
	//check for positive values
	err = binary.Write(b, binary.BigEndian, r.FundingAmount)
	if err != nil {
		return nil, err
	}

	//RequesterChannelMinCapacity (8)
	//The amount needed to accept and sign the channel commit later
	err = binary.Write(b, binary.BigEndian, r.MinTotalFundingAmount)
	if err != nil {
		return nil, err
	}

	//RevocationHash (20)
	//Our revocation hash being contributed (for CLTV/CSV)
	_, err = b.Write(btcutil.Hash160(r.ourRevocation))
	if err != nil {
		return nil, err
	}

	//CommitPubkey (33)
	//Our public key being used for the commitment
	ourPubKey := r.ourKey.PubKey().SerializeCompressed()
	if len(ourPubKey) != 33 { //validation, can remove later? (NO UNCOMPRESSED KEYS!)
		return nil, fmt.Errorf("Serialize FundReq: our Pubkey length incorrect")
	}
	_, err = b.Write(ourPubKey)
	if err != nil {
		return nil, err
	}

	//DeliveryPkHash (20)
	//For now it's a P2PKH, but we will add an extra byte later for the
	//option for P2SH
	//This is the address to send funds to when complete or refunded
	_, err = b.Write(r.ourDeliveryAddress)
	if err != nil {
		return nil, err
	}

	//ReserveAmount (8)
	//Our own reserve amount
	err = binary.Write(b, binary.BigEndian, r.ReserveAmount)
	if err != nil {
		return nil, err
	}

	//Minimum transaction fee per kb (8)
	err = binary.Write(b, binary.BigEndian, r.MinFeePerKb)
	if err != nil {
		return nil, err
	}

	//LockTime (4)
	err = binary.Write(b, binary.BigEndian, r.FundingLockTime)
	if err != nil {
		return nil, err
	}

	//Fee payer (default to split currently) (1)
	err = binary.Write(b, binary.BigEndian, 0)
	if err != nil {
		return nil, err
	}

	//ChangePkScript
	//Length (1)
	changeScriptLength := len(r.ourChange[0].PkScript)
	if changeScriptLength > 255 {
		return nil, fmt.Errorf("Your changeScriptLength is too long!")
	}
	err = binary.Write(b, binary.BigEndian, uint8(changeScriptLength))
	if err != nil {
		return nil, err
	}
	//For now it's a P2PKH, but we will add an extra byte later for the
	//option for P2SH
	//This is the address to send change to (only allow one)
	//ChangePkScript (length of script)
	_, err = b.Write(r.ourChange[0].PkScript)
	if err != nil {
		return nil, err
	}

	//Append the unsigned(!!) txins
	//First one byte for the amount of txins (1)
	if len(r.ourInputs) > 127 {
		return nil, fmt.Errorf("Too many txins")
	}
	err = b.WriteByte(uint8(len(r.ourInputs)))
	if err != nil {
		return nil, err
	}
	//Append the actual Txins (NumOfTxins * 36)
	//Do not include the sequence number to eliminate funny business
	for _, in := range r.ourInputs {
		//Hash
		_, err = b.Write(in.PreviousOutPoint.Hash.Bytes())
		if err != nil {
			return nil, err
		}
		//Index
		err = binary.Write(b, binary.BigEndian, in.PreviousOutPoint.Index)
		if err != nil {
			return nil, err
		}
	}

	return b.Bytes(), err
}

func (r *ChannelReservation) DeserializeFundRequest(wireMsg []byte) error {
	//Make sure we're not overwriting stuff...
	//Update the channelState to 1 before progressing if you want to re-do it.
	//Assumes only one thread is writing at a time
	if r.channelState > 1 {
		return fmt.Errorf("FundRequest: Channel State Mismatch")
	}

	var err error

	b := bytes.NewBuffer(wireMsg)
	msgid, _ := b.ReadByte()
	if msgid != 0x30 {
		return fmt.Errorf("Cannot deserialize: not a funding request")
	}

	if err != nil {
		return err
	}
	//Update the channel state as complete
	r.channelState = 2

	return nil
}

//Validation on the data being supplied from the fund request
func (r *ChannelReservation) ValidateFundRequest(wireMsg []byte) error {
	return nil
}

//Serialize CSV Revocation
//After the Commitment Transaction has been created, send a message to revoke this tx
func (r *ChannelReservation) SerializeCSVRefundRevocation() ([]byte, error) {
	return nil, nil
}

//Deserialize CSV Revocation
//Validate the revocation, after this step, the channel is fully set up
func (r *ChannelReservation) DeserializeCSVRefundRevocation() error {
	return nil
}*/

// OurFunds...
func (r *ChannelReservation) OurFunds() ([]*wire.TxIn, []*wire.TxOut) {
	r.RLock()
	defer r.RUnlock()
	return r.ourInputs, r.ourChange
}

func (r *ChannelReservation) OurKeys() (*btcec.PrivateKey, *btcec.PrivateKey) {
	r.RLock()
	defer r.RUnlock()
	return r.partialState.multiSigKey, r.partialState.ourCommitKey
}

// AddContribution...
func (r *ChannelReservation) AddContribution(theirInputs []*wire.TxIn,
	theirChangeOutputs []*wire.TxOut, commitKey, multiSigKey *btcec.PublicKey,
	deliveryAddress btcutil.Address, initialRevocation [wire.HashSize]byte,
	acceptedDelayPeriod int64) error {

	errChan := make(chan error, 1)

	r.wallet.msgChan <- &addContributionMsg{
		pendingFundingID:   r.reservationID,
		theirInputs:        theirInputs,
		theirChangeOutputs: theirChangeOutputs,
		theirMultiSigKey:   multiSigKey,
		theirCommitKey:     commitKey,
		deliveryAddress:    deliveryAddress,
		revocationHash:     initialRevocation,
		csvDelay:           acceptedDelayPeriod,
		err:                errChan,
	}

	return <-errChan
}

// OurFundingSigs...
func (r *ChannelReservation) OurFundingSigs() [][]byte {
	r.RLock()
	defer r.RUnlock()
	return r.ourFundingSigs
}

// OurCommitmentSig
func (r *ChannelReservation) OurCommitmentSig() []byte {
	r.RLock()
	defer r.RUnlock()
	return r.ourCommitmentSig
}

// TheirFunds...
// TODO(roasbeef): return error if accessors not yet populated?
func (r *ChannelReservation) TheirFunds() ([]*wire.TxIn, []*wire.TxOut) {
	r.RLock()
	defer r.RUnlock()
	return r.theirInputs, r.theirChange
}

func (r *ChannelReservation) TheirKeys() (*btcec.PublicKey, *btcec.PublicKey) {
	r.RLock()
	defer r.RUnlock()
	return r.theirMultiSigKey, r.partialState.theirCommitKey
}

// CompleteFundingReservation...
// TODO(roasbeef): add commit sig also
func (r *ChannelReservation) CompleteReservation(fundingSigs [][]byte, commitmentSig []byte) error {
	errChan := make(chan error, 1)

	r.wallet.msgChan <- &addCounterPartySigsMsg{
		pendingFundingID:   r.reservationID,
		theirFundingSigs:   fundingSigs,
		theirCommitmentSig: commitmentSig,
		err:                errChan,
	}

	return <-errChan
}

// FinalFundingTransaction...
func (r *ChannelReservation) FundingTx() *wire.MsgTx {
	r.RLock()
	defer r.RUnlock()
	return r.partialState.fundingTx
}

// RequestFundingReserveCancellation...
// TODO(roasbeef): also return mutated state?
func (r *ChannelReservation) Cancel() error {
	errChan := make(chan error, 1)
	r.wallet.msgChan <- &fundingReserveCancelMsg{
		pendingFundingID: r.reservationID,
		err:              errChan,
	}

	return <-errChan
}

// WaitForChannelOpen...
func (r *ChannelReservation) WaitForChannelOpen() *LightningChannel {
	return nil
}

// * finish reset of tests
// * comment out stuff that'll need a node.
// * start on commitment side
//   * implement rusty's shachain
//   * set up logic to get notification from node when funding tx gets 6 deep.
//     * prob spawn into ChainNotifier struct
//   * create builder for initial funding transaction
//     * fascade through the wallet, for signing and such.
//   * channel should have active namespace to it's bucket, query at that point fo past commits etc
<commit_msg>lnwallet: add ChannelContribution, finalize ChannelReservation API/workflow<commit_after>package lnwallet

import (
	"sync"

	"github.com/btcsuite/btcd/btcec"
	"github.com/btcsuite/btcd/wire"
	"github.com/btcsuite/btcutil"
)

// ChannelContribution...
type ChannelContribution struct {
	// Amount of funds contributed to the funding transaction.
	FundingAmount btcutil.Amount

	// Inputs to the funding transaction.
	Inputs []*wire.TxIn

	// Outputs to be used in the case that the total value of the fund
	// ing inputs is greather than the total potential channel capacity.
	ChangeOutputs []*wire.TxOut

	// The key to be used for the funding transaction's P2SH multi-sig
	// 2-of-2 output.
	MultiSigKey *btcec.PublicKey

	// The key to be used for this party's version of the commitment
	// transaction.
	CommitKey *btcec.PublicKey

	// Address to be used for delivery of cleared channel funds in the scenario
	// of a cooperative channel closure.
	DeliveryAddress btcutil.Address

	// Hash to be used as the revocation for the initial version of this
	// party's commitment transaction.
	RevocationHash [wire.HashSize]byte

	// The delay (in blocks) to be used for the pay-to-self output in this
	// party's version of the commitment transaction.
	CsvDelay int64
}

// ChannelReservation...
type ChannelReservation struct {
	fundingType FundingType

	// This mutex MUST be held when either reading or modifying any of the
	// fields below.
	sync.RWMutex

	// For CLTV it is nLockTime, for CSV it's nSequence, for segwit it's
	// not needed
	fundingLockTime int64

	// In order of sorted inputs. Sorting is done in accordance
	// to BIP-69: https://github.com/bitcoin/bips/blob/master/bip-0069.mediawiki.
	ourFundingSigs   [][]byte
	theirFundingSigs [][]byte

	// Our signature for their version of the commitment transaction.
	ourCommitmentSig []byte

	ourContribution   *ChannelContribution
	theirContribution *ChannelContribution

	partialState *OpenChannelState

	// The ID of this reservation, used to uniquely track the reservation
	// throughout its lifetime.
	reservationID uint64

	// A channel which will be sent on once the channel is considered
	// 'open'. A channel is open once the funding transaction has reached
	// a sufficient number of confirmations.
	chanOpen chan *LightningChannel

	wallet *LightningWallet
}

// newChannelReservation...
func newChannelReservation(t FundingType, fundingAmt btcutil.Amount,
	minFeeRate btcutil.Amount, wallet *LightningWallet, id uint64) *ChannelReservation {
	// TODO(roasbeef): CSV here, or on delay?
	return &ChannelReservation{
		fundingType: t,
		ourContribution: &ChannelContribution{
			FundingAmount: fundingAmt,
		},
		theirContribution: &ChannelContribution{
			FundingAmount: fundingAmt,
		},
		partialState: &OpenChannelState{
			// TODO(roasbeef): assumes balanced symmetric channels.
			capacity:    fundingAmt * 2,
			minFeePerKb: minFeeRate,
		},
		reservationID: id,
		wallet:        wallet,
	}
}

// OurContribution...
// NOTE: This SHOULD NOT be modified.
func (r *ChannelReservation) OurContribution() *ChannelContribution {
	r.RLock()
	defer r.RUnlock()
	return r.ourContribution
}

// ProcessContribution...
func (r *ChannelReservation) ProcessContribution(theirContribution *ChannelContribution) error {
	errChan := make(chan error, 1)

	r.wallet.msgChan <- &addContributionMsg{
		pendingFundingID: r.reservationID,
		contribution:     theirContribution,
		err:              errChan,
	}

	return <-errChan
}

func (r *ChannelReservation) TheirContribution() *ChannelContribution {
	r.RLock()
	defer r.RUnlock()
	return r.theirContribution
}

// OurSignatures...
func (r *ChannelReservation) OurSignatures() ([][]byte, []byte) {
	r.RLock()
	defer r.RUnlock()
	return r.ourFundingSigs, r.ourCommitmentSig
}

// CompleteFundingReservation...
// TODO(roasbeef): add commit sig also
func (r *ChannelReservation) CompleteReservation(fundingSigs [][]byte, commitmentSig []byte) error {
	errChan := make(chan error, 1)

	r.wallet.msgChan <- &addCounterPartySigsMsg{
		pendingFundingID:   r.reservationID,
		theirFundingSigs:   fundingSigs,
		theirCommitmentSig: commitmentSig,
		err:                errChan,
	}

	return <-errChan
}

// OurSignatures...
func (r *ChannelReservation) TheirSignatures() ([][]byte, []byte) {
	r.RLock()
	defer r.RUnlock()
	return r.theirFundingSigs, r.partialState.theirCommitSig
}

// FinalFundingTransaction...
func (r *ChannelReservation) FinalFundingTx() *wire.MsgTx {
	r.RLock()
	defer r.RUnlock()
	return r.partialState.fundingTx
}

// RequestFundingReserveCancellation...
// TODO(roasbeef): also return mutated state?
func (r *ChannelReservation) Cancel() error {
	errChan := make(chan error, 1)
	r.wallet.msgChan <- &fundingReserveCancelMsg{
		pendingFundingID: r.reservationID,
		err:              errChan,
	}

	return <-errChan
}

// WaitForChannelOpen...
func (r *ChannelReservation) WaitForChannelOpen() *LightningChannel {
	return nil
}

/*//FundRequest serialize
//(reading from ChannelReservation directly to reduce the amount of copies)
//We can move this stuff to another file too if it's too big...
func (r *ChannelReservation) SerializeFundRequest() ([]byte, error) {
	var err error

	//Buffer to dump in the serialized data
	b := new(bytes.Buffer)

	//Fund Request
	err = b.WriteByte(0x30)
	if err != nil {
		return nil, err
	}

	//ChannelType (1)
	//Default to current type
	err = b.WriteByte(uint8(0))
	if err != nil {
		return nil, err
	}

	//RequesterFundingAmount - The amount we are going to fund (8)
	//check for positive values
	err = binary.Write(b, binary.BigEndian, r.FundingAmount)
	if err != nil {
		return nil, err
	}

	//RequesterChannelMinCapacity (8)
	//The amount needed to accept and sign the channel commit later
	err = binary.Write(b, binary.BigEndian, r.MinTotalFundingAmount)
	if err != nil {
		return nil, err
	}

	//RevocationHash (20)
	//Our revocation hash being contributed (for CLTV/CSV)
	_, err = b.Write(btcutil.Hash160(r.ourRevocation))
	if err != nil {
		return nil, err
	}

	//CommitPubkey (33)
	//Our public key being used for the commitment
	ourPubKey := r.ourKey.PubKey().SerializeCompressed()
	if len(ourPubKey) != 33 { //validation, can remove later? (NO UNCOMPRESSED KEYS!)
		return nil, fmt.Errorf("Serialize FundReq: our Pubkey length incorrect")
	}
	_, err = b.Write(ourPubKey)
	if err != nil {
		return nil, err
	}

	//DeliveryPkHash (20)
	//For now it's a P2PKH, but we will add an extra byte later for the
	//option for P2SH
	//This is the address to send funds to when complete or refunded
	_, err = b.Write(r.ourDeliveryAddress)
	if err != nil {
		return nil, err
	}

	//ReserveAmount (8)
	//Our own reserve amount
	err = binary.Write(b, binary.BigEndian, r.ReserveAmount)
	if err != nil {
		return nil, err
	}

	//Minimum transaction fee per kb (8)
	err = binary.Write(b, binary.BigEndian, r.MinFeePerKb)
	if err != nil {
		return nil, err
	}

	//LockTime (4)
	err = binary.Write(b, binary.BigEndian, r.FundingLockTime)
	if err != nil {
		return nil, err
	}

	//Fee payer (default to split currently) (1)
	err = binary.Write(b, binary.BigEndian, 0)
	if err != nil {
		return nil, err
	}

	//ChangePkScript
	//Length (1)
	changeScriptLength := len(r.ourChange[0].PkScript)
	if changeScriptLength > 255 {
		return nil, fmt.Errorf("Your changeScriptLength is too long!")
	}
	err = binary.Write(b, binary.BigEndian, uint8(changeScriptLength))
	if err != nil {
		return nil, err
	}
	//For now it's a P2PKH, but we will add an extra byte later for the
	//option for P2SH
	//This is the address to send change to (only allow one)
	//ChangePkScript (length of script)
	_, err = b.Write(r.ourChange[0].PkScript)
	if err != nil {
		return nil, err
	}

	//Append the unsigned(!!) txins
	//First one byte for the amount of txins (1)
	if len(r.ourInputs) > 127 {
		return nil, fmt.Errorf("Too many txins")
	}
	err = b.WriteByte(uint8(len(r.ourInputs)))
	if err != nil {
		return nil, err
	}
	//Append the actual Txins (NumOfTxins * 36)
	//Do not include the sequence number to eliminate funny business
	for _, in := range r.ourInputs {
		//Hash
		_, err = b.Write(in.PreviousOutPoint.Hash.Bytes())
		if err != nil {
			return nil, err
		}
		//Index
		err = binary.Write(b, binary.BigEndian, in.PreviousOutPoint.Index)
		if err != nil {
			return nil, err
		}
	}

	return b.Bytes(), err
}

func (r *ChannelReservation) DeserializeFundRequest(wireMsg []byte) error {
	//Make sure we're not overwriting stuff...
	//Update the channelState to 1 before progressing if you want to re-do it.
	//Assumes only one thread is writing at a time
	if r.channelState > 1 {
		return fmt.Errorf("FundRequest: Channel State Mismatch")
	}

	var err error

	b := bytes.NewBuffer(wireMsg)
	msgid, _ := b.ReadByte()
	if msgid != 0x30 {
		return fmt.Errorf("Cannot deserialize: not a funding request")
	}

	if err != nil {
		return err
	}
	//Update the channel state as complete
	r.channelState = 2

	return nil
}

//Validation on the data being supplied from the fund request
func (r *ChannelReservation) ValidateFundRequest(wireMsg []byte) error {
	return nil
}

//Serialize CSV Revocation
//After the Commitment Transaction has been created, send a message to revoke this tx
func (r *ChannelReservation) SerializeCSVRefundRevocation() ([]byte, error) {
	return nil, nil
}

//Deserialize CSV Revocation
//Validate the revocation, after this step, the channel is fully set up
func (r *ChannelReservation) DeserializeCSVRefundRevocation() error {
	return nil
}*/

// * finish reset of tests
// * comment out stuff that'll need a node.
// * start on commitment side
//   * implement rusty's shachain
//   * set up logic to get notification from node when funding tx gets 6 deep.
//     * prob spawn into ChainNotifier struct
//   * create builder for initial funding transaction
//     * fascade through the wallet, for signing and such.
//   * channel should have active namespace to it's bucket, query at that point fo past commits etc
<|endoftext|>


<commit_before>/*
Copyright 2018 Gravitational, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package utils

import (
	"crypto/hmac"
	"crypto/sha256"
	"encoding/base64"
	"strings"

	"github.com/gravitational/trace"
)

// Anonymizer defines an interface for anonymizing data
type Anonymizer interface {
	// Anonymize returns anonymized string from the provided data
	Anonymize(data []byte) string
}

// hmacAnonymizer implements anonymization using HMAC
type hmacAnonymizer struct {
	// key is the HMAC key
	key string
}

// NewHMACAnonymizer returns a new HMAC-based anonymizer
func NewHMACAnonymizer(key string) (*hmacAnonymizer, error) {
	if strings.TrimSpace(key) == "" {
		return nil, trace.BadParameter("hmac key must not be empty")
	}
	return &hmacAnonymizer{
		key: key,
	}, nil
}

// Anonymize anonymizes the provided data using HMAC
func (a *hmacAnonymizer) Anonymize(data []byte) string {
	h := hmac.New(sha256.New, []byte(a.key))
	h.Write(data)
	return base64.StdEncoding.EncodeToString(h.Sum(nil))
}
<commit_msg>Fix comment<commit_after>/*
Copyright 2018 Gravitational, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package utils

import (
	"crypto/hmac"
	"crypto/sha256"
	"encoding/base64"
	"strings"

	"github.com/gravitational/trace"
)

// Anonymizer defines an interface for anonymizing data
type Anonymizer interface {
	// Anonymize returns anonymized string from the provided data
	Anonymize(data []byte) string
}

// hmacAnonymizer implements anonymization using HMAC
type hmacAnonymizer struct {
	// key is the HMAC key
	key string
}

// NewHMACAnonymizer returns a new HMAC-based anonymizer
func NewHMACAnonymizer(key string) (*hmacAnonymizer, error) {
	if strings.TrimSpace(key) == "" {
		return nil, trace.BadParameter("HMAC key must not be empty")
	}
	return &hmacAnonymizer{
		key: key,
	}, nil
}

// Anonymize anonymizes the provided data using HMAC
func (a *hmacAnonymizer) Anonymize(data []byte) string {
	h := hmac.New(sha256.New, []byte(a.key))
	h.Write(data)
	return base64.StdEncoding.EncodeToString(h.Sum(nil))
}
<|endoftext|>


<commit_before>// Sliding window download -- Fast parallel block downloading
package catchUp

import (
    "sync"
    "time"
    "github.com/oxfeeefeee/kaiju/log"
    "github.com/oxfeeefeee/kaiju/klib"
    "github.com/oxfeeefeee/kaiju/catma"
    "github.com/oxfeeefeee/kaiju/blockchain"
    "github.com/oxfeeefeee/kaiju/blockchain/cold"
    "github.com/oxfeeefeee/kaiju/knet"
    "github.com/oxfeeefeee/kaiju/knet/btcmsg"
)

type swdl struct {
    begin       int
    end         int
    paral       int
    load        int
    cursor      int
    size        int
    window      []interface{}
    chout       chan map[int]*blockchain.InvElement
    chin        chan map[int]interface{}
    chblock     chan struct{btcmsg.Message; I int}
    done        chan struct{}
    ca          *swdlca
    wg          sync.WaitGroup
}

func newSwdl(begin int, end int, paral int, load int) *swdl {
    // Open a window that wider than paral * load
    maxSlots := (end - begin) / load
    slots := paral * 4
    if slots > maxSlots {
        slots = maxSlots
    }
    s := slots * load
    if slots == 0 {
        s = end - begin
    }
    log.Infof("newSwdl begin %d end %d winsize %d", begin, end, s)
    return &swdl{
        begin: begin,
        end: end,
        paral: paral,
        load: load,
        cursor: begin,
        size: s,
        window: make([]interface{}, 0),
        chout: make(chan map[int]*blockchain.InvElement),
        chin: make(chan map[int]interface{}),
        chblock: make(chan struct{btcmsg.Message; I int}),
        done: make(chan struct{}),
        ca: newSwdlca(),
    }
}

func (sw *swdl) start() {
    sw.wg.Add(1) // For doSaveBlocks
    for i := 0; i < sw.paral; i++ {
        go sw.doDownload()
    }
    go sw.doSaveBlock()
    go sw.doSchedule()

    sw.chin <- nil // Trigger downloading
    sw.wg.Wait()

    db := cold.Get().OutputDB()
    if err := db.Commit(uint32(sw.end-1),true); err != nil {
        log.Panicf("db commit error: %s", err)
    }
    log.Infof("Finished downloading from %d to %d", sw.begin, sw.end)
}

func (sw *swdl) doSchedule() {
    running := true
    for running {
        select {
        case msgs := <- sw.chin:
            sw.schedule(msgs)
        case <- sw.done:
            running = false
        }
    }
    close(sw.chblock) // To end doSaveBlock
}

func (sw *swdl) doDownload() {
    running := true
    for running {
        select {
        case req := <- sw.chout:
            if req != nil {
                sw.chin <- download(req)
            } else {
                time.Sleep(30 * time.Second)
                sw.chin <- nil
            }
        case <- sw.done:
            running = false
        }
    }
    log.Infoln("doDownload exit")
}

func (sw *swdl) doSaveBlock() {
    defer sw.wg.Done()
    for bm := range sw.chblock {
        saveBlock(bm.Message, bm.I, false)
    }
    log.Infoln("doSaveBlock exit")
}

func (sw *swdl) schedule(msgs map[int]interface{}) {
    // 1. Fill blanks with downloaded blocks
    got := 0
    for k, v := range msgs {
        i := k - sw.cursor
        if _, ok := v.(*btcmsg.Message_block); ok {
            got++
        }
        // Do these checks because we might send same work to different workers
        // otherwise we should simple write "sw.window[i] = v"
        if i < 0 || i >= len(sw.window){
            continue
        } else if _, ok := sw.window[i].(*btcmsg.Message_block); !ok {
            sw.window[i] = v
        }
    }
    // 2. Congestion control
    if len(msgs) > 0 && sw.ca.update(got, len(msgs)) {
        // Send nil for a "NOP" download
        sw.sendWork(nil)
        return
    }
    // 3. Slide window and process blocks
    dist := len(sw.window) // Slide distance
    for i, elem := range sw.window {
        if bm, ok := elem.(*btcmsg.Message_block); ok {
            //log.Infoln("save block", i + sw.cursor, i, sw.cursor, bm.Header.Hash())
            sw.chblock <- struct{btcmsg.Message; I int}{bm, i + sw.cursor}
        } else {
            dist = i
            break
        }
    }
    if dist > 0 {
        log.Infof("swdl: window slided %d", dist)
    }
    sw.window = sw.window[dist:]
    sw.cursor += dist
    l := len(sw.window)
    for i := l; i < sw.size; i++ {
        p := sw.cursor + i
        if p >= sw.end {
            break
        }
        ie := blockchain.GetInvElem(p)
        sw.window = append(sw.window, ie)
    }
    if len(sw.window) == 0 {
        close(sw.done)// All blocks downloaded
    }
    // 4. Handle unfinished work
    unfinished := make(map[int]*blockchain.InvElement)
    for i, elem := range sw.window {
        if ie, ok := elem.(*blockchain.InvElement); ok {
            unfinished[sw.cursor+i] = ie
        }
    }
    l = len(unfinished)
    if len(msgs) > 0 {
        log.Debugf("winsize %d left %d got %d cursor %d health %f", len(sw.window), l, got, sw.cursor, sw.ca.health)
    }
    if l == 0 { // No blocks left, do nothing
    } else if l <= sw.load {
        // Not much left, send twice to slide the window faster
        sw.sendWork(unfinished)
        sw.sendWork(unfinished)
    } else {
        work := make(map[int]*blockchain.InvElement)
        for k, v := range unfinished {
            work[k] = v
            if len(work) >= sw.load {
                if !sw.sendWork(work) { // channel is full
                    break
                }
                work = make(map[int]*blockchain.InvElement)
            }
        }
    }
}

func (sw *swdl) sendWork(w map[int]*blockchain.InvElement) bool {
    select {
    case sw.chout <- w:
        for k, _ := range w {
            sw.window[k-sw.cursor] = nil // nil in window means "in process"
        }
        return true
    default:
        return false
    }
}

func download(req map[int]*blockchain.InvElement) map[int]interface{} {
    inv := make([]*blockchain.InvElement, 0, len(req))
    records := make(map[klib.Hash256]interface{})
    for _, v := range req {
        inv = append(inv, v)
        records[v.Hash] = v
    }
    f := func(m btcmsg.Message) bool {
        bmsg, ok := m.(*btcmsg.Message_block)
        if ok {
            hash := bmsg.Header.Hash()
            v, ok := records[*hash]
            if ok {
                _, ok := v.(*blockchain.InvElement)
                records[*hash] = m
                return ok
            }
        }
        return false
    }
    msg := btcmsg.NewGetDataMsg().(*btcmsg.Message_getdata)
    msg.Inventory = inv
    knet.MsgForMsgs(msg, f, len(inv))
    ret := make(map[int]interface{})
    for k, v := range req {
        ret[k] = records[v.Hash] // Either btcmsg.Message or *blockchain.InvElement
    }
    return ret
}

func saveBlock(m btcmsg.Message, i int, verify bool) {
    db := cold.Get().OutputDB()
    bm, _ := m.(*btcmsg.Message_block)
    for _, tx := range bm.Txs {
        ctx := (*catma.Tx)(tx)
        err := catma.VerifyTx(ctx, db, true, false, !verify)
        if err != nil {
            log.Panicf("Process tx %s error: %s", ctx.Hash(), err)
        }
    }
    if err := db.Commit(uint32(i),false); err != nil {
        log.Panicf("db commit error: %s", err)
    }
}

<commit_msg>Removed swdl duel work sent to fix a crash<commit_after>// Sliding window download -- Fast parallel block downloading
package catchUp

import (
    "sync"
    "time"
    "github.com/oxfeeefeee/kaiju/log"
    "github.com/oxfeeefeee/kaiju/klib"
    "github.com/oxfeeefeee/kaiju/catma"
    "github.com/oxfeeefeee/kaiju/blockchain"
    "github.com/oxfeeefeee/kaiju/blockchain/cold"
    "github.com/oxfeeefeee/kaiju/knet"
    "github.com/oxfeeefeee/kaiju/knet/btcmsg"
)

type swdl struct {
    begin       int
    end         int
    paral       int
    load        int
    cursor      int
    size        int
    window      []interface{}
    chout       chan map[int]*blockchain.InvElement
    chin        chan map[int]interface{}
    chblock     chan struct{btcmsg.Message; I int}
    done        chan struct{}
    ca          *swdlca
    wg          sync.WaitGroup
}

func newSwdl(begin int, end int, paral int, load int) *swdl {
    // Open a window that wider than paral * load
    maxSlots := (end - begin) / load
    slots := paral * 4
    if slots > maxSlots {
        slots = maxSlots
    }
    s := slots * load
    if slots == 0 {
        s = end - begin
    }
    log.Infof("newSwdl begin %d end %d winsize %d", begin, end, s)
    return &swdl{
        begin: begin,
        end: end,
        paral: paral,
        load: load,
        cursor: begin,
        size: s,
        window: make([]interface{}, 0),
        chout: make(chan map[int]*blockchain.InvElement),
        chin: make(chan map[int]interface{}),
        chblock: make(chan struct{btcmsg.Message; I int}),
        done: make(chan struct{}),
        ca: newSwdlca(),
    }
}

func (sw *swdl) start() {
    sw.wg.Add(1) // For doSaveBlocks
    for i := 0; i < sw.paral; i++ {
        go sw.doDownload()
    }
    go sw.doSaveBlock()
    go sw.doSchedule()

    sw.chin <- nil // Trigger downloading
    sw.wg.Wait()

    db := cold.Get().OutputDB()
    if err := db.Commit(uint32(sw.end-1),true); err != nil {
        log.Panicf("db commit error: %s", err)
    }
    log.Infof("Finished downloading from %d to %d", sw.begin, sw.end)
}

func (sw *swdl) doSchedule() {
    running := true
    for running {
        select {
        case msgs := <- sw.chin:
            sw.schedule(msgs)
        case <- sw.done:
            running = false
        }
    }
    close(sw.chblock) // To end doSaveBlock
}

func (sw *swdl) doDownload() {
    running := true
    for running {
        select {
        case req := <- sw.chout:
            if req != nil {
                sw.chin <- download(req)
            } else {
                time.Sleep(30 * time.Second)
                sw.chin <- nil
            }
        case <- sw.done:
            running = false
        }
    }
    log.Infoln("doDownload exit")
}

func (sw *swdl) doSaveBlock() {
    defer sw.wg.Done()
    for bm := range sw.chblock {
        saveBlock(bm.Message, bm.I, false)
    }
    log.Infoln("doSaveBlock exit")
}

func (sw *swdl) schedule(msgs map[int]interface{}) {
    // 1. Fill blanks with downloaded blocks
    got := 0
    for k, v := range msgs {
        i := k - sw.cursor
        if _, ok := v.(*btcmsg.Message_block); ok {
            got++
        }
        sw.window[i] = v
    }
    // 2. Congestion control
    if len(msgs) > 0 && sw.ca.update(got, len(msgs)) {
        // Send nil for a "NOP" download
        sw.sendWork(nil)
        return
    }
    // 3. Slide window and process blocks
    dist := len(sw.window) // Slide distance
    for i, elem := range sw.window {
        if bm, ok := elem.(*btcmsg.Message_block); ok {
            //log.Infoln("save block", i + sw.cursor, i, sw.cursor, bm.Header.Hash())
            sw.chblock <- struct{btcmsg.Message; I int}{bm, i + sw.cursor}
        } else {
            dist = i
            break
        }
    }
    if dist > 0 {
        log.Infof("swdl: window slided %d", dist)
    }
    sw.window = sw.window[dist:]
    sw.cursor += dist
    l := len(sw.window)
    for i := l; i < sw.size; i++ {
        p := sw.cursor + i
        if p >= sw.end {
            break
        }
        ie := blockchain.GetInvElem(p)
        sw.window = append(sw.window, ie)
    }
    if len(sw.window) == 0 {
        close(sw.done)// All blocks downloaded
    }
    // 4. Handle unfinished work
    unfinished := make(map[int]*blockchain.InvElement)
    for i, elem := range sw.window {
        if ie, ok := elem.(*blockchain.InvElement); ok {
            unfinished[sw.cursor+i] = ie
        }
    }
    l = len(unfinished)
    if len(msgs) > 0 {
        log.Debugf("winsize %d left %d got %d cursor %d health %f", len(sw.window), l, got, sw.cursor, sw.ca.health)
    }
    if l == 0 { // No blocks left, do nothing
    } else if l <= sw.load {
        sw.sendWork(unfinished)
    } else {
        work := make(map[int]*blockchain.InvElement)
        for k, v := range unfinished {
            work[k] = v
            if len(work) >= sw.load {
                if !sw.sendWork(work) { // channel is full
                    break
                }
                work = make(map[int]*blockchain.InvElement)
            }
        }
    }
}

func (sw *swdl) sendWork(w map[int]*blockchain.InvElement) bool {
    select {
    case sw.chout <- w:
        for k, _ := range w {
            sw.window[k-sw.cursor] = nil // nil in window means "in process"
        }
        return true
    default:
        return false
    }
}

func download(req map[int]*blockchain.InvElement) map[int]interface{} {
    inv := make([]*blockchain.InvElement, 0, len(req))
    records := make(map[klib.Hash256]interface{})
    for _, v := range req {
        inv = append(inv, v)
        records[v.Hash] = v
    }
    f := func(m btcmsg.Message) bool {
        bmsg, ok := m.(*btcmsg.Message_block)
        if ok {
            hash := bmsg.Header.Hash()
            v, ok := records[*hash]
            if ok {
                _, ok := v.(*blockchain.InvElement)
                records[*hash] = m
                return ok
            }
        }
        return false
    }
    msg := btcmsg.NewGetDataMsg().(*btcmsg.Message_getdata)
    msg.Inventory = inv
    knet.MsgForMsgs(msg, f, len(inv))
    ret := make(map[int]interface{})
    for k, v := range req {
        ret[k] = records[v.Hash] // Either btcmsg.Message or *blockchain.InvElement
    }
    return ret
}

func saveBlock(m btcmsg.Message, i int, verify bool) {
    db := cold.Get().OutputDB()
    bm, _ := m.(*btcmsg.Message_block)
    for _, tx := range bm.Txs {
        ctx := (*catma.Tx)(tx)
        err := catma.VerifyTx(ctx, db, true, false, !verify)
        if err != nil {
            log.Panicf("Process tx %s error: %s", ctx.Hash(), err)
        }
    }
    if err := db.Commit(uint32(i),false); err != nil {
        log.Panicf("db commit error: %s", err)
    }
}

<|endoftext|>


<commit_before>package bintree

import "fmt"

// Binary tree with integer values.
type Tree struct {
	Left  *Tree
	Value int
	Right *Tree
}

func (t *Tree) String() string {
	return fmt.Sprintf("{Left node: %p, Value: %d, Right node: %p}", t.Left, t.Value, t.Right)
}

// Traverse a tree depth-first, sending each Value on a channel.
func Walk(t *Tree, ch chan int) {
	if t == nil {
		return
	}
	Walk(t.Left, ch)
	ch <- t.Value
	Walk(t.Right, ch)
}

// Launch Walk in a new goroutine and return read-only channel of values.
func Walker(t *Tree) <-chan int {
	ch := make(chan int)
	go func() {
		Walk(t, ch)
		close(ch)
	}()
	return ch
}

// Create new tree holding the values from 0 to n-1.
func NewRange(n int) *Tree {
	var t *Tree
	for i := 0; i < n; i++ {
		t = Insert(t, i)
	}
	return t
}

// Create new empty tree.
func New() *Tree {
	return &Tree{nil, 0, nil}
}

// Insert new node into the tree.
func Insert(t *Tree, val int) *Tree {
	if t == nil {
		return &Tree{nil, val, nil}
	}
	if val < t.Value {
		t.Left = Insert(t.Left, val)
		return t
	}
	t.Right = Insert(t.Right, val)
	return t
}

// The procedure begins its search at the root and traces a simple path downward in the tree.
func Search(t *Tree, val int) *Tree {
	if t == nil || val == t.Value {
		return t
	}

	if val < t.Value {
		return Search(t.Left, val)
	} else {
		return Search(t.Right, val)
	}
}
<commit_msg>Implement max/min methods for binary tree<commit_after>package bintree

import "fmt"

// Binary tree with integer values.
type Tree struct {
	Left  *Tree
	Value int
	Right *Tree
}

func (t *Tree) String() string {
	return fmt.Sprintf("{Left node: %p, Value: %d, Right node: %p}", t.Left, t.Value, t.Right)
}

// Traverse a tree depth-first, sending each Value on a channel.
func Walk(t *Tree, ch chan int) {
	if t == nil {
		return
	}
	Walk(t.Left, ch)
	ch <- t.Value
	Walk(t.Right, ch)
}

// Launch Walk in a new goroutine and return read-only channel of values.
func Walker(t *Tree) <-chan int {
	ch := make(chan int)
	go func() {
		Walk(t, ch)
		close(ch)
	}()
	return ch
}

// Create new tree holding the values from 0 to n-1.
func NewRange(n int) *Tree {
	var t *Tree
	for i := 0; i < n; i++ {
		t = Insert(t, i)
	}
	return t
}

// Create new empty tree.
func New() *Tree {
	return &Tree{nil, 0, nil}
}

// Insert new node into the tree.
func Insert(t *Tree, val int) *Tree {
	if t == nil {
		return &Tree{nil, val, nil}
	}
	if val < t.Value {
		t.Left = Insert(t.Left, val)
		return t
	}
	t.Right = Insert(t.Right, val)
	return t
}

// The procedure begins its search at the root and traces a simple path downward in the tree.
func Search(t *Tree, val int) *Tree {
	if t == nil || val == t.Value {
		return t
	}

	if val < t.Value {
		return Search(t.Left, val)
	} else {
		return Search(t.Right, val)
	}
}

// Return node with minimum value
func Min(t *Tree) *Tree {
	x := t

	for x.Left != nil {
		x = x.Left
	}

	return x
}

// Return node with maximum value
func Max(t *Tree) *Tree {
	x := t

	for x.Right != nil {
		x = x.Right
	}

	return x
}
<|endoftext|>