<commit_before>// Copyright 2016 The Gosl Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build ignore

package main

import (
	"testing"

	"github.com/cpmech/gosl/chk"
	"github.com/cpmech/gosl/io"
	"github.com/cpmech/gosl/la"
	"github.com/cpmech/gosl/mpi"
)

func main() {

	mpi.Start()
	defer mpi.Stop()

	if mpi.WorldRank() == 0 {
		io.Pf("\n------------------------ Test SP MPI 01------------------------ \n")
	}

	comm := mpi.NewCommunicator(nil)

	M := [][]float64{
		{1000, 1000, 1000, 1011, 1021, 1000},
		{1000, 1000, 1000, 1012, 1022, 1000},
		{1000, 1000, 1000, 1013, 1023, 1000},
		{1011, 1012, 1013, 1000, 1000, 1000},
		{1021, 1022, 1023, 1000, 1000, 1000},
		{1000, 1000, 1000, 1000, 1000, 1000},
	}

	id, sz, m := comm.Rank(), comm.Size(), len(M)
	start, endp1 := (id*m)/sz, ((id+1)*m)/sz

	if sz > 6 {
		chk.Panic("this test works with at most 6 processors")
	}

	var J la.Triplet
	J.Init(m, m, m*m)
	for i := start; i < endp1; i++ {
		for j := 0; j < m; j++ {
			J.Put(i, j, M[i][j])
		}
	}

	la.SpTriReduce(comm, &J)

	chk.Verbose = true
	var tst testing.T

	if comm.Rank() == 0 {
		chk.Matrix(&tst, "J @ proc 0", 1.0e-17, J.GetDenseMatrix().GetSlice(), [][]float64{
			{1000, 1000, 1000, 1011, 1021, 1000},
			{1000, 1000, 1000, 1012, 1022, 1000},
			{1000, 1000, 1000, 1013, 1023, 1000},
			{1011, 1012, 1013, 1000, 1000, 1000},
			{1021, 1022, 1023, 1000, 1000, 1000},
			{1000, 1000, 1000, 1000, 1000, 1000},
		})
	}
}
<commit_msg>Fix use of chk.Deep2<commit_after>// Copyright 2016 The Gosl Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build ignore

package main

import (
	"testing"

	"github.com/cpmech/gosl/chk"
	"github.com/cpmech/gosl/io"
	"github.com/cpmech/gosl/la"
	"github.com/cpmech/gosl/mpi"
)

func main() {

	mpi.Start()
	defer mpi.Stop()

	if mpi.WorldRank() == 0 {
		io.Pf("\n------------------------ Test SP MPI 01------------------------ \n")
	}

	comm := mpi.NewCommunicator(nil)

	M := [][]float64{
		{1000, 1000, 1000, 1011, 1021, 1000},
		{1000, 1000, 1000, 1012, 1022, 1000},
		{1000, 1000, 1000, 1013, 1023, 1000},
		{1011, 1012, 1013, 1000, 1000, 1000},
		{1021, 1022, 1023, 1000, 1000, 1000},
		{1000, 1000, 1000, 1000, 1000, 1000},
	}

	id, sz, m := comm.Rank(), comm.Size(), len(M)
	start, endp1 := (id*m)/sz, ((id+1)*m)/sz

	if sz > 6 {
		chk.Panic("this test works with at most 6 processors")
	}

	var J la.Triplet
	J.Init(m, m, m*m)
	for i := start; i < endp1; i++ {
		for j := 0; j < m; j++ {
			J.Put(i, j, M[i][j])
		}
	}

	la.SpTriReduce(comm, &J)

	chk.Verbose = true
	var tst testing.T

	if comm.Rank() == 0 {
		chk.Deep2(&tst, "J @ proc 0", 1.0e-17, J.GetDenseMatrix().GetDeep2(), [][]float64{
			{1000, 1000, 1000, 1011, 1021, 1000},
			{1000, 1000, 1000, 1012, 1022, 1000},
			{1000, 1000, 1000, 1013, 1023, 1000},
			{1011, 1012, 1013, 1000, 1000, 1000},
			{1021, 1022, 1023, 1000, 1000, 1000},
			{1000, 1000, 1000, 1000, 1000, 1000},
		})
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"

	"github.com/hprose/hprose-golang/rpc"
)

func main() {
	client := rpc.NewTCPClient("tcp4://127.0.0.1:2016/")
	count := 0
	id, _ := client.ID()
	done := make(chan bool)
	client.Subscribe("time", id, nil, func(data string) {
		count++
		if count > 10 {
			client.Unsubscribe("time")
			done <- true
		}
		fmt.Println(data)
	})
	<-done
}
<commit_msg>Fixed example<commit_after>package main

import (
	"fmt"

	"github.com/hprose/hprose-golang/rpc"
)

func main() {
	client := rpc.NewTCPClient("tcp4://127.0.0.1:2016/")
	count := 0
	id, _ := client.AutoID()
	done := make(chan bool)
	client.Subscribe("time", id, nil, func(data string) {
		count++
		if count > 10 {
			client.Unsubscribe("time")
			done <- true
		}
		fmt.Println(data)
	})
	<-done
}
<|endoftext|>


<commit_before>package models

import (
	"errors"
	"github.com/stretchr/testify/assert"
	"gopkg.in/DATA-DOG/go-sqlmock.v1"
	"testing"

	"github.com/eirka/eirka-libs/db"
	e "github.com/eirka/eirka-libs/errors"
)

func TestNewTagValidateInput(t *testing.T) {

	var err error

	tag := NewTagModel{
		Ib:      1,
		Tag:     "test",
		TagType: 0,
	}

	err = tag.ValidateInput()
	if assert.Error(t, err, "An error was expected") {
		assert.Equal(t, err, e.ErrInvalidParam, "Error should match")
	}

}

func TestNewTagIsValid(t *testing.T) {

	tag := NewTagModel{
		Ib:      1,
		Tag:     "",
		TagType: 1,
	}

	assert.False(t, tag.IsValid(), "Should be false")

}

func TestNewTagStatus(t *testing.T) {

	var err error

	mock, err := db.NewTestDb()
	assert.NoError(t, err, "An error was not expected")

	statusrows := sqlmock.NewRows([]string{"count"}).AddRow(0)
	mock.ExpectQuery(`select count\(1\) from tags`).WillReturnRows(statusrows)

	tag := NewTagModel{
		Ib:      1,
		Tag:     "test",
		TagType: 1,
	}

	err = tag.Status()
	assert.NoError(t, err, "An error was not expected")

}

func TestNewTagStatusDuplicate(t *testing.T) {

	var err error

	mock, err := db.NewTestDb()
	assert.NoError(t, err, "An error was not expected")

	statusrows := sqlmock.NewRows([]string{"count"}).AddRow(1)
	mock.ExpectQuery(`select count\(1\) from tags`).WillReturnRows(statusrows)

	tag := NewTagModel{
		Ib:      1,
		Tag:     "test",
		TagType: 1,
	}

	err = tag.Status()
	if assert.Error(t, err, "An error was expected") {
		assert.Equal(t, err, e.ErrDuplicateTag, "Error should match")
	}

}

func TestNewTagPost(t *testing.T) {

	var err error

	mock, err := db.NewTestDb()
	assert.NoError(t, err, "An error was not expected")

	mock.ExpectExec("INSERT into tags").
		WithArgs("test", 1, 1).
		WillReturnResult(sqlmock.NewResult(1, 1))

	tag := NewTagModel{
		Ib:      1,
		Tag:     "test",
		TagType: 1,
	}

	err = tag.Post()
	assert.NoError(t, err, "An error was not expected")

}

func TestNewTagPostInvalid(t *testing.T) {

	var err error

	tag := NewTagModel{
		Ib:      0,
		Tag:     "test",
		TagType: 1,
	}

	err = tag.Post()
	if assert.Error(t, err, "An error was expected") {
		assert.Equal(t, err, errors.New("NewTagModel is not valid"), "Error should match")
	}

}
<commit_msg>add model tests<commit_after>package models

import (
	"errors"
	"github.com/stretchr/testify/assert"
	"gopkg.in/DATA-DOG/go-sqlmock.v1"
	"testing"

	"github.com/eirka/eirka-libs/db"
	e "github.com/eirka/eirka-libs/errors"
)

func TestNewTagValidateInput(t *testing.T) {

	var err error

	tag := NewTagModel{
		Ib:      1,
		Tag:     "test",
		TagType: 0,
	}

	err = tag.ValidateInput()
	if assert.Error(t, err, "An error was expected") {
		assert.Equal(t, err, e.ErrInvalidParam, "Error should match")
	}

}

func TestNewTagValidateInputTagShort(t *testing.T) {

	var err error

	tag := NewTagModel{
		Ib:      1,
		Tag:     "t",
		TagType: 1,
	}

	err = tag.ValidateInput()
	if assert.Error(t, err, "An error was expected") {
		assert.Equal(t, err, e.ErrTagShort, "Error should match")
	}

}

func TestNewTagValidateInputTagEmpty(t *testing.T) {

	var err error

	tag := NewTagModel{
		Ib:      1,
		Tag:     "",
		TagType: 1,
	}

	err = tag.ValidateInput()
	if assert.Error(t, err, "An error was expected") {
		assert.Equal(t, err, e.ErrNoTagName, "Error should match")
	}

}

func TestNewTagIsValid(t *testing.T) {

	tag := NewTagModel{
		Ib:      1,
		Tag:     "",
		TagType: 1,
	}

	assert.False(t, tag.IsValid(), "Should be false")

}

func TestNewTagStatus(t *testing.T) {

	var err error

	mock, err := db.NewTestDb()
	assert.NoError(t, err, "An error was not expected")

	statusrows := sqlmock.NewRows([]string{"count"}).AddRow(0)
	mock.ExpectQuery(`select count\(1\) from tags`).WillReturnRows(statusrows)

	tag := NewTagModel{
		Ib:      1,
		Tag:     "test",
		TagType: 1,
	}

	err = tag.Status()
	assert.NoError(t, err, "An error was not expected")

}

func TestNewTagStatusDuplicate(t *testing.T) {

	var err error

	mock, err := db.NewTestDb()
	assert.NoError(t, err, "An error was not expected")

	statusrows := sqlmock.NewRows([]string{"count"}).AddRow(1)
	mock.ExpectQuery(`select count\(1\) from tags`).WillReturnRows(statusrows)

	tag := NewTagModel{
		Ib:      1,
		Tag:     "test",
		TagType: 1,
	}

	err = tag.Status()
	if assert.Error(t, err, "An error was expected") {
		assert.Equal(t, err, e.ErrDuplicateTag, "Error should match")
	}

}

func TestNewTagPost(t *testing.T) {

	var err error

	mock, err := db.NewTestDb()
	assert.NoError(t, err, "An error was not expected")

	mock.ExpectExec("INSERT into tags").
		WithArgs("test", 1, 1).
		WillReturnResult(sqlmock.NewResult(1, 1))

	tag := NewTagModel{
		Ib:      1,
		Tag:     "test",
		TagType: 1,
	}

	err = tag.Post()
	assert.NoError(t, err, "An error was not expected")

}

func TestNewTagPostInvalid(t *testing.T) {

	var err error

	tag := NewTagModel{
		Ib:      0,
		Tag:     "test",
		TagType: 1,
	}

	err = tag.Post()
	if assert.Error(t, err, "An error was expected") {
		assert.Equal(t, err, errors.New("NewTagModel is not valid"), "Error should match")
	}

}
<|endoftext|>


<commit_before>// Package astcontext provides context aware utilities to be used within
// editors.
package astcontext

import (
	"errors"
	"fmt"
	"go/ast"
	"sort"
)

// Func represents a declared (*ast.FuncDecl) or an anonymous (*ast.FuncLit) Go
// function
type Func struct {
	FuncPos *Position `json:"func" vim:"func"`     // position of the "func" keyword
	Lbrace  *Position `json:"lbrace" vim:"lbrace"` // position of "{"
	Rbrace  *Position `json:"rbrace" vim:"rbrace"` // position of "}"

	// position of the doc comment, only for *ast.FuncDecl
	Doc *Position `json:"doc,omitempty" vim:"doc,omitempty"`

	node ast.Node // either *ast.FuncDecl or *ast.FuncLit
}

// Funcs represents a list of functions
type Funcs []*Func

// IsDeclaration returns true if the given function is a function declaration
// (*ast.FuncDecl)
func (f *Func) IsDeclaration() bool {
	_, ok := f.node.(*ast.FuncDecl)
	return ok
}

// IsLiteral returns true if the given function is a function literal
// (*ast.FuncLit)
func (f *Func) IsLiteral() bool {
	_, ok := f.node.(*ast.FuncLit)
	return ok
}

func (f *Func) String() string {
	switch x := f.node.(type) {
	case *ast.FuncDecl:
		return fmt.Sprintf("line: %d type: %T name: %s",
			f.FuncPos.Line, f.node, x.Name.Name)
	default:
		return fmt.Sprintf("line: %d type: %T",
			f.FuncPos.Line, f.node)
	}
}

// Funcs returns a list of Func's from the parsed source.  Func's are sorted
// according to the order of Go functions in the given source.
func (p *Parser) Funcs() Funcs {
	var funcs []*Func

	// Inspect the AST and find all function declarements and literals
	ast.Inspect(p.file, func(n ast.Node) bool {
		switch x := n.(type) {
		case *ast.FuncDecl:
			fn := &Func{
				Lbrace:  ToPosition(p.fset.Position(x.Body.Lbrace)),
				Rbrace:  ToPosition(p.fset.Position(x.Body.Rbrace)),
				FuncPos: ToPosition(p.fset.Position(x.Type.Func)),
				node:    x,
			}

			if x.Doc != nil {
				fn.Doc = ToPosition(p.fset.Position(x.Doc.Pos()))
			}

			funcs = append(funcs, fn)
		case *ast.FuncLit:
			funcs = append(funcs, &Func{
				Lbrace:  ToPosition(p.fset.Position(x.Body.Lbrace)),
				Rbrace:  ToPosition(p.fset.Position(x.Body.Rbrace)),
				FuncPos: ToPosition(p.fset.Position(x.Type.Func)),
				node:    x,
			})
		}
		return true
	})

	return funcs
}

// EnclosingFunc returns the enclosing *Func for the given offset
func (f Funcs) EnclosingFunc(offset int) (*Func, error) {
	var encFunc *Func

	// TODO(arslan) this is iterating over all functions. Benchmark it and see
	// if it's worth it to change it with a more effiecent search function. For
	// now this is enough for us.
	for _, fn := range f {
		// standard function declaration without any docs. Start from the func
		// keyword
		start := fn.FuncPos.Offset

		// has a doc, also include it
		if fn.Doc != nil && fn.Doc.IsValid() {
			start = fn.Doc.Offset
		}

		// one liner, start from the beginning to make it easier
		if fn.FuncPos.Line == fn.Rbrace.Line {
			start = fn.FuncPos.Offset - fn.FuncPos.Column
		}

		end := fn.Rbrace.Offset

		if start <= offset && offset <= end {
			encFunc = fn
		}
	}

	if encFunc == nil {
		return nil, errors.New("no enclosing functions found")
	}

	return encFunc, nil
}

// // NextFunc returns the nearest next Func for the given offset.
// func (f Funcs) NextFunc(offset int) (*Func, error) {
// 	// find nearest next function
// 	nextIndex := sort.Search(len(f), func(i int) bool {
// 		return f[i].FuncPos.Offset > offset
// 	})

// 	if nextIndex == len(f) {
// 		return nil, errors.New("no functions found")
// 	}
// 	return f[nextIndex], nil
// }

// NextFunc returns the nearest next Func for the given offset.
func (f Funcs) NextFunc(offset int) (*Func, error) {
	return f.nextFuncShift(offset, 0)
}

// NextFuncShift returns the nearest next Func for the given offset. Shift
// shifts the index before returning. This is useful to get the second nearest
// next function (shift being 1), third nearest next function (shift being 2),
// etc...
func (f Funcs) NextFuncShift(offset, shift int) (*Func, error) {
	return f.nextFuncShift(offset, shift)
}

// PrevFunc returns the nearest previous *Func for the given offset.
func (f Funcs) PrevFunc(offset int) (*Func, error) {
	return f.prevFuncShift(offset, 0)
}

// PrevFuncShift returns the nearest previous Func for the given offset. Shift
// shifts the index before returning. This is useful to get the second nearest
// previous function (shift being 1), third nearest previous function (shift
// being 2), etc...
func (f Funcs) PrevFuncShift(offset, shift int) (*Func, error) {
	return f.prevFuncShift(offset, shift)
}

// nextFuncShift returns the nearest next function for the given offset and
// shift index. If index is zero it returns the nearest next function. If shift
// is non zero positive number it returns the function shifted by the given
// number. i.e: [a, b, c, d] if the nearest func is b (shift 0), shift with
// value 1 returns c, 2 returns d and anything larger returns an error.
func (f Funcs) nextFuncShift(offset, shift int) (*Func, error) {
	if shift < 0 {
		return nil, errors.New("shift can't be negative")
	}

	// find nearest next function
	nextIndex := sort.Search(len(f), func(i int) bool {
		return f[i].FuncPos.Offset > offset
	})

	if nextIndex+shift >= len(f) {
		return nil, errors.New("no functions found")
	}

	return f[nextIndex+shift], nil
}

// prevFuncShift returns the nearest previous *Func for the given offset and
// shift index. If index is zero it returns the nearest previous function. If
// shift is non zero positive number it returns the function shifted by the
// given number. i.e: [a, b, c, d] if the nearest previous func is c (shift 0),
// shift with value 1 returns b, 2 returns a and anything larger returns an
// error.
func (f Funcs) prevFuncShift(offset, shift int) (*Func, error) {
	if shift < 0 {
		return nil, errors.New("shift can't be negative")
	}

	// start from the reverse to get the prev function
	f.Reserve()

	prevIndex := sort.Search(len(f), func(i int) bool {
		return f[i].FuncPos.Offset < offset
	})

	if prevIndex+shift >= len(f) {
		return nil, errors.New("no functions found")
	}

	return f[prevIndex+shift], nil
}

func (f Funcs) Len() int           { return len(f) }
func (f Funcs) Swap(i, j int)      { f[i], f[j] = f[j], f[i] }
func (f Funcs) Less(i, j int) bool { return f[i].FuncPos.Offset < f[j].FuncPos.Offset }

// Reserve reserves the Function data
func (f Funcs) Reserve() {
	for start, end := 0, f.Len()-1; start < end; {
		f.Swap(start, end)
		start++
		end--
	}
}

// Declarations returns a copy of funcs with only Function declarations
func (f Funcs) Declarations() Funcs {
	// NOTE(arslan): we can prepopulate these in the future, but again we need
	// to benchmark first
	var decls []*Func
	for _, fn := range f {
		if fn.IsDeclaration() {
			decls = append(decls, fn)
		}
	}
	return decls
}
<commit_msg>motion: remove unneeded func<commit_after>// Package astcontext provides context aware utilities to be used within
// editors.
package astcontext

import (
	"errors"
	"fmt"
	"go/ast"
	"sort"
)

// Func represents a declared (*ast.FuncDecl) or an anonymous (*ast.FuncLit) Go
// function
type Func struct {
	FuncPos *Position `json:"func" vim:"func"`     // position of the "func" keyword
	Lbrace  *Position `json:"lbrace" vim:"lbrace"` // position of "{"
	Rbrace  *Position `json:"rbrace" vim:"rbrace"` // position of "}"

	// position of the doc comment, only for *ast.FuncDecl
	Doc *Position `json:"doc,omitempty" vim:"doc,omitempty"`

	node ast.Node // either *ast.FuncDecl or *ast.FuncLit
}

// Funcs represents a list of functions
type Funcs []*Func

// IsDeclaration returns true if the given function is a function declaration
// (*ast.FuncDecl)
func (f *Func) IsDeclaration() bool {
	_, ok := f.node.(*ast.FuncDecl)
	return ok
}

// IsLiteral returns true if the given function is a function literal
// (*ast.FuncLit)
func (f *Func) IsLiteral() bool {
	_, ok := f.node.(*ast.FuncLit)
	return ok
}

func (f *Func) String() string {
	switch x := f.node.(type) {
	case *ast.FuncDecl:
		return fmt.Sprintf("line: %d type: %T name: %s",
			f.FuncPos.Line, f.node, x.Name.Name)
	default:
		return fmt.Sprintf("line: %d type: %T",
			f.FuncPos.Line, f.node)
	}
}

// Funcs returns a list of Func's from the parsed source.  Func's are sorted
// according to the order of Go functions in the given source.
func (p *Parser) Funcs() Funcs {
	var funcs []*Func

	// Inspect the AST and find all function declarements and literals
	ast.Inspect(p.file, func(n ast.Node) bool {
		switch x := n.(type) {
		case *ast.FuncDecl:
			fn := &Func{
				Lbrace:  ToPosition(p.fset.Position(x.Body.Lbrace)),
				Rbrace:  ToPosition(p.fset.Position(x.Body.Rbrace)),
				FuncPos: ToPosition(p.fset.Position(x.Type.Func)),
				node:    x,
			}

			if x.Doc != nil {
				fn.Doc = ToPosition(p.fset.Position(x.Doc.Pos()))
			}

			funcs = append(funcs, fn)
		case *ast.FuncLit:
			funcs = append(funcs, &Func{
				Lbrace:  ToPosition(p.fset.Position(x.Body.Lbrace)),
				Rbrace:  ToPosition(p.fset.Position(x.Body.Rbrace)),
				FuncPos: ToPosition(p.fset.Position(x.Type.Func)),
				node:    x,
			})
		}
		return true
	})

	return funcs
}

// EnclosingFunc returns the enclosing *Func for the given offset
func (f Funcs) EnclosingFunc(offset int) (*Func, error) {
	var encFunc *Func

	// TODO(arslan) this is iterating over all functions. Benchmark it and see
	// if it's worth it to change it with a more effiecent search function. For
	// now this is enough for us.
	for _, fn := range f {
		// standard function declaration without any docs. Start from the func
		// keyword
		start := fn.FuncPos.Offset

		// has a doc, also include it
		if fn.Doc != nil && fn.Doc.IsValid() {
			start = fn.Doc.Offset
		}

		// one liner, start from the beginning to make it easier
		if fn.FuncPos.Line == fn.Rbrace.Line {
			start = fn.FuncPos.Offset - fn.FuncPos.Column
		}

		end := fn.Rbrace.Offset

		if start <= offset && offset <= end {
			encFunc = fn
		}
	}

	if encFunc == nil {
		return nil, errors.New("no enclosing functions found")
	}

	return encFunc, nil
}

// NextFunc returns the nearest next Func for the given offset.
func (f Funcs) NextFunc(offset int) (*Func, error) {
	return f.nextFuncShift(offset, 0)
}

// NextFuncShift returns the nearest next Func for the given offset. Shift
// shifts the index before returning. This is useful to get the second nearest
// next function (shift being 1), third nearest next function (shift being 2),
// etc...
func (f Funcs) NextFuncShift(offset, shift int) (*Func, error) {
	return f.nextFuncShift(offset, shift)
}

// PrevFunc returns the nearest previous *Func for the given offset.
func (f Funcs) PrevFunc(offset int) (*Func, error) {
	return f.prevFuncShift(offset, 0)
}

// PrevFuncShift returns the nearest previous Func for the given offset. Shift
// shifts the index before returning. This is useful to get the second nearest
// previous function (shift being 1), third nearest previous function (shift
// being 2), etc...
func (f Funcs) PrevFuncShift(offset, shift int) (*Func, error) {
	return f.prevFuncShift(offset, shift)
}

// nextFuncShift returns the nearest next function for the given offset and
// shift index. If index is zero it returns the nearest next function. If shift
// is non zero positive number it returns the function shifted by the given
// number. i.e: [a, b, c, d] if the nearest func is b (shift 0), shift with
// value 1 returns c, 2 returns d and anything larger returns an error.
func (f Funcs) nextFuncShift(offset, shift int) (*Func, error) {
	if shift < 0 {
		return nil, errors.New("shift can't be negative")
	}

	// find nearest next function
	nextIndex := sort.Search(len(f), func(i int) bool {
		return f[i].FuncPos.Offset > offset
	})

	if nextIndex+shift >= len(f) {
		return nil, errors.New("no functions found")
	}

	return f[nextIndex+shift], nil
}

// prevFuncShift returns the nearest previous *Func for the given offset and
// shift index. If index is zero it returns the nearest previous function. If
// shift is non zero positive number it returns the function shifted by the
// given number. i.e: [a, b, c, d] if the nearest previous func is c (shift 0),
// shift with value 1 returns b, 2 returns a and anything larger returns an
// error.
func (f Funcs) prevFuncShift(offset, shift int) (*Func, error) {
	if shift < 0 {
		return nil, errors.New("shift can't be negative")
	}

	// start from the reverse to get the prev function
	f.Reserve()

	prevIndex := sort.Search(len(f), func(i int) bool {
		return f[i].FuncPos.Offset < offset
	})

	if prevIndex+shift >= len(f) {
		return nil, errors.New("no functions found")
	}

	return f[prevIndex+shift], nil
}

func (f Funcs) Len() int           { return len(f) }
func (f Funcs) Swap(i, j int)      { f[i], f[j] = f[j], f[i] }
func (f Funcs) Less(i, j int) bool { return f[i].FuncPos.Offset < f[j].FuncPos.Offset }

// Reserve reserves the Function data
func (f Funcs) Reserve() {
	for start, end := 0, f.Len()-1; start < end; {
		f.Swap(start, end)
		start++
		end--
	}
}

// Declarations returns a copy of funcs with only Function declarations
func (f Funcs) Declarations() Funcs {
	// NOTE(arslan): we can prepopulate these in the future, but again we need
	// to benchmark first
	var decls []*Func
	for _, fn := range f {
		if fn.IsDeclaration() {
			decls = append(decls, fn)
		}
	}
	return decls
}
<|endoftext|>


<commit_before>package lorawan

// CID defines the MAC command identifier.
type CID byte

// MAC commands as specified by the LoRaWAN R1.0 specs. Note that each *Req / *Ans
// has the same value. Based on the fact if a message is uplink or downlink
// you should use on or the other.
const (
	LinkCheckReq     CID = 0x02
	LinkCheckAns     CID = 0x02
	LinkADRReq       CID = 0x03
	LinkADRAns       CID = 0x03
	DutyCycleReq     CID = 0x04
	DutyCycleAns     CID = 0x04
	RXParamSetupReq  CID = 0x05
	RXParamSetupAns  CID = 0x05
	DevStatusReq     CID = 0x06
	DevStatusAns     CID = 0x06
	NewChannelReq    CID = 0x07
	NewChannelAns    CID = 0x07
	RXTimingSetupReq CID = 0x08
	RXTimingSetupAns CID = 0x08
	// 0x80 to 0xFF reserved for proprietary network command extensions
)
<commit_msg>Implement LinkCheckAnsPayload.<commit_after>package lorawan

// CID defines the MAC command identifier.
type CID byte

// MAC commands as specified by the LoRaWAN R1.0 specs. Note that each *Req / *Ans
// has the same value. Based on the fact if a message is uplink or downlink
// you should use on or the other.
const (
	LinkCheckReq     CID = 0x02
	LinkCheckAns     CID = 0x02
	LinkADRReq       CID = 0x03
	LinkADRAns       CID = 0x03
	DutyCycleReq     CID = 0x04
	DutyCycleAns     CID = 0x04
	RXParamSetupReq  CID = 0x05
	RXParamSetupAns  CID = 0x05
	DevStatusReq     CID = 0x06
	DevStatusAns     CID = 0x06
	NewChannelReq    CID = 0x07
	NewChannelAns    CID = 0x07
	RXTimingSetupReq CID = 0x08
	RXTimingSetupAns CID = 0x08
	// 0x80 to 0xFF reserved for proprietary network command extensions
)

// LinkCheckAnsPayload represents the LinkCheckAns payload.
type LinkCheckAnsPayload struct {
	Margin uint8
	GwCnt  uint8
}
<|endoftext|>


<commit_before>package main

import (
	"bsearch/index"
	"bsearch/ops"
	"bsearch/parser"
	"flag"
	"fmt"
	"os"
	"time"
	"log"
 	"runtime/pprof"
)

func usage() {
	fmt.Fprintf(os.Stderr, "usage: bsearch <path to index blob>\n")
	flag.PrintDefaults()
	os.Exit(1)
}

type headers map[string]string

func (h headers) Add(k, v string) {
	h[k] = v
}

func stdtest(in *index.Index) {
	p := parser.Parse(in, "0 lim:10 count_all(hej) root:10 OR magic:boll status:active")
	q := p.Stack[0]

	ops.Dump(q, 0)

	fmt.Printf("%v\n", in.Header())

	s := index.NullDoc()
	for true {
		d := q.NextDoc(s)
		if d == nil {
			break
		}
		fmt.Printf("%v\n", string(in.Docs[d.Id]))
		*s = *d
		s.Inc()
	}
	h := make(headers)
	q.ProcessHeaders(h)
	for k, v := range h {
		fmt.Printf("info:%v:%v\n", k, v)
	}
}

func bltest(in *index.Index) {
	p := parser.Parse(in, "0 count_all(hej) region:11 category:1000 OR category:2000")
	q := p.Stack[0]

	t1 := time.Now()
	s := index.NullDoc()
	for true {
		d := q.NextDoc(s)
		if d == nil {
			break
		}
		*s = *d
		s.Inc()
	}
	h := make(headers)
	q.ProcessHeaders(h)
	t2 := time.Now()
	fmt.Printf("rt: %v\n", t2.Sub(t1))
	for k, v := range h {
		fmt.Printf("info:%v:%v\n", k, v)
	}
}

var cpuprofile = flag.String("cpuprofile", "", "Write cpu profile to file")

func main() {
	flag.Parse()
	if flag.NArg() != 1 {
		usage()
	}
	dbname := flag.Arg(0)
	in, err := index.Open(dbname)
	if err != nil {
		log.Fatal(os.Stderr, "bindex.Open: %v\n", err)
	}
	defer in.Close()

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		defer f.Close()
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	t1 := time.Now()
	for i := 0; i < 10; i++ {
		bltest(in)
	}
	t2 := time.Now()
	fmt.Printf("t: %v\n", t2.Sub(t1))
}
<commit_msg>Improve the test.<commit_after>package main

import (
	"bsearch/index"
	"bsearch/ops"
	"bsearch/parser"
	"flag"
	"fmt"
	"os"
	"time"
	"log"
 	"runtime/pprof"
)

func usage() {
	fmt.Fprintf(os.Stderr, "usage: bsearch <path to index blob>\n")
	flag.PrintDefaults()
	os.Exit(1)
}

type headers map[string]string

func (h headers) Add(k, v string) {
	h[k] = v
}

func stdtest(in *index.Index) {
	p := parser.Parse(in, "0 lim:10 count_all(hej) root:10 OR magic:boll status:active")
	q := p.Stack[0]

	ops.Dump(q, 0)

	fmt.Printf("%v\n", in.Header())

	s := index.NullDoc()
	for true {
		d := q.NextDoc(s)
		if d == nil {
			break
		}
		fmt.Printf("%v\n", string(in.Docs[d.Id]))
		*s = *d
		s.Inc()
	}
	h := make(headers)
	q.ProcessHeaders(h)
	for k, v := range h {
		fmt.Printf("info:%v:%v\n", k, v)
	}
}

func bltest(in *index.Index) {
	p := parser.Parse(in, "0 count_all(hej) status:active region:11 category:1000 OR category:2000")
	q := p.Stack[0]

//	t1 := time.Now()
	s := index.NullDoc()
	for true {
		d := q.NextDoc(s)
		if d == nil {
			break
		}
		*s = *d
		s.Inc()
	}
	h := make(headers)
	q.ProcessHeaders(h)
//	t2 := time.Now()
//	fmt.Printf("rt: %v\n", t2.Sub(t1))
}

var cpuprofile = flag.String("cpuprofile", "", "Write cpu profile to file")

func main() {
	flag.Parse()
	if flag.NArg() != 1 {
		usage()
	}
	dbname := flag.Arg(0)
	in, err := index.Open(dbname)
	if err != nil {
		log.Fatal(os.Stderr, "bindex.Open: %v\n", err)
	}
	defer in.Close()

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		defer f.Close()
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	bltest(in)	// Warm up.
	t1 := time.Now()
	for i := 0; i < 10; i++ {
		bltest(in)
	}
	t2 := time.Now()
	fmt.Printf("t: %v\n", t2.Sub(t1))
}
<|endoftext|>


<commit_before>package market

import (
	"fmt"
	"regexp"
	"sort"

	"github.com/guotie/gogb2312"
	"github.com/nzai/go-utility/net"
)

// China 中国证券市场
type China struct{}

// Name 名称
func (m China) Name() string {
	return "China"
}

// Timezone 时区
func (m China) Timezone() string {
	return "Asia/Shanghai"
}

// Companies 上市公司
func (m China) Companies() ([]Company, error) {

	dict := make(map[string]Company, 0)

	//	上海证券交易所
	sh, err := m.shanghaiCompanies()
	if err != nil {
		return nil, err
	}
	for _, company := range sh {
		//	去重
		if _, found := dict[company.Code]; found {
			continue
		}

		dict[company.Code] = company
	}

	//	深圳证券交易所
	sz, err := m.shenzhenCompanies()
	if err != nil {
		return nil, err
	}
	for _, company := range sz {
		//	去重
		if _, found := dict[company.Code]; found {
			continue
		}

		dict[company.Code] = company
	}

	var companies []Company
	for _, company := range dict {
		companies = append(companies, company)
	}

	//	按Code排序
	sort.Sort(CompanyList(companies))

	return companies, nil
}

// shanghaiCompanies 上海证券交易所上市公司
func (m China) shanghaiCompanies() ([]Company, error) {

	urls := [...]string{
		"http://query.sse.com.cn/security/stock/downloadStockListFile.do?csrcCode=&stockCode=&areaName=&stockType=1",
		"http://query.sse.com.cn/security/stock/downloadStockListFile.do?csrcCode=&stockCode=&areaName=&stockType=2",
	}
	referer := "http://www.sse.com.cn/assortment/stock/list/share/"

	var list []Company
	for _, url := range urls {

		//	尝试从网络获取实时上市公司列表
		text, err := net.DownloadStringRefererRetry(url, referer, retryTimes, retryIntervalSeconds)
		if err != nil {
			return nil, err
		}

		//	解析json
		companies, err := m.parseShanghaiJSON(text)
		if err != nil {
			return nil, err
		}

		list = append(list, companies...)
	}

	return list, nil
}

// parseShanghaiJSON 解析上海证券交易所上市公司
func (m China) parseShanghaiJSON(text string) ([]Company, error) {

	//	深圳证券交易所的查询结果是GBK编码的，需要转成UTF8
	text, err, _, _ := gogb2312.ConvertGB2312String(text)
	if err != nil {
		return nil, err
	}

	//  使用正则分析json
	regex := regexp.MustCompile(`(\d{6})	  (\S+)	  \d{6}	  \S+`)
	group := regex.FindAllStringSubmatch(text, -1)

	var companies []Company
	for _, section := range group {
		companies = append(companies, Company{Code: section[1], Name: section[2]})
	}

	if len(companies) == 0 {
		return nil, fmt.Errorf("错误的上海证券交易所上市公司列表内容:%s", text)
	}

	return companies, nil
}

//	深圳证券交易所上市公司
func (m China) shenzhenCompanies() ([]Company, error) {
	urls := [...]string{
		"http://www.szse.cn/szseWeb/ShowReport.szse?SHOWTYPE=EXCEL&CATALOGID=1110&tab1PAGENUM=1&ENCODE=1&TABKEY=tab1",
	}

	var list []Company
	for _, url := range urls {

		//	尝试从网络获取实时上市公司列表
		html, err := net.DownloadStringRetry(url, retryTimes, retryIntervalSeconds)
		if err != nil {
			return nil, err
		}

		//	深圳证券交易所的查询结果是GBK编码的，需要转成UTF8
		html, err, _, _ = gogb2312.ConvertGB2312String(html)
		if err != nil {
			return nil, err
		}

		//	解析Html
		companies, err := m.parseShenzhenHTML(html)
		if err != nil {
			return nil, err
		}

		list = append(list, companies...)
	}

	return list, nil
}

// parseShenzhenHTML 解析深圳证券交易所上市公司
func (m China) parseShenzhenHTML(html string) ([]Company, error) {
	//  使用正则分析html
	regex := regexp.MustCompile(`null align='center' >(\d{6})</td><td  class='cls-data-td' null align='center' >([^<]*?)</td>`)
	group := regex.FindAllStringSubmatch(html, -1)

	var companies []Company
	for _, section := range group {
		companies = append(companies, Company{Code: section[1], Name: section[2]})
	}

	if len(companies) == 0 {
		return nil, fmt.Errorf("错误的深圳证券交易所上市公司列表内容:%s", html)
	}

	return companies, nil
}

// YahooQueryCode 雅虎查询代码
func (m China) YahooQueryCode(company Company) string {

	var suffix string
	switch company.Code[:1] {
	case "0":
		suffix = "SZ"
	case "2":
		suffix = "SZ"
	case "3":
		suffix = "SZ"
	case "9":
		suffix = "SS"
	case "6":
		suffix = "SS"
	default:
		suffix = "SS"
	}

	return company.Code + "." + suffix
}
<commit_msg>[Misc]适应深圳股市的上市公司列表新格式<commit_after>package market

import (
	"fmt"
	"regexp"
	"sort"

	"github.com/guotie/gogb2312"
	"github.com/nzai/go-utility/net"
)

// China 中国证券市场
type China struct{}

// Name 名称
func (m China) Name() string {
	return "China"
}

// Timezone 时区
func (m China) Timezone() string {
	return "Asia/Shanghai"
}

// Companies 上市公司
func (m China) Companies() ([]Company, error) {

	dict := make(map[string]Company, 0)

	//	上海证券交易所
	sh, err := m.shanghaiCompanies()
	if err != nil {
		return nil, err
	}
	for _, company := range sh {
		//	去重
		if _, found := dict[company.Code]; found {
			continue
		}

		dict[company.Code] = company
	}

	//	深圳证券交易所
	sz, err := m.shenzhenCompanies()
	if err != nil {
		return nil, err
	}
	for _, company := range sz {
		//	去重
		if _, found := dict[company.Code]; found {
			continue
		}

		dict[company.Code] = company
	}

	var companies []Company
	for _, company := range dict {
		companies = append(companies, company)
	}

	//	按Code排序
	sort.Sort(CompanyList(companies))

	return companies, nil
}

// shanghaiCompanies 上海证券交易所上市公司
func (m China) shanghaiCompanies() ([]Company, error) {

	urls := [...]string{
		"http://query.sse.com.cn/security/stock/downloadStockListFile.do?csrcCode=&stockCode=&areaName=&stockType=1",
		"http://query.sse.com.cn/security/stock/downloadStockListFile.do?csrcCode=&stockCode=&areaName=&stockType=2",
	}
	referer := "http://www.sse.com.cn/assortment/stock/list/share/"

	var list []Company
	for _, url := range urls {

		//	尝试从网络获取实时上市公司列表
		text, err := net.DownloadStringRefererRetry(url, referer, retryTimes, retryIntervalSeconds)
		if err != nil {
			return nil, err
		}

		//	解析json
		companies, err := m.parseShanghaiJSON(text)
		if err != nil {
			return nil, err
		}

		list = append(list, companies...)
	}

	return list, nil
}

// parseShanghaiJSON 解析上海证券交易所上市公司
func (m China) parseShanghaiJSON(text string) ([]Company, error) {

	//	深圳证券交易所的查询结果是GBK编码的，需要转成UTF8
	text, err, _, _ := gogb2312.ConvertGB2312String(text)
	if err != nil {
		return nil, err
	}

	//  使用正则分析json
	regex := regexp.MustCompile(`(\d{6})	  (\S+)	  \d{6}	  \S+`)
	group := regex.FindAllStringSubmatch(text, -1)

	var companies []Company
	for _, section := range group {
		companies = append(companies, Company{Code: section[1], Name: section[2]})
	}

	if len(companies) == 0 {
		return nil, fmt.Errorf("错误的上海证券交易所上市公司列表内容:%s", text)
	}

	return companies, nil
}

//	深圳证券交易所上市公司
func (m China) shenzhenCompanies() ([]Company, error) {
	urls := [...]string{
		"http://www.szse.cn/szseWeb/ShowReport.szse?SHOWTYPE=EXCEL&CATALOGID=1110&tab1PAGENUM=1&ENCODE=1&TABKEY=tab1",
	}

	var list []Company
	for _, url := range urls {

		//	尝试从网络获取实时上市公司列表
		html, err := net.DownloadStringRetry(url, retryTimes, retryIntervalSeconds)
		if err != nil {
			return nil, err
		}

		//	深圳证券交易所的查询结果是GBK编码的，需要转成UTF8
		html, err, _, _ = gogb2312.ConvertGB2312String(html)
		if err != nil {
			return nil, err
		}

		//	解析Html
		companies, err := m.parseShenzhenHTML(html)
		if err != nil {
			return nil, err
		}

		list = append(list, companies...)
	}

	return list, nil
}

// parseShenzhenHTML 解析深圳证券交易所上市公司
func (m China) parseShenzhenHTML(html string) ([]Company, error) {
	//  使用正则分析html
	regex := regexp.MustCompile(`@' align='center'  >(\d{6})</td><td  class='cls-data-td' null align='center'  >([^<]*?)</td>`)
	group := regex.FindAllStringSubmatch(html, -1)

	var companies []Company
	for _, section := range group {
		companies = append(companies, Company{Code: section[1], Name: section[2]})
	}

	if len(companies) == 0 {
		return nil, fmt.Errorf("错误的深圳证券交易所上市公司列表内容:%s", html)
	}

	return companies, nil
}

// YahooQueryCode 雅虎查询代码
func (m China) YahooQueryCode(company Company) string {

	var suffix string
	switch company.Code[:1] {
	case "0":
		suffix = "SZ"
	case "2":
		suffix = "SZ"
	case "3":
		suffix = "SZ"
	case "9":
		suffix = "SS"
	case "6":
		suffix = "SS"
	default:
		suffix = "SS"
	}

	return company.Code + "." + suffix
}
<|endoftext|>


<commit_before>package mat

import (
	"math"
)

type Matrix struct {
	Vals []float64
	Width, Height int
}

type LUFactors struct {
	lu Matrix
	pivot []int
	d float64
}

func NewMatrix(vals []float64, width, height int) *Matrix {
	if width <= 0 {
		panic("width must be positive.")
	} else if height <= 0 {
		panic("height must be positive.")
	} else if width * height != len(vals) {
		panic("height * width must equal len(vals).")
	}

	return &Matrix{Vals: vals, Width: width, Height: height}
}

func NewLUFactors(n int) *LUFactors {
	luf := new(LUFactors)

	luf.lu.Vals, luf.lu.Width, luf.lu.Height = make([]float64, n*n), n, n
	luf.pivot = make([]int, n)
	luf.d = 1

	return luf
}

func (m *Matrix) LU() *LUFactors {
	if m.Width != m.Height { panic("m is non-square.") }

	lu := NewLUFactors(m.Width)
	m.LUFactorsAt(lu)
	return lu
}

func (m *Matrix) LUFactorsAt(luf *LUFactors) {
	if luf.lu.Width != m.Width || luf.lu.Height != m.Height {
		panic("luf has different dimenstions than m.")
	}

	n := m.Width
	for i := 0; i < n; i++ { luf.pivot[i] = i }
	lu := luf.lu.Vals
	mat := m.Vals

	// Maintained for determinant calculations.
	luf.d = 1

	// Crout's algorithm.
	copy(lu, m.Vals)

	// Swap rows.
	for k := 0; k < n; k++ {
		maxRow := findMaxRow(n, mat, k)
		luf.pivot[k], luf.pivot[maxRow] = luf.pivot[maxRow], luf.pivot[k]

		if k != maxRow {
			swapRows(k, maxRow, n, lu)
			luf.d = -luf.d
		}
	}

	// This nonsense.
	for k := 0; k < n; k++ {
		kOffset := k*n
		for i := k + 1; i < n; i++ {
			iOffset := i*n
			lu[iOffset + k] /= lu[kOffset + k]
			tmp := lu[iOffset + k]
			for j := k + 1; j < n; j++ {
				lu[iOffset + j] -= tmp * lu[kOffset + j]
			}
		}
	}
}

// Finds the index of the row containing the maximum value in the column.
// Ignores the values above the point m_col,col since those have already been
// swapped.
func findMaxRow(n int, m []float64, col int) int {
	max, maxRow := -1.0, 0

	for i := col; i < n; i++ {
		val := math.Abs(m[i*n + col])
		if val > max {
			max = val
			maxRow = i
		}
	}
	return maxRow
}

func swapRows(i1, i2, n int, lu []float64) {
	i1Offset, i2Offset := n*i1, n*i2
	for j := 0; j < n; j++ {
		idx1, idx2 := i1Offset + j, i2Offset + j
		lu[idx1], lu[idx2] = lu[idx2], lu[idx1]
	}
}

// SolveVector solves M * xs = bs for xs.
//
// bs and xs may poin to the same physical memory.
func (luf *LUFactors) SolveVector(bs, xs []float64) {
	n := luf.lu.Width
	if n != len(bs) {
		panic("len(b) != luf.Width")
	} else if n != len(xs) {
		panic("len(x) != luf.Width")
	}

	// A x = b -> (L U) x = b -> L (U x) = b -> L y = b
	ys := xs
	if &bs[0] == &ys[0] {
		bs = make([]float64, n)
		copy(bs, ys)
	}

	// Solve L * y = b for y.
	forwardSubst(n, luf.pivot, luf.lu.Vals, bs, ys)
	// Solve U * x = y for x.
	backSubst(n, luf.lu.Vals, ys, xs)
}

// Solves L * y = b for y.
// y_i = (b_i - sum_j=0^i-1 (alpha_ij y_j)) / alpha_ij
func forwardSubst(n int, pivot []int, lu, bs, ys []float64) {
	for i := 0; i < n; i++ {
		ys[pivot[i]] = bs[i]
	}
	for i := 0; i < n; i++ {
		sum := 0.0
		for j := 0; j < i; j++ {
			sum += lu[i*n + j] * ys[j]
		}
		ys[i] = (ys[i] - sum)
	}
}

// Solves U * x = y for x.
// x_i = (y_i - sum_j=i+^N-1 (beta_ij x_j)) / beta_ii
func backSubst(n int, lu, ys, xs []float64) {
	for i := n - 1; i >= 0; i-- {
		sum := 0.0
		for j := i + 1; j < n; j++ {
			sum += lu[i*n + j] * xs[j]
		}
		xs[i] = (ys[i] - sum) / lu[i*n + i]
	}
}

// SolveMatrix solves the equation m * x = b.
//
// x and b may point to the same physical memory.
func (luf *LUFactors) SolveMatrix(b, x *Matrix) {
	xs := x.Vals
	n := luf.lu.Width

	if b.Width != b.Height {
		panic("b matrix is non-square.")
	} else if x.Width != x.Height {
		panic("x matrix is non-square.")
	} else if n != b.Width {
		panic("b matrix different size than m matrix.")
	} else if n != x.Width {
		panic("x matrix different size than m matrix.")
	}

	col := make([]float64, n)

	for j := 0; j < n; j++ {
		for i := 0; i < n; i++ {
			col[i] = xs[i*n + j]
		}
		luf.SolveVector(col, col)
		for i := 0; i < n; i++ {
			xs[i*n + j] = col[i]
		}
	}
}

func (luf *LUFactors) Invert(out *Matrix) {
	n := luf.lu.Width
	if out.Width != out.Height {
		panic("out matrix is non-square.")
	} else if n != out.Width {
		panic("out matrix different size than m matrix.")
	}

	for i := range out.Vals {
		out.Vals[i] = 0
	}
	for i := 0; i < n; i++ {
		out.Vals[i*n + i] = 1
	}

	luf.SolveMatrix(out, out)
}

func (luf *LUFactors) Determinant() float64 {
	d := luf.d
	lu := luf.lu.Vals
	n := luf.lu.Width

	for i := 0; i < luf.lu.Width; i++ {
		d *= lu[i*n + i]
	}
	return d
}
<commit_msg>Added documentation to matrix library.<commit_after>/*mat contains routines for executing operations on matrices. Opersions are
split into easy to use methods which might be somewhat wasteful with memory
consumption and execution time and slightly less easy to use methods which
require explictly managing LU decomposition:

Pretty much everything only works on square matrices because that's all I've
needed so far.
*/
package mat

import (
	"math"
)

// Matrix represents a matrix of float64 values.
type Matrix struct {
	Vals []float64
	Width, Height int
}

// LUFactors contains data fields neccessary for a number of matrix operations.
// Exporting this type allows calling routines to better manage their memory
// consumption and to prevent recomputing the same decomposition many times.
type LUFactors struct {
	lu Matrix
	pivot []int
	d float64
}

// New matrix creates a matrix with the specified values and dimensions.
func NewMatrix(vals []float64, width, height int) *Matrix {
	if width <= 0 {
		panic("width must be positive.")
	} else if height <= 0 {
		panic("height must be positive.")
	} else if width * height != len(vals) {
		panic("height * width must equal len(vals).")
	}

	return &Matrix{Vals: vals, Width: width, Height: height}
}

// NewLUFactors creates an LUFactors instance of the requested dimensions.
func NewLUFactors(n int) *LUFactors {
	luf := new(LUFactors)

	luf.lu.Vals, luf.lu.Width, luf.lu.Height = make([]float64, n*n), n, n
	luf.pivot = make([]int, n)
	luf.d = 1

	return luf
}

// LU returns the LU decomposition of a matrix.
func (m *Matrix) LU() *LUFactors {
	if m.Width != m.Height { panic("m is non-square.") }

	lu := NewLUFactors(m.Width)
	m.LUFactorsAt(lu)
	return lu
}

// LUFactorsAt stores the LU decomposition of a matrix at the specified
// location.
func (m *Matrix) LUFactorsAt(luf *LUFactors) {
	if luf.lu.Width != m.Width || luf.lu.Height != m.Height {
		panic("luf has different dimenstions than m.")
	}

	n := m.Width
	for i := 0; i < n; i++ { luf.pivot[i] = i }
	lu := luf.lu.Vals
	mat := m.Vals

	// Maintained for determinant calculations.
	luf.d = 1

	// Crout's algorithm.
	copy(lu, m.Vals)

	// Swap rows.
	for k := 0; k < n; k++ {
		maxRow := findMaxRow(n, mat, k)
		luf.pivot[k], luf.pivot[maxRow] = luf.pivot[maxRow], luf.pivot[k]

		if k != maxRow {
			swapRows(k, maxRow, n, lu)
			luf.d = -luf.d
		}
	}

	// This nonsense.
	for k := 0; k < n; k++ {
		kOffset := k*n
		for i := k + 1; i < n; i++ {
			iOffset := i*n
			lu[iOffset + k] /= lu[kOffset + k]
			tmp := lu[iOffset + k]
			for j := k + 1; j < n; j++ {
				lu[iOffset + j] -= tmp * lu[kOffset + j]
			}
		}
	}
}

// Finds the index of the row containing the maximum value in the column.
// Ignores the values above the point m_col,col since those have already been
// swapped.
func findMaxRow(n int, m []float64, col int) int {
	max, maxRow := -1.0, 0

	for i := col; i < n; i++ {
		val := math.Abs(m[i*n + col])
		if val > max {
			max = val
			maxRow = i
		}
	}
	return maxRow
}

func swapRows(i1, i2, n int, lu []float64) {
	i1Offset, i2Offset := n*i1, n*i2
	for j := 0; j < n; j++ {
		idx1, idx2 := i1Offset + j, i2Offset + j
		lu[idx1], lu[idx2] = lu[idx2], lu[idx1]
	}
}

// SolveVector solves M * xs = bs for xs.
//
// bs and xs may poin to the same physical memory.
func (luf *LUFactors) SolveVector(bs, xs []float64) {
	n := luf.lu.Width
	if n != len(bs) {
		panic("len(b) != luf.Width")
	} else if n != len(xs) {
		panic("len(x) != luf.Width")
	}

	// A x = b -> (L U) x = b -> L (U x) = b -> L y = b
	ys := xs
	if &bs[0] == &ys[0] {
		bs = make([]float64, n)
		copy(bs, ys)
	}

	// Solve L * y = b for y.
	forwardSubst(n, luf.pivot, luf.lu.Vals, bs, ys)
	// Solve U * x = y for x.
	backSubst(n, luf.lu.Vals, ys, xs)
}

// Solves L * y = b for y.
// y_i = (b_i - sum_j=0^i-1 (alpha_ij y_j)) / alpha_ij
func forwardSubst(n int, pivot []int, lu, bs, ys []float64) {
	for i := 0; i < n; i++ {
		ys[pivot[i]] = bs[i]
	}
	for i := 0; i < n; i++ {
		sum := 0.0
		for j := 0; j < i; j++ {
			sum += lu[i*n + j] * ys[j]
		}
		ys[i] = (ys[i] - sum)
	}
}

// Solves U * x = y for x.
// x_i = (y_i - sum_j=i+^N-1 (beta_ij x_j)) / beta_ii
func backSubst(n int, lu, ys, xs []float64) {
	for i := n - 1; i >= 0; i-- {
		sum := 0.0
		for j := i + 1; j < n; j++ {
			sum += lu[i*n + j] * xs[j]
		}
		xs[i] = (ys[i] - sum) / lu[i*n + i]
	}
}

// SolveMatrix solves the equation m * x = b.
//
// x and b may point to the same physical memory.
func (luf *LUFactors) SolveMatrix(b, x *Matrix) {
	xs := x.Vals
	n := luf.lu.Width

	if b.Width != b.Height {
		panic("b matrix is non-square.")
	} else if x.Width != x.Height {
		panic("x matrix is non-square.")
	} else if n != b.Width {
		panic("b matrix different size than m matrix.")
	} else if n != x.Width {
		panic("x matrix different size than m matrix.")
	}

	col := make([]float64, n)

	for j := 0; j < n; j++ {
		for i := 0; i < n; i++ {
			col[i] = xs[i*n + j]
		}
		luf.SolveVector(col, col)
		for i := 0; i < n; i++ {
			xs[i*n + j] = col[i]
		}
	}
}

// InvertAt inverts the matrix represented by the given LU decomposition
// and writes the results into the specified out matrix.
func (luf *LUFactors) InvertAt(out *Matrix) {
	n := luf.lu.Width
	if out.Width != out.Height {
		panic("out matrix is non-square.")
	} else if n != out.Width {
		panic("out matrix different size than m matrix.")
	}

	for i := range out.Vals {
		out.Vals[i] = 0
	}
	for i := 0; i < n; i++ {
		out.Vals[i*n + i] = 1
	}

	luf.SolveMatrix(out, out)
}

// Determinant compute the determinant of of the matrix represented by the
// given LU decomposition.
func (luf *LUFactors) Determinant() float64 {
	d := luf.d
	lu := luf.lu.Vals
	n := luf.lu.Width

	for i := 0; i < luf.lu.Width; i++ {
		d *= lu[i*n + i]
	}
	return d
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/42wim/matterbridge/bridge/config"
	"github.com/42wim/matterbridge/gateway"
	log "github.com/Sirupsen/logrus"
	"github.com/google/gops/agent"
	"os"
	"strings"
)

var (
	version = "1.6.0"
	githash string
)

func init() {
	log.SetFormatter(&log.TextFormatter{FullTimestamp: true})
}

func main() {
	flagConfig := flag.String("conf", "matterbridge.toml", "config file")
	flagDebug := flag.Bool("debug", false, "enable debug")
	flagVersion := flag.Bool("version", false, "show version")
	flagGops := flag.Bool("gops", false, "enable gops agent")
	flag.Parse()
	if *flagGops {
		agent.Listen(&agent.Options{})
		defer agent.Close()
	}
	if *flagVersion {
		fmt.Printf("version: %s %s\n", version, githash)
		return
	}
	if *flagDebug || os.Getenv("DEBUG") == "1" {
		log.Info("Enabling debug")
		log.SetLevel(log.DebugLevel)
	}
	log.Printf("Running version %s %s", version, githash)
	if strings.Contains(version, "-dev") {
		log.Println("WARNING: THIS IS A DEVELOPMENT VERSION. Things may break.")
	}
	cfg := config.NewConfig(*flagConfig)
	r, err := gateway.NewRouter(cfg)
	if err != nil {
		log.Fatalf("Starting gateway failed: %s", err)
	}
	err = r.Start()
	if err != nil {
		log.Fatalf("Starting gateway failed: %s", err)
	}
	log.Printf("Gateway(s) started succesfully. Now relaying messages")
	select {}
}
<commit_msg>Bump version<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/42wim/matterbridge/bridge/config"
	"github.com/42wim/matterbridge/gateway"
	log "github.com/Sirupsen/logrus"
	"github.com/google/gops/agent"
	"os"
	"strings"
)

var (
	version = "1.6.1-dev"
	githash string
)

func init() {
	log.SetFormatter(&log.TextFormatter{FullTimestamp: true})
}

func main() {
	flagConfig := flag.String("conf", "matterbridge.toml", "config file")
	flagDebug := flag.Bool("debug", false, "enable debug")
	flagVersion := flag.Bool("version", false, "show version")
	flagGops := flag.Bool("gops", false, "enable gops agent")
	flag.Parse()
	if *flagGops {
		agent.Listen(&agent.Options{})
		defer agent.Close()
	}
	if *flagVersion {
		fmt.Printf("version: %s %s\n", version, githash)
		return
	}
	if *flagDebug || os.Getenv("DEBUG") == "1" {
		log.Info("Enabling debug")
		log.SetLevel(log.DebugLevel)
	}
	log.Printf("Running version %s %s", version, githash)
	if strings.Contains(version, "-dev") {
		log.Println("WARNING: THIS IS A DEVELOPMENT VERSION. Things may break.")
	}
	cfg := config.NewConfig(*flagConfig)
	r, err := gateway.NewRouter(cfg)
	if err != nil {
		log.Fatalf("Starting gateway failed: %s", err)
	}
	err = r.Start()
	if err != nil {
		log.Fatalf("Starting gateway failed: %s", err)
	}
	log.Printf("Gateway(s) started succesfully. Now relaying messages")
	select {}
}
<|endoftext|>


<commit_before>package letters

const registerBoletoResponse = `{
    {{if (trim .errorCode) ne ""}}
    "Errors": [
        {
            "Code": "{{trim .errorCode}}",
            "Message": "{{trim .errorMessage}}"
        }
    ]
    {{else}}
    "Url": null,
    "DigitableLine": "{{trim .digitableLine}}",
    "BarCodeNumber": "{{trim .barcodeNumber}}"
    {{end}}
}
`

//GetRegisterBoletoAPIResponseTmpl retorna o template de resposta para a Api
func GetRegisterBoletoAPIResponseTmpl() string {
	return registerBoletoResponse
}
<commit_msg>:art: Formata linha digitável na resposta do JSON<commit_after>package letters

const registerBoletoResponse = `{
    {{if (trim .errorCode) ne ""}}
    "Errors": [
        {
            "Code": "{{trim .errorCode}}",
            "Message": "{{trim .errorMessage}}"
        }
    ]
    {{else}}
    "Url": null,
    "DigitableLine": "{{fmtDigitableLine (trim .digitableLine)}}",
    "BarCodeNumber": "{{trim .barcodeNumber}}"
    {{end}}
}
`

//GetRegisterBoletoAPIResponseTmpl retorna o template de resposta para a Api
func GetRegisterBoletoAPIResponseTmpl() string {
	return registerBoletoResponse
}
<|endoftext|>


<commit_before>package slashcommands

import (
	"strings"

	"github.com/otiai10/chant/server/middleware"
	"github.com/otiai10/chant/server/models"
)

// Help ...
type Help struct {
}

// Handle ...
func (cmd Help) Handle(req *SlashCommandRequest) error {
	ctx := middleware.Context(req.Request)
	bot := models.Bot()
	text := strings.Join([]string{
		Hello{}.Help(),
		Help{}.Help(),
	}, "\n")
	message := models.NewMessage(text, bot)
	return message.Push(ctx)
}

// Help ...
func (cmd Help) Help() string {
	return `/help
-- Help!`
}
<commit_msg>/help show help message dynamically<commit_after>package slashcommands

import (
	"strings"

	"github.com/otiai10/chant/server/middleware"
	"github.com/otiai10/chant/server/models"
)

// Help ...
type Help struct {
}

// Handle ...
func (cmd Help) Handle(req *SlashCommandRequest) error {
	ctx := middleware.Context(req.Request)
	bot := models.Bot()
	helps := []string{}
	for _, cmd := range Commands {
		helps = append(helps, cmd.Help())
	}
	text := strings.Join(helps, "\n")
	message := models.NewMessage(text, bot)
	return message.Push(ctx)
}

// Help ...
func (cmd Help) Help() string {
	return `/help
-- Help!`
}
<|endoftext|>


<commit_before>package miner

import (
	"fmt"
	"math/big"
	"sort"
	"sync"
	"sync/atomic"

	"github.com/ethereum/go-ethereum/common"
	"github.com/ethereum/go-ethereum/core"
	"github.com/ethereum/go-ethereum/core/state"
	"github.com/ethereum/go-ethereum/core/types"
	"github.com/ethereum/go-ethereum/event"
	"github.com/ethereum/go-ethereum/logger"
	"github.com/ethereum/go-ethereum/logger/glog"
	"github.com/ethereum/go-ethereum/pow"
	"gopkg.in/fatih/set.v0"
)

var jsonlogger = logger.NewJsonLogger()

type environment struct {
	totalUsedGas *big.Int
	state        *state.StateDB
	coinbase     *state.StateObject
	block        *types.Block
	family       *set.Set
	uncles       *set.Set
}

func env(block *types.Block, eth core.Backend) *environment {
	state := state.New(block.Root(), eth.StateDb())
	env := &environment{
		totalUsedGas: new(big.Int),
		state:        state,
		block:        block,
		family:       set.New(),
		uncles:       set.New(),
		coinbase:     state.GetOrNewStateObject(block.Coinbase()),
	}

	return env
}

type Work struct {
	Number    uint64
	Nonce     uint64
	MixDigest []byte
	SeedHash  []byte
}

type Agent interface {
	Work() chan<- *types.Block
	SetReturnCh(chan<- *types.Block)
	Stop()
	Start()
	GetHashRate() int64
}

type worker struct {
	mu sync.Mutex

	agents []Agent
	recv   chan *types.Block
	mux    *event.TypeMux
	quit   chan struct{}
	pow    pow.PoW
	atWork int64

	eth   core.Backend
	chain *core.ChainManager
	proc  *core.BlockProcessor

	coinbase common.Address
	extra    []byte

	currentMu sync.Mutex
	current   *environment

	uncleMu        sync.Mutex
	possibleUncles map[common.Hash]*types.Block

	txQueueMu sync.Mutex
	txQueue   map[common.Hash]*types.Transaction

	mining int64
}

func newWorker(coinbase common.Address, eth core.Backend) *worker {
	worker := &worker{
		eth:            eth,
		mux:            eth.EventMux(),
		recv:           make(chan *types.Block),
		chain:          eth.ChainManager(),
		proc:           eth.BlockProcessor(),
		possibleUncles: make(map[common.Hash]*types.Block),
		coinbase:       coinbase,
		txQueue:        make(map[common.Hash]*types.Transaction),
		quit:           make(chan struct{}),
	}
	go worker.update()
	go worker.wait()

	worker.commitNewWork()

	return worker
}

func (self *worker) pendingState() *state.StateDB {
	self.currentMu.Lock()
	defer self.currentMu.Unlock()

	return self.current.state
}

func (self *worker) pendingBlock() *types.Block {
	self.currentMu.Lock()
	defer self.currentMu.Unlock()

	return self.current.block
}

func (self *worker) start() {
	// spin up agents
	for _, agent := range self.agents {
		agent.Start()
	}

	atomic.StoreInt64(&self.mining, 1)
}

func (self *worker) stop() {
	if atomic.LoadInt64(&self.mining) == 1 {
		// stop all agents
		for _, agent := range self.agents {
			agent.Stop()
		}
	}

	atomic.StoreInt64(&self.mining, 0)
	atomic.StoreInt64(&self.atWork, 0)
}

func (self *worker) register(agent Agent) {
	self.agents = append(self.agents, agent)
	agent.SetReturnCh(self.recv)
}

func (self *worker) update() {
	events := self.mux.Subscribe(core.ChainHeadEvent{}, core.ChainSideEvent{}, core.TxPreEvent{})

out:
	for {
		select {
		case event := <-events.Chan():
			switch ev := event.(type) {
			case core.ChainHeadEvent:
				self.commitNewWork()
			case core.ChainSideEvent:
				self.uncleMu.Lock()
				self.possibleUncles[ev.Block.Hash()] = ev.Block
				self.uncleMu.Unlock()
			case core.TxPreEvent:
				if atomic.LoadInt64(&self.mining) == 0 {
					self.commitNewWork()
				}
			}
		case <-self.quit:
			break out
		}
	}

	events.Unsubscribe()
}

func (self *worker) wait() {
	for {
		for block := range self.recv {
			atomic.AddInt64(&self.atWork, -1)

			if block == nil {
				continue
			}

			if err := self.chain.InsertChain(types.Blocks{block}); err == nil {
				for _, uncle := range block.Uncles() {
					delete(self.possibleUncles, uncle.Hash())
				}
				self.mux.Post(core.NewMinedBlockEvent{block})

				glog.V(logger.Info).Infof("🔨  Mined block #%v", block.Number())

				jsonlogger.LogJson(&logger.EthMinerNewBlock{
					BlockHash:     block.Hash().Hex(),
					BlockNumber:   block.Number(),
					ChainHeadHash: block.ParentHeaderHash.Hex(),
					BlockPrevHash: block.ParentHeaderHash.Hex(),
				})
			} else {
				self.commitNewWork()
			}
		}
	}
}

func (self *worker) push() {
	if atomic.LoadInt64(&self.mining) == 1 {
		self.current.block.Header().GasUsed = self.current.totalUsedGas
		self.current.block.SetRoot(self.current.state.Root())

		// push new work to agents
		for _, agent := range self.agents {
			atomic.AddInt64(&self.atWork, 1)

			if agent.Work() != nil {
				agent.Work() <- self.current.block.Copy()
			} else {
				common.Report(fmt.Sprintf("%v %T\n", agent, agent))
			}
		}
	}
}

func (self *worker) makeCurrent() {
	block := self.chain.NewBlock(self.coinbase)
	if block.Time() == self.chain.CurrentBlock().Time() {
		block.Header().Time++
	}
	block.Header().Extra = self.extra

	self.current = env(block, self.eth)
	for _, ancestor := range self.chain.GetAncestors(block, 7) {
		self.current.family.Add(ancestor.Hash())
	}

	parent := self.chain.GetBlock(self.current.block.ParentHash())
	self.current.coinbase.SetGasPool(core.CalcGasLimit(parent, self.current.block))
}

func (self *worker) commitNewWork() {
	self.mu.Lock()
	defer self.mu.Unlock()
	self.uncleMu.Lock()
	defer self.uncleMu.Unlock()
	self.currentMu.Lock()
	defer self.currentMu.Unlock()

	self.makeCurrent()

	transactions := self.eth.TxPool().GetTransactions()
	sort.Sort(types.TxByNonce{transactions})

	// Keep track of transactions which return errors so they can be removed
	var (
		remove = set.New()
		tcount = 0
	)
	//gasLimit:
	for _, tx := range transactions {
		self.current.state.StartRecord(tx.Hash(), common.Hash{}, 0)

		err := self.commitTransaction(tx)
		switch {
		case core.IsNonceErr(err) || core.IsInvalidTxErr(err):
			// Remove invalid transactions
			from, _ := tx.From()
			self.chain.TxState().RemoveNonce(from, tx.Nonce())
			remove.Add(tx.Hash())

			if glog.V(logger.Detail) {
				glog.Infof("TX (%x) failed, will be removed: %v\n", tx.Hash().Bytes()[:4], err)
				//glog.Infoln(tx)
			}
		case state.IsGasLimitErr(err):
			//glog.V(logger.Debug).Infof("Gas limit reached for block. %d TXs included in this block\n", i)
			//break gasLimit
		default:
			tcount++
		}
	}
	//self.eth.TxPool().InvalidateSet(remove)

	var (
		uncles    []*types.Header
		badUncles []common.Hash
	)
	for hash, uncle := range self.possibleUncles {
		if len(uncles) == 2 {
			break
		}

		if err := self.commitUncle(uncle.Header()); err != nil {
			if glog.V(logger.Ridiculousness) {
				glog.V(logger.Detail).Infof("Bad uncle found and will be removed (%x)\n", hash[:4])
				glog.V(logger.Detail).Infoln(uncle)
			}

			badUncles = append(badUncles, hash)
		} else {
			glog.V(logger.Debug).Infof("commiting %x as uncle\n", hash[:4])
			uncles = append(uncles, uncle.Header())
		}
	}

	// We only care about logging if we're actually mining
	if atomic.LoadInt64(&self.mining) == 1 {
		glog.V(logger.Info).Infof("commit new work on block %v with %d txs & %d uncles\n", self.current.block.Number(), tcount, len(uncles))
	}

	for _, hash := range badUncles {
		delete(self.possibleUncles, hash)
	}

	self.current.block.SetUncles(uncles)

	core.AccumulateRewards(self.current.state, self.current.block)

	self.current.state.Update()

	self.push()
}

var (
	inclusionReward = new(big.Int).Div(core.BlockReward, big.NewInt(32))
	_uncleReward    = new(big.Int).Mul(core.BlockReward, big.NewInt(15))
	uncleReward     = new(big.Int).Div(_uncleReward, big.NewInt(16))
)

func (self *worker) commitUncle(uncle *types.Header) error {
	if self.current.uncles.Has(uncle.Hash()) {
		// Error not unique
		return core.UncleError("Uncle not unique")
	}
	self.current.uncles.Add(uncle.Hash())

	if !self.current.family.Has(uncle.ParentHash) {
		return core.UncleError(fmt.Sprintf("Uncle's parent unknown (%x)", uncle.ParentHash[0:4]))
	}

	if self.current.family.Has(uncle.Hash()) {
		return core.UncleError(fmt.Sprintf("Uncle already in family (%x)", uncle.Hash()))
	}

	return nil
}

func (self *worker) commitTransaction(tx *types.Transaction) error {
	snap := self.current.state.Copy()
	receipt, _, err := self.proc.ApplyTransaction(self.current.coinbase, self.current.state, self.current.block, tx, self.current.totalUsedGas, true)
	if err != nil && (core.IsNonceErr(err) || state.IsGasLimitErr(err) || core.IsInvalidTxErr(err)) {
		self.current.state.Set(snap)
		return err
	}

	self.current.block.AddTransaction(tx)
	self.current.block.AddReceipt(receipt)

	return nil
}

func (self *worker) HashRate() int64 {
	var tot int64
	for _, agent := range self.agents {
		tot += agent.GetHashRate()
	}

	return tot
}
<commit_msg>miner: ignore a tx's transactor after a gas limit has been returned<commit_after>package miner

import (
	"fmt"
	"math/big"
	"sort"
	"sync"
	"sync/atomic"

	"github.com/ethereum/go-ethereum/common"
	"github.com/ethereum/go-ethereum/core"
	"github.com/ethereum/go-ethereum/core/state"
	"github.com/ethereum/go-ethereum/core/types"
	"github.com/ethereum/go-ethereum/event"
	"github.com/ethereum/go-ethereum/logger"
	"github.com/ethereum/go-ethereum/logger/glog"
	"github.com/ethereum/go-ethereum/pow"
	"gopkg.in/fatih/set.v0"
)

var jsonlogger = logger.NewJsonLogger()

type environment struct {
	totalUsedGas *big.Int
	state        *state.StateDB
	coinbase     *state.StateObject
	block        *types.Block
	family       *set.Set
	uncles       *set.Set
}

func env(block *types.Block, eth core.Backend) *environment {
	state := state.New(block.Root(), eth.StateDb())
	env := &environment{
		totalUsedGas: new(big.Int),
		state:        state,
		block:        block,
		family:       set.New(),
		uncles:       set.New(),
		coinbase:     state.GetOrNewStateObject(block.Coinbase()),
	}

	return env
}

type Work struct {
	Number    uint64
	Nonce     uint64
	MixDigest []byte
	SeedHash  []byte
}

type Agent interface {
	Work() chan<- *types.Block
	SetReturnCh(chan<- *types.Block)
	Stop()
	Start()
	GetHashRate() int64
}

type worker struct {
	mu sync.Mutex

	agents []Agent
	recv   chan *types.Block
	mux    *event.TypeMux
	quit   chan struct{}
	pow    pow.PoW
	atWork int64

	eth   core.Backend
	chain *core.ChainManager
	proc  *core.BlockProcessor

	coinbase common.Address
	extra    []byte

	currentMu sync.Mutex
	current   *environment

	uncleMu        sync.Mutex
	possibleUncles map[common.Hash]*types.Block

	txQueueMu sync.Mutex
	txQueue   map[common.Hash]*types.Transaction

	mining int64
}

func newWorker(coinbase common.Address, eth core.Backend) *worker {
	worker := &worker{
		eth:            eth,
		mux:            eth.EventMux(),
		recv:           make(chan *types.Block),
		chain:          eth.ChainManager(),
		proc:           eth.BlockProcessor(),
		possibleUncles: make(map[common.Hash]*types.Block),
		coinbase:       coinbase,
		txQueue:        make(map[common.Hash]*types.Transaction),
		quit:           make(chan struct{}),
	}
	go worker.update()
	go worker.wait()

	worker.commitNewWork()

	return worker
}

func (self *worker) pendingState() *state.StateDB {
	self.currentMu.Lock()
	defer self.currentMu.Unlock()

	return self.current.state
}

func (self *worker) pendingBlock() *types.Block {
	self.currentMu.Lock()
	defer self.currentMu.Unlock()

	return self.current.block
}

func (self *worker) start() {
	// spin up agents
	for _, agent := range self.agents {
		agent.Start()
	}

	atomic.StoreInt64(&self.mining, 1)
}

func (self *worker) stop() {
	if atomic.LoadInt64(&self.mining) == 1 {
		// stop all agents
		for _, agent := range self.agents {
			agent.Stop()
		}
	}

	atomic.StoreInt64(&self.mining, 0)
	atomic.StoreInt64(&self.atWork, 0)
}

func (self *worker) register(agent Agent) {
	self.agents = append(self.agents, agent)
	agent.SetReturnCh(self.recv)
}

func (self *worker) update() {
	events := self.mux.Subscribe(core.ChainHeadEvent{}, core.ChainSideEvent{}, core.TxPreEvent{})

out:
	for {
		select {
		case event := <-events.Chan():
			switch ev := event.(type) {
			case core.ChainHeadEvent:
				self.commitNewWork()
			case core.ChainSideEvent:
				self.uncleMu.Lock()
				self.possibleUncles[ev.Block.Hash()] = ev.Block
				self.uncleMu.Unlock()
			case core.TxPreEvent:
				if atomic.LoadInt64(&self.mining) == 0 {
					self.commitNewWork()
				}
			}
		case <-self.quit:
			break out
		}
	}

	events.Unsubscribe()
}

func (self *worker) wait() {
	for {
		for block := range self.recv {
			atomic.AddInt64(&self.atWork, -1)

			if block == nil {
				continue
			}

			if err := self.chain.InsertChain(types.Blocks{block}); err == nil {
				for _, uncle := range block.Uncles() {
					delete(self.possibleUncles, uncle.Hash())
				}
				self.mux.Post(core.NewMinedBlockEvent{block})

				glog.V(logger.Info).Infof("🔨  Mined block #%v", block.Number())

				jsonlogger.LogJson(&logger.EthMinerNewBlock{
					BlockHash:     block.Hash().Hex(),
					BlockNumber:   block.Number(),
					ChainHeadHash: block.ParentHeaderHash.Hex(),
					BlockPrevHash: block.ParentHeaderHash.Hex(),
				})
			} else {
				self.commitNewWork()
			}
		}
	}
}

func (self *worker) push() {
	if atomic.LoadInt64(&self.mining) == 1 {
		self.current.block.Header().GasUsed = self.current.totalUsedGas
		self.current.block.SetRoot(self.current.state.Root())

		// push new work to agents
		for _, agent := range self.agents {
			atomic.AddInt64(&self.atWork, 1)

			if agent.Work() != nil {
				agent.Work() <- self.current.block.Copy()
			} else {
				common.Report(fmt.Sprintf("%v %T\n", agent, agent))
			}
		}
	}
}

func (self *worker) makeCurrent() {
	block := self.chain.NewBlock(self.coinbase)
	if block.Time() == self.chain.CurrentBlock().Time() {
		block.Header().Time++
	}
	block.Header().Extra = self.extra

	self.current = env(block, self.eth)
	for _, ancestor := range self.chain.GetAncestors(block, 7) {
		self.current.family.Add(ancestor.Hash())
	}

	parent := self.chain.GetBlock(self.current.block.ParentHash())
	self.current.coinbase.SetGasPool(core.CalcGasLimit(parent, self.current.block))
}

func (self *worker) commitNewWork() {
	self.mu.Lock()
	defer self.mu.Unlock()
	self.uncleMu.Lock()
	defer self.uncleMu.Unlock()
	self.currentMu.Lock()
	defer self.currentMu.Unlock()

	self.makeCurrent()

	transactions := self.eth.TxPool().GetTransactions()
	sort.Sort(types.TxByNonce{transactions})

	// Keep track of transactions which return errors so they can be removed
	var (
		remove             = set.New()
		tcount             = 0
		ignoredTransactors = set.New()
	)
	//gasLimit:
	for _, tx := range transactions {
		// We can skip err. It has already been validated in the tx pool
		from, _ := tx.From()
		// Move on to the next transaction when the transactor is in ignored transactions set
		// This may occur when a transaction hits the gas limit. When a gas limit is hit and
		// the transaction is processed (that could potentially be included in the block) it
		// will throw a nonce error because the previous transaction hasn't been processed.
		// Therefor we need to ignore any transaction after the ignored one.
		if ignoredTransactors.Has(from) {
			continue
		}

		self.current.state.StartRecord(tx.Hash(), common.Hash{}, 0)

		err := self.commitTransaction(tx)
		switch {
		case core.IsNonceErr(err) || core.IsInvalidTxErr(err):
			// Remove invalid transactions
			from, _ := tx.From()

			self.chain.TxState().RemoveNonce(from, tx.Nonce())
			remove.Add(tx.Hash())

			if glog.V(logger.Detail) {
				glog.Infof("TX (%x) failed, will be removed: %v\n", tx.Hash().Bytes()[:4], err)
			}
		case state.IsGasLimitErr(err):
			from, _ := tx.From()
			// ignore the transactor so no nonce errors will be thrown for this account
			// next time the worker is run, they'll be picked up again.
			ignoredTransactors.Add(from)
			//glog.V(logger.Debug).Infof("Gas limit reached for block. %d TXs included in this block\n", i)
			//break gasLimit
		default:
			tcount++
		}
	}
	//self.eth.TxPool().InvalidateSet(remove)

	var (
		uncles    []*types.Header
		badUncles []common.Hash
	)
	for hash, uncle := range self.possibleUncles {
		if len(uncles) == 2 {
			break
		}

		if err := self.commitUncle(uncle.Header()); err != nil {
			if glog.V(logger.Ridiculousness) {
				glog.V(logger.Detail).Infof("Bad uncle found and will be removed (%x)\n", hash[:4])
				glog.V(logger.Detail).Infoln(uncle)
			}

			badUncles = append(badUncles, hash)
		} else {
			glog.V(logger.Debug).Infof("commiting %x as uncle\n", hash[:4])
			uncles = append(uncles, uncle.Header())
		}
	}

	// We only care about logging if we're actually mining
	if atomic.LoadInt64(&self.mining) == 1 {
		glog.V(logger.Info).Infof("commit new work on block %v with %d txs & %d uncles\n", self.current.block.Number(), tcount, len(uncles))
	}

	for _, hash := range badUncles {
		delete(self.possibleUncles, hash)
	}

	self.current.block.SetUncles(uncles)

	core.AccumulateRewards(self.current.state, self.current.block)

	self.current.state.Update()

	self.push()
}

var (
	inclusionReward = new(big.Int).Div(core.BlockReward, big.NewInt(32))
	_uncleReward    = new(big.Int).Mul(core.BlockReward, big.NewInt(15))
	uncleReward     = new(big.Int).Div(_uncleReward, big.NewInt(16))
)

func (self *worker) commitUncle(uncle *types.Header) error {
	if self.current.uncles.Has(uncle.Hash()) {
		// Error not unique
		return core.UncleError("Uncle not unique")
	}
	self.current.uncles.Add(uncle.Hash())

	if !self.current.family.Has(uncle.ParentHash) {
		return core.UncleError(fmt.Sprintf("Uncle's parent unknown (%x)", uncle.ParentHash[0:4]))
	}

	if self.current.family.Has(uncle.Hash()) {
		return core.UncleError(fmt.Sprintf("Uncle already in family (%x)", uncle.Hash()))
	}

	return nil
}

func (self *worker) commitTransaction(tx *types.Transaction) error {
	snap := self.current.state.Copy()
	receipt, _, err := self.proc.ApplyTransaction(self.current.coinbase, self.current.state, self.current.block, tx, self.current.totalUsedGas, true)
	if err != nil && (core.IsNonceErr(err) || state.IsGasLimitErr(err) || core.IsInvalidTxErr(err)) {
		self.current.state.Set(snap)
		return err
	}

	self.current.block.AddTransaction(tx)
	self.current.block.AddReceipt(receipt)

	return nil
}

func (self *worker) HashRate() int64 {
	var tot int64
	for _, agent := range self.agents {
		tot += agent.GetHashRate()
	}

	return tot
}
<|endoftext|>


<commit_before>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package leveldb provides a LevelDB-based implementation of store.Store.
package leveldb

// #cgo LDFLAGS: -lleveldb -lsnappy
// #include <stdlib.h>
// #include "leveldb/c.h"
// #include "syncbase_leveldb.h"
import "C"
import (
	"fmt"
	"sync"
	"unsafe"

	"v.io/v23/verror"
	"v.io/x/ref/services/syncbase/store"
	"v.io/x/ref/services/syncbase/store/transactions"
)

// db is a wrapper around LevelDB that implements the transactions.BatchStore
// interface.
type db struct {
	// mu protects the state of the db.
	mu   sync.RWMutex
	node *store.ResourceNode
	cDb  *C.leveldb_t
	// Default read/write options.
	readOptions  *C.leveldb_readoptions_t
	writeOptions *C.leveldb_writeoptions_t
	err          error
}

type OpenOptions struct {
	CreateIfMissing bool
	ErrorIfExists   bool
}

// Open opens the database located at the given path.
func Open(path string, opts OpenOptions) (store.Store, error) {
	var cError *C.char
	cPath := C.CString(path)
	defer C.free(unsafe.Pointer(cPath))

	var cOptsCreateIfMissing, cOptsErrorIfExists C.uchar
	if opts.CreateIfMissing {
		cOptsCreateIfMissing = 1
	}
	if opts.ErrorIfExists {
		cOptsErrorIfExists = 1
	}

	cOpts := C.leveldb_options_create()
	C.leveldb_options_set_create_if_missing(cOpts, cOptsCreateIfMissing)
	C.leveldb_options_set_error_if_exists(cOpts, cOptsErrorIfExists)
	C.leveldb_options_set_paranoid_checks(cOpts, 1)
	defer C.leveldb_options_destroy(cOpts)

	cDb := C.leveldb_open(cOpts, cPath, &cError)
	if err := goError(cError); err != nil {
		return nil, err
	}
	readOptions := C.leveldb_readoptions_create()
	C.leveldb_readoptions_set_verify_checksums(readOptions, 1)
	return transactions.Wrap(&db{
		node:         store.NewResourceNode(),
		cDb:          cDb,
		readOptions:  readOptions,
		writeOptions: C.leveldb_writeoptions_create(),
	}), nil
}

// Close implements the store.Store interface.
func (d *db) Close() error {
	d.mu.Lock()
	defer d.mu.Unlock()
	if d.err != nil {
		return store.ConvertError(d.err)
	}
	d.node.Close()
	C.leveldb_close(d.cDb)
	d.cDb = nil
	C.leveldb_readoptions_destroy(d.readOptions)
	d.readOptions = nil
	C.leveldb_writeoptions_destroy(d.writeOptions)
	d.writeOptions = nil
	d.err = verror.New(verror.ErrCanceled, nil, store.ErrMsgClosedStore)
	return nil
}

// Destroy removes all physical data of the database located at the given path.
func Destroy(path string) error {
	var cError *C.char
	cPath := C.CString(path)
	defer C.free(unsafe.Pointer(cPath))
	cOpts := C.leveldb_options_create()
	defer C.leveldb_options_destroy(cOpts)
	C.leveldb_destroy_db(cOpts, cPath, &cError)
	return goError(cError)
}

// Get implements the store.StoreReader interface.
func (d *db) Get(key, valbuf []byte) ([]byte, error) {
	return d.getWithOpts(key, valbuf, d.readOptions)
}

// Scan implements the store.StoreReader interface.
func (d *db) Scan(start, limit []byte) store.Stream {
	d.mu.RLock()
	defer d.mu.RUnlock()
	if d.err != nil {
		return &store.InvalidStream{Error: d.err}
	}
	return newStream(d, d.node, start, limit, d.readOptions)
}

// NewSnapshot implements the store.Store interface.
func (d *db) NewSnapshot() store.Snapshot {
	d.mu.RLock()
	defer d.mu.RUnlock()
	if d.err != nil {
		return &store.InvalidSnapshot{Error: d.err}
	}
	return newSnapshot(d, d.node)
}

// WriteBatch implements the transactions.BatchStore interface.
func (d *db) WriteBatch(batch ...transactions.WriteOp) error {
	d.mu.Lock()
	defer d.mu.Unlock()
	if d.err != nil {
		return d.err
	}
	cBatch := C.leveldb_writebatch_create()
	defer C.leveldb_writebatch_destroy(cBatch)
	for _, write := range batch {
		switch write.T {
		case transactions.PutOp:
			cKey, cKeyLen := cSlice(write.Key)
			cVal, cValLen := cSlice(write.Value)
			C.leveldb_writebatch_put(cBatch, cKey, cKeyLen, cVal, cValLen)
		case transactions.DeleteOp:
			cKey, cKeyLen := cSlice(write.Key)
			C.leveldb_writebatch_delete(cBatch, cKey, cKeyLen)
		default:
			panic(fmt.Sprintf("unknown write operation type: %v", write.T))
		}
	}
	var cError *C.char
	C.leveldb_write(d.cDb, d.writeOptions, cBatch, &cError)
	return goError(cError)
}

// getWithOpts returns the value for the given key.
// cOpts may contain a pointer to a snapshot.
func (d *db) getWithOpts(key, valbuf []byte, cOpts *C.leveldb_readoptions_t) ([]byte, error) {
	d.mu.RLock()
	defer d.mu.RUnlock()
	if d.err != nil {
		return valbuf, store.ConvertError(d.err)
	}
	var cError *C.char
	var valLen C.size_t
	cStr, cLen := cSlice(key)
	val := C.leveldb_get(d.cDb, cOpts, cStr, cLen, &valLen, &cError)
	if err := goError(cError); err != nil {
		return valbuf, err
	}
	if val == nil {
		return valbuf, verror.New(store.ErrUnknownKey, nil, string(key))
	}
	defer C.leveldb_free(unsafe.Pointer(val))
	return store.CopyBytes(valbuf, goBytes(val, valLen)), nil
}
<commit_msg>syncbase: Set leveldb max_open_files when opening a database.<commit_after>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package leveldb provides a LevelDB-based implementation of store.Store.
package leveldb

// #cgo LDFLAGS: -lleveldb -lsnappy
// #include <stdlib.h>
// #include "leveldb/c.h"
// #include "syncbase_leveldb.h"
import "C"
import (
	"fmt"
	"sync"
	"unsafe"

	"v.io/v23/verror"
	"v.io/x/ref/services/syncbase/store"
	"v.io/x/ref/services/syncbase/store/transactions"
)

// db is a wrapper around LevelDB that implements the transactions.BatchStore
// interface.
type db struct {
	// mu protects the state of the db.
	mu   sync.RWMutex
	node *store.ResourceNode
	cDb  *C.leveldb_t
	// Default read/write options.
	readOptions  *C.leveldb_readoptions_t
	writeOptions *C.leveldb_writeoptions_t
	err          error
}

const defaultMaxOpenFiles = 100

type OpenOptions struct {
	CreateIfMissing bool
	ErrorIfExists   bool
	MaxOpenFiles    int
}

// Open opens the database located at the given path.
func Open(path string, opts OpenOptions) (store.Store, error) {
	var cError *C.char
	cPath := C.CString(path)
	defer C.free(unsafe.Pointer(cPath))

	var cOptsCreateIfMissing, cOptsErrorIfExists C.uchar
	if opts.CreateIfMissing {
		cOptsCreateIfMissing = 1
	}
	if opts.ErrorIfExists {
		cOptsErrorIfExists = 1
	}

	// If max_open_files is not set, leveldb can open many files, leading to
	// "Too many open files" error, or other strange system behavior.
	// See https://github.com/vanadium/issues/issues/1253
	cOptsMaxOpenFiles := C.int(opts.MaxOpenFiles)
	if cOptsMaxOpenFiles <= 0 {
		cOptsMaxOpenFiles = defaultMaxOpenFiles
	}

	cOpts := C.leveldb_options_create()
	C.leveldb_options_set_create_if_missing(cOpts, cOptsCreateIfMissing)
	C.leveldb_options_set_error_if_exists(cOpts, cOptsErrorIfExists)
	C.leveldb_options_set_max_open_files(cOpts, cOptsMaxOpenFiles)
	C.leveldb_options_set_paranoid_checks(cOpts, 1)
	defer C.leveldb_options_destroy(cOpts)

	cDb := C.leveldb_open(cOpts, cPath, &cError)
	if err := goError(cError); err != nil {
		return nil, err
	}
	readOptions := C.leveldb_readoptions_create()
	C.leveldb_readoptions_set_verify_checksums(readOptions, 1)
	return transactions.Wrap(&db{
		node:         store.NewResourceNode(),
		cDb:          cDb,
		readOptions:  readOptions,
		writeOptions: C.leveldb_writeoptions_create(),
	}), nil
}

// Close implements the store.Store interface.
func (d *db) Close() error {
	d.mu.Lock()
	defer d.mu.Unlock()
	if d.err != nil {
		return store.ConvertError(d.err)
	}
	d.node.Close()
	C.leveldb_close(d.cDb)
	d.cDb = nil
	C.leveldb_readoptions_destroy(d.readOptions)
	d.readOptions = nil
	C.leveldb_writeoptions_destroy(d.writeOptions)
	d.writeOptions = nil
	d.err = verror.New(verror.ErrCanceled, nil, store.ErrMsgClosedStore)
	return nil
}

// Destroy removes all physical data of the database located at the given path.
func Destroy(path string) error {
	var cError *C.char
	cPath := C.CString(path)
	defer C.free(unsafe.Pointer(cPath))
	cOpts := C.leveldb_options_create()
	defer C.leveldb_options_destroy(cOpts)
	C.leveldb_destroy_db(cOpts, cPath, &cError)
	return goError(cError)
}

// Get implements the store.StoreReader interface.
func (d *db) Get(key, valbuf []byte) ([]byte, error) {
	return d.getWithOpts(key, valbuf, d.readOptions)
}

// Scan implements the store.StoreReader interface.
func (d *db) Scan(start, limit []byte) store.Stream {
	d.mu.RLock()
	defer d.mu.RUnlock()
	if d.err != nil {
		return &store.InvalidStream{Error: d.err}
	}
	return newStream(d, d.node, start, limit, d.readOptions)
}

// NewSnapshot implements the store.Store interface.
func (d *db) NewSnapshot() store.Snapshot {
	d.mu.RLock()
	defer d.mu.RUnlock()
	if d.err != nil {
		return &store.InvalidSnapshot{Error: d.err}
	}
	return newSnapshot(d, d.node)
}

// WriteBatch implements the transactions.BatchStore interface.
func (d *db) WriteBatch(batch ...transactions.WriteOp) error {
	d.mu.Lock()
	defer d.mu.Unlock()
	if d.err != nil {
		return d.err
	}
	cBatch := C.leveldb_writebatch_create()
	defer C.leveldb_writebatch_destroy(cBatch)
	for _, write := range batch {
		switch write.T {
		case transactions.PutOp:
			cKey, cKeyLen := cSlice(write.Key)
			cVal, cValLen := cSlice(write.Value)
			C.leveldb_writebatch_put(cBatch, cKey, cKeyLen, cVal, cValLen)
		case transactions.DeleteOp:
			cKey, cKeyLen := cSlice(write.Key)
			C.leveldb_writebatch_delete(cBatch, cKey, cKeyLen)
		default:
			panic(fmt.Sprintf("unknown write operation type: %v", write.T))
		}
	}
	var cError *C.char
	C.leveldb_write(d.cDb, d.writeOptions, cBatch, &cError)
	return goError(cError)
}

// getWithOpts returns the value for the given key.
// cOpts may contain a pointer to a snapshot.
func (d *db) getWithOpts(key, valbuf []byte, cOpts *C.leveldb_readoptions_t) ([]byte, error) {
	d.mu.RLock()
	defer d.mu.RUnlock()
	if d.err != nil {
		return valbuf, store.ConvertError(d.err)
	}
	var cError *C.char
	var valLen C.size_t
	cStr, cLen := cSlice(key)
	val := C.leveldb_get(d.cDb, cOpts, cStr, cLen, &valLen, &cError)
	if err := goError(cError); err != nil {
		return valbuf, err
	}
	if val == nil {
		return valbuf, verror.New(store.ErrUnknownKey, nil, string(key))
	}
	defer C.leveldb_free(unsafe.Pointer(val))
	return store.CopyBytes(valbuf, goBytes(val, valLen)), nil
}
<|endoftext|>


<commit_before>// This file was auto-generated by the veyron vdl tool.
// Source: channel.vdl

package channel

import (
	// The non-user imports are prefixed with "_gen_" to prevent collisions.
	_gen_vdlutil "veyron.io/veyron/veyron2/vdl/vdlutil"
)

type Request struct {
	Type string
	Seq  uint64
	Body _gen_vdlutil.Any
}

type Response struct {
	ReqSeq uint64
	Err    string // TODO(bprosnitz) change this back to error when it is possible to do so. (issue 368)
	Body   _gen_vdlutil.Any
}

type Message struct{ oneof interface{} }

// MakeMessage creates a Message.
// Returns true iff the oneof value has a valid type.
func MakeMessage(oneof interface{}) (x Message, ok bool) {
	ok = x.Assign(oneof)
	return
}

// Assign assigns oneof to x.
// Returns true iff the oneof value has a valid type.
func (x *Message) Assign(oneof interface{}) bool {
	switch oneof.(type) {
	case Request, Response:
		x.oneof = oneof
		return true
	}
	x.oneof = nil
	return false
}

// OneOf returns the underlying typed value of x.
func (x Message) OneOf() interface{} {
	return x.oneof
}

// vdlOneOfTypes identifies Message as a oneof.
func (Message) vdlOneOfTypes(_ Request, _ Response) {}
<commit_msg>wspr: Trivial update to new vdl generator in go/vcl/6373<commit_after>// This file was auto-generated by the veyron vdl tool.
// Source: channel.vdl

package channel

import (
	// The non-user imports are prefixed with "__" to prevent collisions.
	__vdlutil "veyron.io/veyron/veyron2/vdl/vdlutil"
)

type Request struct {
	Type string
	Seq  uint64
	Body __vdlutil.Any
}

type Response struct {
	ReqSeq uint64
	Err    string // TODO(bprosnitz) change this back to error when it is possible to do so. (issue 368)
	Body   __vdlutil.Any
}

type Message struct{ oneof interface{} }

// MakeMessage creates a Message.
// Returns true iff the oneof value has a valid type.
func MakeMessage(oneof interface{}) (x Message, ok bool) {
	ok = x.Assign(oneof)
	return
}

// Assign assigns oneof to x.
// Returns true iff the oneof value has a valid type.
func (x *Message) Assign(oneof interface{}) bool {
	switch oneof.(type) {
	case Request, Response:
		x.oneof = oneof
		return true
	}
	x.oneof = nil
	return false
}

// OneOf returns the underlying typed value of x.
func (x Message) OneOf() interface{} {
	return x.oneof
}

// vdlOneOfTypes identifies Message as a oneof.
func (Message) vdlOneOfTypes(_ Request, _ Response) {}
<|endoftext|>


<commit_before>package archiver

import (
	"context"
	"sync"

	"github.com/restic/restic/internal/debug"
	"github.com/restic/restic/internal/restic"
)

// FutureTree is returned by Save and will return the data once it
// has been processed.
type FutureTree struct {
	ch  <-chan saveTreeResponse
	res saveTreeResponse
}

func (s *FutureTree) wait() {
	res, ok := <-s.ch
	if ok {
		s.res = res
	}
}

// Node returns the node once it is available.
func (s *FutureTree) Node() *restic.Node {
	s.wait()
	return s.res.node
}

// Stats returns the stats for the file once they are available.
func (s *FutureTree) Stats() ItemStats {
	s.wait()
	return s.res.stats
}

// Err returns the error in case an error occurred.
func (s *FutureTree) Err() error {
	s.wait()
	return s.res.err
}

// TreeSaver concurrently saves incoming trees to the repo.
type TreeSaver struct {
	saveTree func(context.Context, *restic.Tree) (restic.ID, ItemStats, error)
	errFn    ErrorFunc

	ch chan<- saveTreeJob
	wg sync.WaitGroup
}

// NewTreeSaver returns a new tree saver. A worker pool with treeWorkers is
// started, it is stopped when ctx is cancelled.
func NewTreeSaver(ctx context.Context, treeWorkers uint, saveTree func(context.Context, *restic.Tree) (restic.ID, ItemStats, error), errFn ErrorFunc) *TreeSaver {
	ch := make(chan saveTreeJob)

	s := &TreeSaver{
		ch:       ch,
		saveTree: saveTree,
		errFn:    errFn,
	}

	for i := uint(0); i < treeWorkers; i++ {
		s.wg.Add(1)
		go s.worker(ctx, &s.wg, ch)
	}

	return s
}

// Save stores the dir d and returns the data once it has been completed.
func (s *TreeSaver) Save(ctx context.Context, snPath string, node *restic.Node, nodes []FutureNode) FutureTree {
	ch := make(chan saveTreeResponse, 1)
	s.ch <- saveTreeJob{
		snPath: snPath,
		node:   node,
		nodes:  nodes,
		ch:     ch,
	}

	return FutureTree{ch: ch}
}

type saveTreeJob struct {
	snPath string
	nodes  []FutureNode
	node   *restic.Node
	ch     chan<- saveTreeResponse
}

type saveTreeResponse struct {
	node  *restic.Node
	stats ItemStats
	err   error
}

// save stores the nodes as a tree in the repo.
func (s *TreeSaver) save(ctx context.Context, snPath string, node *restic.Node, nodes []FutureNode) (*restic.Node, ItemStats, error) {
	var stats ItemStats

	tree := restic.NewTree()
	for _, fn := range nodes {
		fn.wait(ctx)

		// return the error if it wasn't ignored
		if fn.err != nil {
			debug.Log("err for %v: %v", fn.node.Name, fn.err)
			fn.err = s.errFn(fn.target, fn.fi, fn.err)
			if fn.err == nil {
				// ignore error
				continue
			}

			return nil, stats, fn.err
		}

		// when the error is ignored, the node could not be saved, so ignore it
		if fn.node == nil {
			debug.Log("%v excluded: %v", fn.snPath, fn.target)
			continue
		}

		debug.Log("insert %v", fn.node.Name)
		err := tree.Insert(fn.node)
		if err != nil {
			return nil, stats, err
		}
	}

	id, treeStats, err := s.saveTree(ctx, tree)
	stats.Add(treeStats)
	if err != nil {
		return nil, stats, err
	}

	node.Subtree = &id
	return node, stats, nil
}

func (s *TreeSaver) worker(ctx context.Context, wg *sync.WaitGroup, jobs <-chan saveTreeJob) {
	defer wg.Done()
	for {
		var job saveTreeJob
		select {
		case <-ctx.Done():
			return
		case job = <-jobs:
		}

		node, stats, err := s.save(ctx, job.snPath, job.node, job.nodes)
		job.ch <- saveTreeResponse{
			node:  node,
			stats: stats,
			err:   err,
		}
		close(job.ch)
	}
}
<commit_msg>Fix panic for nil-pointer dereference<commit_after>package archiver

import (
	"context"
	"sync"

	"github.com/restic/restic/internal/debug"
	"github.com/restic/restic/internal/restic"
)

// FutureTree is returned by Save and will return the data once it
// has been processed.
type FutureTree struct {
	ch  <-chan saveTreeResponse
	res saveTreeResponse
}

func (s *FutureTree) wait() {
	res, ok := <-s.ch
	if ok {
		s.res = res
	}
}

// Node returns the node once it is available.
func (s *FutureTree) Node() *restic.Node {
	s.wait()
	return s.res.node
}

// Stats returns the stats for the file once they are available.
func (s *FutureTree) Stats() ItemStats {
	s.wait()
	return s.res.stats
}

// Err returns the error in case an error occurred.
func (s *FutureTree) Err() error {
	s.wait()
	return s.res.err
}

// TreeSaver concurrently saves incoming trees to the repo.
type TreeSaver struct {
	saveTree func(context.Context, *restic.Tree) (restic.ID, ItemStats, error)
	errFn    ErrorFunc

	ch chan<- saveTreeJob
	wg sync.WaitGroup
}

// NewTreeSaver returns a new tree saver. A worker pool with treeWorkers is
// started, it is stopped when ctx is cancelled.
func NewTreeSaver(ctx context.Context, treeWorkers uint, saveTree func(context.Context, *restic.Tree) (restic.ID, ItemStats, error), errFn ErrorFunc) *TreeSaver {
	ch := make(chan saveTreeJob)

	s := &TreeSaver{
		ch:       ch,
		saveTree: saveTree,
		errFn:    errFn,
	}

	for i := uint(0); i < treeWorkers; i++ {
		s.wg.Add(1)
		go s.worker(ctx, &s.wg, ch)
	}

	return s
}

// Save stores the dir d and returns the data once it has been completed.
func (s *TreeSaver) Save(ctx context.Context, snPath string, node *restic.Node, nodes []FutureNode) FutureTree {
	ch := make(chan saveTreeResponse, 1)
	s.ch <- saveTreeJob{
		snPath: snPath,
		node:   node,
		nodes:  nodes,
		ch:     ch,
	}

	return FutureTree{ch: ch}
}

type saveTreeJob struct {
	snPath string
	nodes  []FutureNode
	node   *restic.Node
	ch     chan<- saveTreeResponse
}

type saveTreeResponse struct {
	node  *restic.Node
	stats ItemStats
	err   error
}

// save stores the nodes as a tree in the repo.
func (s *TreeSaver) save(ctx context.Context, snPath string, node *restic.Node, nodes []FutureNode) (*restic.Node, ItemStats, error) {
	var stats ItemStats

	tree := restic.NewTree()
	for _, fn := range nodes {
		fn.wait(ctx)

		// return the error if it wasn't ignored
		if fn.err != nil {
			debug.Log("err for %v: %v", fn.snPath, fn.err)
			fn.err = s.errFn(fn.target, fn.fi, fn.err)
			if fn.err == nil {
				// ignore error
				continue
			}

			return nil, stats, fn.err
		}

		// when the error is ignored, the node could not be saved, so ignore it
		if fn.node == nil {
			debug.Log("%v excluded: %v", fn.snPath, fn.target)
			continue
		}

		debug.Log("insert %v", fn.node.Name)
		err := tree.Insert(fn.node)
		if err != nil {
			return nil, stats, err
		}
	}

	id, treeStats, err := s.saveTree(ctx, tree)
	stats.Add(treeStats)
	if err != nil {
		return nil, stats, err
	}

	node.Subtree = &id
	return node, stats, nil
}

func (s *TreeSaver) worker(ctx context.Context, wg *sync.WaitGroup, jobs <-chan saveTreeJob) {
	defer wg.Done()
	for {
		var job saveTreeJob
		select {
		case <-ctx.Done():
			return
		case job = <-jobs:
		}

		node, stats, err := s.save(ctx, job.snPath, job.node, job.nodes)
		job.ch <- saveTreeResponse{
			node:  node,
			stats: stats,
			err:   err,
		}
		close(job.ch)
	}
}
<|endoftext|>


<commit_before>package alita

import (
	"testing"
)

func TestDefaultMargin(t *testing.T) {
	m := NewMargin()
	l, r := 1, 1
	if m.left != l || m.right != r {
		t.Errorf("got %d:%d, want %d:%d", m.left, m.right, l, r)
	}
}

type FormatTest struct {
	format string
	left   int
	right  int
}

var indexTestsFormat = []FormatTest{
	// digit only
	{"1", 1, 1},
	{"2", 2, 2},
	{"10", 10, 10},
	{"00", 0, 0},

	// colon separated digits
	{"1:1", 1, 1},
	{"2:1", 2, 1},
	{"1:4", 1, 4},
	{"10:5", 10, 5},
	{"5:20", 5, 20},
}

func TestParseFormat(t *testing.T) {
	m := NewMargin()
	for _, test := range indexTestsFormat {
		if err := m.Set(test.format); err != nil {
			t.Errorf("Set(%q) returns %q; want nil",
				test.format, err)
		}
		if m.left != test.left || m.right != test.right {
			t.Errorf("got %d:%d, want %d:%d",
				m.left, m.right, test.left, test.right)
		}
	}
}

var indexTestsErrFormat = []string{
	"abc",
	"100000000000000000000000000000",
	"-1",
	":",
	"1:",
	":1",
	"1:-1",
	"-1:1",
	"3:1:4",
}

func TestParseErrFormat(t *testing.T) {
	m := NewMargin()
	for _, format := range indexTestsErrFormat {
		if err := m.Set(format); err == nil {
			t.Errorf("Margin.Set(%q) returns nil; want err", format)
		}
	}
}

type JoinTest struct {
	format string
	src    []string
	dst    string
}

var indexTestJoinStrings = []JoinTest{
	{"1", []string{"n", "=", "100"}, "n = 100"},
	{"2", []string{"n", "=", "100"}, "n  =  100"},

	{"1:0", []string{"n", "=", "100"}, "n =100"},
	{"0:1", []string{"n", "=", "100"}, "n= 100"},

	{"1", []string{"1", "2", "3", "4"}, "1 2 3 4"},
	{"2", []string{"1", "2", "3", "4"}, "1  2  3  4"},
}

func TestJoinStrings(t *testing.T) {
	m := NewMargin()
	for _, test := range indexTestJoinStrings {
		if err := m.Set(test.format); err != nil {
			t.Errorf("Margin.Set(%q) returns %q; want nil",
				err, test.format)
		}

		actual := m.Join(test.src)
		expect := test.dst
		if actual != expect {
			t.Errorf("Margin.Set(%q).Join(%q) = %q; want %q",
				test.format, test.src, actual, expect)
		}
	}
}
<commit_msg>Add some test for Margin.Join<commit_after>package alita

import (
	"testing"
)

func TestDefaultMargin(t *testing.T) {
	m := NewMargin()
	l, r := 1, 1
	if m.left != l || m.right != r {
		t.Errorf("got %d:%d, want %d:%d", m.left, m.right, l, r)
	}
}

type FormatTest struct {
	format string
	left   int
	right  int
}

var indexTestsFormat = []FormatTest{
	// digit only
	{"1", 1, 1},
	{"2", 2, 2},
	{"10", 10, 10},
	{"00", 0, 0},

	// colon separated digits
	{"1:1", 1, 1},
	{"2:1", 2, 1},
	{"1:4", 1, 4},
	{"10:5", 10, 5},
	{"5:20", 5, 20},
}

func TestParseFormat(t *testing.T) {
	m := NewMargin()
	for _, test := range indexTestsFormat {
		if err := m.Set(test.format); err != nil {
			t.Errorf("Set(%q) returns %q; want nil",
				test.format, err)
		}
		if m.left != test.left || m.right != test.right {
			t.Errorf("got %d:%d, want %d:%d",
				m.left, m.right, test.left, test.right)
		}
	}
}

var indexTestsErrFormat = []string{
	"abc",
	"100000000000000000000000000000",
	"-1",
	":",
	"1:",
	":1",
	"1:-1",
	"-1:1",
	"3:1:4",
}

func TestParseErrFormat(t *testing.T) {
	m := NewMargin()
	for _, format := range indexTestsErrFormat {
		if err := m.Set(format); err == nil {
			t.Errorf("Margin.Set(%q) returns nil; want err", format)
		}
	}
}

type JoinTest struct {
	format string
	src    []string
	dst    string
}

var indexTestJoinStrings = []JoinTest{
	{"0", []string{"a"}, "a"},
	{"2", []string{"a"}, "a"},

	{"0", []string{"a", "b"}, "ab"},
	{"2", []string{"a", "b"}, "a  b"},

	{"1", []string{"n", "=", "100"}, "n = 100"},
	{"2", []string{"n", "=", "100"}, "n  =  100"},
	{"1:0", []string{"n", "=", "100"}, "n =100"},
	{"0:1", []string{"n", "=", "100"}, "n= 100"},

	{"1", []string{"1", "2", "3", "4"}, "1 2 3 4"},
	{"2", []string{"1", "2", "3", "4"}, "1  2  3  4"},
	{"0:1", []string{"1", "2", "3", "4"}, "12 34"},
	{"1:0", []string{"1", "2", "3", "4"}, "1 23 4"},

	{"1", []string{"a", ":", "b", ":", "c"}, "a : b : c"},
	{"2", []string{"a", ":", "b", ":", "c"}, "a  :  b  :  c"},
	{"0:1", []string{"a", ":", "b", ":", "c"}, "a: b: c"},
	{"1:0", []string{"a", ":", "b", ":", "c"}, "a :b :c"},

	{"0:1", []string{"1", "2", "3", "4", "5", "6", "7", "8"},
		"12 34 56 78"},
	{"1:0", []string{"1", "2", "3", "4", "5", "6", "7", "8"},
		"1 23 45 67 8"},
}

func TestJoinStrings(t *testing.T) {
	m := NewMargin()
	for _, test := range indexTestJoinStrings {
		if err := m.Set(test.format); err != nil {
			t.Errorf("Margin.Set(%q) returns %q; want nil",
				err, test.format)
		}

		actual := m.Join(test.src)
		expect := test.dst
		if actual != expect {
			t.Errorf("Margin.Set(%q).Join(%q) = %q; want %q",
				test.format, test.src, actual, expect)
		}
	}
}
<|endoftext|>