<commit_before>package main

import (
	"bytes"
	"fmt"
	"sort"
	"strings"

	"github.com/vattle/sqlboiler/bdb"
)

// imports defines the optional standard imports and
// thirdParty imports (from github for example)
type imports struct {
	standard   importList
	thirdParty importList
}

// importList is a list of import names
type importList []string

func (i importList) Len() int {
	return len(i)
}

func (i importList) Swap(k, j int) {
	i[k], i[j] = i[j], i[k]
}

func (i importList) Less(k, j int) bool {
	res := strings.Compare(strings.TrimLeft(i[k], "_ "), strings.TrimLeft(i[j], "_ "))
	if res <= 0 {
		return true
	}

	return false
}

func combineImports(a, b imports) imports {
	var c imports

	c.standard = removeDuplicates(combineStringSlices(a.standard, b.standard))
	c.thirdParty = removeDuplicates(combineStringSlices(a.thirdParty, b.thirdParty))

	sort.Sort(c.standard)
	sort.Sort(c.thirdParty)

	return c
}

func combineTypeImports(a imports, b map[string]imports, columns []bdb.Column) imports {
	tmpImp := imports{
		standard:   make(importList, len(a.standard)),
		thirdParty: make(importList, len(a.thirdParty)),
	}

	copy(tmpImp.standard, a.standard)
	copy(tmpImp.thirdParty, a.thirdParty)

	for _, col := range columns {
		for key, imp := range b {
			if col.Type == key {
				tmpImp.standard = append(tmpImp.standard, imp.standard...)
				tmpImp.thirdParty = append(tmpImp.thirdParty, imp.thirdParty...)
			}
		}
	}

	tmpImp.standard = removeDuplicates(tmpImp.standard)
	tmpImp.thirdParty = removeDuplicates(tmpImp.thirdParty)

	sort.Sort(tmpImp.standard)
	sort.Sort(tmpImp.thirdParty)

	return tmpImp
}

func buildImportString(imps imports) []byte {
	stdlen, thirdlen := len(imps.standard), len(imps.thirdParty)
	if stdlen+thirdlen < 1 {
		return []byte{}
	}

	if stdlen+thirdlen == 1 {
		var imp string
		if stdlen == 1 {
			imp = imps.standard[0]
		} else {
			imp = imps.thirdParty[0]
		}
		return []byte(fmt.Sprintf("import %s", imp))
	}

	buf := &bytes.Buffer{}
	buf.WriteString("import (")
	for _, std := range imps.standard {
		fmt.Fprintf(buf, "\n\t%s", std)
	}
	if stdlen != 0 && thirdlen != 0 {
		buf.WriteString("\n")
	}
	for _, third := range imps.thirdParty {
		fmt.Fprintf(buf, "\n\t%s", third)
	}
	buf.WriteString("\n)\n")

	return buf.Bytes()
}

func combineStringSlices(a, b []string) []string {
	c := make([]string, len(a)+len(b))
	if len(a) > 0 {
		copy(c, a)
	}
	if len(b) > 0 {
		copy(c[len(a):], b)
	}

	return c
}

func removeDuplicates(dedup []string) []string {
	if len(dedup) <= 1 {
		return dedup
	}

	for i := 0; i < len(dedup)-1; i++ {
		for j := i + 1; j < len(dedup); j++ {
			if dedup[i] != dedup[j] {
				continue
			}

			if j != len(dedup)-1 {
				dedup[j] = dedup[len(dedup)-1]
				j--
			}
			dedup = dedup[:len(dedup)-1]
		}
	}

	return dedup
}

var defaultTemplateImports = imports{
	standard: importList{
		`"fmt"`,
		`"strings"`,
		`"database/sql"`,
	},
	thirdParty: importList{
		`"github.com/pkg/errors"`,
		`"github.com/vattle/sqlboiler/boil"`,
		`"github.com/vattle/sqlboiler/boil/qm"`,
		`"github.com/vattle/sqlboiler/strmangle"`,
	},
}

var defaultSingletonTemplateImports = map[string]imports{
	"boil_helpers": {
		standard: importList{
			`"fmt"`,
			`"strings"`,
		},
		thirdParty: importList{
			`"github.com/vattle/sqlboiler/boil"`,
			`"github.com/vattle/sqlboiler/boil/qm"`,
			`"github.com/vattle/sqlboiler/strmangle"`,
		},
	},
	"boil_types": {
		thirdParty: importList{
			`"github.com/pkg/errors"`,
		},
	},
}

var defaultTestTemplateImports = imports{
	standard: importList{
		`"testing"`,
		`"reflect"`,
	},
	thirdParty: importList{
		`"github.com/vattle/sqlboiler/boil"`,
		`"github.com/vattle/sqlboiler/strmangle"`,
	},
}

var defaultSingletonTestTemplateImports = map[string]imports{
	"boil_main_helpers": {
		standard: importList{
			`"database/sql"`,
			`"os"`,
			`"path/filepath"`,
		},
		thirdParty: importList{
			`"github.com/spf13/viper"`,
		},
	},
	"boil_helpers": {
		standard: importList{
			`"crypto/md5"`,
			`"fmt"`,
			`"os"`,
			`"strconv"`,
			`"math/rand"`,
		},
		thirdParty: importList{
			`"github.com/vattle/sqlboiler/boil"`,
		},
	},
	"boil_test_suite": {
		standard: importList{
			`"testing"`,
		},
	},
}

var defaultTestMainImports = map[string]imports{
	"postgres": {
		standard: importList{
			`"testing"`,
			`"os"`,
			`"os/exec"`,
			`"flag"`,
			`"fmt"`,
			`"io/ioutil"`,
			`"bytes"`,
			`"database/sql"`,
			`"path/filepath"`,
			`"time"`,
			`"math/rand"`,
		},
		thirdParty: importList{
			`"github.com/kat-co/vala"`,
			`"github.com/pkg/errors"`,
			`"github.com/spf13/viper"`,
			`"github.com/vattle/sqlboiler/boil"`,
			`"github.com/vattle/sqlboiler/bdb/drivers"`,
			`_ "github.com/lib/pq"`,
		},
	},
}

// importsBasedOnType imports are only included in the template output if the
// database requires one of the following special types. Check
// TranslateColumnType to see the type assignments.
var importsBasedOnType = map[string]imports{
	"null.Float32": {
		thirdParty: importList{`"gopkg.in/nullbio/null.v4"`},
	},
	"null.Float64": {
		thirdParty: importList{`"gopkg.in/nullbio/null.v4"`},
	},
	"null.Int": {
		thirdParty: importList{`"gopkg.in/nullbio/null.v4"`},
	},
	"null.Int8": {
		thirdParty: importList{`"gopkg.in/nullbio/null.v4"`},
	},
	"null.Int16": {
		thirdParty: importList{`"gopkg.in/nullbio/null.v4"`},
	},
	"null.Int32": {
		thirdParty: importList{`"gopkg.in/nullbio/null.v4"`},
	},
	"null.Int64": {
		thirdParty: importList{`"gopkg.in/nullbio/null.v4"`},
	},
	"null.Uint": {
		thirdParty: importList{`"gopkg.in/nullbio/null.v4"`},
	},
	"null.Uint8": {
		thirdParty: importList{`"gopkg.in/nullbio/null.v4"`},
	},
	"null.Uint16": {
		thirdParty: importList{`"gopkg.in/nullbio/null.v4"`},
	},
	"null.Uint32": {
		thirdParty: importList{`"gopkg.in/nullbio/null.v4"`},
	},
	"null.Uint64": {
		thirdParty: importList{`"gopkg.in/nullbio/null.v4"`},
	},
	"null.String": {
		thirdParty: importList{`"gopkg.in/nullbio/null.v4"`},
	},
	"null.Bool": {
		thirdParty: importList{`"gopkg.in/nullbio/null.v4"`},
	},
	"null.Time": {
		thirdParty: importList{`"gopkg.in/nullbio/null.v4"`},
	},
	"time.Time": {
		standard: importList{`"time"`},
	},
}
<commit_msg>Fix imports broken from last test.<commit_after>package main

import (
	"bytes"
	"fmt"
	"sort"
	"strings"

	"github.com/vattle/sqlboiler/bdb"
)

// imports defines the optional standard imports and
// thirdParty imports (from github for example)
type imports struct {
	standard   importList
	thirdParty importList
}

// importList is a list of import names
type importList []string

func (i importList) Len() int {
	return len(i)
}

func (i importList) Swap(k, j int) {
	i[k], i[j] = i[j], i[k]
}

func (i importList) Less(k, j int) bool {
	res := strings.Compare(strings.TrimLeft(i[k], "_ "), strings.TrimLeft(i[j], "_ "))
	if res <= 0 {
		return true
	}

	return false
}

func combineImports(a, b imports) imports {
	var c imports

	c.standard = removeDuplicates(combineStringSlices(a.standard, b.standard))
	c.thirdParty = removeDuplicates(combineStringSlices(a.thirdParty, b.thirdParty))

	sort.Sort(c.standard)
	sort.Sort(c.thirdParty)

	return c
}

func combineTypeImports(a imports, b map[string]imports, columns []bdb.Column) imports {
	tmpImp := imports{
		standard:   make(importList, len(a.standard)),
		thirdParty: make(importList, len(a.thirdParty)),
	}

	copy(tmpImp.standard, a.standard)
	copy(tmpImp.thirdParty, a.thirdParty)

	for _, col := range columns {
		for key, imp := range b {
			if col.Type == key {
				tmpImp.standard = append(tmpImp.standard, imp.standard...)
				tmpImp.thirdParty = append(tmpImp.thirdParty, imp.thirdParty...)
			}
		}
	}

	tmpImp.standard = removeDuplicates(tmpImp.standard)
	tmpImp.thirdParty = removeDuplicates(tmpImp.thirdParty)

	sort.Sort(tmpImp.standard)
	sort.Sort(tmpImp.thirdParty)

	return tmpImp
}

func buildImportString(imps imports) []byte {
	stdlen, thirdlen := len(imps.standard), len(imps.thirdParty)
	if stdlen+thirdlen < 1 {
		return []byte{}
	}

	if stdlen+thirdlen == 1 {
		var imp string
		if stdlen == 1 {
			imp = imps.standard[0]
		} else {
			imp = imps.thirdParty[0]
		}
		return []byte(fmt.Sprintf("import %s", imp))
	}

	buf := &bytes.Buffer{}
	buf.WriteString("import (")
	for _, std := range imps.standard {
		fmt.Fprintf(buf, "\n\t%s", std)
	}
	if stdlen != 0 && thirdlen != 0 {
		buf.WriteString("\n")
	}
	for _, third := range imps.thirdParty {
		fmt.Fprintf(buf, "\n\t%s", third)
	}
	buf.WriteString("\n)\n")

	return buf.Bytes()
}

func combineStringSlices(a, b []string) []string {
	c := make([]string, len(a)+len(b))
	if len(a) > 0 {
		copy(c, a)
	}
	if len(b) > 0 {
		copy(c[len(a):], b)
	}

	return c
}

func removeDuplicates(dedup []string) []string {
	if len(dedup) <= 1 {
		return dedup
	}

	for i := 0; i < len(dedup)-1; i++ {
		for j := i + 1; j < len(dedup); j++ {
			if dedup[i] != dedup[j] {
				continue
			}

			if j != len(dedup)-1 {
				dedup[j] = dedup[len(dedup)-1]
				j--
			}
			dedup = dedup[:len(dedup)-1]
		}
	}

	return dedup
}

var defaultTemplateImports = imports{
	standard: importList{
		`"fmt"`,
		`"strings"`,
		`"database/sql"`,
	},
	thirdParty: importList{
		`"github.com/pkg/errors"`,
		`"github.com/vattle/sqlboiler/boil"`,
		`"github.com/vattle/sqlboiler/boil/qm"`,
		`"github.com/vattle/sqlboiler/strmangle"`,
	},
}

var defaultSingletonTemplateImports = map[string]imports{
	"boil_queries": {
		standard: importList{
			`"fmt"`,
			`"strings"`,
		},
		thirdParty: importList{
			`"github.com/vattle/sqlboiler/boil"`,
			`"github.com/vattle/sqlboiler/boil/qm"`,
			`"github.com/vattle/sqlboiler/strmangle"`,
		},
	},
	"boil_types": {
		thirdParty: importList{
			`"github.com/pkg/errors"`,
		},
	},
}

var defaultTestTemplateImports = imports{
	standard: importList{
		`"testing"`,
		`"reflect"`,
	},
	thirdParty: importList{
		`"github.com/vattle/sqlboiler/boil"`,
		`"github.com/vattle/sqlboiler/strmangle"`,
	},
}

var defaultSingletonTestTemplateImports = map[string]imports{
	"boil_viper_test": {
		standard: importList{
			`"database/sql"`,
			`"os"`,
			`"path/filepath"`,
		},
		thirdParty: importList{
			`"github.com/spf13/viper"`,
		},
	},
	"boil_queries_test": {
		standard: importList{
			`"crypto/md5"`,
			`"fmt"`,
			`"os"`,
			`"strconv"`,
			`"math/rand"`,
		},
		thirdParty: importList{
			`"github.com/vattle/sqlboiler/boil"`,
		},
	},
	"boil_suites_test": {
		standard: importList{
			`"testing"`,
		},
	},
}

var defaultTestMainImports = map[string]imports{
	"postgres": {
		standard: importList{
			`"testing"`,
			`"os"`,
			`"os/exec"`,
			`"flag"`,
			`"fmt"`,
			`"io/ioutil"`,
			`"bytes"`,
			`"database/sql"`,
			`"path/filepath"`,
			`"time"`,
			`"math/rand"`,
		},
		thirdParty: importList{
			`"github.com/kat-co/vala"`,
			`"github.com/pkg/errors"`,
			`"github.com/spf13/viper"`,
			`"github.com/vattle/sqlboiler/boil"`,
			`"github.com/vattle/sqlboiler/bdb/drivers"`,
			`_ "github.com/lib/pq"`,
		},
	},
}

// importsBasedOnType imports are only included in the template output if the
// database requires one of the following special types. Check
// TranslateColumnType to see the type assignments.
var importsBasedOnType = map[string]imports{
	"null.Float32": {
		thirdParty: importList{`"gopkg.in/nullbio/null.v4"`},
	},
	"null.Float64": {
		thirdParty: importList{`"gopkg.in/nullbio/null.v4"`},
	},
	"null.Int": {
		thirdParty: importList{`"gopkg.in/nullbio/null.v4"`},
	},
	"null.Int8": {
		thirdParty: importList{`"gopkg.in/nullbio/null.v4"`},
	},
	"null.Int16": {
		thirdParty: importList{`"gopkg.in/nullbio/null.v4"`},
	},
	"null.Int32": {
		thirdParty: importList{`"gopkg.in/nullbio/null.v4"`},
	},
	"null.Int64": {
		thirdParty: importList{`"gopkg.in/nullbio/null.v4"`},
	},
	"null.Uint": {
		thirdParty: importList{`"gopkg.in/nullbio/null.v4"`},
	},
	"null.Uint8": {
		thirdParty: importList{`"gopkg.in/nullbio/null.v4"`},
	},
	"null.Uint16": {
		thirdParty: importList{`"gopkg.in/nullbio/null.v4"`},
	},
	"null.Uint32": {
		thirdParty: importList{`"gopkg.in/nullbio/null.v4"`},
	},
	"null.Uint64": {
		thirdParty: importList{`"gopkg.in/nullbio/null.v4"`},
	},
	"null.String": {
		thirdParty: importList{`"gopkg.in/nullbio/null.v4"`},
	},
	"null.Bool": {
		thirdParty: importList{`"gopkg.in/nullbio/null.v4"`},
	},
	"null.Time": {
		thirdParty: importList{`"gopkg.in/nullbio/null.v4"`},
	},
	"time.Time": {
		standard: importList{`"time"`},
	},
}
<|endoftext|>


<commit_before>// Copyright 2015 Kazuhisa TAKEI<xtakei@me.com>. All rights reserved.
// Use of this source code is governed by MPL-2.0 license tha can be
// found in the LICENSE file

// Package buffer_list implements a double linked list with sequencial buffer data.
//
// To Get New First Data from buffer_list(l is a *List)
//		type Hoge Struct {
//			a int
//			b int
//		}
//		l := buffer_list.New(Hoge{})
//		hoge := l.GetElement(),Value().(*Hoge)
//		hoge.a = 1
//		hoge.b = 2
// To iterate over a list
//		for e := l.Front(); e != nil ; e = e.Next() {
//			a := (*Hoge)(e.Value())  // Hoge is Value type
//			// do something
//		}

package buffer_list

import (
	//	"fmt" // FIXME remove
	"reflect"
	"sync"
	"unsafe"
)

const (
	DEFAULT_BUF_SIZE = 1024
)

type Element struct {
	list      *List
	next      *Element
	prev      *Element
	old_value unsafe.Pointer
	value     interface{}
}

type List struct {
	Used      *Element
	Freed     *Element
	SizeElm   int64
	SizeData  int64
	Used_idx  int64
	Value_inf interface{}
	elms      []byte
	datas     []byte
	Len       int
	m         sync.Mutex
	cast_f    func(interface{}) interface{}
	pointers  map[uintptr]map[int]unsafe.Pointer
}

func New(first_value interface{}, buf_cnt int) (l *List) {
	l = new(List).Init(first_value, buf_cnt)
	l.pointers = make(map[uintptr]map[int]unsafe.Pointer)
	return l
}

func (e *Element) Commit() {
	e.List().Pick_ptr(e)
}

func (l *List) Pick_ptr(e *Element) {
	f_num := reflect.ValueOf(e.Value()).Elem().NumField()
	v_ptr := reflect.ValueOf(e.Value()).Pointer()
	if l.pointers[uintptr(v_ptr)] == nil {
		l.pointers[uintptr(v_ptr)] = make(map[int]unsafe.Pointer)
	}

	for i := 0; i < f_num; i++ {
		m := reflect.ValueOf(e.Value()).Elem().Field(i)
		switch m.Kind() {
		case reflect.UnsafePointer:
			fallthrough
		case reflect.String:
			fallthrough
		case reflect.Slice:
			fallthrough
		case reflect.Map:
			fallthrough
		case reflect.Chan:
			fallthrough
		case reflect.Array:
			fallthrough
		case reflect.Func:
			fallthrough
		case reflect.Ptr:
			//			fmt.Println("detect ptr member", i, m.Kind(), m.Pointer())
			l.pointers[uintptr(v_ptr)][i] = unsafe.Pointer(m.Pointer())
		default:
		}

	}
}

func (l *List) GetDataPtr() uintptr {
	return uintptr(unsafe.Pointer(&l.datas[0]))
}
func (l *List) getElemData(idx int64) *Element {
	elm := (*Element)(unsafe.Pointer(&l.elms[int(l.SizeElm)*int(idx)]))
	elm.value = reflect.NewAt(l.TypeOfValue_inf(), unsafe.Pointer(&l.datas[int(l.SizeData)*int(idx)])).Interface()
	return elm
}
func (l *List) GetElement() *Element {
	return l.Used
}
func (e *Element) Next() *Element {
	e.list.m.Lock()
	defer e.list.m.Unlock()

	if e.next != nil {
		return e.next
	} else {
		return nil
	}
}

func (e *Element) Prev() *Element {
	e.list.m.Lock()
	defer e.list.m.Unlock()

	if e.prev != nil {
		return e.prev
	} else {
		return nil
	}
}

func (e *Element) Value() interface{} {
	return e.value
}

func (e *Element) Free() {

	e.list.m.Lock()
	defer e.list.m.Unlock()

	for ee := e.list.Used; ee != nil; ee = ee.next {
		if e == ee {
			goto DO_FREE
		}
	}

	//	fmt.Println("dont Free() e is not used ")
	return

DO_FREE:
	//	fmt.Println("do Free()")

	at := e.prev
	n := e.next
	if at.next == e {
		at.next = n
	}
	if n != nil {
		n.prev = at
	}

	e.list.Len -= 1

	if e.list.Used == e {
		e.list.Used = n
	}
	// move to free buffer
	if e.list.Freed == nil {
		e.prev = nil
		e.next = nil
		e.list.Freed = e
	} else {
		f_at := e.list.Freed
		e.next = f_at
		e.prev = nil
		f_at.prev = e
		e.list.Freed = e
	}
}

func (e *Element) InitValue() {

	diff := int(uint64(reflect.ValueOf(e.value).Pointer()) - uint64(uintptr(unsafe.Pointer(&e.list.datas[0]))))

	for i := range e.list.datas[diff : diff+int(e.list.SizeData)-1] {
		e.list.datas[diff+i] = 0
	}

	return
	//	fmt.Println(ref_byte, databyte)
}
func (l *List) newFirstElem() *Element {
	var e *Element

	//	l.m.Lock()
	//	defer l.m.Unlock()

	if l.Freed == nil {
		e = l.getElemData(l.Used_idx)
		l.Used_idx += 1
	} else {
		e = l.Freed
		if l.Freed.next == nil {
			l.Freed = nil
		} else {
			l.Freed = l.Freed.next
			l.Freed.prev = nil
		}
	}
	e.prev = e
	e.next = nil
	e.list = l
	if l.Used == nil {
		l.Used = e
	}
	l.Len++
	return e
}

func (l *List) InsertNewElem(at *Element) *Element {
	var e *Element

	l.m.Lock()
	defer l.m.Unlock()

	if l.Len == 0 && at == nil {
		return l.newFirstElem()
	}

	if l != at.list {
		return nil
	}

	if l.Freed == nil {
		e = l.getElemData(l.Used_idx)
		l.Used_idx += 1
	} else {
		e = l.Freed
		e.prev = nil
		e.next = nil
		if l.Freed.next == nil {
			l.Freed = nil
		} else {
			l.Freed = l.Freed.next
			l.Freed.prev = nil
		}
	}
	e.list = l
	n := at.next
	at.next = e
	e.prev = at
	if n != nil {
		n.prev = e
		e.next = n
	} else {
		e.list.Used.prev = e
	}

	l.Len++
	return e
}

func (l *List) TypeOfValue_inf() reflect.Type {
	if reflect.TypeOf(l.Value_inf).Kind() == reflect.Ptr {
		return reflect.ValueOf(l.Value_inf).Elem().Type()
	} else {
		return reflect.TypeOf(l.Value_inf)
	}
}

func (l *List) Init(first_value interface{}, value_len int) *List {
	l.m.Lock()
	defer l.m.Unlock()
	if l.Used == nil {
		var buf_len int64
		if value_len < 1024 {
			buf_len = int64(DEFAULT_BUF_SIZE)
		} else {
			buf_len = int64(value_len)
		}
		l.Value_inf = first_value
		l.SizeData = int64(l.TypeOfValue_inf().Size())
		l.SizeElm = int64(reflect.TypeOf(Element{}).Size())
		l.elms = make([]byte, buf_len*l.SizeElm,
			buf_len*l.SizeElm)
		l.datas = make([]byte, buf_len*l.SizeData,
			buf_len*l.SizeData)
		elm := (*Element)(unsafe.Pointer(&l.elms[0]))
		elm.value = reflect.NewAt(l.TypeOfValue_inf(), unsafe.Pointer(&l.datas[0])).Interface()
		elm.prev = elm
		elm.next = nil
		elm.list = l
		l.Used = elm
		l.Freed = nil
		l.Used_idx = 1
		l.Len = 1
	}
	return l
}

func (l *List) Front() *Element {
	l.m.Lock()
	defer l.m.Unlock()

	return l.Used
}

func (l *List) Back() *Element {
	l.m.Lock()
	defer l.m.Unlock()

	if l.Used == nil {
		return nil
	} else {
		return l.Used.prev
	}
}

func (l *List) Inf() interface{} {
	return l.Value_inf
}

func (l *List) Value() interface{} {
	return l.Used.value
}
func (l *List) SetCastFunc(f func(val interface{}) interface{}) {
	l.cast_f = f
}

func (e *Element) List() *List {
	return e.list
}

func (e *Element) ValueWithCast() interface{} {
	return e.list.cast_f(e.Value())
}
<commit_msg> free avoided pointer on e.Free() e.InitValue()<commit_after>// Copyright 2015 Kazuhisa TAKEI<xtakei@me.com>. All rights reserved.
// Use of this source code is governed by MPL-2.0 license tha can be
// found in the LICENSE file

// Package buffer_list implements a double linked list with sequencial buffer data.
//
// To Get New First Data from buffer_list(l is a *List)
//		type Hoge Struct {
//			a int
//			b int
//		}
//		l := buffer_list.New(Hoge{})
//		hoge := l.GetElement(),Value().(*Hoge)
//		hoge.a = 1
//		hoge.b = 2
// To iterate over a list
//		for e := l.Front(); e != nil ; e = e.Next() {
//			a := (*Hoge)(e.Value())  // Hoge is Value type
//			// do something
//		}

package buffer_list

import (
	"fmt" // FIXME remove
	"reflect"
	"sync"
	"unsafe"
)

const (
	DEFAULT_BUF_SIZE = 1024
)

type Element struct {
	list      *List
	next      *Element
	prev      *Element
	old_value unsafe.Pointer
	value     interface{}
}

type List struct {
	Used      *Element
	Freed     *Element
	SizeElm   int64
	SizeData  int64
	Used_idx  int64
	Value_inf interface{}
	elms      []byte
	datas     []byte
	Len       int
	m         sync.Mutex
	cast_f    func(interface{}) interface{}
	pointers  map[uintptr]map[int]unsafe.Pointer
}

func New(first_value interface{}, buf_cnt int) (l *List) {
	l = new(List).Init(first_value, buf_cnt)
	l.pointers = make(map[uintptr]map[int]unsafe.Pointer)
	return l
}

func (e *Element) Commit() {
	e.List().Pick_ptr(e)
}

func (e *Element) free_pick_ptr() {
	l := e.list
	f_num := reflect.ValueOf(e.Value()).Elem().NumField()
	v_ptr := reflect.ValueOf(e.Value()).Pointer()

	if l.pointers[uintptr(v_ptr)] == nil {
		return
	}

	for i := 0; i < f_num; i++ {
		if l.pointers[uintptr(v_ptr)][i] != nil {
			fmt.Println("free value.member", i, l.pointers[uintptr(v_ptr)][i])

			delete(l.pointers[uintptr(v_ptr)], i)
		}
	}
}

func (l *List) Pick_ptr(e *Element) {
	f_num := reflect.ValueOf(e.Value()).Elem().NumField()
	v_ptr := reflect.ValueOf(e.Value()).Pointer()
	if l.pointers[uintptr(v_ptr)] == nil {
		l.pointers[uintptr(v_ptr)] = make(map[int]unsafe.Pointer)
	}

	for i := 0; i < f_num; i++ {
		m := reflect.ValueOf(e.Value()).Elem().Field(i)
		switch m.Kind() {
		case reflect.UnsafePointer:
			fallthrough
		case reflect.String:
			fallthrough
		case reflect.Slice:
			fallthrough
		case reflect.Map:
			fallthrough
		case reflect.Chan:
			fallthrough
		case reflect.Array:
			fallthrough
		case reflect.Func:
			fallthrough
		case reflect.Ptr:
			fmt.Println("detect ptr member", i, m.Kind(), m.Pointer())
			l.pointers[uintptr(v_ptr)][i] = unsafe.Pointer(m.Pointer())
		default:
		}

	}
}

func (l *List) GetDataPtr() uintptr {
	return uintptr(unsafe.Pointer(&l.datas[0]))
}
func (l *List) getElemData(idx int64) *Element {
	elm := (*Element)(unsafe.Pointer(&l.elms[int(l.SizeElm)*int(idx)]))
	elm.value = reflect.NewAt(l.TypeOfValue_inf(), unsafe.Pointer(&l.datas[int(l.SizeData)*int(idx)])).Interface()
	return elm
}
func (l *List) GetElement() *Element {
	return l.Used
}
func (e *Element) Next() *Element {
	e.list.m.Lock()
	defer e.list.m.Unlock()

	if e.next != nil {
		return e.next
	} else {
		return nil
	}
}

func (e *Element) Prev() *Element {
	e.list.m.Lock()
	defer e.list.m.Unlock()

	if e.prev != nil {
		return e.prev
	} else {
		return nil
	}
}

func (e *Element) Value() interface{} {
	return e.value
}

func (e *Element) Free() {

	e.list.m.Lock()
	defer e.list.m.Unlock()

	for ee := e.list.Used; ee != nil; ee = ee.next {
		if e == ee {
			goto DO_FREE
		}
	}

	//	fmt.Println("dont Free() e is not used ")
	return

DO_FREE:
	//	fmt.Println("do Free()")

	at := e.prev
	n := e.next
	if at.next == e {
		at.next = n
	}
	if n != nil {
		n.prev = at
	}

	e.list.Len -= 1

	if e.list.Used == e {
		e.list.Used = n
	}
	// move to free buffer
	if e.list.Freed == nil {
		e.prev = nil
		e.next = nil
		e.list.Freed = e
	} else {
		f_at := e.list.Freed
		e.next = f_at
		e.prev = nil
		f_at.prev = e
		e.list.Freed = e
	}
	e.free_pick_ptr()
}

func (e *Element) InitValue() {

	e.free_pick_ptr()

	diff := int(uint64(reflect.ValueOf(e.value).Pointer()) - uint64(uintptr(unsafe.Pointer(&e.list.datas[0]))))

	for i := range e.list.datas[diff : diff+int(e.list.SizeData)-1] {
		e.list.datas[diff+i] = 0
	}

	return
	//	fmt.Println(ref_byte, databyte)
}
func (l *List) newFirstElem() *Element {
	var e *Element

	//	l.m.Lock()
	//	defer l.m.Unlock()

	if l.Freed == nil {
		e = l.getElemData(l.Used_idx)
		l.Used_idx += 1
	} else {
		e = l.Freed
		if l.Freed.next == nil {
			l.Freed = nil
		} else {
			l.Freed = l.Freed.next
			l.Freed.prev = nil
		}
	}
	e.prev = e
	e.next = nil
	e.list = l
	if l.Used == nil {
		l.Used = e
	}
	l.Len++
	return e
}

func (l *List) InsertNewElem(at *Element) *Element {
	var e *Element

	l.m.Lock()
	defer l.m.Unlock()

	if l.Len == 0 && at == nil {
		return l.newFirstElem()
	}

	if l != at.list {
		return nil
	}

	if l.Freed == nil {
		e = l.getElemData(l.Used_idx)
		l.Used_idx += 1
	} else {
		e = l.Freed
		e.prev = nil
		e.next = nil
		if l.Freed.next == nil {
			l.Freed = nil
		} else {
			l.Freed = l.Freed.next
			l.Freed.prev = nil
		}
	}
	e.list = l
	n := at.next
	at.next = e
	e.prev = at
	if n != nil {
		n.prev = e
		e.next = n
	} else {
		e.list.Used.prev = e
	}

	l.Len++
	return e
}

func (l *List) TypeOfValue_inf() reflect.Type {
	if reflect.TypeOf(l.Value_inf).Kind() == reflect.Ptr {
		return reflect.ValueOf(l.Value_inf).Elem().Type()
	} else {
		return reflect.TypeOf(l.Value_inf)
	}
}

func (l *List) Init(first_value interface{}, value_len int) *List {
	l.m.Lock()
	defer l.m.Unlock()
	if l.Used == nil {
		var buf_len int64
		if value_len < 1024 {
			buf_len = int64(DEFAULT_BUF_SIZE)
		} else {
			buf_len = int64(value_len)
		}
		l.Value_inf = first_value
		l.SizeData = int64(l.TypeOfValue_inf().Size())
		l.SizeElm = int64(reflect.TypeOf(Element{}).Size())
		l.elms = make([]byte, buf_len*l.SizeElm,
			buf_len*l.SizeElm)
		l.datas = make([]byte, buf_len*l.SizeData,
			buf_len*l.SizeData)
		elm := (*Element)(unsafe.Pointer(&l.elms[0]))
		elm.value = reflect.NewAt(l.TypeOfValue_inf(), unsafe.Pointer(&l.datas[0])).Interface()
		elm.prev = elm
		elm.next = nil
		elm.list = l
		l.Used = elm
		l.Freed = nil
		l.Used_idx = 1
		l.Len = 1
	}
	return l
}

func (l *List) Front() *Element {
	l.m.Lock()
	defer l.m.Unlock()

	return l.Used
}

func (l *List) Back() *Element {
	l.m.Lock()
	defer l.m.Unlock()

	if l.Used == nil {
		return nil
	} else {
		return l.Used.prev
	}
}

func (l *List) Inf() interface{} {
	return l.Value_inf
}

func (l *List) Value() interface{} {
	return l.Used.value
}
func (l *List) SetCastFunc(f func(val interface{}) interface{}) {
	l.cast_f = f
}

func (e *Element) List() *List {
	return e.list
}

func (e *Element) ValueWithCast() interface{} {
	return e.list.cast_f(e.Value())
}
<|endoftext|>


<commit_before>package cockroachdb

import "github.com/jmoiron/sqlx"

// Link is used to insert and update in mysql
type Link struct{}

// MigrateUp creates the needed tables
func (link *Link) MigrateUp(exec sqlx.Execer) (errExec error) {
	_, errExec = exec.Exec(
		`
        CREATE TABLE IF NOT EXISTS entityone (
            entityone_id BIGSERIAL NOT NULL,
            time_created DATE NOT NULL DEFAULT CURRENT_DATE,
            PRIMARY KEY (entityone_id)
        )
    `)
	if errExec != nil {
		return errExec
	}

	_, errExec = exec.Exec(
		`
        CREATE TABLE IF NOT EXISTS entityone_status (
            entityone_id BIGSERIAL NOT NULL,
            action_id BIGINT NOT NULL DEFAULT 1,
            status_id INT NOT NULL DEFAULT 1,
            time_created DATE NOT NULL DEFAULT CURRENT_DATE,
            is_latest INT NULL DEFAULT 1,
            UNIQUE (is_latest, entityone_id),
            INDEX (status_id, is_latest),
            CONSTRAINT es_fk_e
            FOREIGN KEY (entityone_id)
            REFERENCES entityone (entityone_id),
            INDEX (entityone_id)
        )
    `)
	return errExec
}

// MigrateDown destroys the needed tables
func (link *Link) MigrateDown(exec sqlx.Execer) (errExec error) {
	_, errExec = exec.Exec("DROP TABLE IF EXISTS entityone_status")
	if errExec != nil {
		return errExec
	}

	_, errExec = exec.Exec("DROP TABLE IF EXISTS entityone")
	return errExec
}
<commit_msg>clean statement<commit_after>package cockroachdb

import "github.com/jmoiron/sqlx"

// Link is used to insert and update in mysql
type Link struct{}

// MigrateUp creates the needed tables
func (link *Link) MigrateUp(exec sqlx.Execer) (errExec error) {
	_, errExec = exec.Exec(
		`
        CREATE TABLE IF NOT EXISTS entityone (
            entityone_id BIGSERIAL NOT NULL,
            time_created DATE NOT NULL DEFAULT CURRENT_DATE,
            PRIMARY KEY (entityone_id)
        )
    `)
	if errExec != nil {
		return errExec
	}

	_, errExec = exec.Exec(
		`
        CREATE TABLE IF NOT EXISTS entityone_status (
            entityone_id BIGSERIAL NOT NULL,
            action_id BIGINT NOT NULL DEFAULT 1,
            status_id INT NOT NULL DEFAULT 1,
            time_created DATE NOT NULL DEFAULT CURRENT_DATE,
            is_latest INT NULL DEFAULT 1,
            UNIQUE (is_latest, entityone_id),
            INDEX (status_id, is_latest),
            INDEX (entityone_id),
            CONSTRAINT es_fk_e
            FOREIGN KEY (entityone_id)
            REFERENCES entityone (entityone_id)
        )
    `)
	return errExec
}

// MigrateDown destroys the needed tables
func (link *Link) MigrateDown(exec sqlx.Execer) (errExec error) {
	_, errExec = exec.Exec("DROP TABLE IF EXISTS entityone_status")
	if errExec != nil {
		return errExec
	}

	_, errExec = exec.Exec("DROP TABLE IF EXISTS entityone")
	return errExec
}
<|endoftext|>


<commit_before>// +build example,go1.7

package main

import (
	"context"
	"flag"
	"fmt"
	"os"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
)

// Uploads a file to S3 given a bucket and object key. Also takes a duration
// value to terminate the update if it doesn't complete within that time.
//
// The AWS Region needs to be provided in the AWS shared config or on the
// environment variable as `AWS_REGION`. Credentials also must be provided
// Will default to shared config file, but can load from environment if provided.
//
// Usage:
//   # Upload myfile.txt to myBucket/myKey. Must complete within 10 minutes or will fail
//   go run withContext.go -b mybucket -k myKey -d 10m < myfile.txt
func main() {
	var bucket, key string
	var timeout time.Duration

	flag.StringVar(&bucket, "b", "", "Bucket name.")
	flag.StringVar(&key, "k", "", "Object key name.")
	flag.DurationVar(&timeout, "d", 0, "Upload timeout.")
	flag.Parse()

	sess := session.Must(session.NewSession())
	svc := s3.New(sess)

	ctx := context.Background()
	var cancelFn func()
	if timeout > 0 {
		ctx, cancelFn = context.WithTimeout(ctx, timeout)
	}

	// Uploads the object to S3. The Context will interrupt the request
	resp, err := svc.PutObjectWithContext(ctx, &s3.PutObjectInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(key),
		Body:   os.Stdin,
	})

	fmt.Println(resp, err)

	// Cleanup context
	cancelFn()
}
<commit_msg>example/aws/withContext: Cleanup S3 PutObjectWithContext example (#1167)<commit_after>// +build example,go1.7

package main

import (
	"context"
	"flag"
	"fmt"
	"os"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/request"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
)

// Uploads a file to S3 given a bucket and object key. Also takes a duration
// value to terminate the update if it doesn't complete within that time.
//
// The AWS Region needs to be provided in the AWS shared config or on the
// environment variable as `AWS_REGION`. Credentials also must be provided
// Will default to shared config file, but can load from environment if provided.
//
// Usage:
//   # Upload myfile.txt to myBucket/myKey. Must complete within 10 minutes or will fail
//   go run withContext.go -b mybucket -k myKey -d 10m < myfile.txt
func main() {
	var bucket, key string
	var timeout time.Duration

	flag.StringVar(&bucket, "b", "", "Bucket name.")
	flag.StringVar(&key, "k", "", "Object key name.")
	flag.DurationVar(&timeout, "d", 0, "Upload timeout.")
	flag.Parse()

	sess := session.Must(session.NewSession())
	svc := s3.New(sess)

	// Create a context with a timeout that will abort the upload if it takes
	// more than the passed in timeout.
	ctx := context.Background()
	var cancelFn func()
	if timeout > 0 {
		ctx, cancelFn = context.WithTimeout(ctx, timeout)
	}
	// Ensure the context is canceled to prevent leaking.
	// See context package for more information, https://golang.org/pkg/context/
	defer cancelFn()

	// Uploads the object to S3. The Context will interrupt the request if the
	// timeout expires.
	_, err := svc.PutObjectWithContext(ctx, &s3.PutObjectInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(key),
		Body:   os.Stdin,
	})
	if err != nil {
		if aerr, ok := err.(awserr.Error); ok && aerr.Code() == request.CanceledErrorCode {
			// If the SDK can determine the request or retry delay was canceled
			// by a context the CanceledErrorCode error code will be returned.
			fmt.Fprintf(os.Stderr, "upload canceled due to timeout, %v\n", err)
		} else {
			fmt.Fprintf(os.Stderr, "failed to upload object, %v\n", err)
		}
		os.Exit(1)
	}

	fmt.Printf("successfully uploaded file to %s/%s\n", bucket, key)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mime/multipart"
	"net/http"
	"os"
	"path/filepath"
	"strings"

	"github.com/ginuerzh/weedo"
)

type ResponseData struct {
	Fid      string `json:"fid"`
	FileName string `json:"fileName"`
	FileUrl  string `json:"fileUrl"`
	Size     int    `json:"size"`
}

type FilerResponse struct {
	Name string `json:"name"`
	Size []byte `json:"size"`
}

// Creates a new file upload http request with optional extra params
func newfileUploadRequest(uri string, params map[string]string, paramName, path string) (*http.Request, error) {
	file, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)
	part, err := writer.CreateFormFile(paramName, filepath.Base(path))
	if err != nil {
		return nil, err
	}
	_, err = io.Copy(part, file)
	if err != nil {
		return nil, err
	}

	for key, val := range params {
		err = writer.WriteField(key, val)
		if err != nil {
			return nil, err
		}
	}
	err = writer.Close()
	if err != nil {
		return nil, err
	}
	req, err := http.NewRequest("POST", uri, body)
	if err != nil {
		return nil, err
	}
	//如果不设置会报request Content-Type isn't multipart/form-data
	req.Header.Set("Content-Type", writer.FormDataContentType())
	return req, nil
}

func main() {
	downFile("http://localhost:8888/image/plate_detect.jpg")
	downFile("http://localhost:8888/submit/test.txt")
}

func testSubmit() {
	request, err := newfileUploadRequest("http://localhost:9333/submit", map[string]string{"aa": "12"}, "upload", "plate_detect.jpg")
	if err != nil {
		log.Fatal(err)
		return
	}
	client := &http.Client{}
	resp, err := client.Do(request)
	if err != nil {
		log.Fatal(err)
	} else {
		body := &bytes.Buffer{}
		_, err := body.ReadFrom(resp.Body)
		if err != nil {
			log.Fatal(err)
		}
		resp.Body.Close()
		fmt.Println(resp.StatusCode)
		fmt.Println(resp.Header)
		if resp.StatusCode == http.StatusCreated {
			var rep ResponseData
			err = json.Unmarshal(body.Bytes(), &rep)
			fmt.Println(rep, err)
		} else {
			fmt.Println(body)
		}

	}
}

func testFiler() {
	request, err := newfileUploadRequest("http://localhost:8888/image/", map[string]string{"aa": "12"}, "file", "plate_detect.jpg")
	if err != nil {
		log.Fatal(err)
		return
	}
	client := &http.Client{}
	resp, err := client.Do(request)
	if err != nil {
		log.Fatal(err)
	} else {
		body := &bytes.Buffer{}
		_, err := body.ReadFrom(resp.Body)
		if err != nil {
			log.Fatal(err)
		}
		resp.Body.Close()
		fmt.Println(resp.StatusCode)
		fmt.Println(resp.Header)
		if resp.StatusCode == http.StatusCreated {
			var rep FilerResponse
			err = json.Unmarshal(body.Bytes(), &rep)
			fmt.Println(rep, err)
		} else {
			fmt.Println(body)
		}

	}
}

func testWeedo() {
	client := weedo.NewClient("localhost:9333", "localhost:8888")
	dir, err := client.Filer("localhost:8888").Dir("/")
	if err != nil {
		fmt.Println(err)
		return
	}
	fmt.Println(dir)
}

func downFile(url string) (pix []byte, err error) {
	path := strings.Split(url, "/")
	var name string
	if len(path) > 1 {
		name = path[len(path)-1]
	}
	fmt.Println(name)

	resp, err := http.Get(url)
	if err != nil {
		fmt.Println(err)
		return
	}
	defer resp.Body.Close()
	pix, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Println(err)
		return
	}
	fmt.Println(string(pix))
	return
}

func saveFile(b []byte, name string) (int64, error) {
	out, err := os.Create(name)
	if err != nil {
		fmt.Println(err)
		return 0, err
	}
	defer out.Close()
	return io.Copy(out, bytes.NewReader(b))
}
<commit_msg>no message<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mime/multipart"
	"net/http"
	"os"
	"path/filepath"
	"strings"

	"github.com/ginuerzh/weedo"
)

type ResponseData struct {
	Fid      string `json:"fid"`
	FileName string `json:"fileName"`
	FileUrl  string `json:"fileUrl"`
	Size     int    `json:"size"`
}

type FilerResponse struct {
	Name string `json:"name"`
	Size []byte `json:"size"`
}

// Creates a new file upload http request with optional extra params
func newfileUploadRequest(uri string, params map[string]string, paramName, path string) (*http.Request, error) {
	file, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)
	part, err := writer.CreateFormFile(paramName, filepath.Base(path))
	if err != nil {
		return nil, err
	}
	_, err = io.Copy(part, file)
	if err != nil {
		return nil, err
	}

	for key, val := range params {
		err = writer.WriteField(key, val)
		if err != nil {
			return nil, err
		}
	}
	err = writer.Close()
	if err != nil {
		return nil, err
	}
	req, err := http.NewRequest("POST", uri, body)
	if err != nil {
		return nil, err
	}
	//如果不设置会报request Content-Type isn't multipart/form-data
	req.Header.Set("Content-Type", writer.FormDataContentType())
	return req, nil
}

func main() {
	http.HandleFunc("/hello", HelloServer)
	err := http.ListenAndServe(":8096", nil)
	if err != nil {
		log.Fatal("ListenAndServe: ", err)
	}
	//testWeedoUpload("test.txt")
	//testWeedoDelete("5,06d8deeb6a")
	//downFile("http://localhost:8888/image/plate_detect.jpg")
	//downFile("http://localhost:8888/submit/test.txt")
}

func testSubmit() {
	request, err := newfileUploadRequest("http://localhost:9333/submit", map[string]string{"aa": "12"}, "upload", "plate_detect.jpg")
	if err != nil {
		log.Fatal(err)
		return
	}
	client := &http.Client{}
	resp, err := client.Do(request)
	if err != nil {
		log.Fatal(err)
	} else {
		body := &bytes.Buffer{}
		_, err := body.ReadFrom(resp.Body)
		if err != nil {
			log.Fatal(err)
		}
		resp.Body.Close()
		fmt.Println(resp.StatusCode)
		fmt.Println(resp.Header)
		if resp.StatusCode == http.StatusCreated {
			var rep ResponseData
			err = json.Unmarshal(body.Bytes(), &rep)
			fmt.Println(rep, err)
		} else {
			fmt.Println(body)
		}

	}
}

func testFiler() {
	request, err := newfileUploadRequest("http://localhost:8888/image/", map[string]string{"aa": "12"}, "file", "plate_detect.jpg")
	if err != nil {
		log.Fatal(err)
		return
	}
	client := &http.Client{}
	resp, err := client.Do(request)
	if err != nil {
		log.Fatal(err)
	} else {
		body := &bytes.Buffer{}
		_, err := body.ReadFrom(resp.Body)
		if err != nil {
			log.Fatal(err)
		}
		resp.Body.Close()
		fmt.Println(resp.StatusCode)
		fmt.Println(resp.Header)
		if resp.StatusCode == http.StatusCreated {
			var rep FilerResponse
			err = json.Unmarshal(body.Bytes(), &rep)
			fmt.Println(rep, err)
		} else {
			fmt.Println(body)
		}

	}
}

var client = weedo.NewClient("localhost:9333", "localhost:8888")

func testWeedoUpload(fileName string) {
	file, err := os.Open(fileName)
	if err != nil {
		fmt.Println(err)
		return
	}
	defer file.Close()
	fid, size, err := client.AssignUpload(fileName, "", file)
	if err != nil {
		fmt.Println(err)
		return
	}
	fmt.Println(fid, size)
}

func testWeedoDelete(fid string) {
	fmt.Println(client.Delete(fid, 1))
}

func HelloServer(w http.ResponseWriter, req *http.Request) {
	resp, err := http.Get("http://localhost:8888/image/plate_detect.jpg")
	if err != nil {
		fmt.Println(err)
		http.NotFound(w, req)
		return
	}
	defer resp.Body.Close()
	for k, v := range resp.Header {
		if len(v) > 0 {
			w.Header().Set(k, v[0])
		}
	}
	io.Copy(w, resp.Body)
}

func downFile(url string) (pix []byte, err error) {
	path := strings.Split(url, "/")
	var name string
	if len(path) > 1 {
		name = path[len(path)-1]
	}
	fmt.Println(name)

	resp, err := http.Get(url)
	if err != nil {
		fmt.Println(err)
		return
	}
	defer resp.Body.Close()
	pix, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Println(err)
		return
	}
	//fmt.Println(string(pix))
	fmt.Println(resp.Header)
	return
}

func saveFile(b []byte, name string) (int64, error) {
	out, err := os.Create(name)
	if err != nil {
		fmt.Println(err)
		return 0, err
	}
	defer out.Close()
	return io.Copy(out, bytes.NewReader(b))
}
<|endoftext|>


<commit_before>// Copyright 2017 Volker Dobler.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gui

import (
	"reflect"
	"regexp"
)

// ----------------------------------------------------------------------------
// Interface implementations

// Implements is the global lookup of what types implement a given interface
var Implements = make(map[reflect.Type][]reflect.Type)

// RegisterImplementation records that iface is implemented by typ.
// To register that AbcWriter implements the Writer interface use:
//     	RegisterImplementation((*Writer)(nil), AbcWriter{})
func RegisterImplementation(iface interface{}, typ interface{}) {
	ifaceType := reflect.TypeOf(iface).Elem()
	concreteType := reflect.TypeOf(typ)
	for _, impl := range Implements[ifaceType] {
		if impl == concreteType {
			return // already registered
		}
	}
	Implements[ifaceType] = append(Implements[ifaceType], concreteType)
}

// ----------------------------------------------------------------------------
// Type Data

var Typedata = make(map[reflect.Type]Typeinfo)

func RegisterType(typ interface{}, info Typeinfo) {
	Typedata[reflect.TypeOf(typ)] = info
}

// Typeinfo contains metadata for types.
type Typeinfo struct {
	// Doc is the documentation for the type as a whole.
	Doc string

	// Fields contains field metadata indexed by field name.
	Field map[string]Fieldinfo
}

// Fielddata contains metadata to fields of structs.
type Fieldinfo struct {
	Doc       string         // Doc is the field documentation
	Multiline bool           // Multiline allows multiline strings
	Const     bool           // Const values are unchangable (display only)
	Only      []string       // Only contains the set of allowed values
	Validate  *regexp.Regexp // Validate this field
	Omit      bool           // Omit this field
}

// ----------------------------------------------------------------------------
// CSS

// CSS contains some minimal CSS definitions needed to render the HTML properly.
var CSS = `
body {
  margin: 40px;
}

textarea {
  vertical-align: text-top;
}

.Notrun { color: grey; }
.Skipped { color: grey; }
.Pass { color: darkgreen; }
.Fail { color: red; }
.Bogus { color: magenta; }
.Error { color: magenta; }
.error { colro: darkred; }

p.msg-bogus {
  color: fuchsia;
  font-weigth: bold;
  margin: 2px 0px 2px 10px;
}

p.msg-error {
  color: red;
  font-weigth: bold;
  margin: 2px 0px 2px 10px;
}

p.msg-fail {
  color: tomato;
  margin: 2px 0px 2px 10px;
}

p.msg-pass {
  color: green;
  margin: 2px 0px 2px 10px;
}

p.msg-skipped {
  color: dim-grey;
  margin: 2px 0px 2px 10px;
}

p.msg-notrun {
  color: light-grey;
  margin: 2px 0px 2px 10px;
}

table {
  border-collapse: collapse;
}

table.map>tbody>tr>th, table.map>tbody>tr>td {
  border-top: 1px solid #777;
  border-bottom: 1px solid #777;
  padding-top: 4px;
  padding-bottom: 4px;
}

th {
    text-align: right;
}

td, th {
  vertical-align: top;
}

pre {
  margin: 4px;
}

.tooltip {
  position: relative;
  display: inline-block;
}

.tooltip .tooltiptext {
  visibility: hidden;
  width: 600px;
  background-color: #404040;
  color: #eeeeee;
  text-align: left;
  border-radius: 6px;
  padding: 6px;

  /* Position the tooltip */
  position: absolute;
  z-index: 1;
  top: 20px;
  left: 20%;
}

.tooltip:hover .tooltiptext {
  visibility: visible;
}

input[type="text"] {
  width: 400px;
}

label {
  display: inline-block;
  width: 7em;
  text-align: right;
  vertical-align: text-top;
}

.actionbutton {
  background-color: #4CAF50;
  border: none;
  color: black;
  padding: 15px 32px;
  text-align: center;
  text-decoration: none;
  display: inline-block;
  width: 200px;
  font-size: 18px;
  font-family: "Arial Black", Gadget, sans-serif;
  margin: 4px 2px;
  cursor: pointer;
}
`

// ----------------------------------------------------------------------------
// Favicon

// Favicon is a blue/red "ht" in 16x16 ico format.
var Favicon = []byte{
	0, 0, 1, 0, 1, 0, 16, 16, 16, 0, 1, 0, 4, 0, 40, 1,
	0, 0, 22, 0, 0, 0, 40, 0, 0, 0, 16, 0, 0, 0, 32, 0,
	0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 2, 2, 179, 0, 184, 6, 14, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 2, 32, 0, 34, 0, 1, 16, 0, 2, 32,
	0, 34, 0, 1, 16, 0, 2, 32, 0, 34, 0, 1, 16, 0, 2, 32,
	0, 34, 0, 1, 16, 0, 2, 32, 0, 34, 0, 1, 16, 0, 2, 32,
	0, 34, 0, 1, 16, 0, 2, 34, 0, 34, 0, 1, 16, 0, 2, 34,
	34, 32, 0, 1, 16, 0, 2, 32, 34, 0, 17, 17, 17, 17, 2, 32,
	0, 0, 17, 17, 17, 17, 2, 32, 0, 0, 0, 1, 16, 0, 2, 32,
	0, 0, 0, 1, 16, 0, 2, 32, 0, 0, 0, 1, 16, 0, 2, 32,
	0, 0, 0, 1, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255,
	0, 0, 156, 231, 0, 0, 156, 231, 0, 0, 156, 231, 0, 0, 156, 231,
	0, 0, 156, 231, 0, 0, 156, 231, 0, 0, 140, 231, 0, 0, 129, 231,
	0, 0, 147, 0, 0, 0, 159, 0, 0, 0, 159, 231, 0, 0, 159, 231,
	0, 0, 159, 231, 0, 0, 159, 231, 0, 0, 255, 255, 0, 0,
}
<commit_msg>gui: make tooltip wide enough for 80 chars<commit_after>// Copyright 2017 Volker Dobler.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gui

import (
	"reflect"
	"regexp"
)

// ----------------------------------------------------------------------------
// Interface implementations

// Implements is the global lookup of what types implement a given interface
var Implements = make(map[reflect.Type][]reflect.Type)

// RegisterImplementation records that iface is implemented by typ.
// To register that AbcWriter implements the Writer interface use:
//     	RegisterImplementation((*Writer)(nil), AbcWriter{})
func RegisterImplementation(iface interface{}, typ interface{}) {
	ifaceType := reflect.TypeOf(iface).Elem()
	concreteType := reflect.TypeOf(typ)
	for _, impl := range Implements[ifaceType] {
		if impl == concreteType {
			return // already registered
		}
	}
	Implements[ifaceType] = append(Implements[ifaceType], concreteType)
}

// ----------------------------------------------------------------------------
// Type Data

var Typedata = make(map[reflect.Type]Typeinfo)

func RegisterType(typ interface{}, info Typeinfo) {
	Typedata[reflect.TypeOf(typ)] = info
}

// Typeinfo contains metadata for types.
type Typeinfo struct {
	// Doc is the documentation for the type as a whole.
	Doc string

	// Fields contains field metadata indexed by field name.
	Field map[string]Fieldinfo
}

// Fielddata contains metadata to fields of structs.
type Fieldinfo struct {
	Doc       string         // Doc is the field documentation
	Multiline bool           // Multiline allows multiline strings
	Const     bool           // Const values are unchangable (display only)
	Only      []string       // Only contains the set of allowed values
	Validate  *regexp.Regexp // Validate this field
	Omit      bool           // Omit this field
}

// ----------------------------------------------------------------------------
// CSS

// CSS contains some minimal CSS definitions needed to render the HTML properly.
var CSS = `
body {
  margin: 40px;
}

textarea {
  vertical-align: text-top;
}

.Notrun { color: grey; }
.Skipped { color: grey; }
.Pass { color: darkgreen; }
.Fail { color: red; }
.Bogus { color: magenta; }
.Error { color: magenta; }
.error { colro: darkred; }

p.msg-bogus {
  color: fuchsia;
  font-weigth: bold;
  margin: 2px 0px 2px 10px;
}

p.msg-error {
  color: red;
  font-weigth: bold;
  margin: 2px 0px 2px 10px;
}

p.msg-fail {
  color: tomato;
  margin: 2px 0px 2px 10px;
}

p.msg-pass {
  color: green;
  margin: 2px 0px 2px 10px;
}

p.msg-skipped {
  color: dim-grey;
  margin: 2px 0px 2px 10px;
}

p.msg-notrun {
  color: light-grey;
  margin: 2px 0px 2px 10px;
}

table {
  border-collapse: collapse;
}

table.map>tbody>tr>th, table.map>tbody>tr>td {
  border-top: 1px solid #777;
  border-bottom: 1px solid #777;
  padding-top: 4px;
  padding-bottom: 4px;
}

th {
    text-align: right;
}

td, th {
  vertical-align: top;
}

pre {
  margin: 4px;
}

.tooltip {
  position: relative;
  display: inline-block;
}

.tooltip .tooltiptext {
  visibility: hidden;
  width: 640px;
  background-color: #404040;
  color: #eeeeee;
  text-align: left;
  border-radius: 6px;
  padding: 6px;

  /* Position the tooltip */
  position: absolute;
  z-index: 1;
  top: 20px;
  left: 20%;
}

.tooltip:hover .tooltiptext {
  visibility: visible;
}

input[type="text"] {
  width: 400px;
}

label {
  display: inline-block;
  width: 7em;
  text-align: right;
  vertical-align: text-top;
}

.actionbutton {
  background-color: #4CAF50;
  border: none;
  color: black;
  padding: 15px 32px;
  text-align: center;
  text-decoration: none;
  display: inline-block;
  width: 200px;
  font-size: 18px;
  font-family: "Arial Black", Gadget, sans-serif;
  margin: 4px 2px;
  cursor: pointer;
}
`

// ----------------------------------------------------------------------------
// Favicon

// Favicon is a blue/red "ht" in 16x16 ico format.
var Favicon = []byte{
	0, 0, 1, 0, 1, 0, 16, 16, 16, 0, 1, 0, 4, 0, 40, 1,
	0, 0, 22, 0, 0, 0, 40, 0, 0, 0, 16, 0, 0, 0, 32, 0,
	0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 2, 2, 179, 0, 184, 6, 14, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 2, 32, 0, 34, 0, 1, 16, 0, 2, 32,
	0, 34, 0, 1, 16, 0, 2, 32, 0, 34, 0, 1, 16, 0, 2, 32,
	0, 34, 0, 1, 16, 0, 2, 32, 0, 34, 0, 1, 16, 0, 2, 32,
	0, 34, 0, 1, 16, 0, 2, 34, 0, 34, 0, 1, 16, 0, 2, 34,
	34, 32, 0, 1, 16, 0, 2, 32, 34, 0, 17, 17, 17, 17, 2, 32,
	0, 0, 17, 17, 17, 17, 2, 32, 0, 0, 0, 1, 16, 0, 2, 32,
	0, 0, 0, 1, 16, 0, 2, 32, 0, 0, 0, 1, 16, 0, 2, 32,
	0, 0, 0, 1, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255,
	0, 0, 156, 231, 0, 0, 156, 231, 0, 0, 156, 231, 0, 0, 156, 231,
	0, 0, 156, 231, 0, 0, 156, 231, 0, 0, 140, 231, 0, 0, 129, 231,
	0, 0, 147, 0, 0, 0, 159, 0, 0, 0, 159, 231, 0, 0, 159, 231,
	0, 0, 159, 231, 0, 0, 159, 231, 0, 0, 255, 255, 0, 0,
}
<|endoftext|>


<commit_before>package hana

import (
	"bytes"
	"encoding/gob"
	"errors"
	"fmt"
	"strconv"
	"strings"
	"time"
	"reflect"

	log "github.com/Sirupsen/logrus"

	"github.com/intelsdi-x/pulse/control/plugin"
	"github.com/intelsdi-x/pulse/control/plugin/cpolicy"
	"github.com/intelsdi-x/pulse/core/ctypes"

	"database/sql"
	"github.com/SAP/go-hdb/driver"
)

const (
	name       = "hana"
	version    = 1
	pluginType = plugin.PublisherPluginType
)

type HANAPublisher struct {
}

func NewHANAPublisher() *HANAPublisher {
	return &HANAPublisher{}
}

// Publish sends data to a HANA server
func (s *HANAPublisher) Publish(contentType string, content []byte, config map[string]ctypes.ConfigValue) error {
	logger := log.New()
	logger.Println("Publishing started")
	var metrics []plugin.PluginMetricType

	switch contentType {
	case plugin.PulseGOBContentType:
		dec := gob.NewDecoder(bytes.NewBuffer(content))
		if err := dec.Decode(&metrics); err != nil {
			logger.Printf("Error decoding: error=%v content=%v", err, content)
			return err
		}
	default:
		logger.Printf("Error unknown content type '%v'", contentType)
		return errors.New(fmt.Sprintf("Unknown content type '%s'", contentType))
	}

	logger.Printf("publishing %v to %v", metrics, config)

	// Open connection and ping to make sure it works
	username := config["username"].(ctypes.ConfigValueStr).Value
	password := config["password"].(ctypes.ConfigValueStr).Value
	host := config["host"].(ctypes.ConfigValueStr).Value
	database := config["database"].(ctypes.ConfigValueStr).Value
	port := config["port"].(ctypes.ConfigValueStr).Value
	tableName := config["table name"].(ctypes.ConfigValueStr).Value
	tableColumns := "(time_posted VARCHAR(200), key_column VARCHAR(200), value_column VARCHAR(200))"
	db, err := sql.Open( driver.DriverName, "hdb://"+username+":"+password+"@"+host+":"+port+"/"+database)
	defer db.Close()
	if err != nil {
		logger.Printf("Error: %v", err)
		return err
	}
	err = db.Ping()
	if err != nil {
		logger.Printf("Error: %v", err)
		return err
	}

	// Create the table if it's not already there
	_, err = db.Exec( "DROP PROCEDURE ifexists" )

	if err != nil {
		logger.Printf("Error while dropping procedure: %v", err)
	}

	createTableStr :=
		"CREATE PROCEDURE ifexists( ) LANGUAGE SQLSCRIPT AS myrowid integer;\n" +
		"BEGIN\n" +
		" myrowid := 0;\n" +
		" SELECT COUNT(*) INTO myrowid FROM \"PUBLIC\".\"M_TABLES\" " +
		" WHERE schema_name = '" + database + "' AND table_name = '" + tableName + "';\n" +
		" IF :myrowid = 0 THEN\n" +
		"  exec 'CREATE COLUMN TABLE \"" + database + "\".\"" + tableName + "\" " + tableColumns + "';\n " +
		" END IF;\n" +
	  "END;"

	_, err = db.Exec( createTableStr )

	if err != nil {
		logger.Printf("Error while creating procedure: %v", err)
		logger.Printf( "Query: %v", createTableStr )
	}

	_, err = db.Exec( "CALL ifexists" )

	if err != nil {
		logger.Printf("Error while invoking procedure: %v", err)

		return err
	}

	// Put the values into the database with the current time
	tableValues := "VALUES( ?, ?, ? )"
	insert, err := db.Prepare("INSERT INTO " + database + "." + tableName + " " + tableValues)
	if err != nil {
		logger.Printf("Error: %v", err)
		logger.Printf( "tablename: " + database + "." + tableName + ", tableValues: " + tableValues )
		return err
	}
	nowTime := time.Now()
	var key, value string
	for _, m := range metrics {
		key = sliceToString(m.Namespace())
		value, err = interfaceToString(m.Data())
		if err == nil {
			_, err := insert.Exec(nowTime, key, value)
			if err != nil {
				panic(err)
				logger.Printf("Error: %v", err)
			}
		} else {
			logger.Printf("Error: %v", err)
		}
	}

	return nil
}

func Meta() *plugin.PluginMeta {
	return plugin.NewPluginMeta(name, version, pluginType, []string{plugin.PulseGOBContentType}, []string{plugin.PulseGOBContentType})
}

func (f *HANAPublisher) GetConfigPolicy() cpolicy.ConfigPolicy {
	cp := cpolicy.New()
	config := cpolicy.NewPolicyNode()

	username, err := cpolicy.NewStringRule("username", true, "root")
	handleErr(err)
	username.Description = "Username to login to the HANA server"

	password, err := cpolicy.NewStringRule("password", true, "root")
	handleErr(err)
	password.Description = "Password to login to the HANA server"

	database, err := cpolicy.NewStringRule("database", true, "PULSE_TEST")
	handleErr(err)
	database.Description = "The HANA database that data will be pushed to"

	tableName, err := cpolicy.NewStringRule("table name", true, "info")
	handleErr(err)
	tableName.Description = "The HANA table within the database where information will be stored"

	config.Add(username)
	config.Add(password)
	config.Add(database)
	config.Add(tableName)

	cp.Add([]string{""}, config)
	return *cp
}

func handleErr(e error) {
	if e != nil {
		panic(e)
	}
}

func sliceToString(slice []string) string {
	return strings.Join(slice, ", ")
}

// Supported types: []string, []int, int, string
func interfaceToString(face interface{}) (string, error) {
	var (
		ret string
		err error
	)
	switch val := face.(type) {
	case []string:
		ret = sliceToString(val)
	case []int:
		length := len(val)
		if length == 0 {
			return ret, err
		}
		ret = strconv.Itoa(val[0])
		if length == 1 {
			return ret, err
		}
		for i := 1; i < length; i++ {
			ret += ", "
			ret += strconv.Itoa(val[i])
		}
	case int:
		ret = strconv.Itoa(val)
	case string:
		ret = val
	default:
		err = errors.New("unsupported type: " + reflect.TypeOf(face).String())
	}
	return ret, err
}
<commit_msg>added support for float32, float64, and uint64 values<commit_after>package hana

import (
	"bytes"
	"encoding/gob"
	"errors"
	"fmt"
	"strconv"
	"strings"
	"time"
	"reflect"

	log "github.com/Sirupsen/logrus"

	"github.com/intelsdi-x/pulse/control/plugin"
	"github.com/intelsdi-x/pulse/control/plugin/cpolicy"
	"github.com/intelsdi-x/pulse/core/ctypes"

	"database/sql"
	"github.com/SAP/go-hdb/driver"
)

const (
	name       = "hana"
	version    = 1
	pluginType = plugin.PublisherPluginType
)

type HANAPublisher struct {
}

func NewHANAPublisher() *HANAPublisher {
	return &HANAPublisher{}
}

// Publish sends data to a HANA server
func (s *HANAPublisher) Publish(contentType string, content []byte, config map[string]ctypes.ConfigValue) error {
	logger := log.New()
	logger.Println("Publishing started")
	var metrics []plugin.PluginMetricType

	switch contentType {
	case plugin.PulseGOBContentType:
		dec := gob.NewDecoder(bytes.NewBuffer(content))
		if err := dec.Decode(&metrics); err != nil {
			logger.Printf("Error decoding: error=%v content=%v", err, content)
			return err
		}
	default:
		logger.Printf("Error unknown content type '%v'", contentType)
		return errors.New(fmt.Sprintf("Unknown content type '%s'", contentType))
	}

	logger.Printf("publishing %v to %v", metrics, config)

	// Open connection and ping to make sure it works
	username := config["username"].(ctypes.ConfigValueStr).Value
	password := config["password"].(ctypes.ConfigValueStr).Value
	host := config["host"].(ctypes.ConfigValueStr).Value
	database := config["database"].(ctypes.ConfigValueStr).Value
	port := config["port"].(ctypes.ConfigValueStr).Value
	tableName := config["table name"].(ctypes.ConfigValueStr).Value
	tableColumns := "(time_posted VARCHAR(200), key_column VARCHAR(200), value_column VARCHAR(200))"
	db, err := sql.Open( driver.DriverName, "hdb://"+username+":"+password+"@"+host+":"+port+"/"+database)
	defer db.Close()
	if err != nil {
		logger.Printf("Error: %v", err)
		return err
	}
	err = db.Ping()
	if err != nil {
		logger.Printf("Error: %v", err)
		return err
	}

	// Create the table if it's not already there
	_, err = db.Exec( "DROP PROCEDURE ifexists" )

	if err != nil {
		logger.Printf("Error while dropping procedure: %v", err)
	}

	createTableStr :=
		"CREATE PROCEDURE ifexists( ) LANGUAGE SQLSCRIPT AS myrowid integer;\n" +
		"BEGIN\n" +
		" myrowid := 0;\n" +
		" SELECT COUNT(*) INTO myrowid FROM \"PUBLIC\".\"M_TABLES\" " +
		" WHERE schema_name = '" + database + "' AND table_name = '" + tableName + "';\n" +
		" IF :myrowid = 0 THEN\n" +
		"  exec 'CREATE COLUMN TABLE \"" + database + "\".\"" + tableName + "\" " + tableColumns + "';\n " +
		" END IF;\n" +
	  "END;"

	_, err = db.Exec( createTableStr )

	if err != nil {
		logger.Printf("Error while creating procedure: %v", err)
		logger.Printf( "Query: %v", createTableStr )
	}

	_, err = db.Exec( "CALL ifexists" )

	if err != nil {
		logger.Printf("Error while invoking procedure: %v", err)

		return err
	}

	// Put the values into the database with the current time
	tableValues := "VALUES( ?, ?, ? )"
	insert, err := db.Prepare("INSERT INTO " + database + "." + tableName + " " + tableValues)
	if err != nil {
		logger.Printf("Error: %v", err)
		logger.Printf( "tablename: " + database + "." + tableName + ", tableValues: " + tableValues )
		return err
	}
	nowTime := time.Now()
	var key, value string
	for _, m := range metrics {
		key = sliceToString(m.Namespace())
		value, err = interfaceToString(m.Data())
		if err == nil {
			_, err := insert.Exec(nowTime, key, value)
			if err != nil {
				panic(err)
				logger.Printf("Error: %v", err)
			}
		} else {
			logger.Printf("Error: %v", err)
		}
	}

	return nil
}

func Meta() *plugin.PluginMeta {
	return plugin.NewPluginMeta(name, version, pluginType, []string{plugin.PulseGOBContentType}, []string{plugin.PulseGOBContentType})
}

func (f *HANAPublisher) GetConfigPolicy() cpolicy.ConfigPolicy {
	cp := cpolicy.New()
	config := cpolicy.NewPolicyNode()

	username, err := cpolicy.NewStringRule("username", true, "root")
	handleErr(err)
	username.Description = "Username to login to the HANA server"

	password, err := cpolicy.NewStringRule("password", true, "root")
	handleErr(err)
	password.Description = "Password to login to the HANA server"

	database, err := cpolicy.NewStringRule("database", true, "PULSE_TEST")
	handleErr(err)
	database.Description = "The HANA database that data will be pushed to"

	tableName, err := cpolicy.NewStringRule("table name", true, "info")
	handleErr(err)
	tableName.Description = "The HANA table within the database where information will be stored"

	config.Add(username)
	config.Add(password)
	config.Add(database)
	config.Add(tableName)

	cp.Add([]string{""}, config)
	return *cp
}

func handleErr(e error) {
	if e != nil {
		panic(e)
	}
}

func sliceToString(slice []string) string {
	return strings.Join(slice, ", ")
}

// Supported types: []string, []int, int, uint64, float32, float64, string
func interfaceToString(face interface{}) (string, error) {
	var (
		ret string
		err error
	)
	switch val := face.(type) {
	case []string:
		ret = sliceToString(val)
	case []int:
		length := len(val)
		if length == 0 {
			return ret, err
		}
		ret = strconv.Itoa(val[0])
		if length == 1 {
			return ret, err
		}
		for i := 1; i < length; i++ {
			ret += ", "
			ret += strconv.Itoa(val[i])
		}
	case int:
		ret = strconv.Itoa(val)
	case uint64:
		ret = strconv.FormatUint(val, 10)
	case float32:
		ret = strconv.FormatFloat(float64(val), 'E', -1, 32)
	case float64:
		ret = strconv.FormatFloat(val, 'E', -1, 64)
	case string:
		ret = val
	default:
		err = errors.New("Unsupported type: " + reflect.TypeOf(face).String())
	}
	return ret, err
}
<|endoftext|>


<commit_before>package main

import (
  "bufio"
  "bytes"
  "io"
  "log"
  "os" // for File and friends
  "time"
)

type Harvester struct {
  Path   string /* the file path to harvest */
  Fields map[string]string
  Offset int64

  file *os.File /* the file being watched */
}

func (h *Harvester) Harvest(output chan *FileEvent) {
  if h.Offset > 0 {
    log.Printf("Starting harvester at position %d: %s\n", h.Offset, h.Path)
  } else {
    log.Printf("Starting harvester: %s\n", h.Path)
  }

  h.open()
  info, _ := h.file.Stat() // TODO(sissel): Check error
  defer h.file.Close()
  //info, _ := file.Stat()

  var line uint64 = 0 // Ask registrar about the line number

  // get current offset in file
  offset, _ := h.file.Seek(0, os.SEEK_CUR)

  log.Printf("Current file offset: %d\n", offset)

  // TODO(sissel): Make the buffer size tunable at start-time
  reader := bufio.NewReaderSize(h.file, 16<<10) // 16kb buffer by default

  var read_timeout = 10 * time.Second
  last_read_time := time.Now()
  for {
    text, bytesread, err := h.readline(reader, read_timeout)

    if err != nil {
      if err == io.EOF {
        // timed out waiting for data, got eof.
        // Check to see if the file was truncated
        info, _ := h.file.Stat()
        if info.Size() < offset {
          log.Printf("File truncated, seeking to beginning: %s\n", h.Path)
          h.file.Seek(0, os.SEEK_SET)
          offset = 0
        } else if age := time.Since(last_read_time); age > (24 * time.Hour) {
          // if last_read_time was more than 24 hours ago, this file is probably
          // dead. Stop watching it.
          // TODO(sissel): Make this time configurable
          // This file is idle for more than 24 hours. Give up and stop harvesting.
          log.Printf("Stopping harvest of %s; last change was %d seconds ago\n", h.Path, age.Seconds())
          return
        }
        continue
      } else {
        log.Printf("Unexpected state reading from %s; error: %s\n", h.Path, err)
        return
      }
    }
    last_read_time = time.Now()

    line++
    event := &FileEvent{
      Source:   &h.Path,
      Offset:   offset,
      Line:     line,
      Text:     text,
      Fields:   &h.Fields,
      fileinfo: &info,
    }
    offset += int64(bytesread)

    output <- event // ship the new event downstream
  } /* forever */
}

func (h *Harvester) open() *os.File {
  // Special handling that "-" means to read from standard input
  if h.Path == "-" {
    h.file = os.Stdin
    return h.file
  }

  for {
    var err error
    h.file, err = os.Open(h.Path)

    if err != nil {
      // retry on failure.
      log.Printf("Failed opening %s: %s\n", h.Path, err)
      time.Sleep(5 * time.Second)
    } else {
      break
    }
  }

  // TODO(sissel): Only seek if the file is a file, not a pipe or socket.
  if h.Offset > 0 {
    h.file.Seek(h.Offset, os.SEEK_SET)
  } else if *from_beginning {
    h.file.Seek(0, os.SEEK_SET)
  } else {
    h.file.Seek(0, os.SEEK_END)
  }

  return h.file
}

func (h *Harvester) readline(reader *bufio.Reader, eof_timeout time.Duration) (*string, int, error) {
  var buffer bytes.Buffer
  var is_partial bool = true
  var is_cr_present bool = false
  var bufferSize int = 0;
  start_time := time.Now()

  // Store current offset for seeking back on timeout if the line is not complete
  offset, _ := h.file.Seek(0, os.SEEK_CUR)

  for {
    segment, err := reader.ReadBytes('\n')

    if segment != nil && len(segment) > 0 {
      if segment[len(segment)-1] == '\n' {
        // Found a complete line
        is_partial = false

        // Check if also a CR present
        if len(segment) > 1 && segment[len(segment)-2] == '\r' {
          is_cr_present = true;
        }
      }
    }

    if segment != nil && len(segment) > 0 {

      // TODO(sissel): if buffer exceeds a certain length, maybe report an error condition? chop it?
      writelen,_ := buffer.Write(segment)
      bufferSize += writelen;

    }

    if err != nil {
      if err == io.EOF && is_partial {
        time.Sleep(1 * time.Second) // TODO(sissel): Implement backoff

        // Give up waiting for data after a certain amount of time.
        // If we time out, return the error (eof)
        if time.Since(start_time) > eof_timeout {

          // If we read a partial line then we seek back otherwise we miss this part
          if len(segment) > 0 || bufferSize > 0 {
            h.file.Seek(offset, os.SEEK_SET)
          }
          return nil, 0, err
        }
        continue
      } else {
        log.Println(err)
        return nil, 0, err // TODO(sissel): don't do this?
      }
    }

    if !is_partial {
      // If we got a full line, return the whole line without the EOL chars (CRLF or LF)
      str := new(string)
      if !is_cr_present {
        *str = buffer.String()[:bufferSize-1]
      } else {
        *str = buffer.String()[:bufferSize-2]
      }
      // bufferSize returns the str length with the EOL chars (LF or CRLF)
      return str, bufferSize, nil
    }
  } /* forever read chunks */

  return nil, 0, nil
}
<commit_msg>Small optimizations<commit_after>package main

import (
  "bufio"
  "bytes"
  "io"
  "log"
  "os" // for File and friends
  "time"
)

type Harvester struct {
  Path   string /* the file path to harvest */
  Fields map[string]string
  Offset int64

  file *os.File /* the file being watched */
}

func (h *Harvester) Harvest(output chan *FileEvent) {
  if h.Offset > 0 {
    log.Printf("Starting harvester at position %d: %s\n", h.Offset, h.Path)
  } else {
    log.Printf("Starting harvester: %s\n", h.Path)
  }

  h.open()
  info, _ := h.file.Stat() // TODO(sissel): Check error
  defer h.file.Close()
  //info, _ := file.Stat()

  var line uint64 = 0 // Ask registrar about the line number

  // get current offset in file
  offset, _ := h.file.Seek(0, os.SEEK_CUR)

  log.Printf("Current file offset: %d\n", offset)

  // TODO(sissel): Make the buffer size tunable at start-time
  reader := bufio.NewReaderSize(h.file, 16<<10) // 16kb buffer by default
  buffer := new(bytes.Buffer)

  var read_timeout = 10 * time.Second
  last_read_time := time.Now()
  for {
    text, bytesread, err := h.readline(reader, buffer, read_timeout)

    if err != nil {
      if err == io.EOF {
        // timed out waiting for data, got eof.
        // Check to see if the file was truncated
        info, _ := h.file.Stat()
        if info.Size() < offset {
          log.Printf("File truncated, seeking to beginning: %s\n", h.Path)
          h.file.Seek(0, os.SEEK_SET)
          offset = 0
        } else if age := time.Since(last_read_time); age > (24 * time.Hour) {
          // if last_read_time was more than 24 hours ago, this file is probably
          // dead. Stop watching it.
          // TODO(sissel): Make this time configurable
          // This file is idle for more than 24 hours. Give up and stop harvesting.
          log.Printf("Stopping harvest of %s; last change was %d seconds ago\n", h.Path, age.Seconds())
          return
        }
        continue
      } else {
        log.Printf("Unexpected state reading from %s; error: %s\n", h.Path, err)
        return
      }
    }
    last_read_time = time.Now()

    line++
    event := &FileEvent{
      Source:   &h.Path,
      Offset:   offset,
      Line:     line,
      Text:     text,
      Fields:   &h.Fields,
      fileinfo: &info,
    }
    offset += int64(bytesread)

    output <- event // ship the new event downstream
  } /* forever */
}

func (h *Harvester) open() *os.File {
  // Special handling that "-" means to read from standard input
  if h.Path == "-" {
    h.file = os.Stdin
    return h.file
  }

  for {
    var err error
    h.file, err = os.Open(h.Path)

    if err != nil {
      // retry on failure.
      log.Printf("Failed opening %s: %s\n", h.Path, err)
      time.Sleep(5 * time.Second)
    } else {
      break
    }
  }

  // TODO(sissel): Only seek if the file is a file, not a pipe or socket.
  if h.Offset > 0 {
    h.file.Seek(h.Offset, os.SEEK_SET)
  } else if *from_beginning {
    h.file.Seek(0, os.SEEK_SET)
  } else {
    h.file.Seek(0, os.SEEK_END)
  }

  return h.file
}

func (h *Harvester) readline(reader *bufio.Reader, buffer *bytes.Buffer, eof_timeout time.Duration) (*string, int, error) {
  var is_partial bool = true
  var newline_length int = 1
  start_time := time.Now()

  for {
    segment, err := reader.ReadBytes('\n')

    if segment != nil && len(segment) > 0 {
      if segment[len(segment)-1] == '\n' {
        // Found a complete line
        is_partial = false

        // Check if also a CR present
        if len(segment) > 1 && segment[len(segment)-2] == '\r' {
          newline_length++
        }
      }

      // TODO(sissel): if buffer exceeds a certain length, maybe report an error condition? chop it?
      buffer.Write(segment)
    }

    if err != nil {
      if err == io.EOF && is_partial {
        time.Sleep(1 * time.Second) // TODO(sissel): Implement backoff

        // Give up waiting for data after a certain amount of time.
        // If we time out, return the error (eof)
        if time.Since(start_time) > eof_timeout {
          return nil, 0, err
        }
        continue
      } else {
        log.Println(err)
        return nil, 0, err // TODO(sissel): don't do this?
      }
    }

    // If we got a full line, return the whole line without the EOL chars (CRLF or LF)
    if !is_partial {
      // Get the str length with the EOL chars (LF or CRLF)
      bufferSize := buffer.Len()
      str := new(string)
      *str = buffer.String()[:bufferSize - newline_length]
      // Reset the buffer for the next line
      buffer.Reset()
      return str, bufferSize, nil
    }
  } /* forever read chunks */

  return nil, 0, nil
}
<|endoftext|>


<commit_before>// Copyright 2013 Herbert G. Fischer. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package imagick

import (
	"reflect"
	"testing"
)

var (
	mw *MagickWand
)

func Init() {
	Initialize()
}

func TestNewMagickWand(t *testing.T) {
	mw := NewMagickWand()
	defer mw.Destroy()

	if !mw.IsVerified() {
		t.Fatal("MagickWand not verified")
	}
}

func TestCloningAndDestroying(t *testing.T) {
	mw := NewMagickWand()
	defer mw.Destroy()
	clone := mw.Clone()
	if !clone.IsVerified() {
		t.Fatal("Unsuccessful clone")
	}
	clone.Destroy()
	if clone.IsVerified() || !mw.IsVerified() {
		t.Fatal("MagickWand not properly destroyed")
	}
}

func TestQueryConfigureOptions(t *testing.T) {
	opts := mw.QueryConfigureOptions("*")
	if len(opts) == 0 {
		t.Fatal("QueryConfigureOptions returned an empty array")
	}
	for _, opt := range opts {
		mw.QueryConfigureOption(opt)
	}
}

func TestNonExistingConfigureOption(t *testing.T) {
	_, err := mw.QueryConfigureOption("4321foobaramps1234")
	if err == nil {
		t.Fatal("Missing error when trying to get non-existing configure option")
	}
}

func TestQueryFonts(t *testing.T) {
	fonts := mw.QueryFonts("*")
	if len(fonts) == 0 {
		t.Fatal("ImageMagick have not identified a single font in this system")
	}
}

func TestQueryFormats(t *testing.T) {
	formats := mw.QueryFormats("*")
	if len(formats) == 0 {
		t.Fatal("ImageMagick have not identified a single image format in this system")
	}
}

func TestDeleteImageArtifact(t *testing.T) {
	mw := NewMagickWand()
	defer mw.Destroy()

	mw.ReadImage(`logo:`)

	if err := mw.DeleteImageArtifact("*"); err != nil {
		t.Fatalf("Error calling DeleteImageArtifact: %s", err.Error())
	}
}

func TestReadImageBlob(t *testing.T) {
	mw := NewMagickWand()
	defer mw.Destroy()

	// Read an invalid blob
	blob := []byte{}
	if err := mw.ReadImageBlob(blob); err == nil {
		t.Fatal("Expected a failure when passing a zero length blob")
	}

	mw.ReadImage(`logo:`)
	blob = mw.GetImageBlob()

	// Read a valid blob
	if err := mw.ReadImageBlob(blob); err != nil {
		t.Fatal(err.Error())
	}
}

func TestGetImageFloats(t *testing.T) {
	Initialize()
	mw := NewMagickWand()
	defer mw.Destroy()

	var err error
	if err = mw.ReadImage(`logo:`); err != nil {
		t.Fatal("Failed to read internal logo: image")
	}

	width, height := mw.GetImageWidth(), mw.GetImageHeight()

	val, err := mw.ExportImagePixels(0, 0, width, height, "RGB", PIXEL_FLOAT)
	if err != nil {
		t.Fatal(err.Error())
	}
	pixels := val.([]float32)
	actual := len(pixels)
	expected := (width * height * 3)
	if actual != int(expected) {
		t.Fatalf("Expected RGB image to have %d float vals; Got %d", expected, actual)
	}

	val, err = mw.ExportImagePixels(0, 0, width, height, "RGBA", PIXEL_DOUBLE)
	if err != nil {
		t.Fatal(err.Error())
	}
	pixels64 := val.([]float64)
	actual = len(pixels64)
	expected = (width * height * 4)
	if actual != int(expected) {
		t.Fatalf("Expected RGBA image to have %d float vals; Got %d", expected, actual)
	}

	val, err = mw.ExportImagePixels(0, 0, width, height, "R", PIXEL_FLOAT)
	if err != nil {
		t.Fatal(err.Error())
	}
	pixels = val.([]float32)
	actual = len(pixels)
	expected = (width * height * 1)
	if actual != int(expected) {
		t.Fatalf("Expected RNN image to have %d float vals; Got %d", expected, actual)
	}

	val, err = mw.ExportImagePixels(0, 0, width, height, "GB", PIXEL_FLOAT)
	if err != nil {
		t.Fatal(err.Error())
	}
	pixels = val.([]float32)
	actual = len(pixels)
	expected = (width * height * 2)
	if actual != int(expected) {
		t.Fatalf("Expected NGB image to have %d float vals; Got %d", expected, actual)
	}
}

func TestGetQuantumDepth(t *testing.T) {
	name, depth := GetQuantumDepth()
	if name == "" {
		t.Fatal("Depth name returned was an empty string")
	}
	if depth == 0 {
		t.Fatal("Depth value returned was 0")
	}
}

func TestGetQuantumRange(t *testing.T) {
	name, r := GetQuantumRange()
	if name == "" {
		t.Fatal("Depth name returned was an empty string")
	}
	if r == 0 {
		t.Fatal("Range value returned was 0")
	}
}

func BenchmarkExportImagePixels(b *testing.B) {
	wand := NewMagickWand()
	defer wand.Destroy()

	wand.ReadImage("logo:")
	wand.ScaleImage(1024, 1024)

	var val interface{}
	var pixels []float32

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		val, _ = wand.ExportImagePixels(0, 0, 1024, 1024, "RGB", PIXEL_FLOAT)
		pixels = val.([]float32)
	}

	b.StopTimer()

	if len(pixels) == 0 {
		b.Fatal("Pixel slice is 0")
	}
}

func BenchmarkImportImagePixels(b *testing.B) {
	wand := NewMagickWand()
	defer wand.Destroy()

	wand.ReadImage("logo:")
	wand.ScaleImage(1024, 1024)

	val, _ := wand.ExportImagePixels(0, 0, 1024, 1024, "RGB", PIXEL_FLOAT)
	pixels := val.([]float32)

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		wand.ImportImagePixels(0, 0, 1024, 1024, "RGB", PIXEL_UNDEFINED, pixels)
	}

	b.StopTimer()
}

type testPixelInterfaceValues struct {
	Pixels  interface{}
	Storage StorageType
}

func TestPixelInterfaceToPtr(t *testing.T) {
	Tests := make([]testPixelInterfaceValues, 6)
	Tests[0].Pixels = []byte{0}
	Tests[0].Storage = PIXEL_CHAR
	Tests[1].Pixels = []float64{0}
	Tests[1].Storage = PIXEL_DOUBLE
	Tests[2].Pixels = []float32{0}
	Tests[2].Storage = PIXEL_FLOAT
	Tests[3].Pixels = []int16{0}
	Tests[3].Storage = PIXEL_SHORT
	Tests[4].Pixels = []int32{0}
	Tests[4].Storage = PIXEL_INTEGER
	Tests[5].Pixels = []int64{0}
	Tests[5].Storage = PIXEL_LONG
	for _, value := range Tests {
		_, storageType, err := pixelInterfaceToPtr(value.Pixels)
		if err != nil {
			t.Fatal("Error when passing", reflect.TypeOf(value.Pixels))
		}
		if storageType != value.Storage {
			t.Fatal("Wrong storage type received for", reflect.TypeOf(value.Pixels))
		}
	}

	_, _, err := pixelInterfaceToPtr(32)
	if err == nil {
		t.Fatal("Expected error when passing invalid type")
	}
}
<commit_msg>make the tests more idiomatic go<commit_after>// Copyright 2013 Herbert G. Fischer. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package imagick

import (
	"reflect"
	"testing"
)

var (
	mw *MagickWand
)

func Init() {
	Initialize()
}

func TestNewMagickWand(t *testing.T) {
	mw := NewMagickWand()
	defer mw.Destroy()

	if !mw.IsVerified() {
		t.Fatal("MagickWand not verified")
	}
}

func TestCloningAndDestroying(t *testing.T) {
	mw := NewMagickWand()
	defer mw.Destroy()
	clone := mw.Clone()
	if !clone.IsVerified() {
		t.Fatal("Unsuccessful clone")
	}
	clone.Destroy()
	if clone.IsVerified() || !mw.IsVerified() {
		t.Fatal("MagickWand not properly destroyed")
	}
}

func TestQueryConfigureOptions(t *testing.T) {
	opts := mw.QueryConfigureOptions("*")
	if len(opts) == 0 {
		t.Fatal("QueryConfigureOptions returned an empty array")
	}
	for _, opt := range opts {
		mw.QueryConfigureOption(opt)
	}
}

func TestNonExistingConfigureOption(t *testing.T) {
	_, err := mw.QueryConfigureOption("4321foobaramps1234")
	if err == nil {
		t.Fatal("Missing error when trying to get non-existing configure option")
	}
}

func TestQueryFonts(t *testing.T) {
	fonts := mw.QueryFonts("*")
	if len(fonts) == 0 {
		t.Fatal("ImageMagick have not identified a single font in this system")
	}
}

func TestQueryFormats(t *testing.T) {
	formats := mw.QueryFormats("*")
	if len(formats) == 0 {
		t.Fatal("ImageMagick have not identified a single image format in this system")
	}
}

func TestDeleteImageArtifact(t *testing.T) {
	mw := NewMagickWand()
	defer mw.Destroy()

	mw.ReadImage(`logo:`)

	if err := mw.DeleteImageArtifact("*"); err != nil {
		t.Fatalf("Error calling DeleteImageArtifact: %s", err.Error())
	}
}

func TestReadImageBlob(t *testing.T) {
	mw := NewMagickWand()
	defer mw.Destroy()

	// Read an invalid blob
	blob := []byte{}
	if err := mw.ReadImageBlob(blob); err == nil {
		t.Fatal("Expected a failure when passing a zero length blob")
	}

	mw.ReadImage(`logo:`)
	blob = mw.GetImageBlob()

	// Read a valid blob
	if err := mw.ReadImageBlob(blob); err != nil {
		t.Fatal(err.Error())
	}
}

func TestGetImageFloats(t *testing.T) {
	Initialize()
	mw := NewMagickWand()
	defer mw.Destroy()

	var err error
	if err = mw.ReadImage(`logo:`); err != nil {
		t.Fatal("Failed to read internal logo: image")
	}

	width, height := mw.GetImageWidth(), mw.GetImageHeight()

	val, err := mw.ExportImagePixels(0, 0, width, height, "RGB", PIXEL_FLOAT)
	if err != nil {
		t.Fatal(err.Error())
	}
	pixels := val.([]float32)
	actual := len(pixels)
	expected := (width * height * 3)
	if actual != int(expected) {
		t.Fatalf("Expected RGB image to have %d float vals; Got %d", expected, actual)
	}

	val, err = mw.ExportImagePixels(0, 0, width, height, "RGBA", PIXEL_DOUBLE)
	if err != nil {
		t.Fatal(err.Error())
	}
	pixels64 := val.([]float64)
	actual = len(pixels64)
	expected = (width * height * 4)
	if actual != int(expected) {
		t.Fatalf("Expected RGBA image to have %d float vals; Got %d", expected, actual)
	}

	val, err = mw.ExportImagePixels(0, 0, width, height, "R", PIXEL_FLOAT)
	if err != nil {
		t.Fatal(err.Error())
	}
	pixels = val.([]float32)
	actual = len(pixels)
	expected = (width * height * 1)
	if actual != int(expected) {
		t.Fatalf("Expected RNN image to have %d float vals; Got %d", expected, actual)
	}

	val, err = mw.ExportImagePixels(0, 0, width, height, "GB", PIXEL_FLOAT)
	if err != nil {
		t.Fatal(err.Error())
	}
	pixels = val.([]float32)
	actual = len(pixels)
	expected = (width * height * 2)
	if actual != int(expected) {
		t.Fatalf("Expected NGB image to have %d float vals; Got %d", expected, actual)
	}
}

func TestGetQuantumDepth(t *testing.T) {
	name, depth := GetQuantumDepth()
	if name == "" {
		t.Fatal("Depth name returned was an empty string")
	}
	if depth == 0 {
		t.Fatal("Depth value returned was 0")
	}
}

func TestGetQuantumRange(t *testing.T) {
	name, r := GetQuantumRange()
	if name == "" {
		t.Fatal("Depth name returned was an empty string")
	}
	if r == 0 {
		t.Fatal("Range value returned was 0")
	}
}

func BenchmarkExportImagePixels(b *testing.B) {
	wand := NewMagickWand()
	defer wand.Destroy()

	wand.ReadImage("logo:")
	wand.ScaleImage(1024, 1024)

	var val interface{}
	var pixels []float32

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		val, _ = wand.ExportImagePixels(0, 0, 1024, 1024, "RGB", PIXEL_FLOAT)
		pixels = val.([]float32)
	}

	b.StopTimer()

	if len(pixels) == 0 {
		b.Fatal("Pixel slice is 0")
	}
}

func BenchmarkImportImagePixels(b *testing.B) {
	wand := NewMagickWand()
	defer wand.Destroy()

	wand.ReadImage("logo:")
	wand.ScaleImage(1024, 1024)

	val, _ := wand.ExportImagePixels(0, 0, 1024, 1024, "RGB", PIXEL_FLOAT)
	pixels := val.([]float32)

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		wand.ImportImagePixels(0, 0, 1024, 1024, "RGB", PIXEL_UNDEFINED, pixels)
	}

	b.StopTimer()
}

func TestPixelInterfaceToPtr(t *testing.T) {
	tests := []struct {
		pixels  interface{}
		storage StorageType
	}{
		{[]byte{0}, PIXEL_CHAR},
		{[]float64{0}, PIXEL_DOUBLE},
		{[]float32{0}, PIXEL_FLOAT},
		{[]int16{0}, PIXEL_SHORT},
		{[]int32{0}, PIXEL_INTEGER},
		{[]int64{0}, PIXEL_LONG},
	}
	for _, value := range tests {
		_, storageType, err := pixelInterfaceToPtr(value.pixels)
		if err != nil {
			t.Fatal("Error when passing", reflect.TypeOf(value.pixels))
		}
		if storageType != value.storage {
			t.Fatal("Wrong storage type received for", reflect.TypeOf(value.pixels))
		}
	}

	_, _, err := pixelInterfaceToPtr(32)
	if err == nil {
		t.Fatal("Expected error when passing invalid type")
	}
}
<|endoftext|>


<commit_before>package termite

// This file tests both rpcfs and fsserver by having them talk over a
// socketpair.

import (
	"crypto"
	"github.com/hanwen/go-fuse/fuse"
	"github.com/hanwen/termite/attr"
	"github.com/hanwen/termite/cba"
	"io"
	"io/ioutil"
	"log"
	"net/rpc"
	"os"
	"path/filepath"
	"testing"
	"time"
)

// UGH - copy & paste.
// for tests:
func StatForTest(t *testing.T, n string) *fuse.Attr {
	t.Logf("test stat %q", n)
	f, _ := os.Lstat(n)
	if f == nil {
		return nil
	}
	a := fuse.Attr{}
	a.FromFileInfo(f)
	return &a
}

func GetattrForTest(t *testing.T, n string) *attr.FileAttr {
	t.Logf("test getattr %q", n)
	fi, _ := os.Lstat(n)

	var fa *fuse.Attr
	if fi != nil {
		fa = &fuse.Attr{}
		fa.FromFileInfo(fi)
	}
	a := attr.FileAttr{
		Attr: fa,
	}
	if !a.Deletion() {
		a.ReadFromFs(n, crypto.MD5)
	}
	return &a
}

// TODO - fix this test.
func DisabledTestRpcFsFetchOnce(t *testing.T) {
	me := newRpcFsTestCase(t)
	defer me.Clean()
	ioutil.WriteFile(me.orig+"/file.txt", []byte{42}, 0644)
	me.attr.Refresh("")

	ioutil.ReadFile(me.mnt + "/file.txt")

	stats := me.server.stats.Timings()
	key := "FsServer.FileContent"
	if stats == nil || stats[key] == nil {
		t.Fatalf("Stats %q missing: %v", key, stats)
	}
	if stats[key].N > 1 {
		t.Errorf("File content was served more than once.")
	}
}

func TestFsServerCache(t *testing.T) {
	me := newRpcFsTestCase(t)
	defer me.Clean()

	content := "hello"
	err := ioutil.WriteFile(me.orig+"/file.txt", []byte(content), 0644)
	me.attr.Refresh("")
	c := me.attr.Copy().Files
	if len(c) > 0 {
		t.Errorf("cache not empty? %#v", c)
	}

	os.Lstat(me.mnt + "/file.txt")
	c = me.attr.Copy().Files
	if len(c) != 2 {
		t.Errorf("cache should have 2 entries, got %#v", c)
	}
	name := "file.txt"
	ok := me.server.attributes.Have(name)
	if !ok {
		t.Errorf("no entry for %q", name)
	}

	newName := me.orig + "/new.txt"
	err = os.Rename(me.orig+"/file.txt", newName)
	if err != nil {
		t.Fatal(err)
	}

	me.attr.Refresh("")
	ok = me.server.attributes.Have(name)
	if ok {
		t.Errorf("after rename: entry for %q unexpected", name)
	}
}

type rpcFsTestCase struct {
	tmp  string
	mnt  string
	orig string

	cache  *cba.Store
	attr   *attr.AttributeCache
	server *FsServer
	rpcFs  *RpcFs
	state  *fuse.MountState

	sockL, sockR       io.ReadWriteCloser
	contentL, contentR io.ReadWriteCloser

	tester *testing.T
}

func (me *rpcFsTestCase) getattr(n string) *attr.FileAttr {
	p := filepath.Join(me.orig, n)
	a := GetattrForTest(me.tester, p)
	if a.Hash != "" {
		me.cache.SavePath(p)
	}
	return a
}

func newRpcFsTestCase(t *testing.T) (me *rpcFsTestCase) {
	me = &rpcFsTestCase{tester: t}
	me.tmp, _ = ioutil.TempDir("", "term-fss")

	me.mnt = me.tmp + "/mnt"
	me.orig = me.tmp + "/orig"
	srvCache := me.tmp + "/server-cache"

	os.Mkdir(me.mnt, 0700)
	os.Mkdir(me.orig, 0700)

	copts := cba.StoreOptions{
		Dir: srvCache,
	}
	me.cache = cba.NewStore(&copts)
	me.attr = attr.NewAttributeCache(
		func(n string) *attr.FileAttr { return me.getattr(n) },
		func(n string) *fuse.Attr {
			return StatForTest(t, filepath.Join(me.orig, n))
		})
	me.attr.Paranoia = true
	me.server = NewFsServer(me.attr, me.cache)

	var err error
	me.sockL, me.sockR, err = unixSocketpair()
	if err != nil {
		t.Fatal(err)
	}
	me.contentL, me.contentR, err = unixSocketpair()
	if err != nil {
		t.Fatal(err)
	}

	rpcServer := rpc.NewServer()
	rpcServer.Register(me.server)
	go rpcServer.ServeConn(me.sockL)
	go me.cache.ServeConn(me.contentL)
	rpcClient := rpc.NewClient(me.sockR)
	cOpts := cba.StoreOptions{
		Dir: me.tmp + "/client-cache",
	}
	clientCache := cba.NewStore(&cOpts)
	me.rpcFs = NewRpcFs(rpcClient, clientCache, me.contentR)
	me.rpcFs.id = "rpcfs_test"
	nfs := fuse.NewPathNodeFs(me.rpcFs, nil)
	me.state, _, err = fuse.MountNodeFileSystem(me.mnt, nfs, nil)
	me.state.Debug = fuse.VerboseTest()
	if err != nil {
		t.Fatal("Mount", err)
	}

	go me.state.Loop()
	return me
}

func (me *rpcFsTestCase) Clean() {
	if err := me.state.Unmount(); err != nil {
		log.Panic("fuse unmount failed.", err)
	}
	os.RemoveAll(me.tmp)
	me.sockL.Close()
	me.sockR.Close()
}

func check(err error) {
	if err != nil {
		panic(err)
	}
}

func TestRpcFsReadDirCache(t *testing.T) {
	me := newRpcFsTestCase(t)
	defer me.Clean()

	os.Mkdir(me.orig+"/subdir", 0700)
	content := "hello"
	err := ioutil.WriteFile(me.orig+"/subdir/file.txt", []byte(content), 0644)
	check(err)

	entries, err := ioutil.ReadDir(me.mnt + "/subdir")
	check(err)

	seen := false
	for _, v := range entries {
		if v.Name() == "file.txt" {
			seen = true
		}
	}

	if !seen {
		t.Fatalf("Missing entry %q %v", "file.txt", entries)
	}

	before, _ := os.Lstat(me.orig + "/subdir")
	for {
		err = ioutil.WriteFile(me.orig+"/subdir/unstatted.txt", []byte("somethingelse"), 0644)
		check(err)
		after, _ := os.Lstat(me.orig + "/subdir")
		if !before.ModTime().Equal(after.ModTime()) {
			break
		}
		time.Sleep(10e6)
		os.Remove(me.orig + "/subdir/unstatted.txt")
	}

	err = os.Remove(me.orig + "/subdir/file.txt")
	check(err)

	fset := me.attr.Refresh("")
	me.rpcFs.updateFiles(fset.Files)

	_, err = ioutil.ReadDir(me.mnt + "/subdir")
	check(err)

	dir := me.rpcFs.attr.GetDir("subdir")
	if dir == nil {
		t.Fatalf("Should have cache entry for /subdir")
	}

	if _, ok := dir.NameModeMap["file.txt"]; ok {
		t.Errorf("file.txt should have disappeared: %v", dir.NameModeMap)
	}
	if _, ok := dir.NameModeMap["unstatted.txt"]; !ok {
		t.Errorf("unstatted.txt should have appeared: %v", dir.NameModeMap)
	}
}

func TestRpcFsBasic(t *testing.T) {
	me := newRpcFsTestCase(t)
	defer me.Clean()

	os.Mkdir(me.orig+"/subdir", 0700)
	content := "hello"
	err := ioutil.WriteFile(me.orig+"/file.txt", []byte(content), 0644)
	if err != nil {
		t.Fatal(err)
	}

	fi, err := os.Lstat(me.mnt + "/subdir")
	if fi == nil || !fi.IsDir() {
		t.Fatal("subdir stat", fi, err)
	}

	c, err := ioutil.ReadFile(me.mnt + "/file.txt")
	if err != nil || string(c) != "hello" {
		t.Errorf("Readfile: want 'hello', got '%s', err %v", c, err)
	}

	entries, err := ioutil.ReadDir(me.mnt)
	if err != nil || len(entries) != 2 {
		t.Error("Readdir", err, entries)
	}

	// This test implementation detail - should be separate?
	a := me.server.attributes.Get("file.txt")
	if a == nil || a.Hash == "" || string(a.Hash) != string(md5str(content)) {
		t.Errorf("cache error %v (%x)", a)
	}

	newcontent := "somethingelse"
	err = ioutil.WriteFile(me.orig+"/file.txt", []byte(newcontent), 0644)
	check(err)
	err = ioutil.WriteFile(me.orig+"/foobar.txt", []byte("more content"), 0644)
	check(err)

	me.attr.Refresh("")
	a = me.server.attributes.Get("file.txt")
	if a == nil || a.Hash == "" || a.Hash != md5str(newcontent) {
		t.Errorf("refreshAttributeCache: cache error got %v, want %x", a, md5str(newcontent))
	}
}
<commit_msg>Reinstate disable FetchOnce test.<commit_after>package termite

// This file tests both rpcfs and fsserver by having them talk over a
// socketpair.

import (
	"crypto"
	"github.com/hanwen/go-fuse/fuse"
	"github.com/hanwen/termite/attr"
	"github.com/hanwen/termite/cba"
	"io"
	"io/ioutil"
	"log"
	"net/rpc"
	"os"
	"path/filepath"
	"testing"
	"time"
)

// UGH - copy & paste.
// for tests:
func StatForTest(t *testing.T, n string) *fuse.Attr {
	t.Logf("test stat %q", n)
	f, _ := os.Lstat(n)
	if f == nil {
		return nil
	}
	a := fuse.Attr{}
	a.FromFileInfo(f)
	return &a
}

func GetattrForTest(t *testing.T, n string) *attr.FileAttr {
	t.Logf("test getattr %q", n)
	fi, _ := os.Lstat(n)

	var fa *fuse.Attr
	if fi != nil {
		fa = &fuse.Attr{}
		fa.FromFileInfo(fi)
	}
	a := attr.FileAttr{
		Attr: fa,
	}
	if !a.Deletion() {
		a.ReadFromFs(n, crypto.MD5)
	}
	return &a
}

func TestRpcFsFetchOnce(t *testing.T) {
	me := newRpcFsTestCase(t)
	defer me.Clean()
	ioutil.WriteFile(me.orig+"/file.txt", []byte{42}, 0644)
	me.attr.Refresh("")

	ioutil.ReadFile(me.mnt + "/file.txt")

	stats := me.serverStore.TimingMap()
	key := "ContentStore.Save"
	if stats == nil || stats[key] == nil {
		t.Fatalf("Stats %q missing: %v", key, stats)
	}
	if stats[key].N > 1 {
		t.Errorf("File content was served more than once.")
	}
}

func TestFsServerCache(t *testing.T) {
	me := newRpcFsTestCase(t)
	defer me.Clean()

	content := "hello"
	err := ioutil.WriteFile(me.orig+"/file.txt", []byte(content), 0644)
	me.attr.Refresh("")
	c := me.attr.Copy().Files
	if len(c) > 0 {
		t.Errorf("cache not empty? %#v", c)
	}

	os.Lstat(me.mnt + "/file.txt")
	c = me.attr.Copy().Files
	if len(c) != 2 {
		t.Errorf("cache should have 2 entries, got %#v", c)
	}
	name := "file.txt"
	ok := me.server.attributes.Have(name)
	if !ok {
		t.Errorf("no entry for %q", name)
	}

	newName := me.orig + "/new.txt"
	err = os.Rename(me.orig+"/file.txt", newName)
	if err != nil {
		t.Fatal(err)
	}

	me.attr.Refresh("")
	ok = me.server.attributes.Have(name)
	if ok {
		t.Errorf("after rename: entry for %q unexpected", name)
	}
}

type rpcFsTestCase struct {
	tmp  string
	mnt  string
	orig string

	serverStore, clientStore  *cba.Store
	attr   *attr.AttributeCache
	server *FsServer
	rpcFs  *RpcFs
	state  *fuse.MountState

	sockL, sockR       io.ReadWriteCloser
	contentL, contentR io.ReadWriteCloser

	tester *testing.T
}

func (me *rpcFsTestCase) getattr(n string) *attr.FileAttr {
	p := filepath.Join(me.orig, n)
	a := GetattrForTest(me.tester, p)
	if a.Hash != "" {
		me.serverStore.SavePath(p)
	}
	return a
}

func newRpcFsTestCase(t *testing.T) (me *rpcFsTestCase) {
	me = &rpcFsTestCase{tester: t}
	me.tmp, _ = ioutil.TempDir("", "term-fss")

	me.mnt = me.tmp + "/mnt"
	me.orig = me.tmp + "/orig"
	srvCache := me.tmp + "/server-cache"

	os.Mkdir(me.mnt, 0700)
	os.Mkdir(me.orig, 0700)

	copts := cba.StoreOptions{
		Dir: srvCache,
	}
	me.serverStore = cba.NewStore(&copts)
	me.attr = attr.NewAttributeCache(
		func(n string) *attr.FileAttr { return me.getattr(n) },
		func(n string) *fuse.Attr {
			return StatForTest(t, filepath.Join(me.orig, n))
		})
	me.attr.Paranoia = true
	me.server = NewFsServer(me.attr, me.serverStore)

	var err error
	me.sockL, me.sockR, err = unixSocketpair()
	if err != nil {
		t.Fatal(err)
	}
	me.contentL, me.contentR, err = unixSocketpair()
	if err != nil {
		t.Fatal(err)
	}

	rpcServer := rpc.NewServer()
	rpcServer.Register(me.server)
	go rpcServer.ServeConn(me.sockL)
	go me.serverStore.ServeConn(me.contentL)
	rpcClient := rpc.NewClient(me.sockR)
	cOpts := cba.StoreOptions{
		Dir: me.tmp + "/client-cache",
	}
	me.clientStore = cba.NewStore(&cOpts)
	me.rpcFs = NewRpcFs(rpcClient, me.clientStore, me.contentR)
	me.rpcFs.id = "rpcfs_test"
	nfs := fuse.NewPathNodeFs(me.rpcFs, nil)
	me.state, _, err = fuse.MountNodeFileSystem(me.mnt, nfs, nil)
	me.state.Debug = fuse.VerboseTest()
	if err != nil {
		t.Fatal("Mount", err)
	}

	go me.state.Loop()
	return me
}

func (me *rpcFsTestCase) Clean() {
	if err := me.state.Unmount(); err != nil {
		log.Panic("fuse unmount failed.", err)
	}
	os.RemoveAll(me.tmp)
	me.sockL.Close()
	me.sockR.Close()
}

func check(err error) {
	if err != nil {
		panic(err)
	}
}

func TestRpcFsReadDirCache(t *testing.T) {
	me := newRpcFsTestCase(t)
	defer me.Clean()

	os.Mkdir(me.orig+"/subdir", 0700)
	content := "hello"
	err := ioutil.WriteFile(me.orig+"/subdir/file.txt", []byte(content), 0644)
	check(err)

	entries, err := ioutil.ReadDir(me.mnt + "/subdir")
	check(err)

	seen := false
	for _, v := range entries {
		if v.Name() == "file.txt" {
			seen = true
		}
	}

	if !seen {
		t.Fatalf("Missing entry %q %v", "file.txt", entries)
	}

	before, _ := os.Lstat(me.orig + "/subdir")
	for {
		err = ioutil.WriteFile(me.orig+"/subdir/unstatted.txt", []byte("somethingelse"), 0644)
		check(err)
		after, _ := os.Lstat(me.orig + "/subdir")
		if !before.ModTime().Equal(after.ModTime()) {
			break
		}
		time.Sleep(10e6)
		os.Remove(me.orig + "/subdir/unstatted.txt")
	}

	err = os.Remove(me.orig + "/subdir/file.txt")
	check(err)

	fset := me.attr.Refresh("")
	me.rpcFs.updateFiles(fset.Files)

	_, err = ioutil.ReadDir(me.mnt + "/subdir")
	check(err)

	dir := me.rpcFs.attr.GetDir("subdir")
	if dir == nil {
		t.Fatalf("Should have cache entry for /subdir")
	}

	if _, ok := dir.NameModeMap["file.txt"]; ok {
		t.Errorf("file.txt should have disappeared: %v", dir.NameModeMap)
	}
	if _, ok := dir.NameModeMap["unstatted.txt"]; !ok {
		t.Errorf("unstatted.txt should have appeared: %v", dir.NameModeMap)
	}
}

func TestRpcFsBasic(t *testing.T) {
	me := newRpcFsTestCase(t)
	defer me.Clean()

	os.Mkdir(me.orig+"/subdir", 0700)
	content := "hello"
	err := ioutil.WriteFile(me.orig+"/file.txt", []byte(content), 0644)
	if err != nil {
		t.Fatal(err)
	}

	fi, err := os.Lstat(me.mnt + "/subdir")
	if fi == nil || !fi.IsDir() {
		t.Fatal("subdir stat", fi, err)
	}

	c, err := ioutil.ReadFile(me.mnt + "/file.txt")
	if err != nil || string(c) != "hello" {
		t.Errorf("Readfile: want 'hello', got '%s', err %v", c, err)
	}

	entries, err := ioutil.ReadDir(me.mnt)
	if err != nil || len(entries) != 2 {
		t.Error("Readdir", err, entries)
	}

	// This test implementation detail - should be separate?
	a := me.server.attributes.Get("file.txt")
	if a == nil || a.Hash == "" || string(a.Hash) != string(md5str(content)) {
		t.Errorf("cache error %v (%x)", a)
	}

	newcontent := "somethingelse"
	err = ioutil.WriteFile(me.orig+"/file.txt", []byte(newcontent), 0644)
	check(err)
	err = ioutil.WriteFile(me.orig+"/foobar.txt", []byte("more content"), 0644)
	check(err)

	me.attr.Refresh("")
	a = me.server.attributes.Get("file.txt")
	if a == nil || a.Hash == "" || a.Hash != md5str(newcontent) {
		t.Errorf("refreshAttributeCache: cache error got %v, want %x", a, md5str(newcontent))
	}
}
<|endoftext|>