<commit_before>package gocassa

import (
	"testing"

	//"github.com/stretchr/testify/require"
	"github.com/stretchr/testify/suite"
)

type user struct {
	Pk1  int
	Pk2  int
	Ck1  int
	Ck2  int
	Name string
}

func TestRunMockSuite(t *testing.T) {
	suite.Run(t, new(MockSuite))
}

type MockSuite struct {
	suite.Suite
	//*require.Assertions
	tbl Table
}

func (s *MockSuite) SetupTest() {
	ks := NewMockKeySpace()
	//s.Assertions = require.New(s.T())
	s.tbl = ks.Table("users", user{}, Keys{
		PartitionKeys:     []string{"Pk1", "Pk2"},
		ClusteringColumns: []string{"Ck1", "Ck2"},
	})
}

func (s *MockSuite) TestEmpty() {
	var result []user
	s.NoError(s.tbl.Where(Eq("Pk1", 1), Eq("Pk2", 1), Eq("Ck1", 1), Eq("Ck2", 1)).Query().Read(&result))
	s.Equal(0, len(result))
}

func (s *MockSuite) TestRead() {
	u1, u2, u3, u4 := s.insertUsers()

	var users []user
	s.NoError(s.tbl.Where(Eq("Pk1", 1), Eq("Pk2", 1)).Query().Read(&users))
	s.Equal([]user{u1, u3, u4}, users)

	s.NoError(s.tbl.Where(Eq("Pk1", 1), Eq("Pk2", 2)).Query().Read(&users))
	s.Equal([]user{u2}, users)

	s.NoError(s.tbl.Where(Eq("Pk1", 1), In("Pk2", 1, 2)).Query().Read(&users))
	s.Equal([]user{u1, u3, u4, u2}, users)

	s.NoError(s.tbl.Where(Eq("Pk1", 1), Eq("Pk2", 1), Eq("Ck1", 1)).Query().Read(&users))
	s.Equal([]user{u1, u4}, users)

	s.NoError(s.tbl.Where(Eq("Pk1", 1), Eq("Pk2", 1), Eq("Ck1", 1), Eq("Ck2", 1)).Query().Read(&users))
	s.Equal([]user{u1}, users)

	s.NoError(s.tbl.Where(Eq("Pk1", 1), Eq("Pk2", 1), GT("Ck1", 1)).Query().Read(&users))
	s.Equal([]user{u3}, users)

	s.NoError(s.tbl.Where(Eq("Pk1", 1), Eq("Pk2", 1), Eq("Ck1", 1), LT("Ck2", 2)).Query().Read(&users))
	s.Equal([]user{u1}, users)

	var u user
	s.NoError(s.tbl.Where(Eq("Pk1", 1), Eq("Pk2", 1), Eq("Ck1", 1), Eq("Ck2", 1)).Query().ReadOne(&u))
	s.Equal(u1, u)

	s.NoError(s.tbl.Where(Eq("Pk1", 1), Eq("Pk2", 1), Eq("Ck1", 1), Eq("Ck2", 2)).Query().ReadOne(&u))
	s.Equal(u4, u)
}

func (s *MockSuite) TestUpdate() {
	s.insertUsers()

	relations := []Relation{Eq("Pk1", 1), Eq("Pk2", 1), Eq("Ck1", 1), Eq("Ck2", 2)}

	s.NoError(s.tbl.Where(relations...).Update(map[string]interface{}{
		"Name": "x",
	}))

	var u user
	s.NoError(s.tbl.Where(relations...).Query().ReadOne(&u))
	s.Equal("x", u.Name)

	relations = []Relation{Eq("Pk1", 1), In("Pk2", 1, 2), Eq("Ck1", 1), Eq("Ck2", 1)}

	s.NoError(s.tbl.Where(relations...).Update(map[string]interface{}{
		"Name": "y",
	}))

	var users []user
	s.NoError(s.tbl.Where(relations...).Query().Read(&users))
	for _, u := range users {
		s.Equal("y", u.Name)
	}
}

func (s *MockSuite) TestDeleteOne() {
	s.insertUsers()

	relations := []Relation{Eq("Pk1", 1), Eq("Pk2", 1), Eq("Ck1", 1), Eq("Ck2", 2)}
	s.NoError(s.tbl.Where(relations...).Delete())

	var users []user
	s.NoError(s.tbl.Where(relations...).Query().Read(&users))
	s.Empty(users)
}

func (s *MockSuite) TestDeleteWithIn() {
	s.insertUsers()

	relations := []Relation{Eq("Pk1", 1), In("Pk2", 1, 2), Eq("Ck1", 1), Eq("Ck2", 1)}
	s.NoError(s.tbl.Where(relations...).Delete())

	var users []user
	s.NoError(s.tbl.Where(relations...).Query().Read(&users))
	s.Empty(users)
}

func (s *MockSuite) insertUsers() (user, user, user, user) {
	u1 := user{
		Pk1:  1,
		Pk2:  1,
		Ck1:  1,
		Ck2:  1,
		Name: "John",
	}
	u2 := user{
		Pk1:  1,
		Pk2:  2,
		Ck1:  1,
		Ck2:  1,
		Name: "Joe",
	}
	u3 := user{
		Pk1:  1,
		Pk2:  1,
		Ck1:  2,
		Ck2:  1,
		Name: "Josh",
	}
	u4 := user{
		Pk1:  1,
		Pk2:  1,
		Ck1:  1,
		Ck2:  2,
		Name: "Jane",
	}

	for _, u := range []user{u1, u2, u3, u4} {
		s.NoError(s.tbl.Set(u))
	}

	return u1, u2, u3, u4
}
<commit_msg>Fixing mock tests<commit_after>package gocassa

import (
	"testing"

	//"github.com/stretchr/testify/require"
	"github.com/stretchr/testify/suite"
)

type user struct {
	Pk1  int
	Pk2  int
	Ck1  int
	Ck2  int
	Name string
}

func TestRunMockSuite(t *testing.T) {
	suite.Run(t, new(MockSuite))
}

type MockSuite struct {
	suite.Suite
	//*require.Assertions
	tbl Table
}

func (s *MockSuite) SetupTest() {
	ks := NewMockKeySpace()
	//s.Assertions = require.New(s.T())
	s.tbl = ks.Table("users", user{}, Keys{
		PartitionKeys:     []string{"Pk1", "Pk2"},
		ClusteringColumns: []string{"Ck1", "Ck2"},
	})
}

func (s *MockSuite) TestEmpty() {
	var result []user
	s.NoError(s.tbl.Where(Eq("Pk1", 1), Eq("Pk2", 1), Eq("Ck1", 1), Eq("Ck2", 1)).Query().Read(&result).Run())
	s.Equal(0, len(result))
}

func (s *MockSuite) TestRead() {
	u1, u2, u3, u4 := s.insertUsers()

	var users []user
	s.NoError(s.tbl.Where(Eq("Pk1", 1), Eq("Pk2", 1)).Query().Read(&users).Run())
	s.Equal([]user{u1, u3, u4}, users)

	s.NoError(s.tbl.Where(Eq("Pk1", 1), Eq("Pk2", 2)).Query().Read(&users).Run())
	s.Equal([]user{u2}, users)

	s.NoError(s.tbl.Where(Eq("Pk1", 1), In("Pk2", 1, 2)).Query().Read(&users).Run())
	s.Equal([]user{u1, u3, u4, u2}, users)

	s.NoError(s.tbl.Where(Eq("Pk1", 1), Eq("Pk2", 1), Eq("Ck1", 1)).Query().Read(&users).Run())
	s.Equal([]user{u1, u4}, users)

	s.NoError(s.tbl.Where(Eq("Pk1", 1), Eq("Pk2", 1), Eq("Ck1", 1), Eq("Ck2", 1)).Query().Read(&users).Run())
	s.Equal([]user{u1}, users)

	s.NoError(s.tbl.Where(Eq("Pk1", 1), Eq("Pk2", 1), GT("Ck1", 1)).Query().Read(&users).Run())
	s.Equal([]user{u3}, users)

	s.NoError(s.tbl.Where(Eq("Pk1", 1), Eq("Pk2", 1), Eq("Ck1", 1), LT("Ck2", 2)).Query().Read(&users).Run())
	s.Equal([]user{u1}, users)

	var u user
	s.NoError(s.tbl.Where(Eq("Pk1", 1), Eq("Pk2", 1), Eq("Ck1", 1), Eq("Ck2", 1)).Query().ReadOne(&u).Run())
	s.Equal(u1, u)

	s.NoError(s.tbl.Where(Eq("Pk1", 1), Eq("Pk2", 1), Eq("Ck1", 1), Eq("Ck2", 2)).Query().ReadOne(&u).Run())
	s.Equal(u4, u)
}

func (s *MockSuite) TestUpdate() {
	s.insertUsers()

	relations := []Relation{Eq("Pk1", 1), Eq("Pk2", 1), Eq("Ck1", 1), Eq("Ck2", 2)}

	s.NoError(s.tbl.Where(relations...).Update(map[string]interface{}{
		"Name": "x",
	}).Run())

	var u user
	s.NoError(s.tbl.Where(relations...).Query().ReadOne(&u).Run())
	s.Equal("x", u.Name)

	relations = []Relation{Eq("Pk1", 1), In("Pk2", 1, 2), Eq("Ck1", 1), Eq("Ck2", 1)}

	s.NoError(s.tbl.Where(relations...).Update(map[string]interface{}{
		"Name": "y",
	}).Run())

	var users []user
	s.NoError(s.tbl.Where(relations...).Query().Read(&users).Run())
	for _, u := range users {
		s.Equal("y", u.Name)
	}
}

func (s *MockSuite) TestDeleteOne() {
	s.insertUsers()

	relations := []Relation{Eq("Pk1", 1), Eq("Pk2", 1), Eq("Ck1", 1), Eq("Ck2", 2)}
	s.NoError(s.tbl.Where(relations...).Delete().Run())

	var users []user
	s.NoError(s.tbl.Where(relations...).Query().Read(&users).Run())
	s.Empty(users)
}

func (s *MockSuite) TestDeleteWithIn() {
	s.insertUsers()

	relations := []Relation{Eq("Pk1", 1), In("Pk2", 1, 2), Eq("Ck1", 1), Eq("Ck2", 1)}
	s.NoError(s.tbl.Where(relations...).Delete().Run())

	var users []user
	s.NoError(s.tbl.Where(relations...).Query().Read(&users).Run())
	s.Empty(users)
}

func (s *MockSuite) insertUsers() (user, user, user, user) {
	u1 := user{
		Pk1:  1,
		Pk2:  1,
		Ck1:  1,
		Ck2:  1,
		Name: "John",
	}
	u2 := user{
		Pk1:  1,
		Pk2:  2,
		Ck1:  1,
		Ck2:  1,
		Name: "Joe",
	}
	u3 := user{
		Pk1:  1,
		Pk2:  1,
		Ck1:  2,
		Ck2:  1,
		Name: "Josh",
	}
	u4 := user{
		Pk1:  1,
		Pk2:  1,
		Ck1:  1,
		Ck2:  2,
		Name: "Jane",
	}

	for _, u := range []user{u1, u2, u3, u4} {
		s.NoError(s.tbl.Set(u).Run())
	}

	return u1, u2, u3, u4
}
<|endoftext|>


<commit_before>package lucene49

import (
	"github.com/balzaczyy/golucene/core/codec"
	. "github.com/balzaczyy/golucene/core/index/model"
	"github.com/balzaczyy/golucene/core/store"
	"github.com/balzaczyy/golucene/core/util"
	"github.com/balzaczyy/golucene/core/util/packed"
	"math"
)

// lucene49/Lucene49NormsConsumer.java

const (
	DELTA_COMPRESSED = 0
	TABLE_COMPRESSED = 1
	CONST_COMPRESSED = 2
	UNCOMPRESSED     = 3
)

type NormsConsumer struct {
	data, meta store.IndexOutput
	maxDoc     int
}

func newLucene49NormsConsumer(state *SegmentWriteState,
	dataCodec, dataExtension, metaCodec, metaExtension string) (nc *NormsConsumer, err error) {

	assert(packed.PackedFormat(packed.PACKED_SINGLE_BLOCK).IsSupported(1))
	assert(packed.PackedFormat(packed.PACKED_SINGLE_BLOCK).IsSupported(2))
	assert(packed.PackedFormat(packed.PACKED_SINGLE_BLOCK).IsSupported(4))

	nc = &NormsConsumer{maxDoc: state.SegmentInfo.DocCount()}
	var success = false
	defer func() {
		if !success {
			util.CloseWhileSuppressingError(nc)
		}
	}()

	dataName := util.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, dataExtension)
	if nc.data, err = state.Directory.CreateOutput(dataName, state.Context); err != nil {
		return nil, err
	}
	if err = codec.WriteHeader(nc.data, dataCodec, VERSION_CURRENT); err != nil {
		return nil, err
	}
	metaName := util.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, metaExtension)
	if nc.meta, err = state.Directory.CreateOutput(metaName, state.Context); err != nil {
		return nil, err
	}
	if err = codec.WriteHeader(nc.meta, metaCodec, VERSION_CURRENT); err != nil {
		return nil, err
	}
	success = true
	return nc, nil
}

func (nc *NormsConsumer) AddNumericField(field *FieldInfo,
	iter func() func() (interface{}, bool)) (err error) {

	if err = nc.meta.WriteVInt(field.Number); err != nil {
		return
	}
	minValue, maxValue := int64(math.MaxInt64), int64(math.MinInt64)
	// TODO: more efficient?
	uniqueValues := newNormMap()

	count := int64(0)
	next := iter()
	for {
		nv, ok := next()
		if !ok {
			break
		}
		assert2(nv != nil, "illegal norms data for field %v, got null for value: %v", field.Name, count)
		v := nv.(int64)

		if v < minValue {
			minValue = v
		}
		if v > maxValue {
			maxValue = v
		}

		if uniqueValues != nil && uniqueValues.add(v) && uniqueValues.size > 256 {
			uniqueValues = nil
		}

		count++
	}
	assert2(count == int64(nc.maxDoc),
		"illegal norms data for field %v, expected %v values, got %v",
		field.Name, nc.maxDoc, count)

	if uniqueValues != nil && uniqueValues.size == 1 {
		// 0 bpv
		if err = nc.meta.WriteByte(CONST_COMPRESSED); err != nil {
			return
		}
		if err = nc.meta.WriteLong(minValue); err != nil {
			return
		}
	} else if uniqueValues != nil {
		// small number of unique values; this is the typical case:
		// we only use bpv=1,2,4,8
		format := packed.PackedFormat(packed.PACKED_SINGLE_BLOCK)
		bitsPerValue := packed.BitsRequired(int64(uniqueValues.size) - 1)
		if bitsPerValue == 3 {
			bitsPerValue = 4
		} else if bitsPerValue > 4 {
			bitsPerValue = 8
		}

		if bitsPerValue == 8 && minValue >= 0 && maxValue <= 255 {
			if err = store.Stream(nc.meta).WriteByte(UNCOMPRESSED). // uncompressed []byte
										WriteLong(nc.data.FilePointer()).
										Close(); err != nil {
				return err
			}
			next = iter()
			for {
				nv, ok := next()
				if !ok {
					break
				}
				n := byte(0)
				if nv != nil {
					n = byte(nv.(int64))
				}
				if err = nc.data.WriteByte(byte(n)); err != nil {
					return err
				}
			}
		} else {
			if err = store.Stream(nc.meta).WriteByte(TABLE_COMPRESSED). // table-compressed
											WriteLong(nc.data.FilePointer()).
											Close(); err != nil {
				return err
			}
			if err = nc.data.WriteVInt(packed.VERSION_CURRENT); err != nil {
				return err
			}

			decode := uniqueValues.decodeTable()
			// upgrade to power of two sized array
			size := 1 << uint(bitsPerValue)
			if err = nc.data.WriteVInt(int32(size)); err != nil {
				return err
			}
			for _, v := range decode {
				if err = nc.data.WriteLong(v); err != nil {
					return err
				}
			}
			for i := len(decode); i < size; i++ {
				if err = nc.data.WriteLong(0); err != nil {
					return err
				}
			}

			if err = store.Stream(nc.data).WriteVInt(int32(format.Id())).
				WriteVInt(int32(bitsPerValue)).
				Close(); err != nil {
				return err
			}

			writer := packed.WriterNoHeader(nc.data, format, nc.maxDoc, bitsPerValue, packed.DEFAULT_BUFFER_SIZE)
			next = iter()
			for {
				nv, ok := next()
				if !ok {
					break
				}
				if err = writer.Add(int64(uniqueValues.ord(nv.(int64)))); err != nil {
					return err
				}
			}
			if err = writer.Finish(); err != nil {
				return err
			}
		}
	} else {
		panic("not implemented yet")
	}
	return nil
}

type Longs []int64

func (a Longs) Len() int           { return len(a) }
func (a Longs) Less(i, j int) bool { return a[i] < a[j] }
func (a Longs) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }

func (nc *NormsConsumer) Close() (err error) {
	var success = false
	defer func() {
		if success {
			err = util.Close(nc.data, nc.meta)
		} else {
			util.CloseWhileSuppressingError(nc.data, nc.meta)
		}
	}()

	if nc.meta != nil {
		if err = nc.meta.WriteVInt(-1); err != nil { // write EOF marker
			return
		}
		if err = codec.WriteFooter(nc.meta); err != nil { // write checksum
			return
		}
	}
	if nc.data != nil {
		if err = codec.WriteFooter(nc.data); err != nil { // write checksum
			return
		}
	}
	success = true
	return nil
}

type NormMap struct {
	size int
}

/*
Specialized deduplication of long-ord for norms: 99.99999% of the
time this will be a single-byte range.
*/
func newNormMap() *NormMap {
	panic("niy")
}

/* Adds an item to the mapping. Returns true if actually added. */
func (m *NormMap) add(l int64) bool {
	panic("niy")
}

/* Gets the ordinal for a previously added item. */
func (m *NormMap) ord(l int64) int {
	panic("niy")
}

/* Retrieves the ordinal table for previously added items. */
func (m *NormMap) decodeTable() []int64 {
	panic("niy")
}
<commit_msg>implement newNormMap()<commit_after>package lucene49

import (
	"github.com/balzaczyy/golucene/core/codec"
	. "github.com/balzaczyy/golucene/core/index/model"
	"github.com/balzaczyy/golucene/core/store"
	"github.com/balzaczyy/golucene/core/util"
	"github.com/balzaczyy/golucene/core/util/packed"
	"math"
)

// lucene49/Lucene49NormsConsumer.java

const (
	DELTA_COMPRESSED = 0
	TABLE_COMPRESSED = 1
	CONST_COMPRESSED = 2
	UNCOMPRESSED     = 3
)

type NormsConsumer struct {
	data, meta store.IndexOutput
	maxDoc     int
}

func newLucene49NormsConsumer(state *SegmentWriteState,
	dataCodec, dataExtension, metaCodec, metaExtension string) (nc *NormsConsumer, err error) {

	assert(packed.PackedFormat(packed.PACKED_SINGLE_BLOCK).IsSupported(1))
	assert(packed.PackedFormat(packed.PACKED_SINGLE_BLOCK).IsSupported(2))
	assert(packed.PackedFormat(packed.PACKED_SINGLE_BLOCK).IsSupported(4))

	nc = &NormsConsumer{maxDoc: state.SegmentInfo.DocCount()}
	var success = false
	defer func() {
		if !success {
			util.CloseWhileSuppressingError(nc)
		}
	}()

	dataName := util.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, dataExtension)
	if nc.data, err = state.Directory.CreateOutput(dataName, state.Context); err != nil {
		return nil, err
	}
	if err = codec.WriteHeader(nc.data, dataCodec, VERSION_CURRENT); err != nil {
		return nil, err
	}
	metaName := util.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, metaExtension)
	if nc.meta, err = state.Directory.CreateOutput(metaName, state.Context); err != nil {
		return nil, err
	}
	if err = codec.WriteHeader(nc.meta, metaCodec, VERSION_CURRENT); err != nil {
		return nil, err
	}
	success = true
	return nc, nil
}

func (nc *NormsConsumer) AddNumericField(field *FieldInfo,
	iter func() func() (interface{}, bool)) (err error) {

	if err = nc.meta.WriteVInt(field.Number); err != nil {
		return
	}
	minValue, maxValue := int64(math.MaxInt64), int64(math.MinInt64)
	// TODO: more efficient?
	uniqueValues := newNormMap()

	count := int64(0)
	next := iter()
	for {
		nv, ok := next()
		if !ok {
			break
		}
		assert2(nv != nil, "illegal norms data for field %v, got null for value: %v", field.Name, count)
		v := nv.(int64)

		if v < minValue {
			minValue = v
		}
		if v > maxValue {
			maxValue = v
		}

		if uniqueValues != nil && uniqueValues.add(v) && uniqueValues.size > 256 {
			uniqueValues = nil
		}

		count++
	}
	assert2(count == int64(nc.maxDoc),
		"illegal norms data for field %v, expected %v values, got %v",
		field.Name, nc.maxDoc, count)

	if uniqueValues != nil && uniqueValues.size == 1 {
		// 0 bpv
		if err = nc.meta.WriteByte(CONST_COMPRESSED); err != nil {
			return
		}
		if err = nc.meta.WriteLong(minValue); err != nil {
			return
		}
	} else if uniqueValues != nil {
		// small number of unique values; this is the typical case:
		// we only use bpv=1,2,4,8
		format := packed.PackedFormat(packed.PACKED_SINGLE_BLOCK)
		bitsPerValue := packed.BitsRequired(int64(uniqueValues.size) - 1)
		if bitsPerValue == 3 {
			bitsPerValue = 4
		} else if bitsPerValue > 4 {
			bitsPerValue = 8
		}

		if bitsPerValue == 8 && minValue >= 0 && maxValue <= 255 {
			if err = store.Stream(nc.meta).WriteByte(UNCOMPRESSED). // uncompressed []byte
										WriteLong(nc.data.FilePointer()).
										Close(); err != nil {
				return err
			}
			next = iter()
			for {
				nv, ok := next()
				if !ok {
					break
				}
				n := byte(0)
				if nv != nil {
					n = byte(nv.(int64))
				}
				if err = nc.data.WriteByte(byte(n)); err != nil {
					return err
				}
			}
		} else {
			if err = store.Stream(nc.meta).WriteByte(TABLE_COMPRESSED). // table-compressed
											WriteLong(nc.data.FilePointer()).
											Close(); err != nil {
				return err
			}
			if err = nc.data.WriteVInt(packed.VERSION_CURRENT); err != nil {
				return err
			}

			decode := uniqueValues.decodeTable()
			// upgrade to power of two sized array
			size := 1 << uint(bitsPerValue)
			if err = nc.data.WriteVInt(int32(size)); err != nil {
				return err
			}
			for _, v := range decode {
				if err = nc.data.WriteLong(v); err != nil {
					return err
				}
			}
			for i := len(decode); i < size; i++ {
				if err = nc.data.WriteLong(0); err != nil {
					return err
				}
			}

			if err = store.Stream(nc.data).WriteVInt(int32(format.Id())).
				WriteVInt(int32(bitsPerValue)).
				Close(); err != nil {
				return err
			}

			writer := packed.WriterNoHeader(nc.data, format, nc.maxDoc, bitsPerValue, packed.DEFAULT_BUFFER_SIZE)
			next = iter()
			for {
				nv, ok := next()
				if !ok {
					break
				}
				if err = writer.Add(int64(uniqueValues.ord(nv.(int64)))); err != nil {
					return err
				}
			}
			if err = writer.Finish(); err != nil {
				return err
			}
		}
	} else {
		panic("not implemented yet")
	}
	return nil
}

type Longs []int64

func (a Longs) Len() int           { return len(a) }
func (a Longs) Less(i, j int) bool { return a[i] < a[j] }
func (a Longs) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }

func (nc *NormsConsumer) Close() (err error) {
	var success = false
	defer func() {
		if success {
			err = util.Close(nc.data, nc.meta)
		} else {
			util.CloseWhileSuppressingError(nc.data, nc.meta)
		}
	}()

	if nc.meta != nil {
		if err = nc.meta.WriteVInt(-1); err != nil { // write EOF marker
			return
		}
		if err = codec.WriteFooter(nc.meta); err != nil { // write checksum
			return
		}
	}
	if nc.data != nil {
		if err = codec.WriteFooter(nc.data); err != nil { // write checksum
			return
		}
	}
	success = true
	return nil
}

/*
Specialized deduplication of long-ord for norms: 99.99999% of the
time this will be a single-byte range.
*/
type NormMap struct {
	// we use int16: at most we will add 257 values to this map before its rejected as too big above.
	singleByteRange []int16
	other           map[int64]int16
	size            int
}

func newNormMap() *NormMap {
	ans := &NormMap{
		singleByteRange: make([]int16, 256),
		other:           make(map[int64]int16),
	}
	for i, _ := range ans.singleByteRange {
		ans.singleByteRange[i] = -1
	}
	return ans
}

/* Adds an item to the mapping. Returns true if actually added. */
func (m *NormMap) add(l int64) bool {
	panic("niy")
}

/* Gets the ordinal for a previously added item. */
func (m *NormMap) ord(l int64) int {
	panic("niy")
}

/* Retrieves the ordinal table for previously added items. */
func (m *NormMap) decodeTable() []int64 {
	panic("niy")
}
<|endoftext|>


<commit_before>package statsd

import (
	"time"
)

// Satisfies the StatsReporter interface to make testing easier.
type MockStatsdClient struct {
	Counts map[string]float64
	Gauges map[string]float64
}

func (c *MockStatsdClient) Flush() error {
	return nil
}

func (c *MockStatsdClient) Count(bucket string, value, sampleRate float64) {
	c.Counts[bucket] = value
}

func (c *MockStatsdClient) Gauge(bucket string, value float64) {
	c.Gauges[bucket] = value
}

func (c *MockStatsdClient) Timing(bucket string, value time.Duration) {
}

func (c *MockStatsdClient) CountUnique(bucket, value string) {
}
<commit_msg>Use strings instead of float64 in testing subpackage.<commit_after>package statsd

import (
	"strconv"
	"time"
)

// Satisfies the StatsReporter interface to make testing easier.
type MockStatsdClient struct {
	Counts map[string]string
	Gauges map[string]string
}

func (c *MockStatsdClient) Flush() error {
	return nil
}

func (c *MockStatsdClient) Count(bucket string, value, sampleRate float64) {
	valueString := strconv.FormatFloat(value, 'f', -1, 64)
	c.Counts[bucket] = valueString
}

func (c *MockStatsdClient) Gauge(bucket string, value float64) {
	valueString := strconv.FormatFloat(value, 'f', -1, 64)
	c.Gauges[bucket] = valueString
}

func (c *MockStatsdClient) Timing(bucket string, value time.Duration) {
}

func (c *MockStatsdClient) CountUnique(bucket, value string) {
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/base64"
	"encoding/json"
	"flag"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"

	"golang.org/x/crypto/ssh"
)

type source int

const (
	client source = iota
	server
)

func (src source) String() string {
	switch src {
	case client:
		return "client"
	case server:
		return "server"
	default:
		return "unknown"
	}
}

func (src source) MarshalJSON() ([]byte, error) {
	return json.Marshal(src.String())
}

type channelLog struct {
	ChannelID int `json:"channel_id"`
}

type requestLog struct {
	Type      string `json:"type"`
	WantReply bool   `json:"want_reply"`
	Payload   string `json:"payload"`

	Accepted bool `json:"accepted"`
}

type logEntry interface {
	eventType() string
}

type globalRequestLog struct {
	requestLog

	Response string `json:"response"`
}

func (entry globalRequestLog) eventType() string {
	return "global_request"
}

type newChannelLog struct {
	Type      string `json:"type"`
	ExtraData string `json:"extra_data"`

	Accepted     bool   `json:"accepted"`
	RejectReason uint32 `json:"reject_reason"`
	Message      string `json:"message"`
}

func (entry newChannelLog) eventType() string {
	return "new_channel"
}

type channelRequestLog struct {
	channelLog
	requestLog
}

func (entry channelRequestLog) eventType() string {
	return "channel_request"
}

type channelDataLog struct {
	channelLog
	Data string `json:"data"`
}

func (entry channelDataLog) eventType() string {
	return "channel_data"
}

type channelErrorLog struct {
	channelLog
	Data string `json:"data"`
}

func (entry channelErrorLog) eventType() string {
	return "channel_error"
}

type channelEOFLog struct {
	channelLog
}

func (entry channelEOFLog) eventType() string {
	return "channel_eof"
}

type channelCloseLog struct {
	channelLog
}

func (entry channelCloseLog) eventType() string {
	return "channel_close"
}

type connectionCloseLog struct{}

func (entry connectionCloseLog) eventType() string {
	return "connection_close"
}

func logEvent(entry logEntry, src source) {
	jsonBytes, err := json.Marshal(struct {
		Source    string   `json:"source"`
		EventType string   `json:"event_type"`
		Event     logEntry `json:"event"`
	}{
		Source:    src.String(),
		EventType: entry.eventType(),
		Event:     entry,
	})
	if err != nil {
		panic(err)
	}
	log.Printf("%s", jsonBytes)
}

func streamReader(reader io.Reader) <-chan string {
	input := make(chan string)
	go func() {
		defer close(input)
		buffer := make([]byte, 256)
		for {
			n, err := reader.Read(buffer)
			if n > 0 {
				input <- string(buffer[:n])
			}
			if err != nil {
				if err != io.EOF {
					panic(err)
				}
				return
			}
		}
	}()
	return input
}

func handleChannel(channelID int, clientChannel ssh.Channel, clientRequests <-chan *ssh.Request, serverChannel ssh.Channel, serverRequests <-chan *ssh.Request) {
	clientInputStream := streamReader(clientChannel)
	serverInputStream := streamReader(serverChannel)
	serverErrorStream := streamReader(serverChannel.Stderr())

	for clientInputStream != nil || clientRequests != nil || serverInputStream != nil || serverRequests != nil {
		select {
		case clientInput, ok := <-clientInputStream:
			if !ok {
				if serverInputStream != nil {
					logEvent(channelEOFLog{
						channelLog: channelLog{
							ChannelID: channelID,
						},
					}, client)
					if err := serverChannel.CloseWrite(); err != nil {
						panic(err)
					}
				}
				clientInputStream = nil
				continue
			}
			logEvent(channelDataLog{
				channelLog: channelLog{
					ChannelID: channelID,
				},
				Data: clientInput,
			}, client)
			if _, err := serverChannel.Write([]byte(clientInput)); err != nil {
				panic(err)
			}
		case clientRequest, ok := <-clientRequests:
			if !ok {
				if clientInputStream != nil && serverInputStream != nil {
					continue
				}
				if serverRequests != nil {
					logEvent(channelCloseLog{
						channelLog: channelLog{
							ChannelID: channelID,
						},
					}, client)
					if err := serverChannel.Close(); err != nil {
						panic(err)
					}
				}
				clientRequests = nil
				continue
			}
			accepted, err := serverChannel.SendRequest(clientRequest.Type, clientRequest.WantReply, clientRequest.Payload)
			if err != nil {
				panic(err)
			}
			logEvent(channelRequestLog{
				channelLog: channelLog{
					ChannelID: channelID,
				},
				requestLog: requestLog{
					Type:      clientRequest.Type,
					WantReply: clientRequest.WantReply,
					Payload:   base64.RawStdEncoding.EncodeToString(clientRequest.Payload),
					Accepted:  accepted,
				},
			}, client)
			if clientRequest.WantReply {
				if err := clientRequest.Reply(accepted, nil); err != nil {
					panic(err)
				}
			}
		case serverInput, ok := <-serverInputStream:
			if !ok {
				if clientInputStream != nil {
					logEvent(channelEOFLog{
						channelLog: channelLog{
							ChannelID: channelID,
						},
					}, server)
					if err := clientChannel.CloseWrite(); err != nil {
						panic(err)
					}
				}
				serverInputStream = nil
				continue
			}
			logEvent(channelDataLog{
				channelLog: channelLog{
					ChannelID: channelID,
				},
				Data: serverInput,
			}, server)
			if _, err := clientChannel.Write([]byte(serverInput)); err != nil {
				panic(err)
			}
		case serverError, ok := <-serverErrorStream:
			if !ok {
				serverErrorStream = nil
				continue
			}
			logEvent(channelErrorLog{
				channelLog: channelLog{
					ChannelID: channelID,
				},
				Data: serverError,
			}, server)
			if _, err := clientChannel.Stderr().Write([]byte(serverError)); err != nil {
				panic(err)
			}
		case serverRequest, ok := <-serverRequests:
			if !ok {
				if clientInputStream != nil && serverInputStream != nil {
					continue
				}
				if clientRequests != nil {
					logEvent(channelCloseLog{
						channelLog: channelLog{
							ChannelID: channelID,
						},
					}, server)
					if err := clientChannel.Close(); err != nil {
						panic(err)
					}
				}
				serverRequests = nil
				continue
			}
			accepted, err := clientChannel.SendRequest(serverRequest.Type, serverRequest.WantReply, serverRequest.Payload)
			if err != nil {
				panic(err)
			}
			logEvent(channelRequestLog{
				channelLog: channelLog{
					ChannelID: channelID,
				},
				requestLog: requestLog{
					Type:      serverRequest.Type,
					WantReply: serverRequest.WantReply,
					Payload:   base64.RawStdEncoding.EncodeToString(serverRequest.Payload),
					Accepted:  accepted,
				},
			}, server)
			if serverRequest.WantReply {
				if err := serverRequest.Reply(accepted, nil); err != nil {
					panic(err)
				}
			}
		}
	}
}

func handleConn(clientConn net.Conn, sshServerConfig *ssh.ServerConfig, serverAddress string, clientKey ssh.Signer) {
	clientSSHConn, clientNewChannels, clientRequests, err := ssh.NewServerConn(clientConn, sshServerConfig)
	if err != nil {
		panic(err)
	}

	serverConn, err := net.Dial("tcp", serverAddress)
	if err != nil {
		panic(err)
	}

	serverSSHConn, serverNewChannels, serverRequests, err := ssh.NewClientConn(serverConn, serverAddress, &ssh.ClientConfig{
		User:            clientSSHConn.User(),
		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
		Auth: []ssh.AuthMethod{
			ssh.PublicKeys(clientKey),
		},
		ClientVersion: "SSH-2.0-OpenSSH_7.2",
	})
	if err != nil {
		panic(err)
	}

	channelID := 0

	for clientNewChannels != nil || clientRequests != nil || serverNewChannels != nil || serverRequests != nil {
		select {
		case clientNewChannel, ok := <-clientNewChannels:
			if !ok {
				clientNewChannels = nil
				if serverNewChannels != nil {
					logEvent(connectionCloseLog{}, client)
					if err := serverSSHConn.Close(); err != nil {
						panic(err)
					}
				}
				continue
			}
			serverChannel, serverChannelRequests, err := serverSSHConn.OpenChannel(clientNewChannel.ChannelType(), clientNewChannel.ExtraData())
			accepted := true
			var rejectReason ssh.RejectionReason
			var message string
			if err != nil {
				if err, ok := err.(*ssh.OpenChannelError); ok {
					accepted = false
					rejectReason = err.Reason
					message = err.Message
				} else {
					panic(err)
				}
			}
			logEvent(newChannelLog{
				Type:         clientNewChannel.ChannelType(),
				ExtraData:    base64.RawStdEncoding.EncodeToString(clientNewChannel.ExtraData()),
				Accepted:     err == nil,
				RejectReason: uint32(rejectReason),
				Message:      message,
			}, client)
			if !accepted {
				if err := clientNewChannel.Reject(rejectReason, message); err != nil {
					panic(err)
				}
				continue
			}
			clientChannel, clientChannelRequests, err := clientNewChannel.Accept()
			if err != nil {
				panic(err)
			}
			go handleChannel(channelID, clientChannel, clientChannelRequests, serverChannel, serverChannelRequests)
			channelID++
		case clientRequest, ok := <-clientRequests:
			if !ok {
				clientRequests = nil
				continue
			}
			if clientRequest.Type == "no-more-sessions@openssh.com" {
				logEvent(globalRequestLog{
					requestLog: requestLog{
						Type:      clientRequest.Type,
						WantReply: clientRequest.WantReply,
						Payload:   base64.RawStdEncoding.EncodeToString(clientRequest.Payload),
						Accepted:  clientRequest.WantReply,
					},
					Response: "",
				}, client)
				continue
			}
			accepted, response, err := serverSSHConn.SendRequest(clientRequest.Type, clientRequest.WantReply, clientRequest.Payload)
			if err != nil {
				panic(err)
			}
			logEvent(globalRequestLog{
				requestLog: requestLog{
					Type:      clientRequest.Type,
					WantReply: clientRequest.WantReply,
					Payload:   base64.RawStdEncoding.EncodeToString(clientRequest.Payload),
					Accepted:  accepted,
				},
				Response: base64.RawStdEncoding.EncodeToString(response),
			}, client)
			if err := clientRequest.Reply(accepted, response); err != nil {
				panic(err)
			}
		case serverNewChannel, ok := <-serverNewChannels:
			if !ok {
				if clientNewChannels != nil {
					logEvent(connectionCloseLog{}, server)
					if err := clientSSHConn.Close(); err != nil {
						panic(err)
					}
				}
				serverNewChannels = nil
				continue
			}
			panic(serverNewChannel.ChannelType())
		case serverRequest, ok := <-serverRequests:
			if !ok {
				serverRequests = nil
				continue
			}
			accepted, response, err := clientSSHConn.SendRequest(serverRequest.Type, serverRequest.WantReply, serverRequest.Payload)
			logEvent(globalRequestLog{
				requestLog: requestLog{
					Type:      serverRequest.Type,
					WantReply: serverRequest.WantReply,
					Payload:   base64.RawStdEncoding.EncodeToString(serverRequest.Payload),
					Accepted:  accepted,
				},
				Response: base64.RawStdEncoding.EncodeToString(response),
			}, server)
			if err != nil {
				panic(err)
			}
			if err := serverRequest.Reply(accepted, response); err != nil {
				panic(err)
			}
		}
	}
}

func main() {
	listenAddress := flag.String("listen_address", "127.0.0.1:2022", "listen address")
	hostKeyFile := flag.String("host_key_file", "", "host key file")
	serverAddress := flag.String("server_address", "127.0.0.1:22", "server address")
	clientKeyFile := flag.String("client_key_file", "", "client key file")
	flag.Parse()
	if *listenAddress == "" {
		panic("listen address is required")
	}
	if *hostKeyFile == "" {
		panic("host key file is required")
	}
	if *serverAddress == "" {
		panic("server address is required")
	}
	if *clientKeyFile == "" {
		panic("client key file is required")
	}

	log.SetFlags(0)
	log.SetOutput(os.Stdout)

	serverConfig := &ssh.ServerConfig{
		NoClientAuth:  true,
		ServerVersion: "SSH-2.0-OpenSSH_7.2",
	}
	hostKeyBytes, err := ioutil.ReadFile(*hostKeyFile)
	if err != nil {
		panic(err)
	}
	hostKey, err := ssh.ParsePrivateKey(hostKeyBytes)
	if err != nil {
		panic(err)
	}
	serverConfig.AddHostKey(hostKey)

	clientKeyBytes, err := ioutil.ReadFile(*clientKeyFile)
	if err != nil {
		panic(err)
	}
	clientKey, err := ssh.ParsePrivateKey(clientKeyBytes)
	if err != nil {
		panic(err)
	}

	listener, err := net.Listen("tcp", *listenAddress)
	if err != nil {
		panic(err)
	}
	defer listener.Close()
	for {
		conn, err := listener.Accept()
		if err != nil {
			panic(err)
		}
		go handleConn(conn, serverConfig, *serverAddress, clientKey)
	}
}
<commit_msg>testproxy: record events and log them at once<commit_after>package main

import (
	"encoding/base64"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net"

	"golang.org/x/crypto/ssh"
)

type event struct {
	Source string   `json:"source"`
	Type   string   `json:"type"`
	Entry  logEntry `json:"entry"`
}

type source int

const (
	client source = iota
	server
)

func (src source) String() string {
	switch src {
	case client:
		return "client"
	case server:
		return "server"
	default:
		return "unknown"
	}
}

func (src source) MarshalJSON() ([]byte, error) {
	return json.Marshal(src.String())
}

type logEntry interface {
	eventType() string
}

type channelLog struct {
	ChannelID int `json:"channel_id"`
}

type requestLog struct {
	Type      string `json:"type"`
	WantReply bool   `json:"want_reply"`
	Payload   string `json:"payload"`

	Accepted bool `json:"accepted"`
}

type globalRequestLog struct {
	requestLog

	Response string `json:"response"`
}

func (entry globalRequestLog) eventType() string {
	return "global_request"
}

type newChannelLog struct {
	Type      string `json:"type"`
	ExtraData string `json:"extra_data"`

	Accepted     bool   `json:"accepted"`
	RejectReason uint32 `json:"reject_reason"`
	Message      string `json:"message"`
}

func (entry newChannelLog) eventType() string {
	return "new_channel"
}

type channelRequestLog struct {
	channelLog
	requestLog
}

func (entry channelRequestLog) eventType() string {
	return "channel_request"
}

type channelDataLog struct {
	channelLog
	Data string `json:"data"`
}

func (entry channelDataLog) eventType() string {
	return "channel_data"
}

type channelErrorLog struct {
	channelLog
	Data string `json:"data"`
}

func (entry channelErrorLog) eventType() string {
	return "channel_error"
}

type channelEOFLog struct {
	channelLog
}

func (entry channelEOFLog) eventType() string {
	return "channel_eof"
}

type channelCloseLog struct {
	channelLog
}

func (entry channelCloseLog) eventType() string {
	return "channel_close"
}

type connectionCloseLog struct{}

func (entry connectionCloseLog) eventType() string {
	return "connection_close"
}

var output struct {
	Events []event `json:"events"`
	Logs   struct {
		Text string `json:"text"`
		JSON string `json:"json"`
	} `json:"logs"`
}

func recordEntry(entry logEntry, src source) {
	event := event{
		Source: src.String(),
		Type:   entry.eventType(),
		Entry:  entry,
	}
	output.Events = append(output.Events, event)
}

func streamReader(reader io.Reader) <-chan string {
	input := make(chan string)
	go func() {
		defer close(input)
		buffer := make([]byte, 256)
		for {
			n, err := reader.Read(buffer)
			if n > 0 {
				input <- string(buffer[:n])
			}
			if err != nil {
				if err != io.EOF {
					panic(err)
				}
				return
			}
		}
	}()
	return input
}

func handleChannel(channelID int, clientChannel ssh.Channel, clientRequests <-chan *ssh.Request, serverChannel ssh.Channel, serverRequests <-chan *ssh.Request) {
	clientInputStream := streamReader(clientChannel)
	serverInputStream := streamReader(serverChannel)
	serverErrorStream := streamReader(serverChannel.Stderr())

	for clientInputStream != nil || clientRequests != nil || serverInputStream != nil || serverRequests != nil {
		select {
		case clientInput, ok := <-clientInputStream:
			if !ok {
				if serverInputStream != nil {
					recordEntry(channelEOFLog{
						channelLog: channelLog{
							ChannelID: channelID,
						},
					}, client)
					if err := serverChannel.CloseWrite(); err != nil {
						panic(err)
					}
				}
				clientInputStream = nil
				continue
			}
			recordEntry(channelDataLog{
				channelLog: channelLog{
					ChannelID: channelID,
				},
				Data: clientInput,
			}, client)
			if _, err := serverChannel.Write([]byte(clientInput)); err != nil {
				panic(err)
			}
		case clientRequest, ok := <-clientRequests:
			if !ok {
				if clientInputStream != nil && serverInputStream != nil {
					continue
				}
				if serverRequests != nil {
					recordEntry(channelCloseLog{
						channelLog: channelLog{
							ChannelID: channelID,
						},
					}, client)
					if err := serverChannel.Close(); err != nil {
						panic(err)
					}
				}
				clientRequests = nil
				continue
			}
			accepted, err := serverChannel.SendRequest(clientRequest.Type, clientRequest.WantReply, clientRequest.Payload)
			if err != nil {
				panic(err)
			}
			recordEntry(channelRequestLog{
				channelLog: channelLog{
					ChannelID: channelID,
				},
				requestLog: requestLog{
					Type:      clientRequest.Type,
					WantReply: clientRequest.WantReply,
					Payload:   base64.RawStdEncoding.EncodeToString(clientRequest.Payload),
					Accepted:  accepted,
				},
			}, client)
			if clientRequest.WantReply {
				if err := clientRequest.Reply(accepted, nil); err != nil {
					panic(err)
				}
			}
		case serverInput, ok := <-serverInputStream:
			if !ok {
				if clientInputStream != nil {
					recordEntry(channelEOFLog{
						channelLog: channelLog{
							ChannelID: channelID,
						},
					}, server)
					if err := clientChannel.CloseWrite(); err != nil {
						panic(err)
					}
				}
				serverInputStream = nil
				continue
			}
			recordEntry(channelDataLog{
				channelLog: channelLog{
					ChannelID: channelID,
				},
				Data: serverInput,
			}, server)
			if _, err := clientChannel.Write([]byte(serverInput)); err != nil {
				panic(err)
			}
		case serverError, ok := <-serverErrorStream:
			if !ok {
				serverErrorStream = nil
				continue
			}
			recordEntry(channelErrorLog{
				channelLog: channelLog{
					ChannelID: channelID,
				},
				Data: serverError,
			}, server)
			if _, err := clientChannel.Stderr().Write([]byte(serverError)); err != nil {
				panic(err)
			}
		case serverRequest, ok := <-serverRequests:
			if !ok {
				if clientInputStream != nil && serverInputStream != nil {
					continue
				}
				if clientRequests != nil {
					recordEntry(channelCloseLog{
						channelLog: channelLog{
							ChannelID: channelID,
						},
					}, server)
					if err := clientChannel.Close(); err != nil {
						panic(err)
					}
				}
				serverRequests = nil
				continue
			}
			accepted, err := clientChannel.SendRequest(serverRequest.Type, serverRequest.WantReply, serverRequest.Payload)
			if err != nil {
				panic(err)
			}
			recordEntry(channelRequestLog{
				channelLog: channelLog{
					ChannelID: channelID,
				},
				requestLog: requestLog{
					Type:      serverRequest.Type,
					WantReply: serverRequest.WantReply,
					Payload:   base64.RawStdEncoding.EncodeToString(serverRequest.Payload),
					Accepted:  accepted,
				},
			}, server)
			if serverRequest.WantReply {
				if err := serverRequest.Reply(accepted, nil); err != nil {
					panic(err)
				}
			}
		}
	}
}

func handleConn(clientConn net.Conn, sshServerConfig *ssh.ServerConfig, serverAddress string, clientKey ssh.Signer) {
	clientSSHConn, clientNewChannels, clientRequests, err := ssh.NewServerConn(clientConn, sshServerConfig)
	if err != nil {
		panic(err)
	}

	serverConn, err := net.Dial("tcp", serverAddress)
	if err != nil {
		panic(err)
	}

	serverSSHConn, serverNewChannels, serverRequests, err := ssh.NewClientConn(serverConn, serverAddress, &ssh.ClientConfig{
		User:            clientSSHConn.User(),
		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
		Auth: []ssh.AuthMethod{
			ssh.PublicKeys(clientKey),
		},
		ClientVersion: "SSH-2.0-OpenSSH_7.2",
	})
	if err != nil {
		panic(err)
	}

	channelID := 0

	for clientNewChannels != nil || clientRequests != nil || serverNewChannels != nil || serverRequests != nil {
		select {
		case clientNewChannel, ok := <-clientNewChannels:
			if !ok {
				clientNewChannels = nil
				if serverNewChannels != nil {
					recordEntry(connectionCloseLog{}, client)
					if err := serverSSHConn.Close(); err != nil {
						panic(err)
					}
				}
				continue
			}
			serverChannel, serverChannelRequests, err := serverSSHConn.OpenChannel(clientNewChannel.ChannelType(), clientNewChannel.ExtraData())
			accepted := true
			var rejectReason ssh.RejectionReason
			var message string
			if err != nil {
				if err, ok := err.(*ssh.OpenChannelError); ok {
					accepted = false
					rejectReason = err.Reason
					message = err.Message
				} else {
					panic(err)
				}
			}
			recordEntry(newChannelLog{
				Type:         clientNewChannel.ChannelType(),
				ExtraData:    base64.RawStdEncoding.EncodeToString(clientNewChannel.ExtraData()),
				Accepted:     err == nil,
				RejectReason: uint32(rejectReason),
				Message:      message,
			}, client)
			if !accepted {
				if err := clientNewChannel.Reject(rejectReason, message); err != nil {
					panic(err)
				}
				continue
			}
			clientChannel, clientChannelRequests, err := clientNewChannel.Accept()
			if err != nil {
				panic(err)
			}
			go handleChannel(channelID, clientChannel, clientChannelRequests, serverChannel, serverChannelRequests)
			channelID++
		case clientRequest, ok := <-clientRequests:
			if !ok {
				clientRequests = nil
				continue
			}
			if clientRequest.Type == "no-more-sessions@openssh.com" {
				recordEntry(globalRequestLog{
					requestLog: requestLog{
						Type:      clientRequest.Type,
						WantReply: clientRequest.WantReply,
						Payload:   base64.RawStdEncoding.EncodeToString(clientRequest.Payload),
						Accepted:  clientRequest.WantReply,
					},
					Response: "",
				}, client)
				continue
			}
			accepted, response, err := serverSSHConn.SendRequest(clientRequest.Type, clientRequest.WantReply, clientRequest.Payload)
			if err != nil {
				panic(err)
			}
			recordEntry(globalRequestLog{
				requestLog: requestLog{
					Type:      clientRequest.Type,
					WantReply: clientRequest.WantReply,
					Payload:   base64.RawStdEncoding.EncodeToString(clientRequest.Payload),
					Accepted:  accepted,
				},
				Response: base64.RawStdEncoding.EncodeToString(response),
			}, client)
			if err := clientRequest.Reply(accepted, response); err != nil {
				panic(err)
			}
		case serverNewChannel, ok := <-serverNewChannels:
			if !ok {
				if clientNewChannels != nil {
					recordEntry(connectionCloseLog{}, server)
					if err := clientSSHConn.Close(); err != nil {
						panic(err)
					}
				}
				serverNewChannels = nil
				continue
			}
			panic(serverNewChannel.ChannelType())
		case serverRequest, ok := <-serverRequests:
			if !ok {
				serverRequests = nil
				continue
			}
			accepted, response, err := clientSSHConn.SendRequest(serverRequest.Type, serverRequest.WantReply, serverRequest.Payload)
			recordEntry(globalRequestLog{
				requestLog: requestLog{
					Type:      serverRequest.Type,
					WantReply: serverRequest.WantReply,
					Payload:   base64.RawStdEncoding.EncodeToString(serverRequest.Payload),
					Accepted:  accepted,
				},
				Response: base64.RawStdEncoding.EncodeToString(response),
			}, server)
			if err != nil {
				panic(err)
			}
			if err := serverRequest.Reply(accepted, response); err != nil {
				panic(err)
			}
		}
	}
}

func main() {
	listenAddress := flag.String("listen_address", "127.0.0.1:2022", "listen address")
	hostKeyFile := flag.String("host_key_file", "", "host key file")
	serverAddress := flag.String("server_address", "127.0.0.1:22", "server address")
	clientKeyFile := flag.String("client_key_file", "", "client key file")
	flag.Parse()
	if *listenAddress == "" {
		panic("listen address is required")
	}
	if *hostKeyFile == "" {
		panic("host key file is required")
	}
	if *serverAddress == "" {
		panic("server address is required")
	}
	if *clientKeyFile == "" {
		panic("client key file is required")
	}

	serverConfig := &ssh.ServerConfig{
		NoClientAuth:  true,
		ServerVersion: "SSH-2.0-OpenSSH_7.2",
	}
	hostKeyBytes, err := ioutil.ReadFile(*hostKeyFile)
	if err != nil {
		panic(err)
	}
	hostKey, err := ssh.ParsePrivateKey(hostKeyBytes)
	if err != nil {
		panic(err)
	}
	serverConfig.AddHostKey(hostKey)

	clientKeyBytes, err := ioutil.ReadFile(*clientKeyFile)
	if err != nil {
		panic(err)
	}
	clientKey, err := ssh.ParsePrivateKey(clientKeyBytes)
	if err != nil {
		panic(err)
	}

	listener, err := net.Listen("tcp", *listenAddress)
	if err != nil {
		panic(err)
	}
	defer listener.Close()

	conn, err := listener.Accept()
	if err != nil {
		panic(err)
	}
	handleConn(conn, serverConfig, *serverAddress, clientKey)

	outputBytes, err := json.MarshalIndent(output, "", "  ")
	if err != nil {
		panic(err)
	}
	fmt.Println(string(outputBytes))
}
<|endoftext|>


<commit_before>// Package to parse a tag.
//
// Tags are annotation with struct fields, that allow for some elegant solutions for different problems. Examples are
// validation and alike. This is an example on what that might look like:
//	type Example struct {
//		AField string `tagName1:"tagValue1" tagName2:"tagValue2"`
//	}
//
// As the syntax is somewhat weird and the tag interface only supports a getter, this tag parser was written. It will
// use go's tag parser to retrieve the tag for a given prefix, parse the value, and return a map of strings to strings.
// Keys and values are separated by an equal sign '=', values might be quoted using single quotes "'", and key-value
// pairs are separated using whitespace.
package tagparse

import (
	"fmt"
	"reflect"
	"strings"
)

func tagSplit(tag string) (fields []string, e error) {
	fields = []string{}
	idxStart := 0
	quoted := false
	for i, c := range tag {
		if i == idxStart && (c == ' ' || c == '\t') {
			idxStart = i + 1
			continue
		}
		if c == '\'' {
			quoted = !quoted
		}

		if (c == ' ' || i+1 == len(tag)) && !quoted {
			fields = append(fields, strings.TrimSpace(tag[idxStart:i+1]))
			idxStart = i + 1
		}
	}
	if quoted {
		return nil, fmt.Errorf("failed to parse tag due to erroneous quotes")
	}
	return fields, nil
}

func parseTag(tagString string) (result map[string]string, e error) {
	fields, e := tagSplit(tagString)
	if e != nil {
		return nil, e
	}

	result = map[string]string{}
	for fIdx := range fields {
		kvList := strings.SplitN(fields[fIdx], "=", 2)
		if len(kvList) != 2 {
			return nil, fmt.Errorf("failed to parse annotation (value missing): %q", fields[fIdx])
		}
		key := strings.TrimSpace(kvList[0])
		value := strings.Trim(kvList[1], "'")

		if _, found := result[key]; found {
			return nil, fmt.Errorf("key %q set multiple times", key)
		}
		result[key] = value
	}
	return result, nil
}

// Parse tag with the given prefix of the given field. Return a map of strings to strings. If errors occur they are
// returned accordingly.
func Parse(field reflect.StructField, prefix string) (result map[string]string, e error) {
	tagString := field.Tag.Get(prefix)

	return parseTag(tagString)
}
<commit_msg>improved tagparse to accept a custom splitter for non 'key=value' fields<commit_after>// Package to parse a tag.
//
// Tags are annotation with struct fields, that allow for some elegant solutions for different problems. Examples are
// validation and alike. This is an example on what that might look like:
//	type Example struct {
//		AField string `tagName1:"tagValue1" tagName2:"tagValue2"`
//	}
//
// As the syntax is somewhat weird and the tag interface only supports a getter, this tag parser was written. It will
// use go's tag parser to retrieve the tag for a given prefix, parse the value, and return a map of strings to strings.
// Keys and values are separated by an equal sign '=', values might be quoted using single quotes "'", and key-value
// pairs are separated using whitespace.
package tagparse

import (
	"fmt"
	"reflect"
	"strings"
)

func tagSplit(tag string) (fields []string, e error) {
	fields = []string{}
	idxStart := 0
	quoted := false
	for i, c := range tag {
		if i == idxStart && (c == ' ' || c == '\t') {
			idxStart = i + 1
			continue
		}
		if c == '\'' {
			quoted = !quoted
		}

		if (c == ' ' || i+1 == len(tag)) && !quoted {
			fields = append(fields, strings.TrimSpace(tag[idxStart:i+1]))
			idxStart = i + 1
		}
	}
	if quoted {
		return nil, fmt.Errorf("failed to parse tag due to erroneous quotes")
	}
	return fields, nil
}

func parseTag(tagString string, customS CustomSplitter) (result map[string]string, e error) {
	fields, e := tagSplit(tagString)
	if e != nil {
		return nil, e
	}

	result = map[string]string{}
	for fIdx := range fields {
		kvList := strings.SplitN(fields[fIdx], "=", 2)
		var key, value string
		if len(kvList) == 2 {
			key = strings.TrimSpace(kvList[0])
			value = strings.Trim(kvList[1], "'")
		} else {
			key, value, e = customS(fields[fIdx])
			if e != nil {
				return nil, fmt.Errorf("failed to parse annotation (value missing): %q", fields[fIdx])
			}
		}

		if _, found := result[key]; found {
			return nil, fmt.Errorf("key %q set multiple times", key)
		}
		result[key] = value
	}
	return result, nil
}

type CustomSplitter func(string) (string, string, error)

// Parse tag with the given prefix of the given field. Return a map of strings to strings. If errors occur they are
// returned accordingly.
func Parse(field reflect.StructField, prefix string, customHander func(string) (string, string, error)) (result map[string]string, e error) {
	return ParseCustom(field, prefix, func(_ string) (string, string, error) { return "", "", fmt.Errorf("failed") })
}

// Like Parse, but with a custom splitter used for tag values that don't have the form `key=value`.
func ParseCustom(field reflect.StructField, prefix string, customF CustomSplitter) (result map[string]string, e error) {
	tagString := field.Tag.Get(prefix)

	return parseTag(tagString, customF)
}
<|endoftext|>


<commit_before>package models

import (
	"encoding/json"
	"errors"
	// "fmt"
	"github.com/jinzhu/gorm"
	"github.com/koding/bongo"
	"socialapi/models"
	"time"
)

type Notification struct {
	// unique identifier of Notification
	Id int64 `json:"id"`

	// notification recipient account id
	AccountId int64 `json:"accountId,string" sql:"NOT NULL"`

	// notification content foreign key
	NotificationContentId int64 `json:"notificationContentId" sql:"NOT NULL"`

	// glanced information
	Glanced bool `json:"glanced" sql:"NOT NULL"`

	// last notifier addition time. when user first subscribes it is set to ZeroDate
	ActivatedAt time.Time `json:"activatedAt"`

	// user's subscription time to related content
	SubscribedAt time.Time `json:"subscribedAt"`

	// notification type as subscribed/unsubscribed
	UnsubscribedAt time.Time `json:"unsubscribedAt"`

	SubscribeOnly bool `json:"-" sql:"-"`
}

func (n *Notification) BeforeCreate() {
	if n.UnsubscribedAt.Equal(ZeroDate()) && n.SubscribedAt.Equal(ZeroDate()) {
		n.SubscribedAt = time.Now()
	}
}

func (n *Notification) BeforeUpdate() {
	if n.UnsubscribedAt.Equal(ZeroDate()) && !n.SubscribeOnly {
		n.Glanced = false
		n.ActivatedAt = time.Now()
	}
}

func (n *Notification) AfterCreate() {
	bongo.B.AfterCreate(n)
}

func (n *Notification) AfterUpdate() {
	bongo.B.AfterUpdate(n)
}

func (n *Notification) GetId() int64 {
	return n.Id
}

func (n Notification) TableName() string {
	return "notification.notification"
}

func NewNotification() *Notification {
	return &Notification{}
}

func (n *Notification) One(q *bongo.Query) error {
	return bongo.B.One(n, n, q)
}

func (n *Notification) Create() error {
	// TODO check notification content existence
	if err := n.FetchByContent(); err != nil {
		if err != gorm.RecordNotFound {
			return err
		}

		return bongo.B.Create(n)
	}

	return nil
}

func (n *Notification) Upsert() error {
	unsubscribedAt := n.UnsubscribedAt

	if err := n.FetchByContent(); err != nil {
		if err != gorm.RecordNotFound {
			return err
		}

		return bongo.B.Create(n)
	}

	if !unsubscribedAt.IsZero() {
		n.UnsubscribedAt = unsubscribedAt
	}

	return bongo.B.Update(n)
}

func (n *Notification) Subscribe(nc *NotificationContent) error {
	if nc.TargetId == 0 {
		return errors.New("target id cannot be empty")
	}
	nc.TypeConstant = NotificationContent_TYPE_COMMENT

	if err := nc.Create(); err != nil {
		return err
	}

	n.NotificationContentId = nc.Id
	n.SubscribeOnly = true

	return n.Create()
}

func (n *Notification) Unsubscribe(nc *NotificationContent) error {
	n.UnsubscribedAt = time.Now()

	return n.Subscribe(nc)
}

func (n *Notification) List(q *models.Query) (*NotificationResponse, error) {
	if q.Limit == 0 {
		return nil, errors.New("limit cannot be zero")
	}
	response := &NotificationResponse{}
	result, err := n.getDecoratedList(q)
	if err != nil {
		return response, err
	}

	response.Notifications = result
	response.UnreadCount = getUnreadNotificationCount(result)

	return response, nil
}

func (n *Notification) Some(data interface{}, q *bongo.Query) error {

	return bongo.B.Some(n, data, q)
}

func (n *Notification) fetchByAccountId(q *models.Query) ([]Notification, error) {
	var notifications []Notification

	err := bongo.B.DB.Table(n.TableName()).
		Where("NOT (activated_at IS NULL OR activated_at <= '0001-01-02') AND account_id = ?", q.AccountId).
		Order("activated_at desc").
		Limit(q.Limit).
		Find(&notifications).Error

	if err != nil {
		return nil, err
	}

	return notifications, nil
}

func (n *Notification) FetchByContent() error {
	selector := map[string]interface{}{
		"account_id":              n.AccountId,
		"notification_content_id": n.NotificationContentId,
	}
	q := bongo.NewQS(selector)

	return n.One(q)
}

// getDecoratedList fetches notifications of the given user and decorates it with
// notification activity actors
func (n *Notification) getDecoratedList(q *models.Query) ([]NotificationContainer, error) {
	result := make([]NotificationContainer, 0)

	nList, err := n.fetchByAccountId(q)
	if err != nil {
		return nil, err
	}
	// fetch all notification content relationships
	contentIds := deductContentIds(nList)

	nc := NewNotificationContent()
	ncMap, err := nc.FetchMapByIds(contentIds)
	if err != nil {
		return nil, err
	}

	na := NewNotificationActivity()
	naMap, err := na.FetchMapByContentIds(contentIds)
	if err != nil {
		return nil, err
	}

	for _, n := range nList {
		nc := ncMap[n.NotificationContentId]
		na := naMap[n.NotificationContentId]
		container := n.buildNotificationContainer(q.AccountId, &nc, na)
		result = append(result, container)
	}

	return result, nil
}

func (n *Notification) buildNotificationContainer(actorId int64, nc *NotificationContent, na []NotificationActivity) NotificationContainer {
	ct, err := CreateNotificationContentType(nc.TypeConstant)
	if err != nil {
		return NotificationContainer{}
	}

	ct.SetTargetId(nc.TargetId)
	ct.SetListerId(actorId)
	ac, err := ct.FetchActors(na)
	if err != nil {
		return NotificationContainer{}
	}
	latestActorsOldIds, _ := models.AccountOldsIdByIds(ac.LatestActors)

	return NotificationContainer{
		TargetId:              nc.TargetId,
		TypeConstant:          nc.TypeConstant,
		UpdatedAt:             n.ActivatedAt,
		Glanced:               n.Glanced,
		NotificationContentId: nc.Id,
		LatestActors:          ac.LatestActors,
		LatestActorsOldIds:    latestActorsOldIds,
		ActorCount:            ac.Count,
	}
}

func deductContentIds(nList []Notification) []int64 {
	notificationContentIds := make([]int64, len(nList))
	for _, n := range nList {
		notificationContentIds = append(notificationContentIds, n.NotificationContentId)
	}

	return notificationContentIds
}

func (n *Notification) FetchContent() (*NotificationContent, error) {
	nc := NewNotificationContent()
	if err := nc.ById(n.NotificationContentId); err != nil {
		return nil, err
	}

	return nc, nil
}

func (n *Notification) Glance() error {
	selector := map[string]interface{}{
		"glanced":    false,
		"account_id": n.AccountId,
	}

	set := map[string]interface{}{
		"glanced": true,
	}

	return bongo.B.UpdateMulti(n, selector, set)
}

func getUnreadNotificationCount(notificationList []NotificationContainer) int {
	unreadCount := 0
	for _, nc := range notificationList {
		if !nc.Glanced {
			unreadCount++
		}
	}

	return unreadCount
}

func (n *Notification) MapMessage(data []byte) error {
	if err := json.Unmarshal(data, n); err != nil {
		return err
	}

	return nil
}

func (n *Notification) FetchLastActivity() (*NotificationActivity, *NotificationContent, error) {
	// fetch notification content and get event type
	nc, err := n.FetchContent()
	if err != nil {
		return nil, nil, err
	}

	a := NewNotificationActivity()
	a.NotificationContentId = nc.Id

	if err := a.LastActivity(); err != nil {
		return nil, nil, err
	}

	return a, nc, nil
}
<commit_msg>Notification: time.Equal(ZeroDate) checks are changed with time.IsZero()<commit_after>package models

import (
	"encoding/json"
	"errors"
	// "fmt"
	"github.com/jinzhu/gorm"
	"github.com/koding/bongo"
	"socialapi/models"
	"time"
)

type Notification struct {
	// unique identifier of Notification
	Id int64 `json:"id"`

	// notification recipient account id
	AccountId int64 `json:"accountId,string" sql:"NOT NULL"`

	// notification content foreign key
	NotificationContentId int64 `json:"notificationContentId" sql:"NOT NULL"`

	// glanced information
	Glanced bool `json:"glanced" sql:"NOT NULL"`

	// last notifier addition time. when user first subscribes it is set to ZeroDate
	ActivatedAt time.Time `json:"activatedAt"`

	// user's subscription time to related content
	SubscribedAt time.Time `json:"subscribedAt"`

	// notification type as subscribed/unsubscribed
	UnsubscribedAt time.Time `json:"unsubscribedAt"`

	SubscribeOnly bool `json:"-" sql:"-"`
}

func (n *Notification) BeforeCreate() {
	if n.UnsubscribedAt.IsZero() && n.SubscribedAt.IsZero() {
		n.SubscribedAt = time.Now()
	}
}

func (n *Notification) BeforeUpdate() {
	if n.UnsubscribedAt.IsZero() && !n.SubscribeOnly {
		n.Glanced = false
		n.ActivatedAt = time.Now()
	}
}

func (n *Notification) AfterCreate() {
	bongo.B.AfterCreate(n)
}

func (n *Notification) AfterUpdate() {
	bongo.B.AfterUpdate(n)
}

func (n *Notification) GetId() int64 {
	return n.Id
}

func (n Notification) TableName() string {
	return "notification.notification"
}

func NewNotification() *Notification {
	return &Notification{}
}

func (n *Notification) One(q *bongo.Query) error {
	return bongo.B.One(n, n, q)
}

func (n *Notification) Create() error {
	// TODO check notification content existence
	if err := n.FetchByContent(); err != nil {
		if err != gorm.RecordNotFound {
			return err
		}

		return bongo.B.Create(n)
	}

	return nil
}

func (n *Notification) Upsert() error {
	unsubscribedAt := n.UnsubscribedAt

	if err := n.FetchByContent(); err != nil {
		if err != gorm.RecordNotFound {
			return err
		}

		return bongo.B.Create(n)
	}

	if !unsubscribedAt.IsZero() {
		n.UnsubscribedAt = unsubscribedAt
	}

	return bongo.B.Update(n)
}

func (n *Notification) Subscribe(nc *NotificationContent) error {
	if nc.TargetId == 0 {
		return errors.New("target id cannot be empty")
	}
	nc.TypeConstant = NotificationContent_TYPE_COMMENT

	if err := nc.Create(); err != nil {
		return err
	}

	n.NotificationContentId = nc.Id
	n.SubscribeOnly = true

	return n.Create()
}

func (n *Notification) Unsubscribe(nc *NotificationContent) error {
	n.UnsubscribedAt = time.Now()

	return n.Subscribe(nc)
}

func (n *Notification) List(q *models.Query) (*NotificationResponse, error) {
	if q.Limit == 0 {
		return nil, errors.New("limit cannot be zero")
	}
	response := &NotificationResponse{}
	result, err := n.getDecoratedList(q)
	if err != nil {
		return response, err
	}

	response.Notifications = result
	response.UnreadCount = getUnreadNotificationCount(result)

	return response, nil
}

func (n *Notification) Some(data interface{}, q *bongo.Query) error {

	return bongo.B.Some(n, data, q)
}

func (n *Notification) fetchByAccountId(q *models.Query) ([]Notification, error) {
	var notifications []Notification

	err := bongo.B.DB.Table(n.TableName()).
		Where("NOT (activated_at IS NULL OR activated_at <= '0001-01-02') AND account_id = ?", q.AccountId).
		Order("activated_at desc").
		Limit(q.Limit).
		Find(&notifications).Error

	if err != nil {
		return nil, err
	}

	return notifications, nil
}

func (n *Notification) FetchByContent() error {
	selector := map[string]interface{}{
		"account_id":              n.AccountId,
		"notification_content_id": n.NotificationContentId,
	}
	q := bongo.NewQS(selector)

	return n.One(q)
}

// getDecoratedList fetches notifications of the given user and decorates it with
// notification activity actors
func (n *Notification) getDecoratedList(q *models.Query) ([]NotificationContainer, error) {
	result := make([]NotificationContainer, 0)

	nList, err := n.fetchByAccountId(q)
	if err != nil {
		return nil, err
	}
	// fetch all notification content relationships
	contentIds := deductContentIds(nList)

	nc := NewNotificationContent()
	ncMap, err := nc.FetchMapByIds(contentIds)
	if err != nil {
		return nil, err
	}

	na := NewNotificationActivity()
	naMap, err := na.FetchMapByContentIds(contentIds)
	if err != nil {
		return nil, err
	}

	for _, n := range nList {
		nc := ncMap[n.NotificationContentId]
		na := naMap[n.NotificationContentId]
		container := n.buildNotificationContainer(q.AccountId, &nc, na)
		result = append(result, container)
	}

	return result, nil
}

func (n *Notification) buildNotificationContainer(actorId int64, nc *NotificationContent, na []NotificationActivity) NotificationContainer {
	ct, err := CreateNotificationContentType(nc.TypeConstant)
	if err != nil {
		return NotificationContainer{}
	}

	ct.SetTargetId(nc.TargetId)
	ct.SetListerId(actorId)
	ac, err := ct.FetchActors(na)
	if err != nil {
		return NotificationContainer{}
	}
	latestActorsOldIds, _ := models.AccountOldsIdByIds(ac.LatestActors)

	return NotificationContainer{
		TargetId:              nc.TargetId,
		TypeConstant:          nc.TypeConstant,
		UpdatedAt:             n.ActivatedAt,
		Glanced:               n.Glanced,
		NotificationContentId: nc.Id,
		LatestActors:          ac.LatestActors,
		LatestActorsOldIds:    latestActorsOldIds,
		ActorCount:            ac.Count,
	}
}

func deductContentIds(nList []Notification) []int64 {
	notificationContentIds := make([]int64, len(nList))
	for _, n := range nList {
		notificationContentIds = append(notificationContentIds, n.NotificationContentId)
	}

	return notificationContentIds
}

func (n *Notification) FetchContent() (*NotificationContent, error) {
	nc := NewNotificationContent()
	if err := nc.ById(n.NotificationContentId); err != nil {
		return nil, err
	}

	return nc, nil
}

func (n *Notification) Glance() error {
	selector := map[string]interface{}{
		"glanced":    false,
		"account_id": n.AccountId,
	}

	set := map[string]interface{}{
		"glanced": true,
	}

	return bongo.B.UpdateMulti(n, selector, set)
}

func getUnreadNotificationCount(notificationList []NotificationContainer) int {
	unreadCount := 0
	for _, nc := range notificationList {
		if !nc.Glanced {
			unreadCount++
		}
	}

	return unreadCount
}

func (n *Notification) MapMessage(data []byte) error {
	if err := json.Unmarshal(data, n); err != nil {
		return err
	}

	return nil
}

func (n *Notification) FetchLastActivity() (*NotificationActivity, *NotificationContent, error) {
	// fetch notification content and get event type
	nc, err := n.FetchContent()
	if err != nil {
		return nil, nil, err
	}

	a := NewNotificationActivity()
	a.NotificationContentId = nc.Id

	if err := a.LastActivity(); err != nil {
		return nil, nil, err
	}

	return a, nc, nil
}
<|endoftext|>


<commit_before><commit_msg>Refactor the whole event switch into own function<commit_after><|endoftext|>


<commit_before>// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
// All rights reservefs.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package storage

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/syndtr/goleveldb/leveldb/util"
)

var errFileOpen = errors.New("leveldb/storage: file still open")

type fileLock interface {
	release() error
}

type fileStorageLock struct {
	fs *fileStorage
}

func (lock *fileStorageLock) Release() {
	fs := lock.fs
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if fs.slock == lock {
		fs.slock = nil
	}
	return
}

// fileStorage is a file-system backed storage.
type fileStorage struct {
	path string

	mu    sync.Mutex
	flock fileLock
	slock *fileStorageLock
	logw  *os.File
	buf   []byte
	// Opened file counter; if open < 0 means closed.
	open int
	day  int
}

// OpenFile returns a new filesytem-backed storage implementation with the given
// path. This also hold a file lock, so any subsequent attempt to open the same
// path will fail.
//
// The storage must be closed after use, by calling Close method.
func OpenFile(path string) (Storage, error) {
	if err := os.MkdirAll(path, 0755); err != nil {
		return nil, err
	}

	flock, err := newFileLock(filepath.Join(path, "LOCK"))
	if err != nil {
		return nil, err
	}

	defer func() {
		if err != nil {
			flock.release()
		}
	}()

	rename(filepath.Join(path, "LOG"), filepath.Join(path, "LOG.old"))
	logw, err := os.OpenFile(filepath.Join(path, "LOG"), os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		return nil, err
	}

	fs := &fileStorage{path: path, flock: flock, logw: logw}
	runtime.SetFinalizer(fs, (*fileStorage).Close)
	return fs, nil
}

func (fs *fileStorage) Lock() (util.Releaser, error) {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if fs.open < 0 {
		return nil, ErrClosed
	}
	if fs.slock != nil {
		return nil, ErrLocked
	}
	fs.slock = &fileStorageLock{fs: fs}
	return fs.slock, nil
}

func itoa(buf []byte, i int, wid int) []byte {
	var u uint = uint(i)
	if u == 0 && wid <= 1 {
		return append(buf, '0')
	}

	// Assemble decimal in reverse order.
	var b [32]byte
	bp := len(b)
	for ; u > 0 || wid > 0; u /= 10 {
		bp--
		wid--
		b[bp] = byte(u%10) + '0'
	}
	return append(buf, b[bp:]...)
}

func (fs *fileStorage) printDay(t time.Time) {
	if fs.day == t.Day() {
		return
	}
	fs.day = t.Day()
	fs.logw.Write([]byte("=============== " + t.Format("Jan 2, 2006 (MST)") + " ===============\n"))
}

func (fs *fileStorage) doLog(t time.Time, str string) {
	fs.printDay(t)
	hour, min, sec := t.Clock()
	msec := t.Nanosecond() / 1e3
	// time
	fs.buf = itoa(fs.buf[:0], hour, 2)
	fs.buf = append(fs.buf, ':')
	fs.buf = itoa(fs.buf, min, 2)
	fs.buf = append(fs.buf, ':')
	fs.buf = itoa(fs.buf, sec, 2)
	fs.buf = append(fs.buf, '.')
	fs.buf = itoa(fs.buf, msec, 6)
	fs.buf = append(fs.buf, ' ')
	// write
	fs.buf = append(fs.buf, []byte(str)...)
	fs.buf = append(fs.buf, '\n')
	fs.logw.Write(fs.buf)
}

func (fs *fileStorage) Log(str string) {
	t := time.Now()
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if fs.open < 0 {
		return
	}
	fs.doLog(t, str)
}

func (fs *fileStorage) log(str string) {
	fs.doLog(time.Now(), str)
}

func (fs *fileStorage) GetFile(num uint64, t FileType) File {
	return &file{fs: fs, num: num, t: t}
}

func (fs *fileStorage) GetFiles(t FileType) (ff []File, err error) {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if fs.open < 0 {
		return nil, ErrClosed
	}
	dir, err := os.Open(fs.path)
	if err != nil {
		return
	}
	fnn, err := dir.Readdirnames(0)
	// Close the dir first before checking for Readdirnames error.
	if err := dir.Close(); err != nil {
		fs.log(fmt.Sprintf("close dir: %v", err))
	}
	if err != nil {
		return
	}
	f := &file{fs: fs}
	for _, fn := range fnn {
		if f.parse(fn) && (f.t&t) != 0 {
			ff = append(ff, f)
			f = &file{fs: fs}
		}
	}
	return
}

func (fs *fileStorage) GetManifest() (f File, err error) {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if fs.open < 0 {
		return nil, ErrClosed
	}
	dir, err := os.Open(fs.path)
	if err != nil {
		return
	}
	fnn, err := dir.Readdirnames(0)
	// Close the dir first before checking for Readdirnames error.
	if err := dir.Close(); err != nil {
		fs.log(fmt.Sprintf("close dir: %v", err))
	}
	if err != nil {
		return
	}
	// Find latest CURRENT file.
	var rem []string
	var pend bool
	var cerr error
	for _, fn := range fnn {
		if strings.HasPrefix(fn, "CURRENT") {
			pend1 := len(fn) > 7
			// Make sure it is valid name for a CURRENT file, otherwise skip it.
			if pend1 {
				if fn[7] != '.' || len(fn) < 9 {
					fs.log(fmt.Sprintf("skipping %s: invalid file name", fn))
					continue
				}
				if _, e1 := strconv.ParseUint(fn[7:], 10, 0); e1 != nil {
					fs.log(fmt.Sprintf("skipping %s: invalid file num: %v", fn, e1))
					continue
				}
			}
			path := filepath.Join(fs.path, fn)
			r, e1 := os.OpenFile(path, os.O_RDONLY, 0)
			if e1 != nil {
				return nil, e1
			}
			b, e1 := ioutil.ReadAll(r)
			if e1 != nil {
				r.Close()
				return nil, e1
			}
			f1 := &file{fs: fs}
			if len(b) < 1 || b[len(b)-1] != '\n' || !f1.parse(string(b[:len(b)-1])) {
				fs.log(fmt.Sprintf("skipping %s: corrupted or incomplete", fn))
				if pend1 {
					rem = append(rem, fn)
				}
				if !pend1 || cerr == nil {
					cerr = fmt.Errorf("leveldb/storage: corrupted or incomplete %s file", fn)
				}
			} else if f != nil && f1.Num() < f.Num() {
				fs.log(fmt.Sprintf("skipping %s: obsolete", fn))
				if pend1 {
					rem = append(rem, fn)
				}
			} else {
				f = f1
				pend = pend1
			}
			if err := r.Close(); err != nil {
				fs.log(fmt.Sprintf("close %s: %v", fn, err))
			}
		}
	}
	// Don't remove any files if there is no valid CURRENT file.
	if f == nil {
		if cerr != nil {
			err = cerr
		} else {
			err = os.ErrNotExist
		}
		return
	}
	// Rename pending CURRENT file to an effective CURRENT.
	if pend {
		path := fmt.Sprintf("%s.%d", filepath.Join(fs.path, "CURRENT"), f.Num())
		if err := rename(path, filepath.Join(fs.path, "CURRENT")); err != nil {
			fs.log(fmt.Sprintf("CURRENT.%d -> CURRENT: %d", f.Num(), err))
		}
	}
	// Remove obsolete or incomplete pending CURRENT files.
	for _, fn := range rem {
		path := filepath.Join(fs.path, fn)
		if err := os.Remove(path); err != nil {
			fs.log(fmt.Sprintf("remove %s: %v", fn, err))
		}
	}
	return
}

func (fs *fileStorage) SetManifest(f File) (err error) {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if fs.open < 0 {
		return ErrClosed
	}
	f2, ok := f.(*file)
	if !ok || f2.t != TypeManifest {
		return ErrInvalidFile
	}
	defer func() {
		if err != nil {
			fs.log(fmt.Sprintf("CURRENT: %v", err))
		}
	}()
	path := fmt.Sprintf("%s.%d", filepath.Join(fs.path, "CURRENT"), f2.Num())
	w, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		return err
	}
	_, err = fmt.Fprintln(w, f2.name())
	// Close the file first.
	if err := w.Close(); err != nil {
		fs.log(fmt.Sprintf("close CURRENT.%d: %v", f2.num, err))
	}
	if err != nil {
		return err
	}
	err = rename(path, filepath.Join(fs.path, "CURRENT"))
	return
}

func (fs *fileStorage) Close() error {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if fs.open < 0 {
		return ErrClosed
	}
	// Clear the finalizer.
	runtime.SetFinalizer(fs, nil)

	if fs.open > 0 {
		fs.log(fmt.Sprintf("refuse to close, %d files still open", fs.open))
		return fmt.Errorf("leveldb/storage: cannot close, %d files still open", fs.open)
	}
	fs.open = -1
	e1 := fs.logw.Close()
	err := fs.flock.release()
	if err == nil {
		err = e1
	}
	return err
}

type fileWrap struct {
	*os.File
	f *file
}

func (fw fileWrap) Close() error {
	f := fw.f
	f.fs.mu.Lock()
	defer f.fs.mu.Unlock()
	if !f.open {
		return ErrClosed
	}
	f.open = false
	f.fs.open--
	err := fw.File.Close()
	if err != nil {
		f.fs.log(fmt.Sprint("close %s.%d: %v", f.Type(), f.Num(), err))
	}
	return err
}

type file struct {
	fs   *fileStorage
	num  uint64
	t    FileType
	open bool
}

func (f *file) Open() (Reader, error) {
	f.fs.mu.Lock()
	defer f.fs.mu.Unlock()
	if f.fs.open < 0 {
		return nil, ErrClosed
	}
	if f.open {
		return nil, errFileOpen
	}
	of, err := os.OpenFile(f.path(), os.O_RDONLY, 0)
	if err != nil {
		return nil, err
	}
	f.open = true
	f.fs.open++
	return fileWrap{of, f}, nil
}

func (f *file) Create() (Writer, error) {
	f.fs.mu.Lock()
	defer f.fs.mu.Unlock()
	if f.fs.open < 0 {
		return nil, ErrClosed
	}
	if f.open {
		return nil, errFileOpen
	}
	of, err := os.OpenFile(f.path(), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		return nil, err
	}
	f.open = true
	f.fs.open++
	return fileWrap{of, f}, nil
}

func (f *file) Type() FileType {
	return f.t
}

func (f *file) Num() uint64 {
	return f.num
}

func (f *file) Remove() error {
	f.fs.mu.Lock()
	defer f.fs.mu.Unlock()
	if f.fs.open < 0 {
		return ErrClosed
	}
	if f.open {
		return errFileOpen
	}
	err := os.Remove(f.path())
	if err != nil {
		f.fs.log(fmt.Sprint("remove %s.%d: %v", f.Type(), f.Num(), err))
	}
	return err
}

func (f *file) name() string {
	switch f.t {
	case TypeManifest:
		return fmt.Sprintf("MANIFEST-%06d", f.num)
	case TypeJournal:
		return fmt.Sprintf("%06d.log", f.num)
	case TypeTable:
		return fmt.Sprintf("%06d.sst", f.num)
	default:
		panic("invalid file type")
	}
	return ""
}

func (f *file) path() string {
	return filepath.Join(f.fs.path, f.name())
}

func (f *file) parse(name string) bool {
	var num uint64
	var tail string
	_, err := fmt.Sscanf(name, "%d.%s", &num, &tail)
	if err == nil {
		switch tail {
		case "log":
			f.t = TypeJournal
		case "sst":
			f.t = TypeTable
		default:
			return false
		}
		f.num = num
		return true
	}
	n, _ := fmt.Sscanf(name, "MANIFEST-%d%s", &num, &tail)
	if n == 1 {
		f.t = TypeManifest
		f.num = num
		return true
	}

	return false
}
<commit_msg>storage: Supposed to be Sprintf<commit_after>// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
// All rights reservefs.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package storage

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/syndtr/goleveldb/leveldb/util"
)

var errFileOpen = errors.New("leveldb/storage: file still open")

type fileLock interface {
	release() error
}

type fileStorageLock struct {
	fs *fileStorage
}

func (lock *fileStorageLock) Release() {
	fs := lock.fs
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if fs.slock == lock {
		fs.slock = nil
	}
	return
}

// fileStorage is a file-system backed storage.
type fileStorage struct {
	path string

	mu    sync.Mutex
	flock fileLock
	slock *fileStorageLock
	logw  *os.File
	buf   []byte
	// Opened file counter; if open < 0 means closed.
	open int
	day  int
}

// OpenFile returns a new filesytem-backed storage implementation with the given
// path. This also hold a file lock, so any subsequent attempt to open the same
// path will fail.
//
// The storage must be closed after use, by calling Close method.
func OpenFile(path string) (Storage, error) {
	if err := os.MkdirAll(path, 0755); err != nil {
		return nil, err
	}

	flock, err := newFileLock(filepath.Join(path, "LOCK"))
	if err != nil {
		return nil, err
	}

	defer func() {
		if err != nil {
			flock.release()
		}
	}()

	rename(filepath.Join(path, "LOG"), filepath.Join(path, "LOG.old"))
	logw, err := os.OpenFile(filepath.Join(path, "LOG"), os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		return nil, err
	}

	fs := &fileStorage{path: path, flock: flock, logw: logw}
	runtime.SetFinalizer(fs, (*fileStorage).Close)
	return fs, nil
}

func (fs *fileStorage) Lock() (util.Releaser, error) {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if fs.open < 0 {
		return nil, ErrClosed
	}
	if fs.slock != nil {
		return nil, ErrLocked
	}
	fs.slock = &fileStorageLock{fs: fs}
	return fs.slock, nil
}

func itoa(buf []byte, i int, wid int) []byte {
	var u uint = uint(i)
	if u == 0 && wid <= 1 {
		return append(buf, '0')
	}

	// Assemble decimal in reverse order.
	var b [32]byte
	bp := len(b)
	for ; u > 0 || wid > 0; u /= 10 {
		bp--
		wid--
		b[bp] = byte(u%10) + '0'
	}
	return append(buf, b[bp:]...)
}

func (fs *fileStorage) printDay(t time.Time) {
	if fs.day == t.Day() {
		return
	}
	fs.day = t.Day()
	fs.logw.Write([]byte("=============== " + t.Format("Jan 2, 2006 (MST)") + " ===============\n"))
}

func (fs *fileStorage) doLog(t time.Time, str string) {
	fs.printDay(t)
	hour, min, sec := t.Clock()
	msec := t.Nanosecond() / 1e3
	// time
	fs.buf = itoa(fs.buf[:0], hour, 2)
	fs.buf = append(fs.buf, ':')
	fs.buf = itoa(fs.buf, min, 2)
	fs.buf = append(fs.buf, ':')
	fs.buf = itoa(fs.buf, sec, 2)
	fs.buf = append(fs.buf, '.')
	fs.buf = itoa(fs.buf, msec, 6)
	fs.buf = append(fs.buf, ' ')
	// write
	fs.buf = append(fs.buf, []byte(str)...)
	fs.buf = append(fs.buf, '\n')
	fs.logw.Write(fs.buf)
}

func (fs *fileStorage) Log(str string) {
	t := time.Now()
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if fs.open < 0 {
		return
	}
	fs.doLog(t, str)
}

func (fs *fileStorage) log(str string) {
	fs.doLog(time.Now(), str)
}

func (fs *fileStorage) GetFile(num uint64, t FileType) File {
	return &file{fs: fs, num: num, t: t}
}

func (fs *fileStorage) GetFiles(t FileType) (ff []File, err error) {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if fs.open < 0 {
		return nil, ErrClosed
	}
	dir, err := os.Open(fs.path)
	if err != nil {
		return
	}
	fnn, err := dir.Readdirnames(0)
	// Close the dir first before checking for Readdirnames error.
	if err := dir.Close(); err != nil {
		fs.log(fmt.Sprintf("close dir: %v", err))
	}
	if err != nil {
		return
	}
	f := &file{fs: fs}
	for _, fn := range fnn {
		if f.parse(fn) && (f.t&t) != 0 {
			ff = append(ff, f)
			f = &file{fs: fs}
		}
	}
	return
}

func (fs *fileStorage) GetManifest() (f File, err error) {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if fs.open < 0 {
		return nil, ErrClosed
	}
	dir, err := os.Open(fs.path)
	if err != nil {
		return
	}
	fnn, err := dir.Readdirnames(0)
	// Close the dir first before checking for Readdirnames error.
	if err := dir.Close(); err != nil {
		fs.log(fmt.Sprintf("close dir: %v", err))
	}
	if err != nil {
		return
	}
	// Find latest CURRENT file.
	var rem []string
	var pend bool
	var cerr error
	for _, fn := range fnn {
		if strings.HasPrefix(fn, "CURRENT") {
			pend1 := len(fn) > 7
			// Make sure it is valid name for a CURRENT file, otherwise skip it.
			if pend1 {
				if fn[7] != '.' || len(fn) < 9 {
					fs.log(fmt.Sprintf("skipping %s: invalid file name", fn))
					continue
				}
				if _, e1 := strconv.ParseUint(fn[7:], 10, 0); e1 != nil {
					fs.log(fmt.Sprintf("skipping %s: invalid file num: %v", fn, e1))
					continue
				}
			}
			path := filepath.Join(fs.path, fn)
			r, e1 := os.OpenFile(path, os.O_RDONLY, 0)
			if e1 != nil {
				return nil, e1
			}
			b, e1 := ioutil.ReadAll(r)
			if e1 != nil {
				r.Close()
				return nil, e1
			}
			f1 := &file{fs: fs}
			if len(b) < 1 || b[len(b)-1] != '\n' || !f1.parse(string(b[:len(b)-1])) {
				fs.log(fmt.Sprintf("skipping %s: corrupted or incomplete", fn))
				if pend1 {
					rem = append(rem, fn)
				}
				if !pend1 || cerr == nil {
					cerr = fmt.Errorf("leveldb/storage: corrupted or incomplete %s file", fn)
				}
			} else if f != nil && f1.Num() < f.Num() {
				fs.log(fmt.Sprintf("skipping %s: obsolete", fn))
				if pend1 {
					rem = append(rem, fn)
				}
			} else {
				f = f1
				pend = pend1
			}
			if err := r.Close(); err != nil {
				fs.log(fmt.Sprintf("close %s: %v", fn, err))
			}
		}
	}
	// Don't remove any files if there is no valid CURRENT file.
	if f == nil {
		if cerr != nil {
			err = cerr
		} else {
			err = os.ErrNotExist
		}
		return
	}
	// Rename pending CURRENT file to an effective CURRENT.
	if pend {
		path := fmt.Sprintf("%s.%d", filepath.Join(fs.path, "CURRENT"), f.Num())
		if err := rename(path, filepath.Join(fs.path, "CURRENT")); err != nil {
			fs.log(fmt.Sprintf("CURRENT.%d -> CURRENT: %d", f.Num(), err))
		}
	}
	// Remove obsolete or incomplete pending CURRENT files.
	for _, fn := range rem {
		path := filepath.Join(fs.path, fn)
		if err := os.Remove(path); err != nil {
			fs.log(fmt.Sprintf("remove %s: %v", fn, err))
		}
	}
	return
}

func (fs *fileStorage) SetManifest(f File) (err error) {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if fs.open < 0 {
		return ErrClosed
	}
	f2, ok := f.(*file)
	if !ok || f2.t != TypeManifest {
		return ErrInvalidFile
	}
	defer func() {
		if err != nil {
			fs.log(fmt.Sprintf("CURRENT: %v", err))
		}
	}()
	path := fmt.Sprintf("%s.%d", filepath.Join(fs.path, "CURRENT"), f2.Num())
	w, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		return err
	}
	_, err = fmt.Fprintln(w, f2.name())
	// Close the file first.
	if err := w.Close(); err != nil {
		fs.log(fmt.Sprintf("close CURRENT.%d: %v", f2.num, err))
	}
	if err != nil {
		return err
	}
	err = rename(path, filepath.Join(fs.path, "CURRENT"))
	return
}

func (fs *fileStorage) Close() error {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if fs.open < 0 {
		return ErrClosed
	}
	// Clear the finalizer.
	runtime.SetFinalizer(fs, nil)

	if fs.open > 0 {
		fs.log(fmt.Sprintf("refuse to close, %d files still open", fs.open))
		return fmt.Errorf("leveldb/storage: cannot close, %d files still open", fs.open)
	}
	fs.open = -1
	e1 := fs.logw.Close()
	err := fs.flock.release()
	if err == nil {
		err = e1
	}
	return err
}

type fileWrap struct {
	*os.File
	f *file
}

func (fw fileWrap) Close() error {
	f := fw.f
	f.fs.mu.Lock()
	defer f.fs.mu.Unlock()
	if !f.open {
		return ErrClosed
	}
	f.open = false
	f.fs.open--
	err := fw.File.Close()
	if err != nil {
		f.fs.log(fmt.Sprintf("close %s.%d: %v", f.Type(), f.Num(), err))
	}
	return err
}

type file struct {
	fs   *fileStorage
	num  uint64
	t    FileType
	open bool
}

func (f *file) Open() (Reader, error) {
	f.fs.mu.Lock()
	defer f.fs.mu.Unlock()
	if f.fs.open < 0 {
		return nil, ErrClosed
	}
	if f.open {
		return nil, errFileOpen
	}
	of, err := os.OpenFile(f.path(), os.O_RDONLY, 0)
	if err != nil {
		return nil, err
	}
	f.open = true
	f.fs.open++
	return fileWrap{of, f}, nil
}

func (f *file) Create() (Writer, error) {
	f.fs.mu.Lock()
	defer f.fs.mu.Unlock()
	if f.fs.open < 0 {
		return nil, ErrClosed
	}
	if f.open {
		return nil, errFileOpen
	}
	of, err := os.OpenFile(f.path(), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		return nil, err
	}
	f.open = true
	f.fs.open++
	return fileWrap{of, f}, nil
}

func (f *file) Type() FileType {
	return f.t
}

func (f *file) Num() uint64 {
	return f.num
}

func (f *file) Remove() error {
	f.fs.mu.Lock()
	defer f.fs.mu.Unlock()
	if f.fs.open < 0 {
		return ErrClosed
	}
	if f.open {
		return errFileOpen
	}
	err := os.Remove(f.path())
	if err != nil {
		f.fs.log(fmt.Sprint("remove %s.%d: %v", f.Type(), f.Num(), err))
	}
	return err
}

func (f *file) name() string {
	switch f.t {
	case TypeManifest:
		return fmt.Sprintf("MANIFEST-%06d", f.num)
	case TypeJournal:
		return fmt.Sprintf("%06d.log", f.num)
	case TypeTable:
		return fmt.Sprintf("%06d.sst", f.num)
	default:
		panic("invalid file type")
	}
	return ""
}

func (f *file) path() string {
	return filepath.Join(f.fs.path, f.name())
}

func (f *file) parse(name string) bool {
	var num uint64
	var tail string
	_, err := fmt.Sscanf(name, "%d.%s", &num, &tail)
	if err == nil {
		switch tail {
		case "log":
			f.t = TypeJournal
		case "sst":
			f.t = TypeTable
		default:
			return false
		}
		f.num = num
		return true
	}
	n, _ := fmt.Sscanf(name, "MANIFEST-%d%s", &num, &tail)
	if n == 1 {
		f.t = TypeManifest
		f.num = num
		return true
	}

	return false
}
<|endoftext|>


<commit_before>package levenshtein_test

import (
	"github.com/texttheater/golang-levenshtein/levenshtein"
	"testing"
)

var testCases = []struct {
	source   string
	target   string
	distance int
}{
	{"", "a", 1},
	{"a", "aa", 1},
	{"a", "aaa", 2},
	{"", "", 0},
	{"a", "b", 2},
	{"aaa", "aba", 2},
	{"aaa", "ab", 3},
	{"a", "a", 0},
	{"ab", "ab", 0},
	{"a", "", 1},
	{"aa", "a", 1},
	{"aaa", "a", 2},
}

func TestLevenshtein(t *testing.T) {
	for _, testCase := range testCases {
		distance := levenshtein.DistanceForStrings(
			[]rune(testCase.source),
			[]rune(testCase.target),
			levenshtein.DefaultOptions)
		if distance != testCase.distance {
			t.Log(
				"Distance between",
				testCase.source,
				"and",
				testCase.target,
				"computed as",
				distance,
				", should be",
				testCase.distance)
			t.Fail()
		}
	}
}
<commit_msg>fixed the test to use the local package (as it should be); added a testable example for godoc<commit_after>package levenshtein

import (
	"fmt"
	"testing"
)

var testCases = []struct {
	source   string
	target   string
	distance int
}{
	{"", "a", 1},
	{"a", "aa", 1},
	{"a", "aaa", 2},
	{"", "", 0},
	{"a", "b", 2},
	{"aaa", "aba", 2},
	{"aaa", "ab", 3},
	{"a", "a", 0},
	{"ab", "ab", 0},
	{"a", "", 1},
	{"aa", "a", 1},
	{"aaa", "a", 2},
}

func TestDistanceForStrings(t *testing.T) {
	for _, testCase := range testCases {
		distance := DistanceForStrings(
			[]rune(testCase.source),
			[]rune(testCase.target),
			DefaultOptions)
		if distance != testCase.distance {
			t.Log(
				"Distance between",
				testCase.source,
				"and",
				testCase.target,
				"computed as",
				distance,
				", should be",
				testCase.distance)
			t.Fail()
		}
	}
}

func ExampleDistanceForStrings() {
	source := "a"
	target := "aa"
	distance := DistanceForStrings([]rune(source), []rune(target), DefaultOptions)
	fmt.Printf(`Distance between "%s" and "%s" computed as %d`, source, target, distance)
	// Output: Distance between "a" and "aa" computed as 1
}
<|endoftext|>


<commit_before><commit_msg>higher maxTextBoundsToImageRatioY and maxTextBoundsToImageRatioX<commit_after><|endoftext|>


<commit_before>// Copyright © 2015-2017 Hilko Bengen <bengen@hilluzination.de>
// All rights reserved.
//
// Use of this source code is governed by the license that can be
// found in the LICENSE file.

package yara

/*
#ifdef _WIN32
#define fdopen _fdopen
#define dup _dup
#endif
#include <stdio.h>
#include <unistd.h>

#include <yara.h>

// This signature should be generated by cgo from the exported
// function below
void compilerCallback(int, char*, int, char*, void*);
*/
import "C"
import (
	"errors"
	"os"
	"runtime"
	"unsafe"
)

//export compilerCallback
func compilerCallback(errorLevel C.int, filename *C.char, linenumber C.int, message *C.char, userData unsafe.Pointer) {
	c := callbackData.Get(uintptr(userData)).(*Compiler)
	msg := CompilerMessage{
		Filename: C.GoString(filename),
		Line:     int(linenumber),
		Text:     C.GoString(message),
	}
	switch errorLevel {
	case C.YARA_ERROR_LEVEL_ERROR:
		c.Errors = append(c.Errors, msg)
	case C.YARA_ERROR_LEVEL_WARNING:
		c.Warnings = append(c.Warnings, msg)
	}
}

// A Compiler encapsulates the YARA compiler that transforms rules
// into YARA's internal, binary form which in turn is used for
// scanning files or memory blocks.
type Compiler struct {
	*compiler
	Errors   []CompilerMessage
	Warnings []CompilerMessage
}

type compiler struct {
	cptr *C.YR_COMPILER
}

// A CompilerMessage contains an error or warning message produced
// while compiling sets of rules using AddString or AddFile.
type CompilerMessage struct {
	Filename string
	Line     int
	Text     string
}

// NewCompiler creates a YARA compiler.
func NewCompiler() (*Compiler, error) {
	var yrCompiler *C.YR_COMPILER
	if err := newError(C.yr_compiler_create(&yrCompiler)); err != nil {
		return nil, err
	}
	c := &Compiler{compiler: &compiler{cptr: yrCompiler}}
	runtime.SetFinalizer(c.compiler, (*compiler).finalize)
	return c, nil
}

func (c *compiler) finalize() {
	C.yr_compiler_destroy(c.cptr)
	runtime.SetFinalizer(c, nil)
}

// Destroy destroys the YARA data structure representing a compiler.
// Since a Finalizer for the underlying YR_COMPILER structure is
// automatically set up on creation, it should not be necessary to
// explicitly call this method.
func (c *Compiler) Destroy() {
	if c.compiler != nil {
		c.compiler.finalize()
		c.compiler = nil
	}
}

// AddFile compiles rules from a file. Rules are added to the
// specified namespace.
func (c *Compiler) AddFile(file *os.File, namespace string) (err error) {
	fd := C.dup(C.int(file.Fd()))
	fh, err := C.fdopen(fd, C.CString("r"))
	if err != nil {
		return err
	}
	defer C.fclose(fh)
	var ns *C.char
	if namespace != "" {
		ns = C.CString(namespace)
		defer C.free(unsafe.Pointer(ns))
	}
	filename := C.CString(file.Name())
	defer C.free(unsafe.Pointer(filename))
	id := callbackData.Put(c)
	defer callbackData.Delete(id)
	C.yr_compiler_set_callback(c.cptr, C.YR_COMPILER_CALLBACK_FUNC(C.compilerCallback), unsafe.Pointer(id))
	numErrors := int(C.yr_compiler_add_file(c.cptr, fh, ns, filename))
	if numErrors > 0 {
		var buf [1024]C.char
		msg := C.GoString(C.yr_compiler_get_error_message(
			c.cptr, (*C.char)(unsafe.Pointer(&buf[0])), 1024))
		err = errors.New(msg)
	}
	return
}

// AddString compiles rules from a string. Rules are added to the
// specified namespace.
func (c *Compiler) AddString(rules string, namespace string) (err error) {
	var ns *C.char
	if namespace != "" {
		ns = C.CString(namespace)
		defer C.free(unsafe.Pointer(ns))
	}
	crules := C.CString(rules)
	defer C.free(unsafe.Pointer(crules))
	id := callbackData.Put(c)
	defer callbackData.Delete(id)
	C.yr_compiler_set_callback(c.cptr, C.YR_COMPILER_CALLBACK_FUNC(C.compilerCallback), unsafe.Pointer(id))
	numErrors := int(C.yr_compiler_add_string(c.cptr, crules, ns))
	if numErrors > 0 {
		var buf [1024]C.char
		msg := C.GoString(C.yr_compiler_get_error_message(
			c.cptr, (*C.char)(unsafe.Pointer(&buf[0])), 1024))
		err = errors.New(msg)
	}
	return
}

// DefineVariable defines a named variable for use by the compiler.
// Boolean, int64, float64, and string types are supported.
func (c *Compiler) DefineVariable(name string, value interface{}) (err error) {
	cname := C.CString(name)
	defer C.free(unsafe.Pointer(cname))
	switch value.(type) {
	case bool:
		var v int
		if value.(bool) {
			v = 1
		}
		err = newError(C.yr_compiler_define_boolean_variable(
			c.cptr, cname, C.int(v)))
	case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64:
		value := toint64(value)
		err = newError(C.yr_compiler_define_integer_variable(
			c.cptr, cname, C.int64_t(value)))
	case float64:
		err = newError(C.yr_compiler_define_float_variable(
			c.cptr, cname, C.double(value.(float64))))
	case string:
		cvalue := C.CString(value.(string))
		defer C.free(unsafe.Pointer(cvalue))
		err = newError(C.yr_compiler_define_string_variable(
			c.cptr, cname, cvalue))
	default:
		err = errors.New("wrong value type passed to DefineVariable; bool, int64, float64, string are accepted")
	}
	return
}

// GetRules returns the compiled ruleset.
func (c *Compiler) GetRules() (*Rules, error) {
	var yrRules *C.YR_RULES
	if err := newError(C.yr_compiler_get_rules(c.cptr, &yrRules)); err != nil {
		return nil, err
	}
	r := &Rules{rules: &rules{cptr: yrRules}}
	runtime.SetFinalizer(r.rules, (*rules).finalize)
	return r, nil
}

// Compile compiles rules and an (optional) set of variables into a
// Rules object in a single step.
func Compile(rules string, variables map[string]interface{}) (r *Rules, err error) {
	var c *Compiler
	if c, err = NewCompiler(); err != nil {
		return
	}
	for k, v := range variables {
		if err = c.DefineVariable(k, v); err != nil {
			return
		}
	}
	if err = c.AddString(rules, ""); err != nil {
		return
	}
	r, err = c.GetRules()
	return
}

// MustCompile is like Compile but panics if the rules and optional
// variables can't be compiled. Like regexp.MustCompile, it allows for
// simple, safe initialization of global or test data.
func MustCompile(rules string, variables map[string]interface{}) (r *Rules) {
	r, err := Compile(rules, variables)
	if err != nil {
		panic(err)
	}
	return
}
<commit_msg>Explicitly destroy compiler used in function Compile()<commit_after>// Copyright © 2015-2017 Hilko Bengen <bengen@hilluzination.de>
// All rights reserved.
//
// Use of this source code is governed by the license that can be
// found in the LICENSE file.

package yara

/*
#ifdef _WIN32
#define fdopen _fdopen
#define dup _dup
#endif
#include <stdio.h>
#include <unistd.h>

#include <yara.h>

// This signature should be generated by cgo from the exported
// function below
void compilerCallback(int, char*, int, char*, void*);
*/
import "C"
import (
	"errors"
	"os"
	"runtime"
	"unsafe"
)

//export compilerCallback
func compilerCallback(errorLevel C.int, filename *C.char, linenumber C.int, message *C.char, userData unsafe.Pointer) {
	c := callbackData.Get(uintptr(userData)).(*Compiler)
	msg := CompilerMessage{
		Filename: C.GoString(filename),
		Line:     int(linenumber),
		Text:     C.GoString(message),
	}
	switch errorLevel {
	case C.YARA_ERROR_LEVEL_ERROR:
		c.Errors = append(c.Errors, msg)
	case C.YARA_ERROR_LEVEL_WARNING:
		c.Warnings = append(c.Warnings, msg)
	}
}

// A Compiler encapsulates the YARA compiler that transforms rules
// into YARA's internal, binary form which in turn is used for
// scanning files or memory blocks.
type Compiler struct {
	*compiler
	Errors   []CompilerMessage
	Warnings []CompilerMessage
}

type compiler struct {
	cptr *C.YR_COMPILER
}

// A CompilerMessage contains an error or warning message produced
// while compiling sets of rules using AddString or AddFile.
type CompilerMessage struct {
	Filename string
	Line     int
	Text     string
}

// NewCompiler creates a YARA compiler.
func NewCompiler() (*Compiler, error) {
	var yrCompiler *C.YR_COMPILER
	if err := newError(C.yr_compiler_create(&yrCompiler)); err != nil {
		return nil, err
	}
	c := &Compiler{compiler: &compiler{cptr: yrCompiler}}
	runtime.SetFinalizer(c.compiler, (*compiler).finalize)
	return c, nil
}

func (c *compiler) finalize() {
	C.yr_compiler_destroy(c.cptr)
	runtime.SetFinalizer(c, nil)
}

// Destroy destroys the YARA data structure representing a compiler.
// Since a Finalizer for the underlying YR_COMPILER structure is
// automatically set up on creation, it should not be necessary to
// explicitly call this method.
func (c *Compiler) Destroy() {
	if c.compiler != nil {
		c.compiler.finalize()
		c.compiler = nil
	}
}

// AddFile compiles rules from a file. Rules are added to the
// specified namespace.
func (c *Compiler) AddFile(file *os.File, namespace string) (err error) {
	fd := C.dup(C.int(file.Fd()))
	fh, err := C.fdopen(fd, C.CString("r"))
	if err != nil {
		return err
	}
	defer C.fclose(fh)
	var ns *C.char
	if namespace != "" {
		ns = C.CString(namespace)
		defer C.free(unsafe.Pointer(ns))
	}
	filename := C.CString(file.Name())
	defer C.free(unsafe.Pointer(filename))
	id := callbackData.Put(c)
	defer callbackData.Delete(id)
	C.yr_compiler_set_callback(c.cptr, C.YR_COMPILER_CALLBACK_FUNC(C.compilerCallback), unsafe.Pointer(id))
	numErrors := int(C.yr_compiler_add_file(c.cptr, fh, ns, filename))
	if numErrors > 0 {
		var buf [1024]C.char
		msg := C.GoString(C.yr_compiler_get_error_message(
			c.cptr, (*C.char)(unsafe.Pointer(&buf[0])), 1024))
		err = errors.New(msg)
	}
	return
}

// AddString compiles rules from a string. Rules are added to the
// specified namespace.
func (c *Compiler) AddString(rules string, namespace string) (err error) {
	var ns *C.char
	if namespace != "" {
		ns = C.CString(namespace)
		defer C.free(unsafe.Pointer(ns))
	}
	crules := C.CString(rules)
	defer C.free(unsafe.Pointer(crules))
	id := callbackData.Put(c)
	defer callbackData.Delete(id)
	C.yr_compiler_set_callback(c.cptr, C.YR_COMPILER_CALLBACK_FUNC(C.compilerCallback), unsafe.Pointer(id))
	numErrors := int(C.yr_compiler_add_string(c.cptr, crules, ns))
	if numErrors > 0 {
		var buf [1024]C.char
		msg := C.GoString(C.yr_compiler_get_error_message(
			c.cptr, (*C.char)(unsafe.Pointer(&buf[0])), 1024))
		err = errors.New(msg)
	}
	return
}

// DefineVariable defines a named variable for use by the compiler.
// Boolean, int64, float64, and string types are supported.
func (c *Compiler) DefineVariable(name string, value interface{}) (err error) {
	cname := C.CString(name)
	defer C.free(unsafe.Pointer(cname))
	switch value.(type) {
	case bool:
		var v int
		if value.(bool) {
			v = 1
		}
		err = newError(C.yr_compiler_define_boolean_variable(
			c.cptr, cname, C.int(v)))
	case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64:
		value := toint64(value)
		err = newError(C.yr_compiler_define_integer_variable(
			c.cptr, cname, C.int64_t(value)))
	case float64:
		err = newError(C.yr_compiler_define_float_variable(
			c.cptr, cname, C.double(value.(float64))))
	case string:
		cvalue := C.CString(value.(string))
		defer C.free(unsafe.Pointer(cvalue))
		err = newError(C.yr_compiler_define_string_variable(
			c.cptr, cname, cvalue))
	default:
		err = errors.New("wrong value type passed to DefineVariable; bool, int64, float64, string are accepted")
	}
	return
}

// GetRules returns the compiled ruleset.
func (c *Compiler) GetRules() (*Rules, error) {
	var yrRules *C.YR_RULES
	if err := newError(C.yr_compiler_get_rules(c.cptr, &yrRules)); err != nil {
		return nil, err
	}
	r := &Rules{rules: &rules{cptr: yrRules}}
	runtime.SetFinalizer(r.rules, (*rules).finalize)
	return r, nil
}

// Compile compiles rules and an (optional) set of variables into a
// Rules object in a single step.
func Compile(rules string, variables map[string]interface{}) (r *Rules, err error) {
	var c *Compiler
	if c, err = NewCompiler(); err != nil {
		return
	}
	defer c.Destroy()
	for k, v := range variables {
		if err = c.DefineVariable(k, v); err != nil {
			return
		}
	}
	if err = c.AddString(rules, ""); err != nil {
		return
	}
	r, err = c.GetRules()
	return
}

// MustCompile is like Compile but panics if the rules and optional
// variables can't be compiled. Like regexp.MustCompile, it allows for
// simple, safe initialization of global or test data.
func MustCompile(rules string, variables map[string]interface{}) (r *Rules) {
	r, err := Compile(rules, variables)
	if err != nil {
		panic(err)
	}
	return
}
<|endoftext|>