<commit_before>package disk

import (
	"encoding/binary"
	"errors"
	"hash/crc32"
	"io"
	"os"
)

var (
	crc32cTable = crc32.MakeTable(crc32.Castagnoli)
	crc32Len    = 4
	blockSize   = 4096
	payloadSize = blockSize - crc32Len

	// ErrPayloadSizeTooLarge indicates the input payload size is too big
	ErrPayloadSizeTooLarge = errors.New("disk: bad payload size")
	// ErrBadCRC indicates there is not CRC can be found in the block
	ErrBadCRC = errors.New("disk: not a valid CRC")
)

// Block is a buffer aligned with disk data block with two offset (left and right)
// representing the start and end of the effective payload inside the buffer
type Block struct{
	buf []byte
	left int
	right int
}

// IsEmpty tells if the buffer's effective payload is empty
func (b *Block) IsEmpty() bool {
	return b.left == b.right
}

// GetPayload returns the effective payload as a byte array
func (b *Block) GetPayload() []byte {
	return b.buf[b.left:b.right]
}

// Copy copies a byte array to the offset of the buffer
// It also updates the effective payload offsets
func (b *Block) Copy(offset int, from []byte) int {
	copied := copy(b.buf[offset:], from)
	b.StartFrom(min(b.left, offset))
	b.EndAt(max(b.right, offset+copied))
	return copied
}

// StartFrom sets the block's left offset
func (b *Block) StartFrom(offset int) {
	b.left = offset
}

// EndAt sets the block's right offset
func (b *Block) EndAt(offset int) {
	b.right = offset
}

// GetCRC calculates the crc of the effective payload
func (b *Block) GetCRC() uint32 {
	return crc32.Checksum(b.buf[b.left:b.right], crc32cTable)
}

// IsPartial checks if effective payload is a partial payload
func (b *Block) IsPartial() bool {
	return !(b.left == 0 && b.right == payloadSize)
}

// Merge uses current block as a base and
// merges another block on top of it
func (b *Block) Merge(toMerge *Block) {
	b.StartFrom(min(b.left, toMerge.left))
	b.EndAt(max(b.right, toMerge.right))
	copy(b.buf[toMerge.left:toMerge.right], toMerge.GetPayload())
}

// Reset resets the effective payload to 0
func (b *Block) Reset() {
	b.StartFrom(0)
	b.EndAt(0)
}

func newBlock() *Block {
	return &Block{make([]byte, payloadSize), 0, payloadSize}
}

func seekToIndex(s io.Seeker, index int) error {
	_, err := s.Seek(int64(index * blockSize), os.SEEK_SET)
	return err
}

func seekToOffset(s io.Seeker, offset int) error {
	_, err := s.Seek(int64(offset), os.SEEK_CUR)
	return err
}

func blockIndexAndOffset(dataOffset int) (int, int) {
	index := dataOffset / payloadSize
	offset := dataOffset - index*payloadSize
	return index, offset
}

func readBlock(f io.ReadSeeker, b *Block, index int) error {
	b.Reset()
	crcBuf := make([]byte, crc32Len)
	if err := seekToIndex(f, index); err != nil {
		return err
	}
	n, err := f.Read(crcBuf)
	// Cannot read full crc
	if n > 0 && n < 4 {
		return ErrBadCRC
	}
	if err != nil {
		return err
	}
	crc := binary.BigEndian.Uint32(crcBuf)
	v, err := f.Read(b.buf)
	b.EndAt(v)
	if err != nil {
		return err
	}
	// Invalid crc
	if crc != b.GetCRC() {
		return ErrBadCRC
	}
	return nil
}

func writeBlock(f io.WriteSeeker, b *Block, index int) error {
	if b.right > payloadSize {
		return ErrPayloadSizeTooLarge
	}
	if err := seekToIndex(f, index); err != nil {
		return err
	}

	crcBuf := make([]byte, crc32Len)
	binary.BigEndian.PutUint32(crcBuf, b.GetCRC())
	_, err := f.Write(crcBuf)
	if err != nil {
		return err
	}
	if err := seekToOffset(f, b.left); err != nil {
		return err
	}
	_, err = f.Write(b.GetPayload())
	if err != nil {
		return err
	}
	return nil
}

func min(a, b int) int {
	if a > b {
		return b
	}
	return a
}

func max(a, b int) int {
	if a > b {
		return a
	}
	return b
}
<commit_msg>Make requested changes<commit_after>package disk

import (
	"encoding/binary"
	"errors"
	"hash/crc32"
	"io"
	"os"
)

var (
	crc32cTable = crc32.MakeTable(crc32.Castagnoli)
	crc32Len    = 4
	blockSize   = 4096
	payloadSize = blockSize - crc32Len

	// ErrPayloadSizeTooLarge indicates the input payload size is too big
	ErrPayloadSizeTooLarge = errors.New("disk: bad payload size")
	// ErrBadCRC indicates there is not CRC can be found in the block
	ErrBadCRC = errors.New("disk: not a valid CRC")
)

// Block is a buffer aligned with disk data block with two offset (left and right)
// representing the start and end of the effective payload inside the buffer
type Block struct{
	buf []byte
	left int
	right int
}

// IsEmpty tells if the buffer's effective payload is empty
func (b *Block) IsEmpty() bool {
	return b.left == b.right
}

// GetPayload returns the effective payload as a byte array
func (b *Block) GetPayload() []byte {
	return b.buf[b.left:b.right]
}

// Copy copies a byte array to the offset of the buffer
// It also updates the effective payload offsets
func (b *Block) Copy(offset int, from []byte) int {
	copied := copy(b.buf[offset:], from)
	b.StartFrom(min(b.left, offset))
	b.EndAt(max(b.right, offset+copied))
	return copied
}

// StartFrom sets the block's left offset
func (b *Block) StartFrom(offset int) {
	b.left = offset
}

// EndAt sets the block's right offset
func (b *Block) EndAt(offset int) {
	b.right = offset
}

// GetCRC calculates the crc of the effective payload
func (b *Block) GetCRC() uint32 {
	return crc32.Checksum(b.buf[b.left:b.right], crc32cTable)
}

// IsPartial checks if effective payload is a partial payload
func (b *Block) IsPartial() bool {
	return !(b.left == 0 && b.right == payloadSize)
}

// Merge uses current block as a base and
// merges another block on top of it
func (b *Block) Merge(toMerge *Block) {
	b.StartFrom(min(b.left, toMerge.left))
	b.EndAt(max(b.right, toMerge.right))
	copy(b.buf[toMerge.left:toMerge.right], toMerge.GetPayload())
}

// Reset resets the effective payload to 0
func (b *Block) Reset() {
	b.StartFrom(0)
	b.EndAt(0)
}

func newBlock() *Block {
	return &Block{make([]byte, payloadSize), 0, payloadSize}
}

func seekToIndex(s io.Seeker, index int) error {
	_, err := s.Seek(int64(index * blockSize), os.SEEK_SET)
	return err
}

func seekToOffset(s io.Seeker, offset int) error {
	_, err := s.Seek(int64(offset), os.SEEK_CUR)
	return err
}

func blockIndexAndOffset(dataOffset int) (int, int) {
	index := dataOffset / payloadSize
	offset := dataOffset - index*payloadSize
	return index, offset
}

func readBlock(f io.ReadSeeker, b *Block, index int) error {
	b.Reset()
	crcBuf := make([]byte, crc32Len)
	if err := seekToIndex(f, index); err != nil {
		return err
	}
	n, err := f.Read(crcBuf)
	// Cannot read full crc
	if n > 0 && n < 4 {
		return ErrBadCRC
	}
	if err != nil {
		return err
	}
	crc := binary.BigEndian.Uint32(crcBuf)
	v, err := f.Read(b.buf)
	b.EndAt(v)
	if err != nil {
		return err
	}
	// Invalid crc
	if crc != b.GetCRC() {
		return ErrBadCRC
	}
	return nil
}

func writeBlock(f io.WriteSeeker, b *Block, index int) error {
	if b.right > payloadSize {
		return ErrPayloadSizeTooLarge
	}
	if err := seekToIndex(f, index); err != nil {
		return err
	}

	crcBuf := make([]byte, crc32Len + b.left + len(b.GetPayload()))
	binary.BigEndian.PutUint32(crcBuf, b.GetCRC())
	// _, err := f.Write(crcBuf)
	// if err != nil {
	// 	return err
	// }
	// if err := seekToOffset(f, b.left); err != nil {
	// 	return err
	// }
	copy(crcBuf[b.left:], b.GetPayload())
	_, err := f.Write(crcBuf)
	if err != nil {
		return err
	}
	return nil
}

func min(a, b int) int {
	if a > b {
		return b
	}
	return a
}

func max(a, b int) int {
	if a > b {
		return a
	}
	return b
}
<|endoftext|>


<commit_before>package dbTest

import (
	"testing"

	"github.com/Everlag/poeitemstore/db"
)

// BenchmarkAddStashes determines how fast adding a single
// response to the database is.
func BenchmarkAddStashes(b *testing.B) {

	// Fetch the changes we need
	set := GetChangeSet("testSet - 11 updates.msgp", b)
	change, err := set.GetFirstChange()
	if err != nil {
		b.Fatalf("failed to get first change")
	}

	resp, err := change.Decompress()
	if err != nil {
		b.Fatalf("failed to decompress response")
	}

	b.ReportAllocs()
	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		// Create a database, we want to exclude
		// the time this takes from the benchmark
		b.StopTimer()
		bdb := NewTempDatabase(b)
		b.StartTimer()

		cStashes, cItems, err := db.StashStashToCompact(resp.Stashes, bdb)
		if err != nil {
			b.Fatalf("failed to convert fat stashes to compact, err=%s\n", err)
		}

		_, err = db.AddStashes(cStashes, cItems, bdb)
		if err != nil {
			b.Fatalf("failed to AddStashes, err=%s", err)
		}
	}

}
<commit_msg>dbTest Benchmark AddStashes split into with and without StringHeap<commit_after>package dbTest

import (
	"testing"

	"github.com/Everlag/poeitemstore/db"
)

// BenchmarkAddStashesNoStringHeap determines how fast adding a single
// response to the database is with an uninitialized StringHeap.
func BenchmarkAddStashesNoStringHeapFast(b *testing.B) {

	// Fetch the changes we need
	set := GetChangeSet("testSet - 11 updates.msgp", b)
	change, err := set.GetFirstChange()
	if err != nil {
		b.Fatalf("failed to get first change")
	}

	resp, err := change.Decompress()
	if err != nil {
		b.Fatalf("failed to decompress response")
	}

	b.ReportAllocs()
	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		// Create a database, we want to exclude
		// the time this takes from the benchmark
		b.StopTimer()
		bdb := NewTempDatabase(b)
		b.StartTimer()

		cStashes, cItems, err := db.StashStashToCompact(resp.Stashes, bdb)
		if err != nil {
			b.Fatalf("failed to convert fat stashes to compact, err=%s\n", err)
		}

		_, err = db.AddStashes(cStashes, cItems, bdb)
		if err != nil {
			b.Fatalf("failed to AddStashes, err=%s", err)
		}
	}

}

// BenchmarkAddStashes determines how fast adding a single
// response to the database is with an already initialized StringHeap
func BenchmarkAddStashesFast(b *testing.B) {

	// Fetch the changes we need
	set := GetChangeSet("testSet - 11 updates.msgp", b)
	change, err := set.GetFirstChange()
	if err != nil {
		b.Fatalf("failed to get first change")
	}

	resp, err := change.Decompress()
	if err != nil {
		b.Fatalf("failed to decompress response")
	}

	b.ReportAllocs()
	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		// Create a database and populate StringHeap, we want to exclude
		// the time this takes from the benchmark
		b.StopTimer()
		bdb := NewTempDatabase(b)
		_, _, err := db.StashStashToCompact(resp.Stashes, bdb)
		if err != nil {
			b.Fatalf("failed to convert fat stashes to compact, err=%s\n", err)
		}
		b.StartTimer()

		cStashes, cItems, err := db.StashStashToCompact(resp.Stashes, bdb)
		if err != nil {
			b.Fatalf("failed to convert fat stashes to compact, err=%s\n", err)
		}

		_, err = db.AddStashes(cStashes, cItems, bdb)
		if err != nil {
			b.Fatalf("failed to AddStashes, err=%s", err)
		}
	}

}
<|endoftext|>


<commit_before>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Spencer Kimball (spencer.kimball@gmail.com)

package kv

import (
	"net"
	"time"

	"golang.org/x/net/context"

	"github.com/cockroachdb/cockroach/client"
	"github.com/cockroachdb/cockroach/gossip"
	"github.com/cockroachdb/cockroach/roachpb"
	"github.com/cockroachdb/cockroach/rpc"
	"github.com/cockroachdb/cockroach/storage"
	"github.com/cockroachdb/cockroach/storage/engine"
	"github.com/cockroachdb/cockroach/testutils"
	"github.com/cockroachdb/cockroach/util"
	"github.com/cockroachdb/cockroach/util/hlc"
	"github.com/cockroachdb/cockroach/util/stop"
	"github.com/gogo/protobuf/proto"
)

// A LocalTestCluster encapsulates an in-memory instantiation of a
// cockroach node with a single store using a local sender. Example
// usage of a LocalTestCluster follows:
//
//   s := &server.LocalTestCluster{}
//   s.Start(t)
//   defer s.Stop()
//
// Note that the LocalTestCluster is different from server.TestCluster
// in that although it uses a distributed sender, there is no RPC traffic.
type LocalTestCluster struct {
	Manual     *hlc.ManualClock
	Clock      *hlc.Clock
	Gossip     *gossip.Gossip
	Eng        engine.Engine
	Store      *storage.Store
	DB         *client.DB
	stores     *storage.Stores
	Sender     *TxnCoordSender
	distSender *DistSender
	Stopper    *stop.Stopper
	Latency    time.Duration // sleep for each RPC sent
	tester     util.Tester
}

// Start starts the test cluster by bootstrapping an in-memory store
// (defaults to maximum of 50M). The server is started, launching the
// node RPC server and all HTTP endpoints. Use the value of
// TestServer.Addr after Start() for client connections. Use Stop()
// to shutdown the server after the test completes.
func (ltc *LocalTestCluster) Start(t util.Tester) {

	nodeID := roachpb.NodeID(1)
	nodeDesc := &roachpb.NodeDescriptor{NodeID: nodeID}
	ltc.tester = t
	ltc.Manual = hlc.NewManualClock(0)
	ltc.Clock = hlc.NewClock(ltc.Manual.UnixNano)
	ltc.Stopper = stop.NewStopper()
	rpcContext := rpc.NewContext(testutils.NewNodeTestBaseContext(), ltc.Clock, ltc.Stopper)
	ltc.Gossip = gossip.New(rpcContext, gossip.TestBootstrap)
	ltc.Eng = engine.NewInMem(roachpb.Attributes{}, 50<<20, ltc.Stopper)

	ltc.stores = storage.NewStores()
	var rpcSend rpcSendFn = func(_ rpc.Options, _ string, _ []net.Addr,
		getArgs func(addr net.Addr) proto.Message, getReply func() proto.Message,
		_ *rpc.Context) ([]proto.Message, error) {
		// TODO(tschottdorf): remove getReply().
		if ltc.Latency > 0 {
			time.Sleep(ltc.Latency)
		}
		br, pErr := ltc.stores.Send(context.Background(), *getArgs(nil).(*roachpb.BatchRequest))
		if br == nil {
			br = &roachpb.BatchResponse{}
		}
		if br.Error != nil {
			panic(roachpb.ErrorUnexpectedlySet(ltc.stores, br))
		}
		br.Error = pErr
		return []proto.Message{br}, nil
	}
	ltc.distSender = NewDistSender(&DistSenderContext{
		Clock: ltc.Clock,
		RangeDescriptorCacheSize: defaultRangeDescriptorCacheSize,
		RangeLookupMaxRanges:     defaultRangeLookupMaxRanges,
		LeaderCacheSize:          defaultLeaderCacheSize,
		RPCRetryOptions:          &defaultRPCRetryOptions,
		nodeDescriptor:           nodeDesc,
		RPCSend:                  rpcSend,    // defined above
		RangeDescriptorDB:        ltc.stores, // for descriptor lookup
	}, ltc.Gossip)

	ltc.Sender = NewTxnCoordSender(ltc.distSender, ltc.Clock, false /* !linearizable */, nil /* tracer */, ltc.Stopper)
	ltc.DB = client.NewDB(ltc.Sender)

	transport := storage.NewLocalRPCTransport(ltc.Stopper)
	ltc.Stopper.AddCloser(transport)
	ctx := storage.TestStoreContext
	ctx.Clock = ltc.Clock
	ctx.DB = ltc.DB
	ctx.Gossip = ltc.Gossip
	ctx.Transport = transport
	ltc.Store = storage.NewStore(ctx, ltc.Eng, nodeDesc)
	if err := ltc.Store.Bootstrap(roachpb.StoreIdent{NodeID: nodeID, StoreID: 1}, ltc.Stopper); err != nil {
		t.Fatalf("unable to start local test cluster: %s", err)
	}
	ltc.stores.AddStore(ltc.Store)
	if err := ltc.Store.BootstrapRange(nil); err != nil {
		t.Fatalf("unable to start local test cluster: %s", err)
	}
	if err := ltc.Store.Start(ltc.Stopper); err != nil {
		t.Fatalf("unable to start local test cluster: %s", err)
	}
	ltc.Gossip.SetNodeID(nodeDesc.NodeID)
	if err := ltc.Gossip.SetNodeDescriptor(nodeDesc); err != nil {
		t.Fatalf("unable to set node descriptor: %s", err)
	}
}

// Stop stops the cluster.
func (ltc *LocalTestCluster) Stop() {
	if ltc.tester.Failed() {
		return
	}
	if r := recover(); r != nil {
		panic(r)
	}
	ltc.Stopper.Stop()
}
<commit_msg>LTC: attempt Stop when test fails<commit_after>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Spencer Kimball (spencer.kimball@gmail.com)

package kv

import (
	"net"
	"time"

	"golang.org/x/net/context"

	"github.com/cockroachdb/cockroach/client"
	"github.com/cockroachdb/cockroach/gossip"
	"github.com/cockroachdb/cockroach/roachpb"
	"github.com/cockroachdb/cockroach/rpc"
	"github.com/cockroachdb/cockroach/storage"
	"github.com/cockroachdb/cockroach/storage/engine"
	"github.com/cockroachdb/cockroach/testutils"
	"github.com/cockroachdb/cockroach/util"
	"github.com/cockroachdb/cockroach/util/hlc"
	"github.com/cockroachdb/cockroach/util/stop"
	"github.com/gogo/protobuf/proto"
)

// A LocalTestCluster encapsulates an in-memory instantiation of a
// cockroach node with a single store using a local sender. Example
// usage of a LocalTestCluster follows:
//
//   s := &server.LocalTestCluster{}
//   s.Start(t)
//   defer s.Stop()
//
// Note that the LocalTestCluster is different from server.TestCluster
// in that although it uses a distributed sender, there is no RPC traffic.
type LocalTestCluster struct {
	Manual     *hlc.ManualClock
	Clock      *hlc.Clock
	Gossip     *gossip.Gossip
	Eng        engine.Engine
	Store      *storage.Store
	DB         *client.DB
	stores     *storage.Stores
	Sender     *TxnCoordSender
	distSender *DistSender
	Stopper    *stop.Stopper
	Latency    time.Duration // sleep for each RPC sent
	tester     util.Tester
}

// Start starts the test cluster by bootstrapping an in-memory store
// (defaults to maximum of 50M). The server is started, launching the
// node RPC server and all HTTP endpoints. Use the value of
// TestServer.Addr after Start() for client connections. Use Stop()
// to shutdown the server after the test completes.
func (ltc *LocalTestCluster) Start(t util.Tester) {

	nodeID := roachpb.NodeID(1)
	nodeDesc := &roachpb.NodeDescriptor{NodeID: nodeID}
	ltc.tester = t
	ltc.Manual = hlc.NewManualClock(0)
	ltc.Clock = hlc.NewClock(ltc.Manual.UnixNano)
	ltc.Stopper = stop.NewStopper()
	rpcContext := rpc.NewContext(testutils.NewNodeTestBaseContext(), ltc.Clock, ltc.Stopper)
	ltc.Gossip = gossip.New(rpcContext, gossip.TestBootstrap)
	ltc.Eng = engine.NewInMem(roachpb.Attributes{}, 50<<20, ltc.Stopper)

	ltc.stores = storage.NewStores()
	var rpcSend rpcSendFn = func(_ rpc.Options, _ string, _ []net.Addr,
		getArgs func(addr net.Addr) proto.Message, getReply func() proto.Message,
		_ *rpc.Context) ([]proto.Message, error) {
		// TODO(tschottdorf): remove getReply().
		if ltc.Latency > 0 {
			time.Sleep(ltc.Latency)
		}
		br, pErr := ltc.stores.Send(context.Background(), *getArgs(nil).(*roachpb.BatchRequest))
		if br == nil {
			br = &roachpb.BatchResponse{}
		}
		if br.Error != nil {
			panic(roachpb.ErrorUnexpectedlySet(ltc.stores, br))
		}
		br.Error = pErr
		return []proto.Message{br}, nil
	}
	ltc.distSender = NewDistSender(&DistSenderContext{
		Clock: ltc.Clock,
		RangeDescriptorCacheSize: defaultRangeDescriptorCacheSize,
		RangeLookupMaxRanges:     defaultRangeLookupMaxRanges,
		LeaderCacheSize:          defaultLeaderCacheSize,
		RPCRetryOptions:          &defaultRPCRetryOptions,
		nodeDescriptor:           nodeDesc,
		RPCSend:                  rpcSend,    // defined above
		RangeDescriptorDB:        ltc.stores, // for descriptor lookup
	}, ltc.Gossip)

	ltc.Sender = NewTxnCoordSender(ltc.distSender, ltc.Clock, false /* !linearizable */, nil /* tracer */, ltc.Stopper)
	ltc.DB = client.NewDB(ltc.Sender)

	transport := storage.NewLocalRPCTransport(ltc.Stopper)
	ltc.Stopper.AddCloser(transport)
	ctx := storage.TestStoreContext
	ctx.Clock = ltc.Clock
	ctx.DB = ltc.DB
	ctx.Gossip = ltc.Gossip
	ctx.Transport = transport
	ltc.Store = storage.NewStore(ctx, ltc.Eng, nodeDesc)
	if err := ltc.Store.Bootstrap(roachpb.StoreIdent{NodeID: nodeID, StoreID: 1}, ltc.Stopper); err != nil {
		t.Fatalf("unable to start local test cluster: %s", err)
	}
	ltc.stores.AddStore(ltc.Store)
	if err := ltc.Store.BootstrapRange(nil); err != nil {
		t.Fatalf("unable to start local test cluster: %s", err)
	}
	if err := ltc.Store.Start(ltc.Stopper); err != nil {
		t.Fatalf("unable to start local test cluster: %s", err)
	}
	ltc.Gossip.SetNodeID(nodeDesc.NodeID)
	if err := ltc.Gossip.SetNodeDescriptor(nodeDesc); err != nil {
		t.Fatalf("unable to set node descriptor: %s", err)
	}
}

// Stop stops the cluster.
func (ltc *LocalTestCluster) Stop() {
	if r := recover(); r != nil {
		panic(r)
	}
	ltc.Stopper.Stop()
}
<|endoftext|>


<commit_before>package configr_test

import (
	"io/ioutil"
	"os"
	"testing"

	. "bitbucket.org/homemade/cin/configr"

	"github.com/stretchr/testify/assert"
)

func writeTempFile(t *testing.T, filePath string, content string) {
	err := ioutil.WriteFile(filePath, []byte(content), os.ModePerm)
	assert.NoError(t, err)
}

func Test_ItParsesAllValuesFromJSONConfig(t *testing.T) {
	filePath := "/tmp/test.json"
	writeTempFile(t, filePath, `{
	"t1": "1",
	"t2": {
		"t21": 2,
		"t22": {
			"t221": true
		}
	}
}`)
	defer os.Remove(filePath)
	f := NewFileSource(filePath)

	config := New()
	config.AddSource(f)
	config.RequireKey("t1", "")
	config.RequireKey("t2.t21", "")
	config.RequireKey("t2.t22.t221", "")
	config.RegisterKey("t3", "", 3)

	assert.NoError(t, config.Parse())

	t1, err := config.String("t1")
	assert.NoError(t, err)
	t2t21, err := config.Float64("t2.t21")
	assert.NoError(t, err)
	t2t22t221, err := config.Bool("t2.t22.t221")
	assert.NoError(t, err)
	t3, err := config.Int("t3")
	assert.NoError(t, err)

	assert.Equal(t, "1", t1)
	assert.Equal(t, float64(2), t2t21)
	assert.Equal(t, true, t2t22t221)
	assert.Equal(t, 3, t3)
}

func Test_ItParsesAllValuesFromTOMLConfig(t *testing.T) {
	filePath := "/tmp/test.toml"
	writeTempFile(t, filePath, `
[t1]
t11 = "1"
t12 = 2

[t2]
	[t2.t21]
	t211 = false
		[t2.t21.t212]
		t2121 = "4"
`)
	defer os.Remove(filePath)
	f := NewFileSource(filePath)

	config := New()
	config.AddSource(f)
	config.RequireKey("t1.t11", "")
	config.RequireKey("t1.t12", "")
	config.RequireKey("t2.t21.t211", "")
	config.RequireKey("t2.t21.t212.t2121", "")
	config.RegisterKey("t3", "", "sup")

	assert.NoError(t, config.Parse())

	t1t11, err := config.String("t1.t11")
	assert.NoError(t, err)
	t1t12, err := config.Int("t1.t12")
	assert.NoError(t, err)
	t2t21t211, err := config.Bool("t2.t21.t211")
	assert.NoError(t, err)
	t2t21t212t2121, err := config.Get("t2.t21.t212.t2121")
	assert.NoError(t, err)
	t3, err := config.String("t3")
	assert.NoError(t, err)

	assert.Equal(t, "1", t1t11)
	assert.Equal(t, 2, t1t12)
	assert.Equal(t, false, t2t21t211)
	assert.Equal(t, "4", t2t21t212t2121.(string))
	assert.Equal(t, "sup", t3)
}

func Test_ItGeneratesBlankJSONConfig(t *testing.T) {
	config := New()
	expectedOutput := `{
	"t1": {
		"t11": "*** You need this ***",
		"t12": "*** Me too ***"
	},
	"t2": {
		"t21": {
			"t211": "*** And me ***",
			"t212": {
				"t2121": "*** Also me! ***"
			}
		}
	},
	"t3": 0
}`
	config.RequireKey("t1.t11", "You need this")
	config.RequireKey("t1.t12", "Me too")
	config.RequireKey("t2.t21.t211", "And me")
	config.RequireKey("t2.t21.t212.t2121", "Also me!")
	config.RegisterKey("t3", "", 0)

	f := NewFileSource("")
	f.SetEncoding(JSON)

	configBytes, err := config.GenerateBlank(f)
	assert.NoError(t, err)

	assert.Equal(t, expectedOutput, string(configBytes))
}

func Test_ItGeneratesBlankTOMLConfig(t *testing.T) {
	config := New()
	expectedOutput := `t3 = 0

[t1]
  t11 = "*** You need this ***"
  t12 = "*** Me too ***"

[t2]
  [t2.t21]
    t211 = "*** And me ***"
    [t2.t21.t212]
      t2121 = "*** Also me! ***"
`
	config.RequireKey("t1.t11", "You need this")
	config.RequireKey("t1.t12", "Me too")
	config.RequireKey("t2.t21.t211", "And me")
	config.RequireKey("t2.t21.t212.t2121", "Also me!")
	config.RegisterKey("t3", "", 0)

	f := NewFileSource("")
	f.SetEncoding(TOML)

	configBytes, err := config.GenerateBlank(f)
	assert.NoError(t, err)

	assert.Equal(t, expectedOutput, string(configBytes))
}
<commit_msg>Rewrite import path to fix feature tests<commit_after>package configr_test

import (
	"io/ioutil"
	"os"
	"testing"

	. "github.com/adrianduke/configr"

	"github.com/stretchr/testify/assert"
)

func writeTempFile(t *testing.T, filePath string, content string) {
	err := ioutil.WriteFile(filePath, []byte(content), os.ModePerm)
	assert.NoError(t, err)
}

func Test_ItParsesAllValuesFromJSONConfig(t *testing.T) {
	filePath := "/tmp/test.json"
	writeTempFile(t, filePath, `{
	"t1": "1",
	"t2": {
		"t21": 2,
		"t22": {
			"t221": true
		}
	}
}`)
	defer os.Remove(filePath)
	f := NewFileSource(filePath)

	config := New()
	config.AddSource(f)
	config.RequireKey("t1", "")
	config.RequireKey("t2.t21", "")
	config.RequireKey("t2.t22.t221", "")
	config.RegisterKey("t3", "", 3)

	assert.NoError(t, config.Parse())

	t1, err := config.String("t1")
	assert.NoError(t, err)
	t2t21, err := config.Float64("t2.t21")
	assert.NoError(t, err)
	t2t22t221, err := config.Bool("t2.t22.t221")
	assert.NoError(t, err)
	t3, err := config.Int("t3")
	assert.NoError(t, err)

	assert.Equal(t, "1", t1)
	assert.Equal(t, float64(2), t2t21)
	assert.Equal(t, true, t2t22t221)
	assert.Equal(t, 3, t3)
}

func Test_ItParsesAllValuesFromTOMLConfig(t *testing.T) {
	filePath := "/tmp/test.toml"
	writeTempFile(t, filePath, `
[t1]
t11 = "1"
t12 = 2

[t2]
	[t2.t21]
	t211 = false
		[t2.t21.t212]
		t2121 = "4"
`)
	defer os.Remove(filePath)
	f := NewFileSource(filePath)

	config := New()
	config.AddSource(f)
	config.RequireKey("t1.t11", "")
	config.RequireKey("t1.t12", "")
	config.RequireKey("t2.t21.t211", "")
	config.RequireKey("t2.t21.t212.t2121", "")
	config.RegisterKey("t3", "", "sup")

	assert.NoError(t, config.Parse())

	t1t11, err := config.String("t1.t11")
	assert.NoError(t, err)
	t1t12, err := config.Int("t1.t12")
	assert.NoError(t, err)
	t2t21t211, err := config.Bool("t2.t21.t211")
	assert.NoError(t, err)
	t2t21t212t2121, err := config.Get("t2.t21.t212.t2121")
	assert.NoError(t, err)
	t3, err := config.String("t3")
	assert.NoError(t, err)

	assert.Equal(t, "1", t1t11)
	assert.Equal(t, 2, t1t12)
	assert.Equal(t, false, t2t21t211)
	assert.Equal(t, "4", t2t21t212t2121.(string))
	assert.Equal(t, "sup", t3)
}

func Test_ItGeneratesBlankJSONConfig(t *testing.T) {
	config := New()
	expectedOutput := `{
	"t1": {
		"t11": "*** You need this ***",
		"t12": "*** Me too ***"
	},
	"t2": {
		"t21": {
			"t211": "*** And me ***",
			"t212": {
				"t2121": "*** Also me! ***"
			}
		}
	},
	"t3": 0
}`
	config.RequireKey("t1.t11", "You need this")
	config.RequireKey("t1.t12", "Me too")
	config.RequireKey("t2.t21.t211", "And me")
	config.RequireKey("t2.t21.t212.t2121", "Also me!")
	config.RegisterKey("t3", "", 0)

	f := NewFileSource("")
	f.SetEncoding(JSON)

	configBytes, err := config.GenerateBlank(f)
	assert.NoError(t, err)

	assert.Equal(t, expectedOutput, string(configBytes))
}

func Test_ItGeneratesBlankTOMLConfig(t *testing.T) {
	config := New()
	expectedOutput := `t3 = 0

[t1]
  t11 = "*** You need this ***"
  t12 = "*** Me too ***"

[t2]
  [t2.t21]
    t211 = "*** And me ***"
    [t2.t21.t212]
      t2121 = "*** Also me! ***"
`
	config.RequireKey("t1.t11", "You need this")
	config.RequireKey("t1.t12", "Me too")
	config.RequireKey("t2.t21.t211", "And me")
	config.RequireKey("t2.t21.t212.t2121", "Also me!")
	config.RegisterKey("t3", "", 0)

	f := NewFileSource("")
	f.SetEncoding(TOML)

	configBytes, err := config.GenerateBlank(f)
	assert.NoError(t, err)

	assert.Equal(t, expectedOutput, string(configBytes))
}
<|endoftext|>


<commit_before>/*
 * Copyright (C) 2013 Matthew Dawson <matthew@mjdsystems.ca>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
package main

import (
	"io/ioutil"

	"net/http"
	"net/url"

	"time"
)

type RawFeed struct {
	Data      []byte
	Url       url.URL
	FetchedAt time.Time
}

type FeedError struct {
	Err error
	Url url.URL
}

func FeedFetcher(in <-chan url.URL, out chan<- RawFeed, errChan chan<- FeedError) {
	for {
		if next, ok := <-in; ok {
			resp, err := http.Get(next.String())
			if err != nil {
				errChan <- FeedError{err, next}
				continue
			}

			content, err := ioutil.ReadAll(resp.Body)
			resp.Body.Close()

			if err != nil {
				errChan <- FeedError{err, next}
			} else {
				out <- RawFeed{content, next, time.Now()}
			}
		} else {
			break
		}
	}
}
<commit_msg>Make FeedError be an Error.<commit_after>/*
 * Copyright (C) 2013 Matthew Dawson <matthew@mjdsystems.ca>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
package main

import (
	"fmt"

	"io/ioutil"

	"net/http"
	"net/url"

	"time"
)

type RawFeed struct {
	Data      []byte
	Url       url.URL
	FetchedAt time.Time
}

type FeedError struct {
	Err error
	Url url.URL
}

func (e FeedError) Error() string {
	return fmt.Sprintf("Error while dealing with url %s: %s", e.Url.String(), e.Err)
}

func FeedFetcher(in <-chan url.URL, out chan<- RawFeed, errChan chan<- FeedError) {
	for {
		if next, ok := <-in; ok {
			resp, err := http.Get(next.String())
			if err != nil {
				errChan <- FeedError{err, next}
				continue
			}

			content, err := ioutil.ReadAll(resp.Body)
			resp.Body.Close()

			if err != nil {
				errChan <- FeedError{err, next}
			} else {
				out <- RawFeed{content, next, time.Now()}
			}
		} else {
			break
		}
	}
}
<|endoftext|>


<commit_before>package indicators_test

import (
	. "github.com/onsi/ginkgo"
	"github.com/thetruetrade/gotrade/indicators"
)

var _ = Describe("when calculating an average true range (atr) with DOHLCV source data", func() {
	var (
		period    int = 14
		indicator *indicators.Atr
		inputs    IndicatorWithFloatBoundsSharedSpecInputs
	)

	BeforeEach(func() {
		indicator, _ = indicators.NewAtr(period)

		inputs = NewIndicatorWithFloatBoundsSharedSpecInputs(indicator, len(sourceDOHLCVData), indicator,
			func() float64 {
				return GetFloatDataMax(indicator.Data)
			},
			func() float64 {
				return GetFloatDataMin(indicator.Data)
			})
	})

	Context("and the indicator has not yet received any ticks", func() {
		ShouldBeAnInitialisedIndicator(&inputs)

		ShouldNotHaveAnyFloatBoundsSetYet(&inputs)
	})

	Context("and the indicator has received less ticks than the lookback period", func() {

		BeforeEach(func() {
			for i := 0; i < indicator.GetLookbackPeriod(); i++ {
				indicator.ReceiveDOHLCVTick(sourceDOHLCVData[i], i+1)
			}
		})

		ShouldBeAnIndicatorThatHasReceivedFewerTicksThanItsLookbackPeriod(&inputs)

		ShouldNotHaveAnyFloatBoundsSetYet(&inputs)
	})

	Context("and the indicator has received ticks equal to the lookback period", func() {

		BeforeEach(func() {
			for i := 0; i <= indicator.GetLookbackPeriod(); i++ {
				indicator.ReceiveDOHLCVTick(sourceDOHLCVData[i], i+1)
			}
		})

		ShouldBeAnIndicatorThatHasReceivedTicksEqualToItsLookbackPeriod(&inputs)

		ShouldHaveFloatBoundsSetToMinMaxOfResults(&inputs)
	})

	Context("and the indicator has received more ticks than the lookback period", func() {

		BeforeEach(func() {
			for i := range sourceDOHLCVData {
				indicator.ReceiveDOHLCVTick(sourceDOHLCVData[i], i+1)
			}
		})

		ShouldBeAnIndicatorThatHasReceivedMoreTicksThanItsLookbackPeriod(&inputs)

		ShouldHaveFloatBoundsSetToMinMaxOfResults(&inputs)
	})

	Context("and the indicator has recieved all of its ticks", func() {
		BeforeEach(func() {
			for i := 0; i < len(sourceDOHLCVData); i++ {
				indicator.ReceiveDOHLCVTick(sourceDOHLCVData[i], i+1)
			}
		})

		ShouldBeAnIndicatorThatHasReceivedAllOfItsTicks(&inputs)

		ShouldHaveFloatBoundsSetToMinMaxOfResults(&inputs)
	})
})
<commit_msg>#77 achieve 100% test coverage for indicators - atr<commit_after>package indicators_test

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/thetruetrade/gotrade/indicators"
)

var _ = Describe("when creating an atrwithoutstorage", func() {
	var (
		indicator      *indicators.AtrWithoutStorage
		indicatorError error
	)

	Context("and the indicator was not given a value available action", func() {
		BeforeEach(func() {
			indicator, indicatorError = indicators.NewAtrWithoutStorage(4, nil)
		})

		It("the indicator should not be created and return the appropriate error message", func() {
			Expect(indicator).To(BeNil())
			Expect(indicatorError).To(Equal(indicators.ErrValueAvailableActionIsNil))
		})
	})

	Context("and the indicator was given a timePeriod below the minimum", func() {
		BeforeEach(func() {
			indicator, indicatorError = indicators.NewAtrWithoutStorage(0, fakeFloatValAvailable)
		})

		It("the indicator should not be created and return the appropriate error message", func() {
			Expect(indicator).To(BeNil())
		})
	})

	Context("and the indicator was given a timePeriod above the maximum", func() {
		BeforeEach(func() {
			indicator, indicatorError = indicators.NewAtrWithoutStorage(indicators.MaximumLookbackPeriod+1, fakeFloatValAvailable)
		})

		It("the indicator should not be created and return the appropriate error message", func() {
			Expect(indicator).To(BeNil())
		})
	})
})

var _ = Describe("when calculating an average true range (atr) with DOHLCV source data", func() {
	var (
		period    int = 14
		indicator *indicators.Atr
		inputs    IndicatorWithFloatBoundsSharedSpecInputs
		stream    *fakeDOHLCVStreamSubscriber
	)

	Context("given the indicator is created via the standard constructor", func() {
		BeforeEach(func() {
			indicator, _ = indicators.NewAtr(period)

			inputs = NewIndicatorWithFloatBoundsSharedSpecInputs(indicator, len(sourceDOHLCVData), indicator,
				func() float64 {
					return GetFloatDataMax(indicator.Data)
				},
				func() float64 {
					return GetFloatDataMin(indicator.Data)
				})
		})

		Context("and the indicator has not yet received any ticks", func() {
			ShouldBeAnInitialisedIndicator(&inputs)

			ShouldNotHaveAnyFloatBoundsSetYet(&inputs)
		})

		Context("and the indicator has received less ticks than the lookback period", func() {

			BeforeEach(func() {
				for i := 0; i < indicator.GetLookbackPeriod(); i++ {
					indicator.ReceiveDOHLCVTick(sourceDOHLCVData[i], i+1)
				}
			})

			ShouldBeAnIndicatorThatHasReceivedFewerTicksThanItsLookbackPeriod(&inputs)

			ShouldNotHaveAnyFloatBoundsSetYet(&inputs)
		})

		Context("and the indicator has received ticks equal to the lookback period", func() {

			BeforeEach(func() {
				for i := 0; i <= indicator.GetLookbackPeriod(); i++ {
					indicator.ReceiveDOHLCVTick(sourceDOHLCVData[i], i+1)
				}
			})

			ShouldBeAnIndicatorThatHasReceivedTicksEqualToItsLookbackPeriod(&inputs)

			ShouldHaveFloatBoundsSetToMinMaxOfResults(&inputs)
		})

		Context("and the indicator has received more ticks than the lookback period", func() {

			BeforeEach(func() {
				for i := range sourceDOHLCVData {
					indicator.ReceiveDOHLCVTick(sourceDOHLCVData[i], i+1)
				}
			})

			ShouldBeAnIndicatorThatHasReceivedMoreTicksThanItsLookbackPeriod(&inputs)

			ShouldHaveFloatBoundsSetToMinMaxOfResults(&inputs)
		})

		Context("and the indicator has recieved all of its ticks", func() {
			BeforeEach(func() {
				for i := 0; i < len(sourceDOHLCVData); i++ {
					indicator.ReceiveDOHLCVTick(sourceDOHLCVData[i], i+1)
				}
			})

			ShouldBeAnIndicatorThatHasReceivedAllOfItsTicks(&inputs)

			ShouldHaveFloatBoundsSetToMinMaxOfResults(&inputs)
		})
	})

	Context("given the indicator is created via the standard constructor", func() {
		BeforeEach(func() {
			indicator, _ = indicators.NewAtr(period)
			inputs = NewIndicatorWithFloatBoundsSharedSpecInputs(indicator, len(sourceDOHLCVData), indicator,
				func() float64 {
					return GetFloatDataMax(indicator.Data)
				},
				func() float64 {
					return GetFloatDataMin(indicator.Data)
				})
		})

		Context("and the indicator has not yet received any ticks", func() {
			ShouldBeAnInitialisedIndicator(&inputs)

			ShouldNotHaveAnyFloatBoundsSetYet(&inputs)
		})

		Context("and the indicator has received less ticks than the lookback period", func() {

			BeforeEach(func() {
				for i := 0; i < indicator.GetLookbackPeriod(); i++ {
					indicator.ReceiveDOHLCVTick(sourceDOHLCVData[i], i+1)
				}
			})

			ShouldBeAnIndicatorThatHasReceivedFewerTicksThanItsLookbackPeriod(&inputs)

			ShouldNotHaveAnyFloatBoundsSetYet(&inputs)
		})

		Context("and the indicator has received ticks equal to the lookback period", func() {

			BeforeEach(func() {
				for i := 0; i <= indicator.GetLookbackPeriod(); i++ {
					indicator.ReceiveDOHLCVTick(sourceDOHLCVData[i], i+1)
				}
			})

			ShouldBeAnIndicatorThatHasReceivedTicksEqualToItsLookbackPeriod(&inputs)

			ShouldHaveFloatBoundsSetToMinMaxOfResults(&inputs)
		})

		Context("and the indicator has received more ticks than the lookback period", func() {

			BeforeEach(func() {
				for i := range sourceDOHLCVData {
					indicator.ReceiveDOHLCVTick(sourceDOHLCVData[i], i+1)
				}
			})

			ShouldBeAnIndicatorThatHasReceivedMoreTicksThanItsLookbackPeriod(&inputs)

			ShouldHaveFloatBoundsSetToMinMaxOfResults(&inputs)
		})

		Context("and the indicator has recieved all of its ticks", func() {
			BeforeEach(func() {
				for i := 0; i < len(sourceDOHLCVData); i++ {
					indicator.ReceiveDOHLCVTick(sourceDOHLCVData[i], i+1)
				}
			})

			ShouldBeAnIndicatorThatHasReceivedAllOfItsTicks(&inputs)

			ShouldHaveFloatBoundsSetToMinMaxOfResults(&inputs)
		})
	})

	Context("given the indicator is created via the constructor with defaulted parameters", func() {
		BeforeEach(func() {
			indicator, _ = indicators.NewDefaultAtr()
			inputs = NewIndicatorWithFloatBoundsSharedSpecInputs(indicator, len(sourceDOHLCVData), indicator,
				func() float64 {
					return GetFloatDataMax(indicator.Data)
				},
				func() float64 {
					return GetFloatDataMin(indicator.Data)
				})
		})

		Context("and the indicator has not yet received any ticks", func() {
			ShouldBeAnInitialisedIndicator(&inputs)

			ShouldNotHaveAnyFloatBoundsSetYet(&inputs)
		})

		Context("and the indicator has recieved all of its ticks", func() {
			BeforeEach(func() {
				for i := 0; i < len(sourceDOHLCVData); i++ {
					indicator.ReceiveDOHLCVTick(sourceDOHLCVData[i], i+1)
				}
			})

			ShouldBeAnIndicatorThatHasReceivedAllOfItsTicks(&inputs)

			ShouldHaveFloatBoundsSetToMinMaxOfResults(&inputs)
		})
	})

	Context("given the indicator is created via the constructor with fixed source length", func() {
		BeforeEach(func() {
			indicator, _ = indicators.NewAtrWithSrcLen(uint(len(sourceDOHLCVData)), 4)
			inputs = NewIndicatorWithFloatBoundsSharedSpecInputs(indicator, len(sourceDOHLCVData), indicator,
				func() float64 {
					return GetFloatDataMax(indicator.Data)
				},
				func() float64 {
					return GetFloatDataMin(indicator.Data)
				})
		})

		It("should have pre-allocated storge for the output data", func() {
			Expect(cap(indicator.Data)).To(Equal(len(sourceDOHLCVData) - indicator.GetLookbackPeriod()))
		})

		Context("and the indicator has not yet received any ticks", func() {
			ShouldBeAnInitialisedIndicator(&inputs)

			ShouldNotHaveAnyFloatBoundsSetYet(&inputs)
		})

		Context("and the indicator has recieved all of its ticks", func() {
			BeforeEach(func() {
				for i := 0; i < len(sourceDOHLCVData); i++ {
					indicator.ReceiveDOHLCVTick(sourceDOHLCVData[i], i+1)
				}
			})

			ShouldBeAnIndicatorThatHasReceivedAllOfItsTicks(&inputs)

			ShouldHaveFloatBoundsSetToMinMaxOfResults(&inputs)

			It("no new storage capcity should have been allocated", func() {
				Expect(len(indicator.Data)).To(Equal(cap(indicator.Data)))
			})
		})
	})

	Context("given the indicator is created via the constructor with defaulted parameters and fixed source length", func() {
		BeforeEach(func() {
			indicator, _ = indicators.NewDefaultAtrWithSrcLen(uint(len(sourceDOHLCVData)))
			inputs = NewIndicatorWithFloatBoundsSharedSpecInputs(indicator, len(sourceDOHLCVData), indicator,
				func() float64 {
					return GetFloatDataMax(indicator.Data)
				},
				func() float64 {
					return GetFloatDataMin(indicator.Data)
				})
		})

		It("should have pre-allocated storge for the output data", func() {
			Expect(cap(indicator.Data)).To(Equal(len(sourceDOHLCVData) - indicator.GetLookbackPeriod()))
		})

		Context("and the indicator has not yet received any ticks", func() {
			ShouldBeAnInitialisedIndicator(&inputs)

			ShouldNotHaveAnyFloatBoundsSetYet(&inputs)
		})

		Context("and the indicator has recieved all of its ticks", func() {
			BeforeEach(func() {
				for i := 0; i < len(sourceDOHLCVData); i++ {
					indicator.ReceiveDOHLCVTick(sourceDOHLCVData[i], i+1)
				}
			})

			ShouldBeAnIndicatorThatHasReceivedAllOfItsTicks(&inputs)

			ShouldHaveFloatBoundsSetToMinMaxOfResults(&inputs)

			It("no new storage capcity should have been allocated", func() {
				Expect(len(indicator.Data)).To(Equal(cap(indicator.Data)))
			})
		})
	})

	Context("given the indicator is created via the constructor for use with a price stream", func() {
		BeforeEach(func() {
			stream = newFakeDOHLCVStreamSubscriber()
			indicator, _ = indicators.NewAtrForStream(stream, 4)
			inputs = NewIndicatorWithFloatBoundsSharedSpecInputs(indicator, len(sourceDOHLCVData), indicator,
				func() float64 {
					return GetFloatDataMax(indicator.Data)
				},
				func() float64 {
					return GetFloatDataMin(indicator.Data)
				})
		})

		It("should have requested to be attached to the stream", func() {
			Expect(stream.lastCallToAddTickSubscriptionArg).To(Equal(indicator))
		})

		Context("and the indicator has not yet received any ticks", func() {
			ShouldBeAnInitialisedIndicator(&inputs)

			ShouldNotHaveAnyFloatBoundsSetYet(&inputs)
		})

		Context("and the indicator has recieved all of its ticks", func() {
			BeforeEach(func() {
				for i := 0; i < len(sourceDOHLCVData); i++ {
					indicator.ReceiveDOHLCVTick(sourceDOHLCVData[i], i+1)
				}
			})

			ShouldBeAnIndicatorThatHasReceivedAllOfItsTicks(&inputs)

			ShouldHaveFloatBoundsSetToMinMaxOfResults(&inputs)
		})
	})

	Context("given the indicator is created via the constructor for use with a price stream with defaulted parameters", func() {
		BeforeEach(func() {
			stream = newFakeDOHLCVStreamSubscriber()
			indicator, _ = indicators.NewDefaultAtrForStream(stream)
			inputs = NewIndicatorWithFloatBoundsSharedSpecInputs(indicator, len(sourceDOHLCVData), indicator,
				func() float64 {
					return GetFloatDataMax(indicator.Data)
				},
				func() float64 {
					return GetFloatDataMin(indicator.Data)
				})
		})

		It("should have requested to be attached to the stream", func() {
			Expect(stream.lastCallToAddTickSubscriptionArg).To(Equal(indicator))
		})

		Context("and the indicator has not yet received any ticks", func() {
			ShouldBeAnInitialisedIndicator(&inputs)

			ShouldNotHaveAnyFloatBoundsSetYet(&inputs)
		})

		Context("and the indicator has recieved all of its ticks", func() {
			BeforeEach(func() {
				for i := 0; i < len(sourceDOHLCVData); i++ {
					indicator.ReceiveDOHLCVTick(sourceDOHLCVData[i], i+1)
				}
			})

			ShouldBeAnIndicatorThatHasReceivedAllOfItsTicks(&inputs)

			ShouldHaveFloatBoundsSetToMinMaxOfResults(&inputs)
		})
	})

	Context("given the indicator is created via the constructor for use with a price stream with fixed source length", func() {
		BeforeEach(func() {
			stream = newFakeDOHLCVStreamSubscriber()
			indicator, _ = indicators.NewAtrForStreamWithSrcLen(uint(len(sourceDOHLCVData)), stream, 4)
			inputs = NewIndicatorWithFloatBoundsSharedSpecInputs(indicator, len(sourceDOHLCVData), indicator,
				func() float64 {
					return GetFloatDataMax(indicator.Data)
				},
				func() float64 {
					return GetFloatDataMin(indicator.Data)
				})
		})

		It("should have pre-allocated storge for the output data", func() {
			Expect(cap(indicator.Data)).To(Equal(len(sourceDOHLCVData) - indicator.GetLookbackPeriod()))
		})

		It("should have requested to be attached to the stream", func() {
			Expect(stream.lastCallToAddTickSubscriptionArg).To(Equal(indicator))
		})

		Context("and the indicator has not yet received any ticks", func() {
			ShouldBeAnInitialisedIndicator(&inputs)

			ShouldNotHaveAnyFloatBoundsSetYet(&inputs)
		})

		Context("and the indicator has recieved all of its ticks", func() {
			BeforeEach(func() {
				for i := 0; i < len(sourceDOHLCVData); i++ {
					indicator.ReceiveDOHLCVTick(sourceDOHLCVData[i], i+1)
				}
			})

			ShouldBeAnIndicatorThatHasReceivedAllOfItsTicks(&inputs)

			ShouldHaveFloatBoundsSetToMinMaxOfResults(&inputs)

			It("no new storage capcity should have been allocated", func() {
				Expect(len(indicator.Data)).To(Equal(cap(indicator.Data)))
			})
		})
	})

	Context("given the indicator is created via the constructor for use with a price stream with fixed source length with defaulted parmeters", func() {
		BeforeEach(func() {
			stream = newFakeDOHLCVStreamSubscriber()
			indicator, _ = indicators.NewDefaultAtrForStreamWithSrcLen(uint(len(sourceDOHLCVData)), stream)
			inputs = NewIndicatorWithFloatBoundsSharedSpecInputs(indicator, len(sourceDOHLCVData), indicator,
				func() float64 {
					return GetFloatDataMax(indicator.Data)
				},
				func() float64 {
					return GetFloatDataMin(indicator.Data)
				})
		})

		It("should have pre-allocated storge for the output data", func() {
			Expect(cap(indicator.Data)).To(Equal(len(sourceDOHLCVData) - indicator.GetLookbackPeriod()))
		})

		It("should have requested to be attached to the stream", func() {
			Expect(stream.lastCallToAddTickSubscriptionArg).To(Equal(indicator))
		})

		Context("and the indicator has not yet received any ticks", func() {
			ShouldBeAnInitialisedIndicator(&inputs)

			ShouldNotHaveAnyFloatBoundsSetYet(&inputs)
		})

		Context("and the indicator has recieved all of its ticks", func() {
			BeforeEach(func() {
				for i := 0; i < len(sourceDOHLCVData); i++ {
					indicator.ReceiveDOHLCVTick(sourceDOHLCVData[i], i+1)
				}
			})

			ShouldBeAnIndicatorThatHasReceivedAllOfItsTicks(&inputs)

			ShouldHaveFloatBoundsSetToMinMaxOfResults(&inputs)

			It("no new storage capcity should have been allocated", func() {
				Expect(len(indicator.Data)).To(Equal(cap(indicator.Data)))
			})
		})
	})
})
<|endoftext|>


<commit_before>package main

import (
	"eaciit/gdrj/model"
	"eaciit/gdrj/modules"
	"os"

	"flag"
	"strings"

	"github.com/eaciit/dbox"
	"github.com/eaciit/toolkit"
	// "sync"
	"time"
)

var conn dbox.IConnection
var count int

var (
	t0         time.Time
	fiscalyear int
	data       map[string]float64
	tablename  = "salespls-1"
	alldata    = toolkit.M{}
)

func setinitialconnection() {
	var err error
	conn, err = modules.GetDboxIConnection("db_godrej")

	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}

	err = gdrj.SetDb(conn)
	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}
}

func main() {
	t0 = time.Now()
	data = make(map[string]float64)
	flag.IntVar(&fiscalyear, "year", 2015, "YYYY representation of godrej fiscal year. Default is 2015")
	flag.Parse()

	setinitialconnection()
	defer gdrj.CloseDb()

	toolkit.Println("Preparing data query...")

	eperiode := time.Date(fiscalyear, 4, 1, 0, 0, 0, 0, time.UTC)
	// speriode := eperiode.AddDate(-1, 0, 0)
	speriode := eperiode.AddDate(0, 0, -1)

	seeds := make([]time.Time, 0, 0)
	seeds = append(seeds, speriode)
	for {
		speriode = speriode.AddDate(0, 0, 1)
		if !speriode.Before(eperiode) {
			break
		}
		seeds = append(seeds, speriode)
	}

	toolkit.Println("Starting worker query...")
	result := make(chan toolkit.M, len(seeds))
	for i, v := range seeds {
		filter := dbox.Eq("date.date", v)
		go workerproc(i, filter, result)
	}

	step := len(seeds) / 100
	if step == 0 {
		step = 1
	}
	// "date.fiscal","date.quartertxt","date.month", "customer.branchname", "customer.keyaccount",
	// "customer.channelid", "customer.channelname", "customer.reportchannel", "customer.reportsubchannel",
	// "customer.zone", "customer.region", "customer.areaname","product.brand"
	toolkit.Println("Waiting result query...")
	for i := 1; i <= len(seeds); i++ {
		a := <-result
		for k, v := range a {
			lasttkm := toolkit.M{}
			newtkm, _ := toolkit.ToM(v)
			if alldata.Has(k) {
				lasttkm = alldata[k].(toolkit.M)
			}

			for tk, tv := range newtkm {
				df := toolkit.ToFloat64(tv, 6, toolkit.RoundingAuto) + lasttkm.GetFloat64(tk)
				lasttkm.Set(tk, df)
			}
			alldata.Set(k, lasttkm)
		}

		if i%step == 0 {
			toolkit.Printfn("Worker %d of %d (%d), Done in %s",
				i, len(seeds), (i / step), time.Since(t0).String())
		}
	}

	listdimension := []string{"date.fiscal,customer.channelid,customer.channelname",
		"date.fiscal,customer.channelid,customer.channelname,customer.reportsubchannel",
		"date.fiscal,customer.channelid,customer.channelname,customer.zone",
		"date.fiscal,customer.channelid,customer.channelname,customer.areaname",
		"date.fiscal,customer.channelid,customer.channelname,customer.region"}

	resdimension := make(chan int)
	dimension := make(chan string, 5)
	detaildata := make(chan toolkit.M)
	ressavedata := make(chan int)

	for i := 0; i < 3; i++ {
		go workerbuilddimension(i, dimension, detaildata, resdimension)
	}

	for i := 0; i < 10; i++ {
		go workersavedata(i, detaildata, ressavedata)
	}

	toolkit.Printfn("Prepare saving collection, Create dimension")
	for _, str := range listdimension {
		dimension <- str
		toolkit.Printfn("SEND : %v", str)
	}
	close(dimension)

	alldatarows := 0
	toolkit.Printfn("Waiting dimension result")
	for i := 0; i < len(listdimension); i++ {
		alldatarows += <-resdimension
	}
	close(detaildata)

	step = alldatarows / 100
	if step == 0 {
		step = 1
	}
	toolkit.Printfn("Saving dimension result")
	for i := 0; i < alldatarows; i++ {
		<-ressavedata
		if i%step == 0 {
			toolkit.Printfn("Data saved %d of %d (%d), Done in %s",
				i, alldatarows, (i / step), time.Since(t0).String())
		}
	}

	toolkit.Printfn("Processing done in %s",
		time.Since(t0).String())
}

func workerproc(wi int, filter *dbox.Filter, result chan<- toolkit.M) {
	workerconn, _ := modules.GetDboxIConnection("db_godrej")
	defer workerconn.Close()
	tkm := toolkit.M{}
	// "date.fiscal","date.quartertxt","date.month", "customer.branchname", "customer.keyaccount",
	// "customer.channelid", "customer.channelname", "customer.reportchannel", "customer.reportsubchannel",
	// "customer.zone", "customer.region", "customer.areaname","product.brand"
	csr, _ := workerconn.NewQuery().Select("date", "customer", "product.brand", "pldatas").
		From(tablename).
		Where(filter).
		Cursor(nil)

	defer csr.Close()

	scount := csr.Count()
	iscount := 0
	step := scount / 100

	if step == 0 {
		step = 1
	}

	for {
		iscount++
		spl := new(gdrj.SalesPL)
		e := csr.Fetch(spl, 1, false)
		if e != nil {
			break
		}

		key := toolkit.Sprintf("%s|%s|%d|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s", spl.Date.Fiscal, spl.Date.QuarterTxt, spl.Date.Month,
			spl.Customer.BranchName, spl.Customer.KeyAccount, spl.Customer.ChannelID, spl.Customer.ChannelName,
			spl.Customer.ReportChannel, spl.Customer.ReportSubChannel, spl.Customer.Zone, spl.Customer.Region,
			spl.Customer.AreaName, spl.Product.Brand)

		dtkm := toolkit.M{}
		if tkm.Has(key) {
			dtkm = tkm[key].(toolkit.M)
		}

		tv := spl.GrossAmount + dtkm.GetFloat64("grossamount")
		dtkm.Set("grossamount", tv)

		tv = spl.NetAmount + dtkm.GetFloat64("netamount")
		dtkm.Set("netamount", tv)

		tv = spl.DiscountAmount + dtkm.GetFloat64("discountamount")
		dtkm.Set("discountamount", tv)

		tv = spl.SalesQty + dtkm.GetFloat64("salesqty")
		dtkm.Set("salesqty", tv)

		for k, v := range spl.PLDatas {
			tv := v.Amount + dtkm.GetFloat64(k)
			dtkm.Set(k, tv)
		}

		tkm.Set(key, dtkm)

		if iscount == 3 {
			break
		}

	}

	result <- tkm

	toolkit.Printfn("Go %d. Processing done in %s",
		wi,
		time.Since(t0).String())
}

func workerbuilddimension(wi int, dimension <-chan string, detaildata chan<- toolkit.M, resdimension chan<- int) {
	//"date.fiscal", "customer.channelid", "customer.channelname"
	sortkeys := []string{"date.fiscal", "date.quartertxt", "date.month",
		"customer.branchname", "customer.keyaccount", "customer.channelid", "customer.channelname", "customer.reportchannel",
		"customer.reportsubchannel", "customer.zone", "customer.region", "customer.areaname", "product.brand"}

	str := ""
	for str = range dimension {
		toolkit.Println(str)
		payload := new(gdrj.PLFinderParam)
		payload.Breakdowns = strings.Split(str, ",")
		tablename := toolkit.Sprintf("1-%v", payload.GetTableName())

		tkm := toolkit.M{}
		for key, val := range alldata {
			// toolkit.Printfn("%s", key)
			arrkey := strings.Split(key, "|")
			// toolkit.Printfn("%d - %d", len(sortkeys), len(arrkey))
			dkey := ""
			for i, v := range sortkeys {
				for _, vx := range payload.Breakdowns {
					if vx == v {
						if len(dkey) > 0 {
							dkey = toolkit.Sprintf("%s|%s", dkey, arrkey[i])
						} else {
							dkey = arrkey[i]
						}
					}
				}
			}

			stkm, _ := toolkit.ToM(val)
			dtkm := toolkit.M{}
			if tkm.Has(dkey) {
				dtkm = tkm[dkey].(toolkit.M)
			}

			for k, v := range stkm {
				fv := toolkit.ToFloat64(v, 6, toolkit.RoundingAuto) + dtkm.GetFloat64(k)
				dtkm.Set(k, fv)
			}

			tkm.Set(dkey, dtkm)
		}
		for k, v := range tkm {
			a, _ := toolkit.ToM(v)
			id := toolkit.M{}
			arrk := strings.Split(k, "|")
			for i, sv := range payload.Breakdowns {
				tsv := strings.Replace(sv, ".", "_", -1)
				id.Set(tsv, arrk[i])
			}
			a.Set("_id", id)

			detaildata <- toolkit.M{}.Set(tablename, a)
		}

		resdimension <- len(tkm)
		toolkit.Println("SUM Data : ", len(tkm))
	}
}

func workersavedata(wi int, detaildata <-chan toolkit.M, ressavedata chan<- int) {
	workerconn, _ := modules.GetDboxIConnection("db_godrej")
	defer workerconn.Close()

	tkm := toolkit.M{}
	qs := map[string]*dbox.Query{}
	for tkm = range detaildata {
		for tbl, dt := range tkm {
			q, exist := qs[tbl]
			if !exist {
				q = workerconn.NewQuery().From(tbl).
					Save().
					SetConfig("multiexec", true)
				qs[tbl] = q
			}
			esave := q.Exec(toolkit.M{}.Set("data", dt))
			if esave != nil {
				toolkit.Printfn("Can't save %s - %s : %v", tbl, esave.Error(), dt)
				os.Exit(100)
			}
			ressavedata <- 1
		}
	}
}
<commit_msg>pointer<commit_after>package main

import (
	"eaciit/gdrj/model"
	"eaciit/gdrj/modules"
	"os"

	"flag"
	"strings"

	"github.com/eaciit/dbox"
	"github.com/eaciit/toolkit"
	// "sync"
	"time"
)

var conn dbox.IConnection
var count int

var (
	t0         time.Time
	fiscalyear int
	data       map[string]float64
	tablename  = "salespls-1"
	alldata    = toolkit.M{}
)

func setinitialconnection() {
	var err error
	conn, err = modules.GetDboxIConnection("db_godrej")

	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}

	err = gdrj.SetDb(conn)
	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}
}

func main() {
	t0 = time.Now()
	data = make(map[string]float64)
	flag.IntVar(&fiscalyear, "year", 2015, "YYYY representation of godrej fiscal year. Default is 2015")
	flag.Parse()

	setinitialconnection()
	defer gdrj.CloseDb()

	toolkit.Println("Preparing data query...")

	eperiode := time.Date(fiscalyear, 4, 1, 0, 0, 0, 0, time.UTC)
	// speriode := eperiode.AddDate(-1, 0, 0)
	speriode := eperiode.AddDate(0, 0, -1)

	seeds := make([]time.Time, 0, 0)
	seeds = append(seeds, speriode)
	for {
		speriode = speriode.AddDate(0, 0, 1)
		if !speriode.Before(eperiode) {
			break
		}
		seeds = append(seeds, speriode)
	}

	toolkit.Println("Starting worker query...")
	result := make(chan toolkit.M, len(seeds))
	for i, v := range seeds {
		filter := dbox.Eq("date.date", v)
		go workerproc(i, filter, result)
	}

	step := len(seeds) / 100
	if step == 0 {
		step = 1
	}
	// "date.fiscal","date.quartertxt","date.month", "customer.branchname", "customer.keyaccount",
	// "customer.channelid", "customer.channelname", "customer.reportchannel", "customer.reportsubchannel",
	// "customer.zone", "customer.region", "customer.areaname","product.brand"
	toolkit.Println("Waiting result query...")
	for i := 1; i <= len(seeds); i++ {
		a := <-result
		for k, v := range a {
			lasttkm := toolkit.M{}
			newtkm, _ := toolkit.ToM(v)
			if alldata.Has(k) {
				lasttkm = alldata[k].(toolkit.M)
			}

			for tk, tv := range newtkm {
				df := toolkit.ToFloat64(tv, 6, toolkit.RoundingAuto) + lasttkm.GetFloat64(tk)
				lasttkm.Set(tk, df)
			}
			alldata.Set(k, lasttkm)
		}

		if i%step == 0 {
			toolkit.Printfn("Worker %d of %d (%d), Done in %s",
				i, len(seeds), (i / step), time.Since(t0).String())
		}
	}

	listdimension := []string{"date.fiscal,customer.channelid,customer.channelname",
		"date.fiscal,customer.channelid,customer.channelname,customer.reportsubchannel",
		"date.fiscal,customer.channelid,customer.channelname,customer.zone",
		"date.fiscal,customer.channelid,customer.channelname,customer.areaname",
		"date.fiscal,customer.channelid,customer.channelname,customer.region"}

	resdimension := make(chan int)
	dimension := make(chan string, 5)
	detaildata := make(chan toolkit.M)
	ressavedata := make(chan int)

	for i := 0; i < 3; i++ {
		go workerbuilddimension(i, dimension, detaildata, resdimension)
	}

	for i := 0; i < 10; i++ {
		go workersavedata(i, detaildata, ressavedata)
	}

	toolkit.Printfn("Prepare saving collection, Create dimension")
	for _, str := range listdimension {
		dimension <- str
		toolkit.Printfn("SEND : %v", str)
	}
	close(dimension)

	alldatarows := 0
	toolkit.Printfn("Waiting dimension result")
	for i := 0; i < len(listdimension); i++ {
		alldatarows += <-resdimension
	}
	close(detaildata)

	step = alldatarows / 100
	if step == 0 {
		step = 1
	}
	toolkit.Printfn("Saving dimension result")
	for i := 0; i < alldatarows; i++ {
		<-ressavedata
		if i%step == 0 {
			toolkit.Printfn("Data saved %d of %d (%d), Done in %s",
				i, alldatarows, (i / step), time.Since(t0).String())
		}
	}

	toolkit.Printfn("Processing done in %s",
		time.Since(t0).String())
}

func workerproc(wi int, filter *dbox.Filter, result chan<- toolkit.M) {
	workerconn, _ := modules.GetDboxIConnection("db_godrej")
	defer workerconn.Close()
	tkm := toolkit.M{}
	// "date.fiscal","date.quartertxt","date.month", "customer.branchname", "customer.keyaccount",
	// "customer.channelid", "customer.channelname", "customer.reportchannel", "customer.reportsubchannel",
	// "customer.zone", "customer.region", "customer.areaname","product.brand"
	csr, _ := workerconn.NewQuery().Select("date", "customer", "product.brand", "pldatas").
		From(tablename).
		Where(filter).
		Cursor(nil)

	defer csr.Close()

	scount := csr.Count()
	iscount := 0
	step := scount / 100

	if step == 0 {
		step = 1
	}

	for {
		iscount++
		spl := new(gdrj.SalesPL)
		e := csr.Fetch(spl, 1, false)
		if e != nil {
			break
		}

		key := toolkit.Sprintf("%s|%s|%d|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s", spl.Date.Fiscal, spl.Date.QuarterTxt, spl.Date.Month,
			spl.Customer.BranchName, spl.Customer.KeyAccount, spl.Customer.ChannelID, spl.Customer.ChannelName,
			spl.Customer.ReportChannel, spl.Customer.ReportSubChannel, spl.Customer.Zone, spl.Customer.Region,
			spl.Customer.AreaName, spl.Product.Brand)

		dtkm := toolkit.M{}
		if tkm.Has(key) {
			dtkm = tkm[key].(toolkit.M)
		}

		tv := spl.GrossAmount + dtkm.GetFloat64("grossamount")
		dtkm.Set("grossamount", tv)

		tv = spl.NetAmount + dtkm.GetFloat64("netamount")
		dtkm.Set("netamount", tv)

		tv = spl.DiscountAmount + dtkm.GetFloat64("discountamount")
		dtkm.Set("discountamount", tv)

		tv = spl.SalesQty + dtkm.GetFloat64("salesqty")
		dtkm.Set("salesqty", tv)

		for k, v := range spl.PLDatas {
			tv := v.Amount + dtkm.GetFloat64(k)
			dtkm.Set(k, tv)
		}

		tkm.Set(key, dtkm)

		if iscount == 3 {
			break
		}

	}

	result <- tkm

	toolkit.Printfn("Go %d. Processing done in %s",
		wi,
		time.Since(t0).String())
}

func workerbuilddimension(wi int, dimension <-chan string, detaildata chan<- toolkit.M, resdimension chan<- int) {
	//"date.fiscal", "customer.channelid", "customer.channelname"
	sortkeys := []string{"date.fiscal", "date.quartertxt", "date.month",
		"customer.branchname", "customer.keyaccount", "customer.channelid", "customer.channelname", "customer.reportchannel",
		"customer.reportsubchannel", "customer.zone", "customer.region", "customer.areaname", "product.brand"}

	str := ""
	for str = range dimension {
		toolkit.Println(str)
		payload := new(gdrj.PLFinderParam)
		payload.Breakdowns = strings.Split(str, ",")
		tablename := toolkit.Sprintf("1-%v", payload.GetTableName())

		tkm := toolkit.M{}
		for key, val := range alldata {
			// toolkit.Printfn("%s", key)
			arrkey := strings.Split(key, "|")
			// toolkit.Printfn("%d - %d", len(sortkeys), len(arrkey))
			dkey := ""
			for i, v := range sortkeys {
				for _, vx := range payload.Breakdowns {
					if vx == v {
						if len(dkey) > 0 {
							dkey = toolkit.Sprintf("%s|%s", dkey, arrkey[i])
						} else {
							dkey = arrkey[i]
						}
					}
				}
			}

			stkm, _ := toolkit.ToM(val)
			dtkm := toolkit.M{}
			if tkm.Has(dkey) {
				dtkm = tkm[dkey].(toolkit.M)
			}

			for k, v := range stkm {
				fv := toolkit.ToFloat64(v, 6, toolkit.RoundingAuto) + dtkm.GetFloat64(k)
				dtkm.Set(k, fv)
			}

			tkm.Set(dkey, dtkm)
		}
		for k, v := range tkm {
			a, _ := toolkit.ToM(v)
			id := toolkit.M{}
			arrk := strings.Split(k, "|")
			for i, sv := range payload.Breakdowns {
				tsv := strings.Replace(sv, ".", "_", -1)
				id.Set(tsv, arrk[i])
			}
			a.Set("_id", id)

			detaildata <- toolkit.M{}.Set(tablename, a)
		}

		resdimension <- len(tkm)
		toolkit.Println("SUM Data : ", len(tkm))
	}
}

func workersavedata(wi int, detaildata <-chan toolkit.M, ressavedata chan<- int) {
	workerconn, _ := modules.GetDboxIConnection("db_godrej")
	defer workerconn.Close()

	tkm := toolkit.M{}
	qs := map[string]dbox.Query{}
	for tkm = range detaildata {
		for tbl, dt := range tkm {
			q, exist := qs[tbl]
			if !exist {
				q = workerconn.NewQuery().From(tbl).
					Save().
					SetConfig("multiexec", true)
				qs[tbl] = q
			}
			esave := q.Exec(toolkit.M{}.Set("data", dt))
			if esave != nil {
				toolkit.Printfn("Can't save %s - %s : %v", tbl, esave.Error(), dt)
				os.Exit(100)
			}
			ressavedata <- 1
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Matthew Holt and The Caddy Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package caddyhttp

import (
	"encoding/json"
	"fmt"
	"net/http"
	"reflect"
	"regexp"
	"strings"

	"github.com/caddyserver/caddy/v2"
	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
	"github.com/google/cel-go/cel"
	"github.com/google/cel-go/checker/decls"
	"github.com/google/cel-go/common/types"
	"github.com/google/cel-go/common/types/ref"
	"github.com/google/cel-go/common/types/traits"
	"github.com/google/cel-go/ext"
	"github.com/google/cel-go/interpreter/functions"
	exprpb "google.golang.org/genproto/googleapis/api/expr/v1alpha1"
)

func init() {
	caddy.RegisterModule(MatchExpression{})
}

// MatchExpression matches requests by evaluating a
// [CEL](https://github.com/google/cel-spec) expression.
// This enables complex logic to be expressed using a comfortable,
// familiar syntax.
//
// COMPATIBILITY NOTE: This module is still experimental and is not
// subject to Caddy's compatibility guarantee.
type MatchExpression struct {
	// The CEL expression to evaluate. Any Caddy placeholders
	// will be expanded and situated into proper CEL function
	// calls before evaluating.
	Expr string

	expandedExpr string
	prg          cel.Program
}

// CaddyModule returns the Caddy module information.
func (MatchExpression) CaddyModule() caddy.ModuleInfo {
	return caddy.ModuleInfo{
		ID:  "http.matchers.expression",
		New: func() caddy.Module { return new(MatchExpression) },
	}
}

// MarshalJSON marshals m's expression.
func (m MatchExpression) MarshalJSON() ([]byte, error) {
	return json.Marshal(m.Expr)
}

// UnmarshalJSON unmarshals m's expression.
func (m *MatchExpression) UnmarshalJSON(data []byte) error {
	return json.Unmarshal(data, &m.Expr)
}

// Provision sets ups m.
func (m *MatchExpression) Provision(_ caddy.Context) error {
	// replace placeholders with a function call - this is just some
	// light (and possibly naïve) syntactic sugar
	m.expandedExpr = placeholderRegexp.ReplaceAllString(m.Expr, placeholderExpansion)

	// create the CEL environment
	env, err := cel.NewEnv(
		cel.Declarations(
			decls.NewIdent("request", httpRequestObjectType, nil),
			decls.NewFunction(placeholderFuncName,
				decls.NewOverload(placeholderFuncName+"_httpRequest_string",
					[]*exprpb.Type{httpRequestObjectType, decls.String},
					decls.Any)),
		),
		cel.CustomTypeAdapter(celHTTPRequestTypeAdapter{}),
		ext.Strings(),
	)
	if err != nil {
		return fmt.Errorf("setting up CEL environment: %v", err)
	}

	// parse the expression
	parsed, issues := env.Parse(m.expandedExpr)
	if issues != nil && issues.Err() != nil {
		return fmt.Errorf("parsing CEL program: %s", issues.Err())
	}

	// type-check it
	checked, issues := env.Check(parsed)
	if issues != nil && issues.Err() != nil {
		return fmt.Errorf("type-checking CEL program: %s", issues.Err())
	}

	// compile the "program"
	m.prg, err = env.Program(checked,
		cel.Functions(
			&functions.Overload{
				Operator: placeholderFuncName,
				Binary:   caddyPlaceholderFunc,
			},
		),
	)

	if err != nil {
		return fmt.Errorf("compiling CEL program: %s", err)
	}
	return nil
}

// Match returns true if r matches m.
func (m MatchExpression) Match(r *http.Request) bool {
	out, _, _ := m.prg.Eval(map[string]interface{}{
		"request": celHTTPRequest{r},
	})
	if outBool, ok := out.Value().(bool); ok {
		return outBool
	}
	return false

}

// UnmarshalCaddyfile implements caddyfile.Unmarshaler.
func (m *MatchExpression) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
	for d.Next() {
		m.Expr = strings.Join(d.RemainingArgs(), " ")
	}
	return nil
}

// httpRequestCELType is the type representation of a native HTTP request.
var httpRequestCELType = types.NewTypeValue("http.Request", traits.ReceiverType)

// cellHTTPRequest wraps an http.Request with
// methods to satisfy the ref.Val interface.
type celHTTPRequest struct {
	*http.Request
}

func (cr celHTTPRequest) ConvertToNative(typeDesc reflect.Type) (interface{}, error) {
	return cr.Request, nil
}
func (celHTTPRequest) ConvertToType(typeVal ref.Type) ref.Val {
	panic("not implemented")
}
func (cr celHTTPRequest) Equal(other ref.Val) ref.Val {
	if o, ok := other.Value().(celHTTPRequest); ok {
		return types.Bool(o.Request == cr.Request)
	}
	return types.ValOrErr(other, "%v is not comparable type", other)
}
func (celHTTPRequest) Type() ref.Type        { return httpRequestCELType }
func (cr celHTTPRequest) Value() interface{} { return cr }

// celHTTPRequestTypeAdapter can adapt a
// celHTTPRequest to a CEL value.
type celHTTPRequestTypeAdapter struct{}

func (celHTTPRequestTypeAdapter) NativeToValue(value interface{}) ref.Val {
	if celReq, ok := value.(celHTTPRequest); ok {
		return celReq
	}
	return types.DefaultTypeAdapter.NativeToValue(value)
}

// Variables used for replacing Caddy placeholders in CEL
// expressions with a proper CEL function call; this is
// just for syntactic sugar.
var (
	placeholderRegexp    = regexp.MustCompile(`{([\w.-]+)}`)
	placeholderExpansion = `caddyPlaceholder(request, "${1}")`
)

var httpRequestObjectType = decls.NewObjectType("http.Request")

// The name of the CEL function which accesses Replacer values.
const placeholderFuncName = "caddyPlaceholder"

// caddyPlaceholderFunc implements the custom CEL function that
// accesses the Replacer on a request and gets values from it.
func caddyPlaceholderFunc(lhs, rhs ref.Val) ref.Val {
	celReq, ok := lhs.(celHTTPRequest)
	if !ok {
		return types.NewErr(
			"invalid request of type '%v' to "+placeholderFuncName+"(request, placeholderVarName)",
			lhs.Type())
	}
	phStr, ok := rhs.(types.String)
	if !ok {
		return types.NewErr(
			"invalid placeholder variable name of type '%v' to "+placeholderFuncName+"(request, placeholderVarName)",
			rhs.Type())
	}

	repl := celReq.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer)
	val, _ := repl.Get(string(phStr))

	// TODO: this is... kinda awful and underwhelming, how can we expand CEL's type system more easily?
	switch v := val.(type) {
	case string:
		return types.String(v)
	case fmt.Stringer:
		return types.String(v.String())
	case error:
		return types.NewErr(v.Error())
	case int:
		return types.Int(v)
	case int32:
		return types.Int(v)
	case int64:
		return types.Int(v)
	case uint:
		return types.Int(v)
	case uint32:
		return types.Int(v)
	case uint64:
		return types.Int(v)
	case float32:
		return types.Double(v)
	case float64:
		return types.Double(v)
	case bool:
		return types.Bool(v)
	default:
		return types.String(fmt.Sprintf("%+v", v))
	}
}

// Interface guards
var (
	_ caddy.Provisioner     = (*MatchExpression)(nil)
	_ RequestMatcher        = (*MatchExpression)(nil)
	_ caddyfile.Unmarshaler = (*MatchExpression)(nil)
	_ json.Marshaler        = (*MatchExpression)(nil)
	_ json.Unmarshaler      = (*MatchExpression)(nil)
)
<commit_msg>cel: Leverage DefaultAdapter to extend CEL's type system<commit_after>// Copyright 2015 Matthew Holt and The Caddy Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package caddyhttp

import (
	"encoding/json"
	"fmt"
	"net/http"
	"reflect"
	"regexp"
	"strings"

	"github.com/caddyserver/caddy/v2"
	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
	"github.com/google/cel-go/cel"
	"github.com/google/cel-go/checker/decls"
	"github.com/google/cel-go/common/types"
	"github.com/google/cel-go/common/types/ref"
	"github.com/google/cel-go/common/types/traits"
	"github.com/google/cel-go/ext"
	"github.com/google/cel-go/interpreter/functions"
	exprpb "google.golang.org/genproto/googleapis/api/expr/v1alpha1"
)

func init() {
	caddy.RegisterModule(MatchExpression{})
}

// MatchExpression matches requests by evaluating a
// [CEL](https://github.com/google/cel-spec) expression.
// This enables complex logic to be expressed using a comfortable,
// familiar syntax.
//
// COMPATIBILITY NOTE: This module is still experimental and is not
// subject to Caddy's compatibility guarantee.
type MatchExpression struct {
	// The CEL expression to evaluate. Any Caddy placeholders
	// will be expanded and situated into proper CEL function
	// calls before evaluating.
	Expr string

	expandedExpr string
	prg          cel.Program
	ta           ref.TypeAdapter
}

// CaddyModule returns the Caddy module information.
func (MatchExpression) CaddyModule() caddy.ModuleInfo {
	return caddy.ModuleInfo{
		ID:  "http.matchers.expression",
		New: func() caddy.Module { return new(MatchExpression) },
	}
}

// MarshalJSON marshals m's expression.
func (m MatchExpression) MarshalJSON() ([]byte, error) {
	return json.Marshal(m.Expr)
}

// UnmarshalJSON unmarshals m's expression.
func (m *MatchExpression) UnmarshalJSON(data []byte) error {
	return json.Unmarshal(data, &m.Expr)
}

// Provision sets ups m.
func (m *MatchExpression) Provision(_ caddy.Context) error {
	// replace placeholders with a function call - this is just some
	// light (and possibly naïve) syntactic sugar
	m.expandedExpr = placeholderRegexp.ReplaceAllString(m.Expr, placeholderExpansion)

	// our type adapter expands CEL's standard type support
	m.ta = celTypeAdapter{}

	// create the CEL environment
	env, err := cel.NewEnv(
		cel.Declarations(
			decls.NewIdent("request", httpRequestObjectType, nil),
			decls.NewFunction(placeholderFuncName,
				decls.NewOverload(placeholderFuncName+"_httpRequest_string",
					[]*exprpb.Type{httpRequestObjectType, decls.String},
					decls.Any)),
		),
		cel.CustomTypeAdapter(m.ta),
		ext.Strings(),
	)
	if err != nil {
		return fmt.Errorf("setting up CEL environment: %v", err)
	}

	// parse the expression
	parsed, issues := env.Parse(m.expandedExpr)
	if issues != nil && issues.Err() != nil {
		return fmt.Errorf("parsing CEL program: %s", issues.Err())
	}

	// type-check it
	checked, issues := env.Check(parsed)
	if issues != nil && issues.Err() != nil {
		return fmt.Errorf("type-checking CEL program: %s", issues.Err())
	}

	// compile the "program"
	m.prg, err = env.Program(checked,
		cel.Functions(
			&functions.Overload{
				Operator: placeholderFuncName,
				Binary:   m.caddyPlaceholderFunc,
			},
		),
	)

	if err != nil {
		return fmt.Errorf("compiling CEL program: %s", err)
	}
	return nil
}

// Match returns true if r matches m.
func (m MatchExpression) Match(r *http.Request) bool {
	out, _, _ := m.prg.Eval(map[string]interface{}{
		"request": celHTTPRequest{r},
	})
	if outBool, ok := out.Value().(bool); ok {
		return outBool
	}
	return false

}

// UnmarshalCaddyfile implements caddyfile.Unmarshaler.
func (m *MatchExpression) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
	for d.Next() {
		m.Expr = strings.Join(d.RemainingArgs(), " ")
	}
	return nil
}

// caddyPlaceholderFunc implements the custom CEL function that accesses the
// Replacer on a request and gets values from it.
func (m MatchExpression) caddyPlaceholderFunc(lhs, rhs ref.Val) ref.Val {
	celReq, ok := lhs.(celHTTPRequest)
	if !ok {
		return types.NewErr(
			"invalid request of type '%v' to "+placeholderFuncName+"(request, placeholderVarName)",
			lhs.Type())
	}
	phStr, ok := rhs.(types.String)
	if !ok {
		return types.NewErr(
			"invalid placeholder variable name of type '%v' to "+placeholderFuncName+"(request, placeholderVarName)",
			rhs.Type())
	}

	repl := celReq.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer)
	val, _ := repl.Get(string(phStr))

	return m.ta.NativeToValue(val)
}

// httpRequestCELType is the type representation of a native HTTP request.
var httpRequestCELType = types.NewTypeValue("http.Request", traits.ReceiverType)

// cellHTTPRequest wraps an http.Request with
// methods to satisfy the ref.Val interface.
type celHTTPRequest struct{ *http.Request }

func (cr celHTTPRequest) ConvertToNative(typeDesc reflect.Type) (interface{}, error) {
	return cr.Request, nil
}
func (celHTTPRequest) ConvertToType(typeVal ref.Type) ref.Val {
	panic("not implemented")
}
func (cr celHTTPRequest) Equal(other ref.Val) ref.Val {
	if o, ok := other.Value().(celHTTPRequest); ok {
		return types.Bool(o.Request == cr.Request)
	}
	return types.ValOrErr(other, "%v is not comparable type", other)
}
func (celHTTPRequest) Type() ref.Type        { return httpRequestCELType }
func (cr celHTTPRequest) Value() interface{} { return cr }

// celTypeAdapter can adapt our custom types to a CEL value.
type celTypeAdapter struct{}

func (celTypeAdapter) NativeToValue(value interface{}) ref.Val {
	switch v := value.(type) {
	case celHTTPRequest:
		return v
	case error:
		types.NewErr(v.Error())
	}
	return types.DefaultTypeAdapter.NativeToValue(value)
}

// Variables used for replacing Caddy placeholders in CEL
// expressions with a proper CEL function call; this is
// just for syntactic sugar.
var (
	placeholderRegexp    = regexp.MustCompile(`{([\w.-]+)}`)
	placeholderExpansion = `caddyPlaceholder(request, "${1}")`
)

var httpRequestObjectType = decls.NewObjectType("http.Request")

// The name of the CEL function which accesses Replacer values.
const placeholderFuncName = "caddyPlaceholder"

// Interface guards
var (
	_ caddy.Provisioner     = (*MatchExpression)(nil)
	_ RequestMatcher        = (*MatchExpression)(nil)
	_ caddyfile.Unmarshaler = (*MatchExpression)(nil)
	_ json.Marshaler        = (*MatchExpression)(nil)
	_ json.Unmarshaler      = (*MatchExpression)(nil)
)
<|endoftext|>


<commit_before>// abbreviation-checker checks the names in a file of code for possible abbreviations.
package main

import (
	"bufio"
	"flag"
	"fmt"
	"github.com/BluntSporks/abbreviation"
	"log"
	"os"
	"regexp"
	"strings"
)

func main() {
	// Parsing flags.
	lang := flag.String("lang", "", "Name of language whose keywords should be ignored")
	file := flag.String("file", "", "Name of source code file to check")
	short := flag.String("short", "", "Short word to check if valid abbreviation of long")
	long := flag.String("long", "", "Long word to check if short is valid abbreviation of it")
	flag.Parse()

	// Check arguments.
	if *file != "" {
		checkFile(*file, *lang)
	} else if *short != "" && *long != "" {
		checkWord(strings.ToLower(*short), strings.ToLower(*long))
	} else {
		log.Fatal("Bad arguments")
	}
}

// checkFile checks a file.
func checkFile(file string, lang string) {
	// Match words.
	wordRegExp := regexp.MustCompile(`\w+`)

	// Ignore strings and line comments.
	ignoreRegExp := regexp.MustCompile(`".*?"|//.*`)

	// Open file.
	hdl, err := os.Open(file)
	if err != nil {
		log.Fatal(err)
	}
	defer hdl.Close()

	// Scan file line by line, removing dupes.
	scanner := bufio.NewScanner(hdl)

	// Keep track of whether the last line was empty or not.
	filePrinted := false
	for scanner.Scan() {
		line := scanner.Text()
		if err := scanner.Err(); err != nil {
			log.Fatal(err)
		}
		bare := ignoreRegExp.ReplaceAllString(strings.ToLower(line), "")
		if !filePrinted {
			// Add filename so it gets checked too.
			bare = strings.ToLower(file) + " " + bare
		}
		words := wordRegExp.FindAllString(bare, -1)
		matches := make(map[string]string)
		for _, word := range words {
			short := abbr.LookUp(word)
			if short != "" && !abbr.IsKeyword(lang, word) {
				matches[word] = short
			}
		}
		if len(matches) > 0 {
			if !filePrinted {
				fmt.Printf("File: %s\n", file)
				filePrinted = true
			}
			fmt.Printf("Line: %s\n", strings.TrimSpace(line))
			for long, short := range matches {
				fmt.Printf("%s => %s\n", long, short)
			}
			fmt.Println()
		}
	}
}

// checkWord checks a word.
func checkWord(short string, long string) {
	pass := true
	if !abbr.AllLetters(short, long) {
		fmt.Println("Fail: All letters of long form not found in same order in short form")
		pass = false
	}
	if !abbr.HasVowel(short) {
		fmt.Println("Fail: Short form should have at least one vowel or y to make it pronounceable")
		pass = false
	}
	if !abbr.IsShortEnough(short, long) {
		fmt.Println("Fail: Short form should be no more than half as long as long form")
		pass = false
	}
	if pass {
		fmt.Println("Pass")
	}
}
<commit_msg>Adding long enough test<commit_after>// abbreviation-checker checks the names in a file of code for possible abbreviations.
package main

import (
	"bufio"
	"flag"
	"fmt"
	"github.com/BluntSporks/abbreviation"
	"log"
	"os"
	"regexp"
	"strings"
)

func main() {
	// Parsing flags.
	lang := flag.String("lang", "", "Name of language whose keywords should be ignored")
	file := flag.String("file", "", "Name of source code file to check")
	short := flag.String("short", "", "Short word to check if valid abbreviation of long")
	long := flag.String("long", "", "Long word to check if short is valid abbreviation of it")
	flag.Parse()

	// Check arguments.
	if *file != "" {
		checkFile(*file, *lang)
	} else if *short != "" && *long != "" {
		checkWord(strings.ToLower(*short), strings.ToLower(*long))
	} else {
		log.Fatal("Bad arguments")
	}
}

// checkFile checks a file.
func checkFile(file string, lang string) {
	// Match words.
	wordRegExp := regexp.MustCompile(`\w+`)

	// Ignore strings and line comments.
	ignoreRegExp := regexp.MustCompile(`".*?"|//.*`)

	// Open file.
	hdl, err := os.Open(file)
	if err != nil {
		log.Fatal(err)
	}
	defer hdl.Close()

	// Scan file line by line, removing dupes.
	scanner := bufio.NewScanner(hdl)

	// Keep track of whether the last line was empty or not.
	filePrinted := false
	for scanner.Scan() {
		line := scanner.Text()
		if err := scanner.Err(); err != nil {
			log.Fatal(err)
		}
		bare := ignoreRegExp.ReplaceAllString(strings.ToLower(line), "")
		if !filePrinted {
			// Add filename so it gets checked too.
			bare = strings.ToLower(file) + " " + bare
		}
		words := wordRegExp.FindAllString(bare, -1)
		matches := make(map[string]string)
		for _, word := range words {
			short := abbr.LookUp(word)
			if short != "" && !abbr.IsKeyword(lang, word) {
				matches[word] = short
			}
		}
		if len(matches) > 0 {
			if !filePrinted {
				fmt.Printf("File: %s\n", file)
				filePrinted = true
			}
			fmt.Printf("Line: %s\n", strings.TrimSpace(line))
			for long, short := range matches {
				fmt.Printf("%s => %s\n", long, short)
			}
			fmt.Println()
		}
	}
}

// checkWord checks a word.
func checkWord(short string, long string) {
	pass := true
	if !abbr.AllLetters(short, long) {
		fmt.Println("Fail: All letters of long form not found in same order in short form")
		pass = false
	}
	if !abbr.HasVowel(short) {
		fmt.Println("Fail: Short form should have at least one vowel or y to make it pronounceable")
		pass = false
	}
	if !abbr.IsLongEnough(short) {
		fmt.Println("Fail: Short form should be at least 3 characters long")
		pass = false
	}
	if !abbr.IsShortEnough(short, long) {
		fmt.Println("Fail: Short form should be no more than half as long as long form")
		pass = false
	}
	if pass {
		fmt.Println("Pass")
	}
}
<|endoftext|>


<commit_before>package leveldb

import (
	"path/filepath"
	"strings"

	"github.com/n0stack/n0stack/n0core/pkg/datastore/store"

	"github.com/pkg/errors"
	"github.com/syndtr/goleveldb/leveldb"
	lerrors "github.com/syndtr/goleveldb/leveldb/errors"
)

type LeveldbStore struct {
	db *leveldb.DB

	prefix string
}

func NewLeveldbStore(directory string) (*LeveldbStore, error) {
	db, err := leveldb.OpenFile("test.db", nil)
	if err != nil {
		return nil, errors.Wrap(err, "failed to connect database")
	}

	return &LeveldbStore{
		db: db,
	}, nil
}

func (ds *LeveldbStore) Close() error {
	return ds.db.Close()
}

func (ds *LeveldbStore) AddPrefix(prefix string) *LeveldbStore {
	return &LeveldbStore{
		db:     ds.db,
		prefix: filepath.Join(ds.prefix, prefix),
	}
}

func (ds *LeveldbStore) List() ([][]byte, error) {
	res := make([][]byte, 0)

	iter := ds.db.NewIterator(nil, nil)
	for iter.Next() {
		key := string(iter.Key())

		if strings.HasPrefix(key, ds.prefix) {
			res = append(res, iter.Value())
		}
	}
	iter.Release()

	if err := iter.Error(); err != nil {
		return nil, err
	}

	return res, nil
}

func (ds *LeveldbStore) Get(key string) ([]byte, error) {
	v, err := ds.db.Get(ds.getKey(key), nil)
	if err != nil {
		if err == lerrors.ErrNotFound {
			return nil, store.NewNotFound(key)
		}

		return nil, err
	}

	return v, nil
}

func (ds *LeveldbStore) Apply(key string, value []byte) error {
	return ds.db.Put(ds.getKey(key), value, nil)
}

func (ds *LeveldbStore) Delete(key string) error {
	return ds.db.Delete(ds.getKey(key), nil)
}

func (ds LeveldbStore) getKey(key string) []byte {
	return []byte(filepath.Join(ds.prefix, key))
}
<commit_msg>leveldb list with BytesPrefix<commit_after>package leveldb

import (
	"path/filepath"

	"github.com/n0stack/n0stack/n0core/pkg/datastore/store"

	"github.com/pkg/errors"
	"github.com/syndtr/goleveldb/leveldb"
	lerrors "github.com/syndtr/goleveldb/leveldb/errors"
	"github.com/syndtr/goleveldb/leveldb/util"
)

type LeveldbStore struct {
	db *leveldb.DB

	prefix string
}

func NewLeveldbStore(directory string) (*LeveldbStore, error) {
	db, err := leveldb.OpenFile("test.db", nil)
	if err != nil {
		return nil, errors.Wrap(err, "failed to connect database")
	}

	return &LeveldbStore{
		db: db,
	}, nil
}

func (ds *LeveldbStore) Close() error {
	return ds.db.Close()
}

func (ds *LeveldbStore) AddPrefix(prefix string) *LeveldbStore {
	return &LeveldbStore{
		db:     ds.db,
		prefix: filepath.Join(ds.prefix, prefix),
	}
}

func (ds *LeveldbStore) List() ([][]byte, error) {
	res := make([][]byte, 0)

	iter := ds.db.NewIterator(util.BytesPrefix([]byte(ds.prefix)), nil)
	for iter.Next() {
		res = append(res, iter.Value())
	}
	iter.Release()

	if err := iter.Error(); err != nil {
		return nil, err
	}

	return res, nil
}

func (ds *LeveldbStore) Get(key string) ([]byte, error) {
	v, err := ds.db.Get(ds.getKey(key), nil)
	if err != nil {
		if err == lerrors.ErrNotFound {
			return nil, store.NewNotFound(key)
		}

		return nil, err
	}

	return v, nil
}

func (ds *LeveldbStore) Apply(key string, value []byte) error {
	return ds.db.Put(ds.getKey(key), value, nil)
}

func (ds *LeveldbStore) Delete(key string) error {
	return ds.db.Delete(ds.getKey(key), nil)
}

func (ds LeveldbStore) getKey(key string) []byte {
	return []byte(filepath.Join(ds.prefix, key))
}
<|endoftext|>