<commit_before>package kv

import (
	"context"
	"errors"
	"fmt"
	"sort"
	"strconv"
	"strings"

	"github.com/google/orderedcode"
	dbm "github.com/tendermint/tm-db"

	abci "github.com/tendermint/tendermint/abci/types"
	"github.com/tendermint/tendermint/libs/pubsub/query"
	"github.com/tendermint/tendermint/state/indexer"
	"github.com/tendermint/tendermint/types"
)

var _ indexer.BlockIndexer = (*BlockerIndexer)(nil)

// BlockerIndexer implements a block indexer, indexing BeginBlock and EndBlock
// events with an underlying KV store. Block events are indexed by their height,
// such that matching search criteria returns the respective block height(s).
type BlockerIndexer struct {
	store dbm.DB
}

func New(store dbm.DB) *BlockerIndexer {
	return &BlockerIndexer{
		store: store,
	}
}

// Has returns true if the given height has been indexed. An error is returned
// upon database query failure.
func (idx *BlockerIndexer) Has(height int64) (bool, error) {
	key, err := heightKey(height)
	if err != nil {
		return false, fmt.Errorf("failed to create block height index key: %w", err)
	}

	return idx.store.Has(key)
}

// Index indexes BeginBlock and EndBlock events for a given block by its height.
// The following is indexed:
//
// primary key: encode(block.height | height) => encode(height)
// BeginBlock events: encode(eventType.eventAttr|eventValue|height|begin_block) => encode(height)
// EndBlock events: encode(eventType.eventAttr|eventValue|height|end_block) => encode(height)
func (idx *BlockerIndexer) Index(bh types.EventDataNewBlockHeader) error {
	batch := idx.store.NewBatch()
	defer batch.Close()

	height := bh.Header.Height

	// 1. index by height
	key, err := heightKey(height)
	if err != nil {
		return fmt.Errorf("failed to create block height index key: %w", err)
	}
	if err := batch.Set(key, int64ToBytes(height)); err != nil {
		return err
	}

	// 2. index BeginBlock events
	if err := idx.indexEvents(batch, bh.ResultBeginBlock.Events, "begin_block", height); err != nil {
		return fmt.Errorf("failed to index BeginBlock events: %w", err)
	}

	// 3. index EndBlock events
	if err := idx.indexEvents(batch, bh.ResultEndBlock.Events, "end_block", height); err != nil {
		return fmt.Errorf("failed to index EndBlock events: %w", err)
	}

	return batch.WriteSync()
}

// Search performs a query for block heights that match a given BeginBlock
// and Endblock event search criteria. The given query can match against zero,
// one or more block heights. In the case of height queries, i.e. block.height=H,
// if the height is indexed, that height alone will be returned. An error and
// nil slice is returned. Otherwise, a non-nil slice and nil error is returned.
func (idx *BlockerIndexer) Search(ctx context.Context, q *query.Query) ([]int64, error) {
	results := make([]int64, 0)
	select {
	case <-ctx.Done():
		return results, nil

	default:
	}

	conditions, err := q.Conditions()
	if err != nil {
		return nil, fmt.Errorf("failed to parse query conditions: %w", err)
	}

	// If there is an exact height query, return the result immediately
	// (if it exists).
	height, ok := lookForHeight(conditions)
	if ok {
		ok, err := idx.Has(height)
		if err != nil {
			return nil, err
		}

		if ok {
			return []int64{height}, nil
		}

		return results, nil
	}

	var heightsInitialized bool
	filteredHeights := make(map[string][]byte)

	// conditions to skip because they're handled before "everything else"
	skipIndexes := make([]int, 0)

	// Extract ranges. If both upper and lower bounds exist, it's better to get
	// them in order as to not iterate over kvs that are not within range.
	ranges, rangeIndexes := indexer.LookForRanges(conditions)
	if len(ranges) > 0 {
		skipIndexes = append(skipIndexes, rangeIndexes...)

		for _, qr := range ranges {
			prefix, err := orderedcode.Append(nil, qr.Key)
			if err != nil {
				return nil, fmt.Errorf("failed to create prefix key: %w", err)
			}

			if !heightsInitialized {
				filteredHeights, err = idx.matchRange(ctx, qr, prefix, filteredHeights, true)
				if err != nil {
					return nil, err
				}

				heightsInitialized = true

				// Ignore any remaining conditions if the first condition resulted in no
				// matches (assuming implicit AND operand).
				if len(filteredHeights) == 0 {
					break
				}
			} else {
				filteredHeights, err = idx.matchRange(ctx, qr, prefix, filteredHeights, false)
				if err != nil {
					return nil, err
				}
			}
		}
	}

	// for all other conditions
	for i, c := range conditions {
		if intInSlice(i, skipIndexes) {
			continue
		}

		startKey, err := orderedcode.Append(nil, c.CompositeKey, fmt.Sprintf("%v", c.Operand))
		if err != nil {
			return nil, err
		}

		if !heightsInitialized {
			filteredHeights, err = idx.match(ctx, c, startKey, filteredHeights, true)
			if err != nil {
				return nil, err
			}

			heightsInitialized = true

			// Ignore any remaining conditions if the first condition resulted in no
			// matches (assuming implicit AND operand).
			if len(filteredHeights) == 0 {
				break
			}
		} else {
			filteredHeights, err = idx.match(ctx, c, startKey, filteredHeights, false)
			if err != nil {
				return nil, err
			}
		}
	}

	// fetch matching heights
	results = make([]int64, 0, len(filteredHeights))
	for _, hBz := range filteredHeights {
		h := int64FromBytes(hBz)

		ok, err := idx.Has(h)
		if err != nil {
			return nil, err
		}
		if ok {
			results = append(results, h)
		}

		select {
		case <-ctx.Done():
			break

		default:
		}
	}

	sort.Slice(results, func(i, j int) bool { return results[i] < results[j] })

	return results, nil
}

// matchRange returns all matching block heights that match a given QueryRange
// and start key. An already filtered result (filteredHeights) is provided such
// that any non-intersecting matches are removed.
//
// NOTE: The provided filteredHeights may be empty if no previous condition has
// matched.
func (idx *BlockerIndexer) matchRange(
	ctx context.Context,
	qr indexer.QueryRange,
	startKey []byte,
	filteredHeights map[string][]byte,
	firstRun bool,
) (map[string][]byte, error) {

	// A previous match was attempted but resulted in no matches, so we return
	// no matches (assuming AND operand).
	if !firstRun && len(filteredHeights) == 0 {
		return filteredHeights, nil
	}

	tmpHeights := make(map[string][]byte)
	lowerBound := qr.LowerBoundValue()
	upperBound := qr.UpperBoundValue()

	it, err := dbm.IteratePrefix(idx.store, startKey)
	if err != nil {
		return nil, fmt.Errorf("failed to create prefix iterator: %w", err)
	}
	defer it.Close()

LOOP:
	for ; it.Valid(); it.Next() {
		var (
			eventValue string
			err        error
		)

		if qr.Key == types.BlockHeightKey {
			eventValue, err = parseValueFromPrimaryKey(it.Key())
		} else {
			eventValue, err = parseValueFromEventKey(it.Key())
		}

		if err != nil {
			continue
		}

		if _, ok := qr.AnyBound().(int64); ok {
			v, err := strconv.ParseInt(eventValue, 10, 64)
			if err != nil {
				continue LOOP
			}

			include := true
			if lowerBound != nil && v < lowerBound.(int64) {
				include = false
			}

			if upperBound != nil && v > upperBound.(int64) {
				include = false
			}

			if include {
				tmpHeights[string(it.Value())] = it.Value()
			}
		}

		select {
		case <-ctx.Done():
			break

		default:
		}
	}

	if err := it.Error(); err != nil {
		return nil, err
	}

	if len(tmpHeights) == 0 || firstRun {
		// Either:
		//
		// 1. Regardless if a previous match was attempted, which may have had
		// results, but no match was found for the current condition, then we
		// return no matches (assuming AND operand).
		//
		// 2. A previous match was not attempted, so we return all results.
		return tmpHeights, nil
	}

	// Remove/reduce matches in filteredHashes that were not found in this
	// match (tmpHashes).
	for k := range filteredHeights {
		if tmpHeights[k] == nil {
			delete(filteredHeights, k)

			select {
			case <-ctx.Done():
				break

			default:
			}
		}
	}

	return filteredHeights, nil
}

// match returns all matching heights that meet a given query condition and start
// key. An already filtered result (filteredHeights) is provided such that any
// non-intersecting matches are removed.
//
// NOTE: The provided filteredHeights may be empty if no previous condition has
// matched.
func (idx *BlockerIndexer) match(
	ctx context.Context,
	c query.Condition,
	startKeyBz []byte,
	filteredHeights map[string][]byte,
	firstRun bool,
) (map[string][]byte, error) {

	// A previous match was attempted but resulted in no matches, so we return
	// no matches (assuming AND operand).
	if !firstRun && len(filteredHeights) == 0 {
		return filteredHeights, nil
	}

	tmpHeights := make(map[string][]byte)

	switch {
	case c.Op == query.OpEqual:
		it, err := dbm.IteratePrefix(idx.store, startKeyBz)
		if err != nil {
			return nil, fmt.Errorf("failed to create prefix iterator: %w", err)
		}
		defer it.Close()

		for ; it.Valid(); it.Next() {
			tmpHeights[string(it.Value())] = it.Value()

			if err := ctx.Err(); err != nil {
				break
			}
		}

		if err := it.Error(); err != nil {
			return nil, err
		}

	case c.Op == query.OpExists:
		prefix, err := orderedcode.Append(nil, c.CompositeKey)
		if err != nil {
			return nil, err
		}

		it, err := dbm.IteratePrefix(idx.store, prefix)
		if err != nil {
			return nil, fmt.Errorf("failed to create prefix iterator: %w", err)
		}
		defer it.Close()

		for ; it.Valid(); it.Next() {
			tmpHeights[string(it.Value())] = it.Value()

			select {
			case <-ctx.Done():
				break

			default:
			}
		}

		if err := it.Error(); err != nil {
			return nil, err
		}

	case c.Op == query.OpContains:
		prefix, err := orderedcode.Append(nil, c.CompositeKey)
		if err != nil {
			return nil, err
		}

		it, err := dbm.IteratePrefix(idx.store, prefix)
		if err != nil {
			return nil, fmt.Errorf("failed to create prefix iterator: %w", err)
		}
		defer it.Close()

		for ; it.Valid(); it.Next() {
			eventValue, err := parseValueFromEventKey(it.Key())
			if err != nil {
				continue
			}

			if strings.Contains(eventValue, c.Operand.(string)) {
				tmpHeights[string(it.Value())] = it.Value()
			}

			select {
			case <-ctx.Done():
				break

			default:
			}
		}
		if err := it.Error(); err != nil {
			return nil, err
		}

	default:
		return nil, errors.New("other operators should be handled already")
	}

	if len(tmpHeights) == 0 || firstRun {
		// Either:
		//
		// 1. Regardless if a previous match was attempted, which may have had
		// results, but no match was found for the current condition, then we
		// return no matches (assuming AND operand).
		//
		// 2. A previous match was not attempted, so we return all results.
		return tmpHeights, nil
	}

	// Remove/reduce matches in filteredHeights that were not found in this
	// match (tmpHeights).
	for k := range filteredHeights {
		if tmpHeights[k] == nil {
			delete(filteredHeights, k)

			select {
			case <-ctx.Done():
				break

			default:
			}
		}
	}

	return filteredHeights, nil
}

func (idx *BlockerIndexer) indexEvents(batch dbm.Batch, events []abci.Event, typ string, height int64) error {
	heightBz := int64ToBytes(height)

	for _, event := range events {
		// only index events with a non-empty type
		if len(event.Type) == 0 {
			continue
		}

		for _, attr := range event.Attributes {
			if len(attr.Key) == 0 {
				continue
			}

			// index iff the event specified index:true and it's not a reserved event
			compositeKey := fmt.Sprintf("%s.%s", event.Type, string(attr.Key))
			if compositeKey == types.TxHashKey || compositeKey == types.TxHeightKey {
				return fmt.Errorf("event type and attribute key \"%s\" is reserved; please use a different key", compositeKey)
			}
			if attr.GetIndex() {
				key, err := eventKey(compositeKey, typ, string(attr.Value), height)
				if err != nil {
					return fmt.Errorf("failed to create block index key: %w", err)
				}

				if err := batch.Set(key, heightBz); err != nil {
					return err
				}
			}
		}
	}

	return nil
}
<commit_msg>state: fix block event indexing reserved key check (#6314) (#6315)<commit_after>package kv

import (
	"context"
	"errors"
	"fmt"
	"sort"
	"strconv"
	"strings"

	"github.com/google/orderedcode"
	dbm "github.com/tendermint/tm-db"

	abci "github.com/tendermint/tendermint/abci/types"
	"github.com/tendermint/tendermint/libs/pubsub/query"
	"github.com/tendermint/tendermint/state/indexer"
	"github.com/tendermint/tendermint/types"
)

var _ indexer.BlockIndexer = (*BlockerIndexer)(nil)

// BlockerIndexer implements a block indexer, indexing BeginBlock and EndBlock
// events with an underlying KV store. Block events are indexed by their height,
// such that matching search criteria returns the respective block height(s).
type BlockerIndexer struct {
	store dbm.DB
}

func New(store dbm.DB) *BlockerIndexer {
	return &BlockerIndexer{
		store: store,
	}
}

// Has returns true if the given height has been indexed. An error is returned
// upon database query failure.
func (idx *BlockerIndexer) Has(height int64) (bool, error) {
	key, err := heightKey(height)
	if err != nil {
		return false, fmt.Errorf("failed to create block height index key: %w", err)
	}

	return idx.store.Has(key)
}

// Index indexes BeginBlock and EndBlock events for a given block by its height.
// The following is indexed:
//
// primary key: encode(block.height | height) => encode(height)
// BeginBlock events: encode(eventType.eventAttr|eventValue|height|begin_block) => encode(height)
// EndBlock events: encode(eventType.eventAttr|eventValue|height|end_block) => encode(height)
func (idx *BlockerIndexer) Index(bh types.EventDataNewBlockHeader) error {
	batch := idx.store.NewBatch()
	defer batch.Close()

	height := bh.Header.Height

	// 1. index by height
	key, err := heightKey(height)
	if err != nil {
		return fmt.Errorf("failed to create block height index key: %w", err)
	}
	if err := batch.Set(key, int64ToBytes(height)); err != nil {
		return err
	}

	// 2. index BeginBlock events
	if err := idx.indexEvents(batch, bh.ResultBeginBlock.Events, "begin_block", height); err != nil {
		return fmt.Errorf("failed to index BeginBlock events: %w", err)
	}

	// 3. index EndBlock events
	if err := idx.indexEvents(batch, bh.ResultEndBlock.Events, "end_block", height); err != nil {
		return fmt.Errorf("failed to index EndBlock events: %w", err)
	}

	return batch.WriteSync()
}

// Search performs a query for block heights that match a given BeginBlock
// and Endblock event search criteria. The given query can match against zero,
// one or more block heights. In the case of height queries, i.e. block.height=H,
// if the height is indexed, that height alone will be returned. An error and
// nil slice is returned. Otherwise, a non-nil slice and nil error is returned.
func (idx *BlockerIndexer) Search(ctx context.Context, q *query.Query) ([]int64, error) {
	results := make([]int64, 0)
	select {
	case <-ctx.Done():
		return results, nil

	default:
	}

	conditions, err := q.Conditions()
	if err != nil {
		return nil, fmt.Errorf("failed to parse query conditions: %w", err)
	}

	// If there is an exact height query, return the result immediately
	// (if it exists).
	height, ok := lookForHeight(conditions)
	if ok {
		ok, err := idx.Has(height)
		if err != nil {
			return nil, err
		}

		if ok {
			return []int64{height}, nil
		}

		return results, nil
	}

	var heightsInitialized bool
	filteredHeights := make(map[string][]byte)

	// conditions to skip because they're handled before "everything else"
	skipIndexes := make([]int, 0)

	// Extract ranges. If both upper and lower bounds exist, it's better to get
	// them in order as to not iterate over kvs that are not within range.
	ranges, rangeIndexes := indexer.LookForRanges(conditions)
	if len(ranges) > 0 {
		skipIndexes = append(skipIndexes, rangeIndexes...)

		for _, qr := range ranges {
			prefix, err := orderedcode.Append(nil, qr.Key)
			if err != nil {
				return nil, fmt.Errorf("failed to create prefix key: %w", err)
			}

			if !heightsInitialized {
				filteredHeights, err = idx.matchRange(ctx, qr, prefix, filteredHeights, true)
				if err != nil {
					return nil, err
				}

				heightsInitialized = true

				// Ignore any remaining conditions if the first condition resulted in no
				// matches (assuming implicit AND operand).
				if len(filteredHeights) == 0 {
					break
				}
			} else {
				filteredHeights, err = idx.matchRange(ctx, qr, prefix, filteredHeights, false)
				if err != nil {
					return nil, err
				}
			}
		}
	}

	// for all other conditions
	for i, c := range conditions {
		if intInSlice(i, skipIndexes) {
			continue
		}

		startKey, err := orderedcode.Append(nil, c.CompositeKey, fmt.Sprintf("%v", c.Operand))
		if err != nil {
			return nil, err
		}

		if !heightsInitialized {
			filteredHeights, err = idx.match(ctx, c, startKey, filteredHeights, true)
			if err != nil {
				return nil, err
			}

			heightsInitialized = true

			// Ignore any remaining conditions if the first condition resulted in no
			// matches (assuming implicit AND operand).
			if len(filteredHeights) == 0 {
				break
			}
		} else {
			filteredHeights, err = idx.match(ctx, c, startKey, filteredHeights, false)
			if err != nil {
				return nil, err
			}
		}
	}

	// fetch matching heights
	results = make([]int64, 0, len(filteredHeights))
	for _, hBz := range filteredHeights {
		h := int64FromBytes(hBz)

		ok, err := idx.Has(h)
		if err != nil {
			return nil, err
		}
		if ok {
			results = append(results, h)
		}

		select {
		case <-ctx.Done():
			break

		default:
		}
	}

	sort.Slice(results, func(i, j int) bool { return results[i] < results[j] })

	return results, nil
}

// matchRange returns all matching block heights that match a given QueryRange
// and start key. An already filtered result (filteredHeights) is provided such
// that any non-intersecting matches are removed.
//
// NOTE: The provided filteredHeights may be empty if no previous condition has
// matched.
func (idx *BlockerIndexer) matchRange(
	ctx context.Context,
	qr indexer.QueryRange,
	startKey []byte,
	filteredHeights map[string][]byte,
	firstRun bool,
) (map[string][]byte, error) {

	// A previous match was attempted but resulted in no matches, so we return
	// no matches (assuming AND operand).
	if !firstRun && len(filteredHeights) == 0 {
		return filteredHeights, nil
	}

	tmpHeights := make(map[string][]byte)
	lowerBound := qr.LowerBoundValue()
	upperBound := qr.UpperBoundValue()

	it, err := dbm.IteratePrefix(idx.store, startKey)
	if err != nil {
		return nil, fmt.Errorf("failed to create prefix iterator: %w", err)
	}
	defer it.Close()

LOOP:
	for ; it.Valid(); it.Next() {
		var (
			eventValue string
			err        error
		)

		if qr.Key == types.BlockHeightKey {
			eventValue, err = parseValueFromPrimaryKey(it.Key())
		} else {
			eventValue, err = parseValueFromEventKey(it.Key())
		}

		if err != nil {
			continue
		}

		if _, ok := qr.AnyBound().(int64); ok {
			v, err := strconv.ParseInt(eventValue, 10, 64)
			if err != nil {
				continue LOOP
			}

			include := true
			if lowerBound != nil && v < lowerBound.(int64) {
				include = false
			}

			if upperBound != nil && v > upperBound.(int64) {
				include = false
			}

			if include {
				tmpHeights[string(it.Value())] = it.Value()
			}
		}

		select {
		case <-ctx.Done():
			break

		default:
		}
	}

	if err := it.Error(); err != nil {
		return nil, err
	}

	if len(tmpHeights) == 0 || firstRun {
		// Either:
		//
		// 1. Regardless if a previous match was attempted, which may have had
		// results, but no match was found for the current condition, then we
		// return no matches (assuming AND operand).
		//
		// 2. A previous match was not attempted, so we return all results.
		return tmpHeights, nil
	}

	// Remove/reduce matches in filteredHashes that were not found in this
	// match (tmpHashes).
	for k := range filteredHeights {
		if tmpHeights[k] == nil {
			delete(filteredHeights, k)

			select {
			case <-ctx.Done():
				break

			default:
			}
		}
	}

	return filteredHeights, nil
}

// match returns all matching heights that meet a given query condition and start
// key. An already filtered result (filteredHeights) is provided such that any
// non-intersecting matches are removed.
//
// NOTE: The provided filteredHeights may be empty if no previous condition has
// matched.
func (idx *BlockerIndexer) match(
	ctx context.Context,
	c query.Condition,
	startKeyBz []byte,
	filteredHeights map[string][]byte,
	firstRun bool,
) (map[string][]byte, error) {

	// A previous match was attempted but resulted in no matches, so we return
	// no matches (assuming AND operand).
	if !firstRun && len(filteredHeights) == 0 {
		return filteredHeights, nil
	}

	tmpHeights := make(map[string][]byte)

	switch {
	case c.Op == query.OpEqual:
		it, err := dbm.IteratePrefix(idx.store, startKeyBz)
		if err != nil {
			return nil, fmt.Errorf("failed to create prefix iterator: %w", err)
		}
		defer it.Close()

		for ; it.Valid(); it.Next() {
			tmpHeights[string(it.Value())] = it.Value()

			if err := ctx.Err(); err != nil {
				break
			}
		}

		if err := it.Error(); err != nil {
			return nil, err
		}

	case c.Op == query.OpExists:
		prefix, err := orderedcode.Append(nil, c.CompositeKey)
		if err != nil {
			return nil, err
		}

		it, err := dbm.IteratePrefix(idx.store, prefix)
		if err != nil {
			return nil, fmt.Errorf("failed to create prefix iterator: %w", err)
		}
		defer it.Close()

		for ; it.Valid(); it.Next() {
			tmpHeights[string(it.Value())] = it.Value()

			select {
			case <-ctx.Done():
				break

			default:
			}
		}

		if err := it.Error(); err != nil {
			return nil, err
		}

	case c.Op == query.OpContains:
		prefix, err := orderedcode.Append(nil, c.CompositeKey)
		if err != nil {
			return nil, err
		}

		it, err := dbm.IteratePrefix(idx.store, prefix)
		if err != nil {
			return nil, fmt.Errorf("failed to create prefix iterator: %w", err)
		}
		defer it.Close()

		for ; it.Valid(); it.Next() {
			eventValue, err := parseValueFromEventKey(it.Key())
			if err != nil {
				continue
			}

			if strings.Contains(eventValue, c.Operand.(string)) {
				tmpHeights[string(it.Value())] = it.Value()
			}

			select {
			case <-ctx.Done():
				break

			default:
			}
		}
		if err := it.Error(); err != nil {
			return nil, err
		}

	default:
		return nil, errors.New("other operators should be handled already")
	}

	if len(tmpHeights) == 0 || firstRun {
		// Either:
		//
		// 1. Regardless if a previous match was attempted, which may have had
		// results, but no match was found for the current condition, then we
		// return no matches (assuming AND operand).
		//
		// 2. A previous match was not attempted, so we return all results.
		return tmpHeights, nil
	}

	// Remove/reduce matches in filteredHeights that were not found in this
	// match (tmpHeights).
	for k := range filteredHeights {
		if tmpHeights[k] == nil {
			delete(filteredHeights, k)

			select {
			case <-ctx.Done():
				break

			default:
			}
		}
	}

	return filteredHeights, nil
}

func (idx *BlockerIndexer) indexEvents(batch dbm.Batch, events []abci.Event, typ string, height int64) error {
	heightBz := int64ToBytes(height)

	for _, event := range events {
		// only index events with a non-empty type
		if len(event.Type) == 0 {
			continue
		}

		for _, attr := range event.Attributes {
			if len(attr.Key) == 0 {
				continue
			}

			// index iff the event specified index:true and it's not a reserved event
			compositeKey := fmt.Sprintf("%s.%s", event.Type, string(attr.Key))
			if compositeKey == types.BlockHeightKey {
				return fmt.Errorf("event type and attribute key \"%s\" is reserved; please use a different key", compositeKey)
			}

			if attr.GetIndex() {
				key, err := eventKey(compositeKey, typ, string(attr.Value), height)
				if err != nil {
					return fmt.Errorf("failed to create block index key: %w", err)
				}

				if err := batch.Set(key, heightBz); err != nil {
					return err
				}
			}
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package models

import (
	"bytes"
	"fmt"
	"html/template"
	"net/url"
	"socialapi/config"
	"time"
)

const (
	DATEFORMAT = "Jan 02"
	TIMEFORMAT = "15:04"
)

type TemplateParser struct {
	UserContact *UserContact
}

var (
	mainTemplateFile        = "../socialapi/workers/emailnotifier/templates/main.tmpl"
	footerTemplateFile      = "../socialapi/workers/emailnotifier/templates/footer.tmpl"
	contentTemplateFile     = "../socialapi/workers/emailnotifier/templates/content.tmpl"
	gravatarTemplateFile    = "../socialapi/workers/emailnotifier/templates/gravatar.tmpl"
	groupTemplateFile       = "../socialapi/workers/emailnotifier/templates/group.tmpl"
	previewTemplateFile     = "../socialapi/workers/emailnotifier/templates/preview.tmpl"
	objectTemplateFile      = "../socialapi/workers/emailnotifier/templates/object.tmpl"
	unsubscribeTemplateFile = "../socialapi/workers/emailnotifier/templates/unsubscribe.tmpl"
)

func NewTemplateParser() *TemplateParser {
	return &TemplateParser{}
}

func (tp *TemplateParser) RenderInstantTemplate(mc *MailerContainer) (string, error) {
	if err := tp.validateTemplateParser(); err != nil {
		return "", err
	}

	ec, err := buildEventContent(mc)
	if err != nil {
		return "", err
	}
	content := tp.renderContentTemplate(ec, mc)

	return tp.renderTemplate(mc.Content.TypeConstant, content, "", mc.CreatedAt)
}

func (tp *TemplateParser) RenderDailyTemplate(containers []*MailerContainer) (string, error) {
	if err := tp.validateTemplateParser(); err != nil {
		return "", err
	}

	var contents string
	for _, mc := range containers {
		ec, err := buildEventContent(mc)
		if err != nil {
			continue
		}
		c := tp.renderContentTemplate(ec, mc)
		contents = c + contents
	}

	return tp.renderTemplate(
		"daily",
		contents,
		"Here what's happened in Koding today!",
		time.Now())
}

func (tp *TemplateParser) validateTemplateParser() error {
	if tp.UserContact == nil {
		return fmt.Errorf("TemplateParser UserContact is not set")
	}

	return nil
}

func (tp *TemplateParser) renderTemplate(contentType, content, description string, date time.Time) (string, error) {
	t := template.Must(template.ParseFiles(
		mainTemplateFile, footerTemplateFile, unsubscribeTemplateFile))
	mc := tp.buildMailContent(contentType, getMonthAndDay(date))

	mc.Content = template.HTML(content)
	mc.Description = description

	var doc bytes.Buffer
	if err := t.Execute(&doc, mc); err != nil {
		return "", err
	}

	return doc.String(), nil
}

func (tp *TemplateParser) buildMailContent(contentType string, currentDate string) *MailContent {
	return &MailContent{
		FirstName:   tp.UserContact.FirstName,
		CurrentDate: currentDate,
		Unsubscribe: &UnsubscribeContent{
			Token:       tp.UserContact.Token,
			ContentType: contentType,
			Recipient:   url.QueryEscape(tp.UserContact.Email),
		},
		Uri: config.Get().Uri,
	}
}

func buildEventContent(mc *MailerContainer) (*EventContent, error) {
	ec := &EventContent{
		Action:       mc.ActivityMessage,
		ActivityTime: prepareTime(mc),
		Uri:          config.Get().Uri,
		Slug:         mc.Slug,
		Message:      mc.Message,
		Group:        mc.Group,
		ObjectType:   mc.ObjectType,
		Size:         20,
	}

	actor, err := FetchUserContact(mc.Activity.ActorId)
	if err != nil {
		return nil, fmt.Errorf("an error occurred while retrieving actor details", err)
	}
	ec.ActorContact = *actor

	return ec, nil
}

func appendGroupTemplate(t *template.Template, mc *MailerContainer) {
	var groupTemplate *template.Template
	if mc.Group.Name == "" || mc.Group.Slug == "koding" {
		groupTemplate = getEmptyTemplate()
	} else {
		groupTemplate = template.Must(
			template.ParseFiles(groupTemplateFile))
	}

	t.AddParseTree("group", groupTemplate.Tree)
}

func (tp *TemplateParser) renderContentTemplate(ec *EventContent, mc *MailerContainer) string {
	t := template.Must(template.ParseFiles(contentTemplateFile, gravatarTemplateFile))
	appendPreviewTemplate(t, mc)
	appendGroupTemplate(t, mc)

	buf := bytes.NewBuffer([]byte{})
	t.ExecuteTemplate(buf, "content", ec)

	return buf.String()
}

func appendPreviewTemplate(t *template.Template, mc *MailerContainer) {
	var previewTemplate, objectTemplate *template.Template
	if mc.Message == "" {
		previewTemplate = getEmptyTemplate()
		objectTemplate = getEmptyTemplate()
	} else {
		previewTemplate = template.Must(template.ParseFiles(previewTemplateFile))
		objectTemplate = template.Must(template.ParseFiles(objectTemplateFile))
	}

	t.AddParseTree("preview", previewTemplate.Tree)
	t.AddParseTree("object", objectTemplate.Tree)
}

func getEmptyTemplate() *template.Template {
	return template.Must(template.New("").Parse(""))
}

func getMonthAndDay(t time.Time) string {
	return t.Format(DATEFORMAT)
}

func prepareDate(mc *MailerContainer) string {
	return mc.Activity.CreatedAt.Format(DATEFORMAT)
}

func prepareTime(mc *MailerContainer) string {
	return mc.Activity.CreatedAt.Format(TIMEFORMAT)
}
<commit_msg>Notification: template root directory is relatively set to GOPATH<commit_after>package models

import (
	"bytes"
	"fmt"
	"html/template"
	"net/url"
	"os"
	"socialapi/config"
	"time"
)

const (
	DATEFORMAT = "Jan 02"
	TIMEFORMAT = "15:04"
)

type TemplateParser struct {
	UserContact *UserContact
}

var (
	mainTemplateFile        = os.Getenv("GOPATH") + "/src/socialapi/workers/emailnotifier/templates/main.tmpl"
	footerTemplateFile      = os.Getenv("GOPATH") + "/src/socialapi/workers/emailnotifier/templates/footer.tmpl"
	contentTemplateFile     = os.Getenv("GOPATH") + "/src/socialapi/workers/emailnotifier/templates/content.tmpl"
	gravatarTemplateFile    = os.Getenv("GOPATH") + "/src/socialapi/workers/emailnotifier/templates/gravatar.tmpl"
	groupTemplateFile       = os.Getenv("GOPATH") + "/src/socialapi/workers/emailnotifier/templates/group.tmpl"
	previewTemplateFile     = os.Getenv("GOPATH") + "/src/socialapi/workers/emailnotifier/templates/preview.tmpl"
	objectTemplateFile      = os.Getenv("GOPATH") + "/src/socialapi/workers/emailnotifier/templates/object.tmpl"
	unsubscribeTemplateFile = os.Getenv("GOPATH") + "/src/socialapi/workers/emailnotifier/templates/unsubscribe.tmpl"
)

func NewTemplateParser() *TemplateParser {
	return &TemplateParser{}
}

func (tp *TemplateParser) RenderInstantTemplate(mc *MailerContainer) (string, error) {
	if err := tp.validateTemplateParser(); err != nil {
		return "", err
	}

	ec, err := buildEventContent(mc)
	if err != nil {
		return "", err
	}
	content := tp.renderContentTemplate(ec, mc)

	return tp.renderTemplate(mc.Content.TypeConstant, content, "", mc.CreatedAt)
}

func (tp *TemplateParser) RenderDailyTemplate(containers []*MailerContainer) (string, error) {
	if err := tp.validateTemplateParser(); err != nil {
		return "", err
	}

	var contents string
	for _, mc := range containers {
		ec, err := buildEventContent(mc)
		if err != nil {
			continue
		}
		c := tp.renderContentTemplate(ec, mc)
		contents = c + contents
	}

	return tp.renderTemplate(
		"daily",
		contents,
		"Here what's happened in Koding today!",
		time.Now())
}

func (tp *TemplateParser) validateTemplateParser() error {
	if tp.UserContact == nil {
		return fmt.Errorf("TemplateParser UserContact is not set")
	}

	return nil
}

func (tp *TemplateParser) renderTemplate(contentType, content, description string, date time.Time) (string, error) {
	t := template.Must(template.ParseFiles(
		mainTemplateFile, footerTemplateFile, unsubscribeTemplateFile))
	mc := tp.buildMailContent(contentType, getMonthAndDay(date))

	mc.Content = template.HTML(content)
	mc.Description = description

	var doc bytes.Buffer
	if err := t.Execute(&doc, mc); err != nil {
		return "", err
	}

	return doc.String(), nil
}

func (tp *TemplateParser) buildMailContent(contentType string, currentDate string) *MailContent {
	return &MailContent{
		FirstName:   tp.UserContact.FirstName,
		CurrentDate: currentDate,
		Unsubscribe: &UnsubscribeContent{
			Token:       tp.UserContact.Token,
			ContentType: contentType,
			Recipient:   url.QueryEscape(tp.UserContact.Email),
		},
		Uri: config.Get().Uri,
	}
}

func buildEventContent(mc *MailerContainer) (*EventContent, error) {
	ec := &EventContent{
		Action:       mc.ActivityMessage,
		ActivityTime: prepareTime(mc),
		Uri:          config.Get().Uri,
		Slug:         mc.Slug,
		Message:      mc.Message,
		Group:        mc.Group,
		ObjectType:   mc.ObjectType,
		Size:         20,
	}

	actor, err := FetchUserContact(mc.Activity.ActorId)
	if err != nil {
		return nil, fmt.Errorf("an error occurred while retrieving actor details", err)
	}
	ec.ActorContact = *actor

	return ec, nil
}

func appendGroupTemplate(t *template.Template, mc *MailerContainer) {
	var groupTemplate *template.Template
	if mc.Group.Name == "" || mc.Group.Slug == "koding" {
		groupTemplate = getEmptyTemplate()
	} else {
		groupTemplate = template.Must(
			template.ParseFiles(groupTemplateFile))
	}

	t.AddParseTree("group", groupTemplate.Tree)
}

func (tp *TemplateParser) renderContentTemplate(ec *EventContent, mc *MailerContainer) string {
	t := template.Must(template.ParseFiles(contentTemplateFile, gravatarTemplateFile))
	appendPreviewTemplate(t, mc)
	appendGroupTemplate(t, mc)

	buf := bytes.NewBuffer([]byte{})
	t.ExecuteTemplate(buf, "content", ec)

	return buf.String()
}

func appendPreviewTemplate(t *template.Template, mc *MailerContainer) {
	var previewTemplate, objectTemplate *template.Template
	if mc.Message == "" {
		previewTemplate = getEmptyTemplate()
		objectTemplate = getEmptyTemplate()
	} else {
		previewTemplate = template.Must(template.ParseFiles(previewTemplateFile))
		objectTemplate = template.Must(template.ParseFiles(objectTemplateFile))
	}

	t.AddParseTree("preview", previewTemplate.Tree)
	t.AddParseTree("object", objectTemplate.Tree)
}

func getEmptyTemplate() *template.Template {
	return template.Must(template.New("").Parse(""))
}

func getMonthAndDay(t time.Time) string {
	return t.Format(DATEFORMAT)
}

func prepareDate(mc *MailerContainer) string {
	return mc.Activity.CreatedAt.Format(DATEFORMAT)
}

func prepareTime(mc *MailerContainer) string {
	return mc.Activity.CreatedAt.Format(TIMEFORMAT)
}
<|endoftext|>


<commit_before>package models

import (
	"bytes"
	"fmt"
	"html/template"
	"net/url"
	"socialapi/config"
	// socialmodels "socialapi/models"
	// "socialapi/workers/notification/models"
	"time"
)

type TemplateParser struct {
	UserContact *UserContact
}

var (
	mainTemplateFile        = "../socialapi/workers/emailnotifier/templates/main.tmpl"
	footerTemplateFile      = "../socialapi/workers/emailnotifier/templates/footer.tmpl"
	contentTemplateFile     = "../socialapi/workers/emailnotifier/templates/content.tmpl"
	gravatarTemplateFile    = "../socialapi/workers/emailnotifier/templates/gravatar.tmpl"
	groupTemplateFile       = "../socialapi/workers/emailnotifier/templates/group.tmpl"
	previewTemplateFile     = "../socialapi/workers/emailnotifier/templates/preview.tmpl"
	objectTemplateFile      = "../socialapi/workers/emailnotifier/templates/object.tmpl"
	unsubscribeTemplateFile = "../socialapi/workers/emailnotifier/templates/unsubscribe.tmpl"
)

func NewTemplateParser() *TemplateParser {
	return &TemplateParser{}
}

func (tp *TemplateParser) RenderInstantTemplate(mc *MailerContainer) (string, error) {
	if err := tp.validateTemplateParser(); err != nil {
		return "", err
	}

	ec, err := buildEventContent(mc)
	if err != nil {
		return "", err
	}
	content := tp.renderContentTemplate(ec, mc)

	return tp.renderTemplate(mc.Content.TypeConstant, content, mc.CreatedAt)
}

func (tp *TemplateParser) RenderDailyTemplate(containers []*MailerContainer) (string, error) {
	if err := tp.validateTemplateParser(); err != nil {
		return "", err
	}

	var contents string
	for _, mc := range containers {
		ec, err := buildEventContent(mc)
		if err != nil {
			continue
		}
		c := tp.renderContentTemplate(ec, mc)
		contents = c + contents
	}

	return tp.renderTemplate("daily", contents, time.Now())
}

func (tp *TemplateParser) validateTemplateParser() error {
	if tp.UserContact == nil {
		return fmt.Errorf("TemplateParser UserContact is not set")
	}

	return nil
}

func (tp *TemplateParser) renderTemplate(contentType, content string, date time.Time) (string, error) {
	t := template.Must(template.ParseFiles(
		mainTemplateFile, footerTemplateFile, unsubscribeTemplateFile))
	mc := tp.buildMailContent(contentType, getMonthAndDay(date))

	mc.Content = template.HTML(content)

	var doc bytes.Buffer
	if err := t.Execute(&doc, mc); err != nil {
		return "", err
	}

	return doc.String(), nil
}

func (tp *TemplateParser) buildMailContent(contentType string, currentDate string) *MailContent {
	return &MailContent{
		FirstName:   tp.UserContact.FirstName,
		CurrentDate: currentDate,
		Unsubscribe: &UnsubscribeContent{
			Token:       tp.UserContact.Token,
			ContentType: contentType,
			Recipient:   url.QueryEscape(tp.UserContact.Email),
		},
		Uri: config.Get().Uri,
	}
}

func buildEventContent(mc *MailerContainer) (*EventContent, error) {
	ec := &EventContent{
		Action:       mc.ActivityMessage,
		ActivityTime: prepareTime(mc),
		Uri:          config.Get().Uri,
		Slug:         mc.Slug,
		Message:      mc.Message,
		Group:        mc.Group,
		ObjectType:   mc.ObjectType,
		Size:         20,
	}

	actor, err := FetchUserContact(mc.Activity.ActorId)
	if err != nil {
		return nil, fmt.Errorf("an error occurred while retrieving actor details", err)
	}
	ec.ActorContact = *actor

	return ec, nil
}

func appendGroupTemplate(t *template.Template, mc *MailerContainer) {
	var groupTemplate *template.Template
	if mc.Group.Name == "" || mc.Group.Slug == "koding" {
		groupTemplate = getEmptyTemplate()
	} else {
		groupTemplate = template.Must(
			template.ParseFiles(groupTemplateFile))
	}

	t.AddParseTree("group", groupTemplate.Tree)
}

func (tp *TemplateParser) renderContentTemplate(ec *EventContent, mc *MailerContainer) string {
	t := template.Must(template.ParseFiles(contentTemplateFile, gravatarTemplateFile))
	appendPreviewTemplate(t, mc)
	appendGroupTemplate(t, mc)

	buf := bytes.NewBuffer([]byte{})
	t.ExecuteTemplate(buf, "content", ec)

	return buf.String()
}

func appendPreviewTemplate(t *template.Template, mc *MailerContainer) {
	var previewTemplate, objectTemplate *template.Template
	if mc.Message == "" {
		previewTemplate = getEmptyTemplate()
		objectTemplate = getEmptyTemplate()
	} else {
		previewTemplate = template.Must(template.ParseFiles(previewTemplateFile))
		objectTemplate = template.Must(template.ParseFiles(objectTemplateFile))
	}

	t.AddParseTree("preview", previewTemplate.Tree)
	t.AddParseTree("object", objectTemplate.Tree)
}

func getEmptyTemplate() *template.Template {
	return template.Must(template.New("").Parse(""))
}

func getMonthAndDay(t time.Time) string {
	return t.Format("Jan 02")
}

func prepareDate(mc *MailerContainer) string {
	return mc.Activity.CreatedAt.Format("Jan 02")
}

func prepareTime(mc *MailerContainer) string {
	return mc.Activity.CreatedAt.Format("15:04")
}
<commit_msg>Notification: Description is added for daily mails<commit_after>package models

import (
	"bytes"
	"fmt"
	"html/template"
	"net/url"
	"socialapi/config"
	// socialmodels "socialapi/models"
	// "socialapi/workers/notification/models"
	"time"
)

type TemplateParser struct {
	UserContact *UserContact
}

var (
	mainTemplateFile        = "../socialapi/workers/emailnotifier/templates/main.tmpl"
	footerTemplateFile      = "../socialapi/workers/emailnotifier/templates/footer.tmpl"
	contentTemplateFile     = "../socialapi/workers/emailnotifier/templates/content.tmpl"
	gravatarTemplateFile    = "../socialapi/workers/emailnotifier/templates/gravatar.tmpl"
	groupTemplateFile       = "../socialapi/workers/emailnotifier/templates/group.tmpl"
	previewTemplateFile     = "../socialapi/workers/emailnotifier/templates/preview.tmpl"
	objectTemplateFile      = "../socialapi/workers/emailnotifier/templates/object.tmpl"
	unsubscribeTemplateFile = "../socialapi/workers/emailnotifier/templates/unsubscribe.tmpl"
)

func NewTemplateParser() *TemplateParser {
	return &TemplateParser{}
}

func (tp *TemplateParser) RenderInstantTemplate(mc *MailerContainer) (string, error) {
	if err := tp.validateTemplateParser(); err != nil {
		return "", err
	}

	ec, err := buildEventContent(mc)
	if err != nil {
		return "", err
	}
	content := tp.renderContentTemplate(ec, mc)

	return tp.renderTemplate(mc.Content.TypeConstant, content, "", mc.CreatedAt)
}

func (tp *TemplateParser) RenderDailyTemplate(containers []*MailerContainer) (string, error) {
	if err := tp.validateTemplateParser(); err != nil {
		return "", err
	}

	var contents string
	for _, mc := range containers {
		ec, err := buildEventContent(mc)
		if err != nil {
			continue
		}
		c := tp.renderContentTemplate(ec, mc)
		contents = c + contents
	}

	return tp.renderTemplate(
		"daily",
		contents,
		"Here what's happened in Koding today!",
		time.Now())
}

func (tp *TemplateParser) validateTemplateParser() error {
	if tp.UserContact == nil {
		return fmt.Errorf("TemplateParser UserContact is not set")
	}

	return nil
}

func (tp *TemplateParser) renderTemplate(contentType, content, description string, date time.Time) (string, error) {
	t := template.Must(template.ParseFiles(
		mainTemplateFile, footerTemplateFile, unsubscribeTemplateFile))
	mc := tp.buildMailContent(contentType, getMonthAndDay(date))

	mc.Content = template.HTML(content)
	mc.Description = description

	var doc bytes.Buffer
	if err := t.Execute(&doc, mc); err != nil {
		return "", err
	}

	return doc.String(), nil
}

func (tp *TemplateParser) buildMailContent(contentType string, currentDate string) *MailContent {
	return &MailContent{
		FirstName:   tp.UserContact.FirstName,
		CurrentDate: currentDate,
		Unsubscribe: &UnsubscribeContent{
			Token:       tp.UserContact.Token,
			ContentType: contentType,
			Recipient:   url.QueryEscape(tp.UserContact.Email),
		},
		Uri: config.Get().Uri,
	}
}

func buildEventContent(mc *MailerContainer) (*EventContent, error) {
	ec := &EventContent{
		Action:       mc.ActivityMessage,
		ActivityTime: prepareTime(mc),
		Uri:          config.Get().Uri,
		Slug:         mc.Slug,
		Message:      mc.Message,
		Group:        mc.Group,
		ObjectType:   mc.ObjectType,
		Size:         20,
	}

	actor, err := FetchUserContact(mc.Activity.ActorId)
	if err != nil {
		return nil, fmt.Errorf("an error occurred while retrieving actor details", err)
	}
	ec.ActorContact = *actor

	return ec, nil
}

func appendGroupTemplate(t *template.Template, mc *MailerContainer) {
	var groupTemplate *template.Template
	if mc.Group.Name == "" || mc.Group.Slug == "koding" {
		groupTemplate = getEmptyTemplate()
	} else {
		groupTemplate = template.Must(
			template.ParseFiles(groupTemplateFile))
	}

	t.AddParseTree("group", groupTemplate.Tree)
}

func (tp *TemplateParser) renderContentTemplate(ec *EventContent, mc *MailerContainer) string {
	t := template.Must(template.ParseFiles(contentTemplateFile, gravatarTemplateFile))
	appendPreviewTemplate(t, mc)
	appendGroupTemplate(t, mc)

	buf := bytes.NewBuffer([]byte{})
	t.ExecuteTemplate(buf, "content", ec)

	return buf.String()
}

func appendPreviewTemplate(t *template.Template, mc *MailerContainer) {
	var previewTemplate, objectTemplate *template.Template
	if mc.Message == "" {
		previewTemplate = getEmptyTemplate()
		objectTemplate = getEmptyTemplate()
	} else {
		previewTemplate = template.Must(template.ParseFiles(previewTemplateFile))
		objectTemplate = template.Must(template.ParseFiles(objectTemplateFile))
	}

	t.AddParseTree("preview", previewTemplate.Tree)
	t.AddParseTree("object", objectTemplate.Tree)
}

func getEmptyTemplate() *template.Template {
	return template.Must(template.New("").Parse(""))
}

func getMonthAndDay(t time.Time) string {
	return t.Format("Jan 02")
}

func prepareDate(mc *MailerContainer) string {
	return mc.Activity.CreatedAt.Format("Jan 02")
}

func prepareTime(mc *MailerContainer) string {
	return mc.Activity.CreatedAt.Format("15:04")
}
<|endoftext|>


<commit_before>package controller

import (
	mongomodels "koding/db/models"
	"koding/db/mongodb/modelhelper"
	"koding/helpers"
	"socialapi/models"
	"strconv"
)

func (mwc *Controller) migrateAllAccounts() error {
	errCount := 0
	successCount := 0

	s := modelhelper.Selector{
		"socialApiId": modelhelper.Selector{"$exists": false},
	}

	migrateAccount := func(account interface{}) error {
		oldAccount := account.(*mongomodels.Account)
		if oldAccount.SocialApiId != 0 {
			return nil
		}

		id, err := models.AccountIdByOldId(
			oldAccount.Id.Hex(),
			oldAccount.Profile.Nickname,
		)
		if err != nil {
			errCount++
			mwc.log.Error("Error occurred for account %s: %s", oldAccount.Id.Hex())
			return nil
		}

		s := modelhelper.Selector{"_id": oldAccount.Id}
		o := modelhelper.Selector{"$set": modelhelper.Selector{"socialApiId": strconv.FormatInt(id, 10)}}
		if err := modelhelper.UpdateAccount(s, o); err != nil {
			mwc.log.Warning("Could not update account document: %s", err)
			return nil
		}

		successCount++

		return nil
	}

	iterOptions := helpers.NewIterOptions()
	iterOptions.CollectionName = "jAccounts"
	iterOptions.F = migrateAccount
	iterOptions.Filter = s
	iterOptions.Result = &mongomodels.Account{}
	iterOptions.Limit = 10000000
	iterOptions.Skip = 0

	helpers.Iter(modelhelper.Mongo, iterOptions)

	mwc.log.Notice("Account migration completed for %d account with %d errors", successCount, errCount)

	return nil
}
<commit_msg>Social: update err count if there is any error<commit_after>package controller

import (
	mongomodels "koding/db/models"
	"koding/db/mongodb/modelhelper"
	"koding/helpers"
	"socialapi/models"
	"strconv"
)

func (mwc *Controller) migrateAllAccounts() error {
	errCount := 0
	successCount := 0

	s := modelhelper.Selector{
		"socialApiId": modelhelper.Selector{"$exists": false},
	}

	migrateAccount := func(account interface{}) error {
		oldAccount := account.(*mongomodels.Account)
		if oldAccount.SocialApiId != 0 {
			return nil
		}

		id, err := models.AccountIdByOldId(
			oldAccount.Id.Hex(),
			oldAccount.Profile.Nickname,
		)
		if err != nil {
			errCount++
			mwc.log.Error("Error occurred for account %s: %s", oldAccount.Id.Hex())
			return nil
		}

		s := modelhelper.Selector{"_id": oldAccount.Id}
		o := modelhelper.Selector{"$set": modelhelper.Selector{"socialApiId": strconv.FormatInt(id, 10)}}
		if err := modelhelper.UpdateAccount(s, o); err != nil {
			errCount++
			mwc.log.Warning("Could not update account document: %s", err)
			return nil
		}

		successCount++

		return nil
	}

	iterOptions := helpers.NewIterOptions()
	iterOptions.CollectionName = "jAccounts"
	iterOptions.F = migrateAccount
	iterOptions.Filter = s
	iterOptions.Result = &mongomodels.Account{}
	iterOptions.Limit = 10000000
	iterOptions.Skip = 0

	helpers.Iter(modelhelper.Mongo, iterOptions)

	mwc.log.Notice("Account migration completed for %d account with %d errors", successCount, errCount)

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2014, The Serviced Authors. All rights reserved.
// Use of this source code is governed by a
// license that can be found in the LICENSE file.

package elasticsearch

import (
	"github.com/zenoss/glog"
	"github.com/zenoss/serviced/domain/service"
	"github.com/zenoss/serviced/volume"
	zkSnapshot "github.com/zenoss/serviced/zzk/snapshot"

	"errors"
	"os"
	"os/user"
	"path"
	"path/filepath"
)

func (this *ControlPlaneDao) DeleteSnapshot(snapshotId string, unused *int) error {
	return this.dfs.DeleteSnapshot(snapshotId)
}

func (this *ControlPlaneDao) DeleteSnapshots(serviceId string, unused *int) error {
	var tenantId string
	if err := this.GetTenantId(serviceId, &tenantId); err != nil {
		glog.V(2).Infof("ControlPlaneDao.DeleteSnapshots err=%s", err)
		return err
	}

	if serviceId != tenantId {
		glog.Infof("ControlPlaneDao.DeleteSnapshots service is not the parent, service=%s, tenant=%s", serviceId, tenantId)
		return nil
	}

	return this.dfs.DeleteSnapshots(tenantId)
}

func (this *ControlPlaneDao) Rollback(snapshotId string, unused *int) error {
	return this.dfs.Rollback(snapshotId)
}

// Takes a snapshot of the DFS via the host
func (this *ControlPlaneDao) TakeSnapshot(serviceID string, label *string) error {
	service, err := this.getService(serviceID)
	if err != nil {
		return err
	}
	tenantID, err := service.GetTenantID(this.getService)
	if err != nil {
		return err
	}
	*label, err = this.dfs.Snapshot(tenantID)
	return err
}

// Snapshot is called via RPC by the CLI to take a snapshot for a serviceId
func (this *ControlPlaneDao) Snapshot(serviceID string, label *string) error {
	conn, err := this.zclient.GetConnection()
	if err != nil {
		return err
	}
	defer conn.Close()

	req := zkSnapshot.Snapshot{
		ServiceID: serviceID,
	}

	if err := zkSnapshot.Send(conn, &req); err != nil {
		return err
	}

	res, err := zkSnapshot.Recv(conn, serviceID)
	*label = res.Label
	return err
}

func (this *ControlPlaneDao) Snapshots(serviceId string, labels *[]string) error {

	var tenantId string
	if err := this.GetTenantId(serviceId, &tenantId); err != nil {
		glog.V(2).Infof("ControlPlaneDao.Snapshots service=%+v err=%s", serviceId, err)
		return err
	}
	var service service.Service
	err := this.GetService(tenantId, &service)
	if err != nil {
		glog.V(2).Infof("ControlPlaneDao.Snapshots service=%+v err=%s", serviceId, err)
		return err
	}

	if volume, err := getSubvolume(this.vfs, service.PoolID, tenantId); err != nil {
		glog.V(2).Infof("ControlPlaneDao.Snapshots service=%+v err=%s", serviceId, err)
		return err
	} else {
		if snaplabels, err := volume.Snapshots(); err != nil {
			return err
		} else {
			glog.Infof("Got snap labels %v", snaplabels)
			*labels = snaplabels
		}
	}
	return nil
}
func (this *ControlPlaneDao) GetVolume(serviceId string, theVolume *volume.Volume) error {
	var tenantId string
	if err := this.GetTenantId(serviceId, &tenantId); err != nil {
		glog.V(2).Infof("ControlPlaneDao.GetVolume service=%+v err=%s", serviceId, err)
		return err
	}
	glog.V(3).Infof("ControlPlaneDao.GetVolume service=%+v tenantId=%s", serviceId, tenantId)
	var service service.Service
	if err := this.GetService(tenantId, &service); err != nil {
		glog.V(2).Infof("ControlPlaneDao.GetVolume service=%+v err=%s", serviceId, err)
		return err
	}
	glog.V(3).Infof("ControlPlaneDao.GetVolume service=%+v poolId=%s", service, service.PoolID)

	aVolume, err := getSubvolume(this.vfs, service.PoolID, tenantId)
	if err != nil {
		glog.V(2).Infof("ControlPlaneDao.GetVolume service=%+v err=%s", serviceId, err)
		return err
	}
	if aVolume == nil {
		glog.V(2).Infof("ControlPlaneDao.GetVolume service=%+v volume=nil", serviceId)
		return errors.New("volume is nil")
	}

	glog.V(2).Infof("ControlPlaneDao.GetVolume service=%+v volume2=%+v %v", serviceId, aVolume, aVolume)
	*theVolume = *aVolume
	return nil
}

// Commits a container to an image and saves it on the DFS
func (this *ControlPlaneDao) Commit(containerId string, label *string) error {
	if id, err := this.dfs.Commit(containerId); err != nil {
		glog.V(2).Infof("ControlPlaneDao.GetVolume containerId=%s err=%s", containerId, err)
		return err
	} else {
		*label = id
	}

	return nil
}

func getSubvolume(vfs, poolId, tenantId string) (*volume.Volume, error) {
	baseDir, err := filepath.Abs(path.Join(varPath(), "volumes", poolId))
	if err != nil {
		return nil, err
	}
	glog.Infof("Mounting vfs:%v tenantId:%v baseDir:%v\n", vfs, tenantId, baseDir)
	return volume.Mount(vfs, tenantId, baseDir)
}

func varPath() string {
	if len(os.Getenv("SERVICED_HOME")) > 0 {
		return path.Join(os.Getenv("SERVICED_HOME"), "var")
	} else if user, err := user.Current(); err == nil {
		return path.Join(os.TempDir(), "serviced-"+user.Username, "var")
	} else {
		defaultPath := "/tmp/serviced/var"
		glog.Warningf("Defaulting varPath to:%v\n", defaultPath)
		return defaultPath
	}
}

func (s *ControlPlaneDao) ReadyDFS(unused bool, unusedint *int) (err error) {
	s.dfs.Lock()
	s.dfs.Unlock()
	return
}
<commit_msg>Fixed error receieved<commit_after>// Copyright 2014, The Serviced Authors. All rights reserved.
// Use of this source code is governed by a
// license that can be found in the LICENSE file.

package elasticsearch

import (
	"github.com/zenoss/glog"
	"github.com/zenoss/serviced/domain/service"
	"github.com/zenoss/serviced/volume"
	zkSnapshot "github.com/zenoss/serviced/zzk/snapshot"

	"errors"
	"os"
	"os/user"
	"path"
	"path/filepath"
)

func (this *ControlPlaneDao) DeleteSnapshot(snapshotId string, unused *int) error {
	return this.dfs.DeleteSnapshot(snapshotId)
}

func (this *ControlPlaneDao) DeleteSnapshots(serviceId string, unused *int) error {
	var tenantId string
	if err := this.GetTenantId(serviceId, &tenantId); err != nil {
		glog.V(2).Infof("ControlPlaneDao.DeleteSnapshots err=%s", err)
		return err
	}

	if serviceId != tenantId {
		glog.Infof("ControlPlaneDao.DeleteSnapshots service is not the parent, service=%s, tenant=%s", serviceId, tenantId)
		return nil
	}

	return this.dfs.DeleteSnapshots(tenantId)
}

func (this *ControlPlaneDao) Rollback(snapshotId string, unused *int) error {
	return this.dfs.Rollback(snapshotId)
}

// Takes a snapshot of the DFS via the host
func (this *ControlPlaneDao) TakeSnapshot(serviceID string, label *string) error {
	service, err := this.getService(serviceID)
	if err != nil {
		return err
	}
	tenantID, err := service.GetTenantID(this.getService)
	if err != nil {
		return err
	}
	*label, err = this.dfs.Snapshot(tenantID)
	return err
}

// Snapshot is called via RPC by the CLI to take a snapshot for a serviceId
func (this *ControlPlaneDao) Snapshot(serviceID string, label *string) error {
	conn, err := this.zclient.GetConnection()
	if err != nil {
		return err
	}
	defer conn.Close()

	req := zkSnapshot.Snapshot{
		ServiceID: serviceID,
	}

	if err := zkSnapshot.Send(conn, &req); err != nil {
		return err
	}

	res, err := zkSnapshot.Recv(conn, serviceID)
	if err != nil {
		return err
	}

	*label = res.Label
	return res.Error
}

func (this *ControlPlaneDao) Snapshots(serviceId string, labels *[]string) error {

	var tenantId string
	if err := this.GetTenantId(serviceId, &tenantId); err != nil {
		glog.V(2).Infof("ControlPlaneDao.Snapshots service=%+v err=%s", serviceId, err)
		return err
	}
	var service service.Service
	err := this.GetService(tenantId, &service)
	if err != nil {
		glog.V(2).Infof("ControlPlaneDao.Snapshots service=%+v err=%s", serviceId, err)
		return err
	}

	if volume, err := getSubvolume(this.vfs, service.PoolID, tenantId); err != nil {
		glog.V(2).Infof("ControlPlaneDao.Snapshots service=%+v err=%s", serviceId, err)
		return err
	} else {
		if snaplabels, err := volume.Snapshots(); err != nil {
			return err
		} else {
			glog.Infof("Got snap labels %v", snaplabels)
			*labels = snaplabels
		}
	}
	return nil
}
func (this *ControlPlaneDao) GetVolume(serviceId string, theVolume *volume.Volume) error {
	var tenantId string
	if err := this.GetTenantId(serviceId, &tenantId); err != nil {
		glog.V(2).Infof("ControlPlaneDao.GetVolume service=%+v err=%s", serviceId, err)
		return err
	}
	glog.V(3).Infof("ControlPlaneDao.GetVolume service=%+v tenantId=%s", serviceId, tenantId)
	var service service.Service
	if err := this.GetService(tenantId, &service); err != nil {
		glog.V(2).Infof("ControlPlaneDao.GetVolume service=%+v err=%s", serviceId, err)
		return err
	}
	glog.V(3).Infof("ControlPlaneDao.GetVolume service=%+v poolId=%s", service, service.PoolID)

	aVolume, err := getSubvolume(this.vfs, service.PoolID, tenantId)
	if err != nil {
		glog.V(2).Infof("ControlPlaneDao.GetVolume service=%+v err=%s", serviceId, err)
		return err
	}
	if aVolume == nil {
		glog.V(2).Infof("ControlPlaneDao.GetVolume service=%+v volume=nil", serviceId)
		return errors.New("volume is nil")
	}

	glog.V(2).Infof("ControlPlaneDao.GetVolume service=%+v volume2=%+v %v", serviceId, aVolume, aVolume)
	*theVolume = *aVolume
	return nil
}

// Commits a container to an image and saves it on the DFS
func (this *ControlPlaneDao) Commit(containerId string, label *string) error {
	if id, err := this.dfs.Commit(containerId); err != nil {
		glog.V(2).Infof("ControlPlaneDao.GetVolume containerId=%s err=%s", containerId, err)
		return err
	} else {
		*label = id
	}

	return nil
}

func getSubvolume(vfs, poolId, tenantId string) (*volume.Volume, error) {
	baseDir, err := filepath.Abs(path.Join(varPath(), "volumes", poolId))
	if err != nil {
		return nil, err
	}
	glog.Infof("Mounting vfs:%v tenantId:%v baseDir:%v\n", vfs, tenantId, baseDir)
	return volume.Mount(vfs, tenantId, baseDir)
}

func varPath() string {
	if len(os.Getenv("SERVICED_HOME")) > 0 {
		return path.Join(os.Getenv("SERVICED_HOME"), "var")
	} else if user, err := user.Current(); err == nil {
		return path.Join(os.TempDir(), "serviced-"+user.Username, "var")
	} else {
		defaultPath := "/tmp/serviced/var"
		glog.Warningf("Defaulting varPath to:%v\n", defaultPath)
		return defaultPath
	}
}

func (s *ControlPlaneDao) ReadyDFS(unused bool, unusedint *int) (err error) {
	s.dfs.Lock()
	s.dfs.Unlock()
	return
}
<|endoftext|>


<commit_before>package postgres

import (
	"bytes"
	"database/sql"
	"database/sql/driver"
	"fmt"
	"strings"
	"time"

	"honnef.co/go/tracer"

	"github.com/jmoiron/sqlx"
	"github.com/opentracing/opentracing-go"
)

// timeRange represents a PostgreSQL tstzrange. Caveat: it only
// supports inclusive ranges.
type timeRange struct {
	Start time.Time
	End   time.Time
}

func (t *timeRange) Scan(src interface{}) error {
	const layout = "2006-01-02 15:04:05.999999-07"

	b := src.([]byte)
	b = b[2:]
	idx := bytes.IndexByte(b, '"')
	t1, err := time.Parse(layout, string(b[:idx]))
	if err != nil {
		return err
	}

	b = b[idx+1:]
	idx = bytes.IndexByte(b, '"')
	b = b[idx+1:]
	idx = bytes.IndexByte(b, '"')
	t2, err := time.Parse(layout, string(string(b[:idx])))
	if err != nil {
		return err
	}
	t.Start = t1
	t.End = t2
	return nil
}

func (t timeRange) Value() (driver.Value, error) {
	const layout = "2006-01-02 15:04:05.999999-07"
	return []byte(fmt.Sprintf(`["%s","%s"]`, t.Start.Format(layout), t.End.Format(layout))), nil
}

type Storage struct {
	db *sqlx.DB
}

func New(db *sql.DB) *Storage {
	return &Storage{db: sqlx.NewDb(db, "postgres")}
}

func (st *Storage) Store(sp tracer.RawSpan) (err error) {
	tx, err := st.db.Begin()
	if err != nil {
		return err
	}
	defer func() {
		if err != nil {
			_ = tx.Rollback()
			return
		}
		err = tx.Commit()
	}()

	_, err = tx.Exec(`INSERT INTO spans (id, trace_id, time, operation_name) VALUES ($1, $2, $3, $4) ON CONFLICT (id) DO UPDATE SET time = $3, operation_name = $4`,
		int64(sp.SpanID), int64(sp.TraceID), timeRange{sp.StartTime, sp.FinishTime}, sp.OperationName)
	if err != nil {
		return err
	}

	if sp.ParentID != 0 {
		_, err = tx.Exec(`INSERT INTO spans (id, trace_id, time, operation_name) VALUES ($1, $2, $3, '') ON CONFLICT (id) DO NOTHING`,
			int64(sp.ParentID), int64(sp.TraceID), timeRange{time.Time{}, time.Time{}})
		if err != nil {
			return err
		}
		_, err = tx.Exec(`INSERT INTO spans (id, trace_id, time, operation_name) VALUES ($1, $2, $3, $4) ON CONFLICT (id) DO NOTHING`,
			int64(sp.TraceID), int64(sp.TraceID), timeRange{sp.StartTime, sp.FinishTime}, sp.OperationName)
		if err != nil {
			return err
		}

		_, err = tx.Exec(`INSERT INTO relations (span1_id, span2_id, kind) VALUES ($1, $2, 'parent')`,
			int64(sp.ParentID), int64(sp.SpanID))
		if err != nil {
			return err
		}
	}

	for k, v := range sp.Tags {
		vs := fmt.Sprintf("%v", v) // XXX
		_, err = tx.Exec(`INSERT INTO tags (span_id, trace_id, key, value) VALUES ($1, $2, $3, $4)`,
			int64(sp.SpanID), int64(sp.TraceID), k, vs)
		if err != nil {
			return err
		}
	}
	for _, l := range sp.Logs {
		v := fmt.Sprintf("%v", l.Payload) // XXX
		_, err = tx.Exec(`INSERT INTO tags (span_id, trace_id, key, value, time) VALUES ($1, $2, $3, $4, $5)`,
			int64(sp.SpanID), int64(sp.TraceID), l.Event, v, l.Timestamp)
		if err != nil {
			return err
		}
	}
	return nil
}

func (st *Storage) TraceWithID(id uint64) (tracer.RawTrace, error) {
	tx, err := st.db.Begin()
	if err != nil {
		return tracer.RawTrace{}, err
	}
	defer tx.Rollback()
	return st.traceWithID(tx, id)
}

func (st *Storage) traceWithID(tx *sql.Tx, id uint64) (tracer.RawTrace, error) {
	rows, err := tx.Query(`SELECT spans.id, spans.trace_id, spans.time, spans.operation_name, tags.key, tags.value, tags.time FROM spans LEFT JOIN tags ON spans.id = tags.span_id WHERE spans.trace_id = $1 ORDER BY lower(spans.time) ASC, spans.id`,
		int64(id))
	if err != nil {
		return tracer.RawTrace{}, err
	}

	spans, err := scanSpans(rows)
	if err != nil {
		return tracer.RawTrace{}, err
	}
	return tracer.RawTrace{
		TraceID: id,
		Spans:   spans,
	}, nil
}

func scanSpans(rows *sql.Rows) ([]tracer.RawSpan, error) {
	// TODO select parents
	var spans []tracer.RawSpan
	var (
		prevSpanID int64

		spanID        int64
		traceID       int64
		spanTime      timeRange
		operationName string
		tagKey        string
		tagValue      string
		tagTime       *time.Time
	)
	tagTime = new(time.Time)
	var span tracer.RawSpan
	for rows.Next() {
		if err := rows.Scan(&spanID, &traceID, &spanTime, &operationName, &tagKey, &tagValue, &tagTime); err != nil {
			return nil, err
		}
		if spanID != prevSpanID {
			if prevSpanID != 0 {
				spans = append(spans, span)
			}
			prevSpanID = spanID
			span = tracer.RawSpan{
				Tags: map[string]interface{}{},
			}
		}
		span.SpanID = uint64(spanID)
		span.TraceID = uint64(traceID)
		span.StartTime = spanTime.Start
		span.FinishTime = spanTime.End
		span.OperationName = operationName
		if tagKey != "" {
			if tagTime == nil {
				span.Tags[tagKey] = tagValue
			} else {
				span.Logs = append(span.Logs, opentracing.LogData{
					Timestamp: *tagTime,
					Event:     tagKey,
					Payload:   tagValue,
				})
			}
		}
	}
	if err := rows.Err(); err != nil {
		return nil, err
	}
	if span.SpanID != 0 {
		spans = append(spans, span)
	}
	return spans, nil
}

func (st *Storage) SpanWithID(id uint64) (tracer.RawSpan, error) {
	tx, err := st.db.Begin()
	if err != nil {
		return tracer.RawSpan{}, err
	}
	defer tx.Rollback()
	return st.spanWithID(tx, id)
}

func (st *Storage) spanWithID(tx *sql.Tx, id uint64) (tracer.RawSpan, error) {
	rows, err := tx.Query(`SELECT spans.id, spans.trace_id, spans.time, spans.time, spans.operation_name, tags.key, tags.value, tags.time FROM spans LEFT JOIN tags ON spans.id = tags.span_id WHERE id = $1 LIMIT 1`,
		int64(id))
	if err != nil {
		return tracer.RawSpan{}, err
	}
	spans, err := scanSpans(rows)
	if err != nil {
		return tracer.RawSpan{}, err
	}
	if len(spans) == 0 {
		return tracer.RawSpan{}, sql.ErrNoRows
	}
	return spans[0], nil
}

func (st *Storage) QueryTraces(q tracer.Query) ([]tracer.RawTrace, error) {
	tx, err := st.db.Begin()
	if err != nil {
		return nil, err
	}
	defer tx.Rollback()

	var andConds []string
	var andArgs []interface{}
	var orConds []string
	var orArgs []interface{}
	if q.FinishTime.IsZero() {
		q.FinishTime = time.Now()
	}
	for _, tag := range q.AndTags {
		if tag.CheckValue {
			andConds = append(andConds, `(tags.key = ? AND tags.value = ?)`)
			andArgs = append(andArgs, tag.Key, tag.Value)
		} else {
			andConds = append(andConds, `(tags.key = ?)`)
			andArgs = append(andArgs, tag.Key)
		}
	}

	for _, tag := range q.OrTags {
		if tag.CheckValue {
			orConds = append(orConds, `(tags.key = ? AND tags.value = ?)`)
			orArgs = append(orArgs, tag.Key, tag.Value)
		} else {
			orConds = append(orConds, `(tags.key = ?)`)
			orArgs = append(orArgs, tag.Key)
		}
	}

	and := strings.Join(andConds, " AND ")
	or := strings.Join(orConds, " OR ")
	conds := []string{"true"}
	if and != "" {
		conds = append(conds, and)
	}
	if or != "" {
		conds = append(conds, or)
	}

	query := st.db.Rebind("SELECT spans.trace_id FROM spans WHERE EXISTS (SELECT 1 FROM tags WHERE tags.trace_id = spans.trace_id AND " + strings.Join(conds, " AND ") + ") AND ? @> spans.time AND spans.id = spans.trace_id ORDER BY spans.time ASC, spans.trace_id")
	args := make([]interface{}, 0, len(andArgs)+len(orArgs))
	args = append(args, andArgs...)
	args = append(args, orArgs...)
	args = append(args, timeRange{q.StartTime, q.FinishTime})

	var ids []int64
	rows, err := st.db.Query(query, args...)
	if err != nil {
		return nil, err
	}
	var id int64
	for rows.Next() {
		if err := rows.Scan(&id); err != nil {
			return nil, err
		}
		ids = append(ids, id)
	}
	if err := rows.Err(); err != nil {
		return nil, err
	}

	var traces []tracer.RawTrace
	for _, id := range ids {
		trace, err := st.traceWithID(tx, uint64(id))
		if err != nil {
			return nil, err
		}
		traces = append(traces, trace)
	}
	return traces, nil
}
<commit_msg>Extract SQL queries into constants and format them<commit_after>package postgres

import (
	"bytes"
	"database/sql"
	"database/sql/driver"
	"fmt"
	"strings"
	"time"

	"honnef.co/go/tracer"

	"github.com/jmoiron/sqlx"
	"github.com/opentracing/opentracing-go"
)

// timeRange represents a PostgreSQL tstzrange. Caveat: it only
// supports inclusive ranges.
type timeRange struct {
	Start time.Time
	End   time.Time
}

func (t *timeRange) Scan(src interface{}) error {
	const layout = "2006-01-02 15:04:05.999999-07"

	b := src.([]byte)
	b = b[2:]
	idx := bytes.IndexByte(b, '"')
	t1, err := time.Parse(layout, string(b[:idx]))
	if err != nil {
		return err
	}

	b = b[idx+1:]
	idx = bytes.IndexByte(b, '"')
	b = b[idx+1:]
	idx = bytes.IndexByte(b, '"')
	t2, err := time.Parse(layout, string(string(b[:idx])))
	if err != nil {
		return err
	}
	t.Start = t1
	t.End = t2
	return nil
}

func (t timeRange) Value() (driver.Value, error) {
	const layout = "2006-01-02 15:04:05.999999-07"
	return []byte(fmt.Sprintf(`["%s","%s"]`, t.Start.Format(layout), t.End.Format(layout))), nil
}

type Storage struct {
	db *sqlx.DB
}

func New(db *sql.DB) *Storage {
	return &Storage{db: sqlx.NewDb(db, "postgres")}
}

func (st *Storage) Store(sp tracer.RawSpan) (err error) {
	const upsertSpan = `
INSERT INTO spans (id, trace_id, time, operation_name)
VALUES ($1, $2, $3, $4)
ON CONFLICT (id) DO
  UPDATE SET
    time = $3,
    operation_name = $4`
	const insertTag = `INSERT INTO tags (span_id, trace_id, key, value) VALUES ($1, $2, $3, $4)`
	const insertLog = `INSERT INTO tags (span_id, trace_id, key, value, time) VALUES ($1, $2, $3, $4, $5)`
	const insertParentRelation = `INSERT INTO relations (span1_id, span2_id, kind) VALUES ($1, $2, 'parent')`
	const insertParentSpan = `INSERT INTO spans (id, trace_id, time, operation_name) VALUES ($1, $2, $3, '') ON CONFLICT (id) DO NOTHING`

	tx, err := st.db.Begin()
	if err != nil {
		return err
	}
	defer func() {
		if err != nil {
			_ = tx.Rollback()
			return
		}
		err = tx.Commit()
	}()

	_, err = tx.Exec(upsertSpan,
		int64(sp.SpanID), int64(sp.TraceID), timeRange{sp.StartTime, sp.FinishTime}, sp.OperationName)
	if err != nil {
		return err
	}

	if sp.ParentID != 0 {
		_, err = tx.Exec(insertParentSpan,
			int64(sp.ParentID), int64(sp.TraceID), timeRange{time.Time{}, time.Time{}})
		if err != nil {
			return err
		}
		_, err = tx.Exec(insertParentSpan,
			int64(sp.TraceID), int64(sp.TraceID), timeRange{sp.StartTime, sp.FinishTime})
		if err != nil {
			return err
		}

		_, err = tx.Exec(insertParentRelation,
			int64(sp.ParentID), int64(sp.SpanID))
		if err != nil {
			return err
		}
	}

	for k, v := range sp.Tags {
		vs := fmt.Sprintf("%v", v) // XXX
		_, err = tx.Exec(insertTag,
			int64(sp.SpanID), int64(sp.TraceID), k, vs)
		if err != nil {
			return err
		}
	}
	for _, l := range sp.Logs {
		v := fmt.Sprintf("%v", l.Payload) // XXX
		_, err = tx.Exec(insertLog,
			int64(sp.SpanID), int64(sp.TraceID), l.Event, v, l.Timestamp)
		if err != nil {
			return err
		}
	}
	return nil
}

func (st *Storage) TraceWithID(id uint64) (tracer.RawTrace, error) {
	tx, err := st.db.Begin()
	if err != nil {
		return tracer.RawTrace{}, err
	}
	defer tx.Rollback()
	return st.traceWithID(tx, id)
}

func (st *Storage) traceWithID(tx *sql.Tx, id uint64) (tracer.RawTrace, error) {
	const selectTrace = `
SELECT spans.id, spans.trace_id, spans.time, spans.operation_name, tags.key, tags.value, tags.time
FROM spans
  LEFT JOIN tags
    ON spans.id = tags.span_id
WHERE spans.trace_id = $1
ORDER BY
  lower(spans.time) ASC,
  spans.id`
	rows, err := tx.Query(selectTrace, int64(id))
	if err != nil {
		return tracer.RawTrace{}, err
	}

	spans, err := scanSpans(rows)
	if err != nil {
		return tracer.RawTrace{}, err
	}
	return tracer.RawTrace{
		TraceID: id,
		Spans:   spans,
	}, nil
}

func scanSpans(rows *sql.Rows) ([]tracer.RawSpan, error) {
	// TODO select parents
	var spans []tracer.RawSpan
	var (
		prevSpanID int64

		spanID        int64
		traceID       int64
		spanTime      timeRange
		operationName string
		tagKey        string
		tagValue      string
		tagTime       *time.Time
	)
	tagTime = new(time.Time)
	var span tracer.RawSpan
	for rows.Next() {
		if err := rows.Scan(&spanID, &traceID, &spanTime, &operationName, &tagKey, &tagValue, &tagTime); err != nil {
			return nil, err
		}
		if spanID != prevSpanID {
			if prevSpanID != 0 {
				spans = append(spans, span)
			}
			prevSpanID = spanID
			span = tracer.RawSpan{
				Tags: map[string]interface{}{},
			}
		}
		span.SpanID = uint64(spanID)
		span.TraceID = uint64(traceID)
		span.StartTime = spanTime.Start
		span.FinishTime = spanTime.End
		span.OperationName = operationName
		if tagKey != "" {
			if tagTime == nil {
				span.Tags[tagKey] = tagValue
			} else {
				span.Logs = append(span.Logs, opentracing.LogData{
					Timestamp: *tagTime,
					Event:     tagKey,
					Payload:   tagValue,
				})
			}
		}
	}
	if err := rows.Err(); err != nil {
		return nil, err
	}
	if span.SpanID != 0 {
		spans = append(spans, span)
	}
	return spans, nil
}

func (st *Storage) SpanWithID(id uint64) (tracer.RawSpan, error) {
	tx, err := st.db.Begin()
	if err != nil {
		return tracer.RawSpan{}, err
	}
	defer tx.Rollback()
	return st.spanWithID(tx, id)
}

func (st *Storage) spanWithID(tx *sql.Tx, id uint64) (tracer.RawSpan, error) {
	const selectSpan = `
SELECT spans.id, spans.trace_id, spans.time, spans.time, spans.operation_name, tags.key, tags.value, tags.time
FROM spans
  LEFT JOIN tags
    ON spans.id = tags.span_id
WHERE id = $1
LIMIT 1`
	rows, err := tx.Query(selectSpan, int64(id))
	if err != nil {
		return tracer.RawSpan{}, err
	}
	spans, err := scanSpans(rows)
	if err != nil {
		return tracer.RawSpan{}, err
	}
	if len(spans) == 0 {
		return tracer.RawSpan{}, sql.ErrNoRows
	}
	return spans[0], nil
}

func (st *Storage) QueryTraces(q tracer.Query) ([]tracer.RawTrace, error) {
	tx, err := st.db.Begin()
	if err != nil {
		return nil, err
	}
	defer tx.Rollback()

	var andConds []string
	var andArgs []interface{}
	var orConds []string
	var orArgs []interface{}
	if q.FinishTime.IsZero() {
		q.FinishTime = time.Now()
	}
	for _, tag := range q.AndTags {
		if tag.CheckValue {
			andConds = append(andConds, `(tags.key = ? AND tags.value = ?)`)
			andArgs = append(andArgs, tag.Key, tag.Value)
		} else {
			andConds = append(andConds, `(tags.key = ?)`)
			andArgs = append(andArgs, tag.Key)
		}
	}

	for _, tag := range q.OrTags {
		if tag.CheckValue {
			orConds = append(orConds, `(tags.key = ? AND tags.value = ?)`)
			orArgs = append(orArgs, tag.Key, tag.Value)
		} else {
			orConds = append(orConds, `(tags.key = ?)`)
			orArgs = append(orArgs, tag.Key)
		}
	}

	and := strings.Join(andConds, " AND ")
	or := strings.Join(orConds, " OR ")
	conds := []string{"true"}
	if and != "" {
		conds = append(conds, and)
	}
	if or != "" {
		conds = append(conds, or)
	}

	query := st.db.Rebind(`
SELECT spans.trace_id
FROM spans
WHERE
  EXISTS (
    SELECT 1
    FROM tags
    WHERE
      tags.trace_id = spans.trace_id AND
      ` + strings.Join(conds, " AND ") + `
  ) AND
  ? @> spans.time AND
  spans.id = spans.trace_id
ORDER BY
  spans.time ASC,
  spans.trace_id
`)
	args := make([]interface{}, 0, len(andArgs)+len(orArgs))
	args = append(args, andArgs...)
	args = append(args, orArgs...)
	args = append(args, timeRange{q.StartTime, q.FinishTime})

	var ids []int64
	rows, err := st.db.Query(query, args...)
	if err != nil {
		return nil, err
	}
	var id int64
	for rows.Next() {
		if err := rows.Scan(&id); err != nil {
			return nil, err
		}
		ids = append(ids, id)
	}
	if err := rows.Err(); err != nil {
		return nil, err
	}

	var traces []tracer.RawTrace
	for _, id := range ids {
		trace, err := st.traceWithID(tx, uint64(id))
		if err != nil {
			return nil, err
		}
		traces = append(traces, trace)
	}
	return traces, nil
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"encoding/xml"
	"flag"
	"io"
	"net/http"
	"os"
	"strconv"
	"sync"

	"github.com/geo-stanciu/go-utils/utils"
	"github.com/sirupsen/logrus"

	_ "github.com/denisenkom/go-mssqldb"
	_ "github.com/go-sql-driver/mysql"
	_ "github.com/lib/pq"
	_ "github.com/mattn/go-sqlite3"
	//_ "github.com/mattn/go-oci8"
)

var (
	appName    = "GoExchRates"
	appVersion = "0.0.2.0"
	log        = logrus.New()
	audit      = utils.AuditLog{}
	db         *sql.DB
	dbutl      *utils.DbUtils
	config     = configuration{}
)

func init() {
	// Log as JSON instead of the default ASCII formatter.
	log.Formatter = new(logrus.JSONFormatter)
	log.Level = logrus.DebugLevel

	dbutl = new(utils.DbUtils)
}

// Rate - Exchange rate struct
type Rate struct {
	Currency   string `xml:"currency,attr"`
	Multiplier string `xml:"multiplier,attr"`
	Rate       string `xml:",chardata"`
}

// Cube - colection of exchange rates
type Cube struct {
	Date string `xml:"date,attr"`
	Rate []Rate
}

// ParseSourceStream - Parse Source Stream
type ParseSourceStream func(source io.Reader) error

func main() {
	var err error
	var wg sync.WaitGroup

	cfgPtr := flag.String("c", "conf.json", "config file")

	flag.Parse()

	err = config.ReadFromFile(*cfgPtr)
	if err != nil {
		log.Println(err)
		return
	}

	err = dbutl.Connect2Database(&db, config.DbType, config.DbURL)
	if err != nil {
		log.Println(err)
		return
	}
	defer db.Close()

	audit.SetLogger(appName, appVersion, log, dbutl)
	audit.SetWaitGroup(&wg)
	defer audit.Close()

	mw := io.MultiWriter(os.Stdout, audit)
	log.Out = mw

	err = prepareCurrencies()
	if err != nil {
		log.Println(err)
		return
	}

	err = getStreamFromURL(config.RatesXMLUrl, parseXMLSource)
	if err != nil {
		log.Println(err)
		return
	}

	audit.Log(nil, "import exchange rates", "Import done.")
	wg.Wait()
}

func getStreamFromURL(url string, callback ParseSourceStream) error {
	response, err := http.Get(url)
	if err != nil {
		return err
	}
	defer response.Body.Close()

	err = callback(response.Body)
	if err != nil {
		return err
	}

	return nil
}

func getStreamFromFile(filename string, callback ParseSourceStream) error {
	f, err := os.Open(filename)
	if err != nil {
		return err
	}
	defer f.Close()

	err = callback(f)
	if err != nil {
		return err
	}

	return nil
}

func parseXMLSource(source io.Reader) error {
	decoder := xml.NewDecoder(source)

	for {
		t, err := decoder.Token()
		if t == nil {
			break
		}
		if err != nil && err != io.EOF {
			return err
		}

		switch se := t.(type) {
		case xml.StartElement:
			if se.Name.Local == "Cube" {
				var cube Cube
				decoder.DecodeElement(&cube, &se)

				if err := dealWithRates(&cube); err != nil {
					return err
				}
			}
		}
	}

	return nil
}

func dealWithRates(cube *Cube) error {
	tx, err := db.Begin()
	if err != nil {
		return err
	}
	defer tx.Rollback()

	if err = dbutl.SetAsyncCommit(tx); err != nil {
		tx.Rollback()
		return err
	}

	if err = storeRates(tx, *cube); err != nil {
		return err
	}

	tx.Commit()

	return nil
}

func getCurrencyIfExists(tx *sql.Tx, currency string) (int32, error) {
	var currencyID int32

	pq := dbutl.PQuery(`
		SELECT currency_id FROM currency WHERE currency = ?
	`, currency)

	err := tx.QueryRow(pq.Query, pq.Args...).Scan(&currencyID)
	if err != nil && err != sql.ErrNoRows {
		return -1, err
	}

	return currencyID, nil
}

func addCurrencyIfNotExists(tx *sql.Tx, currency string) (int32, error) {
	currencyID, err := getCurrencyIfExists(tx, currency)
	if err != nil {
		return -1, err
	} else if currencyID > 0 {
		return currencyID, nil
	}

	pq := dbutl.PQuery(`
		INSERT INTO currency (currency) VALUES (?)
	`, currency)

	_, err = dbutl.ExecTx(tx, pq)
	if err != nil {
		return -1, err
	}

	audit.Log(nil, "add currency", "Adding missing currency...", "currency", currency)

	return addCurrencyIfNotExists(tx, currency)
}

func prepareCurrencies() error {
	tx, err := db.Begin()
	if err != nil {
		return err
	}
	defer tx.Rollback()

	if err = dbutl.SetAsyncCommit(tx); err != nil {
		return err
	}

	refCurrencyID, err := addCurrencyIfNotExists(tx, "RON")
	if err != nil {
		return err
	}

	_, err = addCurrencyIfNotExists(tx, "EUR")
	if err != nil {
		return err
	}

	_, err = addCurrencyIfNotExists(tx, "USD")
	if err != nil {
		return err
	}

	_, err = addCurrencyIfNotExists(tx, "CHF")
	if err != nil {
		return err
	}

	err = storeRate(tx, "1970-01-01", refCurrencyID, "RON", 1.0, 1.0)
	if err != nil {
		return err
	}

	tx.Commit()

	return nil
}

func storeRates(tx *sql.Tx, cube Cube) error {
	var err error
	var refCurrencyID int32

	audit.Log(nil, "exchange rates", "Importing exchange rates...", "date", cube.Date)

	refCurrencyID, err = addCurrencyIfNotExists(tx, "RON")
	if err != nil {
		return err
	}

	for _, rate := range cube.Rate {
		multiplier := 1.0
		exchRate := 1.0

		if rate.Multiplier == "-" {
			multiplier = 1.0
		} else if len(rate.Multiplier) > 0 {
			multiplier, err = strconv.ParseFloat(rate.Multiplier, 64)
			if err != nil {
				return err
			}
		}

		if rate.Rate == "-" {
			continue
		} else {
			exchRate, err = strconv.ParseFloat(rate.Rate, 64)
			if err != nil {
				return err
			}
		}

		err = storeRate(tx, cube.Date, refCurrencyID, rate.Currency, multiplier, exchRate)
		if err != nil {
			return err
		}
	}

	return nil
}

func storeRate(tx *sql.Tx, date string, refCurrencyID int32, currency string, multiplier float64, exchRate float64) error {
	found := 0
	var currencyID int32
	var err error
	rate := exchRate / multiplier

	if config.AddMissingCurrencies {
		currencyID, err = addCurrencyIfNotExists(tx, currency)
	} else {
		currencyID, err = getCurrencyIfExists(tx, currency)
	}

	if err != nil {
		return err
	}

	pq := dbutl.PQuery(`
		SELECT CASE WHEN EXISTS (
			SELECT 1
			FROM exchange_rate
		   WHERE currency_id = ?
		     AND exchange_date = DATE ?
		) THEN 1 ELSE 0 END
		FROM dual
	`, currencyID,
		date)

	err = tx.QueryRow(pq.Query, pq.Args...).Scan(&found)
	if err != nil {
		return err
	}

	if found == 0 {
		pq = dbutl.PQuery(`
			INSERT INTO exchange_rate (
				reference_currency_id,
				currency_id,
				exchange_date,
				rate
			)
			VALUES (?, ?, DATE ?, ?)
		`, refCurrencyID,
			currencyID,
			date,
			rate)

		_, err = dbutl.ExecTx(tx, pq)
		if err != nil {
			return err
		}

		audit.Log(nil,
			"add exchange rate",
			"added value",
			"date", date,
			"currency", currency,
			"rate", rate)
	}

	return nil
}
<commit_msg>sqlite3 - open in journal mode<commit_after>package main

import (
	"database/sql"
	"encoding/xml"
	"flag"
	"io"
	"math/big"
	"net/http"
	"os"
	"strconv"
	"sync"

	"github.com/geo-stanciu/go-utils/utils"
	"github.com/sirupsen/logrus"

	_ "github.com/denisenkom/go-mssqldb"
	_ "github.com/go-sql-driver/mysql"
	_ "github.com/lib/pq"
	_ "github.com/mattn/go-sqlite3"
	//_ "github.com/mattn/go-oci8"
)

var (
	appName    = "GoExchRates"
	appVersion = "0.0.2.0"
	log        = logrus.New()
	audit      = utils.AuditLog{}
	db         *sql.DB
	dbutl      *utils.DbUtils
	config     = configuration{}
)

func init() {
	// Log as JSON instead of the default ASCII formatter.
	log.Formatter = new(logrus.JSONFormatter)
	log.Level = logrus.DebugLevel

	dbutl = new(utils.DbUtils)
}

// Rate - Exchange rate struct
type Rate struct {
	Currency   string `xml:"currency,attr"`
	Multiplier string `xml:"multiplier,attr"`
	Rate       string `xml:",chardata"`
}

// Cube - colection of exchange rates
type Cube struct {
	Date string `xml:"date,attr"`
	Rate []Rate
}

// ParseSourceStream - Parse Source Stream
type ParseSourceStream func(source io.Reader) error

func main() {
	var err error
	var wg sync.WaitGroup

	cfgPtr := flag.String("c", "conf.json", "config file")

	flag.Parse()

	err = config.ReadFromFile(*cfgPtr)
	if err != nil {
		log.Println(err)
		return
	}

	err = dbutl.Connect2Database(&db, config.DbType, config.DbURL)
	if err != nil {
		log.Println(err)
		return
	}
	defer db.Close()

	audit.SetLogger(appName, appVersion, log, dbutl)
	audit.SetWaitGroup(&wg)
	defer audit.Close()

	mw := io.MultiWriter(os.Stdout, audit)
	log.Out = mw

	err = prepareCurrencies()
	if err != nil {
		log.Println(err)
		return
	}

	err = getStreamFromURL(config.RatesXMLUrl, parseXMLSource)
	if err != nil {
		log.Println(err)
		return
	}

	audit.Log(nil, "import exchange rates", "Import done.")
	wg.Wait()
}

func getStreamFromURL(url string, callback ParseSourceStream) error {
	response, err := http.Get(url)
	if err != nil {
		return err
	}
	defer response.Body.Close()

	err = callback(response.Body)
	if err != nil {
		return err
	}

	return nil
}

func getStreamFromFile(filename string, callback ParseSourceStream) error {
	f, err := os.Open(filename)
	if err != nil {
		return err
	}
	defer f.Close()

	err = callback(f)
	if err != nil {
		return err
	}

	return nil
}

func parseXMLSource(source io.Reader) error {
	decoder := xml.NewDecoder(source)

	for {
		t, err := decoder.Token()
		if t == nil {
			break
		}
		if err != nil && err != io.EOF {
			return err
		}

		switch se := t.(type) {
		case xml.StartElement:
			if se.Name.Local == "Cube" {
				var cube Cube
				decoder.DecodeElement(&cube, &se)

				if err := dealWithRates(&cube); err != nil {
					return err
				}
			}
		}
	}

	return nil
}

func dealWithRates(cube *Cube) error {
	tx, err := db.Begin()
	if err != nil {
		return err
	}
	defer tx.Rollback()

	if err = dbutl.SetAsyncCommit(tx); err != nil {
		tx.Rollback()
		return err
	}

	if err = storeRates(tx, *cube); err != nil {
		return err
	}

	tx.Commit()

	return nil
}

func getCurrencyIfExists(tx *sql.Tx, currency string) (int32, error) {
	var currencyID int32

	pq := dbutl.PQuery(`
		SELECT currency_id FROM currency WHERE currency = ?
	`, currency)

	err := tx.QueryRow(pq.Query, pq.Args...).Scan(&currencyID)
	if err != nil && err != sql.ErrNoRows {
		return -1, err
	}

	return currencyID, nil
}

func addCurrencyIfNotExists(tx *sql.Tx, currency string) (int32, error) {
	currencyID, err := getCurrencyIfExists(tx, currency)
	if err != nil {
		return -1, err
	} else if currencyID > 0 {
		return currencyID, nil
	}

	pq := dbutl.PQuery(`
		INSERT INTO currency (currency) VALUES (?)
	`, currency)

	_, err = dbutl.ExecTx(tx, pq)
	if err != nil {
		return -1, err
	}

	audit.Log(nil, "add currency", "Adding missing currency...", "currency", currency)

	return addCurrencyIfNotExists(tx, currency)
}

func prepareCurrencies() error {
	tx, err := db.Begin()
	if err != nil {
		return err
	}
	defer tx.Rollback()

	if err = dbutl.SetAsyncCommit(tx); err != nil {
		return err
	}

	refCurrencyID, err := addCurrencyIfNotExists(tx, "RON")
	if err != nil {
		return err
	}

	_, err = addCurrencyIfNotExists(tx, "EUR")
	if err != nil {
		return err
	}

	_, err = addCurrencyIfNotExists(tx, "USD")
	if err != nil {
		return err
	}

	_, err = addCurrencyIfNotExists(tx, "CHF")
	if err != nil {
		return err
	}

	err = storeRate(tx, "1970-01-01", refCurrencyID, "RON", 1.0, 1.0)
	if err != nil {
		return err
	}

	tx.Commit()

	return nil
}

func storeRates(tx *sql.Tx, cube Cube) error {
	var err error
	var refCurrencyID int32

	audit.Log(nil, "exchange rates", "Importing exchange rates...", "date", cube.Date)

	refCurrencyID, err = addCurrencyIfNotExists(tx, "RON")
	if err != nil {
		return err
	}

	for _, rate := range cube.Rate {
		multiplier := 1.0
		exchRate := 1.0

		if rate.Multiplier == "-" {
			multiplier = 1.0
		} else if len(rate.Multiplier) > 0 {
			multiplier, err = strconv.ParseFloat(rate.Multiplier, 64)
			if err != nil {
				return err
			}
		}

		if rate.Rate == "-" {
			continue
		} else {
			exchRate, err = strconv.ParseFloat(rate.Rate, 64)
			if err != nil {
				return err
			}
		}

		err = storeRate(tx, cube.Date, refCurrencyID, rate.Currency, multiplier, exchRate)
		if err != nil {
			return err
		}
	}

	return nil
}

func storeRate(tx *sql.Tx, date string, refCurrencyID int32, currency string, multiplier float64, exchRate float64) error {
	found := 0
	var currencyID int32
	var err error

	exch := big.NewFloat(exchRate)
	mul := big.NewFloat(multiplier)
	srate := new(big.Float).SetMode(big.ToNearestAway).Quo(exch, mul).Text('f', 6)

	rate, err := strconv.ParseFloat(srate, 64)
	if err != nil {
		return err
	}

	if config.AddMissingCurrencies {
		currencyID, err = addCurrencyIfNotExists(tx, currency)
	} else {
		currencyID, err = getCurrencyIfExists(tx, currency)
	}

	if err != nil {
		return err
	}

	pq := dbutl.PQuery(`
		SELECT CASE WHEN EXISTS (
			SELECT 1
			FROM exchange_rate
		   WHERE currency_id = ?
		     AND exchange_date = DATE ?
		) THEN 1 ELSE 0 END
		FROM dual
	`, currencyID,
		date)

	err = tx.QueryRow(pq.Query, pq.Args...).Scan(&found)
	if err != nil {
		return err
	}

	if found == 0 {
		pq = dbutl.PQuery(`
			INSERT INTO exchange_rate (
				reference_currency_id,
				currency_id,
				exchange_date,
				rate
			)
			VALUES (?, ?, DATE ?, ?)
		`, refCurrencyID,
			currencyID,
			date,
			rate)

		_, err = dbutl.ExecTx(tx, pq)
		if err != nil {
			return err
		}

		audit.Log(nil,
			"add exchange rate",
			"added value",
			"date", date,
			"currency", currency,
			"rate", rate)
	}

	return nil
}
<|endoftext|>