<commit_before>// Copyright (c) 2021 The Decred developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package blockchain

// UtxoBackendTx represents a UtxoBackend transaction.
//
// As would be expected with a transaction, no changes will be saved to the
// underlying UtxoBackend until it has been committed.  The transaction will
// only provide a view of the database at the time it was created.  Transactions
// should not be long running operations.
//
// The interface contract requires that these methods are safe for concurrent
// access.
type UtxoBackendTx interface {
	// Get returns the value for the given key.  It returns nil if the key does
	// not exist.  An empty slice is returned for keys that exist but have no
	// value assigned.
	//
	// The returned slice is safe to modify.  Additionally, it is safe to modify
	// the slice passed as an argument after Get returns.
	Get(key []byte) ([]byte, error)

	// Has returns true if the key exists.
	//
	// It is safe to modify the slice passed as an argument after Has returns.
	Has(key []byte) (bool, error)

	// Put sets the value for the given key.  It overwrites any previous value for
	// that key.
	//
	// It is safe to modify the slice passed as an argument after Put returns.
	Put(key, value []byte) error

	// Delete removes the given key.
	//
	// It is safe to modify the slice passed as an argument after Delete returns.
	Delete(key []byte) error

	// NewIterator returns an iterator for the latest snapshot of the transaction.
	// The returned iterator is NOT safe for concurrent use, but it is safe to use
	// multiple iterators concurrently, with each in a dedicated goroutine.
	//
	// The prefix parameter allows for slicing the iterator to only contain keys
	// with the given prefix.  A nil prefix is treated as a key BEFORE all keys.
	//
	// NOTE: The contents of any slice returned by the iterator should NOT be
	// modified unless noted otherwise.
	//
	// The iterator must be released after use, by calling the Release method.
	NewIterator(prefix []byte) UtxoBackendIterator

	// Commit commits the transaction.  If the returned error is not nil, then the
	// transaction is not committed and can either be retried or discarded.
	//
	// Other methods should not be called after the transaction has been
	// committed.
	Commit() error

	// Discard discards the transaction.  This method is a noop if the
	// transaction is already closed (either committed or discarded).
	//
	// Other methods should not be called after the transaction has been
	// discarded.
	Discard()
}
<commit_msg>blockchain: Add levelDbUtxoBackendTx type.<commit_after>// Copyright (c) 2021 The Decred developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package blockchain

import (
	"errors"
	"fmt"

	"github.com/syndtr/goleveldb/leveldb"
	"github.com/syndtr/goleveldb/leveldb/util"
)

// UtxoBackendTx represents a UtxoBackend transaction.
//
// As would be expected with a transaction, no changes will be saved to the
// underlying UtxoBackend until it has been committed.  The transaction will
// only provide a view of the database at the time it was created.  Transactions
// should not be long running operations.
//
// The interface contract requires that these methods are safe for concurrent
// access.
type UtxoBackendTx interface {
	// Get returns the value for the given key.  It returns nil if the key does
	// not exist.  An empty slice is returned for keys that exist but have no
	// value assigned.
	//
	// The returned slice is safe to modify.  Additionally, it is safe to modify
	// the slice passed as an argument after Get returns.
	Get(key []byte) ([]byte, error)

	// Has returns true if the key exists.
	//
	// It is safe to modify the slice passed as an argument after Has returns.
	Has(key []byte) (bool, error)

	// Put sets the value for the given key.  It overwrites any previous value for
	// that key.
	//
	// It is safe to modify the slice passed as an argument after Put returns.
	Put(key, value []byte) error

	// Delete removes the given key.
	//
	// It is safe to modify the slice passed as an argument after Delete returns.
	Delete(key []byte) error

	// NewIterator returns an iterator for the latest snapshot of the transaction.
	// The returned iterator is NOT safe for concurrent use, but it is safe to use
	// multiple iterators concurrently, with each in a dedicated goroutine.
	//
	// The prefix parameter allows for slicing the iterator to only contain keys
	// with the given prefix.  A nil prefix is treated as a key BEFORE all keys.
	//
	// NOTE: The contents of any slice returned by the iterator should NOT be
	// modified unless noted otherwise.
	//
	// The iterator must be released after use, by calling the Release method.
	NewIterator(prefix []byte) UtxoBackendIterator

	// Commit commits the transaction.  If the returned error is not nil, then the
	// transaction is not committed and can either be retried or discarded.
	//
	// Other methods should not be called after the transaction has been
	// committed.
	Commit() error

	// Discard discards the transaction.  This method is a noop if the
	// transaction is already closed (either committed or discarded).
	//
	// Other methods should not be called after the transaction has been
	// discarded.
	Discard()
}

// levelDbUtxoBackendTx represents a UtxoBackend transaction.  It wraps an
// underlying leveldb transaction and implements the UtxoBackendTx interface.
type levelDbUtxoBackendTx struct {
	*leveldb.Transaction
}

// Ensure levelDbUtxoBackendTx implements the UtxoBackendTx interface.
var _ UtxoBackendTx = (*levelDbUtxoBackendTx)(nil)

// Get returns the value for the given key.  It returns nil if the key does not
// exist.  An empty slice is returned for keys that exist but have no value
// assigned.
//
// The returned slice is safe to modify.  Additionally, it is safe to modify the
// slice passed as an argument after Get returns.
func (tx *levelDbUtxoBackendTx) Get(key []byte) ([]byte, error) {
	serialized, err := tx.Transaction.Get(key, nil)
	if err != nil {
		if errors.Is(err, leveldb.ErrNotFound) {
			return nil, nil
		}
		str := fmt.Sprintf("failed to get key %x from leveldb transaction", key)
		return nil, convertLdbErr(err, str)
	}
	return serialized, nil
}

// Has returns true if the key exists.
//
// It is safe to modify the slice passed as an argument after Has returns.
func (tx *levelDbUtxoBackendTx) Has(key []byte) (bool, error) {
	has, err := tx.Transaction.Has(key, nil)
	if err != nil {
		str := fmt.Sprintf("failed to get existence of key %x from leveldb "+
			"transaction", key)
		return false, convertLdbErr(err, str)
	}
	return has, nil
}

// Put sets the value for the given key.  It overwrites any previous value for
// that key.
//
// It is safe to modify the slice passed as an argument after Put returns.
func (tx *levelDbUtxoBackendTx) Put(key, value []byte) error {
	err := tx.Transaction.Put(key, value, nil)
	if err != nil {
		str := fmt.Sprintf("failed to put key %x (value %x) to leveldb transaction",
			key, value)
		return convertLdbErr(err, str)
	}
	return nil
}

// Delete removes the given key.
//
// It is safe to modify the slice passed as an argument after Delete returns.
func (tx *levelDbUtxoBackendTx) Delete(key []byte) error {
	err := tx.Transaction.Delete(key, nil)
	if err != nil {
		str := fmt.Sprintf("failed to delete key %x from leveldb transaction", key)
		return convertLdbErr(err, str)
	}
	return nil
}

// NewIterator returns an iterator for the latest snapshot of the transaction.
// The returned iterator is NOT safe for concurrent use, but it is safe to use
// multiple iterators concurrently, with each in a dedicated goroutine.
//
// The prefix parameter allows for slicing the iterator to only contain keys
// with the given prefix.  A nil prefix is treated as a key BEFORE all keys.
//
// NOTE: The contents of any slice returned by the iterator should NOT be
// modified unless noted otherwise.
//
// The iterator must be released after use, by calling the Release method.
func (tx *levelDbUtxoBackendTx) NewIterator(prefix []byte) UtxoBackendIterator {
	var slice *util.Range
	if prefix != nil {
		slice = util.BytesPrefix(prefix)
	}
	return tx.Transaction.NewIterator(slice, nil)
}
<|endoftext|>


<commit_before>// Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0.

package summary

import (
	"strings"
	"sync"
	"time"

	"github.com/docker/go-units"
	"github.com/pingcap/log"
	"go.uber.org/zap"
)

const (
	// BackupUnit tells summary in backup
	BackupUnit = "backup"
	// RestoreUnit tells summary in restore
	RestoreUnit = "restore"

	// TotalKV is a field we collect during backup/restore
	TotalKV = "total kv"
	// TotalBytes is a field we collect during backup/restore
	TotalBytes = "total bytes"
	// BackupDataSize is a field we collect after backup finish
	BackupDataSize = "backup data size(after compressed)"
	// RestoreDataSize is a field we collection after restore finish
	RestoreDataSize = "restore data size(after compressed)"
)

// LogCollector collects infos into summary log.
type LogCollector interface {
	SetUnit(unit string)

	CollectSuccessUnit(name string, unitCount int, arg interface{})

	CollectFailureUnit(name string, reason error)

	CollectDuration(name string, t time.Duration)

	CollectInt(name string, t int)

	CollectUInt(name string, t uint64)

	SetSuccessStatus(success bool)

	Summary(name string)
}

type logFunc func(msg string, fields ...zap.Field)

var collector LogCollector = NewLogCollector(log.Info)

// InitCollector initilize global collector instance.
func InitCollector( // revive:disable-line:flag-parameter
	hasLogFile bool,
) {
	logF := log.L().Info
	if hasLogFile {
		conf := new(log.Config)
		// Always duplicate summary to stdout.
		logger, _, err := log.InitLogger(conf)
		if err == nil {
			logF = func(msg string, fields ...zap.Field) {
				logger.Info(msg, fields...)
				log.Info(msg, fields...)
			}
		}
	}
	collector = NewLogCollector(logF)
}

type logCollector struct {
	mu               sync.Mutex
	unit             string
	successUnitCount int
	failureUnitCount int
	successCosts     map[string]time.Duration
	successData      map[string]uint64
	failureReasons   map[string]error
	durations        map[string]time.Duration
	ints             map[string]int
	uints            map[string]uint64
	successStatus    bool
	startTime        time.Time

	log logFunc
}

// NewLogCollector returns a new LogCollector.
func NewLogCollector(log logFunc) LogCollector {
	return &logCollector{
		successUnitCount: 0,
		failureUnitCount: 0,
		successCosts:     make(map[string]time.Duration),
		successData:      make(map[string]uint64),
		failureReasons:   make(map[string]error),
		durations:        make(map[string]time.Duration),
		ints:             make(map[string]int),
		uints:            make(map[string]uint64),
		log:              log,
		startTime:        time.Now(),
	}
}

func (tc *logCollector) SetUnit(unit string) {
	tc.mu.Lock()
	defer tc.mu.Unlock()
	tc.unit = unit
}

func (tc *logCollector) CollectSuccessUnit(name string, unitCount int, arg interface{}) {
	tc.mu.Lock()
	defer tc.mu.Unlock()

	switch v := arg.(type) {
	case time.Duration:
		tc.successUnitCount += unitCount
		tc.successCosts[name] += v
	case uint64:
		tc.successData[name] += v
	}
}

func (tc *logCollector) CollectFailureUnit(name string, reason error) {
	tc.mu.Lock()
	defer tc.mu.Unlock()
	if _, ok := tc.failureReasons[name]; !ok {
		tc.failureReasons[name] = reason
		tc.failureUnitCount++
	}
}

func (tc *logCollector) CollectDuration(name string, t time.Duration) {
	tc.mu.Lock()
	defer tc.mu.Unlock()
	tc.durations[name] += t
}

func (tc *logCollector) CollectInt(name string, t int) {
	tc.mu.Lock()
	defer tc.mu.Unlock()
	tc.ints[name] += t
}

func (tc *logCollector) CollectUInt(name string, t uint64) {
	tc.mu.Lock()
	defer tc.mu.Unlock()
	tc.uints[name] += t
}

func (tc *logCollector) SetSuccessStatus(success bool) {
	tc.mu.Lock()
	defer tc.mu.Unlock()
	tc.successStatus = success
}

func logKeyFor(key string) string {
	return strings.ReplaceAll(key, " ", "-")
}

func (tc *logCollector) Summary(name string) {
	tc.mu.Lock()
	defer func() {
		tc.durations = make(map[string]time.Duration)
		tc.ints = make(map[string]int)
		tc.successCosts = make(map[string]time.Duration)
		tc.failureReasons = make(map[string]error)
		tc.mu.Unlock()
	}()

	logFields := make([]zap.Field, 0, len(tc.durations)+len(tc.ints)+3)

	logFields = append(logFields,
		zap.Int("total-ranges", tc.failureUnitCount+tc.successUnitCount),
		zap.Int("ranges-succeed", tc.successUnitCount),
		zap.Int("ranges-failed", tc.failureUnitCount),
	)

	for key, val := range tc.durations {
		logFields = append(logFields, zap.Duration(logKeyFor(key), val))
	}
	for key, val := range tc.ints {
		logFields = append(logFields, zap.Int(logKeyFor(key), val))
	}
	for key, val := range tc.uints {
		logFields = append(logFields, zap.Uint64(logKeyFor(key), val))
	}

	if len(tc.failureReasons) != 0 || !tc.successStatus {
		for unitName, reason := range tc.failureReasons {
			logFields = append(logFields, zap.String("unit-name", unitName), zap.Error(reason))
		}
		tc.log(name+" failed summary", logFields...)
		return
	}

	totalDureTime := time.Since(tc.startTime)
	logFields = append(logFields, zap.Duration("total-take", totalDureTime))
	for name, data := range tc.successData {
		if name == TotalBytes {
			logFields = append(logFields,
				zap.String("total-kv-size", units.HumanSize(float64(data))),
				zap.String("average-speed", units.HumanSize(float64(data)/totalDureTime.Seconds())+"/s"))
			continue
		}
		if name == BackupDataSize {
			if tc.failureUnitCount+tc.successUnitCount == 0 {
				logFields = append(logFields, zap.String("Result", "Nothing to bakcup"))
			} else {
				logFields = append(logFields,
					zap.String(BackupDataSize, units.HumanSize(float64(data))))
			}
			continue
		}
		if name == RestoreDataSize {
			if tc.failureUnitCount+tc.successUnitCount == 0 {
				logFields = append(logFields, zap.String("Result", "Nothing to restore"))
			} else {
				logFields = append(logFields,
					zap.String(RestoreDataSize, units.HumanSize(float64(data))))
			}
			continue
		}
		logFields = append(logFields, zap.Uint64(logKeyFor(name), data))
	}

	tc.log(name+" success summary", logFields...)
}

// SetLogCollector allow pass LogCollector outside.
func SetLogCollector(l LogCollector) {
	collector = l
}
<commit_msg>br: Fix format on summary of br (#28310)<commit_after>// Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0.

package summary

import (
	"strings"
	"sync"
	"time"

	"github.com/docker/go-units"
	"github.com/pingcap/log"
	"go.uber.org/zap"
)

const (
	// BackupUnit tells summary in backup
	BackupUnit = "backup"
	// RestoreUnit tells summary in restore
	RestoreUnit = "restore"

	// TotalKV is a field we collect during backup/restore
	TotalKV = "total kv"
	// TotalBytes is a field we collect during backup/restore
	TotalBytes = "total bytes"
	// BackupDataSize is a field we collect after backup finish
	BackupDataSize = "backup data size(after compressed)"
	// RestoreDataSize is a field we collection after restore finish
	RestoreDataSize = "restore data size(after compressed)"
)

// LogCollector collects infos into summary log.
type LogCollector interface {
	SetUnit(unit string)

	CollectSuccessUnit(name string, unitCount int, arg interface{})

	CollectFailureUnit(name string, reason error)

	CollectDuration(name string, t time.Duration)

	CollectInt(name string, t int)

	CollectUInt(name string, t uint64)

	SetSuccessStatus(success bool)

	Summary(name string)
}

type logFunc func(msg string, fields ...zap.Field)

var collector LogCollector = NewLogCollector(log.Info)

// InitCollector initilize global collector instance.
func InitCollector( // revive:disable-line:flag-parameter
	hasLogFile bool,
) {
	logF := log.L().Info
	if hasLogFile {
		conf := new(log.Config)
		// Always duplicate summary to stdout.
		logger, _, err := log.InitLogger(conf)
		if err == nil {
			logF = func(msg string, fields ...zap.Field) {
				logger.Info(msg, fields...)
				log.Info(msg, fields...)
			}
		}
	}
	collector = NewLogCollector(logF)
}

type logCollector struct {
	mu               sync.Mutex
	unit             string
	successUnitCount int
	failureUnitCount int
	successCosts     map[string]time.Duration
	successData      map[string]uint64
	failureReasons   map[string]error
	durations        map[string]time.Duration
	ints             map[string]int
	uints            map[string]uint64
	successStatus    bool
	startTime        time.Time

	log logFunc
}

// NewLogCollector returns a new LogCollector.
func NewLogCollector(log logFunc) LogCollector {
	return &logCollector{
		successUnitCount: 0,
		failureUnitCount: 0,
		successCosts:     make(map[string]time.Duration),
		successData:      make(map[string]uint64),
		failureReasons:   make(map[string]error),
		durations:        make(map[string]time.Duration),
		ints:             make(map[string]int),
		uints:            make(map[string]uint64),
		log:              log,
		startTime:        time.Now(),
	}
}

func (tc *logCollector) SetUnit(unit string) {
	tc.mu.Lock()
	defer tc.mu.Unlock()
	tc.unit = unit
}

func (tc *logCollector) CollectSuccessUnit(name string, unitCount int, arg interface{}) {
	tc.mu.Lock()
	defer tc.mu.Unlock()

	switch v := arg.(type) {
	case time.Duration:
		tc.successUnitCount += unitCount
		tc.successCosts[name] += v
	case uint64:
		tc.successData[name] += v
	}
}

func (tc *logCollector) CollectFailureUnit(name string, reason error) {
	tc.mu.Lock()
	defer tc.mu.Unlock()
	if _, ok := tc.failureReasons[name]; !ok {
		tc.failureReasons[name] = reason
		tc.failureUnitCount++
	}
}

func (tc *logCollector) CollectDuration(name string, t time.Duration) {
	tc.mu.Lock()
	defer tc.mu.Unlock()
	tc.durations[name] += t
}

func (tc *logCollector) CollectInt(name string, t int) {
	tc.mu.Lock()
	defer tc.mu.Unlock()
	tc.ints[name] += t
}

func (tc *logCollector) CollectUInt(name string, t uint64) {
	tc.mu.Lock()
	defer tc.mu.Unlock()
	tc.uints[name] += t
}

func (tc *logCollector) SetSuccessStatus(success bool) {
	tc.mu.Lock()
	defer tc.mu.Unlock()
	tc.successStatus = success
}

func logKeyFor(key string) string {
	return strings.ReplaceAll(key, " ", "-")
}

func (tc *logCollector) Summary(name string) {
	tc.mu.Lock()
	defer func() {
		tc.durations = make(map[string]time.Duration)
		tc.ints = make(map[string]int)
		tc.successCosts = make(map[string]time.Duration)
		tc.failureReasons = make(map[string]error)
		tc.mu.Unlock()
	}()

	logFields := make([]zap.Field, 0, len(tc.durations)+len(tc.ints)+3)

	logFields = append(logFields,
		zap.Int("total-ranges", tc.failureUnitCount+tc.successUnitCount),
		zap.Int("ranges-succeed", tc.successUnitCount),
		zap.Int("ranges-failed", tc.failureUnitCount),
	)

	for key, val := range tc.durations {
		logFields = append(logFields, zap.Duration(logKeyFor(key), val))
	}
	for key, val := range tc.ints {
		logFields = append(logFields, zap.Int(logKeyFor(key), val))
	}
	for key, val := range tc.uints {
		logFields = append(logFields, zap.Uint64(logKeyFor(key), val))
	}

	if len(tc.failureReasons) != 0 || !tc.successStatus {
		for unitName, reason := range tc.failureReasons {
			logFields = append(logFields, zap.String("unit-name", unitName), zap.Error(reason))
		}
		tc.log(name+" failed summary", logFields...)
		return
	}

	totalDureTime := time.Since(tc.startTime)
	logFields = append(logFields, zap.Duration("total-take", totalDureTime))
	for name, data := range tc.successData {
		if name == TotalBytes {
			logFields = append(logFields,
				zap.String("total-kv-size", units.HumanSize(float64(data))),
				zap.String("average-speed", units.HumanSize(float64(data)/totalDureTime.Seconds())+"/s"))
			continue
		}
		if name == BackupDataSize {
			if tc.failureUnitCount+tc.successUnitCount == 0 {
				logFields = append(logFields, zap.String("Result", "Nothing to bakcup"))
			} else {
				logFields = append(logFields,
					zap.String(logKeyFor(BackupDataSize), units.HumanSize(float64(data))))
			}
			continue
		}
		if name == RestoreDataSize {
			if tc.failureUnitCount+tc.successUnitCount == 0 {
				logFields = append(logFields, zap.String("Result", "Nothing to restore"))
			} else {
				logFields = append(logFields,
					zap.String(logKeyFor(RestoreDataSize), units.HumanSize(float64(data))))
			}
			continue
		}
		logFields = append(logFields, zap.Uint64(logKeyFor(name), data))
	}

	tc.log(name+" success summary", logFields...)
}

// SetLogCollector allow pass LogCollector outside.
func SetLogCollector(l LogCollector) {
	collector = l
}
<|endoftext|>


<commit_before>package btelegram

import (
	"strconv"

	"github.com/42wim/matterbridge/bridge/config"
	log "github.com/Sirupsen/logrus"
	"github.com/go-telegram-bot-api/telegram-bot-api"
)

type Btelegram struct {
	c       *tgbotapi.BotAPI
	Config  *config.Protocol
	Remote  chan config.Message
	Account string
}

var flog *log.Entry
var protocol = "telegram"

func init() {
	flog = log.WithFields(log.Fields{"module": protocol})
}

func New(cfg config.Protocol, account string, c chan config.Message) *Btelegram {
	b := &Btelegram{}
	b.Config = &cfg
	b.Remote = c
	b.Account = account
	return b
}

func (b *Btelegram) Connect() error {
	var err error
	flog.Info("Connecting")
	b.c, err = tgbotapi.NewBotAPI(b.Config.Token)
	if err != nil {
		flog.Debugf("%#v", err)
		return err
	}
	updates, err := b.c.GetUpdatesChan(tgbotapi.NewUpdate(0))
	if err != nil {
		flog.Debugf("%#v", err)
		return err
	}
	flog.Info("Connection succeeded")
	go b.handleRecv(updates)
	return nil
}

func (b *Btelegram) Disconnect() error {
	return nil

}

func (b *Btelegram) JoinChannel(channel config.ChannelInfo) error {
	return nil
}

func (b *Btelegram) Send(msg config.Message) (string, error) {
	flog.Debugf("Receiving %#v", msg)
	chatid, err := strconv.ParseInt(msg.Channel, 10, 64)
	if err != nil {
		return "", err
	}

	if b.Config.MessageFormat == "HTML" {
		msg.Text = makeHTML(msg.Text)
	}
	m := tgbotapi.NewMessage(chatid, msg.Username+msg.Text)
	if b.Config.MessageFormat == "HTML" {
		m.ParseMode = tgbotapi.ModeHTML
	}
	_, err = b.c.Send(m)
	return "", err
}

func (b *Btelegram) handleRecv(updates <-chan tgbotapi.Update) {
	for update := range updates {
		flog.Debugf("Receiving from telegram: %#v", update.Message)
		var message *tgbotapi.Message
		username := ""
		channel := ""
		text := ""
		// handle channels
		if update.ChannelPost != nil {
			message = update.ChannelPost
		}
		if update.EditedChannelPost != nil && !b.Config.EditDisable {
			message = update.EditedChannelPost
			message.Text = message.Text + b.Config.EditSuffix
		}
		// handle groups
		if update.Message != nil {
			message = update.Message
		}
		if update.EditedMessage != nil && !b.Config.EditDisable {
			message = update.EditedMessage
			message.Text = message.Text + b.Config.EditSuffix
		}
		if message.From != nil {
			if b.Config.UseFirstName {
				username = message.From.FirstName
			}
			if username == "" {
				username = message.From.UserName
				if username == "" {
					username = message.From.FirstName
				}
			}
			text = message.Text
			channel = strconv.FormatInt(message.Chat.ID, 10)
		}

		if username == "" {
			username = "unknown"
		}
		if message.Sticker != nil && b.Config.UseInsecureURL {
			text = text + " " + b.getFileDirectURL(message.Sticker.FileID)
		}
		if message.Video != nil && b.Config.UseInsecureURL {
			text = text + " " + b.getFileDirectURL(message.Video.FileID)
		}
		if message.Photo != nil && b.Config.UseInsecureURL {
			photos := *message.Photo
			// last photo is the biggest
			text = text + " " + b.getFileDirectURL(photos[len(photos)-1].FileID)
		}
		if message.Document != nil && b.Config.UseInsecureURL {
			text = text + " " + message.Document.FileName + " : " + b.getFileDirectURL(message.Document.FileID)
		}
		if text != "" {
			flog.Debugf("Sending message from %s on %s to gateway", username, b.Account)
			b.Remote <- config.Message{Username: username, Text: text, Channel: channel, Account: b.Account, UserID: strconv.Itoa(message.From.ID)}
		}
	}
}

func (b *Btelegram) getFileDirectURL(id string) string {
	res, err := b.c.GetFileDirectURL(id)
	if err != nil {
		return ""
	}
	return res
}
<commit_msg>Add support for editing messages (telegram)<commit_after>package btelegram

import (
	"strconv"

	"github.com/42wim/matterbridge/bridge/config"
	log "github.com/Sirupsen/logrus"
	"github.com/go-telegram-bot-api/telegram-bot-api"
)

type Btelegram struct {
	c       *tgbotapi.BotAPI
	Config  *config.Protocol
	Remote  chan config.Message
	Account string
}

var flog *log.Entry
var protocol = "telegram"

func init() {
	flog = log.WithFields(log.Fields{"module": protocol})
}

func New(cfg config.Protocol, account string, c chan config.Message) *Btelegram {
	b := &Btelegram{}
	b.Config = &cfg
	b.Remote = c
	b.Account = account
	return b
}

func (b *Btelegram) Connect() error {
	var err error
	flog.Info("Connecting")
	b.c, err = tgbotapi.NewBotAPI(b.Config.Token)
	if err != nil {
		flog.Debugf("%#v", err)
		return err
	}
	updates, err := b.c.GetUpdatesChan(tgbotapi.NewUpdate(0))
	if err != nil {
		flog.Debugf("%#v", err)
		return err
	}
	flog.Info("Connection succeeded")
	go b.handleRecv(updates)
	return nil
}

func (b *Btelegram) Disconnect() error {
	return nil

}

func (b *Btelegram) JoinChannel(channel config.ChannelInfo) error {
	return nil
}

func (b *Btelegram) Send(msg config.Message) (string, error) {
	flog.Debugf("Receiving %#v", msg)
	chatid, err := strconv.ParseInt(msg.Channel, 10, 64)
	if err != nil {
		return "", err
	}

	if b.Config.MessageFormat == "HTML" {
		msg.Text = makeHTML(msg.Text)
	}

	// edit the message if we have a msg ID
	if msg.ID != "" {
		msgid, err := strconv.Atoi(msg.ID)
		if err != nil {
			return "", err
		}
		m := tgbotapi.NewEditMessageText(chatid, msgid, msg.Username+msg.Text)
		_, err = b.c.Send(m)
		if err != nil {
			return "", err
		}
		return "", nil
	}

	m := tgbotapi.NewMessage(chatid, msg.Username+msg.Text)
	if b.Config.MessageFormat == "HTML" {
		m.ParseMode = tgbotapi.ModeHTML
	}
	res, err := b.c.Send(m)
	if err != nil {
		return "", err
	}
	return strconv.Itoa(res.MessageID), nil

}

func (b *Btelegram) handleRecv(updates <-chan tgbotapi.Update) {
	for update := range updates {
		flog.Debugf("Receiving from telegram: %#v", update.Message)
		var message *tgbotapi.Message
		username := ""
		channel := ""
		text := ""
		// handle channels
		if update.ChannelPost != nil {
			message = update.ChannelPost
		}
		if update.EditedChannelPost != nil && !b.Config.EditDisable {
			message = update.EditedChannelPost
			message.Text = message.Text + b.Config.EditSuffix
		}
		// handle groups
		if update.Message != nil {
			message = update.Message
		}
		if update.EditedMessage != nil && !b.Config.EditDisable {
			message = update.EditedMessage
			message.Text = message.Text + b.Config.EditSuffix
		}
		if message.From != nil {
			if b.Config.UseFirstName {
				username = message.From.FirstName
			}
			if username == "" {
				username = message.From.UserName
				if username == "" {
					username = message.From.FirstName
				}
			}
			text = message.Text
			channel = strconv.FormatInt(message.Chat.ID, 10)
		}

		if username == "" {
			username = "unknown"
		}
		if message.Sticker != nil && b.Config.UseInsecureURL {
			text = text + " " + b.getFileDirectURL(message.Sticker.FileID)
		}
		if message.Video != nil && b.Config.UseInsecureURL {
			text = text + " " + b.getFileDirectURL(message.Video.FileID)
		}
		if message.Photo != nil && b.Config.UseInsecureURL {
			photos := *message.Photo
			// last photo is the biggest
			text = text + " " + b.getFileDirectURL(photos[len(photos)-1].FileID)
		}
		if message.Document != nil && b.Config.UseInsecureURL {
			text = text + " " + message.Document.FileName + " : " + b.getFileDirectURL(message.Document.FileID)
		}
		if text != "" {
			flog.Debugf("Sending message from %s on %s to gateway", username, b.Account)
			b.Remote <- config.Message{Username: username, Text: text, Channel: channel, Account: b.Account, UserID: strconv.Itoa(message.From.ID), ID: strconv.Itoa(message.MessageID)}
		}
	}
}

func (b *Btelegram) getFileDirectURL(id string) string {
	res, err := b.c.GetFileDirectURL(id)
	if err != nil {
		return ""
	}
	return res
}
<|endoftext|>


<commit_before>package monitor

import (
	"expvar"
	"fmt"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"sort"
	"strconv"
	"sync"
	"time"

	"github.com/influxdb/influxdb/influxql"
)

// Client is the interface modules must implement if they wish to register with monitor.
type Client interface {
	Statistics() (map[string]interface{}, error)
	Diagnostics() (map[string]interface{}, error)
}

// Service represents an instance of the monitor service.
type Service struct {
	wg            sync.WaitGroup
	done          chan struct{}
	mu            sync.Mutex
	registrations []*clientWithMeta

	hostname  string
	clusterID uint64
	nodeID    uint64

	storeEnabled  bool
	storeDatabase string
	storeAddress  string
	storeInterval time.Duration

	expvarAddress string

	Logger *log.Logger
}

// NewService returns a new instance of the monitor service.
func NewService(c Config) *Service {
	return &Service{
		registrations: make([]*clientWithMeta, 0),
		storeEnabled:  c.StoreEnabled,
		storeDatabase: c.StoreDatabase,
		storeAddress:  c.StoreAddress,
		storeInterval: time.Duration(c.StoreInterval),
		expvarAddress: c.ExpvarAddress,
		Logger:        log.New(os.Stderr, "[monitor] ", log.LstdFlags),
	}
}

// Open opens the monitoring service, using the given clusterID, node ID, and hostname
// for identification purposes.
func (s *Service) Open(clusterID, nodeID uint64, hostname string) error {
	s.Logger.Printf("starting monitor service for cluster %d, host %s", clusterID, hostname)
	s.clusterID = clusterID
	s.nodeID = nodeID
	s.hostname = hostname

	// Self-register Go runtime stats.
	s.Register("runtime", nil, &goRuntime{})

	// If enabled, record stats in a InfluxDB system.
	if s.storeEnabled {
		s.Logger.Printf("storing in %s, database '%s', interval %s",
			s.storeAddress, s.storeDatabase, s.storeInterval)

		s.Logger.Printf("ensuring database %s exists on %s", s.storeDatabase, s.storeAddress)
		if err := ensureDatabaseExists(s.storeAddress, s.storeDatabase); err != nil {
			return err
		}

		// Start periodic writes to system.
		s.wg.Add(1)
		go s.storeStatistics()
	}

	// If enabled, expose all expvar data over HTTP.
	if s.expvarAddress != "" {
		listener, err := net.Listen("tcp", s.expvarAddress)
		if err != nil {
			return err
		}

		go func() {
			http.Serve(listener, nil)
		}()
		s.Logger.Printf("expvar information available on %s", s.expvarAddress)
	}
	return nil
}

// Close closes the monitor service.
func (s *Service) Close() {
	s.Logger.Println("shutting down monitor service")
	close(s.done)
	s.wg.Wait()
	s.done = nil
}

// SetLogger sets the internal logger to the logger passed in.
func (s *Service) SetLogger(l *log.Logger) {
	s.Logger = l
}

// Register registers a client with the given name and tags.
func (s *Service) Register(name string, tags map[string]string, client Client) error {
	s.mu.Lock()
	defer s.mu.Unlock()
	c := &clientWithMeta{
		Client: client,
		name:   name,
		tags:   tags,
	}
	s.registrations = append(s.registrations, c)
	s.Logger.Printf(`'%s:%v' registered for monitoring`, name, tags)
	return nil
}

// ExecuteStatement executes monitor-related query statements.
func (s *Service) ExecuteStatement(stmt influxql.Statement) *influxql.Result {
	switch stmt := stmt.(type) {
	case *influxql.ShowStatsStatement:
		return s.executeShowStatistics(stmt)
	default:
		panic(fmt.Sprintf("unsupported statement type: %T", stmt))
	}
}

// executeShowStatistics returns the statistics of the registered monitor client in
// the standard form expected by users of the InfluxDB system.
func (s *Service) executeShowStatistics(q *influxql.ShowStatsStatement) *influxql.Result {
	stats, _ := s.statistics()
	rows := make([]*influxql.Row, len(stats))

	for n, stat := range stats {
		row := &influxql.Row{}
		values := make([]interface{}, 0, len(stat.Tags)+len(stat.Values))

		row.Columns = append(row.Columns, "name")
		values = append(values, stat.Name)

		for _, k := range stat.tagNames() {
			row.Columns = append(row.Columns, k)
			values = append(values, stat.Tags[k])
		}
		for _, k := range stat.valueNames() {
			row.Columns = append(row.Columns, k)
			values = append(values, stat.Values[k])
		}
		row.Values = [][]interface{}{values}
		rows[n] = row
	}
	return &influxql.Result{Series: rows}
}

// statistics returns the combined statistics for all registered clients.
func (s *Service) statistics() ([]*statistic, error) {
	s.mu.Lock()
	defer s.mu.Unlock()

	statistics := make([]*statistic, 0, len(s.registrations))
	for _, r := range s.registrations {
		stats, err := r.Client.Statistics()
		if err != nil {
			continue
		}
		statistics[i] = newStatistic(r.name, r.tags, stats)
	}
	return statistics, nil
}

// storeStatistics writes the statistics to an InfluxDB system.
func (s *Service) storeStatistics() {
	// XXX add tags such as local hostname and cluster ID
	//a.Tags["clusterID"] = strconv.FormatUint(s.clusterID, 10)
	//a.Tags["nodeID"] = strconv.FormatUint(s.nodeID, 10)
	//a.Tags["hostname"] = s.hostname
	defer s.wg.Done()

	tick := time.NewTicker(s.storeInterval)
	defer tick.Stop()
	for {
		select {
		case <-tick.C:
			// Write stats here.
		case <-s.done:
			s.Logger.Printf("terminating storage of statistics to %s", s.storeAddress)
			return
		}

	}
}

// statistic represents the information returned by a single monitor client.
type statistic struct {
	Name   string
	Tags   map[string]string
	Values map[string]interface{}
}

// newStatistic returns a new statistic object. It ensures that tags are always non-nil.
func newStatistic(name string, tags map[string]string, values map[string]interface{}) *statistic {
	a := tags
	if a == nil {
		a = make(map[string]string)
	}

	return &statistic{
		Name:   name,
		Tags:   a,
		Values: values,
	}
}

// tagNames returns a sorted list of the tag names, if any.
func (s *statistic) tagNames() []string {
	a := make([]string, 0, len(s.Tags))
	for k, _ := range s.Tags {
		a = append(a, k)
	}
	sort.Strings(a)
	return a
}

// valueNames returns a sorted list of the value names, if any.
func (s *statistic) valueNames() []string {
	a := make([]string, 0, len(s.Values))
	for k, _ := range s.Values {
		a = append(a, k)
	}
	sort.Strings(a)
	return a
}

// clientWithMeta wraps a registered client with its associated name and tags.
type clientWithMeta struct {
	Client
	name string
	tags map[string]string
}

// MonitorClient wraps a *expvar.Map so that it implements the Client interface. It is for
// use by external packages that just record stats in an expvar.Map type.
type MonitorClient struct {
	ep *expvar.Map
}

// NewMonitorClient returns a new MonitorClient using the given expvar.Map.
func NewMonitorClient(ep *expvar.Map) *MonitorClient {
	return &MonitorClient{ep: ep}
}

// Statistics implements the Client interface for a MonitorClient.
func (m MonitorClient) Statistics() (map[string]interface{}, error) {
	values := make(map[string]interface{})
	m.ep.Do(func(kv expvar.KeyValue) {
		var f interface{}
		var err error
		switch v := kv.Value.(type) {
		case *expvar.Float:
			f, err = strconv.ParseFloat(v.String(), 64)
			if err != nil {
				return
			}
		case *expvar.Int:
			f, err = strconv.ParseUint(v.String(), 10, 64)
			if err != nil {
				return
			}
		default:
			return
		}
		values[kv.Key] = f
	})

	return values, nil
}

// Diagnostics implements the Client interface for a MonitorClient.
func (m MonitorClient) Diagnostics() (map[string]interface{}, error) {
	return nil, nil
}

func ensureDatabaseExists(host, database string) error {
	values := url.Values{}
	values.Set("q", fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", database))
	resp, err := http.Get(host + "/query?" + values.Encode())
	if err != nil {
		return fmt.Errorf("failed to create monitoring database on %s: %s", host, err.Error())
	}
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("failed to create monitoring database on %s, received code: %d",
			host, resp.StatusCode)
	}
	return nil
}
<commit_msg>Don't include empty statistical data in results<commit_after>package monitor

import (
	"expvar"
	"fmt"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"sort"
	"strconv"
	"sync"
	"time"

	"github.com/influxdb/influxdb/influxql"
)

// Client is the interface modules must implement if they wish to register with monitor.
type Client interface {
	Statistics() (map[string]interface{}, error)
	Diagnostics() (map[string]interface{}, error)
}

// Service represents an instance of the monitor service.
type Service struct {
	wg            sync.WaitGroup
	done          chan struct{}
	mu            sync.Mutex
	registrations []*clientWithMeta

	hostname  string
	clusterID uint64
	nodeID    uint64

	storeEnabled  bool
	storeDatabase string
	storeAddress  string
	storeInterval time.Duration

	expvarAddress string

	Logger *log.Logger
}

// NewService returns a new instance of the monitor service.
func NewService(c Config) *Service {
	return &Service{
		registrations: make([]*clientWithMeta, 0),
		storeEnabled:  c.StoreEnabled,
		storeDatabase: c.StoreDatabase,
		storeAddress:  c.StoreAddress,
		storeInterval: time.Duration(c.StoreInterval),
		expvarAddress: c.ExpvarAddress,
		Logger:        log.New(os.Stderr, "[monitor] ", log.LstdFlags),
	}
}

// Open opens the monitoring service, using the given clusterID, node ID, and hostname
// for identification purposes.
func (s *Service) Open(clusterID, nodeID uint64, hostname string) error {
	s.Logger.Printf("starting monitor service for cluster %d, host %s", clusterID, hostname)
	s.clusterID = clusterID
	s.nodeID = nodeID
	s.hostname = hostname

	// Self-register Go runtime stats.
	s.Register("runtime", nil, &goRuntime{})

	// If enabled, record stats in a InfluxDB system.
	if s.storeEnabled {
		s.Logger.Printf("storing in %s, database '%s', interval %s",
			s.storeAddress, s.storeDatabase, s.storeInterval)

		s.Logger.Printf("ensuring database %s exists on %s", s.storeDatabase, s.storeAddress)
		if err := ensureDatabaseExists(s.storeAddress, s.storeDatabase); err != nil {
			return err
		}

		// Start periodic writes to system.
		s.wg.Add(1)
		go s.storeStatistics()
	}

	// If enabled, expose all expvar data over HTTP.
	if s.expvarAddress != "" {
		listener, err := net.Listen("tcp", s.expvarAddress)
		if err != nil {
			return err
		}

		go func() {
			http.Serve(listener, nil)
		}()
		s.Logger.Printf("expvar information available on %s", s.expvarAddress)
	}
	return nil
}

// Close closes the monitor service.
func (s *Service) Close() {
	s.Logger.Println("shutting down monitor service")
	close(s.done)
	s.wg.Wait()
	s.done = nil
}

// SetLogger sets the internal logger to the logger passed in.
func (s *Service) SetLogger(l *log.Logger) {
	s.Logger = l
}

// Register registers a client with the given name and tags.
func (s *Service) Register(name string, tags map[string]string, client Client) error {
	s.mu.Lock()
	defer s.mu.Unlock()
	c := &clientWithMeta{
		Client: client,
		name:   name,
		tags:   tags,
	}
	s.registrations = append(s.registrations, c)
	s.Logger.Printf(`'%s:%v' registered for monitoring`, name, tags)
	return nil
}

// ExecuteStatement executes monitor-related query statements.
func (s *Service) ExecuteStatement(stmt influxql.Statement) *influxql.Result {
	switch stmt := stmt.(type) {
	case *influxql.ShowStatsStatement:
		return s.executeShowStatistics(stmt)
	default:
		panic(fmt.Sprintf("unsupported statement type: %T", stmt))
	}
}

// executeShowStatistics returns the statistics of the registered monitor client in
// the standard form expected by users of the InfluxDB system.
func (s *Service) executeShowStatistics(q *influxql.ShowStatsStatement) *influxql.Result {
	stats, _ := s.statistics()
	rows := make([]*influxql.Row, len(stats))

	for n, stat := range stats {
		row := &influxql.Row{}
		values := make([]interface{}, 0, len(stat.Tags)+len(stat.Values))

		row.Columns = append(row.Columns, "name")
		values = append(values, stat.Name)

		for _, k := range stat.tagNames() {
			row.Columns = append(row.Columns, k)
			values = append(values, stat.Tags[k])
		}
		for _, k := range stat.valueNames() {
			row.Columns = append(row.Columns, k)
			values = append(values, stat.Values[k])
		}
		row.Values = [][]interface{}{values}
		rows[n] = row
	}
	return &influxql.Result{Series: rows}
}

// statistics returns the combined statistics for all registered clients.
func (s *Service) statistics() ([]*statistic, error) {
	s.mu.Lock()
	defer s.mu.Unlock()

	statistics := make([]*statistic, 0, len(s.registrations))
	for _, r := range s.registrations {
		stats, err := r.Client.Statistics()
		if err != nil {
			continue
		}

		// If a registered client has no field data, don't include it in the results
		if len(stats) == 0 {
			continue
		}

		statistics = append(statistics, newStatistic(r.name, r.tags, stats))
	}
	return statistics, nil
}

// storeStatistics writes the statistics to an InfluxDB system.
func (s *Service) storeStatistics() {
	// XXX add tags such as local hostname and cluster ID
	//a.Tags["clusterID"] = strconv.FormatUint(s.clusterID, 10)
	//a.Tags["nodeID"] = strconv.FormatUint(s.nodeID, 10)
	//a.Tags["hostname"] = s.hostname
	defer s.wg.Done()

	tick := time.NewTicker(s.storeInterval)
	defer tick.Stop()
	for {
		select {
		case <-tick.C:
			// Write stats here.
		case <-s.done:
			s.Logger.Printf("terminating storage of statistics to %s", s.storeAddress)
			return
		}

	}
}

// statistic represents the information returned by a single monitor client.
type statistic struct {
	Name   string
	Tags   map[string]string
	Values map[string]interface{}
}

// newStatistic returns a new statistic object. It ensures that tags are always non-nil.
func newStatistic(name string, tags map[string]string, values map[string]interface{}) *statistic {
	a := tags
	if a == nil {
		a = make(map[string]string)
	}

	return &statistic{
		Name:   name,
		Tags:   a,
		Values: values,
	}
}

// tagNames returns a sorted list of the tag names, if any.
func (s *statistic) tagNames() []string {
	a := make([]string, 0, len(s.Tags))
	for k, _ := range s.Tags {
		a = append(a, k)
	}
	sort.Strings(a)
	return a
}

// valueNames returns a sorted list of the value names, if any.
func (s *statistic) valueNames() []string {
	a := make([]string, 0, len(s.Values))
	for k, _ := range s.Values {
		a = append(a, k)
	}
	sort.Strings(a)
	return a
}

// clientWithMeta wraps a registered client with its associated name and tags.
type clientWithMeta struct {
	Client
	name string
	tags map[string]string
}

// MonitorClient wraps a *expvar.Map so that it implements the Client interface. It is for
// use by external packages that just record stats in an expvar.Map type.
type MonitorClient struct {
	ep *expvar.Map
}

// NewMonitorClient returns a new MonitorClient using the given expvar.Map.
func NewMonitorClient(ep *expvar.Map) *MonitorClient {
	return &MonitorClient{ep: ep}
}

// Statistics implements the Client interface for a MonitorClient.
func (m MonitorClient) Statistics() (map[string]interface{}, error) {
	values := make(map[string]interface{})
	m.ep.Do(func(kv expvar.KeyValue) {
		var f interface{}
		var err error
		switch v := kv.Value.(type) {
		case *expvar.Float:
			f, err = strconv.ParseFloat(v.String(), 64)
			if err != nil {
				return
			}
		case *expvar.Int:
			f, err = strconv.ParseUint(v.String(), 10, 64)
			if err != nil {
				return
			}
		default:
			return
		}
		values[kv.Key] = f
	})

	return values, nil
}

// Diagnostics implements the Client interface for a MonitorClient.
func (m MonitorClient) Diagnostics() (map[string]interface{}, error) {
	return nil, nil
}

func ensureDatabaseExists(host, database string) error {
	values := url.Values{}
	values.Set("q", fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", database))
	resp, err := http.Get(host + "/query?" + values.Encode())
	if err != nil {
		return fmt.Errorf("failed to create monitoring database on %s: %s", host, err.Error())
	}
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("failed to create monitoring database on %s, received code: %d",
			host, resp.StatusCode)
	}
	return nil
}
<|endoftext|>


<commit_before>package monitor

import (
	"expvar"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"os"
	"sort"
	"strconv"
	"sync"
	"time"

	"github.com/influxdb/influxdb/influxql"
)

// Client is the interface modules must implement if they wish to register with monitor.
type Client interface {
	Statistics() (map[string]interface{}, error)
	Diagnostics() (map[string]interface{}, error)
}

// Service represents an instance of the monitor service.
type Service struct {
	wg            sync.WaitGroup
	done          chan struct{}
	mu            sync.Mutex
	registrations []*clientWithMeta

	hostname  string
	clusterID uint64
	nodeID    uint64

	storeEnabled  bool
	storeDatabase string
	storeAddress  string
	storeInterval time.Duration

	Logger *log.Logger
}

// NewService returns a new instance of the monitor service.
func NewService(c Config) *Service {
	return &Service{
		registrations: make([]*clientWithMeta, 0),
		storeEnabled:  c.StoreEnabled,
		storeDatabase: c.StoreDatabase,
		storeAddress:  c.StoreAddress,
		storeInterval: time.Duration(c.StoreInterval),
		Logger:        log.New(os.Stderr, "[monitor] ", log.LstdFlags),
	}
}

// Open opens the monitoring service, using the given clusterID, node ID, and hostname
// for identification purposes.
func (s *Service) Open(clusterID, nodeID uint64, hostname string) error {
	s.Logger.Printf("starting monitor service for cluster %d, host %s", clusterID, hostname)
	s.clusterID = clusterID
	s.nodeID = nodeID
	s.hostname = hostname

	// Self-register Go runtime stats.
	s.Register("runtime", nil, &goRuntime{})

	// If enabled, record stats in a InfluxDB system.
	if s.storeEnabled {
		s.Logger.Printf("storing in %s, database '%s', interval %s",
			s.storeAddress, s.storeDatabase, s.storeInterval)

		s.Logger.Printf("ensuring database %s exists on %s", s.storeDatabase, s.storeAddress)
		if err := ensureDatabaseExists(s.storeAddress, s.storeDatabase); err != nil {
			return err
		}

		// Start periodic writes to system.
		s.wg.Add(1)
		go s.storeStatistics()
	}

	return nil
}

// Close closes the monitor service.
func (s *Service) Close() {
	s.Logger.Println("shutting down monitor service")
	close(s.done)
	s.wg.Wait()
	s.done = nil
}

// SetLogger sets the internal logger to the logger passed in.
func (s *Service) SetLogger(l *log.Logger) {
	s.Logger = l
}

// Register registers a client with the given name and tags.
func (s *Service) Register(name string, tags map[string]string, client Client) error {
	s.mu.Lock()
	defer s.mu.Unlock()
	c := &clientWithMeta{
		Client: client,
		name:   name,
		tags:   tags,
	}
	s.registrations = append(s.registrations, c)
	s.Logger.Printf(`'%s:%v' registered for monitoring`, name, tags)
	return nil
}

// ExecuteStatement executes monitor-related query statements.
func (s *Service) ExecuteStatement(stmt influxql.Statement) *influxql.Result {
	switch stmt := stmt.(type) {
	case *influxql.ShowStatsStatement:
		return s.executeShowStatistics(stmt)
	default:
		panic(fmt.Sprintf("unsupported statement type: %T", stmt))
	}
}

// executeShowStatistics returns the statistics of the registered monitor client in
// the standard form expected by users of the InfluxDB system.
func (s *Service) executeShowStatistics(q *influxql.ShowStatsStatement) *influxql.Result {
	stats, _ := s.statistics()
	rows := make([]*influxql.Row, len(stats))

	for n, stat := range stats {
		row := &influxql.Row{}
		values := make([]interface{}, 0, len(stat.Tags)+len(stat.Values))

		row.Columns = append(row.Columns, "name")
		values = append(values, stat.Name)

		for _, k := range stat.tagNames() {
			row.Columns = append(row.Columns, k)
			values = append(values, stat.Tags[k])
		}
		for _, k := range stat.valueNames() {
			row.Columns = append(row.Columns, k)
			values = append(values, stat.Values[k])
		}
		row.Values = [][]interface{}{values}
		rows[n] = row
	}
	return &influxql.Result{Series: rows}
}

// statistics returns the combined statistics for all registered clients.
func (s *Service) statistics() ([]*statistic, error) {
	s.mu.Lock()
	defer s.mu.Unlock()

	statistics := make([]*statistic, 0, len(s.registrations))
	for _, r := range s.registrations {
		stats, err := r.Client.Statistics()
		if err != nil {
			continue
		}

		// If a registered client has no field data, don't include it in the results
		if len(stats) == 0 {
			continue
		}

		statistics = append(statistics, newStatistic(r.name, r.tags, stats))
	}
	return statistics, nil
}

// storeStatistics writes the statistics to an InfluxDB system.
func (s *Service) storeStatistics() {
	// XXX add tags such as local hostname and cluster ID
	//a.Tags["clusterID"] = strconv.FormatUint(s.clusterID, 10)
	//a.Tags["nodeID"] = strconv.FormatUint(s.nodeID, 10)
	//a.Tags["hostname"] = s.hostname
	defer s.wg.Done()

	tick := time.NewTicker(s.storeInterval)
	defer tick.Stop()
	for {
		select {
		case <-tick.C:
			// Write stats here.
		case <-s.done:
			s.Logger.Printf("terminating storage of statistics to %s", s.storeAddress)
			return
		}

	}
}

// statistic represents the information returned by a single monitor client.
type statistic struct {
	Name   string
	Tags   map[string]string
	Values map[string]interface{}
}

// newStatistic returns a new statistic object. It ensures that tags are always non-nil.
func newStatistic(name string, tags map[string]string, values map[string]interface{}) *statistic {
	a := tags
	if a == nil {
		a = make(map[string]string)
	}

	return &statistic{
		Name:   name,
		Tags:   a,
		Values: values,
	}
}

// tagNames returns a sorted list of the tag names, if any.
func (s *statistic) tagNames() []string {
	a := make([]string, 0, len(s.Tags))
	for k, _ := range s.Tags {
		a = append(a, k)
	}
	sort.Strings(a)
	return a
}

// valueNames returns a sorted list of the value names, if any.
func (s *statistic) valueNames() []string {
	a := make([]string, 0, len(s.Values))
	for k, _ := range s.Values {
		a = append(a, k)
	}
	sort.Strings(a)
	return a
}

// clientWithMeta wraps a registered client with its associated name and tags.
type clientWithMeta struct {
	Client
	name string
	tags map[string]string
}

// MonitorClient wraps a *expvar.Map so that it implements the Client interface. It is for
// use by external packages that just record stats in an expvar.Map type.
type MonitorClient struct {
	ep *expvar.Map
}

// NewMonitorClient returns a new MonitorClient using the given expvar.Map.
func NewMonitorClient(ep *expvar.Map) *MonitorClient {
	return &MonitorClient{ep: ep}
}

// Statistics implements the Client interface for a MonitorClient.
func (m MonitorClient) Statistics() (map[string]interface{}, error) {
	values := make(map[string]interface{})
	m.ep.Do(func(kv expvar.KeyValue) {
		var f interface{}
		var err error
		switch v := kv.Value.(type) {
		case *expvar.Float:
			f, err = strconv.ParseFloat(v.String(), 64)
			if err != nil {
				return
			}
		case *expvar.Int:
			f, err = strconv.ParseUint(v.String(), 10, 64)
			if err != nil {
				return
			}
		default:
			return
		}
		values[kv.Key] = f
	})

	return values, nil
}

// Diagnostics implements the Client interface for a MonitorClient.
func (m MonitorClient) Diagnostics() (map[string]interface{}, error) {
	return nil, nil
}

func ensureDatabaseExists(host, database string) error {
	values := url.Values{}
	values.Set("q", fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", database))
	resp, err := http.Get(host + "/query?" + values.Encode())
	if err != nil {
		return fmt.Errorf("failed to create monitoring database on %s: %s", host, err.Error())
	}
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("failed to create monitoring database on %s, received code: %d",
			host, resp.StatusCode)
	}
	return nil
}
<commit_msg>Use Measurement name, not a column "name"<commit_after>package monitor

import (
	"expvar"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"os"
	"sort"
	"strconv"
	"sync"
	"time"

	"github.com/influxdb/influxdb/influxql"
)

// Client is the interface modules must implement if they wish to register with monitor.
type Client interface {
	Statistics() (map[string]interface{}, error)
	Diagnostics() (map[string]interface{}, error)
}

// Service represents an instance of the monitor service.
type Service struct {
	wg            sync.WaitGroup
	done          chan struct{}
	mu            sync.Mutex
	registrations []*clientWithMeta

	hostname  string
	clusterID uint64
	nodeID    uint64

	storeEnabled  bool
	storeDatabase string
	storeAddress  string
	storeInterval time.Duration

	Logger *log.Logger
}

// NewService returns a new instance of the monitor service.
func NewService(c Config) *Service {
	return &Service{
		registrations: make([]*clientWithMeta, 0),
		storeEnabled:  c.StoreEnabled,
		storeDatabase: c.StoreDatabase,
		storeAddress:  c.StoreAddress,
		storeInterval: time.Duration(c.StoreInterval),
		Logger:        log.New(os.Stderr, "[monitor] ", log.LstdFlags),
	}
}

// Open opens the monitoring service, using the given clusterID, node ID, and hostname
// for identification purposes.
func (s *Service) Open(clusterID, nodeID uint64, hostname string) error {
	s.Logger.Printf("starting monitor service for cluster %d, host %s", clusterID, hostname)
	s.clusterID = clusterID
	s.nodeID = nodeID
	s.hostname = hostname

	// Self-register Go runtime stats.
	s.Register("runtime", nil, &goRuntime{})

	// If enabled, record stats in a InfluxDB system.
	if s.storeEnabled {
		s.Logger.Printf("storing in %s, database '%s', interval %s",
			s.storeAddress, s.storeDatabase, s.storeInterval)

		s.Logger.Printf("ensuring database %s exists on %s", s.storeDatabase, s.storeAddress)
		if err := ensureDatabaseExists(s.storeAddress, s.storeDatabase); err != nil {
			return err
		}

		// Start periodic writes to system.
		s.wg.Add(1)
		go s.storeStatistics()
	}

	return nil
}

// Close closes the monitor service.
func (s *Service) Close() {
	s.Logger.Println("shutting down monitor service")
	close(s.done)
	s.wg.Wait()
	s.done = nil
}

// SetLogger sets the internal logger to the logger passed in.
func (s *Service) SetLogger(l *log.Logger) {
	s.Logger = l
}

// Register registers a client with the given name and tags.
func (s *Service) Register(name string, tags map[string]string, client Client) error {
	s.mu.Lock()
	defer s.mu.Unlock()
	c := &clientWithMeta{
		Client: client,
		name:   name,
		tags:   tags,
	}
	s.registrations = append(s.registrations, c)
	s.Logger.Printf(`'%s:%v' registered for monitoring`, name, tags)
	return nil
}

// ExecuteStatement executes monitor-related query statements.
func (s *Service) ExecuteStatement(stmt influxql.Statement) *influxql.Result {
	switch stmt := stmt.(type) {
	case *influxql.ShowStatsStatement:
		return s.executeShowStatistics(stmt)
	default:
		panic(fmt.Sprintf("unsupported statement type: %T", stmt))
	}
}

// executeShowStatistics returns the statistics of the registered monitor client in
// the standard form expected by users of the InfluxDB system.
func (s *Service) executeShowStatistics(q *influxql.ShowStatsStatement) *influxql.Result {
	stats, _ := s.statistics()
	rows := make([]*influxql.Row, len(stats))

	for n, stat := range stats {
		row := &influxql.Row{Name: stat.Name}
		values := make([]interface{}, 0, len(stat.Tags)+len(stat.Values))

		for _, k := range stat.tagNames() {
			row.Columns = append(row.Columns, k)
			values = append(values, stat.Tags[k])
		}
		for _, k := range stat.valueNames() {
			row.Columns = append(row.Columns, k)
			values = append(values, stat.Values[k])
		}
		row.Values = [][]interface{}{values}
		rows[n] = row
	}
	return &influxql.Result{Series: rows}
}

// statistics returns the combined statistics for all registered clients.
func (s *Service) statistics() ([]*statistic, error) {
	s.mu.Lock()
	defer s.mu.Unlock()

	statistics := make([]*statistic, 0, len(s.registrations))
	for _, r := range s.registrations {
		stats, err := r.Client.Statistics()
		if err != nil {
			continue
		}

		// If a registered client has no field data, don't include it in the results
		if len(stats) == 0 {
			continue
		}

		statistics = append(statistics, newStatistic(r.name, r.tags, stats))
	}
	return statistics, nil
}

// storeStatistics writes the statistics to an InfluxDB system.
func (s *Service) storeStatistics() {
	// XXX add tags such as local hostname and cluster ID
	//a.Tags["clusterID"] = strconv.FormatUint(s.clusterID, 10)
	//a.Tags["nodeID"] = strconv.FormatUint(s.nodeID, 10)
	//a.Tags["hostname"] = s.hostname
	defer s.wg.Done()

	tick := time.NewTicker(s.storeInterval)
	defer tick.Stop()
	for {
		select {
		case <-tick.C:
			// Write stats here.
		case <-s.done:
			s.Logger.Printf("terminating storage of statistics to %s", s.storeAddress)
			return
		}

	}
}

// statistic represents the information returned by a single monitor client.
type statistic struct {
	Name   string
	Tags   map[string]string
	Values map[string]interface{}
}

// newStatistic returns a new statistic object. It ensures that tags are always non-nil.
func newStatistic(name string, tags map[string]string, values map[string]interface{}) *statistic {
	a := tags
	if a == nil {
		a = make(map[string]string)
	}

	return &statistic{
		Name:   name,
		Tags:   a,
		Values: values,
	}
}

// tagNames returns a sorted list of the tag names, if any.
func (s *statistic) tagNames() []string {
	a := make([]string, 0, len(s.Tags))
	for k, _ := range s.Tags {
		a = append(a, k)
	}
	sort.Strings(a)
	return a
}

// valueNames returns a sorted list of the value names, if any.
func (s *statistic) valueNames() []string {
	a := make([]string, 0, len(s.Values))
	for k, _ := range s.Values {
		a = append(a, k)
	}
	sort.Strings(a)
	return a
}

// clientWithMeta wraps a registered client with its associated name and tags.
type clientWithMeta struct {
	Client
	name string
	tags map[string]string
}

// MonitorClient wraps a *expvar.Map so that it implements the Client interface. It is for
// use by external packages that just record stats in an expvar.Map type.
type MonitorClient struct {
	ep *expvar.Map
}

// NewMonitorClient returns a new MonitorClient using the given expvar.Map.
func NewMonitorClient(ep *expvar.Map) *MonitorClient {
	return &MonitorClient{ep: ep}
}

// Statistics implements the Client interface for a MonitorClient.
func (m MonitorClient) Statistics() (map[string]interface{}, error) {
	values := make(map[string]interface{})
	m.ep.Do(func(kv expvar.KeyValue) {
		var f interface{}
		var err error
		switch v := kv.Value.(type) {
		case *expvar.Float:
			f, err = strconv.ParseFloat(v.String(), 64)
			if err != nil {
				return
			}
		case *expvar.Int:
			f, err = strconv.ParseUint(v.String(), 10, 64)
			if err != nil {
				return
			}
		default:
			return
		}
		values[kv.Key] = f
	})

	return values, nil
}

// Diagnostics implements the Client interface for a MonitorClient.
func (m MonitorClient) Diagnostics() (map[string]interface{}, error) {
	return nil, nil
}

func ensureDatabaseExists(host, database string) error {
	values := url.Values{}
	values.Set("q", fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", database))
	resp, err := http.Get(host + "/query?" + values.Encode())
	if err != nil {
		return fmt.Errorf("failed to create monitoring database on %s: %s", host, err.Error())
	}
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("failed to create monitoring database on %s, received code: %d",
			host, resp.StatusCode)
	}
	return nil
}
<|endoftext|>


<commit_before>package mumble

import (
	"log"
	"sync/atomic"

	"github.com/TF2Stadium/fumble/database"
	"github.com/layeh/gumble/gumble"
)

var channels = new(uint64)

func (l Conn) OnConnect(e *gumble.ConnectEvent) {
	log.Println("Connected to Mumble!")
	l.wait.Done()
}

func (l Conn) OnDisconnect(e *gumble.DisconnectEvent) {
	log.Fatal("Disconnected from Mumble: ", e.String)
}

func (l Conn) OnUserChange(e *gumble.UserChangeEvent) {
	l.client.Do(func() {
		if e.Type.Has(gumble.UserChangeChannel) {
			if allowed, reason := isUserAllowed(e.User, e.User.Channel); !allowed {
				e.User.Send(reason)
				e.User.Move(e.Client.Channels[0])
			}

			if atomic.LoadUint64(channels) == 30 {
				go l.removeEmptyChannels()
			}
		}
		if e.Type.Has(gumble.UserChangeConnected) {
			steamid := database.GetSteamID(e.User.UserID)
			e.User.SetComment("http://steamcommunity.com/profiles/" + steamid)
			e.User.Send("Welcome to TF2Stadium!")
		}
	})
}

func (l Conn) OnChannelChange(e *gumble.ChannelChangeEvent) {
	if e.Type.Has(gumble.ChannelChangeCreated) {
		l.wait.Done()
		atomic.AddUint64(channels, 1)
	} else if e.Type.Has(gumble.ChannelChangeRemoved) {
		l.wait.Done()
		atomic.AddUint64(channels, ^uint64(0))
	}
}

func (l Conn) OnPermissionDenied(e *gumble.PermissionDeniedEvent)       {}
func (l Conn) OnTextMessage(e *gumble.TextMessageEvent)                 {}
func (l Conn) OnUserList(e *gumble.UserListEvent)                       {}
func (l Conn) OnACL(e *gumble.ACLEvent)                                 {}
func (l Conn) OnBanList(e *gumble.BanListEvent)                         {}
func (l Conn) OnContextActionChange(e *gumble.ContextActionChangeEvent) {}
func (l Conn) OnServerConfig(e *gumble.ServerConfigEvent)               {}
<commit_msg>OnUserChange: Deafen and Mute root users<commit_after>package mumble

import (
	"log"
	"sync/atomic"

	"github.com/TF2Stadium/fumble/database"
	"github.com/layeh/gumble/gumble"
)

var channels = new(uint64)

func (l Conn) OnConnect(e *gumble.ConnectEvent) {
	log.Println("Connected to Mumble!")
	l.wait.Done()
}

func (l Conn) OnDisconnect(e *gumble.DisconnectEvent) {
	log.Fatal("Disconnected from Mumble: ", e.String)
}

func (l Conn) OnUserChange(e *gumble.UserChangeEvent) {
	l.client.Do(func() {
		if e.Type.Has(gumble.UserChangeChannel) {
			if e.User.Channel.ID == 0 { // is root
				e.User.SetDeafened(true)
				e.User.SetMuted(true)
			}

			if allowed, reason := isUserAllowed(e.User, e.User.Channel); !allowed {
				e.User.Send(reason)
				e.User.Move(e.Client.Channels[0])
			} else {
				e.User.SetDeafened(false)
				e.User.SetMuted(false)
			}

			if atomic.LoadUint64(channels) == 30 {
				go l.removeEmptyChannels()
			}
		}
		if e.Type.Has(gumble.UserChangeConnected) {
			e.User.Send("Welcome to TF2Stadium!")
			e.User.SetDeafened(true)
			e.User.SetMuted(true)
		}
	})
}

func (l Conn) OnChannelChange(e *gumble.ChannelChangeEvent) {
	if e.Type.Has(gumble.ChannelChangeCreated) {
		l.wait.Done()
		atomic.AddUint64(channels, 1)
	} else if e.Type.Has(gumble.ChannelChangeRemoved) {
		l.wait.Done()
		atomic.AddUint64(channels, ^uint64(0))
	}
}

// func (l Conn) OnPermissionDenied(e *gumble.PermissionDeniedEvent)       {}
// func (l Conn) OnTextMessage(e *gumble.TextMessageEvent)                 {}
// func (l Conn) OnUserList(e *gumble.UserListEvent)                       {}
// func (l Conn) OnACL(e *gumble.ACLEvent)                                 {}
// func (l Conn) OnBanList(e *gumble.BanListEvent)                         {}
// func (l Conn) OnContextActionChange(e *gumble.ContextActionChangeEvent) {}
// func (l Conn) OnServerConfig(e *gumble.ServerConfigEvent)               {}
<|endoftext|>


<commit_before>package main

import (
	"cloud.google.com/go/translate"
	vision "cloud.google.com/go/vision/apiv1"
	"github.com/jjacquay712/GoRODS/msi"
	"golang.org/x/net/context"
	"golang.org/x/text/language"
	"google.golang.org/api/option"
	"log"
	"strings"
	"unsafe"
)

// #cgo CFLAGS: -I/usr/include/irods
// #cgo LDFLAGS: -lirods_server -lirods_common -lpthread
/*
#include "msParam.h"
#include "re_structs.h"
*/
import "C"

//export ExtractImageMetadata
func ExtractImageMetadata(imagePath *C.msParam_t, APIAuthPath *C.msParam_t, rei *C.ruleExecInfo_t) int {
	msi.Configure(unsafe.Pointer(rei))

	// Convert input to Golang strings
	imageFilePath := msi.ToParam(unsafe.Pointer(imagePath)).String()
	apiAuthFile := msi.ToParam(unsafe.Pointer(APIAuthPath)).String()

	// Extract image labels via machine learning API
	enLabels := GetImageLabels(imageFilePath, apiAuthFile)

	// Translate the labels into dutch
	labels := make(ImgLabels, len(enLabels))
	for i, label := range enLabels {
		labels[i].english = label
	}

	labels.FetchTranslations(apiAuthFile)

	metaKVPs := msi.NewParam(msi.KeyValPair_MS_T)

	metaMap := make(map[string]string)
	metaMap["tags_english"] = ""
	metaMap["tags_dutch"] = ""

	for _, label := range labels {
		metaMap["tags_english"] += label.english + ","
		metaMap["tags_dutch"] += label.dutch + ","
	}

	metaMap["tags_english"] = strings.TrimRight(metaMap["tags_english"], ",")
	metaMap["tags_dutch"] = strings.TrimRight(metaMap["tags_dutch"], ",")

	metaKVPs.SetKVP(metaMap)

	if err := msi.Call("msiAssociateKeyValuePairsToObj", metaKVPs, imageFilePath, "-d"); err != nil {
		log.Print(err)
		return -1
	}

	return 0
}

type ImgLabel struct {
	english string
	dutch   string
}

type ImgLabels []ImgLabel

func (labels ImgLabels) FetchTranslations(apiAuthFile string) {
	words := make([]string, len(labels))
	for i, label := range labels {
		words[i] = label.english
	}

	translations := TranslateString("nl", words, apiAuthFile)
	for i, translation := range translations {
		labels[i].dutch = translation
	}
}

func GetImageLabels(filepath, apiAuthFile string) (imageLabels []string) {

	authOption := option.WithServiceAccountFile(apiAuthFile)

	ctx := context.Background()

	// Creates a client.
	client, err := vision.NewImageAnnotatorClient(ctx, authOption)
	if err != nil {
		log.Fatalf("Failed to create client: %v", err)
	}

	file, err := msi.NewObjReader(filepath)
	if err != nil {
		log.Fatalf("Failed to read file: %v", err)
	}
	defer file.Close()

	image, err := vision.NewImageFromReader(file)
	if err != nil {
		log.Fatalf("Failed to create image: %v", err)
	}

	labels, err := client.DetectLabels(ctx, image, nil, 10)
	if err != nil {
		log.Fatalf("Failed to detect labels: %v", err)
	}

	for _, label := range labels {
		imageLabels = append(imageLabels, label.Description)
	}

	return
}

var translateClient *translate.Client

func TranslateString(targetLang string, words []string, apiAuthFile string) []string {

	authOption := option.WithServiceAccountFile(apiAuthFile)

	ctx := context.Background()

	// Lazy load single client
	if translateClient == nil {

		// Creates a client.
		client, err := translate.NewClient(ctx, authOption)
		if err != nil {
			log.Fatalf("Failed to create client: %v", err)
		}
		translateClient = client
	}

	// Sets the target language.
	target, err := language.Parse(targetLang)
	if err != nil {
		log.Fatalf("Failed to parse target language: %v", err)
	}

	translations, err := translateClient.Translate(ctx, words, target, nil)
	if err != nil {
		log.Fatalf("Failed to translate text: %v", err)
	}

	var responseTranslations []string

	for _, translation := range translations {
		responseTranslations = append(responseTranslations, translation.Text)
	}

	return responseTranslations
}

func main() {
	// We need the main function to make possible
	// CGO compiler to compile the package as C static archive
}
<commit_msg>Refactoring, comments<commit_after>package main

import (
	"cloud.google.com/go/translate"
	vision "cloud.google.com/go/vision/apiv1"
	"github.com/jjacquay712/GoRODS/msi"
	"golang.org/x/net/context"
	"golang.org/x/text/language"
	"google.golang.org/api/option"
	"log"
	"strings"
	"unsafe"
)

// #cgo CFLAGS: -I/usr/include/irods
// #cgo LDFLAGS: -lirods_server -lirods_common -lpthread
/*
#include "msParam.h"
#include "re_structs.h"
*/
import "C"

//export ExtractImageMetadata
func ExtractImageMetadata(imagePath *C.msParam_t, APIAuthPath *C.msParam_t, rei *C.ruleExecInfo_t) int {

	// Setup GoRODS/msi
	msi.Configure(unsafe.Pointer(rei))

	// Convert input to Golang strings
	imageFilePath := msi.ToParam(unsafe.Pointer(imagePath)).String()
	apiAuthFile := msi.ToParam(unsafe.Pointer(APIAuthPath)).String()

	// Extract image metadata via machine learning API
	kvp := GetImageLabels(imageFilePath, apiAuthFile).ToKVP()

	// Associate metadata to data object
	if err := msi.Call("msiAssociateKeyValuePairsToObj", kvp, imageFilePath, "-d"); err != nil {
		log.Print(err)
		return -1
	}

	return 0
}

type ImgLabel struct {
	english string
	dutch   string
}

type ImgLabels []ImgLabel

func (labels ImgLabels) FetchTranslations(apiAuthFile string) {
	words := make([]string, len(labels))
	for i, label := range labels {
		words[i] = label.english
	}

	translations := TranslateString("nl", words, apiAuthFile)
	for i, translation := range translations {
		labels[i].dutch = translation
	}
}

func (labels ImgLabels) ToKVP() *msi.Param {
	metaKVPs := msi.NewParam(msi.KeyValPair_MS_T)

	metaMap := make(map[string]string)
	metaMap["tags_english"] = ""
	metaMap["tags_dutch"] = ""

	for _, label := range labels {
		metaMap["tags_english"] += label.english + ","
		metaMap["tags_dutch"] += label.dutch + ","
	}

	metaMap["tags_english"] = strings.TrimRight(metaMap["tags_english"], ",")
	metaMap["tags_dutch"] = strings.TrimRight(metaMap["tags_dutch"], ",")

	return metaKVPs.SetKVP(metaMap)
}

func (labels ImgLabels) SetEnglish(enLabels []string) {
	for i, label := range enLabels {
		labels[i].english = label
	}
}

var translateClient *translate.Client
var visionClient *vision.ImageAnnotatorClient

func GetImageLabels(filepath, apiAuthFile string) (imageLabels ImgLabels) {

	authOption := option.WithServiceAccountFile(apiAuthFile)

	ctx := context.Background()

	// Lazy load single client
	if visionClient == nil {

		// Creates a client.
		client, err := vision.NewImageAnnotatorClient(ctx, authOption)
		if err != nil {
			log.Fatalf("Failed to create client: %v", err)
		}
		visionClient = client
	}

	file, err := msi.NewObjReader(filepath)
	if err != nil {
		log.Fatalf("Failed to read file: %v", err)
	}
	defer file.Close()

	image, err := vision.NewImageFromReader(file)
	if err != nil {
		log.Fatalf("Failed to create image: %v", err)
	}

	labels, err := visionClient.DetectLabels(ctx, image, nil, 10)
	if err != nil {
		log.Fatalf("Failed to detect labels: %v", err)
	}

	enLabels := make([]string, len(labels))
	for i, label := range labels {
		enLabels[i] = label.Description
	}

	imageLabels = make(ImgLabels, len(labels))

	imageLabels.SetEnglish(enLabels)

	// Translate the labels into dutch
	imageLabels.FetchTranslations(apiAuthFile)

	return
}

func TranslateString(targetLang string, words []string, apiAuthFile string) []string {

	authOption := option.WithServiceAccountFile(apiAuthFile)

	ctx := context.Background()

	// Lazy load single client
	if translateClient == nil {

		// Creates a client.
		client, err := translate.NewClient(ctx, authOption)
		if err != nil {
			log.Fatalf("Failed to create client: %v", err)
		}
		translateClient = client
	}

	// Sets the target language.
	target, err := language.Parse(targetLang)
	if err != nil {
		log.Fatalf("Failed to parse target language: %v", err)
	}

	translations, err := translateClient.Translate(ctx, words, target, nil)
	if err != nil {
		log.Fatalf("Failed to translate text: %v", err)
	}

	var responseTranslations []string

	for _, translation := range translations {
		responseTranslations = append(responseTranslations, translation.Text)
	}

	return responseTranslations
}

func main() {
	// We need the main function to make possible
	// CGO compiler to compile the package as C static archive
}
<|endoftext|>


<commit_before>package neptulon

// Logger provides low level request logging, performance metrics, and other metrics data.
type Logger struct{}
<commit_msg>add per logger stub<commit_after>package neptulon

import "github.com/nbusy/devastator/neptulon"

// Logger provides low level request logging, performance metrics, and other metrics data.
type Logger struct{}

func perfLoggerMiddleware(conn *neptulon.Conn, session *neptulon.Session, msg []byte) {
}
<|endoftext|>


<commit_before>// Copyright 2016 Palantir Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package defaulttasks

import (
	"sort"
	"strings"

	"github.com/pkg/errors"

	"github.com/palantir/godel/framework/godel/config"
	"github.com/palantir/godel/framework/internal/pluginsinternal"
)

const defaultResolver = "https://palantir.bintray.com/releases/{{GroupPath}}/{{Product}}/{{Version}}/{{Product}}-{{Version}}-{{OS}}-{{Arch}}.tgz"

var defaultPluginsConfig = config.PluginsConfig{
	DefaultResolvers: []string{
		defaultResolver,
	},
	Plugins: config.ToSinglePluginConfigs([]config.SinglePluginConfig{
		{
			LocatorWithResolverConfig: config.ToLocatorWithResolverConfig(config.LocatorWithResolverConfig{
				Locator: config.ToLocatorConfig(config.LocatorConfig{
					ID: "com.palantir.distgo:dist-plugin:1.0.0-rc15",
				}),
			}),
		},
		{
			LocatorWithResolverConfig: config.ToLocatorWithResolverConfig(config.LocatorWithResolverConfig{
				Locator: config.ToLocatorConfig(config.LocatorConfig{
					ID: "com.palantir.godel-format-plugin:format-plugin:1.0.0-rc7",
				}),
			}),
			Assets: config.ToLocatorWithResolverConfigs([]config.LocatorWithResolverConfig{
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-format-asset-ptimports:ptimports-asset:1.0.0-rc6",
					}),
				},
			}),
		},
		{
			LocatorWithResolverConfig: config.ToLocatorWithResolverConfig(config.LocatorWithResolverConfig{
				Locator: config.ToLocatorConfig(config.LocatorConfig{
					ID: "com.palantir.godel-goland-plugin:goland-plugin:1.0.0-rc2",
				}),
			}),
		},
		{
			LocatorWithResolverConfig: config.ToLocatorWithResolverConfig(config.LocatorWithResolverConfig{
				Locator: config.ToLocatorConfig(config.LocatorConfig{
					ID: "com.palantir.okgo:check-plugin:1.0.0-rc6",
				}),
			}),
			Assets: config.ToLocatorWithResolverConfigs([]config.LocatorWithResolverConfig{
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-compiles:compiles-asset:1.0.0-rc3",
					}),
				},
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-deadcode:deadcode-asset:1.0.0-rc2",
					}),
				},
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-errcheck:errcheck-asset:1.0.0-rc3",
					}),
				},
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-extimport:extimport-asset:1.0.0-rc2",
					}),
				},
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-golint:golint-asset:1.0.0-rc3",
					}),
				},
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-govet:govet-asset:1.0.0-rc3",
					}),
				},
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-importalias:importalias-asset:1.0.0-rc2",
					}),
				},
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-ineffassign:ineffassign-asset:1.0.0-rc2",
					}),
				},
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-novendor:novendor-asset:1.0.0-rc4",
					}),
				},
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-outparamcheck:outparamcheck-asset:1.0.0-rc3",
					}),
				},
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-unconvert:unconvert-asset:1.0.0-rc3",
					}),
				},
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-varcheck:varcheck-asset:1.0.0-rc2",
					}),
				},
			}),
		},
		{
			LocatorWithResolverConfig: config.ToLocatorWithResolverConfig(config.LocatorWithResolverConfig{
				Locator: config.ToLocatorConfig(config.LocatorConfig{
					ID: "com.palantir.godel-license-plugin:license-plugin:1.0.0-rc1",
				}),
			}),
		},
		{
			LocatorWithResolverConfig: config.ToLocatorWithResolverConfig(config.LocatorWithResolverConfig{
				Locator: config.ToLocatorConfig(config.LocatorConfig{
					ID: "com.palantir.godel-test-plugin:test-plugin:1.0.0-rc5",
				}),
			}),
		},
	}),
}

func BuiltinPluginsConfig() config.PluginsConfig {
	return defaultPluginsConfig
}

func PluginsConfig(cfg config.DefaultTasksConfig) (config.PluginsConfig, error) {
	// start with configuration that uses default resolver
	pluginsCfg := config.PluginsConfig{
		DefaultResolvers: defaultPluginsConfig.DefaultResolvers,
	}
	// append default resolvers provided by the configuration and uniquify
	pluginsCfg.DefaultResolvers = pluginsinternal.Uniquify(append(pluginsCfg.DefaultResolvers, cfg.DefaultResolvers...))

	defaultPluginKeys := make(map[string]struct{})
	for _, currPlugin := range defaultPluginsConfig.Plugins {
		currKey := locatorIDWithoutVersion(currPlugin.Locator.ID)
		defaultPluginKeys[currKey] = struct{}{}

		var assets []config.LocatorWithResolverConfig
		for _, asset := range currPlugin.Assets {
			assets = append(assets, config.LocatorWithResolverConfig(asset))
		}
		taskCfgV0, ok := cfg.Tasks[currKey]
		if !ok {
			// if custom configuration is not specified, use default and continue
			pluginsCfg.Plugins = append(pluginsCfg.Plugins, currPlugin)
			continue
		}
		taskCfg := config.SingleDefaultTaskConfig(taskCfgV0)

		// custom configuration was non-empty: start it with default LocatorWithResolver configuration
		currCfg := config.SinglePluginConfig{
			LocatorWithResolverConfig: currPlugin.LocatorWithResolverConfig,
		}
		if taskCfg.Locator.ID != "" {
			currCfg.Locator = taskCfg.Locator
		}
		if taskCfg.Resolver != "" {
			currCfg.Resolver = taskCfg.Resolver
		}

		currCfg.Assets = append(currCfg.Assets, config.ToLocatorWithResolverConfigs(assetConfigFromDefault(assets, taskCfg))...)
		currCfg.Assets = append(currCfg.Assets, taskCfg.Assets...)
		pluginsCfg.Plugins = append(pluginsCfg.Plugins, config.ToSinglePluginConfig(currCfg))
	}

	var invalidKeys []string
	for providedDefaultCfgKey := range cfg.Tasks {
		if _, ok := defaultPluginKeys[providedDefaultCfgKey]; ok {
			continue
		}
		invalidKeys = append(invalidKeys, providedDefaultCfgKey)
	}
	sort.Strings(invalidKeys)

	if len(invalidKeys) > 0 {
		var validKeys []string
		for k := range defaultPluginKeys {
			validKeys = append(validKeys, k)
		}
		sort.Strings(validKeys)
		return config.PluginsConfig{}, errors.Errorf("default-task key(s) specified but are not valid: %v. Valid values: %v", invalidKeys, validKeys)
	}

	return pluginsCfg, nil
}

func assetConfigFromDefault(baseCfg []config.LocatorWithResolverConfig, cfg config.SingleDefaultTaskConfig) []config.LocatorWithResolverConfig {
	if cfg.ExcludeAllDefaultAssets {
		return nil
	}
	exclude := make(map[string]struct{})
	for _, currExclude := range cfg.DefaultAssetsToExclude {
		exclude[currExclude] = struct{}{}
	}
	var out []config.LocatorWithResolverConfig
	for _, asset := range baseCfg {
		if _, ok := exclude[locatorIDWithoutVersion(asset.Locator.ID)]; ok {
			continue
		}
		out = append(out, asset)
	}
	return out
}

func locatorIDWithoutVersion(locatorID string) string {
	parts := strings.Split(locatorID, ":")
	return strings.Join(parts[:2], ":")
}
<commit_msg>Update default plugin and asset versions (#373)<commit_after>// Copyright 2016 Palantir Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package defaulttasks

import (
	"sort"
	"strings"

	"github.com/pkg/errors"

	"github.com/palantir/godel/framework/godel/config"
	"github.com/palantir/godel/framework/internal/pluginsinternal"
)

const defaultResolver = "https://palantir.bintray.com/releases/{{GroupPath}}/{{Product}}/{{Version}}/{{Product}}-{{Version}}-{{OS}}-{{Arch}}.tgz"

var defaultPluginsConfig = config.PluginsConfig{
	DefaultResolvers: []string{
		defaultResolver,
	},
	Plugins: config.ToSinglePluginConfigs([]config.SinglePluginConfig{
		{
			LocatorWithResolverConfig: config.ToLocatorWithResolverConfig(config.LocatorWithResolverConfig{
				Locator: config.ToLocatorConfig(config.LocatorConfig{
					ID: "com.palantir.distgo:dist-plugin:1.0.0",
					Checksums: map[string]string{
						"darwin-amd64": "f0154db51ed36bbd928d294537c426725a2054219dafd9588db1405dde37324a",
						"linux-amd64":  "0948157827cd97a044e6e47e41048b2406f43e290d3ad094491629da0fd7265c",
					},
				}),
			}),
		},
		{
			LocatorWithResolverConfig: config.ToLocatorWithResolverConfig(config.LocatorWithResolverConfig{
				Locator: config.ToLocatorConfig(config.LocatorConfig{
					ID: "com.palantir.godel-format-plugin:format-plugin:1.0.0",
					Checksums: map[string]string{
						"darwin-amd64": "1c6c7e06226efab7213f0b7f2e8ee61f59c0198d7cb72fcee76b2c7d9abddf36",
						"linux-amd64":  "f29f608a13eb1991357a29e6576c4010738fcd43e0930f5e507c02b2b87dfb73",
					},
				}),
			}),
			Assets: config.ToLocatorWithResolverConfigs([]config.LocatorWithResolverConfig{
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-format-asset-ptimports:ptimports-asset:1.0.0",
						Checksums: map[string]string{
							"darwin-amd64": "44f7d2ee0392cb30174a189a03d537cbc24f80f521a1110f74e5d54345d66d93",
							"linux-amd64":  "94ef1e5f8260cc783396e4e72034cb1dba6b25b33b7b2a21371b06ae17376d60",
						},
					}),
				},
			}),
		},
		{
			LocatorWithResolverConfig: config.ToLocatorWithResolverConfig(config.LocatorWithResolverConfig{
				Locator: config.ToLocatorConfig(config.LocatorConfig{
					ID: "com.palantir.godel-goland-plugin:goland-plugin:1.0.0",
					Checksums: map[string]string{
						"darwin-amd64": "5b518708e5c41d81545d89d7224d2b61bf56d953eb560513ad047903eaa11b12",
						"linux-amd64":  "a000f7cd87f878d4c2e51e74f6015beb8fe48ea242c45f7731c7435a93f5a419",
					},
				}),
			}),
		},
		{
			LocatorWithResolverConfig: config.ToLocatorWithResolverConfig(config.LocatorWithResolverConfig{
				Locator: config.ToLocatorConfig(config.LocatorConfig{
					ID: "com.palantir.okgo:check-plugin:1.0.0",
					Checksums: map[string]string{
						"darwin-amd64": "29c47ae4370c1a21f3b7832ec055669d102d4eb019c0d2c5785fcedfe688b867",
						"linux-amd64":  "80d4fa038326048b57507a56a6adbb3d492182fb38093bf9483b27d4c83428b0",
					},
				}),
			}),
			Assets: config.ToLocatorWithResolverConfigs([]config.LocatorWithResolverConfig{
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-compiles:compiles-asset:1.0.0",
						Checksums: map[string]string{
							"darwin-amd64": "d740b5ddf4befcb553ee90ec07a62ce322943069d48688cf48eefae6df45bd62",
							"linux-amd64":  "a01ecaeeb9093b5ba2838bde7ee041c3f181d3f286aeca740c3b39659a04e12f",
						},
					}),
				},
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-deadcode:deadcode-asset:1.0.0",
						Checksums: map[string]string{
							"darwin-amd64": "385437dd9e424b7372934a009cb13d2cba5442d9f6f5ce4d77b7831e789cca10",
							"linux-amd64":  "9d51dd2942a5b2758246ce9ddb7baecf097a9dc83e90aff8ccbd847b92fcc932",
						},
					}),
				},
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-errcheck:errcheck-asset:1.0.0",
						Checksums: map[string]string{
							"darwin-amd64": "52500c7d9abb4f44e3ff5baabe39926622ac383dd1186bffd9d1b45128850b0d",
							"linux-amd64":  "6f508aae2c45dce4d6cdc2bbe3c9e53db195639a63966c96b05215e4a1b900ed",
						},
					}),
				},
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-extimport:extimport-asset:1.0.0",
						Checksums: map[string]string{
							"darwin-amd64": "76a16a0228a1dbb3fbd28614094a52e9a0a320178aa70d045575eb07ca022b15",
							"linux-amd64":  "ca9a0d9044cb626898d3eb5b3f56d85df543d536150180ee506f27489f84aa19",
						},
					}),
				},
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-golint:golint-asset:1.0.0",
						Checksums: map[string]string{
							"darwin-amd64": "23105fb709f06f241be59c8441afa3125191a7c4c3b6e68beabea56955b0a410",
							"linux-amd64":  "b29cdb47fba9127166306ff072285e2c71753b98c8e17fc4646379dc58dab17d",
						},
					}),
				},
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-govet:govet-asset:1.0.0",
						Checksums: map[string]string{
							"darwin-amd64": "8ce27439c844e3617dceeb9b6cb99218f73b17ef56d0eda8d6ea4434825b78ce",
							"linux-amd64":  "43a526f8800dbb6802755747c5a133010e4accaa1afcef39e2d14ffa39bb0dd8",
						},
					}),
				},
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-importalias:importalias-asset:1.0.0",
						Checksums: map[string]string{
							"darwin-amd64": "2453a34101380c82a02427cf04ebe0431ed14c37c87d6d5811adfc268b75f391",
							"linux-amd64":  "8737bf922f7771df845a92339f3479d53a509bab4ebb5b0893899738e800d21a",
						},
					}),
				},
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-ineffassign:ineffassign-asset:1.0.0",
						Checksums: map[string]string{
							"darwin-amd64": "a5722caf601170bf25a6a4d95b75016fd76272413b73541847df47c26c431662",
							"linux-amd64":  "66afe8103ccdd8c35f8f05a8200e5a258ea0cd35762dc3a8f1b7fcfb466e837d",
						},
					}),
				},
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-novendor:novendor-asset:1.0.0",
						Checksums: map[string]string{
							"darwin-amd64": "94e90c93487985b033382f6a92cd1bad410172dcbfa41b336189125be988d771",
							"linux-amd64":  "37b3d2ef4aae0e4a05bdf88062d0359474de81eb54d8542f169ba2377ca3369a",
						},
					}),
				},
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-outparamcheck:outparamcheck-asset:1.0.0",
						Checksums: map[string]string{
							"darwin-amd64": "9612037f770679e7deb9ee3f11f939a7e565193b6b558759199ba3a0635f97b0",
							"linux-amd64":  "3eec093c4c62a5a809ee58acf6a76efeb86a35bdf5416397ff7f5ddef1b8a35a",
						},
					}),
				},
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-unconvert:unconvert-asset:1.0.0",
						Checksums: map[string]string{
							"darwin-amd64": "a6c92dd8659d7b0824d4ef4e653002299a2e3323c1d441f1f9899737da81270d",
							"linux-amd64":  "f429c2361af530d835451edf8447393f221f21b4ad054cc4e44bf4eb51c4de75",
						},
					}),
				},
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-varcheck:varcheck-asset:1.0.0",
						Checksums: map[string]string{
							"darwin-amd64": "3a58db2a6c810c45985970cd85c15a529080b55e2f43c54bd777b4715abb2a69",
							"linux-amd64":  "ab07f4bcb182412696523b252f721c4ba502670cadb2bcf8fc2bef1abd0ed29e",
						},
					}),
				},
			}),
		},
		{
			LocatorWithResolverConfig: config.ToLocatorWithResolverConfig(config.LocatorWithResolverConfig{
				Locator: config.ToLocatorConfig(config.LocatorConfig{
					ID: "com.palantir.godel-license-plugin:license-plugin:1.0.0",
					Checksums: map[string]string{
						"darwin-amd64": "9b3b464ebfcad71718fe31dac47b192457d705e679d05fae564caa92661533f5",
						"linux-amd64":  "5e51f6df8bb3ee77ac25c1c9f9bf84498a34139590efee7ce9deda28d96d52cd",
					},
				}),
			}),
		},
		{
			LocatorWithResolverConfig: config.ToLocatorWithResolverConfig(config.LocatorWithResolverConfig{
				Locator: config.ToLocatorConfig(config.LocatorConfig{
					ID: "com.palantir.godel-test-plugin:test-plugin:1.0.0",
					Checksums: map[string]string{
						"darwin-amd64": "6a40ddbc0c6d1c0f705f455c8efcb76d05766c71c058caa938ead942d60bf190",
						"linux-amd64":  "95716e73a388865fd04881ed2cb7c521ffb9a0ffeb25a3090bd543d34c4983e5",
					},
				}),
			}),
		},
	}),
}

func BuiltinPluginsConfig() config.PluginsConfig {
	return defaultPluginsConfig
}

func PluginsConfig(cfg config.DefaultTasksConfig) (config.PluginsConfig, error) {
	// start with configuration that uses default resolver
	pluginsCfg := config.PluginsConfig{
		DefaultResolvers: defaultPluginsConfig.DefaultResolvers,
	}
	// append default resolvers provided by the configuration and uniquify
	pluginsCfg.DefaultResolvers = pluginsinternal.Uniquify(append(pluginsCfg.DefaultResolvers, cfg.DefaultResolvers...))

	defaultPluginKeys := make(map[string]struct{})
	for _, currPlugin := range defaultPluginsConfig.Plugins {
		currKey := locatorIDWithoutVersion(currPlugin.Locator.ID)
		defaultPluginKeys[currKey] = struct{}{}

		var assets []config.LocatorWithResolverConfig
		for _, asset := range currPlugin.Assets {
			assets = append(assets, config.LocatorWithResolverConfig(asset))
		}
		taskCfgV0, ok := cfg.Tasks[currKey]
		if !ok {
			// if custom configuration is not specified, use default and continue
			pluginsCfg.Plugins = append(pluginsCfg.Plugins, currPlugin)
			continue
		}
		taskCfg := config.SingleDefaultTaskConfig(taskCfgV0)

		// custom configuration was non-empty: start it with default LocatorWithResolver configuration
		currCfg := config.SinglePluginConfig{
			LocatorWithResolverConfig: currPlugin.LocatorWithResolverConfig,
		}
		if taskCfg.Locator.ID != "" {
			currCfg.Locator = taskCfg.Locator
		}
		if taskCfg.Resolver != "" {
			currCfg.Resolver = taskCfg.Resolver
		}

		currCfg.Assets = append(currCfg.Assets, config.ToLocatorWithResolverConfigs(assetConfigFromDefault(assets, taskCfg))...)
		currCfg.Assets = append(currCfg.Assets, taskCfg.Assets...)
		pluginsCfg.Plugins = append(pluginsCfg.Plugins, config.ToSinglePluginConfig(currCfg))
	}

	var invalidKeys []string
	for providedDefaultCfgKey := range cfg.Tasks {
		if _, ok := defaultPluginKeys[providedDefaultCfgKey]; ok {
			continue
		}
		invalidKeys = append(invalidKeys, providedDefaultCfgKey)
	}
	sort.Strings(invalidKeys)

	if len(invalidKeys) > 0 {
		var validKeys []string
		for k := range defaultPluginKeys {
			validKeys = append(validKeys, k)
		}
		sort.Strings(validKeys)
		return config.PluginsConfig{}, errors.Errorf("default-task key(s) specified but are not valid: %v. Valid values: %v", invalidKeys, validKeys)
	}

	return pluginsCfg, nil
}

func assetConfigFromDefault(baseCfg []config.LocatorWithResolverConfig, cfg config.SingleDefaultTaskConfig) []config.LocatorWithResolverConfig {
	if cfg.ExcludeAllDefaultAssets {
		return nil
	}
	exclude := make(map[string]struct{})
	for _, currExclude := range cfg.DefaultAssetsToExclude {
		exclude[currExclude] = struct{}{}
	}
	var out []config.LocatorWithResolverConfig
	for _, asset := range baseCfg {
		if _, ok := exclude[locatorIDWithoutVersion(asset.Locator.ID)]; ok {
			continue
		}
		out = append(out, asset)
	}
	return out
}

func locatorIDWithoutVersion(locatorID string) string {
	parts := strings.Split(locatorID, ":")
	return strings.Join(parts[:2], ":")
}
<|endoftext|>