<commit_before>package nicolive

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/cookiejar"
	"net/url"

	"gopkg.in/yaml.v2"
)

// Account is a niconico account
type Account struct {
	Mail        string
	Pass        string
	Usersession string
}

func (a Account) String() string {
	i, l := 5, len(a.Mail)
	if i > l {
		i = l
	}
	return fmt.Sprintf("Account{%s..}", a.Mail[0:i])
}

// Save save Account to a file
func (a Account) Save(filePath string) error {
	d, err := yaml.Marshal(a)
	if err != nil {
		return err
	}

	err = ioutil.WriteFile(filePath, d, 0600)
	if err != nil {
		return err
	}

	return nil
}

// Load lead from a file and returns a pointer to Account
func (a *Account) Load(filePath string) error {
	d, err := ioutil.ReadFile(filePath)
	if err != nil {
		return err
	}

	err = yaml.Unmarshal(d, a)
	if err != nil {
		return err
	}

	return nil
}

// Login log in to niconico and update Usersession
func (a *Account) Login() NicoError {
	if a.Mail == "" || a.Pass == "" {
		return NicoErr(NicoErrOther, "invalid account info", "mail or pass is not set")
	}

	jar, err := cookiejar.New(nil)
	if err != nil {
		return NicoErrFromStdErr(err)
	}
	cl := http.Client{Jar: jar}

	params := url.Values{
		"mail":     []string{a.Mail},
		"password": []string{a.Pass},
	}
	resp, err := cl.PostForm("https://secure.nicovideo.jp/secure/login?site=nicolive", params)
	if err != nil {
		return NicoErrFromStdErr(err)
	}
	defer resp.Body.Close()

	nicoURL, err := url.Parse("http://nicovideo.jp")
	if err != nil {
		return NicoErrFromStdErr(err)
	}
	for _, ck := range cl.Jar.Cookies(nicoURL) {
		if ck.Name == "user_session" {
			if ck.Value != "deleted" && ck.Value != "" {
				a.Usersession = ck.Value
				return nil
			}
		}
	}

	return NicoErr(NicoErrOther, "login error", "failed log in to niconico")
}
<commit_msg>Usersession should not be exported<commit_after>package nicolive

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/cookiejar"
	"net/url"

	"gopkg.in/yaml.v2"
)

// Account is a niconico account
type Account struct {
	Mail        string `yaml:"mail"`
	Pass        string `yaml:"pass"`
	Usersession string `yaml:"-"`
}

func (a Account) String() string {
	i, l := 5, len(a.Mail)
	if i > l {
		i = l
	}
	return fmt.Sprintf("Account{%s..}", a.Mail[0:i])
}

// Save save Account to a file
func (a Account) Save(filePath string) error {
	d, err := yaml.Marshal(a)
	if err != nil {
		return err
	}

	err = ioutil.WriteFile(filePath, d, 0600)
	if err != nil {
		return err
	}

	return nil
}

// Load lead from a file and returns a pointer to Account
func (a *Account) Load(filePath string) error {
	d, err := ioutil.ReadFile(filePath)
	if err != nil {
		return err
	}

	err = yaml.Unmarshal(d, a)
	if err != nil {
		return err
	}

	return nil
}

// Login log in to niconico and update Usersession
func (a *Account) Login() NicoError {
	if a.Mail == "" || a.Pass == "" {
		return NicoErr(NicoErrOther, "invalid account info", "mail or pass is not set")
	}

	jar, err := cookiejar.New(nil)
	if err != nil {
		return NicoErrFromStdErr(err)
	}
	cl := http.Client{Jar: jar}

	params := url.Values{
		"mail":     []string{a.Mail},
		"password": []string{a.Pass},
	}
	resp, err := cl.PostForm("https://secure.nicovideo.jp/secure/login?site=nicolive", params)
	if err != nil {
		return NicoErrFromStdErr(err)
	}
	defer resp.Body.Close()

	nicoURL, err := url.Parse("http://nicovideo.jp")
	if err != nil {
		return NicoErrFromStdErr(err)
	}
	for _, ck := range cl.Jar.Cookies(nicoURL) {
		if ck.Name == "user_session" {
			if ck.Value != "deleted" && ck.Value != "" {
				a.Usersession = ck.Value
				return nil
			}
		}
	}

	return NicoErr(NicoErrOther, "login error", "failed log in to niconico")
}
<|endoftext|>


<commit_before>/*
Copyright 2018, Cossack Labs Limited

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package server

import (
	"context"
	"google.golang.org/grpc/credentials"
	"net"
	"os"
	"time"

	"bufio"
	"net/http"

	"github.com/cossacklabs/acra/cmd/acra-translator/common"
	"github.com/cossacklabs/acra/cmd/acra-translator/http_api"
	"github.com/cossacklabs/acra/decryptor/base"
	"github.com/cossacklabs/acra/keystore"
	"github.com/cossacklabs/acra/logging"
	"github.com/cossacklabs/acra/network"
	log "github.com/sirupsen/logrus"
	"go.opencensus.io/trace"
	"google.golang.org/grpc"
)

// ReaderServer represents AcraTranslator server, connects with KeyStorage, configuration file,
// gRPC and HTTP request parsers.
type ReaderServer struct {
	config            *common.AcraTranslatorConfig
	keystorage        keystore.MultiKeyStore
	connectionManager *network.ConnectionManager
	grpcServer        *grpc.Server

	httpDecryptor *http_api.HTTPConnectionsDecryptor

	waitTimeout time.Duration

	listenersContextCancel []context.CancelFunc
	grpcServerFactory      common.GRPCServerFactory
}

// NewReaderServer creates Reader server with provided params.
func NewReaderServer(config *common.AcraTranslatorConfig, keystorage keystore.MultiKeyStore, grpcServerFactory common.GRPCServerFactory, waitTimeout time.Duration) (server *ReaderServer, err error) {
	return &ReaderServer{
		grpcServerFactory: grpcServerFactory,
		waitTimeout:       waitTimeout,
		config:            config,
		keystorage:        keystorage,
		connectionManager: network.NewConnectionManager(),
	}, nil
}

// Stop stops AcraTranslator from accepting new connections, and gracefully close existing ones.
func (server *ReaderServer) Stop() {
	log.Infoln("Stop accepting new connections")
	// stop all listeners
	for _, cancelFunc := range server.listenersContextCancel {
		cancelFunc()
	}
	// non block stop
	if server.grpcServer != nil {
		go server.grpcServer.GracefulStop()
	}

	if server.connectionManager.Counter != 0 {
		log.Infof("Wait ending current connections (%v)", server.connectionManager.Counter)
		// wait existing connections to end request
		<-time.NewTimer(server.waitTimeout).C
	}

	log.Infof("Stop all connections that not closed (%v)", server.connectionManager.Counter)
	if server.grpcServer != nil {
		// force stop of grpc server
		server.grpcServer.Stop()
	}
	// force close all connections
	if err := server.connectionManager.CloseConnections(); err != nil {
		log.WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantCloseConnection).WithError(err).Errorln("Took error on closing available connections")
	}
}

func (server *ReaderServer) listenerContext(parentContext context.Context) context.Context {
	ctx, cancel := context.WithCancel(parentContext)
	server.listenersContextCancel = append(server.listenersContextCancel, cancel)
	return ctx
}

// HandleConnectionString handles each connection with gRPC request handler or HTTP request handler
// depending on connection string.
func (server *ReaderServer) HandleConnectionString(parentContext context.Context, connectionString string, processingFunc ProcessingFunc) error {
	logger := logging.GetLoggerFromContext(parentContext)
	if logger == nil {
		logger = log.NewEntry(log.StandardLogger())
	}
	logger = log.WithField("connection_string", connectionString)

	errCh := make(chan error)

	listenerContext := server.listenerContext(parentContext)

	// start accept new connections from connectionString
	connectionChannel, err := common.AcceptConnections(listenerContext, connectionString, errCh)
	if err != nil {
		logger.WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantAcceptNewConnections).WithError(err).Errorf("Can't start to handle connection string %v", connectionString)
		return err
	}
	// use to send close packets to all unclosed connections at end
	go func() {
		logger.WithField("connection_string", connectionString).Debugln("Start wrap new connections")
		for {
			var connection net.Conn
			select {
			case connection = <-connectionChannel:
				break
			case <-parentContext.Done():
				logger.WithError(parentContext.Err()).Debugln("Stop wrapping new connections")
				return
			}

			wrappedConnection, clientID, err := server.config.ConnectionWrapper.WrapServer(context.TODO(), connection)
			if err != nil {
				logger.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTranslatorCantWrapConnectionToSS).
					Errorln("Can't wrap new connection")
				if err := connection.Close(); err != nil {
					logger.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTranslatorCantCloseConnection).
						Errorln("Can't close connection")
				}
				continue
			}
			logger = logger.WithField("client_id", string(clientID))
			logger.Debugln("Read trace")
			spanContext, err := network.ReadTrace(wrappedConnection)
			if err != nil {
				logger.WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTracingCantReadTrace).WithError(err).Errorln("Can't read trace from wrapped connection")
				if err := wrappedConnection.Close(); err != nil {
					log.WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantWrapConnection).WithError(err).Errorln("Can't close wrapped connection")
				}
				continue
			}
			ctx, span := trace.StartSpanWithRemoteParent(listenerContext, getHandlerName(listenerContext), spanContext, server.config.GetTraceOptions()...)
			logger.Debugln("Pass wrapped connection to processing function")
			logging.SetLoggerToContext(ctx, logger)

			go func() {
				defer func() {
					span.End()
					err := wrappedConnection.Close()
					if err != nil {
						logger.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTranslatorCantCloseConnection).
							Errorln("Can't close wrapped connection")
					}
					logger.Infoln("Connection closed")
				}()

				if err := server.connectionManager.AddConnection(wrappedConnection); err != nil {
					logger.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTranslatorCantHandleHTTPConnection).
						Errorln("Can't add connection to connection manager")
					return
				}

				processingFunc(ctx, clientID, wrappedConnection)

				if err := server.connectionManager.RemoveConnection(wrappedConnection); err != nil {
					logger.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTranslatorCantHandleHTTPConnection).
						Errorln("Can't remove connection from connection manager")
				}
			}()
		}
	}()
	var outErr error
	select {
	case <-parentContext.Done():
		log.WithError(parentContext.Err()).Debugln("Exit from handling connection string. Close all connections")
		outErr = parentContext.Err()
	case outErr = <-errCh:
		log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTranslatorCantAcceptNewHTTPConnection).
			Errorln("Error on accepting new connections")
		server.Stop()
	}
	return outErr
}

// Constants show possible connection types.
const (
	ConnectionTypeKey  = "connection_type"
	HTTPConnectionType = "http"
	GRPCConnectionType = "grpc"
)

type handlerName struct{}

func withHandlerName(ctx context.Context, name string) context.Context {
	return context.WithValue(ctx, handlerName{}, name)
}

func getHandlerName(ctx context.Context) string {
	if s, ok := ctx.Value(handlerName{}).(string); ok {
		return s
	}
	return "undefined"
}

// Start setups gRPC handler or HTTP handler, poison records callbacks and starts listening to connections.
func (server *ReaderServer) Start(parentContext context.Context) {
	logger := logging.GetLoggerFromContext(parentContext)
	poisonCallbacks := base.NewPoisonCallbackStorage()
	if server.config.DetectPoisonRecords() {
		if server.config.ScriptOnPoison() != "" {
			log.Infof("Add poison record callback with script execution %v", server.config.ScriptOnPoison())
			poisonCallbacks.AddCallback(base.NewExecuteScriptCallback(server.config.ScriptOnPoison()))
		}

		// must be last
		if server.config.StopOnPoison() {
			log.Infoln("Add poison record callback with AcraTranslator termination")
			poisonCallbacks.AddCallback(&base.StopCallback{})
		}
	}
	decryptorData := &common.TranslatorData{Keystorage: server.keystorage, PoisonRecordCallbacks: poisonCallbacks, CheckPoisonRecords: server.config.DetectPoisonRecords()}
	if server.config.IncomingConnectionHTTPString() != "" {
		go func() {
			httpContext := logging.SetLoggerToContext(parentContext, logger.WithField(ConnectionTypeKey, HTTPConnectionType))
			httpDecryptor, err := http_api.NewHTTPConnectionsDecryptor(decryptorData)
			logger.WithField("connection_string", server.config.IncomingConnectionHTTPString()).Infof("Start process HTTP requests")
			if err != nil {
				log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTranslatorCantHandleHTTPConnection).
					Errorln("Can't create HTTP decryptor")
			}
			server.httpDecryptor = httpDecryptor
			err = server.HandleConnectionString(withHandlerName(httpContext, "processHTTPConnection"), server.config.IncomingConnectionHTTPString(), server.processHTTPConnection)
			if err != nil {
				log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTranslatorCantHandleHTTPConnection).
					Errorln("Took error on handling HTTP requests")
				server.Stop()
				os.Exit(1)
			}
		}()
	}
	// provide way to register new services and custom server
	if server.config.IncomingConnectionGRPCString() != "" {
		go func() {
			grpcLogger := logger.WithField(ConnectionTypeKey, GRPCConnectionType)
			logger.WithField("connection_string", server.config.IncomingConnectionGRPCString()).Infof("Start process gRPC requests")
			var listener net.Listener
			var err error
			var opts []grpc.ServerOption
			if server.config.WithTLS() {
				listener, err = network.Listen(server.config.IncomingConnectionGRPCString())
				if err != nil {
					grpcLogger.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTranslatorCantHandleGRPCConnection).
						Errorln("Can't create gRPC connection listener")
					return
				}
				opts = append(opts, grpc.Creds(credentials.NewTLS(server.config.GetTLSConfig())))
			} else {
				listener, err = network.Listen(server.config.IncomingConnectionGRPCString())
				if err != nil {
					grpcLogger.WithError(err).Errorln("Can't initialize connection listener")
					return
				}
				wrapper, err := network.NewSecureSessionConnectionWrapper(server.config.ServerID(), server.keystorage)
				if err != nil {
					grpcLogger.WithError(err).Errorln("Can't initialize Secure Session wrapper")
					return
				}
				opts = append(opts, grpc.Creds(wrapper))
			}

			grpcListener := common.WrapListenerWithMetrics(listener)

			grpcServer, err := server.grpcServerFactory.New(decryptorData, opts...)
			if err != nil {
				logger.WithError(err).Errorln("Can't create new grpc server")
				os.Exit(1)
			}
			server.grpcServer = grpcServer
			if err := grpcServer.Serve(grpcListener); err != nil {
				grpcLogger.Errorf("failed to serve: %v", err)
				server.Stop()
				os.Exit(1)
				return
			}
		}()
	}
	<-parentContext.Done()
}

// ProcessingFunc redirects processing of connection to HTTP handler or gRPC handler.
type ProcessingFunc func(context.Context, []byte, net.Conn)

func (server *ReaderServer) processHTTPConnection(parentContext context.Context, clientID []byte, connection net.Conn) {
	connection.SetDeadline(time.Now().Add(network.DefaultNetworkTimeout))
	defer connection.SetDeadline(time.Time{})

	spanCtx, span := trace.StartSpan(parentContext, "processHTTPConnection")
	defer span.End()

	// processing HTTP connection
	logger := logging.LoggerWithTrace(spanCtx, logging.GetLoggerFromContext(parentContext))
	httpLogger := logger.WithField(ConnectionTypeKey, HTTPConnectionType)
	httpLogger.Debugln("HTTP handler")

	reader := bufio.NewReader(connection)
	request, err := http.ReadRequest(reader)

	// TODO: handle keep alive

	if err != nil {
		logger.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTranslatorCantHandleHTTPRequest).
			Warningln("Got new HTTP request, but can't read it")
		server.httpDecryptor.SendResponse(logger,
			server.httpDecryptor.EmptyResponseWithStatus(request, http.StatusBadRequest), connection)
		return
	}

	response := server.httpDecryptor.ParseRequestPrepareResponse(logger, request, clientID)
	server.httpDecryptor.SendResponse(logger, response, connection)
}
<commit_msg>user waitGroup mechanism (with timeout) on ReaderServer level<commit_after>/*
Copyright 2018, Cossack Labs Limited

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package server

import (
	"context"
	"errors"
	"google.golang.org/grpc/credentials"
	"net"
	"os"
	"sync"
	"time"

	"bufio"
	"net/http"

	"github.com/cossacklabs/acra/cmd/acra-translator/common"
	"github.com/cossacklabs/acra/cmd/acra-translator/http_api"
	"github.com/cossacklabs/acra/decryptor/base"
	"github.com/cossacklabs/acra/keystore"
	"github.com/cossacklabs/acra/logging"
	"github.com/cossacklabs/acra/network"
	log "github.com/sirupsen/logrus"
	"go.opencensus.io/trace"
	"google.golang.org/grpc"
)

// ReaderServer represents AcraTranslator server, connects with KeyStorage, configuration file,
// gRPC and HTTP request parsers.
type ReaderServer struct {
	config            *common.AcraTranslatorConfig
	keystorage        keystore.MultiKeyStore
	connectionManager *network.ConnectionManager
	grpcServer        *grpc.Server
	httpDecryptor *http_api.HTTPConnectionsDecryptor
	waitTimeout time.Duration
	listenersContextCancel []context.CancelFunc
	grpcServerFactory      common.GRPCServerFactory
	waitGroup              sync.WaitGroup
}

// NewReaderServer creates Reader server with provided params.
func NewReaderServer(config *common.AcraTranslatorConfig, keystorage keystore.MultiKeyStore, grpcServerFactory common.GRPCServerFactory, waitTimeout time.Duration) (server *ReaderServer, err error) {
	var wg sync.WaitGroup
	return &ReaderServer{
		grpcServerFactory: grpcServerFactory,
		waitTimeout:       waitTimeout,
		config:            config,
		keystorage:        keystorage,
		connectionManager: network.NewConnectionManager(),
		waitGroup: wg,
	}, nil
}

// Stop stops AcraTranslator from accepting new connections, and gracefully close existing ones.
func (server *ReaderServer) Stop() {
	log.Infoln("Stop accepting new connections")
	// stop all listeners
	for _, cancelFunc := range server.listenersContextCancel {
		cancelFunc()
	}
	// non block stop
	if server.grpcServer != nil {
		server.waitGroup.Add(1)
		go func() {
			defer server.waitGroup.Done()
			server.grpcServer.GracefulStop()
		}()
	}

	if server.connectionManager.Counter != 0 {
		log.Infof("Wait ending current connections (%v)", server.connectionManager.Counter)
		// wait existing connections to end request
		<-time.NewTimer(server.waitTimeout).C
	}

	log.Infof("Stop all connections that not closed (%v)", server.connectionManager.Counter)
	if server.grpcServer != nil {
		// force stop of grpc server
		server.grpcServer.Stop()
	}
	// force close all connections
	if err := server.connectionManager.CloseConnections(); err != nil {
		log.WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantCloseConnection).WithError(err).Errorln("Took error on closing available connections")
	}
}

func (server *ReaderServer) listenerContext(parentContext context.Context) context.Context {
	ctx, cancel := context.WithCancel(parentContext)
	server.listenersContextCancel = append(server.listenersContextCancel, cancel)
	return ctx
}

// HandleConnectionString handles each connection with gRPC request handler or HTTP request handler
// depending on connection string.
func (server *ReaderServer) HandleConnectionString(parentContext context.Context, connectionString string, processingFunc ProcessingFunc) error {
	logger := logging.GetLoggerFromContext(parentContext)
	if logger == nil {
		logger = log.NewEntry(log.StandardLogger())
	}
	logger = log.WithField("connection_string", connectionString)

	errCh := make(chan error)

	listenerContext := server.listenerContext(parentContext)

	// start accept new connections from connectionString
	connectionChannel, err := common.AcceptConnections(listenerContext, connectionString, errCh)
	if err != nil {
		logger.WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantAcceptNewConnections).WithError(err).Errorf("Can't start to handle connection string %v", connectionString)
		return err
	}
	// use to send close packets to all unclosed connections at end
	server.waitGroup.Add(1)
	go func() {
		defer server.waitGroup.Done()
		logger.WithField("connection_string", connectionString).Debugln("Start wrap new connections")
		for {
			var connection net.Conn
			select {
			case connection = <-connectionChannel:
				break
			case <-parentContext.Done():
				logger.WithError(parentContext.Err()).Debugln("Stop wrapping new connections")
				return
			}

			wrappedConnection, clientID, err := server.config.ConnectionWrapper.WrapServer(context.TODO(), connection)
			if err != nil {
				logger.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTranslatorCantWrapConnectionToSS).
					Errorln("Can't wrap new connection")
				if err := connection.Close(); err != nil {
					logger.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTranslatorCantCloseConnection).
						Errorln("Can't close connection")
				}
				continue
			}
			logger = logger.WithField("client_id", string(clientID))
			logger.Debugln("Read trace")
			spanContext, err := network.ReadTrace(wrappedConnection)
			if err != nil {
				logger.WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTracingCantReadTrace).WithError(err).Errorln("Can't read trace from wrapped connection")
				if err := wrappedConnection.Close(); err != nil {
					log.WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantWrapConnection).WithError(err).Errorln("Can't close wrapped connection")
				}
				continue
			}
			ctx, span := trace.StartSpanWithRemoteParent(listenerContext, getHandlerName(listenerContext), spanContext, server.config.GetTraceOptions()...)
			logger.Debugln("Pass wrapped connection to processing function")
			logging.SetLoggerToContext(ctx, logger)

			server.waitGroup.Add(1)
			go func() {
				defer func() {
					span.End()
					err := wrappedConnection.Close()
					if err != nil {
						logger.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTranslatorCantCloseConnection).
							Errorln("Can't close wrapped connection")
					}
					logger.Infoln("Connection closed")
					server.waitGroup.Done()
				}()

				if err := server.connectionManager.AddConnection(wrappedConnection); err != nil {
					logger.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTranslatorCantHandleHTTPConnection).
						Errorln("Can't add connection to connection manager")
					return
				}

				processingFunc(ctx, clientID, wrappedConnection)

				if err := server.connectionManager.RemoveConnection(wrappedConnection); err != nil {
					logger.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTranslatorCantHandleHTTPConnection).
						Errorln("Can't remove connection from connection manager")
				}
			}()
		}
	}()
	var outErr error
	select {
	case <-parentContext.Done():
		log.WithError(parentContext.Err()).Debugln("Exit from handling connection string. Close all connections")
		outErr = parentContext.Err()
	case outErr = <-errCh:
		log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTranslatorCantAcceptNewHTTPConnection).
			Errorln("Error on accepting new connections")
		server.Stop()
	}
	return outErr
}

// Constants show possible connection types.
const (
	ConnectionTypeKey  = "connection_type"
	HTTPConnectionType = "http"
	GRPCConnectionType = "grpc"
)

type handlerName struct{}

func withHandlerName(ctx context.Context, name string) context.Context {
	return context.WithValue(ctx, handlerName{}, name)
}

func getHandlerName(ctx context.Context) string {
	if s, ok := ctx.Value(handlerName{}).(string); ok {
		return s
	}
	return "undefined"
}

// Start setups gRPC handler or HTTP handler, poison records callbacks and starts listening to connections.
func (server *ReaderServer) Start(parentContext context.Context) {
	logger := logging.GetLoggerFromContext(parentContext)
	poisonCallbacks := base.NewPoisonCallbackStorage()
	if server.config.DetectPoisonRecords() {
		if server.config.ScriptOnPoison() != "" {
			log.Infof("Add poison record callback with script execution %v", server.config.ScriptOnPoison())
			poisonCallbacks.AddCallback(base.NewExecuteScriptCallback(server.config.ScriptOnPoison()))
		}

		// must be last
		if server.config.StopOnPoison() {
			log.Infoln("Add poison record callback with AcraTranslator termination")
			poisonCallbacks.AddCallback(&base.StopCallback{})
		}
	}
	decryptorData := &common.TranslatorData{Keystorage: server.keystorage, PoisonRecordCallbacks: poisonCallbacks, CheckPoisonRecords: server.config.DetectPoisonRecords()}
	if server.config.IncomingConnectionHTTPString() != "" {
		server.waitGroup.Add(1)
		go func() {
			defer server.waitGroup.Done()
			httpContext := logging.SetLoggerToContext(parentContext, logger.WithField(ConnectionTypeKey, HTTPConnectionType))
			httpDecryptor, err := http_api.NewHTTPConnectionsDecryptor(decryptorData)
			logger.WithField("connection_string", server.config.IncomingConnectionHTTPString()).Infof("Start process HTTP requests")
			if err != nil {
				log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTranslatorCantHandleHTTPConnection).
					Errorln("Can't create HTTP decryptor")
			}
			server.httpDecryptor = httpDecryptor
			err = server.HandleConnectionString(withHandlerName(httpContext, "processHTTPConnection"), server.config.IncomingConnectionHTTPString(), server.processHTTPConnection)
			if err != nil {
				log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTranslatorCantHandleHTTPConnection).
					Errorln("Took error on handling HTTP requests")
				server.Stop()
				os.Exit(1)
			}
		}()
	}
	// provide way to register new services and custom server
	if server.config.IncomingConnectionGRPCString() != "" {
		server.waitGroup.Add(1)
		go func() {
			defer server.waitGroup.Done()
			grpcLogger := logger.WithField(ConnectionTypeKey, GRPCConnectionType)
			logger.WithField("connection_string", server.config.IncomingConnectionGRPCString()).Infof("Start process gRPC requests")
			var listener net.Listener
			var err error
			var opts []grpc.ServerOption
			if server.config.WithTLS() {
				listener, err = network.Listen(server.config.IncomingConnectionGRPCString())
				if err != nil {
					grpcLogger.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTranslatorCantHandleGRPCConnection).
						Errorln("Can't create gRPC connection listener")
					return
				}
				opts = append(opts, grpc.Creds(credentials.NewTLS(server.config.GetTLSConfig())))
			} else {
				listener, err = network.Listen(server.config.IncomingConnectionGRPCString())
				if err != nil {
					grpcLogger.WithError(err).Errorln("Can't initialize connection listener")
					return
				}
				wrapper, err := network.NewSecureSessionConnectionWrapper(server.config.ServerID(), server.keystorage)
				if err != nil {
					grpcLogger.WithError(err).Errorln("Can't initialize Secure Session wrapper")
					return
				}
				opts = append(opts, grpc.Creds(wrapper))
			}

			grpcListener := common.WrapListenerWithMetrics(listener)

			grpcServer, err := server.grpcServerFactory.New(decryptorData, opts...)

			err = errors.New("syntetic error")

			if err != nil {
				logger.WithError(err).Errorln("Can't create new grpc server")
			}
			server.grpcServer = grpcServer
			if err := grpcServer.Serve(grpcListener); err != nil {
				grpcLogger.Errorf("failed to serve: %v", err)
				server.Stop()
				return
			}
		}()
	}
	<-parentContext.Done()

	// global 'cancel' has been called called. Now we should wait (not more than specified duration) until all
	// background goroutines spawned by readerServer will finish their execution
	if timeoutExpired(&server.waitGroup, time.Second) {
		log.Errorf("Couldn't stop all background goroutines spawned by readerServer. Exited by timeout")
	}
	return
}

// timeoutExpired waits for the waitgroup for the specified max timeout.
// Returns true if waiting timed out.
func timeoutExpired(wg *sync.WaitGroup, timeout time.Duration) bool {
	c := make(chan struct{})
	go func() {
		defer close(c)
		wg.Wait()
	}()
	select {
	case <-c:
		return false // completed normally
	case <-time.After(timeout):
		return true // timed out
	}
}

// ProcessingFunc redirects processing of connection to HTTP handler or gRPC handler.
type ProcessingFunc func(context.Context, []byte, net.Conn)

func (server *ReaderServer) processHTTPConnection(parentContext context.Context, clientID []byte, connection net.Conn) {
	connection.SetDeadline(time.Now().Add(network.DefaultNetworkTimeout))
	defer connection.SetDeadline(time.Time{})

	spanCtx, span := trace.StartSpan(parentContext, "processHTTPConnection")
	defer span.End()

	// processing HTTP connection
	logger := logging.LoggerWithTrace(spanCtx, logging.GetLoggerFromContext(parentContext))
	httpLogger := logger.WithField(ConnectionTypeKey, HTTPConnectionType)
	httpLogger.Debugln("HTTP handler")

	reader := bufio.NewReader(connection)
	request, err := http.ReadRequest(reader)

	// TODO: handle keep alive

	if err != nil {
		logger.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTranslatorCantHandleHTTPRequest).
			Warningln("Got new HTTP request, but can't read it")
		server.httpDecryptor.SendResponse(logger,
			server.httpDecryptor.EmptyResponseWithStatus(request, http.StatusBadRequest), connection)
		return
	}

	response := server.httpDecryptor.ParseRequestPrepareResponse(logger, request, clientID)
	server.httpDecryptor.SendResponse(logger, response, connection)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"io"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/fatih/color"
)

type warning struct {
	format string
	vars   []interface{}
	token.Position
}

type warningPrinter struct {
	warnings []warning
}

func (w warningPrinter) print(writer io.Writer) {
	for _, warning := range w.warnings {
		coloredVars := make([]interface{}, len(warning.vars))
		for i, v := range warning.vars {
			coloredVars[i] = color.CyanString(v.(string))
		}

		message := fmt.Sprintf(warning.format, coloredVars...)

		fmt.Printf(
			"%s %s %s\n",
			color.MagentaString(warning.Position.Filename),
			color.MagentaString(fmt.Sprintf("+%d", warning.Position.Line)),
			message)
	}
}

type visitor struct {
	fileSet *token.FileSet

	lastConstSpec    string
	lastFuncDecl     string
	lastReceiverFunc string
	lastReceiver     string
	lastVarSpec      string
	typeSpecs        []string

	warnings []warning

	previousPass *visitor
}

func (v *visitor) Visit(node ast.Node) ast.Visitor {
	switch typedNode := node.(type) {
	case *ast.File:
		return v
	case *ast.GenDecl:
		if typedNode.Tok == token.CONST {
			v.checkConst(typedNode)
		} else if typedNode.Tok == token.VAR {
			v.checkVar(typedNode)
		}
		return v
	case *ast.FuncDecl:
		v.checkFunc(typedNode)
	case *ast.TypeSpec:
		v.checkType(typedNode)
	}

	return nil
}

func (v *visitor) addWarning(pos token.Pos, format string, vars ...interface{}) {
	v.warnings = append(v.warnings, warning{
		format:   format,
		vars:     vars,
		Position: v.fileSet.Position(pos),
	})
}

func (v *visitor) checkConst(node *ast.GenDecl) {
	constName := node.Specs[0].(*ast.ValueSpec).Names[0].Name

	if v.lastFuncDecl != "" {
		v.addWarning(node.Pos(), "constant %s defined after a function declaration", constName)
	}
	if len(v.typeSpecs) != 0 {
		v.addWarning(node.Pos(), "constant %s defined after a type declaration", constName)
	}
	if v.lastVarSpec != "" {
		v.addWarning(node.Pos(), "constant %s defined after a variable declaration", constName)
	}

	if strings.Compare(constName, v.lastConstSpec) == -1 {
		v.addWarning(node.Pos(), "constant %s defined after constant %s", constName, v.lastConstSpec)
	}

	v.lastConstSpec = constName
}

func (v *visitor) checkFunc(node *ast.FuncDecl) {
	if node.Recv != nil {
		v.checkFuncWithReceiver(node)
	} else {
		funcName := node.Name.Name

		if strings.Compare(funcName, v.lastFuncDecl) == -1 {
			v.addWarning(node.Pos(), "function %s defined after function %s", funcName, v.lastFuncDecl)
		}

		v.lastFuncDecl = funcName
	}
}

func (v *visitor) checkFuncWithReceiver(node *ast.FuncDecl) {
	funcName := node.Name.Name

	var receiver string
	switch typedType := node.Recv.List[0].Type.(type) {
	case *ast.Ident:
		receiver = typedType.Name
	case *ast.StarExpr:
		receiver = typedType.X.(*ast.Ident).Name
	}
	if v.lastFuncDecl != "" {
		v.addWarning(node.Pos(), "method %s.%s defined after function %s", receiver, funcName, v.lastFuncDecl)
	}
	if len(v.typeSpecs) > 0 {
		lastTypeSpec := v.typeSpecs[len(v.typeSpecs)-1]
		if v.typeDefinedInFile(receiver) && receiver != lastTypeSpec {
			v.addWarning(node.Pos(), "method %s.%s should be defined immediately after type %s", receiver, funcName, receiver)
		}
	}
	if receiver == v.lastReceiver {
		if strings.Compare(funcName, v.lastReceiverFunc) == -1 {
			v.addWarning(node.Pos(), "method %s.%s defined after method %s.%s", receiver, funcName, receiver, v.lastReceiverFunc)
		}
	}

	v.lastReceiver = receiver
	v.lastReceiverFunc = funcName
}

func (v *visitor) checkType(node *ast.TypeSpec) {
	typeName := node.Name.Name
	if v.lastFuncDecl != "" {
		v.addWarning(node.Pos(), "type declaration %s defined after a function declaration", typeName)
	}
	v.typeSpecs = append(v.typeSpecs, typeName)
}

func (v *visitor) checkVar(node *ast.GenDecl) {
	varName := node.Specs[0].(*ast.ValueSpec).Names[0].Name

	if v.lastFuncDecl != "" {
		v.addWarning(node.Pos(), "variable %s defined after a function declaration", varName)
	}
	if len(v.typeSpecs) != 0 {
		v.addWarning(node.Pos(), "variable %s defined after a type declaration", varName)
	}

	if strings.Compare(varName, v.lastVarSpec) == -1 {
		v.addWarning(node.Pos(), "variable %s defined after variable %s", varName, v.lastVarSpec)
	}

	v.lastVarSpec = varName
}

func (v *visitor) typeDefinedInFile(typeName string) bool {
	if v.previousPass == nil {
		return true
	}

	for _, definedTypeName := range v.previousPass.typeSpecs {
		if definedTypeName == typeName {
			return true
		}
	}

	return false
}

func main() {
	var allWarnings []warning

	fileSet := token.NewFileSet()

	err := filepath.Walk(os.Args[1], func(path string, info os.FileInfo, err error) error {
		if !info.IsDir() {
			return nil
		}

		base := filepath.Base(path)
		if base == "vendor" || base == ".git" || strings.HasSuffix(base, "fakes") {
			return filepath.SkipDir
		}

		packages, err := parser.ParseDir(fileSet, path, shouldParseFile, 0)
		if err != nil {
			return err
		}

		var packageNames []string
		for packageName, _ := range packages {
			packageNames = append(packageNames, packageName)
		}
		sort.Strings(packageNames)

		for _, packageName := range packageNames {
			var fileNames []string
			for fileName, _ := range packages[packageName].Files {
				fileNames = append(fileNames, fileName)
			}
			sort.Strings(fileNames)

			for _, fileName := range fileNames {
				firstPass := visitor{
					fileSet: fileSet,
				}
				ast.Walk(&firstPass, packages[packageName].Files[fileName])

				v := visitor{
					fileSet:      fileSet,
					previousPass: &firstPass,
				}
				ast.Walk(&v, packages[packageName].Files[fileName])
				allWarnings = append(allWarnings, v.warnings...)
			}
		}

		return nil
	})

	if err != nil {
		panic(err)
	}

	warningPrinter := warningPrinter{
		warnings: allWarnings,
	}
	warningPrinter.print(os.Stdout)

	if len(allWarnings) > 0 {
		os.Exit(1)
	}
}

func shouldParseFile(info os.FileInfo) bool {
	return !strings.HasSuffix(info.Name(), "_test.go")
}
<commit_msg>command line arguments are now any number of files or directories<commit_after>package main

import (
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"io"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/fatih/color"
)

type warning struct {
	format string
	vars   []interface{}
	token.Position
}

type warningPrinter struct {
	warnings []warning
}

func (w warningPrinter) print(writer io.Writer) {
	w.sortWarnings()

	for _, warning := range w.warnings {
		coloredVars := make([]interface{}, len(warning.vars))
		for i, v := range warning.vars {
			coloredVars[i] = color.CyanString(v.(string))
		}

		message := fmt.Sprintf(warning.format, coloredVars...)

		fmt.Printf(
			"%s %s %s\n",
			color.MagentaString(warning.Position.Filename),
			color.MagentaString(fmt.Sprintf("+%d", warning.Position.Line)),
			message)
	}
}

func (w warningPrinter) sortWarnings() {
	sort.Slice(w.warnings, func(i int, j int) bool {
		if w.warnings[i].Position.Filename < w.warnings[j].Position.Filename {
			return true
		}
		if w.warnings[i].Position.Filename > w.warnings[j].Position.Filename {
			return false
		}

		if w.warnings[i].Position.Line < w.warnings[j].Position.Line {
			return true
		}
		if w.warnings[i].Position.Line > w.warnings[j].Position.Line {
			return false
		}

		iMessage := fmt.Sprintf(w.warnings[i].format, w.warnings[i].vars...)
		jMessage := fmt.Sprintf(w.warnings[j].format, w.warnings[j].vars...)

		return iMessage < jMessage
	})
}

type visitor struct {
	fileSet *token.FileSet

	lastConstSpec    string
	lastFuncDecl     string
	lastReceiverFunc string
	lastReceiver     string
	lastVarSpec      string
	typeSpecs        []string

	warnings []warning

	previousPass *visitor
}

func (v *visitor) Visit(node ast.Node) ast.Visitor {
	switch typedNode := node.(type) {
	case *ast.File:
		return v
	case *ast.GenDecl:
		if typedNode.Tok == token.CONST {
			v.checkConst(typedNode)
		} else if typedNode.Tok == token.VAR {
			v.checkVar(typedNode)
		}
		return v
	case *ast.FuncDecl:
		v.checkFunc(typedNode)
	case *ast.TypeSpec:
		v.checkType(typedNode)
	}

	return nil
}

func (v *visitor) addWarning(pos token.Pos, format string, vars ...interface{}) {
	v.warnings = append(v.warnings, warning{
		format:   format,
		vars:     vars,
		Position: v.fileSet.Position(pos),
	})
}

func (v *visitor) checkConst(node *ast.GenDecl) {
	constName := node.Specs[0].(*ast.ValueSpec).Names[0].Name

	if v.lastFuncDecl != "" {
		v.addWarning(node.Pos(), "constant %s defined after a function declaration", constName)
	}
	if len(v.typeSpecs) != 0 {
		v.addWarning(node.Pos(), "constant %s defined after a type declaration", constName)
	}
	if v.lastVarSpec != "" {
		v.addWarning(node.Pos(), "constant %s defined after a variable declaration", constName)
	}

	if strings.Compare(constName, v.lastConstSpec) == -1 {
		v.addWarning(node.Pos(), "constant %s defined after constant %s", constName, v.lastConstSpec)
	}

	v.lastConstSpec = constName
}

func (v *visitor) checkFunc(node *ast.FuncDecl) {
	if node.Recv != nil {
		v.checkFuncWithReceiver(node)
	} else {
		funcName := node.Name.Name

		if strings.Compare(funcName, v.lastFuncDecl) == -1 {
			v.addWarning(node.Pos(), "function %s defined after function %s", funcName, v.lastFuncDecl)
		}

		v.lastFuncDecl = funcName
	}
}

func (v *visitor) checkFuncWithReceiver(node *ast.FuncDecl) {
	funcName := node.Name.Name

	var receiver string
	switch typedType := node.Recv.List[0].Type.(type) {
	case *ast.Ident:
		receiver = typedType.Name
	case *ast.StarExpr:
		receiver = typedType.X.(*ast.Ident).Name
	}
	if v.lastFuncDecl != "" {
		v.addWarning(node.Pos(), "method %s.%s defined after function %s", receiver, funcName, v.lastFuncDecl)
	}
	if len(v.typeSpecs) > 0 {
		lastTypeSpec := v.typeSpecs[len(v.typeSpecs)-1]
		if v.typeDefinedInFile(receiver) && receiver != lastTypeSpec {
			v.addWarning(node.Pos(), "method %s.%s should be defined immediately after type %s", receiver, funcName, receiver)
		}
	}
	if receiver == v.lastReceiver {
		if strings.Compare(funcName, v.lastReceiverFunc) == -1 {
			v.addWarning(node.Pos(), "method %s.%s defined after method %s.%s", receiver, funcName, receiver, v.lastReceiverFunc)
		}
	}

	v.lastReceiver = receiver
	v.lastReceiverFunc = funcName
}

func (v *visitor) checkType(node *ast.TypeSpec) {
	typeName := node.Name.Name
	if v.lastFuncDecl != "" {
		v.addWarning(node.Pos(), "type declaration %s defined after a function declaration", typeName)
	}
	v.typeSpecs = append(v.typeSpecs, typeName)
}

func (v *visitor) checkVar(node *ast.GenDecl) {
	varName := node.Specs[0].(*ast.ValueSpec).Names[0].Name

	if v.lastFuncDecl != "" {
		v.addWarning(node.Pos(), "variable %s defined after a function declaration", varName)
	}
	if len(v.typeSpecs) != 0 {
		v.addWarning(node.Pos(), "variable %s defined after a type declaration", varName)
	}

	if strings.Compare(varName, v.lastVarSpec) == -1 {
		v.addWarning(node.Pos(), "variable %s defined after variable %s", varName, v.lastVarSpec)
	}

	v.lastVarSpec = varName
}

func (v *visitor) typeDefinedInFile(typeName string) bool {
	if v.previousPass == nil {
		return true
	}

	for _, definedTypeName := range v.previousPass.typeSpecs {
		if definedTypeName == typeName {
			return true
		}
	}

	return false
}

func check(fileSet *token.FileSet, path string) ([]warning, error) {
	stat, err := os.Stat(path)
	if err != nil {
		return nil, err
	}

	if stat.IsDir() {
		return checkDir(fileSet, path)
	} else {
		return checkFile(fileSet, path)
	}
}

func checkDir(fileSet *token.FileSet, path string) ([]warning, error) {
	var warnings []warning

	err := filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
		if !info.IsDir() {
			return nil
		}

		if shouldSkipDir(path) {
			return filepath.SkipDir
		}

		packages, err := parser.ParseDir(fileSet, path, shouldParseFile, 0)
		if err != nil {
			return err
		}

		for _, packag := range packages {
			for _, file := range packag.Files {
				warnings = append(warnings, walkFile(fileSet, file)...)
			}
		}

		return nil
	})

	return warnings, err
}

func checkFile(fileSet *token.FileSet, path string) ([]warning, error) {
	file, err := parser.ParseFile(fileSet, path, nil, 0)
	if err != nil {
		return nil, err
	}

	return walkFile(fileSet, file), nil
}

func main() {
	var allWarnings []warning

	args := os.Args[1:]
	if args[0] == "--" {
		args = args[1:]
	}

	fileSet := token.NewFileSet()

	for _, arg := range args {
		warnings, err := check(fileSet, arg)
		if err != nil {
			panic(err)
		}
		allWarnings = append(allWarnings, warnings...)
	}

	warningPrinter := warningPrinter{
		warnings: allWarnings,
	}
	warningPrinter.print(os.Stdout)

	if len(allWarnings) > 0 {
		os.Exit(1)
	}
}

func shouldParseFile(info os.FileInfo) bool {
	return !strings.HasSuffix(info.Name(), "_test.go")
}

func shouldSkipDir(path string) bool {
	base := filepath.Base(path)
	return base == "vendor" || base == ".git" || strings.HasSuffix(base, "fakes")
}

func walkFile(fileSet *token.FileSet, file *ast.File) []warning {
	firstPass := visitor{
		fileSet: fileSet,
	}
	ast.Walk(&firstPass, file)

	v := visitor{
		fileSet:      fileSet,
		previousPass: &firstPass,
	}
	ast.Walk(&v, file)

	return v.warnings
}
<|endoftext|>


<commit_before>package users

import (
	"sort"

	"github.com/aerogo/aero"
	"github.com/animenotifier/arn"
	"github.com/animenotifier/notify.moe/components"
	"github.com/animenotifier/notify.moe/utils"
)

// Active ...
func Active(ctx *aero.Context) string {
	users := arn.FilterUsers(func(user *arn.User) bool {
		return user.HasAvatar() && user.HasNick() && user.IsActive()
	})

	followCount := arn.UserFollowerCountMap()

	sort.Slice(users, func(i, j int) bool {
		if users[i].HasAvatar() != users[j].HasAvatar() {
			return users[i].HasAvatar()
		}

		followersA := followCount[users[i].ID]
		followersB := followCount[users[j].ID]

		if followersA == followersB {
			return users[i].Nick < users[j].Nick
		}

		return followersA > followersB
	})

	return ctx.HTML(components.Users(users, ctx.URI()))
}

// Pro ...
func Pro(ctx *aero.Context) string {
	users := arn.FilterUsers(func(user *arn.User) bool {
		return user.IsPro()
	})

	sort.Slice(users, func(i, j int) bool {
		return users[i].Registered > users[j].Registered
	})

	return ctx.HTML(components.ProUsers(users, ctx.URI()))
}

// Editors ...
func Editors(ctx *aero.Context) string {
	score := map[string]int{}
	users := []*arn.User{}

	for entry := range arn.StreamEditLogEntries() {
		entryScore := entry.EditorScore()

		if entryScore == 0 {
			continue
		}

		current, exists := score[entry.UserID]

		if !exists {
			users = append(users, entry.User())
		}

		score[entry.UserID] = current + entryScore
	}

	for ignore := range arn.StreamIgnoreAnimeDifferences() {
		score[ignore.CreatedBy] += arn.IgnoreAnimeDifferenceEditorScore
	}

	sort.Slice(users, func(i, j int) bool {
		scoreA := score[users[i].ID]
		scoreB := score[users[j].ID]

		if scoreA == scoreB {
			return users[i].Registered > users[j].Registered
		}

		return scoreA > scoreB
	})

	return ctx.HTML(components.EditorRankingList(users, score, ctx.URI()))
}

// ActiveNoAvatar ...
func ActiveNoAvatar(ctx *aero.Context) string {
	users := arn.FilterUsers(func(user *arn.User) bool {
		return user.IsActive() && !user.HasAvatar()
	})

	followCount := arn.UserFollowerCountMap()

	sort.Slice(users, func(i, j int) bool {
		if users[i].HasAvatar() != users[j].HasAvatar() {
			return users[i].HasAvatar()
		}

		followersA := followCount[users[i].ID]
		followersB := followCount[users[j].ID]

		if followersA == followersB {
			return users[i].Nick < users[j].Nick
		}

		return followersA > followersB
	})

	return ctx.HTML(components.Users(users, ctx.URI()))
}

// Osu ...
func Osu(ctx *aero.Context) string {
	users := arn.FilterUsers(func(user *arn.User) bool {
		return user.HasAvatar() && user.HasNick() && user.IsActive() && user.Accounts.Osu.PP > 0
	})

	// Sort by pp
	sort.Slice(users, func(i, j int) bool {
		return users[i].Accounts.Osu.PP > users[j].Accounts.Osu.PP
	})

	if len(users) > 50 {
		users = users[:50]
	}

	return ctx.HTML(components.OsuRankingList(users, ctx.URI()))
}

// Overwatch ...
func Overwatch(ctx *aero.Context) string {
	users := arn.FilterUsers(func(user *arn.User) bool {
		return user.HasAvatar() && user.HasNick() && user.IsActive() && user.Accounts.Overwatch.SkillRating > 0
	})

	// Sort by Skill Ratings
	sort.Slice(users, func(i, j int) bool {
		return users[i].Accounts.Overwatch.SkillRating > users[j].Accounts.Overwatch.SkillRating
	})

	if len(users) > 50 {
		users = users[:50]
	}

	return ctx.HTML(components.OverwatchRankingList(users, ctx.URI()))
}

// FFXIV ...
func FFXIV(ctx *aero.Context) string {
	users := arn.FilterUsers(func(user *arn.User) bool {
		return user.HasAvatar() && user.HasNick() && user.IsActive() && user.Accounts.FinalFantasyXIV.ItemLevel > 0
	})

	// Sort by item level
	sort.Slice(users, func(i, j int) bool {
		return users[i].Accounts.FinalFantasyXIV.ItemLevel > users[j].Accounts.FinalFantasyXIV.ItemLevel
	})

	if len(users) > 50 {
		users = users[:50]
	}

	return ctx.HTML(components.FinalFantasyXIVRankingList(users, ctx.URI()))
}

// Staff ...
func Staff(ctx *aero.Context) string {
	users := arn.FilterUsers(func(user *arn.User) bool {
		return user.HasAvatar() && user.HasNick() && user.IsActive() && user.Role != ""
	})

	// Make order deterministic
	sort.Slice(users, func(i, j int) bool {
		return users[i].Nick < users[j].Nick
	})

	admins := &utils.UserList{
		Name:  "Developer",
		Users: []*arn.User{},
	}

	contributors := &utils.UserList{
		Name:  "Contributors",
		Users: []*arn.User{},
	}

	// contributors.Users = append(contributors.Users, )

	editors := &utils.UserList{
		Name:  "Editors",
		Users: []*arn.User{},
	}

	contributorIDs := []string{
		"VJOK1ckvx", // Scott
		"SUQOAFFkR", // Allen
		"KQgtMWOiR", // Franksks
		"9NMYrAHiR", // Amatrelan
	}

	for _, user := range users {
		if user.Role == "admin" {
			admins.Users = append(admins.Users, user)
			continue
		}

		if arn.Contains(contributorIDs, user.ID) {
			contributors.Users = append(contributors.Users, user)
			continue
		}

		if user.Role == "editor" {
			editors.Users = append(editors.Users, user)
			continue
		}
	}

	userLists := []*utils.UserList{
		admins,
		contributors,
		editors,
	}

	return ctx.HTML(components.UserLists(userLists, ctx.URI()) + components.StaffRecruitment())
}
<commit_msg>Limit game ranking lists to the top 10<commit_after>package users

import (
	"sort"

	"github.com/aerogo/aero"
	"github.com/animenotifier/arn"
	"github.com/animenotifier/notify.moe/components"
	"github.com/animenotifier/notify.moe/utils"
)

// Active ...
func Active(ctx *aero.Context) string {
	users := arn.FilterUsers(func(user *arn.User) bool {
		return user.HasAvatar() && user.HasNick() && user.IsActive()
	})

	followCount := arn.UserFollowerCountMap()

	sort.Slice(users, func(i, j int) bool {
		if users[i].HasAvatar() != users[j].HasAvatar() {
			return users[i].HasAvatar()
		}

		followersA := followCount[users[i].ID]
		followersB := followCount[users[j].ID]

		if followersA == followersB {
			return users[i].Nick < users[j].Nick
		}

		return followersA > followersB
	})

	return ctx.HTML(components.Users(users, ctx.URI()))
}

// Pro ...
func Pro(ctx *aero.Context) string {
	users := arn.FilterUsers(func(user *arn.User) bool {
		return user.IsPro()
	})

	sort.Slice(users, func(i, j int) bool {
		return users[i].Registered > users[j].Registered
	})

	return ctx.HTML(components.ProUsers(users, ctx.URI()))
}

// Editors ...
func Editors(ctx *aero.Context) string {
	score := map[string]int{}
	users := []*arn.User{}

	for entry := range arn.StreamEditLogEntries() {
		entryScore := entry.EditorScore()

		if entryScore == 0 {
			continue
		}

		current, exists := score[entry.UserID]

		if !exists {
			users = append(users, entry.User())
		}

		score[entry.UserID] = current + entryScore
	}

	for ignore := range arn.StreamIgnoreAnimeDifferences() {
		score[ignore.CreatedBy] += arn.IgnoreAnimeDifferenceEditorScore
	}

	sort.Slice(users, func(i, j int) bool {
		scoreA := score[users[i].ID]
		scoreB := score[users[j].ID]

		if scoreA == scoreB {
			return users[i].Registered > users[j].Registered
		}

		return scoreA > scoreB
	})

	if len(users) > 10 {
		users = users[:10]
	}

	return ctx.HTML(components.EditorRankingList(users, score, ctx.URI()))
}

// ActiveNoAvatar ...
func ActiveNoAvatar(ctx *aero.Context) string {
	users := arn.FilterUsers(func(user *arn.User) bool {
		return user.IsActive() && !user.HasAvatar()
	})

	followCount := arn.UserFollowerCountMap()

	sort.Slice(users, func(i, j int) bool {
		if users[i].HasAvatar() != users[j].HasAvatar() {
			return users[i].HasAvatar()
		}

		followersA := followCount[users[i].ID]
		followersB := followCount[users[j].ID]

		if followersA == followersB {
			return users[i].Nick < users[j].Nick
		}

		return followersA > followersB
	})

	return ctx.HTML(components.Users(users, ctx.URI()))
}

// Osu ...
func Osu(ctx *aero.Context) string {
	users := arn.FilterUsers(func(user *arn.User) bool {
		return user.HasAvatar() && user.HasNick() && user.IsActive() && user.Accounts.Osu.PP > 0
	})

	// Sort by pp
	sort.Slice(users, func(i, j int) bool {
		return users[i].Accounts.Osu.PP > users[j].Accounts.Osu.PP
	})

	if len(users) > 10 {
		users = users[:10]
	}

	return ctx.HTML(components.OsuRankingList(users, ctx.URI()))
}

// Overwatch ...
func Overwatch(ctx *aero.Context) string {
	users := arn.FilterUsers(func(user *arn.User) bool {
		return user.HasAvatar() && user.HasNick() && user.IsActive() && user.Accounts.Overwatch.SkillRating > 0
	})

	// Sort by Skill Ratings
	sort.Slice(users, func(i, j int) bool {
		return users[i].Accounts.Overwatch.SkillRating > users[j].Accounts.Overwatch.SkillRating
	})

	if len(users) > 10 {
		users = users[:10]
	}

	return ctx.HTML(components.OverwatchRankingList(users, ctx.URI()))
}

// FFXIV ...
func FFXIV(ctx *aero.Context) string {
	users := arn.FilterUsers(func(user *arn.User) bool {
		return user.HasAvatar() && user.HasNick() && user.IsActive() && user.Accounts.FinalFantasyXIV.ItemLevel > 0
	})

	// Sort by item level
	sort.Slice(users, func(i, j int) bool {
		return users[i].Accounts.FinalFantasyXIV.ItemLevel > users[j].Accounts.FinalFantasyXIV.ItemLevel
	})

	if len(users) > 10 {
		users = users[:10]
	}

	return ctx.HTML(components.FinalFantasyXIVRankingList(users, ctx.URI()))
}

// Staff ...
func Staff(ctx *aero.Context) string {
	users := arn.FilterUsers(func(user *arn.User) bool {
		return user.HasAvatar() && user.HasNick() && user.IsActive() && user.Role != ""
	})

	// Make order deterministic
	sort.Slice(users, func(i, j int) bool {
		return users[i].Nick < users[j].Nick
	})

	admins := &utils.UserList{
		Name:  "Developer",
		Users: []*arn.User{},
	}

	contributors := &utils.UserList{
		Name:  "Contributors",
		Users: []*arn.User{},
	}

	// contributors.Users = append(contributors.Users, )

	editors := &utils.UserList{
		Name:  "Editors",
		Users: []*arn.User{},
	}

	contributorIDs := []string{
		"VJOK1ckvx", // Scott
		"SUQOAFFkR", // Allen
		"KQgtMWOiR", // Franksks
		"9NMYrAHiR", // Amatrelan
	}

	for _, user := range users {
		if user.Role == "admin" {
			admins.Users = append(admins.Users, user)
			continue
		}

		if arn.Contains(contributorIDs, user.ID) {
			contributors.Users = append(contributors.Users, user)
			continue
		}

		if user.Role == "editor" {
			editors.Users = append(editors.Users, user)
			continue
		}
	}

	userLists := []*utils.UserList{
		admins,
		contributors,
		editors,
	}

	return ctx.HTML(components.UserLists(userLists, ctx.URI()) + components.StaffRecruitment())
}
<|endoftext|>


<commit_before>package confDir

import (
	"path"
)

var (
	configDir = path.Join("Library","Application Support", "containerum")
	logDir := path.Join("Library", "Logs")
)
<commit_msg>fix logdir init<commit_after>package confDir

import (
	"path"
)

var (
	configDir = path.Join("Library", "Application Support", "containerum")
	logDir    = path.Join("Library", "Logs")
)
<|endoftext|>


<commit_before>package sql

import (
	"crypto/rand"
	"crypto/sha256"
	"database/sql"
	"encoding/binary"
	"encoding/hex"
	"errors"
	"io"
	"os"
	"path"
	"sync"
	"time"

	"github.com/agl/ed25519"
	"github.com/repbin/repbin/hashcash"
	"github.com/repbin/repbin/message"
	"github.com/repbin/repbin/utils/keyproof"
)

// Version of this release
const Version = "0.0.1 very alpha"

var shardRand = make([]byte, 16)

func init() {
	io.ReadFull(rand.Reader, shardRand)
}

var (
	// ErrNoModify is returned if a row was not modified
	ErrNoModify = errors.New("storage: Row not modified")
)

// CurrentTime returns the current time in UTC
var CurrentTime = func() int64 { return time.Now().UTC().Unix() }

// MessageDB implements a message database
type MessageDB struct {
	db                     *sql.DB
	NumShards              uint64
	shardMutexes           []*sync.Mutex
	dir                    string
	driver                 string
	queries                map[string]string
	signerInsertQ          *sql.Stmt
	signerSelectPublicKeyQ *sql.Stmt
	signerSelectIDQ        *sql.Stmt
	signerUpdateQ          *sql.Stmt
	signerUpdateInsertQ    *sql.Stmt
	signerAddMessageQ      *sql.Stmt
	signerDelMessageQ      *sql.Stmt
	signerPrepareExpireQ   *sql.Stmt
	signerExpireQ          *sql.Stmt
	signerSetExpireQ       *sql.Stmt
	peerInsertQ            *sql.Stmt
	peerUpdateStatQ        *sql.Stmt
	peerUpdateTokenQ       *sql.Stmt
	peerUpdateNotifyQ      *sql.Stmt
	peerSelectQ            *sql.Stmt
	nextMessageCounterQ    *sql.Stmt
	incrMessageCounterQ    *sql.Stmt
	expireMessageCounterQ  *sql.Stmt
	insertMessageCounterQ  *sql.Stmt
	insertMessageQ         *sql.Stmt
	selectMessageQ         *sql.Stmt
	deleteMessageQ         *sql.Stmt
	updateExpireMessageQ   *sql.Stmt
	selectExpireMessageQ   *sql.Stmt
	globalIndexAddQ        *sql.Stmt
	getKeyIndexQ           *sql.Stmt
	getGlobalIndexQ        *sql.Stmt
	messageBlobInsertQ     *sql.Stmt
	messageBlobSelectQ     *sql.Stmt
	messageBlobDeleteQ     *sql.Stmt
	messageExistInsertQ    *sql.Stmt
	messageExistSelectQ    *sql.Stmt
	messageExistExpireQ    *sql.Stmt
}

// New returns a new message database. driver is the database driver to use,
// url the database url. dir is the optional directory in which to store the
// raw message blobs. If dir is empty blobs will be stored in the database (which
// may not be a good idea at all). Shards is the number of lock shards to use
// for sequence generation (memory/lock-probability tradeoff)
func New(driver, url, dir string, shards int) (*MessageDB, error) {
	var db *sql.DB
	var err error
	if driver == "sqlite3" {
		os.MkdirAll(path.Dir(url), 0700)
	}
	db, err = sql.Open(driver, url)
	if err != nil {
		return nil, err
	}
	mdb := &MessageDB{
		NumShards: uint64(shards),
		queries:   queries[driver],
		db:        db,
		dir:       dir,
		driver:    driver,
	}

	if _, err := mdb.db.Exec(mdb.queries["SignerCreate"]); err != nil {
		return nil, err
	}
	if _, err := mdb.db.Exec(mdb.queries["PeerCreate"]); err != nil {
		return nil, err
	}
	if _, err := mdb.db.Exec(mdb.queries["MessageCreate"]); err != nil {
		return nil, err
	}
	if _, err := mdb.db.Exec(mdb.queries["GlobalIndexCreate"]); err != nil {
		return nil, err
	}
	if _, err := mdb.db.Exec(mdb.queries["messageBlobCreate"]); err != nil {
		return nil, err
	}
	if _, err := mdb.db.Exec(mdb.queries["MessageCounterCreate"]); err != nil {
		return nil, err
	}
	if _, err := mdb.db.Exec(mdb.queries["messageExistCreate"]); err != nil {
		return nil, err
	}
	if mdb.signerInsertQ, err = mdb.db.Prepare(mdb.queries["SignerInsert"]); err != nil {
		return nil, err
	}
	if mdb.signerSelectPublicKeyQ, err = mdb.db.Prepare(mdb.queries["SelectSignerPublicKey"]); err != nil {
		return nil, err
	}
	if mdb.signerSelectIDQ, err = mdb.db.Prepare(mdb.queries["SelectSignerID"]); err != nil {
		return nil, err
	}
	if mdb.signerUpdateQ, err = mdb.db.Prepare(mdb.queries["UpdateSigner"]); err != nil {
		return nil, err
	}
	if mdb.signerAddMessageQ, err = mdb.db.Prepare(mdb.queries["AddMessageSigner"]); err != nil {
		return nil, err
	}
	if mdb.signerDelMessageQ, err = mdb.db.Prepare(mdb.queries["DelMessageSigner"]); err != nil {
		return nil, err
	}
	if mdb.signerPrepareExpireQ, err = mdb.db.Prepare(mdb.queries["PrepareExpireSigner"]); err != nil {
		return nil, err
	}
	if mdb.signerExpireQ, err = mdb.db.Prepare(mdb.queries["DeleteExpireSigner"]); err != nil {
		return nil, err
	}

	if mdb.peerInsertQ, err = mdb.db.Prepare(mdb.queries["InsertPeer"]); err != nil {
		return nil, err
	}
	if mdb.peerUpdateStatQ, err = mdb.db.Prepare(mdb.queries["UpdateStatPeer"]); err != nil {
		return nil, err
	}
	if mdb.peerUpdateNotifyQ, err = mdb.db.Prepare(mdb.queries["UpdateNotifyPeer"]); err != nil {
		return nil, err
	}
	if mdb.peerUpdateTokenQ, err = mdb.db.Prepare(mdb.queries["UpdateTokenPeer"]); err != nil {
		return nil, err
	}
	if mdb.peerSelectQ, err = mdb.db.Prepare(mdb.queries["SelectPeer"]); err != nil {
		return nil, err
	}

	if mdb.nextMessageCounterQ, err = mdb.db.Prepare(mdb.queries["NextMessageCounter"]); err != nil {
		return nil, err
	}
	if mdb.incrMessageCounterQ, err = mdb.db.Prepare(mdb.queries["IncreaseMessageCounter"]); err != nil {
		return nil, err
	}
	if mdb.insertMessageCounterQ, err = mdb.db.Prepare(mdb.queries["InsertMessageCounter"]); err != nil {
		return nil, err
	}
	if mdb.expireMessageCounterQ, err = mdb.db.Prepare(mdb.queries["ExpireMessageCounter"]); err != nil {
		return nil, err
	}

	if mdb.insertMessageQ, err = mdb.db.Prepare(mdb.queries["InsertMessage"]); err != nil {
		return nil, err
	}
	if mdb.selectMessageQ, err = mdb.db.Prepare(mdb.queries["SelectMessage"]); err != nil {
		return nil, err
	}
	if mdb.deleteMessageQ, err = mdb.db.Prepare(mdb.queries["DeleteMessage"]); err != nil {
		return nil, err
	}
	if mdb.updateExpireMessageQ, err = mdb.db.Prepare(mdb.queries["UpdateExpireMessage"]); err != nil {
		return nil, err
	}
	if mdb.selectExpireMessageQ, err = mdb.db.Prepare(mdb.queries["SelectExpireMessage"]); err != nil {
		return nil, err
	}

	if mdb.globalIndexAddQ, err = mdb.db.Prepare(mdb.queries["globalIndexAdd"]); err != nil {
		return nil, err
	}
	if mdb.getKeyIndexQ, err = mdb.db.Prepare(mdb.queries["getKeyIndex"]); err != nil {
		return nil, err
	}
	if mdb.getGlobalIndexQ, err = mdb.db.Prepare(mdb.queries["getGlobalIndex"]); err != nil {
		return nil, err
	}

	if mdb.messageBlobInsertQ, err = mdb.db.Prepare(mdb.queries["messageBlobInsert"]); err != nil {
		return nil, err
	}
	if mdb.messageBlobSelectQ, err = mdb.db.Prepare(mdb.queries["messageBlobSelect"]); err != nil {
		return nil, err
	}
	if mdb.messageBlobDeleteQ, err = mdb.db.Prepare(mdb.queries["messageBlobDelete"]); err != nil {
		return nil, err
	}

	if mdb.messageExistInsertQ, err = mdb.db.Prepare(mdb.queries["messageExistInsert"]); err != nil {
		return nil, err
	}
	if mdb.messageExistSelectQ, err = mdb.db.Prepare(mdb.queries["messageExistSelect"]); err != nil {
		return nil, err
	}
	if mdb.messageExistExpireQ, err = mdb.db.Prepare(mdb.queries["messageExistExpire"]); err != nil {
		return nil, err
	}

	if driver == "mysql" {
		if mdb.signerUpdateInsertQ, err = mdb.db.Prepare(mdb.queries["UpdateOrInsertSigner"]); err != nil {
			return nil, err
		}
	}
	if dir != "" {
		if err := os.MkdirAll(dir, 0700); err != nil {
			return nil, err
		}
	}
	mdb.setMutexes()
	return mdb, nil
}

// LockShard locks shard s
func (db *MessageDB) LockShard(s []byte) {
	db.shardMutexes[db.calcShard(s)].Lock()
}

// UnlockShard unlocks shard s, if locked. Runtime error otherwise
func (db *MessageDB) UnlockShard(s []byte) {
	db.shardMutexes[db.calcShard(s)].Unlock()
}

func (db *MessageDB) setMutexes() {
	db.shardMutexes = make([]*sync.Mutex, db.NumShards)
	for i := range db.shardMutexes {
		db.shardMutexes[i] = new(sync.Mutex)
	}
}

// Close the database
func (db *MessageDB) Close() error {
	return db.db.Close()
}

func toHex(d []byte) string {
	return hex.EncodeToString(d)
}

func fromHex(s string) []byte {
	d, _ := hex.DecodeString(s)
	return d
}

func sliceToSignerPubKey(d []byte) *[message.SignerPubKeySize]byte {
	r := new([message.SignerPubKeySize]byte)
	copy(r[:], d)
	return r
}

func sliceToNonce(d []byte) *[hashcash.NonceSize]byte {
	r := new([hashcash.NonceSize]byte)
	copy(r[:], d)
	return r
}

func sliceToProofTokenSigned(d []byte) *[keyproof.ProofTokenSignedSize]byte {
	r := new([keyproof.ProofTokenSignedSize]byte)
	copy(r[:], d)
	return r
}

func sliceToEDPublicKey(d []byte) *[ed25519.PublicKeySize]byte {
	r := new([ed25519.PublicKeySize]byte)
	copy(r[:], d)
	return r
}

func sliceToCurve25519Key(d []byte) *message.Curve25519Key {
	r := new(message.Curve25519Key)
	copy(r[:], d)
	return r
}

func sliceToMessageID(d []byte) *[message.MessageIDSize]byte {
	r := new([message.MessageIDSize]byte)
	copy(r[:], d)
	return r
}

func boolToInt(b bool) int {
	if b {
		return 1
	}
	return 0
}

func intToBool(i int) bool {
	if i > 0 {
		return true
	}
	return false
}

func updateConvertNilError(res sql.Result, err error) error {
	if err != nil {
		return err
	}
	n, err := res.RowsAffected()
	if err != nil {
		return err
	}
	if n < 1 {
		return ErrNoModify
	}
	return nil
}

func (db *MessageDB) calcShard(d []byte) uint64 {
	h := sha256.Sum256(append(shardRand, d...))
	return binary.BigEndian.Uint64(h[:]) % db.NumShards
}
<commit_msg>err fix sqlite mkdir<commit_after>package sql

import (
	"crypto/rand"
	"crypto/sha256"
	"database/sql"
	"encoding/binary"
	"encoding/hex"
	"errors"
	"io"
	"os"
	"path"
	"sync"
	"time"

	"github.com/agl/ed25519"
	"github.com/repbin/repbin/hashcash"
	"github.com/repbin/repbin/message"
	"github.com/repbin/repbin/utils/keyproof"
)

// Version of this release
const Version = "0.0.1 very alpha"

var shardRand = make([]byte, 16)

func init() {
	io.ReadFull(rand.Reader, shardRand)
}

var (
	// ErrNoModify is returned if a row was not modified
	ErrNoModify = errors.New("storage: Row not modified")
)

// CurrentTime returns the current time in UTC
var CurrentTime = func() int64 { return time.Now().UTC().Unix() }

// MessageDB implements a message database
type MessageDB struct {
	db                     *sql.DB
	NumShards              uint64
	shardMutexes           []*sync.Mutex
	dir                    string
	driver                 string
	queries                map[string]string
	signerInsertQ          *sql.Stmt
	signerSelectPublicKeyQ *sql.Stmt
	signerSelectIDQ        *sql.Stmt
	signerUpdateQ          *sql.Stmt
	signerUpdateInsertQ    *sql.Stmt
	signerAddMessageQ      *sql.Stmt
	signerDelMessageQ      *sql.Stmt
	signerPrepareExpireQ   *sql.Stmt
	signerExpireQ          *sql.Stmt
	signerSetExpireQ       *sql.Stmt
	peerInsertQ            *sql.Stmt
	peerUpdateStatQ        *sql.Stmt
	peerUpdateTokenQ       *sql.Stmt
	peerUpdateNotifyQ      *sql.Stmt
	peerSelectQ            *sql.Stmt
	nextMessageCounterQ    *sql.Stmt
	incrMessageCounterQ    *sql.Stmt
	expireMessageCounterQ  *sql.Stmt
	insertMessageCounterQ  *sql.Stmt
	insertMessageQ         *sql.Stmt
	selectMessageQ         *sql.Stmt
	deleteMessageQ         *sql.Stmt
	updateExpireMessageQ   *sql.Stmt
	selectExpireMessageQ   *sql.Stmt
	globalIndexAddQ        *sql.Stmt
	getKeyIndexQ           *sql.Stmt
	getGlobalIndexQ        *sql.Stmt
	messageBlobInsertQ     *sql.Stmt
	messageBlobSelectQ     *sql.Stmt
	messageBlobDeleteQ     *sql.Stmt
	messageExistInsertQ    *sql.Stmt
	messageExistSelectQ    *sql.Stmt
	messageExistExpireQ    *sql.Stmt
}

// New returns a new message database. driver is the database driver to use,
// url the database url. dir is the optional directory in which to store the
// raw message blobs. If dir is empty blobs will be stored in the database (which
// may not be a good idea at all). Shards is the number of lock shards to use
// for sequence generation (memory/lock-probability tradeoff)
func New(driver, url, dir string, shards int) (*MessageDB, error) {
	var db *sql.DB
	var err error
	if driver == "sqlite3" {
		if err := os.MkdirAll(path.Dir(url), 0700); err != nil {
			return nil, err
		}
	}
	db, err = sql.Open(driver, url)
	if err != nil {
		return nil, err
	}
	mdb := &MessageDB{
		NumShards: uint64(shards),
		queries:   queries[driver],
		db:        db,
		dir:       dir,
		driver:    driver,
	}

	if _, err := mdb.db.Exec(mdb.queries["SignerCreate"]); err != nil {
		return nil, err
	}
	if _, err := mdb.db.Exec(mdb.queries["PeerCreate"]); err != nil {
		return nil, err
	}
	if _, err := mdb.db.Exec(mdb.queries["MessageCreate"]); err != nil {
		return nil, err
	}
	if _, err := mdb.db.Exec(mdb.queries["GlobalIndexCreate"]); err != nil {
		return nil, err
	}
	if _, err := mdb.db.Exec(mdb.queries["messageBlobCreate"]); err != nil {
		return nil, err
	}
	if _, err := mdb.db.Exec(mdb.queries["MessageCounterCreate"]); err != nil {
		return nil, err
	}
	if _, err := mdb.db.Exec(mdb.queries["messageExistCreate"]); err != nil {
		return nil, err
	}
	if mdb.signerInsertQ, err = mdb.db.Prepare(mdb.queries["SignerInsert"]); err != nil {
		return nil, err
	}
	if mdb.signerSelectPublicKeyQ, err = mdb.db.Prepare(mdb.queries["SelectSignerPublicKey"]); err != nil {
		return nil, err
	}
	if mdb.signerSelectIDQ, err = mdb.db.Prepare(mdb.queries["SelectSignerID"]); err != nil {
		return nil, err
	}
	if mdb.signerUpdateQ, err = mdb.db.Prepare(mdb.queries["UpdateSigner"]); err != nil {
		return nil, err
	}
	if mdb.signerAddMessageQ, err = mdb.db.Prepare(mdb.queries["AddMessageSigner"]); err != nil {
		return nil, err
	}
	if mdb.signerDelMessageQ, err = mdb.db.Prepare(mdb.queries["DelMessageSigner"]); err != nil {
		return nil, err
	}
	if mdb.signerPrepareExpireQ, err = mdb.db.Prepare(mdb.queries["PrepareExpireSigner"]); err != nil {
		return nil, err
	}
	if mdb.signerExpireQ, err = mdb.db.Prepare(mdb.queries["DeleteExpireSigner"]); err != nil {
		return nil, err
	}

	if mdb.peerInsertQ, err = mdb.db.Prepare(mdb.queries["InsertPeer"]); err != nil {
		return nil, err
	}
	if mdb.peerUpdateStatQ, err = mdb.db.Prepare(mdb.queries["UpdateStatPeer"]); err != nil {
		return nil, err
	}
	if mdb.peerUpdateNotifyQ, err = mdb.db.Prepare(mdb.queries["UpdateNotifyPeer"]); err != nil {
		return nil, err
	}
	if mdb.peerUpdateTokenQ, err = mdb.db.Prepare(mdb.queries["UpdateTokenPeer"]); err != nil {
		return nil, err
	}
	if mdb.peerSelectQ, err = mdb.db.Prepare(mdb.queries["SelectPeer"]); err != nil {
		return nil, err
	}

	if mdb.nextMessageCounterQ, err = mdb.db.Prepare(mdb.queries["NextMessageCounter"]); err != nil {
		return nil, err
	}
	if mdb.incrMessageCounterQ, err = mdb.db.Prepare(mdb.queries["IncreaseMessageCounter"]); err != nil {
		return nil, err
	}
	if mdb.insertMessageCounterQ, err = mdb.db.Prepare(mdb.queries["InsertMessageCounter"]); err != nil {
		return nil, err
	}
	if mdb.expireMessageCounterQ, err = mdb.db.Prepare(mdb.queries["ExpireMessageCounter"]); err != nil {
		return nil, err
	}

	if mdb.insertMessageQ, err = mdb.db.Prepare(mdb.queries["InsertMessage"]); err != nil {
		return nil, err
	}
	if mdb.selectMessageQ, err = mdb.db.Prepare(mdb.queries["SelectMessage"]); err != nil {
		return nil, err
	}
	if mdb.deleteMessageQ, err = mdb.db.Prepare(mdb.queries["DeleteMessage"]); err != nil {
		return nil, err
	}
	if mdb.updateExpireMessageQ, err = mdb.db.Prepare(mdb.queries["UpdateExpireMessage"]); err != nil {
		return nil, err
	}
	if mdb.selectExpireMessageQ, err = mdb.db.Prepare(mdb.queries["SelectExpireMessage"]); err != nil {
		return nil, err
	}

	if mdb.globalIndexAddQ, err = mdb.db.Prepare(mdb.queries["globalIndexAdd"]); err != nil {
		return nil, err
	}
	if mdb.getKeyIndexQ, err = mdb.db.Prepare(mdb.queries["getKeyIndex"]); err != nil {
		return nil, err
	}
	if mdb.getGlobalIndexQ, err = mdb.db.Prepare(mdb.queries["getGlobalIndex"]); err != nil {
		return nil, err
	}

	if mdb.messageBlobInsertQ, err = mdb.db.Prepare(mdb.queries["messageBlobInsert"]); err != nil {
		return nil, err
	}
	if mdb.messageBlobSelectQ, err = mdb.db.Prepare(mdb.queries["messageBlobSelect"]); err != nil {
		return nil, err
	}
	if mdb.messageBlobDeleteQ, err = mdb.db.Prepare(mdb.queries["messageBlobDelete"]); err != nil {
		return nil, err
	}

	if mdb.messageExistInsertQ, err = mdb.db.Prepare(mdb.queries["messageExistInsert"]); err != nil {
		return nil, err
	}
	if mdb.messageExistSelectQ, err = mdb.db.Prepare(mdb.queries["messageExistSelect"]); err != nil {
		return nil, err
	}
	if mdb.messageExistExpireQ, err = mdb.db.Prepare(mdb.queries["messageExistExpire"]); err != nil {
		return nil, err
	}

	if driver == "mysql" {
		if mdb.signerUpdateInsertQ, err = mdb.db.Prepare(mdb.queries["UpdateOrInsertSigner"]); err != nil {
			return nil, err
		}
	}
	if dir != "" {
		if err := os.MkdirAll(dir, 0700); err != nil {
			return nil, err
		}
	}
	mdb.setMutexes()
	return mdb, nil
}

// LockShard locks shard s
func (db *MessageDB) LockShard(s []byte) {
	db.shardMutexes[db.calcShard(s)].Lock()
}

// UnlockShard unlocks shard s, if locked. Runtime error otherwise
func (db *MessageDB) UnlockShard(s []byte) {
	db.shardMutexes[db.calcShard(s)].Unlock()
}

func (db *MessageDB) setMutexes() {
	db.shardMutexes = make([]*sync.Mutex, db.NumShards)
	for i := range db.shardMutexes {
		db.shardMutexes[i] = new(sync.Mutex)
	}
}

// Close the database
func (db *MessageDB) Close() error {
	return db.db.Close()
}

func toHex(d []byte) string {
	return hex.EncodeToString(d)
}

func fromHex(s string) []byte {
	d, _ := hex.DecodeString(s)
	return d
}

func sliceToSignerPubKey(d []byte) *[message.SignerPubKeySize]byte {
	r := new([message.SignerPubKeySize]byte)
	copy(r[:], d)
	return r
}

func sliceToNonce(d []byte) *[hashcash.NonceSize]byte {
	r := new([hashcash.NonceSize]byte)
	copy(r[:], d)
	return r
}

func sliceToProofTokenSigned(d []byte) *[keyproof.ProofTokenSignedSize]byte {
	r := new([keyproof.ProofTokenSignedSize]byte)
	copy(r[:], d)
	return r
}

func sliceToEDPublicKey(d []byte) *[ed25519.PublicKeySize]byte {
	r := new([ed25519.PublicKeySize]byte)
	copy(r[:], d)
	return r
}

func sliceToCurve25519Key(d []byte) *message.Curve25519Key {
	r := new(message.Curve25519Key)
	copy(r[:], d)
	return r
}

func sliceToMessageID(d []byte) *[message.MessageIDSize]byte {
	r := new([message.MessageIDSize]byte)
	copy(r[:], d)
	return r
}

func boolToInt(b bool) int {
	if b {
		return 1
	}
	return 0
}

func intToBool(i int) bool {
	if i > 0 {
		return true
	}
	return false
}

func updateConvertNilError(res sql.Result, err error) error {
	if err != nil {
		return err
	}
	n, err := res.RowsAffected()
	if err != nil {
		return err
	}
	if n < 1 {
		return ErrNoModify
	}
	return nil
}

func (db *MessageDB) calcShard(d []byte) uint64 {
	h := sha256.Sum256(append(shardRand, d...))
	return binary.BigEndian.Uint64(h[:]) % db.NumShards
}
<|endoftext|>


<commit_before>package main_test

import (
	"encoding/json"
	"fmt"
	"net/url"
	"os"
	"os/exec"
	"strings"
	"testing"
	"time"

	"github.com/cloudfoundry/gunk/diegonats"
	"github.com/cloudfoundry/gunk/diegonats/gnatsdrunner"
	"github.com/cloudfoundry/storeadapter"
	"github.com/cloudfoundry/storeadapter/storerunner/etcdstorerunner"
	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/config"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
	"github.com/pivotal-golang/lager/lagertest"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"

	"github.com/cloudfoundry-incubator/bbs"
	bbstestrunner "github.com/cloudfoundry-incubator/bbs/cmd/bbs/testrunner"
	"github.com/cloudfoundry-incubator/consuladapter/consulrunner"
)

const heartbeatInterval = 1 * time.Second

var (
	emitterPath string

	etcdPort int

	natsPort int

	bbsPath string
	bbsURL  *url.URL
)

var bbsArgs bbstestrunner.Args
var bbsRunner *ginkgomon.Runner
var bbsProcess ifrit.Process

var etcdRunner *etcdstorerunner.ETCDClusterRunner
var consulRunner *consulrunner.ClusterRunner
var gnatsdRunner ifrit.Process
var natsClient diegonats.NATSClient
var store storeadapter.StoreAdapter
var bbsClient bbs.Client
var logger *lagertest.TestLogger
var syncInterval time.Duration

func TestRouteEmitter(t *testing.T) {
	RegisterFailHandler(Fail)
	SetDefaultEventuallyTimeout(2 * time.Second)
	RunSpecs(t, "Route Emitter Suite")
}

func createEmitterRunner(sessionName string) *ginkgomon.Runner {
	return ginkgomon.New(ginkgomon.Config{
		Command: exec.Command(
			string(emitterPath),
			"-sessionName", sessionName,
			"-natsAddresses", fmt.Sprintf("127.0.0.1:%d", natsPort),
			"-bbsAddress", bbsURL.String(),
			"-communicationTimeout", "100ms",
			"-syncInterval", syncInterval.String(),
			"-lockRetryInterval", "1s",
			"-consulCluster", consulRunner.ConsulCluster(),
		),

		StartCheck: "route-emitter.started",

		AnsiColorCode: "97m",
	})
}

var _ = SynchronizedBeforeSuite(func() []byte {
	emitter, err := gexec.Build("github.com/cloudfoundry-incubator/route-emitter/cmd/route-emitter", "-race")
	Expect(err).NotTo(HaveOccurred())

	bbs, err := gexec.Build("github.com/cloudfoundry-incubator/bbs/cmd/bbs", "-race")
	Expect(err).NotTo(HaveOccurred())

	payload, err := json.Marshal(map[string]string{
		"emitter": emitter,
		"bbs":     bbs,
	})

	Expect(err).NotTo(HaveOccurred())

	return payload
}, func(payload []byte) {
	binaries := map[string]string{}

	err := json.Unmarshal(payload, &binaries)
	Expect(err).NotTo(HaveOccurred())

	etcdPort = 5001 + GinkgoParallelNode()
	natsPort = 4001 + GinkgoParallelNode()

	etcdRunner = etcdstorerunner.NewETCDClusterRunner(etcdPort, 1, nil)

	emitterPath = string(binaries["emitter"])
	store = etcdRunner.Adapter(nil)

	consulRunner = consulrunner.NewClusterRunner(
		9001+config.GinkgoConfig.ParallelNode*consulrunner.PortOffsetLength,
		1,
		"http",
	)

	logger = lagertest.NewTestLogger("test")

	syncInterval = 200 * time.Millisecond

	bbsPath = string(binaries["bbs"])
	bbsAddress := fmt.Sprintf("127.0.0.1:%d", 13000+GinkgoParallelNode())

	bbsURL = &url.URL{
		Scheme: "http",
		Host:   bbsAddress,
	}

	bbsClient = bbs.NewClient(bbsURL.String())

	bbsArgs = bbstestrunner.Args{
		Address:           bbsAddress,
		AdvertiseURL:      bbsURL.String(),
		AuctioneerAddress: "some-address",
		EtcdCluster:       strings.Join(etcdRunner.NodeURLS(), ","),
		ConsulCluster:     consulRunner.ConsulCluster(),
	}
})

var _ = BeforeEach(func() {
	etcdRunner.Start()
	consulRunner.Start()
	consulRunner.WaitUntilReady()

	bbsRunner = bbstestrunner.New(bbsPath, bbsArgs)
	bbsProcess = ginkgomon.Invoke(bbsRunner)

	gnatsdRunner, natsClient = gnatsdrunner.StartGnatsd(natsPort)
})

var _ = AfterEach(func() {
	ginkgomon.Kill(bbsProcess)
	etcdRunner.Stop()
	consulRunner.Stop()
	gnatsdRunner.Signal(os.Interrupt)
	Eventually(gnatsdRunner.Wait(), 5).Should(Receive())
})

var _ = SynchronizedAfterSuite(func() {
	if etcdRunner != nil {
		etcdRunner.Stop()
	}
}, func() {
	gexec.CleanupBuildArtifacts()
})
<commit_msg>Pass encryption key to BBS<commit_after>package main_test

import (
	"encoding/json"
	"fmt"
	"net/url"
	"os"
	"os/exec"
	"strings"
	"testing"
	"time"

	"github.com/cloudfoundry/gunk/diegonats"
	"github.com/cloudfoundry/gunk/diegonats/gnatsdrunner"
	"github.com/cloudfoundry/storeadapter"
	"github.com/cloudfoundry/storeadapter/storerunner/etcdstorerunner"
	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/config"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
	"github.com/pivotal-golang/lager/lagertest"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"

	"github.com/cloudfoundry-incubator/bbs"
	bbstestrunner "github.com/cloudfoundry-incubator/bbs/cmd/bbs/testrunner"
	"github.com/cloudfoundry-incubator/consuladapter/consulrunner"
)

const heartbeatInterval = 1 * time.Second

var (
	emitterPath string

	etcdPort int

	natsPort int

	bbsPath string
	bbsURL  *url.URL
)

var bbsArgs bbstestrunner.Args
var bbsRunner *ginkgomon.Runner
var bbsProcess ifrit.Process

var etcdRunner *etcdstorerunner.ETCDClusterRunner
var consulRunner *consulrunner.ClusterRunner
var gnatsdRunner ifrit.Process
var natsClient diegonats.NATSClient
var store storeadapter.StoreAdapter
var bbsClient bbs.Client
var logger *lagertest.TestLogger
var syncInterval time.Duration

func TestRouteEmitter(t *testing.T) {
	RegisterFailHandler(Fail)
	SetDefaultEventuallyTimeout(2 * time.Second)
	RunSpecs(t, "Route Emitter Suite")
}

func createEmitterRunner(sessionName string) *ginkgomon.Runner {
	return ginkgomon.New(ginkgomon.Config{
		Command: exec.Command(
			string(emitterPath),
			"-sessionName", sessionName,
			"-natsAddresses", fmt.Sprintf("127.0.0.1:%d", natsPort),
			"-bbsAddress", bbsURL.String(),
			"-communicationTimeout", "100ms",
			"-syncInterval", syncInterval.String(),
			"-lockRetryInterval", "1s",
			"-consulCluster", consulRunner.ConsulCluster(),
		),

		StartCheck: "route-emitter.started",

		AnsiColorCode: "97m",
	})
}

var _ = SynchronizedBeforeSuite(func() []byte {
	emitter, err := gexec.Build("github.com/cloudfoundry-incubator/route-emitter/cmd/route-emitter", "-race")
	Expect(err).NotTo(HaveOccurred())

	bbs, err := gexec.Build("github.com/cloudfoundry-incubator/bbs/cmd/bbs", "-race")
	Expect(err).NotTo(HaveOccurred())

	payload, err := json.Marshal(map[string]string{
		"emitter": emitter,
		"bbs":     bbs,
	})

	Expect(err).NotTo(HaveOccurred())

	return payload
}, func(payload []byte) {
	binaries := map[string]string{}

	err := json.Unmarshal(payload, &binaries)
	Expect(err).NotTo(HaveOccurred())

	etcdPort = 5001 + GinkgoParallelNode()
	natsPort = 4001 + GinkgoParallelNode()

	etcdRunner = etcdstorerunner.NewETCDClusterRunner(etcdPort, 1, nil)

	emitterPath = string(binaries["emitter"])
	store = etcdRunner.Adapter(nil)

	consulRunner = consulrunner.NewClusterRunner(
		9001+config.GinkgoConfig.ParallelNode*consulrunner.PortOffsetLength,
		1,
		"http",
	)

	logger = lagertest.NewTestLogger("test")

	syncInterval = 200 * time.Millisecond

	bbsPath = string(binaries["bbs"])
	bbsAddress := fmt.Sprintf("127.0.0.1:%d", 13000+GinkgoParallelNode())

	bbsURL = &url.URL{
		Scheme: "http",
		Host:   bbsAddress,
	}

	bbsClient = bbs.NewClient(bbsURL.String())

	bbsArgs = bbstestrunner.Args{
		Address:           bbsAddress,
		AdvertiseURL:      bbsURL.String(),
		AuctioneerAddress: "some-address",
		EtcdCluster:       strings.Join(etcdRunner.NodeURLS(), ","),
		ConsulCluster:     consulRunner.ConsulCluster(),

		EncryptionKeys: []string{"label:key"},
		ActiveKeyLabel: "label",
	}
})

var _ = BeforeEach(func() {
	etcdRunner.Start()
	consulRunner.Start()
	consulRunner.WaitUntilReady()

	bbsRunner = bbstestrunner.New(bbsPath, bbsArgs)
	bbsProcess = ginkgomon.Invoke(bbsRunner)

	gnatsdRunner, natsClient = gnatsdrunner.StartGnatsd(natsPort)
})

var _ = AfterEach(func() {
	ginkgomon.Kill(bbsProcess)
	etcdRunner.Stop()
	consulRunner.Stop()
	gnatsdRunner.Signal(os.Interrupt)
	Eventually(gnatsdRunner.Wait(), 5).Should(Receive())
})

var _ = SynchronizedAfterSuite(func() {
	if etcdRunner != nil {
		etcdRunner.Stop()
	}
}, func() {
	gexec.CleanupBuildArtifacts()
})
<|endoftext|>


<commit_before>package elasticsearch

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/vektra/cypress"
	"github.com/vektra/errors"
)

type Connection interface {
	Do(*http.Request) (*http.Response, error)
}

type Send struct {
	conn     Connection
	Host     string `short:"H" long:"host" default:"localhost:9200" description:"Address of elasticsearch"`
	Index    string `short:"i" long:"index" description:"Store all messages in one index rather than date driven indexes"`
	Prefix   string `short:"p" long:"prefix" default:"cypress" description:"Prefix to apply to date driven indexes"`
	Logstash bool   `short:"l" long:"logstash" description:"Store messages like logstash does"`

	template string
}

func (s *Send) nextIndex() string {
	if s.Index != "" {
		return s.Index
	}

	return s.Prefix + "-" + time.Now().Format("2006.01.02")
}

func (s *Send) fixupHost() {
	if strings.HasPrefix(s.Host, "http://") ||
		strings.HasPrefix(s.Host, "https://") {
		return
	}

	if !strings.Contains(s.Host, ":") {
		s.Host = s.Host + ":9200"
	}

	s.Host = "http://" + s.Host
}

func (s *Send) connection() Connection {
	if s.conn == nil {
		return http.DefaultClient
	}

	return s.conn
}

// Check and write an index template for the indexes used
func (s *Send) SetupTemplate() error {
	req, err := http.NewRequest("GET", s.Host+"/_template/"+s.template, nil)
	if err != nil {
		return err
	}

	resp, err := s.connection().Do(req)
	if err != nil {
		return err
	}

	defer resp.Body.Close()

	if resp.StatusCode < 300 {
		return nil
	}

	data, ok := Templates[s.template]
	if !ok {
		return fmt.Errorf("Unknown template: %s", s.template)
	}

	body := strings.NewReader(data)

	req, err = http.NewRequest("PUT", s.Host+"/_template/"+s.template, body)
	if err != nil {
		return err
	}

	resp, err = s.connection().Do(req)
	if err != nil {
		return err
	}

	defer resp.Body.Close()

	if resp.StatusCode >= 400 {
		str, _ := ioutil.ReadAll(resp.Body)

		return errors.Context(ErrFromElasticsearch, string(str))
	}

	return nil
}

var ErrFromElasticsearch = errors.New("elasticsearch reported an error")

// Write a Message to Elasticsearch
func (s *Send) Receive(m *cypress.Message) error {
	idx := s.nextIndex()
	url := s.Host + "/" + idx + "/" + m.StringType()

	data, err := json.Marshal(m)
	if err != nil {
		return err
	}

	body := bytes.NewReader(data)

	req, err := http.NewRequest("POST", url, body)
	if err != nil {
		return err
	}

	conn := s.conn
	if conn == nil {
		conn = http.DefaultClient
	}

	resp, err := conn.Do(req)
	defer resp.Body.Close()

	if resp.StatusCode >= 400 {
		str, _ := ioutil.ReadAll(resp.Body)

		return errors.Context(ErrFromElasticsearch, string(str))
	}

	return err
}

// Check all the options and get ready to run.
func (s *Send) Init() error {
	if s.Logstash {
		s.template = "logstash"
		s.Index = ""
		s.Prefix = "logstash"
	} else {
		s.template = "cypress"
	}

	s.fixupHost()
	err := s.SetupTemplate()
	if err != nil {
		return err
	}

	return nil
}

// Called when used via the CLI
func (s *Send) Execute(args []string) error {
	dec, err := cypress.NewStreamDecoder(os.Stdin)
	if err != nil {
		return err
	}

	err = s.Init()
	if err != nil {
		return err
	}

	return cypress.Glue(dec, s)
}

// To fit the Generator interface
func (s *Send) Close() error {
	return nil
}
<commit_msg>Use elastigo and the bulk indexer<commit_after>package elasticsearch

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
	"time"

	elastigo "github.com/mattbaird/elastigo/lib"
	"github.com/vektra/cypress"
	"github.com/vektra/errors"
)

type Connection interface {
	Do(*http.Request) (*http.Response, error)
}

type Send struct {
	conn     Connection
	Host     string `short:"H" long:"host" default:"localhost:9200" description:"Address of elasticsearch"`
	Index    string `short:"i" long:"index" description:"Store all messages in one index rather than date driven indexes"`
	Prefix   string `short:"p" long:"prefix" default:"cypress" description:"Prefix to apply to date driven indexes"`
	Logstash bool   `short:"l" long:"logstash" description:"Store messages like logstash does"`

	template string

	econn *elastigo.Conn
	bulk  *elastigo.BulkIndexer
}

func (s *Send) nextIndex() string {
	if s.Index != "" {
		return s.Index
	}

	return s.Prefix + "-" + time.Now().Format("2006.01.02")
}

func (s *Send) fixupHost() {
	if strings.HasPrefix(s.Host, "http://") ||
		strings.HasPrefix(s.Host, "https://") {
		return
	}

	if !strings.Contains(s.Host, ":") {
		s.Host = s.Host + ":9200"
	}

	s.Host = "http://" + s.Host
}

func (s *Send) connection() Connection {
	if s.conn == nil {
		return http.DefaultClient
	}

	return s.conn
}

// Check and write an index template for the indexes used
func (s *Send) SetupTemplate() error {
	req, err := http.NewRequest("GET", s.Host+"/_template/"+s.template, nil)
	if err != nil {
		return err
	}

	resp, err := s.connection().Do(req)
	if err != nil {
		return err
	}

	defer resp.Body.Close()

	if resp.StatusCode < 300 {
		return nil
	}

	data, ok := Templates[s.template]
	if !ok {
		return fmt.Errorf("Unknown template: %s", s.template)
	}

	body := strings.NewReader(data)

	req, err = http.NewRequest("PUT", s.Host+"/_template/"+s.template, body)
	if err != nil {
		return err
	}

	resp, err = s.connection().Do(req)
	if err != nil {
		return err
	}

	defer resp.Body.Close()

	if resp.StatusCode >= 400 {
		str, _ := ioutil.ReadAll(resp.Body)

		return errors.Context(ErrFromElasticsearch, string(str))
	}

	return nil
}

var ErrFromElasticsearch = errors.New("elasticsearch reported an error")

// Write a Message to Elasticsearch
func (s *Send) Receive(m *cypress.Message) error {
	idx := s.nextIndex()

	if s.econn == nil {
		s.econn = elastigo.NewConn()
		s.bulk = s.econn.NewBulkIndexer(10)
	}

	t := m.GetTimestamp().Time()

	return s.bulk.Index(idx, m.StringType(), "", "", &t, m, false)
}

func (s *Send) Close() error {
	if s.bulk != nil {
		s.bulk.Flush()
	}

	return nil
}

// Check all the options and get ready to run.
func (s *Send) Init() error {
	if s.Logstash {
		s.template = "logstash"
		s.Index = ""
		s.Prefix = "logstash"
	} else {
		s.template = "cypress"
	}

	s.fixupHost()
	err := s.SetupTemplate()
	if err != nil {
		return err
	}

	return nil
}

// Called when used via the CLI
func (s *Send) Execute(args []string) error {
	dec, err := cypress.NewStreamDecoder(os.Stdin)
	if err != nil {
		return err
	}

	err = s.Init()
	if err != nil {
		return err
	}

	defer s.Close()

	return cypress.Glue(dec, s)
}
<|endoftext|>


<commit_before>// Copyright 2017 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package e2e

import (
	"testing"

	"istio.io/api/mixer/adapter/model/v1beta1"
	pb "istio.io/api/policy/v1beta1"
	"istio.io/istio/mixer/test/spyAdapter"
	e2eTmpl "istio.io/istio/mixer/test/spyAdapter/template"
	reportTmpl "istio.io/istio/mixer/test/spyAdapter/template/report"
)

const (
	reportGlobalCfg = `
apiVersion: "config.istio.io/v1alpha2"
kind: attributemanifest
metadata:
  name: istio-proxy
  namespace: default
spec:
    attributes:
      source.name:
        value_type: STRING
      target.name:
        value_type: STRING
      response.count:
        value_type: INT64
      attr.bool:
        value_type: BOOL
      attr.string:
        value_type: STRING
      attr.double:
        value_type: DOUBLE
      attr.int64:
        value_type: INT64
---
`
	reportSvcCfg = `
apiVersion: "config.istio.io/v1alpha2"
kind: fakeHandler
metadata:
  name: fakeHandlerConfig
  namespace: istio-system

---

apiVersion: "config.istio.io/v1alpha2"
kind: samplereport
metadata:
  name: reportInstance
  namespace: istio-system
spec:
  value: "2"
  dimensions:
    source: source.name | "mysrc"
    target_ip: target.name | "mytarget"

---

apiVersion: "config.istio.io/v1alpha2"
kind: rule
metadata:
  name: rule1
  namespace: istio-system
spec:
  selector: match(target.name, "*")
  actions:
  - handler: fakeHandlerConfig.fakeHandler
    instances:
    - reportInstance.samplereport

---
`
)

func TestReport(t *testing.T) {
	tests := []testData{
		{
			name: "Basic Report",

			attrs: map[string]interface{}{
				"target.name": "somesrvcname",
			},

			expectSetTypes: map[string]interface{}{
				"reportInstance.samplereport.istio-system": &reportTmpl.Type{
					Value:      pb.INT64,
					Dimensions: map[string]pb.ValueType{"source": pb.STRING, "target_ip": pb.STRING},
				},
			},

			expectCalls: []spyAdapter.CapturedCall{
				{
					Name: "HandleSampleReport",
					Instances: []interface{}{
						&reportTmpl.Instance{
							Name:       "reportInstance.samplereport.istio-system",
							Value:      int64(2),
							Dimensions: map[string]interface{}{"source": "mysrc", "target_ip": "somesrvcname"},
						},
					},
				},
			},
		},
	}
	for _, tt := range tests {
		// Set the defaults for the test.
		if tt.cfg == "" {
			tt.cfg = reportSvcCfg
		}

		if tt.templates == nil {
			tt.templates = e2eTmpl.SupportedTmplInfo
		}

		if tt.behaviors == nil {
			tt.behaviors = []spyAdapter.AdapterBehavior{{Name: "fakeHandler"}}
		}

		t.Run(tt.name, func(t *testing.T) {
			tt.run(t, v1beta1.TEMPLATE_VARIETY_REPORT, reportGlobalCfg)
		})
	}
}
<commit_msg>Add more end-to-end Mixer tests. (#4076)<commit_after>// Copyright 2017 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package e2e

import (
	"testing"

	"istio.io/api/mixer/adapter/model/v1beta1"
	pb "istio.io/api/policy/v1beta1"
	"istio.io/istio/mixer/test/spyAdapter"
	e2eTmpl "istio.io/istio/mixer/test/spyAdapter/template"
	reportTmpl "istio.io/istio/mixer/test/spyAdapter/template/report"
)

const (
	reportGlobalCfg = `
apiVersion: "config.istio.io/v1alpha2"
kind: attributemanifest
metadata:
  name: istio-proxy
  namespace: default
spec:
    attributes:
      source.name:
        value_type: STRING
      target.name:
        value_type: STRING
      response.count:
        value_type: INT64
      attr.bool:
        value_type: BOOL
      attr.string:
        value_type: STRING
      attr.double:
        value_type: DOUBLE
      attr.int64:
        value_type: INT64
---
`
)

func TestReport(t *testing.T) {
	tests := []testData{

		{
			name: "Basic Report",
			cfg: `
apiVersion: "config.istio.io/v1alpha2"
kind: fakeHandler
metadata:
  name: fakeHandlerConfig
  namespace: istio-system

---

apiVersion: "config.istio.io/v1alpha2"
kind: samplereport
metadata:
  name: reportInstance
  namespace: istio-system
spec:
  value: "2"
  dimensions:
    source: source.name | "mysrc"
    target_ip: target.name | "mytarget"

---

apiVersion: "config.istio.io/v1alpha2"
kind: rule
metadata:
  name: rule1
  namespace: istio-system
spec:
  selector: match(target.name, "*")
  actions:
  - handler: fakeHandlerConfig.fakeHandler
    instances:
    - reportInstance.samplereport

---
`,
			attrs: map[string]interface{}{
				"target.name": "somesrvcname",
			},

			expectSetTypes: map[string]interface{}{
				"reportInstance.samplereport.istio-system": &reportTmpl.Type{
					Value:      pb.INT64,
					Dimensions: map[string]pb.ValueType{"source": pb.STRING, "target_ip": pb.STRING},
				},
			},

			expectCalls: []spyAdapter.CapturedCall{
				{
					Name: "HandleSampleReport",
					Instances: []interface{}{
						&reportTmpl.Instance{
							Name:       "reportInstance.samplereport.istio-system",
							Value:      int64(2),
							Dimensions: map[string]interface{}{"source": "mysrc", "target_ip": "somesrvcname"},
						},
					},
				},
			},
		},

		{
			name: "Multi Instance Report",
			cfg: `
apiVersion: "config.istio.io/v1alpha2"
kind: fakeHandler
metadata:
  name: fakeHandlerConfig
  namespace: istio-system

---
# Instance 1
apiVersion: "config.istio.io/v1alpha2"
kind: samplereport
metadata:
  name: reportInstance1
  namespace: istio-system
spec:
  value: "2"
  dimensions:
    source: source.name | "mysrc"
    target_ip: target.name | "mytarget"

---
# Instance 2
apiVersion: "config.istio.io/v1alpha2"
kind: samplereport
metadata:
  name: reportInstance2
  namespace: istio-system
spec:
  value: "5"
  dimensions:
    source: source.name | "yoursrc"
    target_ip: target.name | "yourtarget"

---

apiVersion: "config.istio.io/v1alpha2"
kind: rule
metadata:
  name: rule1
  namespace: istio-system
spec:
  selector: match(target.name, "*")
  actions:
  - handler: fakeHandlerConfig.fakeHandler
    instances:
    - reportInstance1.samplereport
    - reportInstance2.samplereport

---
`,
			attrs: map[string]interface{}{
				"target.name": "somesrvcname",
			},

			expectSetTypes: map[string]interface{}{
				"reportInstance1.samplereport.istio-system": &reportTmpl.Type{
					Value:      pb.INT64,
					Dimensions: map[string]pb.ValueType{"source": pb.STRING, "target_ip": pb.STRING},
				},
				"reportInstance2.samplereport.istio-system": &reportTmpl.Type{
					Value:      pb.INT64,
					Dimensions: map[string]pb.ValueType{"source": pb.STRING, "target_ip": pb.STRING},
				},
			},

			expectCalls: []spyAdapter.CapturedCall{
				{
					Name: "HandleSampleReport",
					Instances: []interface{}{
						&reportTmpl.Instance{
							Name:       "reportInstance1.samplereport.istio-system",
							Value:      int64(2),
							Dimensions: map[string]interface{}{"source": "mysrc", "target_ip": "somesrvcname"},
						},
						&reportTmpl.Instance{
							Name:       "reportInstance2.samplereport.istio-system",
							Value:      int64(5),
							Dimensions: map[string]interface{}{"source": "yoursrc", "target_ip": "somesrvcname"},
						},
					},
				},
			},
		},

		{
			name: "Conditional Report with No Success",
			cfg: `
apiVersion: "config.istio.io/v1alpha2"
kind: fakeHandler
metadata:
  name: fakeHandlerConfig
  namespace: istio-system

---

apiVersion: "config.istio.io/v1alpha2"
kind: samplereport
metadata:
  name: reportInstance
  namespace: istio-system
spec:
  value: "2"
  dimensions:
    source: source.name | "mysrc"
    target_ip: target.name | "mytarget"

---

apiVersion: "config.istio.io/v1alpha2"
kind: rule
metadata:
  name: rule1
  namespace: istio-system
spec:
  selector: match(target.name, "some unknown thing")
  actions:
  - handler: fakeHandlerConfig.fakeHandler
    instances:
    - reportInstance.samplereport

---
`,
			attrs: map[string]interface{}{
				"target.name": "somesrvcname",
			},

			expectCalls: nil,
		},
	}

	for _, tt := range tests {
		if tt.templates == nil {
			tt.templates = e2eTmpl.SupportedTmplInfo
		}

		if tt.behaviors == nil {
			tt.behaviors = []spyAdapter.AdapterBehavior{{Name: "fakeHandler"}}
		}

		t.Run(tt.name, func(t *testing.T) {
			tt.run(t, v1beta1.TEMPLATE_VARIETY_REPORT, reportGlobalCfg)
		})
	}
}
<|endoftext|>