<commit_before>/*
Copyright 2018, Cossack Labs Limited

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package server

import (
	"context"
	"crypto/tls"
	"net"
	"os"
	"time"

	"bufio"
	"net/http"

	"github.com/cossacklabs/acra/cmd/acra-translator/common"
	"github.com/cossacklabs/acra/cmd/acra-translator/http_api"
	"github.com/cossacklabs/acra/decryptor/base"
	"github.com/cossacklabs/acra/keystore"
	"github.com/cossacklabs/acra/logging"
	"github.com/cossacklabs/acra/network"
	log "github.com/sirupsen/logrus"
	"go.opencensus.io/trace"
	"google.golang.org/grpc"
)

// ReaderServer represents AcraTranslator server, connects with KeyStorage, configuration file,
// gRPC and HTTP request parsers.
type ReaderServer struct {
	config            *common.AcraTranslatorConfig
	keystorage        keystore.MultiKeyStore
	connectionManager *network.ConnectionManager
	grpcServer        *grpc.Server

	httpDecryptor *http_api.HTTPConnectionsDecryptor

	waitTimeout time.Duration

	listenersContextCancel []context.CancelFunc
	grpcServerFactory      common.GRPCServerFactory
}

// NewReaderServer creates Reader server with provided params.
func NewReaderServer(config *common.AcraTranslatorConfig, keystorage keystore.MultiKeyStore, grpcServerFactory common.GRPCServerFactory, waitTimeout time.Duration) (server *ReaderServer, err error) {
	return &ReaderServer{
		grpcServerFactory: grpcServerFactory,
		waitTimeout:       waitTimeout,
		config:            config,
		keystorage:        keystorage,
		connectionManager: network.NewConnectionManager(),
	}, nil
}

// Stop stops AcraTranslator from accepting new connections, and gracefully close existing ones.
func (server *ReaderServer) Stop() {
	log.Infoln("Stop accepting new connections")
	// stop all listeners
	for _, cancelFunc := range server.listenersContextCancel {
		cancelFunc()
	}
	// non block stop
	if server.grpcServer != nil {
		go server.grpcServer.GracefulStop()
	}

	if server.connectionManager.Counter != 0 {
		log.Infof("Wait ending current connections (%v)", server.connectionManager.Counter)
		// wait existing connections to end request
		<-time.NewTimer(server.waitTimeout).C
	}

	log.Infof("Stop all connections that not closed (%v)", server.connectionManager.Counter)
	if server.grpcServer != nil {
		// force stop of grpc server
		server.grpcServer.Stop()
	}
	// force close all connections
	if err := server.connectionManager.CloseConnections(); err != nil {
		log.WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantCloseConnection).WithError(err).Errorln("Took error on closing available connections")
	}
}

func (server *ReaderServer) listenerContext(parentContext context.Context) context.Context {
	ctx, cancel := context.WithCancel(parentContext)
	server.listenersContextCancel = append(server.listenersContextCancel, cancel)
	return ctx
}

// HandleConnectionString handles each connection with gRPC request handler or HTTP request handler
// depending on connection string.
func (server *ReaderServer) HandleConnectionString(parentContext context.Context, connectionString string, processingFunc ProcessingFunc) error {
	logger := logging.GetLoggerFromContext(parentContext)
	if logger == nil {
		logger = log.NewEntry(log.StandardLogger())
	}
	logger = log.WithField("connection_string", connectionString)

	errCh := make(chan error)

	listenerContext := server.listenerContext(parentContext)

	// start accept new connections from connectionString
	connectionChannel, err := common.AcceptConnections(listenerContext, connectionString, errCh)
	if err != nil {
		logger.WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantAcceptNewConnections).WithError(err).Errorf("Can't start to handle connection string %v", connectionString)
		return err
	}
	// use to send close packets to all unclosed connections at end
	go func() {
		logger.WithField("connection_string", connectionString).Debugln("Start wrap new connections")
		for {
			var connection net.Conn
			select {
			case connection = <-connectionChannel:
				break
			case <-parentContext.Done():
				logger.WithError(parentContext.Err()).Debugln("Stop wrapping new connections")
				return
			}

			wrappedConnection, clientID, err := server.config.ConnectionWrapper.WrapServer(context.TODO(), connection)
			if err != nil {
				logger.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTranslatorCantWrapConnectionToSS).
					Errorln("Can't wrap new connection")
				if err := connection.Close(); err != nil {
					logger.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTranslatorCantCloseConnection).
						Errorln("Can't close connection")
				}
				continue
			}
			logger = logger.WithField("client_id", string(clientID))
			logger.Debugln("Read trace")
			spanContext, err := network.ReadTrace(wrappedConnection)
			if err != nil {
				logger.WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTracingCantReadTrace).WithError(err).Errorln("Can't read trace from wrapped connection")
				if err := wrappedConnection.Close(); err != nil {
					log.WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantWrapConnection).WithError(err).Errorln("Can't close wrapped connection")
				}
				continue
			}
			ctx, span := trace.StartSpanWithRemoteParent(listenerContext, getHandlerName(listenerContext), spanContext, server.config.GetTraceOptions()...)
			logger.Debugln("Pass wrapped connection to processing function")
			logging.SetLoggerToContext(ctx, logger)

			go func() {
				defer func() {
					span.End()
					err := wrappedConnection.Close()
					if err != nil {
						logger.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTranslatorCantCloseConnection).
							Errorln("Can't close wrapped connection")
					}
					logger.Infoln("Connection closed")
				}()

				if err := server.connectionManager.AddConnection(wrappedConnection); err != nil {
					logger.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTranslatorCantHandleHTTPConnection).
						Errorln("Can't add connection to connection manager")
					return
				}

				processingFunc(ctx, clientID, wrappedConnection)

				if err := server.connectionManager.RemoveConnection(wrappedConnection); err != nil {
					logger.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTranslatorCantHandleHTTPConnection).
						Errorln("Can't remove connection from connection manager")
				}
			}()
		}
	}()
	var outErr error
	select {
	case <-parentContext.Done():
		log.WithError(parentContext.Err()).Debugln("Exit from handling connection string. Close all connections")
		outErr = parentContext.Err()
	case outErr = <-errCh:
		log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTranslatorCantAcceptNewHTTPConnection).
			Errorln("Error on accepting new connections")
		server.Stop()
	}
	return outErr
}

// Constants show possible connection types.
const (
	ConnectionTypeKey  = "connection_type"
	HTTPConnectionType = "http"
	GRPCConnectionType = "grpc"
)

type handlerName struct{}

func withHandlerName(ctx context.Context, name string) context.Context {
	return context.WithValue(ctx, handlerName{}, name)
}

func getHandlerName(ctx context.Context) string {
	if s, ok := ctx.Value(handlerName{}).(string); ok {
		return s
	}
	return "undefined"
}

// Start setups gRPC handler or HTTP handler, poison records callbacks and starts listening to connections.
func (server *ReaderServer) Start(parentContext context.Context) {
	logger := logging.GetLoggerFromContext(parentContext)
	poisonCallbacks := base.NewPoisonCallbackStorage()
	if server.config.DetectPoisonRecords() {
		if server.config.ScriptOnPoison() != "" {
			log.Infof("Add poison record callback with script execution %v", server.config.ScriptOnPoison())
			poisonCallbacks.AddCallback(base.NewExecuteScriptCallback(server.config.ScriptOnPoison()))
		}

		// must be last
		if server.config.StopOnPoison() {
			log.Infoln("Add poison record callback with AcraTranslator termination")
			poisonCallbacks.AddCallback(&base.StopCallback{})
		}
	}
	decryptorData := &common.TranslatorData{Keystorage: server.keystorage, PoisonRecordCallbacks: poisonCallbacks, CheckPoisonRecords: server.config.DetectPoisonRecords()}
	if server.config.IncomingConnectionHTTPString() != "" {
		go func() {
			httpContext := logging.SetLoggerToContext(parentContext, logger.WithField(ConnectionTypeKey, HTTPConnectionType))
			httpDecryptor, err := http_api.NewHTTPConnectionsDecryptor(decryptorData)
			logger.WithField("connection_string", server.config.IncomingConnectionHTTPString()).Infof("Start process HTTP requests")
			if err != nil {
				log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTranslatorCantHandleHTTPConnection).
					Errorln("Can't create HTTP decryptor")
			}
			server.httpDecryptor = httpDecryptor
			err = server.HandleConnectionString(withHandlerName(httpContext, "processHTTPConnection"), server.config.IncomingConnectionHTTPString(), server.processHTTPConnection)
			if err != nil {
				log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTranslatorCantHandleHTTPConnection).
					Errorln("Took error on handling HTTP requests")
				server.Stop()
				os.Exit(1)
			}
		}()
	}
	// provide way to register new services and custom server
	if server.config.IncomingConnectionGRPCString() != "" {
		go func() {
			grpcLogger := logger.WithField(ConnectionTypeKey, GRPCConnectionType)
			logger.WithField("connection_string", server.config.IncomingConnectionGRPCString()).Infof("Start process gRPC requests")
			var listener net.Listener
			var err error
			var opts []grpc.ServerOption
			if server.config.WithTLS() {
				listener, err = network.Listen(server.config.IncomingConnectionGRPCString())
				if err != nil {
					grpcLogger.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTranslatorCantHandleGRPCConnection).
						Errorln("Can't create gRPC connection listener")
					return
				}
				listener = tls.NewListener(listener, server.config.GetTLSConfig())
			} else {
				//listener, err = network.NewSecureSessionListener(server.config.ServerID(), server.config.IncomingConnectionGRPCString(), server.keystorage)
				//if err != nil {
				//	grpcLogger.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTranslatorCantHandleGRPCConnection).
				//		Errorln("Can't create secure session listener")
				//	return
				//}
				listener, err = network.Listen(server.config.IncomingConnectionGRPCString())
				if err != nil {
					grpcLogger.WithError(err).Errorln("Can't initialize connection listener")
					return
				}
				wrapper, err := network.NewSecureSessionConnectionWrapper(server.config.ServerID(), server.keystorage)
				if err != nil {
					grpcLogger.WithError(err).Errorln("Can't initialize Secure Session wrapper")
					return
				}
				opts = append(opts, grpc.Creds(wrapper))
			}

			grpcListener := common.WrapListenerWithMetrics(listener)

			grpcServer, err := server.grpcServerFactory.New(decryptorData, opts...)
			if err != nil {
				logger.WithError(err).Errorln("Can't create new grpc server")
				os.Exit(1)
			}
			server.grpcServer = grpcServer
			if err := grpcServer.Serve(grpcListener); err != nil {
				grpcLogger.Errorf("failed to serve: %v", err)
				server.Stop()
				os.Exit(1)
				return
			}
		}()
	}
	<-parentContext.Done()
}

// ProcessingFunc redirects processing of connection to HTTP handler or gRPC handler.
type ProcessingFunc func(context.Context, []byte, net.Conn)

func (server *ReaderServer) processHTTPConnection(parentContext context.Context, clientID []byte, connection net.Conn) {
	connection.SetDeadline(time.Now().Add(network.DefaultNetworkTimeout))
	defer connection.SetDeadline(time.Time{})

	spanCtx, span := trace.StartSpan(parentContext, "processHTTPConnection")
	defer span.End()

	// processing HTTP connection
	logger := logging.LoggerWithTrace(spanCtx, logging.GetLoggerFromContext(parentContext))
	httpLogger := logger.WithField(ConnectionTypeKey, HTTPConnectionType)
	httpLogger.Debugln("HTTP handler")

	reader := bufio.NewReader(connection)
	request, err := http.ReadRequest(reader)

	// TODO: handle keep alive

	if err != nil {
		logger.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTranslatorCantHandleHTTPRequest).
			Warningln("Got new HTTP request, but can't read it")
		server.httpDecryptor.SendResponse(logger,
			server.httpDecryptor.EmptyResponseWithStatus(request, http.StatusBadRequest), connection)
		return
	}

	response := server.httpDecryptor.ParseRequestPrepareResponse(logger, request, clientID)
	server.httpDecryptor.SendResponse(logger, response, connection)
}
<commit_msg>use grpc tls extension instead pure go tls<commit_after>/*
Copyright 2018, Cossack Labs Limited

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package server

import (
	"context"
	"google.golang.org/grpc/credentials"
	"net"
	"os"
	"time"

	"bufio"
	"net/http"

	"github.com/cossacklabs/acra/cmd/acra-translator/common"
	"github.com/cossacklabs/acra/cmd/acra-translator/http_api"
	"github.com/cossacklabs/acra/decryptor/base"
	"github.com/cossacklabs/acra/keystore"
	"github.com/cossacklabs/acra/logging"
	"github.com/cossacklabs/acra/network"
	log "github.com/sirupsen/logrus"
	"go.opencensus.io/trace"
	"google.golang.org/grpc"
)

// ReaderServer represents AcraTranslator server, connects with KeyStorage, configuration file,
// gRPC and HTTP request parsers.
type ReaderServer struct {
	config            *common.AcraTranslatorConfig
	keystorage        keystore.MultiKeyStore
	connectionManager *network.ConnectionManager
	grpcServer        *grpc.Server

	httpDecryptor *http_api.HTTPConnectionsDecryptor

	waitTimeout time.Duration

	listenersContextCancel []context.CancelFunc
	grpcServerFactory      common.GRPCServerFactory
}

// NewReaderServer creates Reader server with provided params.
func NewReaderServer(config *common.AcraTranslatorConfig, keystorage keystore.MultiKeyStore, grpcServerFactory common.GRPCServerFactory, waitTimeout time.Duration) (server *ReaderServer, err error) {
	return &ReaderServer{
		grpcServerFactory: grpcServerFactory,
		waitTimeout:       waitTimeout,
		config:            config,
		keystorage:        keystorage,
		connectionManager: network.NewConnectionManager(),
	}, nil
}

// Stop stops AcraTranslator from accepting new connections, and gracefully close existing ones.
func (server *ReaderServer) Stop() {
	log.Infoln("Stop accepting new connections")
	// stop all listeners
	for _, cancelFunc := range server.listenersContextCancel {
		cancelFunc()
	}
	// non block stop
	if server.grpcServer != nil {
		go server.grpcServer.GracefulStop()
	}

	if server.connectionManager.Counter != 0 {
		log.Infof("Wait ending current connections (%v)", server.connectionManager.Counter)
		// wait existing connections to end request
		<-time.NewTimer(server.waitTimeout).C
	}

	log.Infof("Stop all connections that not closed (%v)", server.connectionManager.Counter)
	if server.grpcServer != nil {
		// force stop of grpc server
		server.grpcServer.Stop()
	}
	// force close all connections
	if err := server.connectionManager.CloseConnections(); err != nil {
		log.WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantCloseConnection).WithError(err).Errorln("Took error on closing available connections")
	}
}

func (server *ReaderServer) listenerContext(parentContext context.Context) context.Context {
	ctx, cancel := context.WithCancel(parentContext)
	server.listenersContextCancel = append(server.listenersContextCancel, cancel)
	return ctx
}

// HandleConnectionString handles each connection with gRPC request handler or HTTP request handler
// depending on connection string.
func (server *ReaderServer) HandleConnectionString(parentContext context.Context, connectionString string, processingFunc ProcessingFunc) error {
	logger := logging.GetLoggerFromContext(parentContext)
	if logger == nil {
		logger = log.NewEntry(log.StandardLogger())
	}
	logger = log.WithField("connection_string", connectionString)

	errCh := make(chan error)

	listenerContext := server.listenerContext(parentContext)

	// start accept new connections from connectionString
	connectionChannel, err := common.AcceptConnections(listenerContext, connectionString, errCh)
	if err != nil {
		logger.WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantAcceptNewConnections).WithError(err).Errorf("Can't start to handle connection string %v", connectionString)
		return err
	}
	// use to send close packets to all unclosed connections at end
	go func() {
		logger.WithField("connection_string", connectionString).Debugln("Start wrap new connections")
		for {
			var connection net.Conn
			select {
			case connection = <-connectionChannel:
				break
			case <-parentContext.Done():
				logger.WithError(parentContext.Err()).Debugln("Stop wrapping new connections")
				return
			}

			wrappedConnection, clientID, err := server.config.ConnectionWrapper.WrapServer(context.TODO(), connection)
			if err != nil {
				logger.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTranslatorCantWrapConnectionToSS).
					Errorln("Can't wrap new connection")
				if err := connection.Close(); err != nil {
					logger.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTranslatorCantCloseConnection).
						Errorln("Can't close connection")
				}
				continue
			}
			logger = logger.WithField("client_id", string(clientID))
			logger.Debugln("Read trace")
			spanContext, err := network.ReadTrace(wrappedConnection)
			if err != nil {
				logger.WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTracingCantReadTrace).WithError(err).Errorln("Can't read trace from wrapped connection")
				if err := wrappedConnection.Close(); err != nil {
					log.WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantWrapConnection).WithError(err).Errorln("Can't close wrapped connection")
				}
				continue
			}
			ctx, span := trace.StartSpanWithRemoteParent(listenerContext, getHandlerName(listenerContext), spanContext, server.config.GetTraceOptions()...)
			logger.Debugln("Pass wrapped connection to processing function")
			logging.SetLoggerToContext(ctx, logger)

			go func() {
				defer func() {
					span.End()
					err := wrappedConnection.Close()
					if err != nil {
						logger.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTranslatorCantCloseConnection).
							Errorln("Can't close wrapped connection")
					}
					logger.Infoln("Connection closed")
				}()

				if err := server.connectionManager.AddConnection(wrappedConnection); err != nil {
					logger.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTranslatorCantHandleHTTPConnection).
						Errorln("Can't add connection to connection manager")
					return
				}

				processingFunc(ctx, clientID, wrappedConnection)

				if err := server.connectionManager.RemoveConnection(wrappedConnection); err != nil {
					logger.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTranslatorCantHandleHTTPConnection).
						Errorln("Can't remove connection from connection manager")
				}
			}()
		}
	}()
	var outErr error
	select {
	case <-parentContext.Done():
		log.WithError(parentContext.Err()).Debugln("Exit from handling connection string. Close all connections")
		outErr = parentContext.Err()
	case outErr = <-errCh:
		log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTranslatorCantAcceptNewHTTPConnection).
			Errorln("Error on accepting new connections")
		server.Stop()
	}
	return outErr
}

// Constants show possible connection types.
const (
	ConnectionTypeKey  = "connection_type"
	HTTPConnectionType = "http"
	GRPCConnectionType = "grpc"
)

type handlerName struct{}

func withHandlerName(ctx context.Context, name string) context.Context {
	return context.WithValue(ctx, handlerName{}, name)
}

func getHandlerName(ctx context.Context) string {
	if s, ok := ctx.Value(handlerName{}).(string); ok {
		return s
	}
	return "undefined"
}

// Start setups gRPC handler or HTTP handler, poison records callbacks and starts listening to connections.
func (server *ReaderServer) Start(parentContext context.Context) {
	logger := logging.GetLoggerFromContext(parentContext)
	poisonCallbacks := base.NewPoisonCallbackStorage()
	if server.config.DetectPoisonRecords() {
		if server.config.ScriptOnPoison() != "" {
			log.Infof("Add poison record callback with script execution %v", server.config.ScriptOnPoison())
			poisonCallbacks.AddCallback(base.NewExecuteScriptCallback(server.config.ScriptOnPoison()))
		}

		// must be last
		if server.config.StopOnPoison() {
			log.Infoln("Add poison record callback with AcraTranslator termination")
			poisonCallbacks.AddCallback(&base.StopCallback{})
		}
	}
	decryptorData := &common.TranslatorData{Keystorage: server.keystorage, PoisonRecordCallbacks: poisonCallbacks, CheckPoisonRecords: server.config.DetectPoisonRecords()}
	if server.config.IncomingConnectionHTTPString() != "" {
		go func() {
			httpContext := logging.SetLoggerToContext(parentContext, logger.WithField(ConnectionTypeKey, HTTPConnectionType))
			httpDecryptor, err := http_api.NewHTTPConnectionsDecryptor(decryptorData)
			logger.WithField("connection_string", server.config.IncomingConnectionHTTPString()).Infof("Start process HTTP requests")
			if err != nil {
				log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTranslatorCantHandleHTTPConnection).
					Errorln("Can't create HTTP decryptor")
			}
			server.httpDecryptor = httpDecryptor
			err = server.HandleConnectionString(withHandlerName(httpContext, "processHTTPConnection"), server.config.IncomingConnectionHTTPString(), server.processHTTPConnection)
			if err != nil {
				log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTranslatorCantHandleHTTPConnection).
					Errorln("Took error on handling HTTP requests")
				server.Stop()
				os.Exit(1)
			}
		}()
	}
	// provide way to register new services and custom server
	if server.config.IncomingConnectionGRPCString() != "" {
		go func() {
			grpcLogger := logger.WithField(ConnectionTypeKey, GRPCConnectionType)
			logger.WithField("connection_string", server.config.IncomingConnectionGRPCString()).Infof("Start process gRPC requests")
			var listener net.Listener
			var err error
			var opts []grpc.ServerOption
			if server.config.WithTLS() {
				listener, err = network.Listen(server.config.IncomingConnectionGRPCString())
				if err != nil {
					grpcLogger.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTranslatorCantHandleGRPCConnection).
						Errorln("Can't create gRPC connection listener")
					return
				}
				//listener = tls.NewListener(listener, server.config.GetTLSConfig())
				opts = append(opts, grpc.Creds(credentials.NewTLS(server.config.GetTLSConfig())))
			} else {
				//listener, err = network.NewSecureSessionListener(server.config.ServerID(), server.config.IncomingConnectionGRPCString(), server.keystorage)
				//if err != nil {
				//	grpcLogger.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTranslatorCantHandleGRPCConnection).
				//		Errorln("Can't create secure session listener")
				//	return
				//}
				listener, err = network.Listen(server.config.IncomingConnectionGRPCString())
				if err != nil {
					grpcLogger.WithError(err).Errorln("Can't initialize connection listener")
					return
				}
				wrapper, err := network.NewSecureSessionConnectionWrapper(server.config.ServerID(), server.keystorage)
				if err != nil {
					grpcLogger.WithError(err).Errorln("Can't initialize Secure Session wrapper")
					return
				}
				opts = append(opts, grpc.Creds(wrapper))
			}

			grpcListener := common.WrapListenerWithMetrics(listener)

			grpcServer, err := server.grpcServerFactory.New(decryptorData, opts...)
			if err != nil {
				logger.WithError(err).Errorln("Can't create new grpc server")
				os.Exit(1)
			}
			server.grpcServer = grpcServer
			if err := grpcServer.Serve(grpcListener); err != nil {
				grpcLogger.Errorf("failed to serve: %v", err)
				server.Stop()
				os.Exit(1)
				return
			}
		}()
	}
	<-parentContext.Done()
}

// ProcessingFunc redirects processing of connection to HTTP handler or gRPC handler.
type ProcessingFunc func(context.Context, []byte, net.Conn)

func (server *ReaderServer) processHTTPConnection(parentContext context.Context, clientID []byte, connection net.Conn) {
	connection.SetDeadline(time.Now().Add(network.DefaultNetworkTimeout))
	defer connection.SetDeadline(time.Time{})

	spanCtx, span := trace.StartSpan(parentContext, "processHTTPConnection")
	defer span.End()

	// processing HTTP connection
	logger := logging.LoggerWithTrace(spanCtx, logging.GetLoggerFromContext(parentContext))
	httpLogger := logger.WithField(ConnectionTypeKey, HTTPConnectionType)
	httpLogger.Debugln("HTTP handler")

	reader := bufio.NewReader(connection)
	request, err := http.ReadRequest(reader)

	// TODO: handle keep alive

	if err != nil {
		logger.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorTranslatorCantHandleHTTPRequest).
			Warningln("Got new HTTP request, but can't read it")
		server.httpDecryptor.SendResponse(logger,
			server.httpDecryptor.EmptyResponseWithStatus(request, http.StatusBadRequest), connection)
		return
	}

	response := server.httpDecryptor.ParseRequestPrepareResponse(logger, request, clientID)
	server.httpDecryptor.SendResponse(logger, response, connection)
}
<|endoftext|>


<commit_before>// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"net/http"

	"github.com/matrix-org/dendrite/appservice"
	"github.com/matrix-org/dendrite/clientapi"
	"github.com/matrix-org/dendrite/clientapi/producers"
	"github.com/matrix-org/dendrite/eduserver"
	"github.com/matrix-org/dendrite/eduserver/cache"
	"github.com/matrix-org/dendrite/federationapi"
	"github.com/matrix-org/dendrite/federationsender"
	"github.com/matrix-org/dendrite/internal"
	"github.com/matrix-org/dendrite/internal/basecomponent"
	"github.com/matrix-org/dendrite/internal/config"
	"github.com/matrix-org/dendrite/internal/keydb"
	"github.com/matrix-org/dendrite/internal/transactions"
	"github.com/matrix-org/dendrite/keyserver"
	"github.com/matrix-org/dendrite/mediaapi"
	"github.com/matrix-org/dendrite/publicroomsapi"
	"github.com/matrix-org/dendrite/publicroomsapi/storage"
	"github.com/matrix-org/dendrite/roomserver"
	"github.com/matrix-org/dendrite/syncapi"

	"github.com/sirupsen/logrus"
)

var (
	httpBindAddr   = flag.String("http-bind-address", ":8008", "The HTTP listening port for the server")
	httpsBindAddr  = flag.String("https-bind-address", ":8448", "The HTTPS listening port for the server")
	certFile       = flag.String("tls-cert", "", "The PEM formatted X509 certificate to use for TLS")
	keyFile        = flag.String("tls-key", "", "The PEM private key to use for TLS")
	enableHTTPAPIs = flag.Bool("api", false, "Use HTTP APIs instead of short-circuiting (warning: exposes API endpoints!)")
)

func main() {
	cfg := basecomponent.ParseMonolithFlags()
	if *enableHTTPAPIs {
		// If the HTTP APIs are enabled then we need to update the Listen
		// statements in the configuration so that we know where to find
		// the API endpoints. They'll listen on the same port as the monolith
		// itself.
		addr := config.Address(*httpBindAddr)
		cfg.Listen.RoomServer = addr
		cfg.Listen.EDUServer = addr
		cfg.Listen.AppServiceAPI = addr
		cfg.Listen.FederationSender = addr
	}

	base := basecomponent.NewBaseDendrite(cfg, "Monolith", *enableHTTPAPIs)
	defer base.Close() // nolint: errcheck

	accountDB := base.CreateAccountsDB()
	deviceDB := base.CreateDeviceDB()
	keyDB := base.CreateKeyDB()
	federation := base.CreateFederationClient()
	keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives)

	rsAPI := roomserver.SetupRoomServerComponent(
		base, keyRing, federation,
	)
	if base.EnableHTTPAPIs {
		rsAPI = base.CreateHTTPRoomserverAPIs()
	}

	eduInputAPI := eduserver.SetupEDUServerComponent(
		base, cache.New(),
	)
	if base.EnableHTTPAPIs {
		eduInputAPI = base.CreateHTTPEDUServerAPIs()
	}

	asAPI := appservice.SetupAppServiceAPIComponent(
		base, accountDB, deviceDB, federation, rsAPI, transactions.New(),
	)
	if base.EnableHTTPAPIs {
		asAPI = base.CreateHTTPAppServiceAPIs()
	}

	fsAPI := federationsender.SetupFederationSenderComponent(
		base, federation, rsAPI, &keyRing,
	)
	if base.EnableHTTPAPIs {
		fsAPI = base.CreateHTTPFederationSenderAPIs()
	}
	rsAPI.SetFederationSenderAPI(fsAPI)

	clientapi.SetupClientAPIComponent(
		base, deviceDB, accountDB,
		federation, &keyRing, rsAPI,
		eduInputAPI, asAPI, transactions.New(), fsAPI,
	)

	keyserver.SetupKeyServerComponent(
		base, deviceDB, accountDB,
	)
	eduProducer := producers.NewEDUServerProducer(eduInputAPI)
	federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, rsAPI, asAPI, fsAPI, eduProducer)
	mediaapi.SetupMediaAPIComponent(base, deviceDB)
	publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties())
	if err != nil {
		logrus.WithError(err).Panicf("failed to connect to public rooms db")
	}
	publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, rsAPI, federation, nil)
	syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, rsAPI, federation, cfg)

	internal.SetupHTTPAPI(
		http.DefaultServeMux,
		base.PublicAPIMux,
		base.InternalAPIMux,
		cfg,
		base.EnableHTTPAPIs,
	)

	// Expose the matrix APIs directly rather than putting them under a /api path.
	go func() {
		serv := http.Server{
			Addr:         *httpBindAddr,
			WriteTimeout: basecomponent.HTTPServerTimeout,
		}

		logrus.Info("Listening on ", serv.Addr)
		logrus.Fatal(serv.ListenAndServe())
	}()
	// Handle HTTPS if certificate and key are provided
	if *certFile != "" && *keyFile != "" {
		go func() {
			serv := http.Server{
				Addr:         *httpsBindAddr,
				WriteTimeout: basecomponent.HTTPServerTimeout,
			}

			logrus.Info("Listening on ", serv.Addr)
			logrus.Fatal(serv.ListenAndServeTLS(*certFile, *keyFile))
		}()
	}

	// We want to block forever to let the HTTP and HTTPS handler serve the APIs
	select {}
}
<commit_msg>Fix monolith room server-federation sender connection<commit_after>// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"net/http"

	"github.com/matrix-org/dendrite/appservice"
	"github.com/matrix-org/dendrite/clientapi"
	"github.com/matrix-org/dendrite/clientapi/producers"
	"github.com/matrix-org/dendrite/eduserver"
	"github.com/matrix-org/dendrite/eduserver/cache"
	"github.com/matrix-org/dendrite/federationapi"
	"github.com/matrix-org/dendrite/federationsender"
	"github.com/matrix-org/dendrite/internal"
	"github.com/matrix-org/dendrite/internal/basecomponent"
	"github.com/matrix-org/dendrite/internal/config"
	"github.com/matrix-org/dendrite/internal/keydb"
	"github.com/matrix-org/dendrite/internal/transactions"
	"github.com/matrix-org/dendrite/keyserver"
	"github.com/matrix-org/dendrite/mediaapi"
	"github.com/matrix-org/dendrite/publicroomsapi"
	"github.com/matrix-org/dendrite/publicroomsapi/storage"
	"github.com/matrix-org/dendrite/roomserver"
	"github.com/matrix-org/dendrite/syncapi"

	"github.com/sirupsen/logrus"
)

var (
	httpBindAddr   = flag.String("http-bind-address", ":8008", "The HTTP listening port for the server")
	httpsBindAddr  = flag.String("https-bind-address", ":8448", "The HTTPS listening port for the server")
	certFile       = flag.String("tls-cert", "", "The PEM formatted X509 certificate to use for TLS")
	keyFile        = flag.String("tls-key", "", "The PEM private key to use for TLS")
	enableHTTPAPIs = flag.Bool("api", false, "Use HTTP APIs instead of short-circuiting (warning: exposes API endpoints!)")
)

func main() {
	cfg := basecomponent.ParseMonolithFlags()
	if *enableHTTPAPIs {
		// If the HTTP APIs are enabled then we need to update the Listen
		// statements in the configuration so that we know where to find
		// the API endpoints. They'll listen on the same port as the monolith
		// itself.
		addr := config.Address(*httpBindAddr)
		cfg.Listen.RoomServer = addr
		cfg.Listen.EDUServer = addr
		cfg.Listen.AppServiceAPI = addr
		cfg.Listen.FederationSender = addr
	}

	base := basecomponent.NewBaseDendrite(cfg, "Monolith", *enableHTTPAPIs)
	defer base.Close() // nolint: errcheck

	accountDB := base.CreateAccountsDB()
	deviceDB := base.CreateDeviceDB()
	keyDB := base.CreateKeyDB()
	federation := base.CreateFederationClient()
	keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives)

	rsComponent := roomserver.SetupRoomServerComponent(
		base, keyRing, federation,
	)
	rsAPI := rsComponent
	if base.EnableHTTPAPIs {
		rsAPI = base.CreateHTTPRoomserverAPIs()
	}

	eduInputAPI := eduserver.SetupEDUServerComponent(
		base, cache.New(),
	)
	if base.EnableHTTPAPIs {
		eduInputAPI = base.CreateHTTPEDUServerAPIs()
	}

	asAPI := appservice.SetupAppServiceAPIComponent(
		base, accountDB, deviceDB, federation, rsAPI, transactions.New(),
	)
	if base.EnableHTTPAPIs {
		asAPI = base.CreateHTTPAppServiceAPIs()
	}

	fsAPI := federationsender.SetupFederationSenderComponent(
		base, federation, rsAPI, &keyRing,
	)
	if base.EnableHTTPAPIs {
		fsAPI = base.CreateHTTPFederationSenderAPIs()
	}
	rsComponent.SetFederationSenderAPI(fsAPI)

	clientapi.SetupClientAPIComponent(
		base, deviceDB, accountDB,
		federation, &keyRing, rsAPI,
		eduInputAPI, asAPI, transactions.New(), fsAPI,
	)

	keyserver.SetupKeyServerComponent(
		base, deviceDB, accountDB,
	)
	eduProducer := producers.NewEDUServerProducer(eduInputAPI)
	federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, rsAPI, asAPI, fsAPI, eduProducer)
	mediaapi.SetupMediaAPIComponent(base, deviceDB)
	publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties())
	if err != nil {
		logrus.WithError(err).Panicf("failed to connect to public rooms db")
	}
	publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, rsAPI, federation, nil)
	syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, rsAPI, federation, cfg)

	internal.SetupHTTPAPI(
		http.DefaultServeMux,
		base.PublicAPIMux,
		base.InternalAPIMux,
		cfg,
		base.EnableHTTPAPIs,
	)

	// Expose the matrix APIs directly rather than putting them under a /api path.
	go func() {
		serv := http.Server{
			Addr:         *httpBindAddr,
			WriteTimeout: basecomponent.HTTPServerTimeout,
		}

		logrus.Info("Listening on ", serv.Addr)
		logrus.Fatal(serv.ListenAndServe())
	}()
	// Handle HTTPS if certificate and key are provided
	if *certFile != "" && *keyFile != "" {
		go func() {
			serv := http.Server{
				Addr:         *httpsBindAddr,
				WriteTimeout: basecomponent.HTTPServerTimeout,
			}

			logrus.Info("Listening on ", serv.Addr)
			logrus.Fatal(serv.ListenAndServeTLS(*certFile, *keyFile))
		}()
	}

	// We want to block forever to let the HTTP and HTTPS handler serve the APIs
	select {}
}
<|endoftext|>


<commit_before>package metatype

import "github.com/qlova/uct/compiler"
import "github.com/qlova/ilang/syntax/symbols"

import "github.com/qlova/ilang/types/number"
import "github.com/qlova/ilang/types/text"

import "strconv"

type Data struct {
	Type compiler.Type
}

func (Data) Name(l compiler.Language) string {
	return ""
}

func (Data) Equals(d compiler.Data) bool {
	return false
}

var Type = compiler.Type {
	Name: compiler.Translatable{
		compiler.English: "type",
	},

	//TODO allow types to be passed and compared as strings. This may open up powerful types of reflection.
	Base: compiler.NULL,
}

func init() {
	Type.Shunt = func(c *compiler.Compiler, symbol string, a, b compiler.Type) *compiler.Type {

		if symbol == symbols.Equals {

			if !a.Equals(b) {
				return nil
			}

			//TODO use fixed type so if statements using this expression can be optimised away.
			if a.Data.(Data).Type.Equals(b.Data.(Data).Type) {
				c.Int(1)
			} else {
				c.Int(0)
			}

			return &number.Type
		}

		return nil
	}

	Type.Cast = func(c *compiler.Compiler, a, b compiler.Type) bool {
		if b.Equals(text.Type) {

			c.SwapOutput()
			c.Data("text_literal"+strconv.Itoa(text.Tmp), []byte(a.Data.(Data).Type.String()))
			c.SwapOutput()

			c.PushList("text_literal"+strconv.Itoa(text.Tmp))

			text.Tmp++

			return true
		}
		return false
	}
}

var Expression = compiler.Expression {
	Name: Type.Name,

	Detect: func(c *compiler.Compiler) *compiler.Type {
		if c.GetType(c.Token()) != nil {
			if c.Peek() != symbols.FunctionCallBegin && c.Peek() != symbols.Index {
				var result = Type.With(Data{
					Type: *c.GetType(c.Token()),
				})
				return &result
			}
		}
		return nil
	},

	OnScan: func(c *compiler.Compiler) compiler.Type {
		c.Expecting(symbols.FunctionCallBegin)
		var arg = c.ScanExpression()
		c.Expecting(symbols.FunctionCallEnd)

		c.DropType(arg)

		return Type.With(Data{
			Type: arg,
		})
	},
}
<commit_msg>Fix type comparison.<commit_after>package metatype

import "github.com/qlova/uct/compiler"
import "github.com/qlova/ilang/syntax/symbols"

import "github.com/qlova/ilang/types/number"
import "github.com/qlova/ilang/types/text"

import "strconv"

type Data struct {
	Type compiler.Type
}

func (Data) Name(l compiler.Language) string {
	return ""
}

func (Data) Equals(d compiler.Data) bool {
	return false
}

var Type = compiler.Type {
	Name: compiler.Translatable{
		compiler.English: "type",
	},

	//TODO allow types to be passed and compared as strings. This may open up powerful types of reflection.
	Base: compiler.NULL,
}

func init() {
	Type.Shunt = func(c *compiler.Compiler, symbol string, a, b compiler.Type) *compiler.Type {

		if symbol == symbols.Equals {

			//if !a.Equals(b) {
			//	return nil
			//}

			//TODO use fixed type so if statements using this expression can be optimised away.
			if a.Data.(Data).Type.Equals(b.Data.(Data).Type) {
				c.Int(1)
			} else {
				c.Int(0)
			}

			return &number.Type
		}

		return nil
	}

	Type.Cast = func(c *compiler.Compiler, a, b compiler.Type) bool {
		if b.Equals(text.Type) {

			c.SwapOutput()
			c.Data("text_literal"+strconv.Itoa(text.Tmp), []byte(a.Data.(Data).Type.String()))
			c.SwapOutput()

			c.PushList("text_literal"+strconv.Itoa(text.Tmp))

			text.Tmp++

			return true
		}
		return false
	}
}

var Expression = compiler.Expression {
	Name: Type.Name,

	Detect: func(c *compiler.Compiler) *compiler.Type {
		if c.GetType(c.Token()) != nil {
			if c.Peek() != symbols.FunctionCallBegin && c.Peek() != symbols.Index {
				var result = Type.With(Data{
					Type: *c.GetType(c.Token()),
				})
				return &result
			}
		}
		return nil
	},

	OnScan: func(c *compiler.Compiler) compiler.Type {
		c.Expecting(symbols.FunctionCallBegin)
		var arg = c.ScanExpression()
		c.Expecting(symbols.FunctionCallEnd)

		c.DropType(arg)

		return Type.With(Data{
			Type: arg,
		})
	},
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"net/http"

	"gopkg.in/mgo.v2/bson"
	"github.com/freehaha/token-auth"
	"github.com/gorilla/mux"
)

// GetUserController will answer a JSON of the user
// linked to the given id in the URL
func GetUserController(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	userID := vars["id"]
	var res = GetUser(bson.ObjectIdHex(userID))
	json.NewEncoder(w).Encode(res)
}

func GetAllUserController(w http.ResponseWriter, r *http.Request) {
	var res = GetAllUser()
	json.NewEncoder(w).Encode(res)
}

// AddUserController will answer a JSON of the
// brand new created user (from the JSON Body)
func AddUserController(w http.ResponseWriter, r *http.Request) {
	decoder := json.NewDecoder(r.Body)
	var user User
	decoder.Decode(&user)
	res := AddUser(user)
	json.NewEncoder(w).Encode(res)
}

// UpdateUserController will answer the JSON of the
// modified user (from the JSON Body)
func UpdateUserController(w http.ResponseWriter, r *http.Request) {
	decoder := json.NewDecoder(r.Body)
	var user User
	decoder.Decode(&user)
	vars := mux.Vars(r)
	userID := vars["id"]
	isValid := VerifyUserRequest(r, bson.ObjectIdHex(userID))
	if !isValid {
		w.WriteHeader(http.StatusUnauthorized)
		json.NewEncoder(w).Encode(bson.M{"error": "Contenu Protégé"})
		return
	}
	res := UpdateUser(bson.ObjectIdHex(userID), user)
	json.NewEncoder(w).Encode(res)
}

// DeleteUserController will answer a JSON of an
// empty user if the deletation has succeed
func DeleteUserController(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	userID := vars["id"]
	isValid := VerifyUserRequest(r, bson.ObjectIdHex(userID))
	if !isValid {
		w.WriteHeader(http.StatusUnauthorized)
		json.NewEncoder(w).Encode(bson.M{"error": "Contenu Protégé"})
		return
	}
	user := GetUser(bson.ObjectIdHex(userID))
	res := DeleteUser(user)
	json.NewEncoder(w).Encode(res)
}

func SearchUserController(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	users := SearchUser(vars["username"])
	json.NewEncoder(w).Encode(bson.M{"users": users})
}

func ReportUserController(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	userID := vars["id"]
	token := tauth.Get(r)
	reporterID := token.Claims("id").(string)
	ReportUser(bson.ObjectIdHex(userID), bson.ObjectIdHex(reporterID))
	json.NewEncoder(w).Encode(bson.M{})
}

func VerifyUserRequest(r *http.Request, userId bson.ObjectId) bool {
	token := tauth.Get(r)
	id := token.Claims("id").(string)
	return bson.ObjectIdHex(id) == userId
}
<commit_msg>master association can delete user<commit_after>package main

import (
	"encoding/json"
	"net/http"

	"gopkg.in/mgo.v2/bson"
	"github.com/freehaha/token-auth"
	"github.com/gorilla/mux"
)

// GetUserController will answer a JSON of the user
// linked to the given id in the URL
func GetUserController(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	userID := vars["id"]
	var res = GetUser(bson.ObjectIdHex(userID))
	json.NewEncoder(w).Encode(res)
}

func GetAllUserController(w http.ResponseWriter, r *http.Request) {
	var res = GetAllUser()
	json.NewEncoder(w).Encode(res)
}

// AddUserController will answer a JSON of the
// brand new created user (from the JSON Body)
func AddUserController(w http.ResponseWriter, r *http.Request) {
	decoder := json.NewDecoder(r.Body)
	var user User
	decoder.Decode(&user)
	res := AddUser(user)
	json.NewEncoder(w).Encode(res)
}

// UpdateUserController will answer the JSON of the
// modified user (from the JSON Body)
func UpdateUserController(w http.ResponseWriter, r *http.Request) {
	decoder := json.NewDecoder(r.Body)
	var user User
	decoder.Decode(&user)
	vars := mux.Vars(r)
	userID := vars["id"]
	isValidUser := VerifyUserRequest(r, bson.ObjectIdHex(userID))
	isValidAssociation := VerifyAssociationRequest(r, bson.ObjectIdHex(userID))
	if !isValidUser && !isValidAssociation {
		w.WriteHeader(http.StatusUnauthorized)
		json.NewEncoder(w).Encode(bson.M{"error": "Contenu Protégé"})
		return
	}
	res := UpdateUser(bson.ObjectIdHex(userID), user)
	json.NewEncoder(w).Encode(res)
}

// DeleteUserController will answer a JSON of an
// empty user if the deletation has succeed
func DeleteUserController(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	userID := vars["id"]
	isValid := VerifyUserRequest(r, bson.ObjectIdHex(userID))
	if !isValid {
		w.WriteHeader(http.StatusUnauthorized)
		json.NewEncoder(w).Encode(bson.M{"error": "Contenu Protégé"})
		return
	}
	user := GetUser(bson.ObjectIdHex(userID))
	res := DeleteUser(user)
	json.NewEncoder(w).Encode(res)
}

func SearchUserController(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	users := SearchUser(vars["username"])
	json.NewEncoder(w).Encode(bson.M{"users": users})
}

func ReportUserController(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	userID := vars["id"]
	token := tauth.Get(r)
	reporterID := token.Claims("id").(string)
	ReportUser(bson.ObjectIdHex(userID), bson.ObjectIdHex(reporterID))
	json.NewEncoder(w).Encode(bson.M{})
}

func VerifyUserRequest(r *http.Request, userId bson.ObjectId) bool {
	token := tauth.Get(r)
	id := token.Claims("id").(string)
	return bson.ObjectIdHex(id) == userId
}
<|endoftext|>


<commit_before>package ultralist

import (
	"encoding/json"
	"fmt"
	"regexp"
	"strconv"
	"strings"

	"github.com/manifoldco/promptui"
	"github.com/skratchdot/open-golang/open"
)

// the current version of ultralist
const (
	VERSION string = "0.9.2"
)

type App struct {
	EventLogger *EventLogger
	TodoStore   Store
	Printer     Printer
	TodoList    *TodoList
}

func NewApp() *App {
	app := &App{
		TodoList:  &TodoList{},
		Printer:   NewScreenPrinter(),
		TodoStore: NewFileStore(),
	}
	return app
}

func (a *App) InitializeRepo() {
	a.TodoStore.Initialize()
	fmt.Println("Repo initialized.")

	backend := NewBackend()
	eventLogger := &EventLogger{Store: a.TodoStore}
	eventLogger.LoadSyncedLists()

	if !backend.CredsFileExists() {
		return
	}

	prompt := promptui.Prompt{
		Label:     "Do you wish to sync this list with ultralist.io",
		IsConfirm: true,
	}

	result, _ := prompt.Run()
	if result != "y" {
		return
	}

	if !backend.CanConnect() {
		fmt.Println("I cannot connect to ultralist.io right now.")
		return
	}

	// fetch lists from ultralist.io, or allow user to create a new list
	// use the "select_add" example in promptui as a way to do this
	type Response struct {
		Todolists []TodoList `json:"todolists"`
	}

	var response *Response

	resp := backend.PerformRequest("GET", "/api/v1/todo_lists", []byte{})
	json.Unmarshal(resp, &response)

	var todolistNames []string
	for _, todolist := range response.Todolists {
		todolistNames = append(todolistNames, todolist.Name)
	}

	prompt2 := promptui.SelectWithAdd{
		Label:    "You can sync with an existing list on ultralist, or create a new list.",
		Items:    todolistNames,
		AddLabel: "New list...",
	}

	idx, name, _ := prompt2.Run()
	if idx == -1 {
		eventLogger.CurrentSyncedList.Name = name
	} else {
		eventLogger.CurrentSyncedList.Name = response.Todolists[idx].Name
		eventLogger.CurrentSyncedList.UUID = response.Todolists[idx].UUID
		a.TodoList = &response.Todolists[idx]
		a.save()
	}

	eventLogger.WriteSyncedLists()
}

func (a *App) AddTodo(input string) {
	a.Load()
	parser := &Parser{}
	todo := parser.ParseNewTodo(input)
	if todo == nil {
		fmt.Println("I need more information. Try something like 'todo a chat with @bob due tom'")
		return
	}

	id := a.TodoList.NextId()
	a.TodoList.Add(todo)
	a.save()
	fmt.Printf("Todo %d added.\n", id)
}

// AddDoneTodo Adds a todo and immediately completed it.
func (a *App) AddDoneTodo(input string) {
	a.Load()

	r, _ := regexp.Compile(`^(done)(\s*|)`)
	input = r.ReplaceAllString(input, "")
	parser := &Parser{}
	todo := parser.ParseNewTodo(input)
	if todo == nil {
		fmt.Println("I need more information. Try something like 'todo done chating with @bob'")
		return
	}

	id := a.TodoList.NextId()
	a.TodoList.Add(todo)
	a.TodoList.Complete(id)
	a.save()
	fmt.Printf("Completed Todo %d added.\n", id)
}

// DeleteTodo deletes a todo
func (a *App) DeleteTodo(input string) {
	a.Load()
	ids := a.getIds(input)
	if len(ids) == 0 {
		return
	}
	a.TodoList.Delete(ids...)
	a.save()
	fmt.Printf("%s deleted.\n", pluralize(len(ids), "Todo", "Todos"))
}

// CompleteTodo completes a todo
func (a *App) CompleteTodo(input string) {
	a.Load()
	ids := a.getIds(input)
	if len(ids) == 0 {
		return
	}
	a.TodoList.Complete(ids...)
	a.save()
	fmt.Println("Todo completed.")
}

// UncompleteTodo marks a todo as not completed
func (a *App) UncompleteTodo(input string) {
	a.Load()
	ids := a.getIds(input)
	if len(ids) == 0 {
		return
	}
	a.TodoList.Uncomplete(ids...)
	a.save()
	fmt.Println("Todo uncompleted.")
}

// ArchiveTodo marks a todo as archived
func (a *App) ArchiveTodo(input string) {
	a.Load()
	ids := a.getIds(input)
	if len(ids) == 0 {
		return
	}
	a.TodoList.Archive(ids...)
	a.save()
	fmt.Println("Todo archived.")
}

// UnarchiveTodo marks a todo as unarchived
func (a *App) UnarchiveTodo(input string) {
	a.Load()
	ids := a.getIds(input)
	if len(ids) == 0 {
		return
	}
	a.TodoList.Unarchive(ids...)
	a.save()
	fmt.Println("Todo unarchived.")
}

// EditTodo edit a todo with the input
func (a *App) EditTodo(input string) {
	a.Load()
	id := a.getId(input)
	if id == -1 {
		return
	}
	todo := a.TodoList.FindById(id)
	if todo == nil {
		fmt.Println("No such id.")
		return
	}
	parser := &Parser{}

	if parser.ParseEditTodo(todo, input) {
		a.save()
		fmt.Println("Todo updated.")
	}
}

// ExpandTodo expands a todo
func (a *App) ExpandTodo(input string) {
	a.Load()
	id := a.getId(input)
	parser := &Parser{}
	if id == -1 {
		return
	}

	commonProject := parser.ExpandProject(input)
	todos := strings.LastIndex(input, ":")
	if commonProject == "" || len(input) <= todos+1 || todos == -1 {
		fmt.Println("I'm expecting a format like \"todolist ex <project>: <todo1>, <todo2>, ...\"")
		return
	}

	newTodos := strings.Split(input[todos+1:], ",")

	for _, todo := range newTodos {
		args := []string{"add ", commonProject, " ", todo}
		a.AddTodo(strings.Join(args, ""))
	}

	a.TodoList.Delete(id)
	a.save()
	fmt.Println("Todo expanded.")
}

// HandleNotes is a sub-function that will handle notes on a todo.
func (a *App) HandleNotes(input string) {
	a.Load()
	id := a.getId(input)
	if id == -1 {
		return
	}
	todo := a.TodoList.FindById(id)
	if todo == nil {
		fmt.Println("No such id.")
		return
	}
	parser := &Parser{}

	if parser.ParseAddNote(todo, input) {
		fmt.Println("Note added.")
	} else if parser.ParseDeleteNote(todo, input) {
		fmt.Println("Note deleted.")
	} else if parser.ParseEditNote(todo, input) {
		fmt.Println("Note edited.")
	} else if parser.ParseShowNote(todo, input) {
		groups := map[string][]*Todo{}
		groups[""] = append(groups[""], todo)
		a.Printer.Print(&GroupedTodos{Groups: groups}, true)
		return
	}
	a.save()
}

// ArchiveCompleted will archive all completed todos
func (a *App) ArchiveCompleted() {
	a.Load()
	for _, todo := range a.TodoList.Todos() {
		if todo.Completed {
			todo.Archive()
		}
	}
	a.save()
	fmt.Println("All completed todos have been archived.")
}

// ListTodos will list all todos
func (a *App) ListTodos(input string) {
	a.Load()
	filtered := NewFilter(a.TodoList.Todos()).Filter(input)
	grouped := a.getGroups(input, filtered)

	re, _ := regexp.Compile(`^ln`)
	a.Printer.Print(grouped, re.MatchString(input))
}

// PrioritizeTodo will prioritize a todo
func (a *App) PrioritizeTodo(input string) {
	a.Load()
	ids := a.getIds(input)
	if len(ids) == 0 {
		return
	}
	a.TodoList.Prioritize(ids...)
	a.save()
	fmt.Println("Todo prioritized.")
}

// UnprioritizeTodo un-prioritizes a todo
func (a *App) UnprioritizeTodo(input string) {
	a.Load()
	ids := a.getIds(input)
	if len(ids) == 0 {
		return
	}
	a.TodoList.Unprioritize(ids...)
	a.save()
	fmt.Println("Todo un-prioritized.")
}

// GarbageCollect will delete all archived todos
func (a *App) GarbageCollect() {
	a.Load()
	a.TodoList.GarbageCollect()
	a.save()
	fmt.Println("Garbage collection complete.")
}

// Sync will sync the todolist with ultralist.io
func (a *App) Sync(input string) {
	a.Load()

	if a.EventLogger.CurrentSyncedList.Name == "" {
		prompt := promptui.Prompt{
			Label: "Give this list a name",
		}

		result, _ := prompt.Run()
		a.EventLogger.CurrentSyncedList.Name = result
	}

	synchronizer := NewSynchronizerWithInput(input)
	synchronizer.Sync(a.TodoList, a.EventLogger.CurrentSyncedList)

	if synchronizer.WasSuccessful() {
		a.EventLogger.ClearEventLogs()
		a.TodoStore.Save(a.TodoList.Data)
	}
}

func (a *App) CheckAuth() {
	synchronizer := NewSynchronizer()
	synchronizer.CheckAuth()
}

func (a *App) AuthWorkflow() {
	webapp := &Webapp{}
	backend := NewBackend()

	open.Start(backend.AuthUrl())
	fmt.Println("Listening for auth response...")
	webapp.Run()
}

// Load the todolist from the store
func (a *App) Load() error {
	todos, err := a.TodoStore.Load()
	if err != nil {
		return err
	}
	a.TodoList.Load(todos)
	a.EventLogger = NewEventLogger(a.TodoList, a.TodoStore)
	a.EventLogger.LoadSyncedLists()
	return nil
}

func (a *App) OpenWeb() {
	a.Load()
	if !a.TodoList.IsSynced {
		fmt.Println("This list isn't synced!  Use 'ultralist sync' to synchronize this list with ultralist.io.")
		return
	}

	open.Start("https://app.ultralist.io/todolist/" + a.TodoList.UUID)
}

// Save the todolist to the store
func (a *App) save() {
	a.TodoStore.Save(a.TodoList.Data)
	if a.TodoList.IsSynced {
		a.EventLogger.ProcessEvents()

		synchronizer := NewSynchronizer()
		synchronizer.ExecSyncInBackground()
	}
}

func (a *App) getId(input string) int {
	re, _ := regexp.Compile("\\d+")
	if re.MatchString(input) {
		id, _ := strconv.Atoi(re.FindString(input))
		return id
	}

	fmt.Println("Invalid id.")
	return -1
}

func (a *App) getIds(input string) (ids []int) {
	idGroups := strings.Split(input, ",")
	for _, idGroup := range idGroups {
		if rangedIds, err := a.parseRangedIds(idGroup); len(rangedIds) > 0 || err != nil {
			if err != nil {
				fmt.Printf("Invalid id group: %s.\n", input)
				continue
			}
			ids = append(ids, rangedIds...)
		} else if id := a.getId(idGroup); id != -1 {
			ids = append(ids, id)
		} else {
			fmt.Printf("Invalid id: %s.\n", idGroup)
		}
	}
	return ids
}

func (a *App) parseRangedIds(input string) (ids []int, err error) {
	rangeNumberRE, _ := regexp.Compile("(\\d+)-(\\d+)")
	if matches := rangeNumberRE.FindStringSubmatch(input); len(matches) > 0 {
		lowerID, _ := strconv.Atoi(matches[1])
		upperID, _ := strconv.Atoi(matches[2])
		if lowerID >= upperID {
			return ids, fmt.Errorf("Invalid id group: %s.\n", input)
		}
		for id := lowerID; id <= upperID; id++ {
			ids = append(ids, id)
		}
	}
	return ids, err
}

func (a *App) getGroups(input string, todos []*Todo) *GroupedTodos {
	grouper := &Grouper{}
	contextRegex, _ := regexp.Compile(`by c.*$`)
	projectRegex, _ := regexp.Compile(`by p.*$`)

	var grouped *GroupedTodos

	if contextRegex.MatchString(input) {
		grouped = grouper.GroupByContext(todos)
	} else if projectRegex.MatchString(input) {
		grouped = grouper.GroupByProject(todos)
	} else {
		grouped = grouper.GroupByNothing(todos)
	}
	return grouped
}
<commit_msg>Bugfix - ultralist web<commit_after>package ultralist

import (
	"encoding/json"
	"fmt"
	"regexp"
	"strconv"
	"strings"

	"github.com/manifoldco/promptui"
	"github.com/skratchdot/open-golang/open"
)

// the current version of ultralist
const (
	VERSION string = "0.9.2"
)

type App struct {
	EventLogger *EventLogger
	TodoStore   Store
	Printer     Printer
	TodoList    *TodoList
}

func NewApp() *App {
	app := &App{
		TodoList:  &TodoList{},
		Printer:   NewScreenPrinter(),
		TodoStore: NewFileStore(),
	}
	return app
}

func (a *App) InitializeRepo() {
	a.TodoStore.Initialize()
	fmt.Println("Repo initialized.")

	backend := NewBackend()
	eventLogger := &EventLogger{Store: a.TodoStore}
	eventLogger.LoadSyncedLists()

	if !backend.CredsFileExists() {
		return
	}

	prompt := promptui.Prompt{
		Label:     "Do you wish to sync this list with ultralist.io",
		IsConfirm: true,
	}

	result, _ := prompt.Run()
	if result != "y" {
		return
	}

	if !backend.CanConnect() {
		fmt.Println("I cannot connect to ultralist.io right now.")
		return
	}

	// fetch lists from ultralist.io, or allow user to create a new list
	// use the "select_add" example in promptui as a way to do this
	type Response struct {
		Todolists []TodoList `json:"todolists"`
	}

	var response *Response

	resp := backend.PerformRequest("GET", "/api/v1/todo_lists", []byte{})
	json.Unmarshal(resp, &response)

	var todolistNames []string
	for _, todolist := range response.Todolists {
		todolistNames = append(todolistNames, todolist.Name)
	}

	prompt2 := promptui.SelectWithAdd{
		Label:    "You can sync with an existing list on ultralist, or create a new list.",
		Items:    todolistNames,
		AddLabel: "New list...",
	}

	idx, name, _ := prompt2.Run()
	if idx == -1 {
		eventLogger.CurrentSyncedList.Name = name
	} else {
		eventLogger.CurrentSyncedList.Name = response.Todolists[idx].Name
		eventLogger.CurrentSyncedList.UUID = response.Todolists[idx].UUID
		a.TodoList = &response.Todolists[idx]
		a.save()
	}

	eventLogger.WriteSyncedLists()
}

func (a *App) AddTodo(input string) {
	a.Load()
	parser := &Parser{}
	todo := parser.ParseNewTodo(input)
	if todo == nil {
		fmt.Println("I need more information. Try something like 'todo a chat with @bob due tom'")
		return
	}

	id := a.TodoList.NextId()
	a.TodoList.Add(todo)
	a.save()
	fmt.Printf("Todo %d added.\n", id)
}

// AddDoneTodo Adds a todo and immediately completed it.
func (a *App) AddDoneTodo(input string) {
	a.Load()

	r, _ := regexp.Compile(`^(done)(\s*|)`)
	input = r.ReplaceAllString(input, "")
	parser := &Parser{}
	todo := parser.ParseNewTodo(input)
	if todo == nil {
		fmt.Println("I need more information. Try something like 'todo done chating with @bob'")
		return
	}

	id := a.TodoList.NextId()
	a.TodoList.Add(todo)
	a.TodoList.Complete(id)
	a.save()
	fmt.Printf("Completed Todo %d added.\n", id)
}

// DeleteTodo deletes a todo
func (a *App) DeleteTodo(input string) {
	a.Load()
	ids := a.getIds(input)
	if len(ids) == 0 {
		return
	}
	a.TodoList.Delete(ids...)
	a.save()
	fmt.Printf("%s deleted.\n", pluralize(len(ids), "Todo", "Todos"))
}

// CompleteTodo completes a todo
func (a *App) CompleteTodo(input string) {
	a.Load()
	ids := a.getIds(input)
	if len(ids) == 0 {
		return
	}
	a.TodoList.Complete(ids...)
	a.save()
	fmt.Println("Todo completed.")
}

// UncompleteTodo marks a todo as not completed
func (a *App) UncompleteTodo(input string) {
	a.Load()
	ids := a.getIds(input)
	if len(ids) == 0 {
		return
	}
	a.TodoList.Uncomplete(ids...)
	a.save()
	fmt.Println("Todo uncompleted.")
}

// ArchiveTodo marks a todo as archived
func (a *App) ArchiveTodo(input string) {
	a.Load()
	ids := a.getIds(input)
	if len(ids) == 0 {
		return
	}
	a.TodoList.Archive(ids...)
	a.save()
	fmt.Println("Todo archived.")
}

// UnarchiveTodo marks a todo as unarchived
func (a *App) UnarchiveTodo(input string) {
	a.Load()
	ids := a.getIds(input)
	if len(ids) == 0 {
		return
	}
	a.TodoList.Unarchive(ids...)
	a.save()
	fmt.Println("Todo unarchived.")
}

// EditTodo edit a todo with the input
func (a *App) EditTodo(input string) {
	a.Load()
	id := a.getId(input)
	if id == -1 {
		return
	}
	todo := a.TodoList.FindById(id)
	if todo == nil {
		fmt.Println("No such id.")
		return
	}
	parser := &Parser{}

	if parser.ParseEditTodo(todo, input) {
		a.save()
		fmt.Println("Todo updated.")
	}
}

// ExpandTodo expands a todo
func (a *App) ExpandTodo(input string) {
	a.Load()
	id := a.getId(input)
	parser := &Parser{}
	if id == -1 {
		return
	}

	commonProject := parser.ExpandProject(input)
	todos := strings.LastIndex(input, ":")
	if commonProject == "" || len(input) <= todos+1 || todos == -1 {
		fmt.Println("I'm expecting a format like \"todolist ex <project>: <todo1>, <todo2>, ...\"")
		return
	}

	newTodos := strings.Split(input[todos+1:], ",")

	for _, todo := range newTodos {
		args := []string{"add ", commonProject, " ", todo}
		a.AddTodo(strings.Join(args, ""))
	}

	a.TodoList.Delete(id)
	a.save()
	fmt.Println("Todo expanded.")
}

// HandleNotes is a sub-function that will handle notes on a todo.
func (a *App) HandleNotes(input string) {
	a.Load()
	id := a.getId(input)
	if id == -1 {
		return
	}
	todo := a.TodoList.FindById(id)
	if todo == nil {
		fmt.Println("No such id.")
		return
	}
	parser := &Parser{}

	if parser.ParseAddNote(todo, input) {
		fmt.Println("Note added.")
	} else if parser.ParseDeleteNote(todo, input) {
		fmt.Println("Note deleted.")
	} else if parser.ParseEditNote(todo, input) {
		fmt.Println("Note edited.")
	} else if parser.ParseShowNote(todo, input) {
		groups := map[string][]*Todo{}
		groups[""] = append(groups[""], todo)
		a.Printer.Print(&GroupedTodos{Groups: groups}, true)
		return
	}
	a.save()
}

// ArchiveCompleted will archive all completed todos
func (a *App) ArchiveCompleted() {
	a.Load()
	for _, todo := range a.TodoList.Todos() {
		if todo.Completed {
			todo.Archive()
		}
	}
	a.save()
	fmt.Println("All completed todos have been archived.")
}

// ListTodos will list all todos
func (a *App) ListTodos(input string) {
	a.Load()
	filtered := NewFilter(a.TodoList.Todos()).Filter(input)
	grouped := a.getGroups(input, filtered)

	re, _ := regexp.Compile(`^ln`)
	a.Printer.Print(grouped, re.MatchString(input))
}

// PrioritizeTodo will prioritize a todo
func (a *App) PrioritizeTodo(input string) {
	a.Load()
	ids := a.getIds(input)
	if len(ids) == 0 {
		return
	}
	a.TodoList.Prioritize(ids...)
	a.save()
	fmt.Println("Todo prioritized.")
}

// UnprioritizeTodo un-prioritizes a todo
func (a *App) UnprioritizeTodo(input string) {
	a.Load()
	ids := a.getIds(input)
	if len(ids) == 0 {
		return
	}
	a.TodoList.Unprioritize(ids...)
	a.save()
	fmt.Println("Todo un-prioritized.")
}

// GarbageCollect will delete all archived todos
func (a *App) GarbageCollect() {
	a.Load()
	a.TodoList.GarbageCollect()
	a.save()
	fmt.Println("Garbage collection complete.")
}

// Sync will sync the todolist with ultralist.io
func (a *App) Sync(input string) {
	a.Load()

	if a.EventLogger.CurrentSyncedList.Name == "" {
		prompt := promptui.Prompt{
			Label: "Give this list a name",
		}

		result, _ := prompt.Run()
		a.EventLogger.CurrentSyncedList.Name = result
	}

	synchronizer := NewSynchronizerWithInput(input)
	synchronizer.Sync(a.TodoList, a.EventLogger.CurrentSyncedList)

	if synchronizer.WasSuccessful() {
		a.EventLogger.ClearEventLogs()
		a.TodoStore.Save(a.TodoList.Data)
	}
}

func (a *App) CheckAuth() {
	synchronizer := NewSynchronizer()
	synchronizer.CheckAuth()
}

func (a *App) AuthWorkflow() {
	webapp := &Webapp{}
	backend := NewBackend()

	open.Start(backend.AuthUrl())
	fmt.Println("Listening for auth response...")
	webapp.Run()
}

// Load the todolist from the store
func (a *App) Load() error {
	todos, err := a.TodoStore.Load()
	if err != nil {
		return err
	}
	a.TodoList.Load(todos)
	a.EventLogger = NewEventLogger(a.TodoList, a.TodoStore)
	a.EventLogger.LoadSyncedLists()
	return nil
}

func (a *App) OpenWeb() {
	a.Load()
	if !a.TodoList.IsSynced {
		fmt.Println("This list isn't synced!  Use 'ultralist sync' to synchronize this list with ultralist.io.")
		return
	}

	fmt.Println("Opening this list on your browser...")
	open.Start("https://app.ultralist.io/todolist/" + a.EventLogger.CurrentSyncedList.UUID)
}

// Save the todolist to the store
func (a *App) save() {
	a.TodoStore.Save(a.TodoList.Data)
	if a.TodoList.IsSynced {
		a.EventLogger.ProcessEvents()

		synchronizer := NewSynchronizer()
		synchronizer.ExecSyncInBackground()
	}
}

func (a *App) getId(input string) int {
	re, _ := regexp.Compile("\\d+")
	if re.MatchString(input) {
		id, _ := strconv.Atoi(re.FindString(input))
		return id
	}

	fmt.Println("Invalid id.")
	return -1
}

func (a *App) getIds(input string) (ids []int) {
	idGroups := strings.Split(input, ",")
	for _, idGroup := range idGroups {
		if rangedIds, err := a.parseRangedIds(idGroup); len(rangedIds) > 0 || err != nil {
			if err != nil {
				fmt.Printf("Invalid id group: %s.\n", input)
				continue
			}
			ids = append(ids, rangedIds...)
		} else if id := a.getId(idGroup); id != -1 {
			ids = append(ids, id)
		} else {
			fmt.Printf("Invalid id: %s.\n", idGroup)
		}
	}
	return ids
}

func (a *App) parseRangedIds(input string) (ids []int, err error) {
	rangeNumberRE, _ := regexp.Compile("(\\d+)-(\\d+)")
	if matches := rangeNumberRE.FindStringSubmatch(input); len(matches) > 0 {
		lowerID, _ := strconv.Atoi(matches[1])
		upperID, _ := strconv.Atoi(matches[2])
		if lowerID >= upperID {
			return ids, fmt.Errorf("Invalid id group: %s.\n", input)
		}
		for id := lowerID; id <= upperID; id++ {
			ids = append(ids, id)
		}
	}
	return ids, err
}

func (a *App) getGroups(input string, todos []*Todo) *GroupedTodos {
	grouper := &Grouper{}
	contextRegex, _ := regexp.Compile(`by c.*$`)
	projectRegex, _ := regexp.Compile(`by p.*$`)

	var grouped *GroupedTodos

	if contextRegex.MatchString(input) {
		grouped = grouper.GroupByContext(todos)
	} else if projectRegex.MatchString(input) {
		grouped = grouper.GroupByProject(todos)
	} else {
		grouped = grouper.GroupByNothing(todos)
	}
	return grouped
}
<|endoftext|>


<commit_before>package util

import (
	"crypto/sha1"
	"encoding/hex"
	"encoding/json"
)

// Serialize an object to compact, deterministic JSON
// (no whitespace, fields in struct order or alphabetical),
// then take the SHA1 of that, and return the hex digest.
//
// This is the algorithm we use for hashing events, quorums,
// and IRC locations.
func HashObject(object interface{}) (string, error) {
	serialized, err := json.Marshal(object)
	if err != nil {
		return "", err
	}

	sum := sha1.Sum(serialized)
	return hex.EncodeToString(sum[:]), nil
}
<commit_msg>Make compatible with older golang versions<commit_after>package util

import (
	"crypto/sha1"
	"encoding/hex"
	"encoding/json"
)

// Serialize an object to compact, deterministic JSON
// (no whitespace, fields in struct order or alphabetical),
// then take the SHA1 of that, and return the hex digest.
//
// This is the algorithm we use for hashing events, quorums,
// and IRC locations.
func HashObject(object interface{}) (string, error) {
	serialized, err := json.Marshal(object)
	if err != nil {
		return "", err
	}

	hasher := sha1.New()
	_, _ = hasher.Write(serialized)

	sum := hasher.Sum(nil)
	return hex.EncodeToString(sum[:]), nil
}
<|endoftext|>


<commit_before>package packer

import (
	"fmt"
	"log"
	"sync"
)

// This is the key in configurations that is set to the name of the
// build.
const BuildNameConfigKey = "packer_build_name"

// This is the key in configurations that is set to "true" when Packer
// debugging is enabled.
const DebugConfigKey = "packer_debug"

// This is the key in configurations that is set to "true" when Packer
// force build is enabled.
const ForceConfigKey = "packer_force"

// A Build represents a single job within Packer that is responsible for
// building some machine image artifact. Builds are meant to be parallelized.
type Build interface {
	// Name is the name of the build. This is unique across a single template,
	// but not absolutely unique. This is meant more to describe to the user
	// what is being built rather than being a unique identifier.
	Name() string

	// Prepare configures the various components of this build and reports
	// any errors in doing so (such as syntax errors, validation errors, etc.)
	Prepare() error

	// Run runs the actual builder, returning an artifact implementation
	// of what is built. If anything goes wrong, an error is returned.
	Run(Ui, Cache) ([]Artifact, error)

	// Cancel will cancel a running build. This will block until the build
	// is actually completely cancelled.
	Cancel()

	// SetDebug will enable/disable debug mode. Debug mode is always
	// enabled by adding the additional key "packer_debug" to boolean
	// true in the configuration of the various components. This must
	// be called prior to Prepare.
	//
	// When SetDebug is set to true, parallelism between builds is
	// strictly prohibited.
	SetDebug(bool)

	// SetForce will enable/disable forcing a build when artifacts exist.
	//
	// When SetForce is set to true, existing artifacts from the build are
	// deleted prior to the build.
	SetForce(bool)
}

// A build struct represents a single build job, the result of which should
// be a single machine image artifact. This artifact may be comprised of
// multiple files, of course, but it should be for only a single provider
// (such as VirtualBox, EC2, etc.).
type coreBuild struct {
	name           string
	builder        Builder
	builderConfig  interface{}
	builderType    string
	hooks          map[string][]Hook
	postProcessors [][]coreBuildPostProcessor
	provisioners   []coreBuildProvisioner

	debug         bool
	force         bool
	l             sync.Mutex
	prepareCalled bool
}

// Keeps track of the post-processor and the configuration of the
// post-processor used within a build.
type coreBuildPostProcessor struct {
	processor         PostProcessor
	processorType     string
	config            interface{}
	keepInputArtifact bool
}

// Keeps track of the provisioner and the configuration of the provisioner
// within the build.
type coreBuildProvisioner struct {
	provisioner Provisioner
	config      []interface{}
}

// Returns the name of the build.
func (b *coreBuild) Name() string {
	return b.name
}

// Prepare prepares the build by doing some initialization for the builder
// and any hooks. This _must_ be called prior to Run.
func (b *coreBuild) Prepare() (err error) {
	b.l.Lock()
	defer b.l.Unlock()

	if b.prepareCalled {
		panic("prepare already called")
	}

	b.prepareCalled = true

	packerConfig := map[string]interface{}{
		BuildNameConfigKey: b.name,
		DebugConfigKey:     b.debug,
		ForceConfigKey:     b.force,
	}

	// Prepare the builder
	err = b.builder.Prepare(b.builderConfig, packerConfig)
	if err != nil {
		log.Printf("Build '%s' prepare failure: %s\n", b.name, err)
		return
	}

	// Prepare the provisioners
	for _, coreProv := range b.provisioners {
		configs := make([]interface{}, len(coreProv.config), len(coreProv.config)+1)
		copy(configs, coreProv.config)
		configs = append(configs, packerConfig)

		if err = coreProv.provisioner.Prepare(configs...); err != nil {
			return
		}
	}

	// Prepare the post-processors
	for _, ppSeq := range b.postProcessors {
		for _, corePP := range ppSeq {
			err = corePP.processor.Configure(corePP.config, packerConfig)
			if err != nil {
				return
			}
		}
	}

	return
}

// Runs the actual build. Prepare must be called prior to running this.
func (b *coreBuild) Run(originalUi Ui, cache Cache) ([]Artifact, error) {
	if !b.prepareCalled {
		panic("Prepare must be called first")
	}

	// Copy the hooks
	hooks := make(map[string][]Hook)
	for hookName, hookList := range b.hooks {
		hooks[hookName] = make([]Hook, len(hookList))
		copy(hooks[hookName], hookList)
	}

	// Add a hook for the provisioners if we have provisioners
	if len(b.provisioners) > 0 {
		provisioners := make([]Provisioner, len(b.provisioners))
		for i, p := range b.provisioners {
			provisioners[i] = p.provisioner
		}

		if _, ok := hooks[HookProvision]; !ok {
			hooks[HookProvision] = make([]Hook, 0, 1)
		}

		hooks[HookProvision] = append(hooks[HookProvision], &ProvisionHook{provisioners})
	}

	hook := &DispatchHook{hooks}
	artifacts := make([]Artifact, 0, 1)

	// The builder just has a normal Ui, but prefixed
	builderUi := &PrefixedUi{
		fmt.Sprintf("==> %s", b.Name()),
		fmt.Sprintf("    %s", b.Name()),
		originalUi,
	}

	log.Printf("Running builder: %s", b.builderType)
	builderArtifact, err := b.builder.Run(builderUi, hook, cache)
	if err != nil {
		return nil, err
	}

	// If there was no result, don't worry about running post-processors
	// because there is nothing they can do, just return.
	if builderArtifact == nil {
		return nil, nil
	}

	errors := make([]error, 0)
	keepOriginalArtifact := len(b.postProcessors) == 0

	// Run the post-processors
PostProcessorRunSeqLoop:
	for _, ppSeq := range b.postProcessors {
		priorArtifact := builderArtifact
		for i, corePP := range ppSeq {
			ppUi := &PrefixedUi{
				fmt.Sprintf("==> %s (%s)", b.Name(), corePP.processorType),
				fmt.Sprintf("    %s (%s)", b.Name(), corePP.processorType),
				originalUi,
			}

			builderUi.Say(fmt.Sprintf("Running post-processor: %s", corePP.processorType))
			artifact, keep, err := corePP.processor.PostProcess(ppUi, priorArtifact)
			if err != nil {
				errors = append(errors, fmt.Errorf("Post-processor failed: %s", err))
				continue PostProcessorRunSeqLoop
			}

			if artifact == nil {
				log.Println("Nil artifact, halting post-processor chain.")
				continue PostProcessorRunSeqLoop
			}

			keep = keep || corePP.keepInputArtifact
			if i == 0 {
				// This is the first post-processor. We handle deleting
				// previous artifacts a bit different because multiple
				// post-processors may be using the original and need it.
				if !keepOriginalArtifact && keep {
					log.Printf(
						"Flagging to keep original artifact from post-processor '%s'",
						corePP.processorType)
					keepOriginalArtifact = true
				}
			} else {
				// We have a prior artifact. If we want to keep it, we append
				// it to the results list. Otherwise, we destroy it.
				if keep {
					artifacts = append(artifacts, priorArtifact)
				} else {
					log.Printf("Deleting prior artifact from post-processor '%s'", corePP.processorType)
					if err := priorArtifact.Destroy(); err != nil {
						errors = append(errors, fmt.Errorf("Failed cleaning up prior artifact: %s", err))
					}
				}
			}

			priorArtifact = artifact
		}

		// Add on the last artifact to the results
		if priorArtifact != nil {
			artifacts = append(artifacts, priorArtifact)
		}
	}

	if keepOriginalArtifact {
		artifacts = append(artifacts, nil)
		copy(artifacts[1:], artifacts)
		artifacts[0] = builderArtifact
	} else {
		log.Printf("Deleting original artifact for build '%s'", b.name)
		if err := builderArtifact.Destroy(); err != nil {
			errors = append(errors, fmt.Errorf("Error destroying builder artifact: %s", err))
		}
	}

	if len(errors) > 0 {
		err = &MultiError{errors}
	}

	return artifacts, err
}

func (b *coreBuild) SetDebug(val bool) {
	if b.prepareCalled {
		panic("prepare has already been called")
	}

	b.debug = val
}

func (b *coreBuild) SetForce(val bool) {
	if b.prepareCalled {
		panic("prepare has already been called")
	}

	b.force = val
}

// Cancels the build if it is running.
func (b *coreBuild) Cancel() {
	b.builder.Cancel()
}
<commit_msg>packer: move constants into a single const()<commit_after>package packer

import (
	"fmt"
	"log"
	"sync"
)

const (
	// This is the key in configurations that is set to the name of the
	// build.
	BuildNameConfigKey = "packer_build_name"

	// This is the key in configurations that is set to "true" when Packer
	// debugging is enabled.
	DebugConfigKey = "packer_debug"

	// This is the key in configurations that is set to "true" when Packer
	// force build is enabled.
	ForceConfigKey = "packer_force"
)

// A Build represents a single job within Packer that is responsible for
// building some machine image artifact. Builds are meant to be parallelized.
type Build interface {
	// Name is the name of the build. This is unique across a single template,
	// but not absolutely unique. This is meant more to describe to the user
	// what is being built rather than being a unique identifier.
	Name() string

	// Prepare configures the various components of this build and reports
	// any errors in doing so (such as syntax errors, validation errors, etc.)
	Prepare() error

	// Run runs the actual builder, returning an artifact implementation
	// of what is built. If anything goes wrong, an error is returned.
	Run(Ui, Cache) ([]Artifact, error)

	// Cancel will cancel a running build. This will block until the build
	// is actually completely cancelled.
	Cancel()

	// SetDebug will enable/disable debug mode. Debug mode is always
	// enabled by adding the additional key "packer_debug" to boolean
	// true in the configuration of the various components. This must
	// be called prior to Prepare.
	//
	// When SetDebug is set to true, parallelism between builds is
	// strictly prohibited.
	SetDebug(bool)

	// SetForce will enable/disable forcing a build when artifacts exist.
	//
	// When SetForce is set to true, existing artifacts from the build are
	// deleted prior to the build.
	SetForce(bool)
}

// A build struct represents a single build job, the result of which should
// be a single machine image artifact. This artifact may be comprised of
// multiple files, of course, but it should be for only a single provider
// (such as VirtualBox, EC2, etc.).
type coreBuild struct {
	name           string
	builder        Builder
	builderConfig  interface{}
	builderType    string
	hooks          map[string][]Hook
	postProcessors [][]coreBuildPostProcessor
	provisioners   []coreBuildProvisioner

	debug         bool
	force         bool
	l             sync.Mutex
	prepareCalled bool
}

// Keeps track of the post-processor and the configuration of the
// post-processor used within a build.
type coreBuildPostProcessor struct {
	processor         PostProcessor
	processorType     string
	config            interface{}
	keepInputArtifact bool
}

// Keeps track of the provisioner and the configuration of the provisioner
// within the build.
type coreBuildProvisioner struct {
	provisioner Provisioner
	config      []interface{}
}

// Returns the name of the build.
func (b *coreBuild) Name() string {
	return b.name
}

// Prepare prepares the build by doing some initialization for the builder
// and any hooks. This _must_ be called prior to Run.
func (b *coreBuild) Prepare() (err error) {
	b.l.Lock()
	defer b.l.Unlock()

	if b.prepareCalled {
		panic("prepare already called")
	}

	b.prepareCalled = true

	packerConfig := map[string]interface{}{
		BuildNameConfigKey: b.name,
		DebugConfigKey:     b.debug,
		ForceConfigKey:     b.force,
	}

	// Prepare the builder
	err = b.builder.Prepare(b.builderConfig, packerConfig)
	if err != nil {
		log.Printf("Build '%s' prepare failure: %s\n", b.name, err)
		return
	}

	// Prepare the provisioners
	for _, coreProv := range b.provisioners {
		configs := make([]interface{}, len(coreProv.config), len(coreProv.config)+1)
		copy(configs, coreProv.config)
		configs = append(configs, packerConfig)

		if err = coreProv.provisioner.Prepare(configs...); err != nil {
			return
		}
	}

	// Prepare the post-processors
	for _, ppSeq := range b.postProcessors {
		for _, corePP := range ppSeq {
			err = corePP.processor.Configure(corePP.config, packerConfig)
			if err != nil {
				return
			}
		}
	}

	return
}

// Runs the actual build. Prepare must be called prior to running this.
func (b *coreBuild) Run(originalUi Ui, cache Cache) ([]Artifact, error) {
	if !b.prepareCalled {
		panic("Prepare must be called first")
	}

	// Copy the hooks
	hooks := make(map[string][]Hook)
	for hookName, hookList := range b.hooks {
		hooks[hookName] = make([]Hook, len(hookList))
		copy(hooks[hookName], hookList)
	}

	// Add a hook for the provisioners if we have provisioners
	if len(b.provisioners) > 0 {
		provisioners := make([]Provisioner, len(b.provisioners))
		for i, p := range b.provisioners {
			provisioners[i] = p.provisioner
		}

		if _, ok := hooks[HookProvision]; !ok {
			hooks[HookProvision] = make([]Hook, 0, 1)
		}

		hooks[HookProvision] = append(hooks[HookProvision], &ProvisionHook{provisioners})
	}

	hook := &DispatchHook{hooks}
	artifacts := make([]Artifact, 0, 1)

	// The builder just has a normal Ui, but prefixed
	builderUi := &PrefixedUi{
		fmt.Sprintf("==> %s", b.Name()),
		fmt.Sprintf("    %s", b.Name()),
		originalUi,
	}

	log.Printf("Running builder: %s", b.builderType)
	builderArtifact, err := b.builder.Run(builderUi, hook, cache)
	if err != nil {
		return nil, err
	}

	// If there was no result, don't worry about running post-processors
	// because there is nothing they can do, just return.
	if builderArtifact == nil {
		return nil, nil
	}

	errors := make([]error, 0)
	keepOriginalArtifact := len(b.postProcessors) == 0

	// Run the post-processors
PostProcessorRunSeqLoop:
	for _, ppSeq := range b.postProcessors {
		priorArtifact := builderArtifact
		for i, corePP := range ppSeq {
			ppUi := &PrefixedUi{
				fmt.Sprintf("==> %s (%s)", b.Name(), corePP.processorType),
				fmt.Sprintf("    %s (%s)", b.Name(), corePP.processorType),
				originalUi,
			}

			builderUi.Say(fmt.Sprintf("Running post-processor: %s", corePP.processorType))
			artifact, keep, err := corePP.processor.PostProcess(ppUi, priorArtifact)
			if err != nil {
				errors = append(errors, fmt.Errorf("Post-processor failed: %s", err))
				continue PostProcessorRunSeqLoop
			}

			if artifact == nil {
				log.Println("Nil artifact, halting post-processor chain.")
				continue PostProcessorRunSeqLoop
			}

			keep = keep || corePP.keepInputArtifact
			if i == 0 {
				// This is the first post-processor. We handle deleting
				// previous artifacts a bit different because multiple
				// post-processors may be using the original and need it.
				if !keepOriginalArtifact && keep {
					log.Printf(
						"Flagging to keep original artifact from post-processor '%s'",
						corePP.processorType)
					keepOriginalArtifact = true
				}
			} else {
				// We have a prior artifact. If we want to keep it, we append
				// it to the results list. Otherwise, we destroy it.
				if keep {
					artifacts = append(artifacts, priorArtifact)
				} else {
					log.Printf("Deleting prior artifact from post-processor '%s'", corePP.processorType)
					if err := priorArtifact.Destroy(); err != nil {
						errors = append(errors, fmt.Errorf("Failed cleaning up prior artifact: %s", err))
					}
				}
			}

			priorArtifact = artifact
		}

		// Add on the last artifact to the results
		if priorArtifact != nil {
			artifacts = append(artifacts, priorArtifact)
		}
	}

	if keepOriginalArtifact {
		artifacts = append(artifacts, nil)
		copy(artifacts[1:], artifacts)
		artifacts[0] = builderArtifact
	} else {
		log.Printf("Deleting original artifact for build '%s'", b.name)
		if err := builderArtifact.Destroy(); err != nil {
			errors = append(errors, fmt.Errorf("Error destroying builder artifact: %s", err))
		}
	}

	if len(errors) > 0 {
		err = &MultiError{errors}
	}

	return artifacts, err
}

func (b *coreBuild) SetDebug(val bool) {
	if b.prepareCalled {
		panic("prepare has already been called")
	}

	b.debug = val
}

func (b *coreBuild) SetForce(val bool) {
	if b.prepareCalled {
		panic("prepare has already been called")
	}

	b.force = val
}

// Cancels the build if it is running.
func (b *coreBuild) Cancel() {
	b.builder.Cancel()
}
<|endoftext|>


<commit_before>//这里实现bitmap索引
package utils

import (
	"fmt"
	"math/big"
	"sort"
)

type BitmapIndex struct { // bitmap索引数据结构
	Data  []byte //数据用[]byte存放,一个元素(block)8bit
	Ext   []byte //扩展数据用[]byte存放,一个元素(block)8bit
	Start int    //开始的block
	End   int    //结束的block
}

/* {{{ func ReadIntFromBytes(bs []byte) (r int)
 *
 */
func ReadIntFromBytes(bs []byte) (r int) {
	l := len(bs)
	for i, b := range bs {
		shift := uint((l - i - 1) * 8)
		r |= int(b) << shift
	}
	return
}

/* }}} */

/* {{{ func NewBitmapIndex(s []int) *BitmapIndex
 * 根据一个整数slice建立索引
 */
func NewBitmapIndex(s []int) *BitmapIndex {
	sort.Ints(s) //先排序
	bi := new(BitmapIndex)
	bi.Start = s[0] / 8
	bi.End = s[len(s)-1] / 8

	b := big.NewInt(0)
	one := big.NewInt(1)
	rcver := big.NewInt(0)

	for _, sv := range s {
		offset := sv - bi.Start*8 //差多少就有多少位
		b.Or(b, rcver.Lsh(one, uint(offset)))
	}
	bi.Data = b.Bytes()
	return bi
}

/* }}} */

/* {{{ func ReadBitmapIndex(ib []byte) (*BitmapIndex,error)
 * 从[]byte里读取BitmapIndex,应用场景为从文件或者内存中拿到[]byte,转化为索引
 */
func ReadBitmapIndex(ib []byte) (bi *BitmapIndex, err error) {
	il := len(ib)
	if il <= 8 { //至少比8大
		return nil, fmt.Errorf("can't read from %s", ib)
	}

	bi = new(BitmapIndex)
	start := ReadIntFromBytes(ib[:4])
	end := ReadIntFromBytes(ib[4:8])
	bi.Start = start / 8
	bi.End = end / 8
	dl := bi.End - bi.Start + 1 //数据长度
	bi.Data = ib[8:(8 + dl)]
	bi.Ext = ib[(8 + dl):] //剩余长度为额外的

	return
}

/* }}} */

/* {{{ func (bi *BitmapIndex) Bytes() ([]byte, error)
 * 将BitmapIndex转化为[]byte,方便存放到文件或者内存中去
 */
func (bi *BitmapIndex) Bytes() (ib []byte, err error) {
	bl := len(bi.Data)
	if bl <= 0 {
		return nil, fmt.Errorf("index data empty")
	}
	start := bi.Start * 8
	end := bi.End * 8
	bs := big.NewInt(int64(start)).Bytes()
	be := big.NewInt(int64(end)).Bytes()

	ib = make([]byte, 8+bl)   //8+数据长度
	copy(ib[4-len(bs):4], bs) //1-4字节保存开始block
	copy(ib[8-len(be):8], be) // 5-8字节保存结束block
	copy(ib[8:], bi.Data)

	return
}

/* }}} */

/* {{{ func (bi *BitmapIndex) Slices() ([]int, error)
 * 将BitmapIndex转化为[]int
 */
func (bi *BitmapIndex) Slices() (s []int, err error) {
	if bi == nil || len(bi.Data) <= 0 {
		return nil, fmt.Errorf("not found item")
	}
	s = make([]int, 0)
	Len := len(bi.Data)
	//for i, b := range bi.Data {
	for i := (Len - 1); i >= 0; i-- {
		b := bi.Data[i]
		if b > 0 { //双方都大于0才有比较的意义
			for ii := 0; ii < 8; ii++ { //遍历8bit
				if b&(1<<uint(ii)) > 0 { //找到交集位置！
					shift := ii + (Len-i-1)*8 //偏移量
					s = append(s, bi.Start*8+shift)
				}
			}
		}
	}

	return
}

/* }}} */

/* {{{ func (bi *BitmapIndex) And(obi *BitmapIndex) *BitmapIndex
 * 求交集
 */
func (bi *BitmapIndex) And(obi *BitmapIndex) (nbi *BitmapIndex) {
	if bi == nil {
		return nil
	}
	if bi.End < obi.Start || obi.End < bi.Start {
		//不可能有交集
		return nil
	}
	var start, end int
	if bi.Start < obi.Start {
		//以大的start为准
		start = obi.Start
	} else {
		start = bi.Start
	}
	if bi.End < obi.End {
		//以小的end为准
		end = bi.End
	} else {
		end = obi.End
	}

	//得到两个索引的重叠部分(注意是从右往左)
	data1 := bi.Data[bi.End-end : len(bi.Data)-(start-bi.Start)]
	data2 := obi.Data[obi.End-end : len(obi.Data)-(start-obi.Start)]

	nbi = new(BitmapIndex)
	nbi.Start = start
	nbi.End = end

	Len := end - start + 1
	nbi.Data = make([]byte, Len)

	var matched bool
	for i, b1 := range data1 {
		b2 := data2[i]
		if b1 > 0 && b2 > 0 {
			if b3 := b1 & b2; b3 > 0 {
				if matched == false {
					matched = true
				}
				nbi.Data[i] = b3
			} else {
				nbi.Data[i] = 0
			}
		} else {
			nbi.Data[i] = 0
		}
	}

	if !matched { //没有重合的部分,返回空
		return nil
	}

	return
}

/* }}} */

/* {{{ func (bi *BitmapIndex) Not(obi *BitmapIndex) *BitmapIndex
 * 求差集
 */
func (bi *BitmapIndex) Not(obi *BitmapIndex) (nbi *BitmapIndex) {
	nbi = bi
	if bi == nil || obi == nil {
		return
	}
	if bi.End < obi.Start || obi.End < bi.Start {
		//直接返回bi
		return
	}
	var cStart, cEnd int //交集部分的起止
	if bi.Start < obi.Start {
		//以大的Start为准
		cStart = obi.Start
	} else {
		cStart = bi.Start
	}
	if bi.End < obi.End {
		//以小的End为准
		cEnd = bi.End
	} else {
		cEnd = obi.End
	}

	//得到两个索引的重叠部分
	data1 := bi.Data[bi.End-cEnd : len(bi.Data)-(cStart-bi.Start)]
	data2 := obi.Data[obi.End-cEnd : len(obi.Data)-(cStart-obi.Start)]

	//重叠部分求差集, 其余部分保持原状
	for i, b1 := range data1 {
		b2 := data2[i]
		offset := nbi.End - cEnd + i
		if b1 > 0 && b2 > 0 {
			nbi.Data[offset] = b1 &^ b2 //b2为1的位都清零
		}
	}

	return
}

/* }}} */

/* {{{ func (bi *BitmapIndex) Or(obi *BitmapIndex) *BitmapIndex
 * 求合集
 */
func (bi *BitmapIndex) Or(obi *BitmapIndex) (nbi *BitmapIndex) {
	if bi == nil {
		return obi
	} else if obi == nil {
		return bi
	}
	var start, end int
	var cStart, cEnd int //交集部分的起止
	if bi.Start < obi.Start {
		//以小的start为准
		start = bi.Start
		cStart = obi.Start
	} else {
		start = obi.Start
		cStart = bi.Start
	}
	if bi.End < obi.End {
		//以大的end为准
		end = obi.End
		cEnd = bi.End
	} else {
		end = bi.End
		cEnd = obi.End
	}

	//得到两个索引的重叠部分
	data1 := bi.Data[bi.End-cEnd : len(bi.Data)-(cStart-bi.Start)]
	data2 := obi.Data[obi.End-cEnd : len(obi.Data)-(cStart-obi.Start)]

	nbi = new(BitmapIndex)
	nbi.Start = start
	nbi.End = end

	Len := end - start + 1
	nbi.Data = make([]byte, Len)

	//copy
	copy(nbi.Data[end-bi.End:Len-(bi.Start-start)], bi.Data)
	copy(nbi.Data[end-obi.End:Len-(obi.Start-start)], obi.Data)

	for i, b1 := range data1 {
		b2 := data2[i]
		offset := nbi.End - cEnd + i
		if b1 > 0 || b2 > 0 {
			nbi.Data[offset] = b1 | b2 // or操作
		}
	}

	return
}

/* }}} */

/* {{{ func OR(bis []*BitmapIndex) *BitmapIndex
 * 求所有索引并集
 */
func OR(bis []*BitmapIndex) (obi *BitmapIndex) {
	if len(bis) <= 0 {
		return nil
	}
	for _, bi := range bis {
		obi = obi.Or(bi)
	}
	return
}

/* }}} */
<commit_msg>update<commit_after>//这里实现bitmap索引
package utils

import (
	"fmt"
	"math/big"
	"sort"
)

type BitmapIndex struct { // bitmap索引数据结构
	Data  []byte //数据用[]byte存放,一个元素(block)8bit
	Ext   []byte //扩展数据,也是[]byte
	Start int    //开始的block
	End   int    //结束的block
}

/* {{{ func ReadIntFromBytes(bs []byte) (r int)
 *
 */
func ReadIntFromBytes(bs []byte) (r int) {
	l := len(bs)
	for i, b := range bs {
		shift := uint((l - i - 1) * 8)
		r |= int(b) << shift
	}
	return
}

/* }}} */

/* {{{ func NewBitmapIndex(s []int) *BitmapIndex
 * 根据一个整数slice建立索引
 */
func NewBitmapIndex(s []int) *BitmapIndex {
	if len(s) == 0 {
		return nil
	}
	sort.Ints(s) //先排序
	bi := new(BitmapIndex)
	bi.Start = s[0] / 8
	bi.End = s[len(s)-1] / 8

	b := big.NewInt(0)
	one := big.NewInt(1)
	rcver := big.NewInt(0)

	for _, sv := range s {
		offset := sv - bi.Start*8 //差多少就有多少位
		b.Or(b, rcver.Lsh(one, uint(offset)))
	}
	bi.Data = b.Bytes()
	return bi
}

/* }}} */

/* {{{ func ReadBitmapIndex(ib []byte) (*BitmapIndex,error)
 * 从[]byte里读取BitmapIndex,应用场景为从文件或者内存中拿到[]byte,转化为索引
 */
func ReadBitmapIndex(ib []byte) (bi *BitmapIndex, err error) {
	il := len(ib)
	if il <= 8 { //至少比8大
		return nil, fmt.Errorf("can't read from %s", ib)
	}

	bi = new(BitmapIndex)
	start := ReadIntFromBytes(ib[:4])
	end := ReadIntFromBytes(ib[4:8])
	bi.Start = start / 8
	bi.End = end / 8
	dl := bi.End - bi.Start + 1 //数据长度
	bi.Data = ib[8:(8 + dl)]
	bi.Ext = ib[(8 + dl):] //剩余长度为额外的

	return
}

/* }}} */

/* {{{ func (bi *BitmapIndex) Bytes() ([]byte, error)
 * 将BitmapIndex转化为[]byte,方便存放到文件或者内存中去
 */
func (bi *BitmapIndex) Bytes() (ib []byte, err error) {
	bl := len(bi.Data)
	if bl <= 0 {
		return nil, fmt.Errorf("index data empty")
	}
	start := bi.Start * 8
	end := bi.End * 8
	bs := big.NewInt(int64(start)).Bytes()
	be := big.NewInt(int64(end)).Bytes()

	ib = make([]byte, 8+bl)   //8+数据长度
	copy(ib[4-len(bs):4], bs) //1-4字节保存开始block
	copy(ib[8-len(be):8], be) // 5-8字节保存结束block
	copy(ib[8:], bi.Data)

	return
}

/* }}} */

/* {{{ func (bi *BitmapIndex) Slices() ([]int, error)
 * 将BitmapIndex转化为[]int
 */
func (bi *BitmapIndex) Slices() (s []int, err error) {
	if bi == nil || len(bi.Data) <= 0 {
		return nil, fmt.Errorf("not found item")
	}
	s = make([]int, 0)
	Len := len(bi.Data)
	//for i, b := range bi.Data {
	for i := (Len - 1); i >= 0; i-- {
		b := bi.Data[i]
		if b > 0 { //双方都大于0才有比较的意义
			for ii := 0; ii < 8; ii++ { //遍历8bit
				if b&(1<<uint(ii)) > 0 { //找到交集位置！
					shift := ii + (Len-i-1)*8 //偏移量
					s = append(s, bi.Start*8+shift)
				}
			}
		}
	}

	return
}

/* }}} */

/* {{{ func (bi *BitmapIndex) And(obi *BitmapIndex) *BitmapIndex
 * 求交集
 */
func (bi *BitmapIndex) And(obi *BitmapIndex) (nbi *BitmapIndex) {
	if bi == nil {
		return nil
	}
	if bi.End < obi.Start || obi.End < bi.Start {
		//不可能有交集
		return nil
	}
	var start, end int
	if bi.Start < obi.Start {
		//以大的start为准
		start = obi.Start
	} else {
		start = bi.Start
	}
	if bi.End < obi.End {
		//以小的end为准
		end = bi.End
	} else {
		end = obi.End
	}

	//得到两个索引的重叠部分(注意是从右往左)
	data1 := bi.Data[bi.End-end : len(bi.Data)-(start-bi.Start)]
	data2 := obi.Data[obi.End-end : len(obi.Data)-(start-obi.Start)]

	nbi = new(BitmapIndex)
	nbi.Start = start
	nbi.End = end

	Len := end - start + 1
	nbi.Data = make([]byte, Len)

	var matched bool
	for i, b1 := range data1 {
		b2 := data2[i]
		if b1 > 0 && b2 > 0 {
			if b3 := b1 & b2; b3 > 0 {
				if matched == false {
					matched = true
				}
				nbi.Data[i] = b3
			} else {
				nbi.Data[i] = 0
			}
		} else {
			nbi.Data[i] = 0
		}
	}

	if !matched { //没有重合的部分,返回空
		return nil
	}

	return
}

/* }}} */

/* {{{ func (bi *BitmapIndex) AndBreak(obis []*BitmapIndex) *BitmapIndex
 *  第一个交集不为空
 */
func (bi *BitmapIndex) AndBreak(obis []*BitmapIndex) (nbi *BitmapIndex) {
	if bi == nil {
		return nil
	}
	for _, obi := range obis {
		if tbi := bi.And(obi); tbi != nil {
			return tbi
		}
	}

	return
}

/* }}} */

/* {{{ func (bi *BitmapIndex) Not(obi *BitmapIndex) *BitmapIndex
 * 求差集
 */
func (bi *BitmapIndex) Not(obi *BitmapIndex) (nbi *BitmapIndex) {
	nbi = bi
	if bi == nil || obi == nil {
		return
	}
	if bi.End < obi.Start || obi.End < bi.Start {
		//直接返回bi
		return
	}
	var cStart, cEnd int //交集部分的起止
	if bi.Start < obi.Start {
		//以大的Start为准
		cStart = obi.Start
	} else {
		cStart = bi.Start
	}
	if bi.End < obi.End {
		//以小的End为准
		cEnd = bi.End
	} else {
		cEnd = obi.End
	}

	//得到两个索引的重叠部分
	data1 := bi.Data[bi.End-cEnd : len(bi.Data)-(cStart-bi.Start)]
	data2 := obi.Data[obi.End-cEnd : len(obi.Data)-(cStart-obi.Start)]

	//重叠部分求差集, 其余部分保持原状
	for i, b1 := range data1 {
		b2 := data2[i]
		offset := nbi.End - cEnd + i
		if b1 > 0 && b2 > 0 {
			nbi.Data[offset] = b1 &^ b2 //b2为1的位都清零
		}
	}

	return
}

/* }}} */

/* {{{ func (bi *BitmapIndex) Or(obi *BitmapIndex) *BitmapIndex
 * 求合集
 */
func (bi *BitmapIndex) Or(obi *BitmapIndex) (nbi *BitmapIndex) {
	if bi == nil {
		return obi
	} else if obi == nil {
		return bi
	}
	var start, end int
	var cStart, cEnd int //交集部分的起止
	if bi.Start < obi.Start {
		//以小的start为准
		start = bi.Start
		cStart = obi.Start
	} else {
		start = obi.Start
		cStart = bi.Start
	}
	if bi.End < obi.End {
		//以大的end为准
		end = obi.End
		cEnd = bi.End
	} else {
		end = bi.End
		cEnd = obi.End
	}

	nbi = new(BitmapIndex)
	nbi.Start = start
	nbi.End = end

	Len := end - start + 1
	nbi.Data = make([]byte, Len)

	//copy
	copy(nbi.Data[end-bi.End:Len-(bi.Start-start)], bi.Data)
	copy(nbi.Data[end-obi.End:Len-(obi.Start-start)], obi.Data)

	if bi.End < obi.Start || obi.End < bi.Start {
		//没有有交集, 直接相加返回
		return
	}

	//有交集, 得到两个索引的重叠部分
	data1 := bi.Data[bi.End-cEnd : len(bi.Data)-(cStart-bi.Start)]
	data2 := obi.Data[obi.End-cEnd : len(obi.Data)-(cStart-obi.Start)]

	for i, b1 := range data1 {
		b2 := data2[i]
		offset := nbi.End - cEnd + i
		if b1 > 0 || b2 > 0 {
			nbi.Data[offset] = b1 | b2 // or操作
		}
	}

	return
}

/* }}} */

/* {{{ func OR(bis []*BitmapIndex) *BitmapIndex
 * 求所有索引并集
 */
func OR(bis []*BitmapIndex) (obi *BitmapIndex) {
	if len(bis) <= 0 {
		return nil
	}
	for _, bi := range bis {
		obi = obi.Or(bi)
	}
	return
}

/* }}} */
<|endoftext|>


<commit_before>package utils

import (
	"errors"

	"github.com/oklog/ulid"
)

// ErrDisposed is returned when a queue is accessed after being disposed.
var ErrDisposed = errors.New(`queue: disposed`)

// ErrTimeout is returned by queues after the provided timeout is expired.
var ErrTimeout = errors.New(`queue: poll timed out`)

// ErrEmptyQueue is returned when an non-applicable queue operation was called
var ErrEmptyQueue = errors.New(`queue: empty queue`)

// EmptyUID is a zero ULID
var EmptyUID ulid.ULID
<commit_msg>IsBrokenPipe<commit_after>package utils

import (
	"errors"
	"net"
	"os"
	"syscall"

	"github.com/oklog/ulid"
)

// ErrDisposed is returned when a queue is accessed after being disposed.
var ErrDisposed = errors.New(`queue: disposed`)

// ErrTimeout is returned by queues after the provided timeout is expired.
var ErrTimeout = errors.New(`queue: poll timed out`)

// ErrEmptyQueue is returned when an non-applicable queue operation was called
var ErrEmptyQueue = errors.New(`queue: empty queue`)

// EmptyUID is a zero ULID
var EmptyUID ulid.ULID

func IsBrokenPipe(err error) bool {
	if err == nil {
		return false
	}
	if perr, ok := err.(*os.PathError); ok {
		if serr, ok := (perr.Err).(*os.SyscallError); ok {
			return serr.Err == syscall.EPIPE
		}
	}
	if operr, ok := err.(*net.OpError); ok {
		if serr, ok := (operr.Err).(*os.SyscallError); ok {
			return serr.Err == syscall.EPIPE
		}
	}
	return false
}
<|endoftext|>