<commit_before>package twitch

import (
	"bufio"
	"fmt"
	"net"
	"strings"
	"time"

	"github.com/sorcix/irc"
)

// a message on the queue, this is not what the outside world sees
type queueItem struct {
	message irc.Message
	signal  chan bool
}

type TwitchClient struct {
	server   string
	username string
	password string

	// connection handling
	conn   net.Conn
	reader *bufio.Reader
	writer *irc.Encoder

	// handlers for incoming messages
	handlers map[string]HandlerFunc

	// time between two regular messages are sent
	delay time.Duration

	// this signal is sent when the client has sent the CAP REQ commands
	ready chan struct{}

	// this signal is sent when we disconnected
	alive chan struct{}

	// these are fired when .Disconnect() is called
	stopSending   chan struct{}
	stopReceiving chan struct{}

	// this is fired when .sender() / .receiver() stop
	stoppedSending   chan struct{}
	stoppedReceiving chan struct{}

	// on this channel incoming messages from the network are sent
	incoming chan Message

	// list of ougtoing messages (sent by us)
	outgoing chan queueItem
}

func NewTwitchClient(server string, username string, password string, delay time.Duration) *TwitchClient {
	client := &TwitchClient{
		server:           server,
		username:         username,
		password:         password,
		delay:            delay,
		conn:             nil,
		reader:           nil,
		writer:           nil,
		ready:            make(chan struct{}),
		alive:            make(chan struct{}),
		stopReceiving:    make(chan struct{}),
		stopSending:      make(chan struct{}),
		stoppedReceiving: make(chan struct{}),
		stoppedSending:   make(chan struct{}),
		incoming:         make(chan Message, 50),
		outgoing:         make(chan queueItem, 50),
	}

	// setup vital message listeners
	client.setupHandlers()

	return client
}

func (client *TwitchClient) Ready() <-chan struct{} {
	return client.ready
}

func (client *TwitchClient) Alive() <-chan struct{} {
	return client.alive
}

func (client *TwitchClient) Incoming() <-chan Message {
	return client.incoming
}

func (client *TwitchClient) Connect() error {
	conn, err := net.Dial("tcp", client.server)
	if err != nil {
		return err
	}

	client.conn = conn
	client.reader = bufio.NewReader(conn) // we manually read to properly handle tags
	client.writer = irc.NewEncoder(conn)

	// start working on the queue
	go client.sender()

	// start receiving
	go client.receiver()

	// send login info before anything else
	client.Send(irc.Message{
		Command: irc.PASS,
		Params:  []string{client.password},
	})

	client.Send(irc.Message{
		Command: irc.NICK,
		Params:  []string{client.username},
	})

	client.Send(irc.Message{
		Command: irc.USER,
		Params:  []string{"kabukibot", "8", "*", client.username},
	})

	return nil
}

func (client *TwitchClient) Disconnect() error {
	// stop the sender/receiver and wait for them to stop (maybe it will drain the
	// outgoing queue, maybe it won't, but let's give it time)
	close(client.stopReceiving)
	<-client.stoppedReceiving

	close(client.stopSending)
	<-client.stoppedSending

	// for all intents and purposes, we are not alive anymore
	close(client.alive)

	// close the IRC connection
	return client.conn.Close()
}

func (client *TwitchClient) Send(msg irc.Message) <-chan bool {
	signal := make(chan bool, 1)
	outgoing := queueItem{msg, signal}

	// if queue is not full, then
	client.outgoing <- outgoing
	// else
	// 	signal <- false // means "not sent"
	// 	close(signal)

	return signal
}

func (client *TwitchClient) sender() {
	for {
		select {
		case msg := <-client.outgoing:
			fmt.Println("< " + msg.message.String())
			client.writer.Encode(&msg.message)

			// signal to the one who sent the message that it was in fact sent
			msg.signal <- true
			close(msg.signal)

		case <-client.stopSending:
			break
		}
	}

	close(client.stoppedSending)
}

func (client *TwitchClient) receiver() {
	reading := make(chan struct{})

	// a buffer between the raw irc input from the net and the goroutine channels
	buffer := make(chan string, 10)

	// fork a reader loop, which could block and needs special handling as it's not a channel
	// (but it will pump its messages into a channel)
	go func() {
		for {
			select {
			case <-client.stopReceiving:
				break

			default:
				// set a 5min timeout
				client.conn.SetDeadline(time.Now().Add(300 * time.Second))

				line, err := client.reader.ReadString('\n')
				if err != nil {
					break
				}

				buffer <- line
			}
		}

		close(reading)
	}()

	for {
		select {
		case rawLine := <-buffer:
			fmt.Println("> " + strings.TrimSpace(rawLine))
			// if the message begins with a '@', we have some tags (IRCv3). The default
			// IRC decoder will not have properly detected it and mangled its output.
			// We fix that by manually splitting the tags from the rest of the message
			// and parse each part individually.
			tags := make(irc.Tags)
			msg := &irc.Message{}

			if strings.HasPrefix(rawLine, "@") {
				parts := strings.SplitN(rawLine, " ", 2)

				tags = irc.ParseTags(strings.TrimPrefix(parts[0], "@"))
				msg = irc.ParseMessage(parts[1])
			} else {
				msg = irc.ParseMessage(rawLine)
			}

			// hand it over to the message handler
			handler, ok := client.handlers[msg.Command]
			if ok {
				handler(msg, tags)
			}

		case <-client.stopReceiving:
			break
		}
	}

	close(client.stoppedReceiving)
}
<commit_msg>handle each message in its own goroutine, until all are synced again at the incoming queue channel that is to be consumed by the bot<commit_after>package twitch

import (
	"bufio"
	"fmt"
	"net"
	"strings"
	"time"

	"github.com/sorcix/irc"
)

// a message on the queue, this is not what the outside world sees
type queueItem struct {
	message irc.Message
	signal  chan bool
}

type TwitchClient struct {
	server   string
	username string
	password string

	// connection handling
	conn   net.Conn
	reader *bufio.Reader
	writer *irc.Encoder

	// handlers for incoming messages
	handlers map[string]HandlerFunc

	// time between two regular messages are sent
	delay time.Duration

	// this signal is sent when the client has sent the CAP REQ commands
	ready chan struct{}

	// this signal is sent when we disconnected
	alive chan struct{}

	// these are fired when .Disconnect() is called
	stopSending   chan struct{}
	stopReceiving chan struct{}

	// this is fired when .sender() / .receiver() stop
	stoppedSending   chan struct{}
	stoppedReceiving chan struct{}

	// on this channel incoming messages from the network are sent
	incoming chan Message

	// list of ougtoing messages (sent by us)
	outgoing chan queueItem
}

func NewTwitchClient(server string, username string, password string, delay time.Duration) *TwitchClient {
	client := &TwitchClient{
		server:           server,
		username:         username,
		password:         password,
		delay:            delay,
		conn:             nil,
		reader:           nil,
		writer:           nil,
		ready:            make(chan struct{}),
		alive:            make(chan struct{}),
		stopReceiving:    make(chan struct{}),
		stopSending:      make(chan struct{}),
		stoppedReceiving: make(chan struct{}),
		stoppedSending:   make(chan struct{}),
		incoming:         make(chan Message, 50),
		outgoing:         make(chan queueItem, 50),
	}

	// setup vital message listeners
	client.setupHandlers()

	return client
}

func (client *TwitchClient) Ready() <-chan struct{} {
	return client.ready
}

func (client *TwitchClient) Alive() <-chan struct{} {
	return client.alive
}

func (client *TwitchClient) Incoming() <-chan Message {
	return client.incoming
}

func (client *TwitchClient) Connect() error {
	conn, err := net.Dial("tcp", client.server)
	if err != nil {
		return err
	}

	client.conn = conn
	client.reader = bufio.NewReader(conn) // we manually read to properly handle tags
	client.writer = irc.NewEncoder(conn)

	// start working on the queue
	go client.sender()

	// start receiving
	go client.receiver()

	// send login info before anything else
	client.Send(irc.Message{
		Command: irc.PASS,
		Params:  []string{client.password},
	})

	client.Send(irc.Message{
		Command: irc.NICK,
		Params:  []string{client.username},
	})

	client.Send(irc.Message{
		Command: irc.USER,
		Params:  []string{"kabukibot", "8", "*", client.username},
	})

	return nil
}

func (client *TwitchClient) Disconnect() error {
	// stop the sender/receiver and wait for them to stop (maybe it will drain the
	// outgoing queue, maybe it won't, but let's give it time)
	close(client.stopReceiving)
	<-client.stoppedReceiving

	close(client.stopSending)
	<-client.stoppedSending

	// for all intents and purposes, we are not alive anymore
	close(client.alive)

	// close the IRC connection
	return client.conn.Close()
}

func (client *TwitchClient) Send(msg irc.Message) <-chan bool {
	signal := make(chan bool, 1)
	outgoing := queueItem{msg, signal}

	// if queue is not full, then
	client.outgoing <- outgoing
	// else
	// 	signal <- false // means "not sent"
	// 	close(signal)

	return signal
}

func (client *TwitchClient) sender() {
	for {
		select {
		case msg := <-client.outgoing:
			fmt.Println("< " + msg.message.String())
			client.writer.Encode(&msg.message)

			// signal to the one who sent the message that it was in fact sent
			msg.signal <- true
			close(msg.signal)

		case <-client.stopSending:
			break
		}
	}

	close(client.stoppedSending)
}

func (client *TwitchClient) receiver() {
	reading := make(chan struct{})

	// a buffer between the raw irc input from the net and the goroutine channels
	buffer := make(chan string, 10)

	// fork a reader loop, which could block and needs special handling as it's not a channel
	// (but it will pump its messages into a channel)
	go func() {
		for {
			select {
			case <-client.stopReceiving:
				break

			default:
				// set a 5min timeout
				client.conn.SetDeadline(time.Now().Add(300 * time.Second))

				line, err := client.reader.ReadString('\n')
				if err != nil {
					break
				}

				buffer <- line
			}
		}

		close(reading)
	}()

	for {
		select {
		case rawLine := <-buffer:
			fmt.Println("> " + strings.TrimSpace(rawLine))
			// if the message begins with a '@', we have some tags (IRCv3). The default
			// IRC decoder will not have properly detected it and mangled its output.
			// We fix that by manually splitting the tags from the rest of the message
			// and parse each part individually.
			tags := make(irc.Tags)
			msg := &irc.Message{}

			if strings.HasPrefix(rawLine, "@") {
				parts := strings.SplitN(rawLine, " ", 2)

				tags = irc.ParseTags(strings.TrimPrefix(parts[0], "@"))
				msg = irc.ParseMessage(parts[1])
			} else {
				msg = irc.ParseMessage(rawLine)
			}

			// hand it over to the message handler
			handler, ok := client.handlers[msg.Command]
			if ok {
				go handler(msg, tags)
			}

		case <-client.stopReceiving:
			break
		}
	}

	close(client.stoppedReceiving)
}
<|endoftext|>


<commit_before>// Package registry provides a dynamic api service router
package registry

import (
	"errors"
	"fmt"
	"net/http"
	"regexp"
	"strings"
	"sync"
	"time"

	"github.com/micro/go-micro/v2/api"
	"github.com/micro/go-micro/v2/api/router"
	"github.com/micro/go-micro/v2/logger"
	"github.com/micro/go-micro/v2/registry"
	"github.com/micro/go-micro/v2/registry/cache"
)

// router is the default router
type registryRouter struct {
	exit chan bool
	opts router.Options

	// registry cache
	rc cache.Cache

	sync.RWMutex
	eps map[string]*api.Service
}

func setNamespace(ns, name string) string {
	ns = strings.TrimSpace(ns)
	name = strings.TrimSpace(name)

	// no namespace
	if len(ns) == 0 {
		return name
	}

	switch {
	// has - suffix
	case strings.HasSuffix(ns, "-"):
		return strings.Replace(ns+name, ".", "-", -1)
	// has . suffix
	case strings.HasSuffix(ns, "."):
		return ns + name
	}

	// default join .
	return strings.Join([]string{ns, name}, ".")
}

func (r *registryRouter) isClosed() bool {
	select {
	case <-r.exit:
		return true
	default:
		return false
	}
}

// refresh list of api services
func (r *registryRouter) refresh() {
	var attempts int

	for {
		services, err := r.opts.Registry.ListServices()
		if err != nil {
			attempts++
			if logger.V(logger.ErrorLevel, logger.DefaultLogger) {
				logger.Errorf("unable to list services: %v", err)
			}
			time.Sleep(time.Duration(attempts) * time.Second)
			continue
		}

		attempts = 0

		// for each service, get service and store endpoints
		for _, s := range services {
			// only get services for this namespace
			if !strings.HasPrefix(s.Name, r.opts.Namespace) {
				continue
			}
			service, err := r.rc.GetService(s.Name)
			if err != nil {
				if logger.V(logger.ErrorLevel, logger.DefaultLogger) {
					logger.Errorf("unable to get service: %v", err)
				}
				continue
			}
			r.store(service)
		}

		// refresh list in 10 minutes... cruft
		select {
		case <-time.After(time.Minute * 10):
		case <-r.exit:
			return
		}
	}
}

// process watch event
func (r *registryRouter) process(res *registry.Result) {
	// skip these things
	if res == nil || res.Service == nil || !strings.HasPrefix(res.Service.Name, r.opts.Namespace) {
		return
	}

	// get entry from cache
	service, err := r.rc.GetService(res.Service.Name)
	if err != nil {
		if logger.V(logger.ErrorLevel, logger.DefaultLogger) {
			logger.Errorf("unable to get service: %v", err)
		}
		return
	}

	// update our local endpoints
	r.store(service)
}

// store local endpoint cache
func (r *registryRouter) store(services []*registry.Service) {
	// endpoints
	eps := map[string]*api.Service{}

	// services
	names := map[string]bool{}

	// create a new endpoint mapping
	for _, service := range services {
		// set names we need later
		names[service.Name] = true

		// map per endpoint
		for _, endpoint := range service.Endpoints {
			// create a key service:endpoint_name
			key := fmt.Sprintf("%s:%s", service.Name, endpoint.Name)
			// decode endpoint
			end := api.Decode(endpoint.Metadata)

			// if we got nothing skip
			if err := api.Validate(end); err != nil {
				if logger.V(logger.ErrorLevel, logger.DefaultLogger) {
					logger.Errorf("endpoint validation failed: %v", err)
				}
				continue
			}

			// try get endpoint
			ep, ok := eps[key]
			if !ok {
				ep = &api.Service{Name: service.Name}
			}

			// overwrite the endpoint
			ep.Endpoint = end
			// append services
			ep.Services = append(ep.Services, service)
			// store it
			eps[key] = ep
		}
	}

	r.Lock()
	defer r.Unlock()

	// delete any existing eps for services we know
	for key, service := range r.eps {
		// skip what we don't care about
		if !names[service.Name] {
			continue
		}

		// ok we know this thing
		// delete delete delete
		delete(r.eps, key)
	}

	// now set the eps we have
	for name, endpoint := range eps {
		r.eps[name] = endpoint
	}
}

// watch for endpoint changes
func (r *registryRouter) watch() {
	var attempts int

	for {
		if r.isClosed() {
			return
		}

		// watch for changes
		w, err := r.opts.Registry.Watch()
		if err != nil {
			attempts++
			if logger.V(logger.ErrorLevel, logger.DefaultLogger) {
				logger.Errorf("error watching endpoints: %v", err)
			}
			time.Sleep(time.Duration(attempts) * time.Second)
			continue
		}

		ch := make(chan bool)

		go func() {
			select {
			case <-ch:
				w.Stop()
			case <-r.exit:
				w.Stop()
			}
		}()

		// reset if we get here
		attempts = 0

		for {
			// process next event
			res, err := w.Next()
			if err != nil {
				if logger.V(logger.ErrorLevel, logger.DefaultLogger) {
					logger.Errorf("error getting next endoint: %v", err)
				}
				close(ch)
				break
			}
			r.process(res)
		}
	}
}

func (r *registryRouter) Options() router.Options {
	return r.opts
}

func (r *registryRouter) Close() error {
	select {
	case <-r.exit:
		return nil
	default:
		close(r.exit)
		r.rc.Stop()
	}
	return nil
}

func (r *registryRouter) Endpoint(req *http.Request) (*api.Service, error) {
	if r.isClosed() {
		return nil, errors.New("router closed")
	}

	r.RLock()
	defer r.RUnlock()

	// use the first match
	// TODO: weighted matching
	for _, e := range r.eps {
		ep := e.Endpoint

		// match
		var pathMatch, hostMatch, methodMatch bool

		// 1. try method GET, POST, PUT, etc
		// 2. try host example.com, foobar.com, etc
		// 3. try path /foo/bar, /bar/baz, etc

		// 1. try match method
		for _, m := range ep.Method {
			if req.Method == m {
				methodMatch = true
				break
			}
		}

		// no match on method pass
		if len(ep.Method) > 0 && !methodMatch {
			continue
		}

		// 2. try match host
		for _, h := range ep.Host {
			if req.Host == h {
				hostMatch = true
				break
			}
		}

		// no match on host pass
		if len(ep.Host) > 0 && !hostMatch {
			continue
		}

		// 3. try match paths
		for _, p := range ep.Path {
			re, err := regexp.CompilePOSIX(p)
			if err == nil && re.MatchString(req.URL.Path) {
				pathMatch = true
				break
			}
		}

		// no match pass
		if len(ep.Path) > 0 && !pathMatch {
			continue
		}

		// TODO: Percentage traffic

		// we got here, so its a match
		return e, nil
	}

	// no match
	return nil, errors.New("not found")
}

func (r *registryRouter) Route(req *http.Request) (*api.Service, error) {
	if r.isClosed() {
		return nil, errors.New("router closed")
	}

	// try get an endpoint
	ep, err := r.Endpoint(req)
	if err == nil {
		return ep, nil
	}

	// error not nil
	// ignore that shit
	// TODO: don't ignore that shit

	// get the service name
	rp, err := r.opts.Resolver.Resolve(req)
	if err != nil {
		return nil, err
	}

	// service name
	name := setNamespace(r.opts.Namespace, rp.Name)

	// get service
	services, err := r.rc.GetService(name)
	if err != nil {
		return nil, err
	}

	// only use endpoint matching when the meta handler is set aka api.Default
	switch r.opts.Handler {
	// rpc handlers
	case "meta", "api", "rpc":
		handler := r.opts.Handler

		// set default handler to api
		if r.opts.Handler == "meta" {
			handler = "rpc"
		}

		// construct api service
		return &api.Service{
			Name: name,
			Endpoint: &api.Endpoint{
				Name:    rp.Method,
				Handler: handler,
			},
			Services: services,
		}, nil
	// http handler
	case "http", "proxy", "web":
		// construct api service
		return &api.Service{
			Name: name,
			Endpoint: &api.Endpoint{
				Name:    req.URL.String(),
				Handler: r.opts.Handler,
				Host:    []string{req.Host},
				Method:  []string{req.Method},
				Path:    []string{req.URL.Path},
			},
			Services: services,
		}, nil
	}

	return nil, errors.New("unknown handler")
}

func newRouter(opts ...router.Option) *registryRouter {
	options := router.NewOptions(opts...)
	r := &registryRouter{
		exit: make(chan bool),
		opts: options,
		rc:   cache.New(options.Registry),
		eps:  make(map[string]*api.Service),
	}
	go r.watch()
	go r.refresh()
	return r
}

// NewRouter returns the default router
func NewRouter(opts ...router.Option) router.Router {
	return newRouter(opts...)
}
<commit_msg>Move error for api validation to trace level (#1432)<commit_after>// Package registry provides a dynamic api service router
package registry

import (
	"errors"
	"fmt"
	"net/http"
	"regexp"
	"strings"
	"sync"
	"time"

	"github.com/micro/go-micro/v2/api"
	"github.com/micro/go-micro/v2/api/router"
	"github.com/micro/go-micro/v2/logger"
	"github.com/micro/go-micro/v2/registry"
	"github.com/micro/go-micro/v2/registry/cache"
)

// router is the default router
type registryRouter struct {
	exit chan bool
	opts router.Options

	// registry cache
	rc cache.Cache

	sync.RWMutex
	eps map[string]*api.Service
}

func setNamespace(ns, name string) string {
	ns = strings.TrimSpace(ns)
	name = strings.TrimSpace(name)

	// no namespace
	if len(ns) == 0 {
		return name
	}

	switch {
	// has - suffix
	case strings.HasSuffix(ns, "-"):
		return strings.Replace(ns+name, ".", "-", -1)
	// has . suffix
	case strings.HasSuffix(ns, "."):
		return ns + name
	}

	// default join .
	return strings.Join([]string{ns, name}, ".")
}

func (r *registryRouter) isClosed() bool {
	select {
	case <-r.exit:
		return true
	default:
		return false
	}
}

// refresh list of api services
func (r *registryRouter) refresh() {
	var attempts int

	for {
		services, err := r.opts.Registry.ListServices()
		if err != nil {
			attempts++
			if logger.V(logger.ErrorLevel, logger.DefaultLogger) {
				logger.Errorf("unable to list services: %v", err)
			}
			time.Sleep(time.Duration(attempts) * time.Second)
			continue
		}

		attempts = 0

		// for each service, get service and store endpoints
		for _, s := range services {
			// only get services for this namespace
			if !strings.HasPrefix(s.Name, r.opts.Namespace) {
				continue
			}
			service, err := r.rc.GetService(s.Name)
			if err != nil {
				if logger.V(logger.ErrorLevel, logger.DefaultLogger) {
					logger.Errorf("unable to get service: %v", err)
				}
				continue
			}
			r.store(service)
		}

		// refresh list in 10 minutes... cruft
		select {
		case <-time.After(time.Minute * 10):
		case <-r.exit:
			return
		}
	}
}

// process watch event
func (r *registryRouter) process(res *registry.Result) {
	// skip these things
	if res == nil || res.Service == nil || !strings.HasPrefix(res.Service.Name, r.opts.Namespace) {
		return
	}

	// get entry from cache
	service, err := r.rc.GetService(res.Service.Name)
	if err != nil {
		if logger.V(logger.ErrorLevel, logger.DefaultLogger) {
			logger.Errorf("unable to get service: %v", err)
		}
		return
	}

	// update our local endpoints
	r.store(service)
}

// store local endpoint cache
func (r *registryRouter) store(services []*registry.Service) {
	// endpoints
	eps := map[string]*api.Service{}

	// services
	names := map[string]bool{}

	// create a new endpoint mapping
	for _, service := range services {
		// set names we need later
		names[service.Name] = true

		// map per endpoint
		for _, endpoint := range service.Endpoints {
			// create a key service:endpoint_name
			key := fmt.Sprintf("%s:%s", service.Name, endpoint.Name)
			// decode endpoint
			end := api.Decode(endpoint.Metadata)

			// if we got nothing skip
			if err := api.Validate(end); err != nil {
				if logger.V(logger.TraceLevel, logger.DefaultLogger) {
					logger.Tracef("endpoint validation failed: %v", err)
				}
				continue
			}

			// try get endpoint
			ep, ok := eps[key]
			if !ok {
				ep = &api.Service{Name: service.Name}
			}

			// overwrite the endpoint
			ep.Endpoint = end
			// append services
			ep.Services = append(ep.Services, service)
			// store it
			eps[key] = ep
		}
	}

	r.Lock()
	defer r.Unlock()

	// delete any existing eps for services we know
	for key, service := range r.eps {
		// skip what we don't care about
		if !names[service.Name] {
			continue
		}

		// ok we know this thing
		// delete delete delete
		delete(r.eps, key)
	}

	// now set the eps we have
	for name, endpoint := range eps {
		r.eps[name] = endpoint
	}
}

// watch for endpoint changes
func (r *registryRouter) watch() {
	var attempts int

	for {
		if r.isClosed() {
			return
		}

		// watch for changes
		w, err := r.opts.Registry.Watch()
		if err != nil {
			attempts++
			if logger.V(logger.ErrorLevel, logger.DefaultLogger) {
				logger.Errorf("error watching endpoints: %v", err)
			}
			time.Sleep(time.Duration(attempts) * time.Second)
			continue
		}

		ch := make(chan bool)

		go func() {
			select {
			case <-ch:
				w.Stop()
			case <-r.exit:
				w.Stop()
			}
		}()

		// reset if we get here
		attempts = 0

		for {
			// process next event
			res, err := w.Next()
			if err != nil {
				if logger.V(logger.ErrorLevel, logger.DefaultLogger) {
					logger.Errorf("error getting next endoint: %v", err)
				}
				close(ch)
				break
			}
			r.process(res)
		}
	}
}

func (r *registryRouter) Options() router.Options {
	return r.opts
}

func (r *registryRouter) Close() error {
	select {
	case <-r.exit:
		return nil
	default:
		close(r.exit)
		r.rc.Stop()
	}
	return nil
}

func (r *registryRouter) Endpoint(req *http.Request) (*api.Service, error) {
	if r.isClosed() {
		return nil, errors.New("router closed")
	}

	r.RLock()
	defer r.RUnlock()

	// use the first match
	// TODO: weighted matching
	for _, e := range r.eps {
		ep := e.Endpoint

		// match
		var pathMatch, hostMatch, methodMatch bool

		// 1. try method GET, POST, PUT, etc
		// 2. try host example.com, foobar.com, etc
		// 3. try path /foo/bar, /bar/baz, etc

		// 1. try match method
		for _, m := range ep.Method {
			if req.Method == m {
				methodMatch = true
				break
			}
		}

		// no match on method pass
		if len(ep.Method) > 0 && !methodMatch {
			continue
		}

		// 2. try match host
		for _, h := range ep.Host {
			if req.Host == h {
				hostMatch = true
				break
			}
		}

		// no match on host pass
		if len(ep.Host) > 0 && !hostMatch {
			continue
		}

		// 3. try match paths
		for _, p := range ep.Path {
			re, err := regexp.CompilePOSIX(p)
			if err == nil && re.MatchString(req.URL.Path) {
				pathMatch = true
				break
			}
		}

		// no match pass
		if len(ep.Path) > 0 && !pathMatch {
			continue
		}

		// TODO: Percentage traffic

		// we got here, so its a match
		return e, nil
	}

	// no match
	return nil, errors.New("not found")
}

func (r *registryRouter) Route(req *http.Request) (*api.Service, error) {
	if r.isClosed() {
		return nil, errors.New("router closed")
	}

	// try get an endpoint
	ep, err := r.Endpoint(req)
	if err == nil {
		return ep, nil
	}

	// error not nil
	// ignore that shit
	// TODO: don't ignore that shit

	// get the service name
	rp, err := r.opts.Resolver.Resolve(req)
	if err != nil {
		return nil, err
	}

	// service name
	name := setNamespace(r.opts.Namespace, rp.Name)

	// get service
	services, err := r.rc.GetService(name)
	if err != nil {
		return nil, err
	}

	// only use endpoint matching when the meta handler is set aka api.Default
	switch r.opts.Handler {
	// rpc handlers
	case "meta", "api", "rpc":
		handler := r.opts.Handler

		// set default handler to api
		if r.opts.Handler == "meta" {
			handler = "rpc"
		}

		// construct api service
		return &api.Service{
			Name: name,
			Endpoint: &api.Endpoint{
				Name:    rp.Method,
				Handler: handler,
			},
			Services: services,
		}, nil
	// http handler
	case "http", "proxy", "web":
		// construct api service
		return &api.Service{
			Name: name,
			Endpoint: &api.Endpoint{
				Name:    req.URL.String(),
				Handler: r.opts.Handler,
				Host:    []string{req.Host},
				Method:  []string{req.Method},
				Path:    []string{req.URL.Path},
			},
			Services: services,
		}, nil
	}

	return nil, errors.New("unknown handler")
}

func newRouter(opts ...router.Option) *registryRouter {
	options := router.NewOptions(opts...)
	r := &registryRouter{
		exit: make(chan bool),
		opts: options,
		rc:   cache.New(options.Registry),
		eps:  make(map[string]*api.Service),
	}
	go r.watch()
	go r.refresh()
	return r
}

// NewRouter returns the default router
func NewRouter(opts ...router.Option) router.Router {
	return newRouter(opts...)
}
<|endoftext|>


<commit_before>package certmagic

import (
	"bytes"
	"encoding/gob"
	"errors"
	"fmt"
	"path"
	"strings"
	"time"

	"github.com/mholt/certmagic"
	"github.com/micro/go-micro/v2/store"
	"github.com/micro/go-micro/v2/sync/lock"
)

// File represents a "File" that will be stored in store.Store - the contents and last modified time
type File struct {
	// last modified time
	LastModified time.Time
	// Contents
	Contents []byte
}

// storage is an implementation of certmagic.Storage using micro's sync.Map and store.Store interfaces.
// As certmagic storage expects a filesystem (with stat() abilities) we have to implement
// the bare minimum of metadata.
type storage struct {
	lock  lock.Lock
	store store.Store
}

func (s *storage) Lock(key string) error {
	return s.lock.Acquire(key, lock.TTL(10*time.Minute))
}

func (s *storage) Unlock(key string) error {
	return s.lock.Release(key)
}

func (s *storage) Store(key string, value []byte) error {
	f := File{
		LastModified: time.Now(),
		Contents:     value,
	}
	buf := &bytes.Buffer{}
	e := gob.NewEncoder(buf)
	if err := e.Encode(f); err != nil {
		return err
	}
	r := &store.Record{
		Key:   key,
		Value: buf.Bytes(),
	}
	return s.store.Write(r)
}

func (s *storage) Load(key string) ([]byte, error) {
	if !s.Exists(key) {
		return nil, certmagic.ErrNotExist(errors.New(key + " doesn't exist"))
	}
	records, err := s.store.Read(key)
	if err != nil {
		return nil, err
	}
	if len(records) != 1 {
		return nil, fmt.Errorf("ACME Storage: multiple records matched key %s", key)
	}
	b := bytes.NewBuffer(records[0].Value)
	d := gob.NewDecoder(b)
	var f File
	err = d.Decode(&f)
	if err != nil {
		return nil, err
	}
	return f.Contents, nil
}

func (s *storage) Delete(key string) error {
	return s.store.Delete(key)
}

func (s *storage) Exists(key string) bool {
	if _, err := s.store.Read(key); err != nil {
		return false
	}
	return true
}

func (s *storage) List(prefix string, recursive bool) ([]string, error) {
	keys, err := s.store.List()
	if err != nil {
		return nil, err
	}

	//nolint:prealloc
	var results []string
	for _, k := range keys {
		if strings.HasPrefix(k, prefix) {
			results = append(results, k)
		}
	}
	if recursive {
		return results, nil
	}
	keysMap := make(map[string]bool)
	for _, key := range results {
		dir := strings.Split(strings.TrimPrefix(key, prefix+"/"), "/")
		keysMap[dir[0]] = true
	}
	results = make([]string, 0)
	for k := range keysMap {
		results = append(results, path.Join(prefix, k))
	}
	return results, nil
}

func (s *storage) Stat(key string) (certmagic.KeyInfo, error) {
	records, err := s.store.Read(key)
	if err != nil {
		return certmagic.KeyInfo{}, err
	}
	if len(records) != 1 {
		return certmagic.KeyInfo{}, fmt.Errorf("ACME Storage: multiple records matched key %s", key)
	}
	b := bytes.NewBuffer(records[0].Value)
	d := gob.NewDecoder(b)
	var f File
	err = d.Decode(&f)
	if err != nil {
		return certmagic.KeyInfo{}, err
	}
	return certmagic.KeyInfo{
		Key:        key,
		Modified:   f.LastModified,
		Size:       int64(len(f.Contents)),
		IsTerminal: false,
	}, nil
}

// NewStorage returns a certmagic.Storage backed by a go-micro/lock and go-micro/store
func NewStorage(lock lock.Lock, store store.Store) certmagic.Storage {
	return &storage{
		lock:  lock,
		store: store,
	}
}
<commit_msg>fix compilation issues<commit_after>package certmagic

import (
	"bytes"
	"encoding/gob"
	"errors"
	"fmt"
	"path"
	"strings"
	"time"

	"github.com/mholt/certmagic"
	"github.com/micro/go-micro/v2/store"
	"github.com/micro/go-micro/v2/sync"
)

// File represents a "File" that will be stored in store.Store - the contents and last modified time
type File struct {
	// last modified time
	LastModified time.Time
	// Contents
	Contents []byte
}

// storage is an implementation of certmagic.Storage using micro's sync.Map and store.Store interfaces.
// As certmagic storage expects a filesystem (with stat() abilities) we have to implement
// the bare minimum of metadata.
type storage struct {
	lock  sync.Sync
	store store.Store
}

func (s *storage) Lock(key string) error {
	return s.lock.Lock(key, sync.LockTTL(10*time.Minute))
}

func (s *storage) Unlock(key string) error {
	return s.lock.Unlock(key)
}

func (s *storage) Store(key string, value []byte) error {
	f := File{
		LastModified: time.Now(),
		Contents:     value,
	}
	buf := &bytes.Buffer{}
	e := gob.NewEncoder(buf)
	if err := e.Encode(f); err != nil {
		return err
	}
	r := &store.Record{
		Key:   key,
		Value: buf.Bytes(),
	}
	return s.store.Write(r)
}

func (s *storage) Load(key string) ([]byte, error) {
	if !s.Exists(key) {
		return nil, certmagic.ErrNotExist(errors.New(key + " doesn't exist"))
	}
	records, err := s.store.Read(key)
	if err != nil {
		return nil, err
	}
	if len(records) != 1 {
		return nil, fmt.Errorf("ACME Storage: multiple records matched key %s", key)
	}
	b := bytes.NewBuffer(records[0].Value)
	d := gob.NewDecoder(b)
	var f File
	err = d.Decode(&f)
	if err != nil {
		return nil, err
	}
	return f.Contents, nil
}

func (s *storage) Delete(key string) error {
	return s.store.Delete(key)
}

func (s *storage) Exists(key string) bool {
	if _, err := s.store.Read(key); err != nil {
		return false
	}
	return true
}

func (s *storage) List(prefix string, recursive bool) ([]string, error) {
	keys, err := s.store.List()
	if err != nil {
		return nil, err
	}

	//nolint:prealloc
	var results []string
	for _, k := range keys {
		if strings.HasPrefix(k, prefix) {
			results = append(results, k)
		}
	}
	if recursive {
		return results, nil
	}
	keysMap := make(map[string]bool)
	for _, key := range results {
		dir := strings.Split(strings.TrimPrefix(key, prefix+"/"), "/")
		keysMap[dir[0]] = true
	}
	results = make([]string, 0)
	for k := range keysMap {
		results = append(results, path.Join(prefix, k))
	}
	return results, nil
}

func (s *storage) Stat(key string) (certmagic.KeyInfo, error) {
	records, err := s.store.Read(key)
	if err != nil {
		return certmagic.KeyInfo{}, err
	}
	if len(records) != 1 {
		return certmagic.KeyInfo{}, fmt.Errorf("ACME Storage: multiple records matched key %s", key)
	}
	b := bytes.NewBuffer(records[0].Value)
	d := gob.NewDecoder(b)
	var f File
	err = d.Decode(&f)
	if err != nil {
		return certmagic.KeyInfo{}, err
	}
	return certmagic.KeyInfo{
		Key:        key,
		Modified:   f.LastModified,
		Size:       int64(len(f.Contents)),
		IsTerminal: false,
	}, nil
}

// NewStorage returns a certmagic.Storage backed by a go-micro/lock and go-micro/store
func NewStorage(lock sync.Sync, store store.Store) certmagic.Storage {
	return &storage{
		lock:  lock,
		store: store,
	}
}
<|endoftext|>


<commit_before>package version

import "fmt"

const (
	// VersionMajor is for an API incompatible changes
	VersionMajor = 5
	// VersionMinor is for functionality in a backwards-compatible manner
	VersionMinor = 13
	// VersionPatch is for backwards-compatible bug fixes
	VersionPatch = 1

	// VersionDev indicates development branch. Releases will be empty string.
	VersionDev = "-dev"
)

// Version is the specification version that the package types support.
var Version = fmt.Sprintf("%d.%d.%d%s", VersionMajor, VersionMinor, VersionPatch, VersionDev)
<commit_msg>Bump to v5.13.1<commit_after>package version

import "fmt"

const (
	// VersionMajor is for an API incompatible changes
	VersionMajor = 5
	// VersionMinor is for functionality in a backwards-compatible manner
	VersionMinor = 13
	// VersionPatch is for backwards-compatible bug fixes
	VersionPatch = 1

	// VersionDev indicates development branch. Releases will be empty string.
	VersionDev = ""
)

// Version is the specification version that the package types support.
var Version = fmt.Sprintf("%d.%d.%d%s", VersionMajor, VersionMinor, VersionPatch, VersionDev)
<|endoftext|>


<commit_before>package mounts

import (
	"fmt"
	"reflect"
	"sort"
	"testing"

	"koding/klient/machine"
	"koding/klient/machine/mount"
)

func TestMountsPath(t *testing.T) {
	tests := map[string]struct {
		Path    string
		MountID mount.ID
		Valid   bool
	}{
		"valid mount from A machine": {
			Path:    "/home/koding/b",
			MountID: "mountAB",
			Valid:   true,
		},
		"valid mount from B machine": {
			Path:    "/home/koding/d",
			MountID: "mountBA",
			Valid:   true,
		},
		"non absolute path": {
			Path:    "../.",
			MountID: "",
			Valid:   false,
		},
		"unknown path": {
			Path:    "/home/koding/unknown",
			MountID: "",
			Valid:   false,
		},
	}

	ms, err := mountsObject()
	if err != nil {
		t.Fatal(err)
	}

	for name, test := range tests {
		t.Run(name, func(t *testing.T) {
			mountID, err := ms.Path(test.Path)
			if (err == nil) != test.Valid {
				t.Fatalf("want err == nil => %t; got err %v", test.Valid, err)
			}

			if err == nil && test.MountID != mountID {
				t.Fatalf("want mount ID = %s; got %s", test.MountID, mountID)
			}
		})
	}
}

func TestMountsRemotePath(t *testing.T) {
	tests := map[string]struct {
		RemotePath string
		MountIDs   mount.IDSlice
		Valid      bool
	}{
		"remote from A machine": {
			RemotePath: "/home/koding/remote/c",
			MountIDs:   mount.IDSlice{"mountAC"},
			Valid:      true,
		},
		"remote from A and B machines": {
			RemotePath: "/home/koding/remote/a",
			MountIDs:   mount.IDSlice{"mountAA", "mountBA"},
			Valid:      true,
		},
		"unknown path": {
			RemotePath: "/home/koding/unknown",
			MountIDs:   nil,
			Valid:      false,
		},
	}

	ms, err := mountsObject()
	if err != nil {
		t.Fatal(err)
	}

	for name, test := range tests {
		t.Run(name, func(t *testing.T) {
			mountIDs, err := ms.RemotePath(test.RemotePath)
			sort.Sort(mountIDs)
			if (err == nil) != test.Valid {
				t.Fatalf("want err == nil => %t; got err %v", test.Valid, err)
			}

			if err == nil && !reflect.DeepEqual(mountIDs, test.MountIDs) {
				t.Fatalf("want mount ID = %v; got %v", test.MountIDs, mountIDs)
			}
		})
	}
}

func TestMountsMachineID(t *testing.T) {
	tests := map[string]struct {
		MountID mount.ID
		ID      machine.ID
		Valid   bool
	}{
		"machine A mount": {
			MountID: "mountAC",
			ID:      "machineA",
			Valid:   true,
		},
		"machine B mount": {
			MountID: "mountBA",
			ID:      "machineB",
			Valid:   true,
		},
		"unknown mount ID": {
			MountID: "unknown",
			ID:      "",
			Valid:   false,
		},
	}

	ms, err := mountsObject()
	if err != nil {
		t.Fatal(err)
	}

	for name, test := range tests {
		t.Run(name, func(t *testing.T) {
			id, err := ms.MachineID(test.MountID)
			if (err == nil) != test.Valid {
				t.Fatalf("want err == nil => %t; got err %v", test.Valid, err)
			}

			if err == nil && test.ID != id {
				t.Fatalf("want machine ID = %s; got %s", test.ID, id)
			}
		})
	}
}

func TestMountsRemove(t *testing.T) {
	ms, err := mountsObject()
	if err != nil {
		t.Fatal(err)
	}

	if err := ms.Remove("mountAA"); err != nil {
		t.Errorf("want err = nil; got %v", err)
	}
	all, err := ms.All("machineA")
	if err != nil {
		t.Errorf("want err = nil; got %v", err)
	}
	if l := len(all); l != 2 {
		t.Errorf("want 2 mounts in machine A; got %d", l)
	}

	if err := ms.Remove("mountBA"); err != nil {
		t.Errorf("want err = nil; got %v", err)
	}
	if _, err := ms.All("machineB"); err == nil {
		t.Errorf("want err != nil; got nil")
	}
}

func mountsObject() (*Mounts, error) {
	var data = []struct {
		ID      machine.ID
		MountID mount.ID
		Mount   mount.Mount
	}{
		{
			ID:      "machineA",
			MountID: "mountAA",
			Mount: mount.Mount{
				Path:       "/home/koding/a",
				RemotePath: "/home/koding/remote/a",
			},
		}, {
			ID:      "machineA",
			MountID: "mountAB",
			Mount: mount.Mount{
				Path:       "/home/koding/b",
				RemotePath: "/home/koding/remote/b",
			},
		}, {
			ID:      "machineA",
			MountID: "mountAC",
			Mount: mount.Mount{
				Path:       "/home/koding/c",
				RemotePath: "/home/koding/remote/c",
			},
		}, {
			ID:      "machineB",
			MountID: "mountBA",
			Mount: mount.Mount{
				Path:       "/home/koding/d",
				RemotePath: "/home/koding/remote/a",
			},
		},
	}

	ms := New()
	for i, d := range data {
		if err := ms.Add(d.ID, d.MountID, d.Mount); err != nil {
			return nil, fmt.Errorf("want err = nil; got %v (i:%d)", err, i)
		}
	}

	return ms, nil
}
<commit_msg>klient/mount: add tests for validate logic of mounts add method<commit_after>package mounts

import (
	"fmt"
	"reflect"
	"sort"
	"testing"

	"koding/klient/machine"
	"koding/klient/machine/mount"
)

func TestMountsPath(t *testing.T) {
	tests := map[string]struct {
		Path    string
		MountID mount.ID
		Valid   bool
	}{
		"valid mount from A machine": {
			Path:    "/home/koding/b",
			MountID: "mountAB",
			Valid:   true,
		},
		"valid mount from B machine": {
			Path:    "/home/koding/d",
			MountID: "mountBA",
			Valid:   true,
		},
		"non absolute path": {
			Path:    "../.",
			MountID: "",
			Valid:   false,
		},
		"unknown path": {
			Path:    "/home/koding/unknown",
			MountID: "",
			Valid:   false,
		},
	}

	ms, err := mountsObject()
	if err != nil {
		t.Fatal(err)
	}

	for name, test := range tests {
		t.Run(name, func(t *testing.T) {
			mountID, err := ms.Path(test.Path)
			if (err == nil) != test.Valid {
				t.Fatalf("want err == nil => %t; got err %v", test.Valid, err)
			}

			if err == nil && test.MountID != mountID {
				t.Fatalf("want mount ID = %s; got %s", test.MountID, mountID)
			}
		})
	}
}

func TestMountsRemotePath(t *testing.T) {
	tests := map[string]struct {
		RemotePath string
		MountIDs   mount.IDSlice
		Valid      bool
	}{
		"remote from A machine": {
			RemotePath: "/home/koding/remote/c",
			MountIDs:   mount.IDSlice{"mountAC"},
			Valid:      true,
		},
		"remote from A and B machines": {
			RemotePath: "/home/koding/remote/a",
			MountIDs:   mount.IDSlice{"mountAA", "mountBA"},
			Valid:      true,
		},
		"unknown path": {
			RemotePath: "/home/koding/unknown",
			MountIDs:   nil,
			Valid:      false,
		},
	}

	ms, err := mountsObject()
	if err != nil {
		t.Fatal(err)
	}

	for name, test := range tests {
		t.Run(name, func(t *testing.T) {
			mountIDs, err := ms.RemotePath(test.RemotePath)
			sort.Sort(mountIDs)
			if (err == nil) != test.Valid {
				t.Fatalf("want err == nil => %t; got err %v", test.Valid, err)
			}

			if err == nil && !reflect.DeepEqual(mountIDs, test.MountIDs) {
				t.Fatalf("want mount ID = %v; got %v", test.MountIDs, mountIDs)
			}
		})
	}
}

func TestMountsMachineID(t *testing.T) {
	tests := map[string]struct {
		MountID mount.ID
		ID      machine.ID
		Valid   bool
	}{
		"machine A mount": {
			MountID: "mountAC",
			ID:      "machineA",
			Valid:   true,
		},
		"machine B mount": {
			MountID: "mountBA",
			ID:      "machineB",
			Valid:   true,
		},
		"unknown mount ID": {
			MountID: "unknown",
			ID:      "",
			Valid:   false,
		},
	}

	ms, err := mountsObject()
	if err != nil {
		t.Fatal(err)
	}

	for name, test := range tests {
		t.Run(name, func(t *testing.T) {
			id, err := ms.MachineID(test.MountID)
			if (err == nil) != test.Valid {
				t.Fatalf("want err == nil => %t; got err %v", test.Valid, err)
			}

			if err == nil && test.ID != id {
				t.Fatalf("want machine ID = %s; got %s", test.ID, id)
			}
		})
	}
}

func TestMountsRemove(t *testing.T) {
	ms, err := mountsObject()
	if err != nil {
		t.Fatal(err)
	}

	if err := ms.Remove("mountAA"); err != nil {
		t.Errorf("want err = nil; got %v", err)
	}
	all, err := ms.All("machineA")
	if err != nil {
		t.Errorf("want err = nil; got %v", err)
	}
	if l := len(all); l != 2 {
		t.Errorf("want 2 mounts in machine A; got %d", l)
	}

	if err := ms.Remove("mountBA"); err != nil {
		t.Errorf("want err = nil; got %v", err)
	}
	if _, err := ms.All("machineB"); err == nil {
		t.Errorf("want err != nil; got nil")
	}
}

func TestMountsAddValidate(t *testing.T) {
	tests := map[string]struct {
		ID      machine.ID
		MountID mount.ID
		Mount   mount.Mount
	}{
		"local path already taken": {
			ID:      "machineX",
			MountID: "mountAAX",
			Mount: mount.Mount{
				Path:       "/home/koding/a",
				RemotePath: "/home/koding/remote/a",
			},
		},
		"mount ID already exist": {
			ID:      "machineX",
			MountID: "mountAB",
			Mount: mount.Mount{
				Path:       "/home/koding/X",
				RemotePath: "/home/koding/remote/b",
			},
		},
	}

	ms, err := mountsObject()
	if err != nil {
		t.Fatal(err)
	}

	for name, test := range tests {
		t.Run(name, func(t *testing.T) {
			if err := ms.Add(test.ID, test.MountID, test.Mount); err == nil {
				fmt.Errorf("want err != nil; got nil")
			}
		})
	}
}

func mountsObject() (*Mounts, error) {
	data := []struct {
		ID      machine.ID
		MountID mount.ID
		Mount   mount.Mount
	}{
		{
			ID:      "machineA",
			MountID: "mountAA",
			Mount: mount.Mount{
				Path:       "/home/koding/a",
				RemotePath: "/home/koding/remote/a",
			},
		}, {
			ID:      "machineA",
			MountID: "mountAB",
			Mount: mount.Mount{
				Path:       "/home/koding/b",
				RemotePath: "/home/koding/remote/b",
			},
		}, {
			ID:      "machineA",
			MountID: "mountAC",
			Mount: mount.Mount{
				Path:       "/home/koding/c",
				RemotePath: "/home/koding/remote/c",
			},
		}, {
			ID:      "machineB",
			MountID: "mountBA",
			Mount: mount.Mount{
				Path:       "/home/koding/d",
				RemotePath: "/home/koding/remote/a",
			},
		},
	}

	ms := New()
	for i, d := range data {
		if err := ms.Add(d.ID, d.MountID, d.Mount); err != nil {
			return nil, fmt.Errorf("want err = nil; got %v (i:%d)", err, i)
		}
	}

	return ms, nil
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package list implements a doubly linked list.
//
// To iterate over a list (where l is a *List):
//	for e := l.Front(); e != nil; e = e.Next() {
//		// do something with e.Value
//	}
//
package list

// Element is an element of a linked list.
type Element struct {
	// Next and previous pointers in the doubly-linked list of elements.
	// To simplify the implementation, internally a list l is implemented
	// as a ring, such that &l.root is both the next element of the last
	// list element (l.Back()) and the previous element of the first list
	// element (l.Front()).
	next, prev *Element

	// The list to which this element belongs.
	list *List

	// The value stored with this element.
	Value interface{}
}

// Next returns the next list element or nil.
func (e *Element) Next() *Element {
	if p := e.next; e.list != nil && p != &e.list.root {
		return p
	}
	return nil
}

// Prev returns the previous list element or nil.
func (e *Element) Prev() *Element {
	if p := e.prev; e.list != nil && p != &e.list.root {
		return p
	}
	return nil
}

// List represents a doubly linked list.
// The zero value for List is an empty list ready to use.
type List struct {
	root Element // sentinel list element, only &root, root.prev, and root.next are used
	len  int     // current list length excluding (this) sentinel element
}

// Init initializes or clears list l.
func (l *List) Init() *List {
	l.root.next = &l.root
	l.root.prev = &l.root
	l.len = 0
	return l
}

// New returns an initialized list.
func New() *List { return new(List).Init() }

// Len returns the number of elements of list l.
// The complexity is O(1).
func (l *List) Len() int { return l.len }

// Front returns the first element of list l or nil if the list is empty.
func (l *List) Front() *Element {
	if l.len == 0 {
		return nil
	}
	return l.root.next
}

// Back returns the last element of list l or nil if the list is empty.
func (l *List) Back() *Element {
	if l.len == 0 {
		return nil
	}
	return l.root.prev
}

// lazyInit lazily initializes a zero List value.
func (l *List) lazyInit() {
	if l.root.next == nil {
		l.Init()
	}
}

// insert inserts e after at, increments l.len, and returns e.
func (l *List) insert(e, at *Element) *Element {
	n := at.next
	at.next = e
	e.prev = at
	e.next = n
	n.prev = e
	e.list = l
	l.len++
	return e
}

// insertValue is a convenience wrapper for insert(&Element{Value: v}, at).
func (l *List) insertValue(v interface{}, at *Element) *Element {
	return l.insert(&Element{Value: v}, at)
}

// remove removes e from its list, decrements l.len, and returns e.
func (l *List) remove(e *Element) *Element {
	e.prev.next = e.next
	e.next.prev = e.prev
	e.next = nil // avoid memory leaks
	e.prev = nil // avoid memory leaks
	e.list = nil
	l.len--
	return e
}

// Remove removes e from l if e is an element of list l.
// It returns the element value e.Value.
// The element must not be nil.
func (l *List) Remove(e *Element) interface{} {
	if e.list == l {
		// if e.list == l, l must have been initialized when e was inserted
		// in l or l == nil (e is a zero Element) and l.remove will crash
		l.remove(e)
	}
	return e.Value
}

// PushFront inserts a new element e with value v at the front of list l and returns e.
func (l *List) PushFront(v interface{}) *Element {
	l.lazyInit()
	return l.insertValue(v, &l.root)
}

// PushBack inserts a new element e with value v at the back of list l and returns e.
func (l *List) PushBack(v interface{}) *Element {
	l.lazyInit()
	return l.insertValue(v, l.root.prev)
}

// InsertBefore inserts a new element e with value v immediately before mark and returns e.
// If mark is not an element of l, the list is not modified.
// The mark must not be nil.
func (l *List) InsertBefore(v interface{}, mark *Element) *Element {
	if mark.list != l {
		return nil
	}
	// see comment in List.Remove about initialization of l
	return l.insertValue(v, mark.prev)
}

// InsertAfter inserts a new element e with value v immediately after mark and returns e.
// If mark is not an element of l, the list is not modified.
// The mark must not be nil.
func (l *List) InsertAfter(v interface{}, mark *Element) *Element {
	if mark.list != l {
		return nil
	}
	// see comment in List.Remove about initialization of l
	return l.insertValue(v, mark)
}

// MoveToFront moves element e to the front of list l.
// If e is not an element of l, the list is not modified.
// The element must not be nil.
func (l *List) MoveToFront(e *Element) {
	if e.list != l || l.root.next == e {
		return
	}
	// see comment in List.Remove about initialization of l
	l.insert(l.remove(e), &l.root)
}

// MoveToBack moves element e to the back of list l.
// If e is not an element of l, the list is not modified.
// The element must not be nil.
func (l *List) MoveToBack(e *Element) {
	if e.list != l || l.root.prev == e {
		return
	}
	// see comment in List.Remove about initialization of l
	l.insert(l.remove(e), l.root.prev)
}

// MoveBefore moves element e to its new position before mark.
// If e or mark is not an element of l, or e == mark, the list is not modified.
// The element and mark must not be nil.
func (l *List) MoveBefore(e, mark *Element) {
	if e.list != l || e == mark || mark.list != l {
		return
	}
	l.insert(l.remove(e), mark.prev)
}

// MoveAfter moves element e to its new position after mark.
// If e or mark is not an element of l, or e == mark, the list is not modified.
// The element and mark must not be nil.
func (l *List) MoveAfter(e, mark *Element) {
	if e.list != l || e == mark || mark.list != l {
		return
	}
	l.insert(l.remove(e), mark)
}

// PushBackList inserts a copy of an other list at the back of list l.
// The lists l and other may be the same. They must not be nil.
func (l *List) PushBackList(other *List) {
	l.lazyInit()
	for i, e := other.Len(), other.Front(); i > 0; i, e = i-1, e.Next() {
		l.insertValue(e.Value, l.root.prev)
	}
}

// PushFrontList inserts a copy of an other list at the front of list l.
// The lists l and other may be the same. They must not be nil.
func (l *List) PushFrontList(other *List) {
	l.lazyInit()
	for i, e := other.Len(), other.Back(); i > 0; i, e = i-1, e.Prev() {
		l.insertValue(e.Value, &l.root)
	}
}
<commit_msg>container/list: combining insert and remove operations while moving elements within a list.<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package list implements a doubly linked list.
//
// To iterate over a list (where l is a *List):
//	for e := l.Front(); e != nil; e = e.Next() {
//		// do something with e.Value
//	}
//
package list

// Element is an element of a linked list.
type Element struct {
	// Next and previous pointers in the doubly-linked list of elements.
	// To simplify the implementation, internally a list l is implemented
	// as a ring, such that &l.root is both the next element of the last
	// list element (l.Back()) and the previous element of the first list
	// element (l.Front()).
	next, prev *Element

	// The list to which this element belongs.
	list *List

	// The value stored with this element.
	Value interface{}
}

// Next returns the next list element or nil.
func (e *Element) Next() *Element {
	if p := e.next; e.list != nil && p != &e.list.root {
		return p
	}
	return nil
}

// Prev returns the previous list element or nil.
func (e *Element) Prev() *Element {
	if p := e.prev; e.list != nil && p != &e.list.root {
		return p
	}
	return nil
}

// List represents a doubly linked list.
// The zero value for List is an empty list ready to use.
type List struct {
	root Element // sentinel list element, only &root, root.prev, and root.next are used
	len  int     // current list length excluding (this) sentinel element
}

// Init initializes or clears list l.
func (l *List) Init() *List {
	l.root.next = &l.root
	l.root.prev = &l.root
	l.len = 0
	return l
}

// New returns an initialized list.
func New() *List { return new(List).Init() }

// Len returns the number of elements of list l.
// The complexity is O(1).
func (l *List) Len() int { return l.len }

// Front returns the first element of list l or nil if the list is empty.
func (l *List) Front() *Element {
	if l.len == 0 {
		return nil
	}
	return l.root.next
}

// Back returns the last element of list l or nil if the list is empty.
func (l *List) Back() *Element {
	if l.len == 0 {
		return nil
	}
	return l.root.prev
}

// lazyInit lazily initializes a zero List value.
func (l *List) lazyInit() {
	if l.root.next == nil {
		l.Init()
	}
}

// insert inserts e after at, increments l.len, and returns e.
func (l *List) insert(e, at *Element) *Element {
	n := at.next
	at.next = e
	e.prev = at
	e.next = n
	n.prev = e
	e.list = l
	l.len++
	return e
}

// insertValue is a convenience wrapper for insert(&Element{Value: v}, at).
func (l *List) insertValue(v interface{}, at *Element) *Element {
	return l.insert(&Element{Value: v}, at)
}

// remove removes e from its list, decrements l.len, and returns e.
func (l *List) remove(e *Element) *Element {
	e.prev.next = e.next
	e.next.prev = e.prev
	e.next = nil // avoid memory leaks
	e.prev = nil // avoid memory leaks
	e.list = nil
	l.len--
	return e
}

// move moves e to next to at and returns e.
func (l *List) move(e, at *Element) *Element {
	if e == at {
		return e
	}
	e.prev.next = e.next
	e.next.prev = e.prev

	n := at.next
	at.next = e
	e.prev = at
	e.next = n
	n.prev = e

	return e
}

// Remove removes e from l if e is an element of list l.
// It returns the element value e.Value.
// The element must not be nil.
func (l *List) Remove(e *Element) interface{} {
	if e.list == l {
		// if e.list == l, l must have been initialized when e was inserted
		// in l or l == nil (e is a zero Element) and l.remove will crash
		l.remove(e)
	}
	return e.Value
}

// PushFront inserts a new element e with value v at the front of list l and returns e.
func (l *List) PushFront(v interface{}) *Element {
	l.lazyInit()
	return l.insertValue(v, &l.root)
}

// PushBack inserts a new element e with value v at the back of list l and returns e.
func (l *List) PushBack(v interface{}) *Element {
	l.lazyInit()
	return l.insertValue(v, l.root.prev)
}

// InsertBefore inserts a new element e with value v immediately before mark and returns e.
// If mark is not an element of l, the list is not modified.
// The mark must not be nil.
func (l *List) InsertBefore(v interface{}, mark *Element) *Element {
	if mark.list != l {
		return nil
	}
	// see comment in List.Remove about initialization of l
	return l.insertValue(v, mark.prev)
}

// InsertAfter inserts a new element e with value v immediately after mark and returns e.
// If mark is not an element of l, the list is not modified.
// The mark must not be nil.
func (l *List) InsertAfter(v interface{}, mark *Element) *Element {
	if mark.list != l {
		return nil
	}
	// see comment in List.Remove about initialization of l
	return l.insertValue(v, mark)
}

// MoveToFront moves element e to the front of list l.
// If e is not an element of l, the list is not modified.
// The element must not be nil.
func (l *List) MoveToFront(e *Element) {
	if e.list != l || l.root.next == e {
		return
	}
	// see comment in List.Remove about initialization of l
	l.move(e, &l.root)
}

// MoveToBack moves element e to the back of list l.
// If e is not an element of l, the list is not modified.
// The element must not be nil.
func (l *List) MoveToBack(e *Element) {
	if e.list != l || l.root.prev == e {
		return
	}
	// see comment in List.Remove about initialization of l
	l.move(e, l.root.prev)
}

// MoveBefore moves element e to its new position before mark.
// If e or mark is not an element of l, or e == mark, the list is not modified.
// The element and mark must not be nil.
func (l *List) MoveBefore(e, mark *Element) {
	if e.list != l || e == mark || mark.list != l {
		return
	}
	l.move(e, mark.prev)
}

// MoveAfter moves element e to its new position after mark.
// If e or mark is not an element of l, or e == mark, the list is not modified.
// The element and mark must not be nil.
func (l *List) MoveAfter(e, mark *Element) {
	if e.list != l || e == mark || mark.list != l {
		return
	}
	l.move(e, mark)
}

// PushBackList inserts a copy of an other list at the back of list l.
// The lists l and other may be the same. They must not be nil.
func (l *List) PushBackList(other *List) {
	l.lazyInit()
	for i, e := other.Len(), other.Front(); i > 0; i, e = i-1, e.Next() {
		l.insertValue(e.Value, l.root.prev)
	}
}

// PushFrontList inserts a copy of an other list at the front of list l.
// The lists l and other may be the same. They must not be nil.
func (l *List) PushFrontList(other *List) {
	l.lazyInit()
	for i, e := other.Len(), other.Back(); i > 0; i, e = i-1, e.Prev() {
		l.insertValue(e.Value, &l.root)
	}
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"
	"runtime"
	"runtime/pprof"
	"syscall"

	"github.com/mozilla-services/pushgo/simplepush"
)

var (
	configFile *string = flag.String("config", "config.toml", "Configuration File")
	profile    *string = flag.String("profile", "", "Profile file output")
	memProfile *string = flag.String("memProfile", "", "Profile file output")
	logging    *int    = flag.Int("logging", 0,
		"logging level (0=none,1=critical ... 10=verbose")
	version *bool = flag.Bool("version", false, "Print the version and exit")
)

const SIGUSR1 = syscall.SIGUSR1

// -- main
func main() {
	flag.Parse()

	if *version {
		fmt.Println(simplepush.VERSION)
		return
	}

	runtime.GOMAXPROCS(runtime.NumCPU())
	// Only create profiles if requested. To view the application profiles,
	// see http://blog.golang.org/profiling-go-programs
	if *profile != "" {
		log.Printf("Creating profile...")
		f, err := os.Create(*profile)
		if err != nil {
			log.Fatal(err)
		}
		defer func() {
			log.Printf("Closing profile...")
			pprof.StopCPUProfile()
		}()
		pprof.StartCPUProfile(f)
	}
	if *memProfile != "" {
		defer func() {
			profFile, err := os.Create(*memProfile)
			if err != nil {
				log.Fatalln(err)
			}
			pprof.WriteHeapProfile(profFile)
			profFile.Close()
		}()
	}

	// Load the app from the config file
	app, err := simplepush.LoadApplicationFromFileName(*configFile, *logging)
	if err != nil {
		log.Fatalf("Error loading application: %s", err)
	}

	// Report what the app believes the current host to be, and what version.
	log.Printf("CurrentHost: %s, Version: %s", app.Hostname(), simplepush.VERSION)

	// wait for sigint
	sigChan := make(chan os.Signal)
	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGHUP, SIGUSR1)

	// And we're underway!
	errChan := app.Run()

	exitCode := 0
	select {
	case err = <-errChan:
		exitCode = 1
		log.Printf("Run: %s", err)

	case <-sigChan:
		app.Logger().Info("main", "Recieved signal, shutting down.", nil)
	}
	if err = app.Close(); err != nil {
		log.Fatalf("Error shutting down: %s", err)
	}
	os.Exit(exitCode)
}

// 04fs
// vim: set tabstab=4 softtabstop=4 shiftwidth=4 noexpandtab
<commit_msg>Use the application logger to report run errors.<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"
	"runtime"
	"runtime/pprof"
	"syscall"

	"github.com/mozilla-services/pushgo/simplepush"
)

var (
	configFile *string = flag.String("config", "config.toml", "Configuration File")
	profile    *string = flag.String("profile", "", "Profile file output")
	memProfile *string = flag.String("memProfile", "", "Profile file output")
	logging    *int    = flag.Int("logging", 0,
		"logging level (0=none,1=critical ... 10=verbose")
	version *bool = flag.Bool("version", false, "Print the version and exit")
)

const SIGUSR1 = syscall.SIGUSR1

// -- main
func main() {
	flag.Parse()

	if *version {
		fmt.Println(simplepush.VERSION)
		return
	}

	runtime.GOMAXPROCS(runtime.NumCPU())
	// Only create profiles if requested. To view the application profiles,
	// see http://blog.golang.org/profiling-go-programs
	if *profile != "" {
		log.Printf("Creating profile...")
		f, err := os.Create(*profile)
		if err != nil {
			log.Fatal(err)
		}
		defer func() {
			log.Printf("Closing profile...")
			pprof.StopCPUProfile()
		}()
		pprof.StartCPUProfile(f)
	}
	if *memProfile != "" {
		defer func() {
			profFile, err := os.Create(*memProfile)
			if err != nil {
				log.Fatalln(err)
			}
			pprof.WriteHeapProfile(profFile)
			profFile.Close()
		}()
	}

	// Load the app from the config file
	app, err := simplepush.LoadApplicationFromFileName(*configFile, *logging)
	if err != nil {
		log.Fatalf("Error loading application: %s", err)
	}

	// Report what the app believes the current host to be, and what version.
	log.Printf("CurrentHost: %s, Version: %s", app.Hostname(), simplepush.VERSION)

	// wait for sigint
	sigChan := make(chan os.Signal)
	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGHUP, SIGUSR1)

	// And we're underway!
	errChan := app.Run()

	logger := app.Logger()
	exitCode := 0
	select {
	case err = <-errChan:
		exitCode = 1
		if logger.ShouldLog(simplepush.ERROR) {
			logger.Error("main", "Run encountered an error; shutting down.",
				simplepush.LogFields{"error": err.Error()})
		}

	case <-sigChan:
		if logger.ShouldLog(simplepush.INFO) {
			logger.Info("main", "Recieved signal, shutting down.", nil)
		}
	}
	if err = app.Close(); err != nil {
		log.Fatalf("Error shutting down: %s", err)
	}
	os.Exit(exitCode)
}

// 04fs
// vim: set tabstab=4 softtabstop=4 shiftwidth=4 noexpandtab
<|endoftext|>


<commit_before>package base

import (
	"encoding/json"
	"errors"
	"io"
	"log"
	"strconv"
	"sync"

	"github.com/couchbase/cbgt"
	"github.com/couchbase/go-couchbase/cbdatasource"
	"github.com/couchbase/sg-bucket"
)

// The two "handles" we have for CBGT are the manager and Cfg objects.
// This struct makes it easy to pass them around together as a unit.
type CbgtContext struct {
	Manager *cbgt.Manager
	Cfg     cbgt.Cfg
}

type SyncGatewayIndexParams struct {
	BucketName string `json:"bucket_name"`
}

const (
	SourceTypeCouchbase      = "couchbase"
	IndexTypeSyncGateway     = "sync_gateway" // Used by CBGT for its data path
	IndexCategorySyncGateway = "general"      // CBGT expects this index to fit into a category (general vs advanced)
)

type CBGTDCPFeed struct {
	eventFeed chan sgbucket.TapEvent
}

func (c *CBGTDCPFeed) Events() <-chan sgbucket.TapEvent {
	return c.eventFeed
}

func (c *CBGTDCPFeed) WriteEvents() chan<- sgbucket.TapEvent {
	return c.eventFeed
}

func (c *CBGTDCPFeed) Close() error { // TODO
	log.Fatalf("CBGTDCPFeed.Close() called but not implemented")
	return nil
}

type SyncGatewayPIndex struct {
	mutex        sync.Mutex               // mutex used to protect meta and seqs
	seqs         map[uint16]uint64        // To track max seq #'s we received per partition (vbucketId).
	meta         map[uint16][]byte        // To track metadata blob's per partition (vbucketId).
	feedEvents   chan<- sgbucket.TapEvent // The channel to forward TapEvents
	bucket       CouchbaseBucket          // the couchbase bucket
	tapArguments sgbucket.TapArguments    // tap args
	stableClock  SequenceClock            // The stable clock when this PIndex object was created
}

func NewSyncGatewayPIndex(feedEvents chan<- sgbucket.TapEvent, bucket CouchbaseBucket, args sgbucket.TapArguments, stableClock SequenceClock) *SyncGatewayPIndex {
	pindex := &SyncGatewayPIndex{
		feedEvents:   feedEvents,
		bucket:       bucket,
		tapArguments: args,
		stableClock:  stableClock,
	}

	if err := pindex.SeedSeqnos(); err != nil {
		log.Fatalf("Error calling SeedSeqnos for pindex: %v", err)
	}

	return pindex
}

func (s *SyncGatewayPIndex) SeedSeqnos() error {
	s.mutex.Lock()
	defer s.mutex.Unlock()

	maxVbno, err := s.bucket.GetMaxVbno()
	if err != nil {
		return err
	}

	startSeqnos := make(map[uint16]uint64, maxVbno)
	vbuuids := make(map[uint16]uint64, maxVbno)

	highSeqnos := s.stableClock.ValueAsMap()

	// GetStatsVbSeqno retrieves high sequence number for each vbucket, to enable starting
	// DCP stream from that position.  Also being used as a check on whether the server supports
	// DCP.
	statsUuids, highSeqnosFromBucket, err := s.bucket.GetStatsVbSeqno(maxVbno, true)
	if err != nil {
		return errors.New("Error retrieving stats-vbseqno - DCP not supported")
	}

	if s.tapArguments.Backfill == sgbucket.TapNoBackfill {
		// For non-backfill, use vbucket uuids, high sequence numbers
		LogTo("Feed+", "Seeding seqnos: %v", highSeqnos)
		vbuuids = statsUuids
		startSeqnos = highSeqnos
	}

	// Set the high seqnos as-is
	s.seqs = startSeqnos

	// For metadata, we need to do more work to build metadata based on uuid and map values.  This
	// isn't strictly to the design of cbdatasource.Receiver, which intends metadata to be opaque, but
	// is required in order to have the BucketDataSource start the UPRStream as needed.
	// The implementation has been reviewed with the cbdatasource owners and they agree this is a
	// reasonable approach, as the structure of VBucketMetaData is expected to rarely change.
	for vbucketId, vbuuid := range vbuuids {

		failOver := make([][]uint64, 1)
		failOverEntry := []uint64{vbuuid, 0}
		failOver[0] = failOverEntry

		highSeqnoFromStableClock := s.seqs[vbucketId]
		highSeqnoFromBucket := highSeqnosFromBucket[vbucketId]

		if highSeqnoFromStableClock > highSeqnoFromBucket {
			Warn("issue_1259 highSeqnoFromStableClock (%d) > highSeqnoFromBucket (%d) for vb %d", highSeqnoFromStableClock, highSeqnoFromBucket, vbucketId)
		}

		metadata := &cbdatasource.VBucketMetaData{
			SeqStart:    s.seqs[vbucketId],
			SeqEnd:      uint64(0xFFFFFFFFFFFFFFFF),
			SnapStart:   s.seqs[vbucketId],
			SnapEnd:     s.seqs[vbucketId],
			FailOverLog: failOver,
		}
		buf, err := json.Marshal(metadata)
		if err == nil {
			if s.meta == nil {
				s.meta = make(map[uint16][]byte)
			}
			s.meta[vbucketId] = buf
		}
	}

	return nil

}

func (s *SyncGatewayPIndex) Close() error {
	return nil
}

// CBGT gives us "partition" which is a more generic version of "VbucketId".
// The partition is in string form (to be more generic), but we want numeric VBucketId's,
// so convert here.
func partitionToVbucketId(partition string) uint16 {

	vbucketNumber, err := strconv.ParseUint(partition, 10, 16) // base 10, 16 bit uint
	if err != nil {
		log.Fatalf("Expected a numeric vbucket (partition), got %v.  Err: %v", partition, err)
	}
	return uint16(vbucketNumber)

}

func (s *SyncGatewayPIndex) DataUpdate(partition string, key []byte, seq uint64, val []byte,
	cas uint64, extrasType cbgt.DestExtrasType, extras []byte) error {

	LogTo("DCP", "DataUpdate for pindex %p called with vbucket: %v.  key: %v seq: %v", s, partition, string(key), seq)

	vbucketNumber := partitionToVbucketId(partition)

	s.updateSeq(partition, seq, true)

	event := sgbucket.TapEvent{
		Opcode:   sgbucket.TapMutation,
		Key:      key,
		Value:    val,
		Sequence: seq,
		VbNo:     vbucketNumber,
	}

	s.feedEvents <- event

	return nil
}

func (s *SyncGatewayPIndex) DataDelete(partition string, key []byte, seq uint64,
	cas uint64, extrasType cbgt.DestExtrasType, extras []byte) error {

	LogTo("DCP", "DataDelete called with vbucket: %v.  key: %v", partition, string(key))

	s.updateSeq(partition, seq, true)

	event := sgbucket.TapEvent{
		Opcode:   sgbucket.TapDeletion,
		Key:      key,
		Sequence: seq,
	}

	s.feedEvents <- event

	return nil
}

func (s *SyncGatewayPIndex) SnapshotStart(partition string, snapStart, snapEnd uint64) error {

	return nil

}

// OpaqueGet() should return the opaque value previously
// provided by an earlier call to OpaqueSet().  If there was no
// previous call to OpaqueSet(), such as in the case of a brand
// new instance of a Dest (as opposed to a restarted or reloaded
// Dest), the Dest should return (nil, 0, nil) for (value,
// lastSeq, err), respectively.  The lastSeq should be the last
// sequence number received and persisted during calls to the
// Dest's DataUpdate() & DataDelete() methods.
func (s *SyncGatewayPIndex) OpaqueGet(partition string) (value []byte, lastSeq uint64, err error) {

	s.mutex.Lock()
	defer s.mutex.Unlock()

	vbucketNumber := partitionToVbucketId(partition)

	value = []byte(nil)
	if s.meta != nil {
		value = s.meta[vbucketNumber]
	}

	if s.seqs != nil {
		lastSeq = s.seqs[vbucketNumber]
	}

	return value, lastSeq, nil

}

// The Dest implementation should persist the value parameter of
// OpaqueSet() for retrieval during some future call to
// OpaqueGet() by the system.  The metadata value should be
// considered "in-stream", or as part of the sequence history of
// mutations.  That is, a later Rollback() to some previous
// sequence number for a particular partition should rollback
// both persisted metadata and regular data.  The Dest
// implementation should make its own copy of the value data.
func (s *SyncGatewayPIndex) OpaqueSet(partition string, value []byte) error {

	s.mutex.Lock()
	defer s.mutex.Unlock()

	vbucketNumber := partitionToVbucketId(partition)

	if s.meta == nil {
		s.meta = make(map[uint16][]byte)
	}
	s.meta[vbucketNumber] = value
	return nil
}

func (s *SyncGatewayPIndex) rollbackSeq(partition string, seq uint64) {

	s.updateSeq(partition, seq, false)

	if err := s.updateMeta(partition, seq); err != nil {
		Warn("RollbackSeq() unable to update meta: %v", err)
	}

}

func (s *SyncGatewayPIndex) updateMeta(partition string, seq uint64) error {

	s.mutex.Lock()
	defer s.mutex.Unlock()

	maxVbno, err := s.bucket.GetMaxVbno()
	if err != nil {
		return err
	}

	// GetStatsVbSeqno retrieves high sequence number for each vbucket, to enable starting
	// DCP stream from that position.
	vbuuids, _, err := s.bucket.GetStatsVbSeqno(maxVbno, true)
	if err != nil {
		return errors.New("Error retrieving stats-vbseqno - DCP not supported")
	}

	vbucketId := partitionToVbucketId(partition)
	vbuuid := vbuuids[vbucketId]

	failOver := make([][]uint64, 1)
	failOverEntry := []uint64{vbuuid, 0}
	failOver[0] = failOverEntry
	metadata := &cbdatasource.VBucketMetaData{
		SeqStart:    seq,
		SeqEnd:      uint64(0xFFFFFFFFFFFFFFFF),
		SnapStart:   seq,
		SnapEnd:     seq,
		FailOverLog: failOver,
	}
	buf, err := json.Marshal(metadata)
	if err == nil {
		if s.meta == nil {
			s.meta = make(map[uint16][]byte)
		}
		s.meta[vbucketId] = buf
	}

	return nil

}

// This updates the value stored in s.seqs with the given seq number for the given partition
// (which is a string value of vbucket id).  Setting warnOnLowerSeqNo to true will check
// if we are setting the seq number to a _lower_ value than we already have stored for that
// vbucket and log a warning in that case.  The valid case for setting warnOnLowerSeqNo to
// false is when it's a rollback scenario.  See https://github.com/couchbase/sync_gateway/issues/1098 for dev notes.
func (s *SyncGatewayPIndex) updateSeq(partition string, seq uint64, warnOnLowerSeqNo bool) {
	s.mutex.Lock()
	defer s.mutex.Unlock()

	vbucketNumber := partitionToVbucketId(partition)

	if s.seqs == nil {
		s.seqs = make(map[uint16]uint64)
	}
	if seq < s.seqs[vbucketNumber] && warnOnLowerSeqNo == true {
		Warn("Setting to _lower_ sequence number than previous: %v -> %v", s.seqs[vbucketNumber], seq)
	}

	s.seqs[vbucketNumber] = seq // Remember the max seq for GetMetaData().

}

func (s *SyncGatewayPIndex) Rollback(partition string, rollbackSeq uint64) error {

	// TODO: this should rollback the relevant metadata too (vals set via OpaqueSet())
	// As of the time of this writing, I believe this is also broken in the master branch
	// of Sync Gateway.

	Warn("DCP Rollback request SyncGatewayPIndex - rolling back DCP feed for: vbucketId: %s, rollbackSeq: %x", partition, rollbackSeq)

	s.rollbackSeq(partition, rollbackSeq)

	return nil
}

func (s *SyncGatewayPIndex) ConsistencyWait(partition, partitionUUID string,
	consistencyLevel string,
	consistencySeq uint64,
	cancelCh <-chan bool) error {
	return nil
}

func (s *SyncGatewayPIndex) Count(pindex *cbgt.PIndex, cancelCh <-chan bool) (uint64, error) {
	return 0, nil
}

func (s *SyncGatewayPIndex) Query(pindex *cbgt.PIndex, req []byte, w io.Writer,
	cancelCh <-chan bool) error {
	return nil
}

func (s *SyncGatewayPIndex) Stats(io.Writer) error {
	return nil
}

// When we detect other nodes have stopped pushing heartbeats, remove from CBGT cluster
type HeartbeatStoppedHandler struct {
	Cfg         cbgt.Cfg
	Manager     *cbgt.Manager
	CbgtVersion string
}

func (h HeartbeatStoppedHandler) StaleHeartBeatDetected(nodeUuid string) {

	LogTo("DIndex+", "StaleHeartBeatDetected for node: %v", nodeUuid)

	kinds := []string{cbgt.NODE_DEFS_KNOWN, cbgt.NODE_DEFS_WANTED}
	for _, kind := range kinds {
		LogTo("DIndex+", "Telling CBGT to remove node: %v (kind: %v, cbgt version: %v)", nodeUuid, kind, h.CbgtVersion)
		if err := cbgt.CfgRemoveNodeDef(
			h.Cfg,
			kind,
			nodeUuid,
			h.CbgtVersion,
		); err != nil {
			Warn("Warning: attempted to remove %v (%v) from CBGT but failed: %v", nodeUuid, kind, err)
		}

	}

}

func CBGTPlanParams(numShards, numVbuckets uint16) cbgt.PlanParams {

	// Make sure the number of vbuckets is a power of two, since it's possible
	// (but not common) to configure the number of vbuckets as such.
	if !IsPowerOfTwo(numVbuckets) {
		LogPanic("The number of vbuckets is %v, but Sync Gateway expects this to be a power of two", numVbuckets)
	}

	// We can't allow more shards than vbuckets, that makes no sense because each
	// shard would be responsible for less than one vbucket.
	if numShards > numVbuckets {
		LogPanic("The number of shards (%v) must be less than the number of vbuckets (%v)", numShards, numVbuckets)
	}

	// Calculate numVbucketsPerShard based on numVbuckets and num_shards.
	// Due to the guarantees above and the ValidateOrPanic() method, this
	// is guaranteed to divide evenly.
	numVbucketsPerShard := numVbuckets / numShards

	return cbgt.PlanParams{
		MaxPartitionsPerPIndex: int(numVbucketsPerShard),
		NumReplicas:            0, // no use case for Sync Gateway to have pindex replicas
	}

}
<commit_msg>Fix stats in order to make CBGT diag output valid json<commit_after>package base

import (
	"encoding/json"
	"errors"
	"io"
	"log"
	"strconv"
	"sync"

	"github.com/couchbase/cbgt"
	"github.com/couchbase/go-couchbase/cbdatasource"
	"github.com/couchbase/sg-bucket"
)

// The two "handles" we have for CBGT are the manager and Cfg objects.
// This struct makes it easy to pass them around together as a unit.
type CbgtContext struct {
	Manager *cbgt.Manager
	Cfg     cbgt.Cfg
}

type SyncGatewayIndexParams struct {
	BucketName string `json:"bucket_name"`
}

const (
	SourceTypeCouchbase      = "couchbase"
	IndexTypeSyncGateway     = "sync_gateway" // Used by CBGT for its data path
	IndexCategorySyncGateway = "general"      // CBGT expects this index to fit into a category (general vs advanced)
)

type CBGTDCPFeed struct {
	eventFeed chan sgbucket.TapEvent
}

func (c *CBGTDCPFeed) Events() <-chan sgbucket.TapEvent {
	return c.eventFeed
}

func (c *CBGTDCPFeed) WriteEvents() chan<- sgbucket.TapEvent {
	return c.eventFeed
}

func (c *CBGTDCPFeed) Close() error { // TODO
	log.Fatalf("CBGTDCPFeed.Close() called but not implemented")
	return nil
}

type SyncGatewayPIndex struct {
	mutex        sync.Mutex               // mutex used to protect meta and seqs
	seqs         map[uint16]uint64        // To track max seq #'s we received per partition (vbucketId).
	meta         map[uint16][]byte        // To track metadata blob's per partition (vbucketId).
	feedEvents   chan<- sgbucket.TapEvent // The channel to forward TapEvents
	bucket       CouchbaseBucket          // the couchbase bucket
	tapArguments sgbucket.TapArguments    // tap args
	stableClock  SequenceClock            // The stable clock when this PIndex object was created
}

func NewSyncGatewayPIndex(feedEvents chan<- sgbucket.TapEvent, bucket CouchbaseBucket, args sgbucket.TapArguments, stableClock SequenceClock) *SyncGatewayPIndex {
	pindex := &SyncGatewayPIndex{
		feedEvents:   feedEvents,
		bucket:       bucket,
		tapArguments: args,
		stableClock:  stableClock,
	}

	if err := pindex.SeedSeqnos(); err != nil {
		log.Fatalf("Error calling SeedSeqnos for pindex: %v", err)
	}

	return pindex
}

func (s *SyncGatewayPIndex) SeedSeqnos() error {
	s.mutex.Lock()
	defer s.mutex.Unlock()

	maxVbno, err := s.bucket.GetMaxVbno()
	if err != nil {
		return err
	}

	startSeqnos := make(map[uint16]uint64, maxVbno)
	vbuuids := make(map[uint16]uint64, maxVbno)

	highSeqnos := s.stableClock.ValueAsMap()

	// GetStatsVbSeqno retrieves high sequence number for each vbucket, to enable starting
	// DCP stream from that position.  Also being used as a check on whether the server supports
	// DCP.
	statsUuids, highSeqnosFromBucket, err := s.bucket.GetStatsVbSeqno(maxVbno, true)
	if err != nil {
		return errors.New("Error retrieving stats-vbseqno - DCP not supported")
	}

	if s.tapArguments.Backfill == sgbucket.TapNoBackfill {
		// For non-backfill, use vbucket uuids, high sequence numbers
		LogTo("Feed+", "Seeding seqnos: %v", highSeqnos)
		vbuuids = statsUuids
		startSeqnos = highSeqnos
	}

	// Set the high seqnos as-is
	s.seqs = startSeqnos

	// For metadata, we need to do more work to build metadata based on uuid and map values.  This
	// isn't strictly to the design of cbdatasource.Receiver, which intends metadata to be opaque, but
	// is required in order to have the BucketDataSource start the UPRStream as needed.
	// The implementation has been reviewed with the cbdatasource owners and they agree this is a
	// reasonable approach, as the structure of VBucketMetaData is expected to rarely change.
	for vbucketId, vbuuid := range vbuuids {

		failOver := make([][]uint64, 1)
		failOverEntry := []uint64{vbuuid, 0}
		failOver[0] = failOverEntry

		highSeqnoFromStableClock := s.seqs[vbucketId]
		highSeqnoFromBucket := highSeqnosFromBucket[vbucketId]

		if highSeqnoFromStableClock > highSeqnoFromBucket {
			Warn("issue_1259 highSeqnoFromStableClock (%d) > highSeqnoFromBucket (%d) for vb %d", highSeqnoFromStableClock, highSeqnoFromBucket, vbucketId)
		}

		metadata := &cbdatasource.VBucketMetaData{
			SeqStart:    s.seqs[vbucketId],
			SeqEnd:      uint64(0xFFFFFFFFFFFFFFFF),
			SnapStart:   s.seqs[vbucketId],
			SnapEnd:     s.seqs[vbucketId],
			FailOverLog: failOver,
		}
		buf, err := json.Marshal(metadata)
		if err == nil {
			if s.meta == nil {
				s.meta = make(map[uint16][]byte)
			}
			s.meta[vbucketId] = buf
		}
	}

	return nil

}

func (s *SyncGatewayPIndex) Close() error {
	return nil
}

// CBGT gives us "partition" which is a more generic version of "VbucketId".
// The partition is in string form (to be more generic), but we want numeric VBucketId's,
// so convert here.
func partitionToVbucketId(partition string) uint16 {

	vbucketNumber, err := strconv.ParseUint(partition, 10, 16) // base 10, 16 bit uint
	if err != nil {
		log.Fatalf("Expected a numeric vbucket (partition), got %v.  Err: %v", partition, err)
	}
	return uint16(vbucketNumber)

}

func (s *SyncGatewayPIndex) DataUpdate(partition string, key []byte, seq uint64, val []byte,
	cas uint64, extrasType cbgt.DestExtrasType, extras []byte) error {

	LogTo("DCP", "DataUpdate for pindex %p called with vbucket: %v.  key: %v seq: %v", s, partition, string(key), seq)

	vbucketNumber := partitionToVbucketId(partition)

	s.updateSeq(partition, seq, true)

	event := sgbucket.TapEvent{
		Opcode:   sgbucket.TapMutation,
		Key:      key,
		Value:    val,
		Sequence: seq,
		VbNo:     vbucketNumber,
	}

	s.feedEvents <- event

	return nil
}

func (s *SyncGatewayPIndex) DataDelete(partition string, key []byte, seq uint64,
	cas uint64, extrasType cbgt.DestExtrasType, extras []byte) error {

	LogTo("DCP", "DataDelete called with vbucket: %v.  key: %v", partition, string(key))

	s.updateSeq(partition, seq, true)

	event := sgbucket.TapEvent{
		Opcode:   sgbucket.TapDeletion,
		Key:      key,
		Sequence: seq,
	}

	s.feedEvents <- event

	return nil
}

func (s *SyncGatewayPIndex) SnapshotStart(partition string, snapStart, snapEnd uint64) error {

	return nil

}

// OpaqueGet() should return the opaque value previously
// provided by an earlier call to OpaqueSet().  If there was no
// previous call to OpaqueSet(), such as in the case of a brand
// new instance of a Dest (as opposed to a restarted or reloaded
// Dest), the Dest should return (nil, 0, nil) for (value,
// lastSeq, err), respectively.  The lastSeq should be the last
// sequence number received and persisted during calls to the
// Dest's DataUpdate() & DataDelete() methods.
func (s *SyncGatewayPIndex) OpaqueGet(partition string) (value []byte, lastSeq uint64, err error) {

	s.mutex.Lock()
	defer s.mutex.Unlock()

	vbucketNumber := partitionToVbucketId(partition)

	value = []byte(nil)
	if s.meta != nil {
		value = s.meta[vbucketNumber]
	}

	if s.seqs != nil {
		lastSeq = s.seqs[vbucketNumber]
	}

	return value, lastSeq, nil

}

// The Dest implementation should persist the value parameter of
// OpaqueSet() for retrieval during some future call to
// OpaqueGet() by the system.  The metadata value should be
// considered "in-stream", or as part of the sequence history of
// mutations.  That is, a later Rollback() to some previous
// sequence number for a particular partition should rollback
// both persisted metadata and regular data.  The Dest
// implementation should make its own copy of the value data.
func (s *SyncGatewayPIndex) OpaqueSet(partition string, value []byte) error {

	s.mutex.Lock()
	defer s.mutex.Unlock()

	vbucketNumber := partitionToVbucketId(partition)

	if s.meta == nil {
		s.meta = make(map[uint16][]byte)
	}
	s.meta[vbucketNumber] = value
	return nil
}

func (s *SyncGatewayPIndex) rollbackSeq(partition string, seq uint64) {

	s.updateSeq(partition, seq, false)

	if err := s.updateMeta(partition, seq); err != nil {
		Warn("RollbackSeq() unable to update meta: %v", err)
	}

}

func (s *SyncGatewayPIndex) updateMeta(partition string, seq uint64) error {

	s.mutex.Lock()
	defer s.mutex.Unlock()

	maxVbno, err := s.bucket.GetMaxVbno()
	if err != nil {
		return err
	}

	// GetStatsVbSeqno retrieves high sequence number for each vbucket, to enable starting
	// DCP stream from that position.
	vbuuids, _, err := s.bucket.GetStatsVbSeqno(maxVbno, true)
	if err != nil {
		return errors.New("Error retrieving stats-vbseqno - DCP not supported")
	}

	vbucketId := partitionToVbucketId(partition)
	vbuuid := vbuuids[vbucketId]

	failOver := make([][]uint64, 1)
	failOverEntry := []uint64{vbuuid, 0}
	failOver[0] = failOverEntry
	metadata := &cbdatasource.VBucketMetaData{
		SeqStart:    seq,
		SeqEnd:      uint64(0xFFFFFFFFFFFFFFFF),
		SnapStart:   seq,
		SnapEnd:     seq,
		FailOverLog: failOver,
	}
	buf, err := json.Marshal(metadata)
	if err == nil {
		if s.meta == nil {
			s.meta = make(map[uint16][]byte)
		}
		s.meta[vbucketId] = buf
	}

	return nil

}

// This updates the value stored in s.seqs with the given seq number for the given partition
// (which is a string value of vbucket id).  Setting warnOnLowerSeqNo to true will check
// if we are setting the seq number to a _lower_ value than we already have stored for that
// vbucket and log a warning in that case.  The valid case for setting warnOnLowerSeqNo to
// false is when it's a rollback scenario.  See https://github.com/couchbase/sync_gateway/issues/1098 for dev notes.
func (s *SyncGatewayPIndex) updateSeq(partition string, seq uint64, warnOnLowerSeqNo bool) {
	s.mutex.Lock()
	defer s.mutex.Unlock()

	vbucketNumber := partitionToVbucketId(partition)

	if s.seqs == nil {
		s.seqs = make(map[uint16]uint64)
	}
	if seq < s.seqs[vbucketNumber] && warnOnLowerSeqNo == true {
		Warn("Setting to _lower_ sequence number than previous: %v -> %v", s.seqs[vbucketNumber], seq)
	}

	s.seqs[vbucketNumber] = seq // Remember the max seq for GetMetaData().

}

func (s *SyncGatewayPIndex) Rollback(partition string, rollbackSeq uint64) error {

	// TODO: this should rollback the relevant metadata too (vals set via OpaqueSet())
	// As of the time of this writing, I believe this is also broken in the master branch
	// of Sync Gateway.

	Warn("DCP Rollback request SyncGatewayPIndex - rolling back DCP feed for: vbucketId: %s, rollbackSeq: %x", partition, rollbackSeq)

	s.rollbackSeq(partition, rollbackSeq)

	return nil
}

func (s *SyncGatewayPIndex) ConsistencyWait(partition, partitionUUID string,
	consistencyLevel string,
	consistencySeq uint64,
	cancelCh <-chan bool) error {
	return nil
}

func (s *SyncGatewayPIndex) Count(pindex *cbgt.PIndex, cancelCh <-chan bool) (uint64, error) {
	return 0, nil
}

func (s *SyncGatewayPIndex) Query(pindex *cbgt.PIndex, req []byte, w io.Writer,
	cancelCh <-chan bool) error {
	return nil
}

func (s *SyncGatewayPIndex) Stats(w io.Writer) error {
	_, err := w.Write(cbgt.JsonNULL)
	return err
}

// When we detect other nodes have stopped pushing heartbeats, remove from CBGT cluster
type HeartbeatStoppedHandler struct {
	Cfg         cbgt.Cfg
	Manager     *cbgt.Manager
	CbgtVersion string
}

func (h HeartbeatStoppedHandler) StaleHeartBeatDetected(nodeUuid string) {

	LogTo("DIndex+", "StaleHeartBeatDetected for node: %v", nodeUuid)

	kinds := []string{cbgt.NODE_DEFS_KNOWN, cbgt.NODE_DEFS_WANTED}
	for _, kind := range kinds {
		LogTo("DIndex+", "Telling CBGT to remove node: %v (kind: %v, cbgt version: %v)", nodeUuid, kind, h.CbgtVersion)
		if err := cbgt.CfgRemoveNodeDef(
			h.Cfg,
			kind,
			nodeUuid,
			h.CbgtVersion,
		); err != nil {
			Warn("Warning: attempted to remove %v (%v) from CBGT but failed: %v", nodeUuid, kind, err)
		}

	}

}

func CBGTPlanParams(numShards, numVbuckets uint16) cbgt.PlanParams {

	// Make sure the number of vbuckets is a power of two, since it's possible
	// (but not common) to configure the number of vbuckets as such.
	if !IsPowerOfTwo(numVbuckets) {
		LogPanic("The number of vbuckets is %v, but Sync Gateway expects this to be a power of two", numVbuckets)
	}

	// We can't allow more shards than vbuckets, that makes no sense because each
	// shard would be responsible for less than one vbucket.
	if numShards > numVbuckets {
		LogPanic("The number of shards (%v) must be less than the number of vbuckets (%v)", numShards, numVbuckets)
	}

	// Calculate numVbucketsPerShard based on numVbuckets and num_shards.
	// Due to the guarantees above and the ValidateOrPanic() method, this
	// is guaranteed to divide evenly.
	numVbucketsPerShard := numVbuckets / numShards

	return cbgt.PlanParams{
		MaxPartitionsPerPIndex: int(numVbucketsPerShard),
		NumReplicas:            0, // no use case for Sync Gateway to have pindex replicas
	}

}
<|endoftext|>


<commit_before>package version

// Version is the version of the build.
const Version = "0.1.38"
<commit_msg>bump to v0.1.39-dev<commit_after>package version

// Version is the version of the build.
const Version = "0.1.39-dev"
<|endoftext|>