<commit_before>package fcm

import (
	"fmt"

	"github.com/Bogh/gcm"
	"github.com/smancke/guble/protocol"
	"github.com/smancke/guble/server/connector"
	"github.com/smancke/guble/server/metrics"
	"github.com/smancke/guble/server/router"
	"time"
)

const (
	// schema is the default database schema for FCM
	schema = "fcm_registration"

	deviceTokenKey = "device_token"
	userIDKEy      = "user_id"
)

// Config is used for configuring the Firebase Cloud Messaging component.
type Config struct {
	Enabled              *bool
	APIKey               *string
	Workers              *int
	Endpoint             *string
	Prefix               *string
	AfterMessageDelivery protocol.MessageDeliveryCallback
}

// Connector is the structure for handling the communication with Firebase Cloud Messaging
type fcm struct {
	Config
	connector.Connector
}

// New creates a new *fcm and returns it as an connector.ResponsiveConnector
func New(router router.Router, sender connector.Sender, config Config) (connector.ResponsiveConnector, error) {
	baseConn, err := connector.NewConnector(router, sender, connector.Config{
		Name:       "fcm",
		Schema:     schema,
		Prefix:     *config.Prefix,
		URLPattern: fmt.Sprintf("/{%s}/{%s}/{%s:.*}", deviceTokenKey, userIDKEy, connector.TopicParam),
		Workers:    *config.Workers,
	})
	if err != nil {
		logger.WithError(err).Error("Base connector error")
		return nil, err
	}

	f := &fcm{config, baseConn}
	f.SetResponseHandler(f)
	return f, nil
}

func (f *fcm) Start() error {
	err := f.Connector.Start()
	if err == nil {
		f.StartMetrics()
	}
	return err
}

func (f *fcm) StartMetrics() {
	mTotalSentMessages.Set(0)
	mTotalSendErrors.Set(0)
	mTotalResponseErrors.Set(0)
	mTotalResponseInternalErrors.Set(0)
	mTotalResponseNotRegisteredErrors.Set(0)
	mTotalReplacedCanonicalErrors.Set(0)
	mTotalResponseOtherErrors.Set(0)

	metrics.RegisterInterval(f.Context(), mMinute, time.Minute, resetIntervalMetrics, processAndResetIntervalMetrics)
	metrics.RegisterInterval(f.Context(), mHour, time.Hour, resetIntervalMetrics, processAndResetIntervalMetrics)
	metrics.RegisterInterval(f.Context(), mDay, time.Hour*24, resetIntervalMetrics, processAndResetIntervalMetrics)
}

func (f *fcm) HandleResponse(request connector.Request, responseIface interface{}, metadata *connector.Metadata, err error) error {
	if err != nil && !isValidResponseError(err) {
		logger.WithField("error", err.Error()).Error("Error sending message to FCM")
		mTotalSendErrors.Add(1)
		addToLatenciesAndCountsMaps(currentTotalErrorsLatenciesKey, currentTotalErrorsKey, metadata.Latency)
		return err
	}
	message := request.Message()
	subscriber := request.Subscriber()

	response, ok := responseIface.(*gcm.Response)
	if !ok {
		mTotalResponseErrors.Add(1)
		return fmt.Errorf("Invalid FCM Response")
	}

	logger.WithField("messageID", message.ID).Debug("Delivered message to FCM")
	subscriber.SetLastID(message.ID)
	if err := f.Manager().Update(request.Subscriber()); err != nil {
		mTotalResponseInternalErrors.Add(1)
		return err
	}
	if response.Ok() {
		mTotalSentMessages.Add(1)
		addToLatenciesAndCountsMaps(currentTotalMessagesLatenciesKey, currentTotalMessagesKey, metadata.Latency)
		return nil
	}

	logger.WithField("success", response.Success).Debug("Handling FCM Error")

	switch errText := response.Error.Error(); errText {
	case "NotRegistered":
		logger.Debug("Removing not registered FCM subscription")
		f.Manager().Remove(subscriber)
		mTotalResponseNotRegisteredErrors.Add(1)
		return response.Error
	case "InvalidRegistration":
		logger.WithField("jsonError", errText).Error("InvalidRegistration of FCM subscription")
	default:
		logger.WithField("jsonError", errText).Error("Unexpected error while sending to FCM")
	}

	if response.CanonicalIDs != 0 {
		mTotalReplacedCanonicalErrors.Add(1)
		// we only send to one receiver, so we know that we can replace the old id with the first registration id (=canonical id)
		return f.replaceCanonical(request.Subscriber(), response.Results[0].RegistrationID)
	}
	mTotalResponseOtherErrors.Add(1)
	return nil
}

func (f *fcm) replaceCanonical(subscriber connector.Subscriber, newToken string) error {
	manager := f.Manager()
	err := manager.Remove(subscriber)
	if err != nil {
		return err
	}

	topic := subscriber.Route().Path
	params := subscriber.Route().RouteParams.Copy()

	params[deviceTokenKey] = newToken

	newSubscriber, err := manager.Create(topic, params)
	go f.Run(newSubscriber)
	return err
}
<commit_msg>private fcm.startMetrics<commit_after>package fcm

import (
	"fmt"

	"github.com/Bogh/gcm"
	"github.com/smancke/guble/protocol"
	"github.com/smancke/guble/server/connector"
	"github.com/smancke/guble/server/metrics"
	"github.com/smancke/guble/server/router"
	"time"
)

const (
	// schema is the default database schema for FCM
	schema = "fcm_registration"

	deviceTokenKey = "device_token"
	userIDKEy      = "user_id"
)

// Config is used for configuring the Firebase Cloud Messaging component.
type Config struct {
	Enabled              *bool
	APIKey               *string
	Workers              *int
	Endpoint             *string
	Prefix               *string
	AfterMessageDelivery protocol.MessageDeliveryCallback
}

// Connector is the structure for handling the communication with Firebase Cloud Messaging
type fcm struct {
	Config
	connector.Connector
}

// New creates a new *fcm and returns it as an connector.ResponsiveConnector
func New(router router.Router, sender connector.Sender, config Config) (connector.ResponsiveConnector, error) {
	baseConn, err := connector.NewConnector(router, sender, connector.Config{
		Name:       "fcm",
		Schema:     schema,
		Prefix:     *config.Prefix,
		URLPattern: fmt.Sprintf("/{%s}/{%s}/{%s:.*}", deviceTokenKey, userIDKEy, connector.TopicParam),
		Workers:    *config.Workers,
	})
	if err != nil {
		logger.WithError(err).Error("Base connector error")
		return nil, err
	}

	f := &fcm{config, baseConn}
	f.SetResponseHandler(f)
	return f, nil
}

func (f *fcm) Start() error {
	err := f.Connector.Start()
	if err == nil {
		f.startMetrics()
	}
	return err
}

func (f *fcm) startMetrics() {
	mTotalSentMessages.Set(0)
	mTotalSendErrors.Set(0)
	mTotalResponseErrors.Set(0)
	mTotalResponseInternalErrors.Set(0)
	mTotalResponseNotRegisteredErrors.Set(0)
	mTotalReplacedCanonicalErrors.Set(0)
	mTotalResponseOtherErrors.Set(0)

	metrics.RegisterInterval(f.Context(), mMinute, time.Minute, resetIntervalMetrics, processAndResetIntervalMetrics)
	metrics.RegisterInterval(f.Context(), mHour, time.Hour, resetIntervalMetrics, processAndResetIntervalMetrics)
	metrics.RegisterInterval(f.Context(), mDay, time.Hour*24, resetIntervalMetrics, processAndResetIntervalMetrics)
}

func (f *fcm) HandleResponse(request connector.Request, responseIface interface{}, metadata *connector.Metadata, err error) error {
	if err != nil && !isValidResponseError(err) {
		logger.WithField("error", err.Error()).Error("Error sending message to FCM")
		mTotalSendErrors.Add(1)
		addToLatenciesAndCountsMaps(currentTotalErrorsLatenciesKey, currentTotalErrorsKey, metadata.Latency)
		return err
	}
	message := request.Message()
	subscriber := request.Subscriber()

	response, ok := responseIface.(*gcm.Response)
	if !ok {
		mTotalResponseErrors.Add(1)
		return fmt.Errorf("Invalid FCM Response")
	}

	logger.WithField("messageID", message.ID).Debug("Delivered message to FCM")
	subscriber.SetLastID(message.ID)
	if err := f.Manager().Update(request.Subscriber()); err != nil {
		mTotalResponseInternalErrors.Add(1)
		return err
	}
	if response.Ok() {
		mTotalSentMessages.Add(1)
		addToLatenciesAndCountsMaps(currentTotalMessagesLatenciesKey, currentTotalMessagesKey, metadata.Latency)
		return nil
	}

	logger.WithField("success", response.Success).Debug("Handling FCM Error")

	switch errText := response.Error.Error(); errText {
	case "NotRegistered":
		logger.Debug("Removing not registered FCM subscription")
		f.Manager().Remove(subscriber)
		mTotalResponseNotRegisteredErrors.Add(1)
		return response.Error
	case "InvalidRegistration":
		logger.WithField("jsonError", errText).Error("InvalidRegistration of FCM subscription")
	default:
		logger.WithField("jsonError", errText).Error("Unexpected error while sending to FCM")
	}

	if response.CanonicalIDs != 0 {
		mTotalReplacedCanonicalErrors.Add(1)
		// we only send to one receiver, so we know that we can replace the old id with the first registration id (=canonical id)
		return f.replaceCanonical(request.Subscriber(), response.Results[0].RegistrationID)
	}
	mTotalResponseOtherErrors.Add(1)
	return nil
}

func (f *fcm) replaceCanonical(subscriber connector.Subscriber, newToken string) error {
	manager := f.Manager()
	err := manager.Remove(subscriber)
	if err != nil {
		return err
	}

	topic := subscriber.Route().Path
	params := subscriber.Route().RouteParams.Copy()

	params[deviceTokenKey] = newToken

	newSubscriber, err := manager.Create(topic, params)
	go f.Run(newSubscriber)
	return err
}
<|endoftext|>


<commit_before>// Copyright (c) 2014 The SurgeMQ Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package service

import (
	"bufio"
	"fmt"
	"io"
	"sync/atomic"
	//"github.com/surgemq/message"
	"encoding/binary"
	"math"
	"sync"
)

var (
	bufcnt            int64
	DefaultBufferSize int64

	DeviceInBufferSize  int64
	DeviceOutBufferSize int64

	MasterInBufferSize  int64
	MasterOutBufferSize int64
)

const (
	smallReadBlockSize    = 512
	defaultReadBlockSize  = 8192
	defaultWriteBlockSize = 8192
)

/**
2016.03.03 修改
bingbuffer结构体
*/
type buffer struct {
	readIndex  int64        //读序号
	writeIndex int64        //写序号
	ringBuffer []*ByteArray //环形buffer指针数组
	bufferSize int64        //初始化环形buffer指针数组大小
	mask       int64        //掩码：bufferSize-1
	done       int64        //是否完成
	rcond      *sync.Cond
	wcond      *sync.Cond
}

type ByteArray struct {
	bArray []byte
}

func (this *buffer) ReadCommit(index int64) {
	this.rcond.L.Lock()
	defer this.rcond.L.Unlock()
	this.ringBuffer[index] = nil
	this.rcond.Broadcast()
	this.wcond.Broadcast()
}

func (this *buffer) Len() int {
	cpos := this.GetCurrentReadIndex()
	ppos := this.GetCurrentWriteIndex()
	return int(ppos - cpos)
}

/**
2016.03.03 添加
初始化ringbuffer
参数bufferSize：初始化环形buffer指针数组大小
*/
func newBuffer(size int64) (*buffer, error) {
	if size < 0 {
		return nil, bufio.ErrNegativeCount
	}
	if size == 0 {
		size = DefaultBufferSize
	}
	if !powerOfTwo64(size) {
		fmt.Printf("Size must be power of two. Try %d.", roundUpPowerOfTwo64(size))
		return nil, fmt.Errorf("Size must be power of two. Try %d.", roundUpPowerOfTwo64(size))
	}

	return &buffer{
		readIndex:  int64(0),                 //读序号
		writeIndex: int64(0),                 //写序号
		ringBuffer: make([]*ByteArray, size), //环形buffer指针数组
		bufferSize: size,                     //初始化环形buffer指针数组大小
		mask:       size - 1,
		rcond:      sync.NewCond(new(sync.Mutex)),
		wcond:      sync.NewCond(new(sync.Mutex)),
	}, nil
}

/**
2016.03.03 添加
获取当前读序号
*/
func (this *buffer) GetCurrentReadIndex() int64 {
	return atomic.LoadInt64(&this.readIndex)
}

/**
2016.03.03 添加
获取当前写序号
*/
func (this *buffer) GetCurrentWriteIndex() int64 {
	return atomic.LoadInt64(&this.writeIndex)
}

/**
2016.03.03 添加
读取ringbuffer指定的buffer指针，返回该指针并清空ringbuffer该位置存在的指针内容，以及将读序号加1
*/
func (this *buffer) ReadBuffer() ([]byte, int64, bool) {
	this.rcond.L.Lock()
	defer this.rcond.L.Unlock()

	for {
		readIndex := atomic.LoadInt64(&this.readIndex)
		writeIndex := atomic.LoadInt64(&this.writeIndex)
		switch {
		case readIndex >= writeIndex:
			this.rcond.Wait()
		case writeIndex-readIndex > this.bufferSize:
			this.rcond.Wait()
		default:
			if readIndex == math.MaxInt64 {
				atomic.StoreInt64(&this.readIndex, int64(0))
			} else {
				atomic.AddInt64(&this.readIndex, int64(1))
			}

			index := readIndex & this.mask

			p_ := this.ringBuffer[index]
			//this.ringBuffer[index] = nil
			if p_ == nil {
				return nil, -1, false
			}
			p := p_.bArray

			if p == nil {
				return nil, -1, false
			}

			return p, index, true
		}
	}

}

/**
2016.03.03 添加
写入ringbuffer指针，以及将写序号加1
*/
func (this *buffer) WriteBuffer(in []byte) bool {
	this.wcond.L.Lock()
	defer this.wcond.L.Unlock()

	for {
		readIndex := atomic.LoadInt64(&this.readIndex)
		writeIndex := atomic.LoadInt64(&this.writeIndex)
		switch {
		case writeIndex-readIndex < 0:
			this.wcond.Wait()
		default:
			index := writeIndex & this.mask
			if writeIndex == math.MaxInt64 {
				atomic.StoreInt64(&this.writeIndex, int64(0))
			} else {
				atomic.AddInt64(&this.writeIndex, int64(1))
			}
			if this.ringBuffer[index] == nil {
				this.ringBuffer[index] = &ByteArray{bArray: in}
				this.rcond.Broadcast()
				return true
			} else {
				this.rcond.Broadcast()
				return false
			}
		}
	}

}

/**
2016.03.03 修改
完成
*/
func (this *buffer) Close() error {
	//atomic.StoreInt64(&this.done, 1)

	this.wcond.L.Lock()
	this.rcond.Broadcast()
	this.wcond.L.Unlock()

	this.rcond.L.Lock()
	this.wcond.Broadcast()
	this.rcond.L.Unlock()

	return nil
}

/*

/**
2016.03.03 修改
向ringbuffer中写数据（从connection的中向ringbuffer中写）--生产者
*/
func (this *buffer) ReadFrom(r io.Reader) (int64, error) {
	defer this.Close()
	total := int64(0)
	//for {

	//if this.isDone() {
	//	return total, io.EOF
	//}
	b := make([]byte, int64(5))
	n, err := r.Read(b[0:1])

	if n > 0 {
		total += int64(n)
		if err != nil {
			return total, err
		}
	}

	/**************************/
	cnt := 1

	// Let's read enough bytes to get the message header (msg type, remaining length)
	for {
		// If we have read 5 bytes and still not done, then there's a problem.
		if cnt > 4 {
			return 0, fmt.Errorf("sendrecv/peekMessageSize: 4th byte of remaining length has continuation bit set")
		}

		Log.Infoc(func() string {
			return fmt.Sprintf("sendrecv/peekMessageSize: %d=========", cnt)
		})
		// Peek cnt bytes from the input buffer.

		_, err := r.Read(b[cnt:(cnt + 1)])
		//fmt.Println(b)
		if err != nil {
			return 0, err
		}
		// If we got enough bytes, then check the last byte to see if the continuation
		// bit is set. If so, increment cnt and continue peeking
		if b[cnt] >= 0x80 {
			cnt++
		} else {
			break
		}
	}

	// Get the remaining length of the message
	remlen, m := binary.Uvarint(b[1:])
	//Log.Infoc(func() string {
	//	return fmt.Sprintf("b[cnt:(cnt + 1)]==end")
	//})
	// Total message length is remlen + 1 (msg type) + m (remlen bytes)

	total = int64(remlen) + int64(1) + int64(m)
	//Log.Infoc(func() string {
	//	return fmt.Sprintf("remlen===n===totle: %d===%d===%d", remlen, m, total)
	//})
	//mtype := message.MessageType(b[0] >> 4)
	/****************/
	//var msg message.Message
	//
	//msg, err = mtype.New()
	//if err != nil {
	//	return 0, err
	//}
	b_ := make([]byte, int64(remlen))
	_, err = r.Read(b_[0:])
	if err != nil {
		fmt.Println("写入buffer失败，total：%d", total)
		return total, err
	}
	b = append(b, b_...)

	//n, err = msg.Decode(b)
	//if err != nil {
	//	return 0, err
	//}

	/*************************/

	if !this.WriteBuffer(b) {
		return total, err
	}

	return total, nil
	//}
}

/**
2016.03.03 修改
*/
func (this *buffer) WriteTo(w io.Writer) (int64, error) {
	defer this.Close()
	total := int64(0)
	//for {
	//if this.isDone() {
	//	return total, io.EOF
	//}
	p, index, ok := this.ReadBuffer()
	defer this.ReadCommit(index)
	if !ok {
		return total, io.EOF
	}

	Log.Debugc(func() string {
		return fmt.Sprintf("defer this.ReadCommit(%s)", index)
	})
	Log.Debugc(func() string {
		return fmt.Sprintf("WriteTo函数》》读取*p：" + string(p))
	})

	Log.Debugc(func() string {
		return fmt.Sprintf(" WriteTo(w io.Writer)(7)")
	})
	//
	//Log.Errorc(func() string {
	//	return fmt.Sprintf("msg::" + msg.Name())
	//})
	//
	//p := make([]byte, msg.Len())
	//_, err := msg.Encode(p)
	//if err != nil {
	//	Log.Errorc(func() string {
	//		return fmt.Sprintf("msg.Encode(p)")
	//	})
	//	return total, io.EOF
	//}
	// There's some data, let's process it first
	if len(p) > 0 {
		n, err := w.Write(p)
		total += int64(n)
		Log.Debugc(func() string {
			return fmt.Sprintf("Wrote %d bytes, totaling %d bytes", n, total)
		})

		if err != nil {
			Log.Errorc(func() string {
				return fmt.Sprintf("w.Write(p) error")
			})
			return total, err
		}
	}

	return total, nil
	//}
}

/**
2016.03.03 修改
*/
func (this *buffer) isDone() bool {
	//if atomic.LoadInt64(&this.done) == 1 {
	//	return true
	//}

	return false
}

func powerOfTwo64(n int64) bool {
	return n != 0 && (n&(n-1)) == 0
}

func roundUpPowerOfTwo64(n int64) int64 {
	n--
	n |= n >> 1
	n |= n >> 2
	n |= n >> 4
	n |= n >> 8
	n |= n >> 16
	n |= n >> 32
	n++

	return n
}
<commit_msg>添加测试代码<commit_after>// Copyright (c) 2014 The SurgeMQ Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package service

import (
	"bufio"
	"fmt"
	"io"
	"sync/atomic"
	//"github.com/surgemq/message"
	"encoding/binary"
	"math"
	"sync"
)

var (
	bufcnt            int64
	DefaultBufferSize int64

	DeviceInBufferSize  int64
	DeviceOutBufferSize int64

	MasterInBufferSize  int64
	MasterOutBufferSize int64
)

const (
	smallReadBlockSize    = 512
	defaultReadBlockSize  = 8192
	defaultWriteBlockSize = 8192
)

/**
2016.03.03 修改
bingbuffer结构体
*/
type buffer struct {
	readIndex  int64        //读序号
	writeIndex int64        //写序号
	ringBuffer []*ByteArray //环形buffer指针数组
	bufferSize int64        //初始化环形buffer指针数组大小
	mask       int64        //掩码：bufferSize-1
	done       int64        //是否完成
	rcond      *sync.Cond
	wcond      *sync.Cond
}

type ByteArray struct {
	bArray []byte
}

func (this *buffer) ReadCommit(index int64) {
	this.rcond.L.Lock()
	defer this.rcond.L.Unlock()
	this.ringBuffer[index] = nil
	this.rcond.Broadcast()
	this.wcond.Broadcast()
}

func (this *buffer) Len() int {
	cpos := this.GetCurrentReadIndex()
	ppos := this.GetCurrentWriteIndex()
	return int(ppos - cpos)
}

/**
2016.03.03 添加
初始化ringbuffer
参数bufferSize：初始化环形buffer指针数组大小
*/
func newBuffer(size int64) (*buffer, error) {
	if size < 0 {
		return nil, bufio.ErrNegativeCount
	}
	if size == 0 {
		size = DefaultBufferSize
	}
	if !powerOfTwo64(size) {
		fmt.Printf("Size must be power of two. Try %d.", roundUpPowerOfTwo64(size))
		return nil, fmt.Errorf("Size must be power of two. Try %d.", roundUpPowerOfTwo64(size))
	}

	return &buffer{
		readIndex:  int64(0),                 //读序号
		writeIndex: int64(0),                 //写序号
		ringBuffer: make([]*ByteArray, size), //环形buffer指针数组
		bufferSize: size,                     //初始化环形buffer指针数组大小
		mask:       size - 1,
		rcond:      sync.NewCond(new(sync.Mutex)),
		wcond:      sync.NewCond(new(sync.Mutex)),
	}, nil
}

/**
2016.03.03 添加
获取当前读序号
*/
func (this *buffer) GetCurrentReadIndex() int64 {
	return atomic.LoadInt64(&this.readIndex)
}

/**
2016.03.03 添加
获取当前写序号
*/
func (this *buffer) GetCurrentWriteIndex() int64 {
	return atomic.LoadInt64(&this.writeIndex)
}

/**
2016.03.03 添加
读取ringbuffer指定的buffer指针，返回该指针并清空ringbuffer该位置存在的指针内容，以及将读序号加1
*/
func (this *buffer) ReadBuffer() ([]byte, int64, bool) {
	this.rcond.L.Lock()
	defer this.rcond.L.Unlock()

	for {
		readIndex := atomic.LoadInt64(&this.readIndex)
		writeIndex := atomic.LoadInt64(&this.writeIndex)
		switch {
		case readIndex >= writeIndex:
			this.rcond.Wait()
		case writeIndex-readIndex > this.bufferSize:
			this.rcond.Wait()
		default:
			if readIndex == math.MaxInt64 {
				atomic.StoreInt64(&this.readIndex, int64(0))
			} else {
				atomic.AddInt64(&this.readIndex, int64(1))
			}

			index := readIndex & this.mask

			p_ := this.ringBuffer[index]
			//this.ringBuffer[index] = nil
			if p_ == nil {
				return nil, -1, false
			}
			p := p_.bArray

			if p == nil {
				return nil, -1, false
			}

			return p, index, true
		}
	}

}

/**
2016.03.03 添加
写入ringbuffer指针，以及将写序号加1
*/
func (this *buffer) WriteBuffer(in []byte) bool {
	this.wcond.L.Lock()
	defer this.wcond.L.Unlock()

	for {
		readIndex := atomic.LoadInt64(&this.readIndex)
		writeIndex := atomic.LoadInt64(&this.writeIndex)
		switch {
		case writeIndex-readIndex < 0:
			this.wcond.Wait()
		default:
			index := writeIndex & this.mask
			if writeIndex == math.MaxInt64 {
				atomic.StoreInt64(&this.writeIndex, int64(0))
			} else {
				atomic.AddInt64(&this.writeIndex, int64(1))
			}
			if this.ringBuffer[index] == nil {
				this.ringBuffer[index] = &ByteArray{bArray: in}
				this.rcond.Broadcast()
				return true
			} else {
				this.rcond.Broadcast()
				return false
			}
		}
	}

}

/**
2016.03.03 修改
完成
*/
func (this *buffer) Close() error {
	//atomic.StoreInt64(&this.done, 1)

	this.wcond.L.Lock()
	this.rcond.Broadcast()
	this.wcond.L.Unlock()

	this.rcond.L.Lock()
	this.wcond.Broadcast()
	this.rcond.L.Unlock()

	return nil
}

/*

/**
2016.03.03 修改
向ringbuffer中写数据（从connection的中向ringbuffer中写）--生产者
*/
func (this *buffer) ReadFrom(r io.Reader) (int64, error) {
	defer this.Close()
	total := int64(0)
	//for {

	//if this.isDone() {
	//	return total, io.EOF
	//}
	b := make([]byte, int64(5))
	n, err := r.Read(b[0:1])

	if n > 0 {
		total += int64(n)
		if err != nil {
			return total, err
		}
	}

	/**************************/
	cnt := 1

	// Let's read enough bytes to get the message header (msg type, remaining length)
	for {
		// If we have read 5 bytes and still not done, then there's a problem.
		if cnt > 4 {
			return 0, fmt.Errorf("sendrecv/peekMessageSize: 4th byte of remaining length has continuation bit set")
		}

		Log.Infoc(func() string {
			return fmt.Sprintf("sendrecv/peekMessageSize: %d=========", cnt)
		})
		// Peek cnt bytes from the input buffer.

		_, err := r.Read(b[cnt:(cnt + 1)])
		//fmt.Println(b)
		if err != nil {
			return 0, err
		}
		// If we got enough bytes, then check the last byte to see if the continuation
		// bit is set. If so, increment cnt and continue peeking
		if b[cnt] >= 0x80 {
			cnt++
		} else {
			break
		}
	}

	// Get the remaining length of the message
	remlen, m := binary.Uvarint(b[1:])
	//Log.Infoc(func() string {
	//	return fmt.Sprintf("b[cnt:(cnt + 1)]==end")
	//})
	// Total message length is remlen + 1 (msg type) + m (remlen bytes)

	total = int64(remlen) + int64(1) + int64(m)
	//Log.Infoc(func() string {
	//	return fmt.Sprintf("remlen===n===totle: %d===%d===%d", remlen, m, total)
	//})
	//mtype := message.MessageType(b[0] >> 4)
	/****************/
	//var msg message.Message
	//
	//msg, err = mtype.New()
	//if err != nil {
	//	return 0, err
	//}
	b_ := make([]byte, int64(remlen))
	_, err = r.Read(b_[0:])
	if err != nil {
		fmt.Println("写入buffer失败，total：%d", total)
		return total, err
	}
	b = append(b, b_[0:]...)

	//n, err = msg.Decode(b)
	//if err != nil {
	//	return 0, err
	//}

	/*************************/

	if !this.WriteBuffer(b) {
		return total, err
	}

	return total, nil
	//}
}

/**
2016.03.03 修改
*/
func (this *buffer) WriteTo(w io.Writer) (int64, error) {
	defer this.Close()
	total := int64(0)
	//for {
	//if this.isDone() {
	//	return total, io.EOF
	//}
	p, index, ok := this.ReadBuffer()
	defer this.ReadCommit(index)
	if !ok {
		return total, io.EOF
	}

	Log.Debugc(func() string {
		return fmt.Sprintf("defer this.ReadCommit(%s)", index)
	})
	Log.Debugc(func() string {
		return fmt.Sprintf("WriteTo函数》》读取*p：" + string(p))
	})

	Log.Debugc(func() string {
		return fmt.Sprintf(" WriteTo(w io.Writer)(7)")
	})
	//
	//Log.Errorc(func() string {
	//	return fmt.Sprintf("msg::" + msg.Name())
	//})
	//
	//p := make([]byte, msg.Len())
	//_, err := msg.Encode(p)
	//if err != nil {
	//	Log.Errorc(func() string {
	//		return fmt.Sprintf("msg.Encode(p)")
	//	})
	//	return total, io.EOF
	//}
	// There's some data, let's process it first
	if len(p) > 0 {
		n, err := w.Write(p)
		total += int64(n)
		Log.Debugc(func() string {
			return fmt.Sprintf("Wrote %d bytes, totaling %d bytes", n, total)
		})

		if err != nil {
			Log.Errorc(func() string {
				return fmt.Sprintf("w.Write(p) error")
			})
			return total, err
		}
	}

	return total, nil
	//}
}

/**
2016.03.03 修改
*/
func (this *buffer) isDone() bool {
	//if atomic.LoadInt64(&this.done) == 1 {
	//	return true
	//}

	return false
}

func powerOfTwo64(n int64) bool {
	return n != 0 && (n&(n-1)) == 0
}

func roundUpPowerOfTwo64(n int64) int64 {
	n--
	n |= n >> 1
	n |= n >> 2
	n |= n >> 4
	n |= n >> 8
	n |= n >> 16
	n |= n >> 32
	n++

	return n
}
<|endoftext|>


<commit_before>package service

import (
	"crypto/md5"
	"encoding/json"
	"fmt"
	"strconv"
	"strings"
	"time"

	"github.com/aosfather/bingo"
	"github.com/aosfather/bingo/utils"
	"github.com/go-redis/redis"
)

/**
  搜索实现
  通过倒排实现关键信息的实现
  规则：
   1、原始内容使用hashmap存储，对象【ID，Content】，键值 indexname，二级key根据md5 对象转json字符串
   2、针对原始内容带的标签，key value，生成set，名称为 indexname_key_value的形式，set内容放 通过内容md5出来的键值
   3、搜索的时候，根据传递的搜索条件 key，value 数组，对找到的set（形如indexname_key_value ），实行找交集
   4、根据交集的结果二级key，并从indexname的hashmap中获取json内容

*/

type Field struct {
	Key   string `json:"key"`
	Value string `json:"value"`
}

type PageSearchResult struct {
	Id    string `json:"uuid"` //查询的请求id
	Index int64  `json:"page"` //页码
	Data  []TargetObject
}

type TargetObject struct {
	Id   string          `json:"id"`
	Data json.RawMessage `json:"data"`
}
type SourceObject struct {
	TargetObject
	Fields map[string][]string `json:"fields"`
}

type FieldType byte
const (
	FT_TEXT FieldType =11  //文本类型
	FT_NUMBER FieldType =9 //数字
	FT_ENUM   FieldType =8 //枚举
	FT_ID FieldType =7  //id唯一标识
	FT_DATE FieldType =6 //日期
)
//索引的元数据信息
type IndexMeta struct {
	Name string  //索引名称
	Fields []FieldMeta //字段
}

type FieldMeta struct {
	Name string //字段名称
	Type FieldType //类型

}

type SearchEngine struct {
	indexs   map[string]*searchIndex
	safeIndexs map[string]*searchIndex  //安全flush时候的临时索引
	client   *redis.Client
	logger   utils.Log
	pageSize int64
	pageLife int64 //分钟
}

func (this *SearchEngine) Init(context *bingo.ApplicationContext) {
	fmt.Println("init .....")
	db, err := strconv.Atoi(context.GetPropertyFromConfig("service.search.db"))
	if err != nil {
		db = 0
	}

	size, err := strconv.Atoi(context.GetPropertyFromConfig("service.search.pagesize"))
	if err != nil {
		size = 20 //默认大小20条
	}
	this.pageSize = int64(size)

	life, err := strconv.Atoi(context.GetPropertyFromConfig("service.search.pagelife"))
	if err != nil {
		life = 10 //默认时间10分钟
	}
	this.pageLife = int64(life)

	this.client = redis.NewClient(&redis.Options{
		Addr:     context.GetPropertyFromConfig("service.search.redis"),
		Password: "", // no password set
		DB:       db,
	})
	fmt.Println(context.GetPropertyFromConfig("service.search.redis"))
	this.indexs = make(map[string]*searchIndex)
	this.safeIndexs = make(map[string]*searchIndex)
	this.logger = context.GetLog("bingo_search")
}

//创建索引
func (this *SearchEngine) CreateIndex(name string) *searchIndex {
	if name != "" {
		index := this.indexs[name]
		if index == nil {
			index = &searchIndex{name, this}
			this.indexs[name] = index
		}
		return index
	}

	return nil
}

//清除索引，将整个索引的数据摧毁
func (this *SearchEngine)FlushIndex(name string) {
	//1、删除存放的数据
	this.client.Del(name)

	//2、删除所有的索引key
	keys,err:=this.client.Keys(name+"_*").Result()
	if err!=nil {
		this.logger.Debug("get index keys error:%s",err.Error())
	}

	if keys!=nil && len(keys)>0 {
		this.client.Del(keys...)
	}
}

//开始安全flush
func (this *SearchEngine)BeginSafeFlush(name string) {
	index := this.indexs[name]
	if index!=nil {
		var nindexName string
		if strings.HasSuffix(index.name,"#") {
			nindexName=index.name[0:len(index.name)-2]
		}else {
			nindexName=index.name+"#"
		}

		nindex:=&searchIndex{nindexName, this}
		this.safeIndexs[name]=nindex
	}
}

//结束安全flush，自动替换掉查询用的索引
func (this *SearchEngine)EndSafeFlush(name string) {
	index:=this.safeIndexs[name]
	if index!=nil {
		delete(this.safeIndexs,name)
		oldindex:=this.indexs[name]
		this.indexs[name]=index
        //删除旧的索引数据
		if oldindex!=nil {
			this.FlushIndex(oldindex.name)
		}
	}
}

//加载和刷新数据
func (this *SearchEngine) LoadSource(name string, obj *SourceObject) {
	if name==""||obj==nil {
		return
	}

	//判断是否正在进行安全flush，如果是安全flush，所有的更改只发生在正在替换的索引上
	index:=this.safeIndexs[name]
	if index==nil {
		index = this.CreateIndex(name)
	}

	if index != nil {
		index.LoadObject(obj)
	}

}

//删除数据
func (this *SearchEngine)RemoveSource(name string, obj *SourceObject){
   if name!=""&&obj!=nil{
	   index := this.CreateIndex(name)
	   if index != nil {
		   index.RemoveObject(obj)
	   }
   }
}

//按页获取数据
func (this *SearchEngine) FetchByPage(request string, page int64) *PageSearchResult {
	if request != "" {
		//获取request的name
		index := strings.Index(request, ":")
		if index < 0 {
			this.logger.Error("pagerequest's index name not found !")
			return nil //找不到对应的索引类型
		}

		name := request[0:index]
		return this.fetchByPage(name,request,page)
	}

	return nil
}

func (this *SearchEngine)fetchByPage(name string,request string,page int64)*PageSearchResult {
	if page <= 0 {
		page = 1
	}
	startIndex := (page - 1) * this.pageSize //+ 1 //从1开始计数
	endIndex := page*this.pageSize - 1
	this.logger.Debug(request,startIndex,endIndex)
	keys, err := this.client.LRange(request, startIndex, endIndex).Result()

	if err != nil || len(keys) == 0 {
		if err!=nil {
			this.logger.Debug(err.Error())
		}
		this.logger.Debug("no content by page!")
		return nil
	}
	go this.client.Expire(request, time.Duration(this.pageLife)*time.Minute) //更新重置失效时间
	return &PageSearchResult{request, page, this.fetch(name, keys...)}
}

//获取内容
func (this *SearchEngine) fetch(name string, keys ...string) []TargetObject {
	datas, err1 := this.client.HMGet(name, keys...).Result()
	if err1 == nil && len(datas) > 0 {

		var targets []TargetObject

		for _, v := range datas {
			if v != nil {
				t := TargetObject{}
				json.Unmarshal([]byte(fmt.Sprintf("%v", v)), &t)
				targets = append(targets, t)
			}
		}

		return targets

	} else {
		this.logger.Error("get data by index error!%s", err1.Error())
	}

	return nil
}

func (this *SearchEngine) Search(name string, input ...Field) *PageSearchResult {
	if name != "" {
		index := this.indexs[name]
		if index != nil {
			return index.Search(input...)

		}
		this.logger.Info("not found index %s", name)
	}

	return nil
}

type searchIndex struct {
	name   string
	engine *SearchEngine
}

//搜索信息
func (this *searchIndex) Search(input ...Field) *PageSearchResult {
	//生成索引搜索请求号
	requestkey := getSearchRequestUuid(this.name)

	//搜索索引
	var searchkeys []string
	var tmpkeys []string
	var notTmpkeys[]string
	for _, f := range input {

		//处理not的情况
		if strings.HasPrefix(f.Key,"!") {//字段使用！表示"非"的意思，就是不等于
		    k:=f.Key[1:len(f.Key)]
			v := f.Value
			arrays := strings.Split(v, "|")
			if len(arrays) > 1 {
				for _, subvalue := range arrays {
					notTmpkeys = append(notTmpkeys, this.buildTheKeyByItem(k, subvalue))
				}
			}else {
				notTmpkeys=append(notTmpkeys,this.buildTheKeyByItem(k, v))
			}

			continue
		}

		//处理并集
		v := f.Value
		arrays := strings.Split(v, "|")  //使用 | 分割多个取值
		if len(arrays) > 1 {
			skey := requestkey + ":" + f.Key
			var subkeys []string
			for _, subkey := range arrays {
				subkeys = append(subkeys, this.buildTheKeyByItem(f.Key, subkey))
			}

			//取并集，将结果存入临时的key中，用于后续取交集用
			this.engine.client.SUnionStore(skey, subkeys...)
			tmpkeys = append(tmpkeys, skey) //放入临时组中用于使用后删除
			searchkeys = append(searchkeys, skey)
		} else {
			searchkeys = append(searchkeys, this.buildTheKey(f))
		}

	}
	defer this.deleteTempkeys(tmpkeys) //删除临时创建的key

	var values []string
	//有差集
	if notTmpkeys!=nil && len(notTmpkeys) >1 {
		interTmpkey:=requestkey+"_tmp"
		//取交集
		result := this.engine.client.SInterStore(interTmpkey,searchkeys...)
		_, err := result.Result()
		if err != nil {
			this.engine.logger.Error("inter key error!%s", err.Error())
			return  nil
		}
		//取差集合，存储
		diffkeys:=make([]string,len(notTmpkeys)+1)
		diffkeys[0]=interTmpkey
		copy(diffkeys[1:len(diffkeys)],notTmpkeys)

		defer this.deleteTempkeys([]string{interTmpkey})//删除临时key
		data:=this.engine.client.SDiff(diffkeys...)
		values, _ = data.Result()

	} else {//如果无差集合，则直接返回交集
		result:=this.engine.client.SInter(searchkeys...)
		values, _ = result.Result()
	}

	//存储数据
	var datas []interface{}

	for _, v := range values {
		datas = append(datas, v)
	}
	this.engine.client.RPush(requestkey,datas...)

	this.engine.client.Expire(requestkey, time.Duration(this.engine.pageLife)*time.Minute) //指定x分钟后失效
	//取出第一页的数据返回
	return this.engine.fetchByPage(this.name,requestkey,1)

}

//删除临时创建的key
func (this *searchIndex) deleteTempkeys(keys []string) {
	if keys != nil && len(keys) > 0 {
		this.engine.client.Del(keys...)
	}
}

//刷新索引，加载信息到存储中
func (this *searchIndex) LoadObject(obj *SourceObject) {
	data, _ := json.Marshal(obj)
	key:=obj.Id
	//如果没有指明对象id，则使用md5作为数据的唯一标识
	if key=="" {
		key= getMd5str(string(data))
	}

	//1、放入数据到目标集合中
	this.engine.client.HSet(this.name, key, string(data))

	//2、根据field存储到各个对应的索引中
	for k, v := range obj.Fields {
		for _,vkey:=range v{
			this.engine.client.SAdd(this.buildTheKeyByItem(k, vkey), key)
		}

	}

}

//删除数据：删除数据及索引存的值
func (this *searchIndex)RemoveObject(obj *SourceObject) {
	data, _ := json.Marshal(obj)
	key:=obj.Id
	if key=="" {
		key=getMd5str(string(data))
	}
	//删除数据
	this.engine.client.HDel(this.name,key)
	//删除索引里的记录
	for k,v:=range obj.Fields {
		for _,vkey:=range v {
			this.engine.client.SRem(this.buildTheKeyByItem(k, vkey), key)
		}
	}

}

func (this *searchIndex) buildTheKey(f Field) string {
	return this.buildTheKeyByItem(f.Key, f.Value)
}

func (this *searchIndex) buildTheKeyByItem(key, value string) string {
	return fmt.Sprintf("%s_%s_%s", this.name, key, value)
}

func getMd5str(value string) string {
	data := []byte(value)
	has := md5.Sum(data)
	md5str1 := fmt.Sprintf("%x", has) //将[]byte转成16进制

	return md5str1

}

func getSearchRequestUuid(prefix string) string {
	return fmt.Sprintf("%s:%d", prefix, time.Now().UnixNano())
}
<commit_msg>增加索引删除某个词条功能<commit_after>package service

import (
	"crypto/md5"
	"encoding/json"
	"fmt"
	"strconv"
	"strings"
	"time"

	"github.com/aosfather/bingo"
	"github.com/aosfather/bingo/utils"
	"github.com/go-redis/redis"
)

/**
  搜索实现
  通过倒排实现关键信息的实现
  规则：
   1、原始内容使用hashmap存储，对象【ID，Content】，键值 indexname，二级key根据md5 对象转json字符串
   2、针对原始内容带的标签，key value，生成set，名称为 indexname_key_value的形式，set内容放 通过内容md5出来的键值
   3、搜索的时候，根据传递的搜索条件 key，value 数组，对找到的set（形如indexname_key_value ），实行找交集
   4、根据交集的结果二级key，并从indexname的hashmap中获取json内容

*/

type Field struct {
	Key   string `json:"key"`
	Value string `json:"value"`
}

type PageSearchResult struct {
	Id    string `json:"uuid"` //查询的请求id
	Index int64  `json:"page"` //页码
	Data  []TargetObject
}

type TargetObject struct {
	Id   string          `json:"id"`
	Data json.RawMessage `json:"data"`
}
type SourceObject struct {
	TargetObject
	Fields map[string][]string `json:"fields"`
}

type FieldType byte
const (
	FT_TEXT FieldType =11  //文本类型
	FT_NUMBER FieldType =9 //数字
	FT_ENUM   FieldType =8 //枚举
	FT_ID FieldType =7  //id唯一标识
	FT_DATE FieldType =6 //日期
)
//索引的元数据信息
type IndexMeta struct {
	Name string  //索引名称
	Fields []FieldMeta //字段
}

type FieldMeta struct {
	Name string //字段名称
	Type FieldType //类型

}

type SearchEngine struct {
	indexs   map[string]*searchIndex
	safeIndexs map[string]*searchIndex  //安全flush时候的临时索引
	client   *redis.Client
	logger   utils.Log
	pageSize int64
	pageLife int64 //分钟
}

func (this *SearchEngine) Init(context *bingo.ApplicationContext) {
	fmt.Println("init .....")
	db, err := strconv.Atoi(context.GetPropertyFromConfig("service.search.db"))
	if err != nil {
		db = 0
	}

	size, err := strconv.Atoi(context.GetPropertyFromConfig("service.search.pagesize"))
	if err != nil {
		size = 20 //默认大小20条
	}
	this.pageSize = int64(size)

	life, err := strconv.Atoi(context.GetPropertyFromConfig("service.search.pagelife"))
	if err != nil {
		life = 10 //默认时间10分钟
	}
	this.pageLife = int64(life)

	this.client = redis.NewClient(&redis.Options{
		Addr:     context.GetPropertyFromConfig("service.search.redis"),
		Password: "", // no password set
		DB:       db,
	})
	fmt.Println(context.GetPropertyFromConfig("service.search.redis"))
	this.indexs = make(map[string]*searchIndex)
	this.safeIndexs = make(map[string]*searchIndex)
	this.logger = context.GetLog("bingo_search")
}

//创建索引
func (this *SearchEngine) CreateIndex(name string) *searchIndex {
	if name != "" {
		index := this.indexs[name]
		if index == nil {
			index = &searchIndex{name, this}
			this.indexs[name] = index
		}
		return index
	}

	return nil
}

//清除索引，将整个索引的数据摧毁
func (this *SearchEngine)FlushIndex(name string) {
	//1、删除存放的数据
	this.client.Del(name)

	//2、删除所有的索引key
	keys,err:=this.client.Keys(name+"_*").Result()
	if err!=nil {
		this.logger.Debug("get index keys error:%s",err.Error())
	}

	if keys!=nil && len(keys)>0 {
		this.client.Del(keys...)
	}
}

//开始安全flush
func (this *SearchEngine)BeginSafeFlush(name string) {
	index := this.indexs[name]
	if index!=nil {
		var nindexName string
		if strings.HasSuffix(index.name,"#") {
			nindexName=index.name[0:len(index.name)-2]
		}else {
			nindexName=index.name+"#"
		}

		nindex:=&searchIndex{nindexName, this}
		this.safeIndexs[name]=nindex
	}
}

//结束安全flush，自动替换掉查询用的索引
func (this *SearchEngine)EndSafeFlush(name string) {
	index:=this.safeIndexs[name]
	if index!=nil {
		delete(this.safeIndexs,name)
		oldindex:=this.indexs[name]
		this.indexs[name]=index
        //删除旧的索引数据
		if oldindex!=nil {
			this.FlushIndex(oldindex.name)
		}
	}
}

//加载和刷新数据
func (this *SearchEngine) LoadSource(name string, obj *SourceObject) {
	if name==""||obj==nil {
		return
	}

	//判断是否正在进行安全flush，如果是安全flush，所有的更改只发生在正在替换的索引上
	index:=this.safeIndexs[name]
	if index==nil {
		index = this.CreateIndex(name)
	}

	if index != nil {
		index.LoadObject(obj)
	}

}

//删除数据
func (this *SearchEngine)RemoveSource(name string, obj *SourceObject){
   if name!=""&&obj!=nil{
	   index := this.CreateIndex(name)
	   if index != nil {
		   index.RemoveObject(obj)
	   }
   }
}

//删除索引中的某个词条
func (this *SearchEngine)RemoveKeyword(name,field,word string)  {
	if name!="" {
		index := this.CreateIndex(name)
		if index != nil {
			index.RemoveKeyword(field,word)
		}
	}

}

//按页获取数据
func (this *SearchEngine) FetchByPage(request string, page int64) *PageSearchResult {
	if request != "" {
		//获取request的name
		index := strings.Index(request, ":")
		if index < 0 {
			this.logger.Error("pagerequest's index name not found !")
			return nil //找不到对应的索引类型
		}

		name := request[0:index]
		return this.fetchByPage(name,request,page)
	}

	return nil
}

func (this *SearchEngine)fetchByPage(name string,request string,page int64)*PageSearchResult {
	if page <= 0 {
		page = 1
	}
	startIndex := (page - 1) * this.pageSize //+ 1 //从1开始计数
	endIndex := page*this.pageSize - 1
	this.logger.Debug(request,startIndex,endIndex)
	keys, err := this.client.LRange(request, startIndex, endIndex).Result()

	if err != nil || len(keys) == 0 {
		if err!=nil {
			this.logger.Debug(err.Error())
		}
		this.logger.Debug("no content by page!")
		return nil
	}
	go this.client.Expire(request, time.Duration(this.pageLife)*time.Minute) //更新重置失效时间
	return &PageSearchResult{request, page, this.fetch(name, keys...)}
}

//获取内容
func (this *SearchEngine) fetch(name string, keys ...string) []TargetObject {
	datas, err1 := this.client.HMGet(name, keys...).Result()
	if err1 == nil && len(datas) > 0 {

		var targets []TargetObject

		for _, v := range datas {
			if v != nil {
				t := TargetObject{}
				json.Unmarshal([]byte(fmt.Sprintf("%v", v)), &t)
				targets = append(targets, t)
			}
		}

		return targets

	} else {
		this.logger.Error("get data by index error!%s", err1.Error())
	}

	return nil
}

func (this *SearchEngine) Search(name string, input ...Field) *PageSearchResult {
	if name != "" {
		index := this.indexs[name]
		if index != nil {
			return index.Search(input...)

		}
		this.logger.Info("not found index %s", name)
	}

	return nil
}

type searchIndex struct {
	name   string
	engine *SearchEngine
}

//搜索信息
func (this *searchIndex) Search(input ...Field) *PageSearchResult {
	//生成索引搜索请求号
	requestkey := getSearchRequestUuid(this.name)

	//搜索索引
	var searchkeys []string
	var tmpkeys []string
	var notTmpkeys[]string
	for _, f := range input {

		//处理not的情况
		if strings.HasPrefix(f.Key,"!") {//字段使用！表示"非"的意思，就是不等于
		    k:=f.Key[1:len(f.Key)]
			v := f.Value
			arrays := strings.Split(v, "|")
			if len(arrays) > 1 {
				for _, subvalue := range arrays {
					notTmpkeys = append(notTmpkeys, this.buildTheKeyByItem(k, subvalue))
				}
			}else {
				notTmpkeys=append(notTmpkeys,this.buildTheKeyByItem(k, v))
			}

			continue
		}

		//处理并集
		v := f.Value
		arrays := strings.Split(v, "|")  //使用 | 分割多个取值
		if len(arrays) > 1 {
			skey := requestkey + ":" + f.Key
			var subkeys []string
			for _, subkey := range arrays {
				subkeys = append(subkeys, this.buildTheKeyByItem(f.Key, subkey))
			}

			//取并集，将结果存入临时的key中，用于后续取交集用
			this.engine.client.SUnionStore(skey, subkeys...)
			tmpkeys = append(tmpkeys, skey) //放入临时组中用于使用后删除
			searchkeys = append(searchkeys, skey)
		} else {
			searchkeys = append(searchkeys, this.buildTheKey(f))
		}

	}
	defer this.deleteTempkeys(tmpkeys) //删除临时创建的key

	var values []string
	//有差集
	if notTmpkeys!=nil && len(notTmpkeys) >1 {
		interTmpkey:=requestkey+"_tmp"
		//取交集
		result := this.engine.client.SInterStore(interTmpkey,searchkeys...)
		_, err := result.Result()
		if err != nil {
			this.engine.logger.Error("inter key error!%s", err.Error())
			return  nil
		}
		//取差集合，存储
		diffkeys:=make([]string,len(notTmpkeys)+1)
		diffkeys[0]=interTmpkey
		copy(diffkeys[1:len(diffkeys)],notTmpkeys)

		defer this.deleteTempkeys([]string{interTmpkey})//删除临时key
		data:=this.engine.client.SDiff(diffkeys...)
		values, _ = data.Result()

	} else {//如果无差集合，则直接返回交集
		result:=this.engine.client.SInter(searchkeys...)
		values, _ = result.Result()
	}

	//存储数据
	var datas []interface{}

	for _, v := range values {
		datas = append(datas, v)
	}
	this.engine.client.RPush(requestkey,datas...)

	this.engine.client.Expire(requestkey, time.Duration(this.engine.pageLife)*time.Minute) //指定x分钟后失效
	//取出第一页的数据返回
	return this.engine.fetchByPage(this.name,requestkey,1)

}

//删除某个字段等于某值得列表，也就是删除一个词条
func (this *searchIndex)RemoveKeyword(field,word string){
	if field!=""&&word!="" {
		thekey:=this.buildTheKeyByItem(field,word)
		this.engine.client.Del(thekey)
	}

}

//删除临时创建的key
func (this *searchIndex) deleteTempkeys(keys []string) {
	if keys != nil && len(keys) > 0 {
		this.engine.client.Del(keys...)
	}
}

//刷新索引，加载信息到存储中
func (this *searchIndex) LoadObject(obj *SourceObject) {
	data, _ := json.Marshal(obj)
	key:=obj.Id
	//如果没有指明对象id，则使用md5作为数据的唯一标识
	if key=="" {
		key= getMd5str(string(data))
	}

	//1、放入数据到目标集合中
	this.engine.client.HSet(this.name, key, string(data))

	//2、根据field存储到各个对应的索引中
	for k, v := range obj.Fields {
		for _,vkey:=range v{
			this.engine.client.SAdd(this.buildTheKeyByItem(k, vkey), key)
		}

	}

}

//删除数据：删除数据及索引存的值
func (this *searchIndex)RemoveObject(obj *SourceObject) {
	data, _ := json.Marshal(obj)
	key:=obj.Id
	if key=="" {
		key=getMd5str(string(data))
	}
	//删除数据
	this.engine.client.HDel(this.name,key)
	//删除索引里的记录
	for k,v:=range obj.Fields {
		for _,vkey:=range v {
			this.engine.client.SRem(this.buildTheKeyByItem(k, vkey), key)
		}
	}

}

func (this *searchIndex) buildTheKey(f Field) string {
	return this.buildTheKeyByItem(f.Key, f.Value)
}

func (this *searchIndex) buildTheKeyByItem(key, value string) string {
	return fmt.Sprintf("%s_%s_%s", this.name, key, value)
}

func getMd5str(value string) string {
	data := []byte(value)
	has := md5.Sum(data)
	md5str1 := fmt.Sprintf("%x", has) //将[]byte转成16进制

	return md5str1

}

func getSearchRequestUuid(prefix string) string {
	return fmt.Sprintf("%s:%d", prefix, time.Now().UnixNano())
}
<|endoftext|>


<commit_before>package jwp

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
)

func (c *Client) GetCurrentWindowSize() (size *WindowSize, err error) {
	var (
		req  *http.Request
		res  *http.Response
		body []byte
		resp ApiWindowSize
	)

	req, err = http.NewRequest("GET", c.Server+"/session/"+c.SessionID+"/window/current/size", nil)
	if err != nil {
		return
	}

	res, err = http.DefaultClient.Do(req)
	if err != nil {
		return
	}

	defer res.Body.Close()
	body, err = ioutil.ReadAll(res.Body)
	if err != nil {
		return
	}

	if err = CheckHTTPStatus(res, body); err != nil {
		return
	}

	json.Unmarshal(body, &resp)
	if err = CheckStatus(resp.Status); err != nil {
		return
	}

	size = &resp.Value

	return
}
<commit_msg>Add method to set current window size.<commit_after>package jwp

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"net/http"
)

func (c *Client) GetCurrentWindowSize() (size *WindowSize, err error) {
	var (
		req  *http.Request
		res  *http.Response
		body []byte
		resp ApiWindowSize
	)

	req, err = http.NewRequest("GET", c.Server+"/session/"+c.SessionID+"/window/current/size", nil)
	if err != nil {
		return
	}

	res, err = http.DefaultClient.Do(req)
	if err != nil {
		return
	}

	defer res.Body.Close()
	body, err = ioutil.ReadAll(res.Body)
	if err != nil {
		return
	}

	if err = CheckHTTPStatus(res, body); err != nil {
		return
	}

	json.Unmarshal(body, &resp)
	if err = CheckStatus(resp.Status); err != nil {
		return
	}

	size = &resp.Value

	return
}

func (c *Client) SetCurrentWindowSize(size *WindowSize) (err error) {
	var (
		req  *http.Request
		res  *http.Response
		body []byte
		buf  = new(bytes.Buffer)
		resp ApiWindowSize
	)

	if size.Height < 100 {
		size.Height = 100
	}
	if size.Width < 100 {
		size.Width = 100
	}

	err = json.NewEncoder(buf).Encode(*size)
	if err != nil {
		return
	}

	req, err = http.NewRequest("POST", c.Server+"/session/"+c.SessionID+"/window/current/size", buf)
	if err != nil {
		return
	}

	res, err = http.DefaultClient.Do(req)
	if err != nil {
		return
	}

	defer res.Body.Close()
	body, err = ioutil.ReadAll(res.Body)
	if err != nil {
		return
	}

	if err = CheckHTTPStatus(res, body); err != nil {
		return
	}

	json.Unmarshal(body, &resp)
	if err = CheckStatus(resp.Status); err != nil {
		return
	}

	return
}
<|endoftext|>


<commit_before>package graph

import (
	"alg"
	"geo"
	"mm"
	"path"
)

const Sentinel uint32 = 0xffffffff

type GraphFile struct {
	// vertex -> first out/in edge
	FirstOut      []uint32
	FirstIn       []uint32
	// positions (at index 2 * i, 2 * i + 1)
	Coordinates   []int32

	// Accessibility bit vectors
	Access        [TransportMax][]byte
	AccessEdge    [TransportMax][]byte
	Oneway        []byte // should be distinguished by transport type

	// edge -> next edge (or to the same edge if this is the last in edge)
	NextIn        []uint32
	// for edge {u,v}, this array contains u^v
	Edges         []uint32

	// edge weights
	Weights       [MetricMax][]uint16

	// edge -> first step
	Steps         []uint32
	StepPositions []byte
}

// I/O

func OpenGraphFile(base string, ignoreErrors bool) (*GraphFile, error) {
	g := &GraphFile{}
	files := []struct{name string; p interface{}} {
		{"vertices.ftf",       &g.FirstOut},
		{"vertices-in.ftf",    &g.FirstIn},
		{"positions.ftf",      &g.Coordinates},
		{"vaccess-car.ftf",    &g.Access[Car]},
		{"vaccess-bike.ftf",   &g.Access[Bike]},
		{"vaccess-foot.ftf",   &g.Access[Foot]},
		{"access-car.ftf",     &g.AccessEdge[Car]},
		{"access-bike.ftf",    &g.AccessEdge[Bike]},
		{"access-foot.ftf",    &g.AccessEdge[Foot]},
		{"oneway.ftf",         &g.Oneway},
		{"edges-next.ftf",     &g.NextIn},
		{"edges.ftf",          &g.Edges},
		{"distances.ftf",      &g.Weights[Distance]},
		{"steps.ftf",          &g.Steps},
		{"step_positions.ftf", &g.StepPositions},
	}

	for _, file := range files {
		err := mm.Open(path.Join(base, file.name), file.p)
		if err != nil && !ignoreErrors {
			return nil, err
		}
	}
	return g, nil
}

func CloseGraphFile(g *GraphFile) error {
	files := []interface{} {
		&g.FirstOut, &g.FirstIn, &g.Coordinates,
		&g.Access[Car], &g.Access[Bike], &g.Access[Foot],
		&g.AccessEdge[Car], &g.AccessEdge[Bike], &g.AccessEdge[Foot],
		&g.Oneway, &g.NextIn, &g.Edges, &g.Weights[Distance],
		&g.Steps, &g.StepPositions,
	}
	for _, p := range files {
		err := mm.Close(p)
		if err != nil {
			return err
		}
	}
	return nil
}

// Graph Interface

func (g *GraphFile) VertexCount() int {
	return len(g.FirstIn)
}

func (g *GraphFile) EdgeCount() int {
	return len(g.Edges)
}

func (g *GraphFile) VertexAccessible(v Vertex, t Transport) bool {
	return alg.GetBit(g.Access[t], uint(v))
}

func (g *GraphFile) VertexCoordinate(v Vertex) geo.Coordinate {
	lat := g.Coordinates[2 * int(v)]
	lng := g.Coordinates[2 * int(v) + 1]
	return geo.DecodeCoordinate(lat, lng)
}

func (g *GraphFile) VertexEdges(v Vertex, forward bool, t Transport, buf []Edge) []Edge {
	// This is rather nice: buf[:0] sets the length to 0 but does not change the capacity.
	// In effect calls to append will not allocate a new array if the capacity is already
	// sufficient. This is much faster than using an iterator, since every interface call
	// is indirect.
	result := buf[:0]

	// So, at this point you are probably thinking:
	// "What The Fuck? Are you implementing common compiler optimizations by hand?"
	// To which I am forced to answer that yes, I am. I wish this were a joke, but this
	// saved 8 seconds (~25%) of the running time in the scc finding program.
	// TODO: Implement loop unswitching, cse and some algebraic identities in the go compiler.

	// Add the out edges for v
	first := g.FirstOut[v]
	last  := g.FirstOut[v+1]
	access := g.AccessEdge[t]
	if forward || t == Foot {
		// No need to consider the oneway flags
		for i := first; i < last; i++ {
			index := i >> 3
			bit := byte(1 << (i & 7))
			if access[index] & bit == 0 {
				continue
			}
			result = append(result, Edge(i))
		}
	} else {
		// Consider the oneway flags...
		oneway := g.Oneway
		for i := first; i < last; i++ {
			index := i >> 3
			bit := byte(1 << (i & 7))
			if access[index] & bit == 0 || oneway[index] & bit != 0 {
				continue
			}
			result = append(result, Edge(i))
		}
	}

	// The in edges are stored as a linked list.
	i := g.FirstIn[v]
	if i == Sentinel {
		return result
	}

	if !forward || t == Foot {
		// As above, no need to consider the oneway flags
		for {
			index := i >> 3
			bit := byte(1 << (i & 7))
			if access[index] & bit != 0 {
				result = append(result, Edge(i))
			}
			if i == g.NextIn[i] {
				break
			}
			i = g.NextIn[i]
		}
	} else {
		// Need to consider the oneway flags.
		oneway := g.Oneway
		for {
			index := i >> 3
			bit := byte(1 << (i & 7))
			if access[index] & bit != 0 && oneway[index] & bit == 0 {
				result = append(result, Edge(i))
			}
			if i == g.NextIn[i] {
				break
			}
			i = g.NextIn[i]
		}
	}

	return result
}

func (g *GraphFile) EdgeOpposite(e Edge, from Vertex) Vertex {
	return Vertex(g.Edges[e]) ^ from
}

func (g *GraphFile) EdgeSteps(e Edge, from Vertex) []geo.Coordinate {
	// In order to decode the step positions we need the starting vertex.
	// Additionally, if this vertex is not "from", we will need to reverse
	// the steps positions before returning.
	firstEdge := g.FirstOut[from]
	lastEdge  := g.FirstOut[from+1]
	forward   := firstEdge <= uint32(e) && uint32(e) < lastEdge
	start     := from
	if !forward {
		start = g.EdgeOpposite(e, from)
	}

	firstStep := g.Steps[e]
	lastStep  := g.Steps[e+1]
	initial   := g.VertexCoordinate(start)
	step      := geo.DecodeStep(initial, g.StepPositions[firstStep:lastStep])

	if !forward {
		for i, j := 0, len(step)-1; i < j; i, j = i+1, j-1 {
			step[i], step[j] = step[j], step[i]
		}
	}

	return step
}

func (g *GraphFile) EdgeWeight(e Edge, t Transport, m Metric) float64 {
	return alg.HalfToFloat64(g.Weights[m][e])
}

// Raw Interface (used to implement other tools working with GraphFiles)

func (g *GraphFile) VertexRawEdges(v Vertex, buf []Edge) []Edge {
	result := buf[:0]

	for i := g.FirstOut[v]; i < g.FirstOut[v+1]; i++ {
		result = append(result, Edge(i))
	}

	i := g.FirstIn[v]
	if i == Sentinel {
		return result
	}

	for {
		result = append(result, Edge(i))
		if i == g.NextIn[i] {
			break
		}
		i = g.NextIn[i]
	}

	return result
}
<commit_msg>Limit the number of open files and add (*GraphFile).EdgeAccessible<commit_after>package graph

import (
	"alg"
	"geo"
	"mm"
	"path"
)

const Sentinel uint32 = 0xffffffff

type GraphFile struct {
	// vertex -> first out/in edge
	FirstOut      []uint32
	FirstIn       []uint32
	// positions (at index 2 * i, 2 * i + 1)
	Coordinates   []int32

	// Accessibility bit vectors
	Access        [TransportMax][]byte
	AccessEdge    [TransportMax][]byte
	Oneway        []byte // should be distinguished by transport type

	// edge -> next edge (or to the same edge if this is the last in edge)
	NextIn        []uint32
	// for edge {u,v}, this array contains u^v
	Edges         []uint32

	// edge weights
	Weights       [MetricMax][]uint16

	// edge -> first step
	Steps         []uint32
	StepPositions []byte
}

// I/O

func OpenGraphFile(base string, ignoreErrors bool) (*GraphFile, error) {
	g := &GraphFile{}
	files := []struct{name string; p interface{}} {
		{"vertices.ftf",       &g.FirstOut},
		{"vertices-in.ftf",    &g.FirstIn},
		{"positions.ftf",      &g.Coordinates},
		{"vaccess-car.ftf",    &g.Access[Car]},
		{"vaccess-bike.ftf",   &g.Access[Bike]},
		{"vaccess-foot.ftf",   &g.Access[Foot]},
		{"access-car.ftf",     &g.AccessEdge[Car]},
		{"access-bike.ftf",    &g.AccessEdge[Bike]},
		{"access-foot.ftf",    &g.AccessEdge[Foot]},
		{"oneway.ftf",         &g.Oneway},
		{"edges-next.ftf",     &g.NextIn},
		{"edges.ftf",          &g.Edges},
		{"distances.ftf",      &g.Weights[Distance]},
		{"steps.ftf",          &g.Steps},
		{"step_positions.ftf", &g.StepPositions},
	}

	for _, file := range files {
		err := mm.Open(path.Join(base, file.name), file.p)
		if err != nil && !ignoreErrors {
			return nil, err
		}
	}

	// Ugly hack: we can't have too many open files...
	bitvectors := []*[]byte {
		&g.Access[Car], &g.Access[Bike], &g.Access[Foot],
		&g.AccessEdge[Car], &g.AccessEdge[Bike], &g.AccessEdge[Foot],
		&g.Oneway,
	}
	for _, bv := range bitvectors {
		p := *bv
		*bv = make([]byte, len(p))
		copy(*bv, p)
		err := mm.Close(&p)
		if err != nil {
			return nil, err
		}
	}

	return g, nil
}

func CloseGraphFile(g *GraphFile) error {
	files := []interface{} {
		&g.FirstOut, &g.FirstIn, &g.Coordinates,
		//&g.Access[Car], &g.Access[Bike], &g.Access[Foot],
		//&g.AccessEdge[Car], &g.AccessEdge[Bike], &g.AccessEdge[Foot],
		//&g.Oneway,
		&g.NextIn, &g.Edges, &g.Weights[Distance],
		&g.Steps, &g.StepPositions,
	}
	for _, p := range files {
		err := mm.Close(p)
		if err != nil {
			return err
		}
	}
	return nil
}

// Graph Interface

func (g *GraphFile) VertexCount() int {
	return len(g.FirstIn)
}

func (g *GraphFile) EdgeCount() int {
	return len(g.Edges)
}

func (g *GraphFile) VertexAccessible(v Vertex, t Transport) bool {
	return alg.GetBit(g.Access[t], uint(v))
}

func (g *GraphFile) VertexCoordinate(v Vertex) geo.Coordinate {
	lat := g.Coordinates[2 * int(v)]
	lng := g.Coordinates[2 * int(v) + 1]
	return geo.DecodeCoordinate(lat, lng)
}

func (g *GraphFile) VertexEdges(v Vertex, forward bool, t Transport, buf []Edge) []Edge {
	// This is rather nice: buf[:0] sets the length to 0 but does not change the capacity.
	// In effect calls to append will not allocate a new array if the capacity is already
	// sufficient. This is much faster than using an iterator, since every interface call
	// is indirect.
	result := buf[:0]

	// So, at this point you are probably thinking:
	// "What The Fuck? Are you implementing common compiler optimizations by hand?"
	// To which I am forced to answer that yes, I am. I wish this were a joke, but this
	// saved 8 seconds (~25%) of the running time in the scc finding program.
	// TODO: Implement loop unswitching, cse and some algebraic identities in the go compiler.

	// Add the out edges for v
	first := g.FirstOut[v]
	last  := g.FirstOut[v+1]
	access := g.AccessEdge[t]
	if forward || t == Foot {
		// No need to consider the oneway flags
		for i := first; i < last; i++ {
			index := i >> 3
			bit := byte(1 << (i & 7))
			if access[index] & bit == 0 {
				continue
			}
			result = append(result, Edge(i))
		}
	} else {
		// Consider the oneway flags...
		oneway := g.Oneway
		for i := first; i < last; i++ {
			index := i >> 3
			bit := byte(1 << (i & 7))
			if access[index] & bit == 0 || oneway[index] & bit != 0 {
				continue
			}
			result = append(result, Edge(i))
		}
	}

	// The in edges are stored as a linked list.
	i := g.FirstIn[v]
	if i == Sentinel {
		return result
	}

	if !forward || t == Foot {
		// As above, no need to consider the oneway flags
		for {
			index := i >> 3
			bit := byte(1 << (i & 7))
			if access[index] & bit != 0 {
				result = append(result, Edge(i))
			}
			if i == g.NextIn[i] {
				break
			}
			i = g.NextIn[i]
		}
	} else {
		// Need to consider the oneway flags.
		oneway := g.Oneway
		for {
			index := i >> 3
			bit := byte(1 << (i & 7))
			if access[index] & bit != 0 && oneway[index] & bit == 0 {
				result = append(result, Edge(i))
			}
			if i == g.NextIn[i] {
				break
			}
			i = g.NextIn[i]
		}
	}

	return result
}

func (g *GraphFile) EdgeOpposite(e Edge, from Vertex) Vertex {
	return Vertex(g.Edges[e]) ^ from
}

func (g *GraphFile) EdgeSteps(e Edge, from Vertex) []geo.Coordinate {
	// In order to decode the step positions we need the starting vertex.
	// Additionally, if this vertex is not "from", we will need to reverse
	// the steps positions before returning.
	firstEdge := g.FirstOut[from]
	lastEdge  := g.FirstOut[from+1]
	forward   := firstEdge <= uint32(e) && uint32(e) < lastEdge
	start     := from
	if !forward {
		start = g.EdgeOpposite(e, from)
	}

	firstStep := g.Steps[e]
	lastStep  := g.Steps[e+1]
	initial   := g.VertexCoordinate(start)
	step      := geo.DecodeStep(initial, g.StepPositions[firstStep:lastStep])

	if !forward {
		for i, j := 0, len(step)-1; i < j; i, j = i+1, j-1 {
			step[i], step[j] = step[j], step[i]
		}
	}

	return step
}

func (g *GraphFile) EdgeWeight(e Edge, t Transport, m Metric) float64 {
	return alg.HalfToFloat64(g.Weights[m][e])
}

// Raw Interface (used to implement other tools working with GraphFiles)

func (g *GraphFile) EdgeAccessible(v Edge, t Transport) bool {
	return alg.GetBit(g.AccessEdge[t], uint(v))
}

func (g *GraphFile) VertexRawEdges(v Vertex, buf []Edge) []Edge {
	result := buf[:0]

	for i := g.FirstOut[v]; i < g.FirstOut[v+1]; i++ {
		result = append(result, Edge(i))
	}

	i := g.FirstIn[v]
	if i == Sentinel {
		return result
	}

	for {
		result = append(result, Edge(i))
		if i == g.NextIn[i] {
			break
		}
		i = g.NextIn[i]
	}

	return result
}
<|endoftext|>


<commit_before>// Copyright 2018 The Nakama Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import (
	"context"
	"crypto"
	"database/sql"
	"encoding/base64"
	"fmt"
	"net"
	"net/http"
	"strings"
	"time"

	"github.com/dgrijalva/jwt-go"

	"github.com/gorilla/handlers"
	"github.com/gorilla/mux"
	"github.com/grpc-ecosystem/grpc-gateway/runtime"
	"github.com/heroiclabs/nakama/v2/console"
	"go.opencensus.io/plugin/ocgrpc"
	"go.opencensus.io/zpages"
	"go.uber.org/zap"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/metadata"
	"google.golang.org/grpc/status"
)

type ConsoleServer struct {
	logger            *zap.Logger
	db                *sql.DB
	config            Config
	tracker           Tracker
	router            MessageRouter
	statusHandler     StatusHandler
	configWarnings    map[string]string
	serverVersion     string
	grpcServer        *grpc.Server
	grpcGatewayServer *http.Server
}

func StartConsoleServer(logger *zap.Logger, startupLogger *zap.Logger, db *sql.DB, config Config, tracker Tracker, router MessageRouter, statusHandler StatusHandler, configWarnings map[string]string, serverVersion string) *ConsoleServer {
	var gatewayContextTimeoutMs string
	if config.GetConsole().IdleTimeoutMs > 500 {
		// Ensure the GRPC Gateway timeout is just under the idle timeout (if possible) to ensure it has priority.
		gatewayContextTimeoutMs = fmt.Sprintf("%vm", config.GetConsole().IdleTimeoutMs-500)
	} else {
		gatewayContextTimeoutMs = fmt.Sprintf("%vm", config.GetConsole().IdleTimeoutMs)
	}

	serverOpts := []grpc.ServerOption{
		grpc.StatsHandler(&ocgrpc.ServerHandler{IsPublicEndpoint: true}),
		grpc.MaxRecvMsgSize(int(config.GetConsole().MaxMessageSizeBytes)),
		grpc.UnaryInterceptor(consoleInterceptorFunc(logger, config)),
	}
	grpcServer := grpc.NewServer(serverOpts...)

	s := &ConsoleServer{
		logger:         logger,
		db:             db,
		config:         config,
		tracker:        tracker,
		router:         router,
		statusHandler:  statusHandler,
		configWarnings: configWarnings,
		serverVersion:  serverVersion,
		grpcServer:     grpcServer,
	}

	console.RegisterConsoleServer(grpcServer, s)
	startupLogger.Info("Starting Console server for gRPC requests", zap.Int("port", config.GetConsole().Port-3))
	go func() {
		listener, err := net.Listen("tcp", fmt.Sprintf("%v:%d", config.GetConsole().Address, config.GetConsole().Port-3))
		if err != nil {
			startupLogger.Fatal("Console server listener failed to start", zap.Error(err))
		}

		if err := grpcServer.Serve(listener); err != nil {
			startupLogger.Fatal("Console server listener failed", zap.Error(err))
		}
	}()

	ctx := context.Background()
	grpcGateway := runtime.NewServeMux()
	if err := console.RegisterConsoleHandlerServer(ctx, grpcGateway, s); err != nil {
		startupLogger.Fatal("Console server gateway registration failed", zap.Error(err))
	}

	grpcGatewayRouter := mux.NewRouter()

	grpcGatewayRouter.Handle("/", console.UI).Methods("GET")
	grpcGatewayRouter.Handle("/manifest.json", console.UI).Methods("GET")
	grpcGatewayRouter.Handle("/favicon.ico", console.UI).Methods("GET")
	grpcGatewayRouter.PathPrefix("/static/").Handler(console.UI).Methods("GET")

	zpagesMux := http.NewServeMux()
	zpages.Handle(zpagesMux, "/metrics/")
	grpcGatewayRouter.NewRoute().PathPrefix("/metrics").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		zpagesMux.ServeHTTP(w, r)
	})

	grpcGatewayRouter.HandleFunc("/v2/console/storage/import", s.importStorage)

	// Enable max size check on requests coming arriving the gateway.
	// Enable compression on responses sent by the gateway.
	handlerWithCompressResponse := handlers.CompressHandler(grpcGateway)
	maxMessageSizeBytes := config.GetConsole().MaxMessageSizeBytes
	handlerWithMaxBody := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		// Check max body size before decompressing incoming request body.
		r.Body = http.MaxBytesReader(w, r.Body, maxMessageSizeBytes)
		handlerWithCompressResponse.ServeHTTP(w, r)
	})
	grpcGatewayRouter.NewRoute().HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		// Ensure some headers have required values.
		// Override any value set by the client if needed.
		r.Header.Set("Grpc-Timeout", gatewayContextTimeoutMs)

		// Allow GRPC Gateway to handle the request.
		handlerWithMaxBody.ServeHTTP(w, r)
	})

	// Enable CORS on all requests.
	CORSHeaders := handlers.AllowedHeaders([]string{"Authorization", "Content-Type", "User-Agent"})
	CORSOrigins := handlers.AllowedOrigins([]string{"*"})
	CORSMethods := handlers.AllowedMethods([]string{"GET", "HEAD", "POST", "PUT", "DELETE", "PATCH"})
	handlerWithCORS := handlers.CORS(CORSHeaders, CORSOrigins, CORSMethods)(grpcGatewayRouter)

	// Set up and start GRPC Gateway server.
	s.grpcGatewayServer = &http.Server{
		Addr:         fmt.Sprintf("%v:%d", config.GetConsole().Address, config.GetConsole().Port),
		ReadTimeout:  time.Millisecond * time.Duration(int64(config.GetConsole().ReadTimeoutMs)),
		WriteTimeout: time.Millisecond * time.Duration(int64(config.GetConsole().WriteTimeoutMs)),
		IdleTimeout:  time.Millisecond * time.Duration(int64(config.GetConsole().IdleTimeoutMs)),
		Handler:      handlerWithCORS,
	}

	startupLogger.Info("Starting Console server gateway for HTTP requests", zap.Int("port", config.GetConsole().Port))
	go func() {
		if err := s.grpcGatewayServer.ListenAndServe(); err != nil && err != http.ErrServerClosed {
			startupLogger.Fatal("Console server gateway listener failed", zap.Error(err))
		}
	}()

	return s
}

func (s *ConsoleServer) Stop() {
	// 1. Stop GRPC Gateway server first as it sits above GRPC server.
	if err := s.grpcGatewayServer.Shutdown(context.Background()); err != nil {
		s.logger.Error("API server gateway listener shutdown failed", zap.Error(err))
	}
	// 2. Stop GRPC server.
	s.grpcServer.GracefulStop()
}

func consoleInterceptorFunc(logger *zap.Logger, config Config) func(context.Context, interface{}, *grpc.UnaryServerInfo, grpc.UnaryHandler) (interface{}, error) {
	return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {

		switch info.FullMethod {
		// skip authentication check for Login endpoint
		case "/nakama.console.Console/Authenticate":
			return handler(ctx, req)
		}

		md, ok := metadata.FromIncomingContext(ctx)
		if !ok {
			logger.Error("Cannot extract metadata from incoming context")
			return nil, status.Error(codes.FailedPrecondition, "Cannot extract metadata from incoming context")
		}
		auth, ok := md["authorization"]
		if !ok {
			auth, ok = md["grpcgateway-authorization"]
		}
		if !ok {
			return nil, status.Error(codes.Unauthenticated, "Console authentication required.")
		}
		if len(auth) != 1 {
			return nil, status.Error(codes.Unauthenticated, "Console authentication required.")
		}

		if !checkAuth(config, auth[0]) {
			return nil, status.Error(codes.Unauthenticated, "Console authentication invalid.")
		}

		return handler(ctx, req)
	}
}

func checkAuth(config Config, auth string) bool {
	const basicPrefix = "Basic "
	const bearerPrefix = "Bearer "

	if strings.HasPrefix(auth, basicPrefix) {
		// Basic authentication.
		c, err := base64.StdEncoding.DecodeString(auth[len(basicPrefix):])
		if err != nil {
			// Not valid Base64.
			return false
		}
		cs := string(c)
		s := strings.IndexByte(cs, ':')
		if s < 0 {
			// Format is not "username:password".
			return false
		}

		if cs[:s] != config.GetConsole().Username || cs[s+1:] != config.GetConsole().Password {
			// Username and/or password do not match.
			return false
		}

		// Basic authentication successful.
		return true
	} else if strings.HasPrefix(auth, bearerPrefix) {
		// Bearer token authentication.
		token, err := jwt.Parse(auth[len(bearerPrefix):], func(token *jwt.Token) (interface{}, error) {
			if s, ok := token.Method.(*jwt.SigningMethodHMAC); !ok || s.Hash != crypto.SHA256 {
				return nil, fmt.Errorf("unexpected signing method: %v", token.Header["alg"])
			}
			return []byte(config.GetConsole().SigningKey), nil
		})
		if err != nil {
			// Token verification failed.
			return false
		}
		claims, ok := token.Claims.(jwt.MapClaims)
		if !ok || !token.Valid {
			// The token or its claims are invalid.
			return false
		}

		exp, ok := claims["exp"].(float64)
		if !ok {
			// Expiry time claim is invalid.
			return false
		}
		if int64(exp) <= time.Now().UTC().Unix() {
			// Token expired.
			return false
		}

		// Bearer token authentication successful.
		return true
	}

	return false
}
<commit_msg>Refactor GRPC Gateway wiring for devconsole.<commit_after>// Copyright 2018 The Nakama Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import (
	"context"
	"crypto"
	"database/sql"
	"encoding/base64"
	"fmt"
	"net"
	"net/http"
	"strings"
	"time"

	"github.com/dgrijalva/jwt-go"

	"github.com/gorilla/handlers"
	"github.com/gorilla/mux"
	"github.com/grpc-ecosystem/grpc-gateway/runtime"
	"github.com/heroiclabs/nakama/v2/console"
	"go.opencensus.io/plugin/ocgrpc"
	"go.opencensus.io/zpages"
	"go.uber.org/zap"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/metadata"
	"google.golang.org/grpc/status"
)

var (
	consoleAuthRequired = []byte(`{"error":"Console authentication required.","message":"Console authentication required.","code":16}`)
)

type ConsoleServer struct {
	logger            *zap.Logger
	db                *sql.DB
	config            Config
	tracker           Tracker
	router            MessageRouter
	statusHandler     StatusHandler
	configWarnings    map[string]string
	serverVersion     string
	grpcServer        *grpc.Server
	grpcGatewayServer *http.Server
}

func StartConsoleServer(logger *zap.Logger, startupLogger *zap.Logger, db *sql.DB, config Config, tracker Tracker, router MessageRouter, statusHandler StatusHandler, configWarnings map[string]string, serverVersion string) *ConsoleServer {
	var gatewayContextTimeoutMs string
	if config.GetConsole().IdleTimeoutMs > 500 {
		// Ensure the GRPC Gateway timeout is just under the idle timeout (if possible) to ensure it has priority.
		gatewayContextTimeoutMs = fmt.Sprintf("%vm", config.GetConsole().IdleTimeoutMs-500)
	} else {
		gatewayContextTimeoutMs = fmt.Sprintf("%vm", config.GetConsole().IdleTimeoutMs)
	}

	serverOpts := []grpc.ServerOption{
		grpc.StatsHandler(&ocgrpc.ServerHandler{IsPublicEndpoint: true}),
		grpc.MaxRecvMsgSize(int(config.GetConsole().MaxMessageSizeBytes)),
		grpc.UnaryInterceptor(consoleInterceptorFunc(logger, config)),
	}
	grpcServer := grpc.NewServer(serverOpts...)

	s := &ConsoleServer{
		logger:         logger,
		db:             db,
		config:         config,
		tracker:        tracker,
		router:         router,
		statusHandler:  statusHandler,
		configWarnings: configWarnings,
		serverVersion:  serverVersion,
		grpcServer:     grpcServer,
	}

	console.RegisterConsoleServer(grpcServer, s)
	startupLogger.Info("Starting Console server for gRPC requests", zap.Int("port", config.GetConsole().Port-3))
	go func() {
		listener, err := net.Listen("tcp", fmt.Sprintf("%v:%d", config.GetConsole().Address, config.GetConsole().Port-3))
		if err != nil {
			startupLogger.Fatal("Console server listener failed to start", zap.Error(err))
		}

		if err := grpcServer.Serve(listener); err != nil {
			startupLogger.Fatal("Console server listener failed", zap.Error(err))
		}
	}()

	ctx := context.Background()
	grpcGateway := runtime.NewServeMux()
	if err := console.RegisterConsoleHandlerServer(ctx, grpcGateway, s); err != nil {
		startupLogger.Fatal("Console server gateway registration failed", zap.Error(err))
	}

	grpcGatewayRouter := mux.NewRouter()

	grpcGatewayRouter.Handle("/", console.UI).Methods("GET")
	grpcGatewayRouter.Handle("/manifest.json", console.UI).Methods("GET")
	grpcGatewayRouter.Handle("/favicon.ico", console.UI).Methods("GET")
	grpcGatewayRouter.PathPrefix("/static/").Handler(console.UI).Methods("GET")

	zpagesMux := http.NewServeMux()
	zpages.Handle(zpagesMux, "/metrics/")
	grpcGatewayRouter.NewRoute().PathPrefix("/metrics").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		zpagesMux.ServeHTTP(w, r)
	})

	grpcGatewayRouter.HandleFunc("/v2/console/storage/import", s.importStorage)

	grpcGatewaySecure := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		switch r.URL.Path {
		case "/v2/console/authenticate":
			// Authentication endpoint doesn't require security.
			grpcGateway.ServeHTTP(w, r)
		default:
			// All other endpoints are secured.
			auth, ok := r.Header["Authorization"]
			if !ok || len(auth) != 1 || !checkAuth(config, auth[0]) {
				// Auth token not valid or expired.
				w.WriteHeader(http.StatusUnauthorized)
				w.Header().Set("content-type", "application/json")
				_, err := w.Write(consoleAuthRequired)
				if err != nil {
					s.logger.Debug("Error writing response to client", zap.Error(err))
				}
				return
			}
			grpcGateway.ServeHTTP(w, r)
		}
	})

	// Enable max size check on requests coming arriving the gateway.
	// Enable compression on responses sent by the gateway.
	handlerWithCompressResponse := handlers.CompressHandler(grpcGatewaySecure)
	maxMessageSizeBytes := config.GetConsole().MaxMessageSizeBytes
	handlerWithMaxBody := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		// Check max body size before decompressing incoming request body.
		r.Body = http.MaxBytesReader(w, r.Body, maxMessageSizeBytes)
		handlerWithCompressResponse.ServeHTTP(w, r)
	})
	grpcGatewayRouter.NewRoute().HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		// Ensure some headers have required values.
		// Override any value set by the client if needed.
		r.Header.Set("Grpc-Timeout", gatewayContextTimeoutMs)

		// Allow GRPC Gateway to handle the request.
		handlerWithMaxBody.ServeHTTP(w, r)
	})

	// Enable CORS on all requests.
	CORSHeaders := handlers.AllowedHeaders([]string{"Authorization", "Content-Type", "User-Agent"})
	CORSOrigins := handlers.AllowedOrigins([]string{"*"})
	CORSMethods := handlers.AllowedMethods([]string{"GET", "HEAD", "POST", "PUT", "DELETE", "PATCH"})
	handlerWithCORS := handlers.CORS(CORSHeaders, CORSOrigins, CORSMethods)(grpcGatewayRouter)

	// Set up and start GRPC Gateway server.
	s.grpcGatewayServer = &http.Server{
		Addr:         fmt.Sprintf("%v:%d", config.GetConsole().Address, config.GetConsole().Port),
		ReadTimeout:  time.Millisecond * time.Duration(int64(config.GetConsole().ReadTimeoutMs)),
		WriteTimeout: time.Millisecond * time.Duration(int64(config.GetConsole().WriteTimeoutMs)),
		IdleTimeout:  time.Millisecond * time.Duration(int64(config.GetConsole().IdleTimeoutMs)),
		Handler:      handlerWithCORS,
	}

	startupLogger.Info("Starting Console server gateway for HTTP requests", zap.Int("port", config.GetConsole().Port))
	go func() {
		if err := s.grpcGatewayServer.ListenAndServe(); err != nil && err != http.ErrServerClosed {
			startupLogger.Fatal("Console server gateway listener failed", zap.Error(err))
		}
	}()

	return s
}

func (s *ConsoleServer) Stop() {
	// 1. Stop GRPC Gateway server first as it sits above GRPC server.
	if err := s.grpcGatewayServer.Shutdown(context.Background()); err != nil {
		s.logger.Error("API server gateway listener shutdown failed", zap.Error(err))
	}
	// 2. Stop GRPC server.
	s.grpcServer.GracefulStop()
}

func consoleInterceptorFunc(logger *zap.Logger, config Config) func(context.Context, interface{}, *grpc.UnaryServerInfo, grpc.UnaryHandler) (interface{}, error) {
	return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {

		switch info.FullMethod {
		// skip authentication check for Login endpoint
		case "/nakama.console.Console/Authenticate":
			return handler(ctx, req)
		}

		md, ok := metadata.FromIncomingContext(ctx)
		if !ok {
			logger.Error("Cannot extract metadata from incoming context")
			return nil, status.Error(codes.FailedPrecondition, "Cannot extract metadata from incoming context")
		}
		auth, ok := md["authorization"]
		if !ok {
			auth, ok = md["grpcgateway-authorization"]
		}
		if !ok {
			return nil, status.Error(codes.Unauthenticated, "Console authentication required.")
		}
		if len(auth) != 1 {
			return nil, status.Error(codes.Unauthenticated, "Console authentication required.")
		}

		if !checkAuth(config, auth[0]) {
			return nil, status.Error(codes.Unauthenticated, "Console authentication invalid.")
		}

		return handler(ctx, req)
	}
}

func checkAuth(config Config, auth string) bool {
	const basicPrefix = "Basic "
	const bearerPrefix = "Bearer "

	if strings.HasPrefix(auth, basicPrefix) {
		// Basic authentication.
		c, err := base64.StdEncoding.DecodeString(auth[len(basicPrefix):])
		if err != nil {
			// Not valid Base64.
			return false
		}
		cs := string(c)
		s := strings.IndexByte(cs, ':')
		if s < 0 {
			// Format is not "username:password".
			return false
		}

		if cs[:s] != config.GetConsole().Username || cs[s+1:] != config.GetConsole().Password {
			// Username and/or password do not match.
			return false
		}

		// Basic authentication successful.
		return true
	} else if strings.HasPrefix(auth, bearerPrefix) {
		// Bearer token authentication.
		token, err := jwt.Parse(auth[len(bearerPrefix):], func(token *jwt.Token) (interface{}, error) {
			if s, ok := token.Method.(*jwt.SigningMethodHMAC); !ok || s.Hash != crypto.SHA256 {
				return nil, fmt.Errorf("unexpected signing method: %v", token.Header["alg"])
			}
			return []byte(config.GetConsole().SigningKey), nil
		})
		if err != nil {
			// Token verification failed.
			return false
		}
		claims, ok := token.Claims.(jwt.MapClaims)
		if !ok || !token.Valid {
			// The token or its claims are invalid.
			return false
		}

		exp, ok := claims["exp"].(float64)
		if !ok {
			// Expiry time claim is invalid.
			return false
		}
		if int64(exp) <= time.Now().UTC().Unix() {
			// Token expired.
			return false
		}

		// Bearer token authentication successful.
		return true
	}

	return false
}
<|endoftext|>


<commit_before>//go:generate esc -o static.go -prefix ../web/build -pkg server ../web/build
package server

import (
	"encoding/hex"
	"encoding/xml"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"strings"

	"github.com/Sirupsen/logrus"
	"github.com/cardigann/cardigann/config"
	"github.com/cardigann/cardigann/indexer"
	"github.com/cardigann/cardigann/logger"
	"github.com/cardigann/cardigann/torznab"
	"github.com/gorilla/mux"
)

const (
	buildDir = "/web/build"
)

var (
	log              = logger.Logger
	apiRoutePrefixes = []string{
		"/torznab/",
		"/download/",
		"/xhr/",
		"/debug/",
	}
)

type Params struct {
	BaseURL    string
	APIKey     []byte
	Passphrase string
	Config     config.Config
	Version    string
}

type handler struct {
	http.Handler
	Params      Params
	FileHandler http.Handler
	indexers    map[string]torznab.Indexer
}

func NewHandler(p Params) (http.Handler, error) {
	h := &handler{
		Params:      p,
		FileHandler: http.FileServer(FS(false)),
		indexers:    map[string]torznab.Indexer{},
	}

	router := mux.NewRouter()

	// torznab routes
	router.HandleFunc("/torznab/{indexer}", h.torznabHandler).Methods("GET")
	router.HandleFunc("/torznab/{indexer}/api", h.torznabHandler).Methods("GET")
	router.HandleFunc("/download/{indexer}/{token}/{filename}", h.downloadHandler).Methods("HEAD")
	router.HandleFunc("/download/{indexer}/{token}/{filename}", h.downloadHandler).Methods("GET")
	router.HandleFunc("/download/{token}/{filename}", h.downloadHandler).Methods("HEAD")
	router.HandleFunc("/download/{token}/{filename}", h.downloadHandler).Methods("GET")

	// xhr routes for the webapp
	router.HandleFunc("/xhr/indexers/{indexer}/test", h.getIndexerTestHandler).Methods("GET")
	router.HandleFunc("/xhr/indexers/{indexer}/config", h.getIndexersConfigHandler).Methods("GET")
	router.HandleFunc("/xhr/indexers/{indexer}/config", h.patchIndexersConfigHandler).Methods("PATCH")
	router.HandleFunc("/xhr/indexers", h.getIndexersHandler).Methods("GET")
	router.HandleFunc("/xhr/indexers", h.patchIndexersHandler).Methods("PATCH")
	router.HandleFunc("/xhr/auth", h.getAuthHandler).Methods("GET")
	router.HandleFunc("/xhr/auth", h.postAuthHandler).Methods("POST")
	router.HandleFunc("/xhr/version", h.getVersionHandler).Methods("GET")

	h.Handler = router
	return h, h.initialize()
}

func (h *handler) initialize() error {
	if h.Params.Passphrase == "" {
		pass, hasPassphrase, _ := h.Params.Config.Get("global", "passphrase")
		if hasPassphrase {
			h.Params.Passphrase = pass
			return nil
		}
		apiKey, hasApiKey, _ := h.Params.Config.Get("global", "apikey")
		if !hasApiKey {
			k, err := h.sharedKey()
			if err != nil {
				return err
			}
			h.Params.APIKey = k
			return h.Params.Config.Set("global", "apikey", fmt.Sprintf("%x", k))
		}
		k, err := hex.DecodeString(apiKey)
		if err != nil {
			return err
		}
		h.Params.APIKey = k
	}
	return nil
}

func (h *handler) baseURL(r *http.Request, path string) (*url.URL, error) {
	if h.Params.BaseURL != "" {
		return url.Parse(h.Params.BaseURL)
	}
	proto := "http"
	if r.TLS != nil {
		proto = "https"
	}
	return url.Parse(fmt.Sprintf("%s://%s%s", proto, r.Host, path))
}

func (h *handler) createIndexer(key string) (torznab.Indexer, error) {
	def, err := indexer.DefaultDefinitionLoader.Load(key)
	if err != nil {
		log.WithError(err).Warnf("Failed to load definition for %q", key)
		return nil, err
	}

	log.WithFields(logrus.Fields{"indexer": key}).Debugf("Loaded indexer")
	indexer, err := indexer.NewRunner(def, indexer.RunnerOpts{
		Config: h.Params.Config,
	}), nil
	if err != nil {
		return nil, err
	}

	return indexer, nil
}

func (h *handler) lookupIndexer(key string) (torznab.Indexer, error) {
	if key == "aggregate" {
		return h.createAggregate()
	}
	if _, ok := h.indexers[key]; !ok {
		indexer, err := h.createIndexer(key)
		if err != nil {
			return nil, err
		}
		h.indexers[key] = indexer
	}

	return h.indexers[key], nil
}

func (h *handler) createAggregate() (torznab.Indexer, error) {
	keys, err := indexer.DefaultDefinitionLoader.List()
	if err != nil {
		return nil, err
	}

	agg := indexer.Aggregate{}
	for _, key := range keys {
		if config.IsSectionEnabled(key, h.Params.Config) {
			indexer, err := h.lookupIndexer(key)
			if err != nil {
				return nil, err
			}
			agg = append(agg, indexer)
		}
	}

	return agg, nil
}

func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if origin := r.Header.Get("Origin"); origin != "" {
		w.Header().Set("Access-Control-Allow-Origin", origin)
		w.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, DELETE, PATCH")
		w.Header().Set("Access-Control-Allow-Credentials", "true")
		w.Header().Set("Access-Control-Allow-Headers",
			"Accept, Cache-Control, Content-Type, Content-Length, Accept-Encoding, Authorization, Last-Event-ID")
	}
	if r.Method == "OPTIONS" {
		return
	}

	log.WithFields(logrus.Fields{
		"method": r.Method,
		"path":   r.URL.RequestURI(),
		"remote": r.RemoteAddr,
	}).Debugf("%s %s", r.Method, r.URL.RequestURI())

	for _, prefix := range apiRoutePrefixes {
		if strings.HasPrefix(r.URL.Path, prefix) {
			h.Handler.ServeHTTP(w, r)
			return
		}
	}

	h.FileHandler.ServeHTTP(w, r)
}

func (h *handler) torznabHandler(w http.ResponseWriter, r *http.Request) {
	params := mux.Vars(r)
	indexerID := params["indexer"]

	apiKey := r.URL.Query().Get("apikey")
	if !h.checkAPIKey(apiKey) {
		torznab.Error(w, "Invalid apikey parameter", torznab.ErrInsufficientPrivs)
		return
	}

	indexer, err := h.lookupIndexer(indexerID)
	if err != nil {
		torznab.Error(w, err.Error(), torznab.ErrIncorrectParameter)
		return
	}

	t := r.URL.Query().Get("t")

	if t == "" {
		http.Redirect(w, r, r.URL.Path+"?t=caps", http.StatusTemporaryRedirect)
		return
	}

	switch t {
	case "caps":
		indexer.Capabilities().ServeHTTP(w, r)

	case "search", "tvsearch", "tv-search":
		feed, err := h.search(r, indexer, indexerID)
		if err != nil {
			torznab.Error(w, err.Error(), torznab.ErrUnknownError)
			return
		}
		switch r.URL.Query().Get("format") {
		case "", "xml":
			x, err := xml.MarshalIndent(feed, "", "  ")
			if err != nil {
				torznab.Error(w, err.Error(), torznab.ErrUnknownError)
				return
			}
			w.Header().Set("Content-Type", "application/rss+xml")
			w.Write(x)
		case "json":
			jsonOutput(w, feed)
		}

	default:
		torznab.Error(w, "Unknown type parameter", torznab.ErrIncorrectParameter)
	}
}

func (h *handler) downloadHandler(w http.ResponseWriter, r *http.Request) {
	params := mux.Vars(r)
	token := params["token"]
	filename := params["filename"]

	log.WithFields(logrus.Fields{"filename": filename}).Debugf("Processing download via handler")

	k, err := h.sharedKey()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	t, err := decodeToken(token, k)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadGateway)
		return
	}

	indexer, err := h.lookupIndexer(t.Site)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadGateway)
		return
	}

	rc, _, err := indexer.Download(t.Link)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadGateway)
		return
	}

	w.Header().Set("Content-Type", "application/x-bittorrent")
	w.Header().Set("Content-Disposition", "attachment; filename="+filename)
	w.Header().Set("Content-Transfer-Encoding", "binary")

	defer rc.Close()
	io.Copy(w, rc)
}

func (h *handler) search(r *http.Request, indexer torznab.Indexer, siteKey string) (*torznab.ResultFeed, error) {
	baseURL, err := h.baseURL(r, "/download")
	if err != nil {
		return nil, err
	}

	query, err := torznab.ParseQuery(r.URL.Query())
	if err != nil {
		return nil, err
	}

	items, err := indexer.Search(query)
	if err != nil {
		return nil, err
	}

	feed := &torznab.ResultFeed{
		Info:  indexer.Info(),
		Items: items,
	}

	k, err := h.sharedKey()
	if err != nil {
		return nil, err
	}

	// rewrite links to use the server
	for idx, item := range feed.Items {
		t := &token{
			Site: item.Site,
			Link: item.Link,
		}

		te, err := t.Encode(k)
		if err != nil {
			log.Debugf("Error encoding token: %v", err)
			return nil, err
		}

		log.Debugf("Generated signed token %q", te)
		feed.Items[idx].Link = fmt.Sprintf("%s/%s/%s.torrent", baseURL.String(), te, item.Title)
	}

	return feed, err
}
<commit_msg>Remove "generating signed token" logging<commit_after>//go:generate esc -o static.go -prefix ../web/build -pkg server ../web/build
package server

import (
	"encoding/hex"
	"encoding/xml"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"strings"

	"github.com/Sirupsen/logrus"
	"github.com/cardigann/cardigann/config"
	"github.com/cardigann/cardigann/indexer"
	"github.com/cardigann/cardigann/logger"
	"github.com/cardigann/cardigann/torznab"
	"github.com/gorilla/mux"
)

const (
	buildDir = "/web/build"
)

var (
	log              = logger.Logger
	apiRoutePrefixes = []string{
		"/torznab/",
		"/download/",
		"/xhr/",
		"/debug/",
	}
)

type Params struct {
	BaseURL    string
	APIKey     []byte
	Passphrase string
	Config     config.Config
	Version    string
}

type handler struct {
	http.Handler
	Params      Params
	FileHandler http.Handler
	indexers    map[string]torznab.Indexer
}

func NewHandler(p Params) (http.Handler, error) {
	h := &handler{
		Params:      p,
		FileHandler: http.FileServer(FS(false)),
		indexers:    map[string]torznab.Indexer{},
	}

	router := mux.NewRouter()

	// torznab routes
	router.HandleFunc("/torznab/{indexer}", h.torznabHandler).Methods("GET")
	router.HandleFunc("/torznab/{indexer}/api", h.torznabHandler).Methods("GET")
	router.HandleFunc("/download/{indexer}/{token}/{filename}", h.downloadHandler).Methods("HEAD")
	router.HandleFunc("/download/{indexer}/{token}/{filename}", h.downloadHandler).Methods("GET")
	router.HandleFunc("/download/{token}/{filename}", h.downloadHandler).Methods("HEAD")
	router.HandleFunc("/download/{token}/{filename}", h.downloadHandler).Methods("GET")

	// xhr routes for the webapp
	router.HandleFunc("/xhr/indexers/{indexer}/test", h.getIndexerTestHandler).Methods("GET")
	router.HandleFunc("/xhr/indexers/{indexer}/config", h.getIndexersConfigHandler).Methods("GET")
	router.HandleFunc("/xhr/indexers/{indexer}/config", h.patchIndexersConfigHandler).Methods("PATCH")
	router.HandleFunc("/xhr/indexers", h.getIndexersHandler).Methods("GET")
	router.HandleFunc("/xhr/indexers", h.patchIndexersHandler).Methods("PATCH")
	router.HandleFunc("/xhr/auth", h.getAuthHandler).Methods("GET")
	router.HandleFunc("/xhr/auth", h.postAuthHandler).Methods("POST")
	router.HandleFunc("/xhr/version", h.getVersionHandler).Methods("GET")

	h.Handler = router
	return h, h.initialize()
}

func (h *handler) initialize() error {
	if h.Params.Passphrase == "" {
		pass, hasPassphrase, _ := h.Params.Config.Get("global", "passphrase")
		if hasPassphrase {
			h.Params.Passphrase = pass
			return nil
		}
		apiKey, hasApiKey, _ := h.Params.Config.Get("global", "apikey")
		if !hasApiKey {
			k, err := h.sharedKey()
			if err != nil {
				return err
			}
			h.Params.APIKey = k
			return h.Params.Config.Set("global", "apikey", fmt.Sprintf("%x", k))
		}
		k, err := hex.DecodeString(apiKey)
		if err != nil {
			return err
		}
		h.Params.APIKey = k
	}
	return nil
}

func (h *handler) baseURL(r *http.Request, path string) (*url.URL, error) {
	if h.Params.BaseURL != "" {
		return url.Parse(h.Params.BaseURL)
	}
	proto := "http"
	if r.TLS != nil {
		proto = "https"
	}
	return url.Parse(fmt.Sprintf("%s://%s%s", proto, r.Host, path))
}

func (h *handler) createIndexer(key string) (torznab.Indexer, error) {
	def, err := indexer.DefaultDefinitionLoader.Load(key)
	if err != nil {
		log.WithError(err).Warnf("Failed to load definition for %q", key)
		return nil, err
	}

	log.WithFields(logrus.Fields{"indexer": key}).Debugf("Loaded indexer")
	indexer, err := indexer.NewRunner(def, indexer.RunnerOpts{
		Config: h.Params.Config,
	}), nil
	if err != nil {
		return nil, err
	}

	return indexer, nil
}

func (h *handler) lookupIndexer(key string) (torznab.Indexer, error) {
	if key == "aggregate" {
		return h.createAggregate()
	}
	if _, ok := h.indexers[key]; !ok {
		indexer, err := h.createIndexer(key)
		if err != nil {
			return nil, err
		}
		h.indexers[key] = indexer
	}

	return h.indexers[key], nil
}

func (h *handler) createAggregate() (torznab.Indexer, error) {
	keys, err := indexer.DefaultDefinitionLoader.List()
	if err != nil {
		return nil, err
	}

	agg := indexer.Aggregate{}
	for _, key := range keys {
		if config.IsSectionEnabled(key, h.Params.Config) {
			indexer, err := h.lookupIndexer(key)
			if err != nil {
				return nil, err
			}
			agg = append(agg, indexer)
		}
	}

	return agg, nil
}

func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if origin := r.Header.Get("Origin"); origin != "" {
		w.Header().Set("Access-Control-Allow-Origin", origin)
		w.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, DELETE, PATCH")
		w.Header().Set("Access-Control-Allow-Credentials", "true")
		w.Header().Set("Access-Control-Allow-Headers",
			"Accept, Cache-Control, Content-Type, Content-Length, Accept-Encoding, Authorization, Last-Event-ID")
	}
	if r.Method == "OPTIONS" {
		return
	}

	log.WithFields(logrus.Fields{
		"method": r.Method,
		"path":   r.URL.RequestURI(),
		"remote": r.RemoteAddr,
	}).Debugf("%s %s", r.Method, r.URL.RequestURI())

	for _, prefix := range apiRoutePrefixes {
		if strings.HasPrefix(r.URL.Path, prefix) {
			h.Handler.ServeHTTP(w, r)
			return
		}
	}

	h.FileHandler.ServeHTTP(w, r)
}

func (h *handler) torznabHandler(w http.ResponseWriter, r *http.Request) {
	params := mux.Vars(r)
	indexerID := params["indexer"]

	apiKey := r.URL.Query().Get("apikey")
	if !h.checkAPIKey(apiKey) {
		torznab.Error(w, "Invalid apikey parameter", torznab.ErrInsufficientPrivs)
		return
	}

	indexer, err := h.lookupIndexer(indexerID)
	if err != nil {
		torznab.Error(w, err.Error(), torznab.ErrIncorrectParameter)
		return
	}

	t := r.URL.Query().Get("t")

	if t == "" {
		http.Redirect(w, r, r.URL.Path+"?t=caps", http.StatusTemporaryRedirect)
		return
	}

	switch t {
	case "caps":
		indexer.Capabilities().ServeHTTP(w, r)

	case "search", "tvsearch", "tv-search":
		feed, err := h.search(r, indexer, indexerID)
		if err != nil {
			torznab.Error(w, err.Error(), torznab.ErrUnknownError)
			return
		}
		switch r.URL.Query().Get("format") {
		case "", "xml":
			x, err := xml.MarshalIndent(feed, "", "  ")
			if err != nil {
				torznab.Error(w, err.Error(), torznab.ErrUnknownError)
				return
			}
			w.Header().Set("Content-Type", "application/rss+xml")
			w.Write(x)
		case "json":
			jsonOutput(w, feed)
		}

	default:
		torznab.Error(w, "Unknown type parameter", torznab.ErrIncorrectParameter)
	}
}

func (h *handler) downloadHandler(w http.ResponseWriter, r *http.Request) {
	params := mux.Vars(r)
	token := params["token"]
	filename := params["filename"]

	log.WithFields(logrus.Fields{"filename": filename}).Debugf("Processing download via handler")

	k, err := h.sharedKey()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	t, err := decodeToken(token, k)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadGateway)
		return
	}

	indexer, err := h.lookupIndexer(t.Site)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadGateway)
		return
	}

	rc, _, err := indexer.Download(t.Link)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadGateway)
		return
	}

	w.Header().Set("Content-Type", "application/x-bittorrent")
	w.Header().Set("Content-Disposition", "attachment; filename="+filename)
	w.Header().Set("Content-Transfer-Encoding", "binary")

	defer rc.Close()
	io.Copy(w, rc)
}

func (h *handler) search(r *http.Request, indexer torznab.Indexer, siteKey string) (*torznab.ResultFeed, error) {
	baseURL, err := h.baseURL(r, "/download")
	if err != nil {
		return nil, err
	}

	query, err := torznab.ParseQuery(r.URL.Query())
	if err != nil {
		return nil, err
	}

	items, err := indexer.Search(query)
	if err != nil {
		return nil, err
	}

	feed := &torznab.ResultFeed{
		Info:  indexer.Info(),
		Items: items,
	}

	k, err := h.sharedKey()
	if err != nil {
		return nil, err
	}

	// rewrite links to use the server
	for idx, item := range feed.Items {
		t := &token{
			Site: item.Site,
			Link: item.Link,
		}

		te, err := t.Encode(k)
		if err != nil {
			log.Debugf("Error encoding token: %v", err)
			return nil, err
		}

		feed.Items[idx].Link = fmt.Sprintf("%s/%s/%s.torrent", baseURL.String(), te, item.Title)
	}

	return feed, err
}
<|endoftext|>