<commit_before>package aerospike

import (
	"github.com/aerospike/aerospike-client-go"
	"github.com/plimble/aerosingle"
	. "github.com/plimble/clover"
	"github.com/plimble/utils/errors2"
)

var errNoPublicKey = errors2.NewInternal("No publickey in store")
var errNotFound = errors2.NewNotFound("not found")

type GetUserFunc func(username, password string) (string, []string, error)
type GetClientFunc func(clientID string) (Client, error)

type Store struct {
	client        *aerosingle.Client
	key           *PublicKey
	getUserFunc   GetUserFunc
	getClientFunc GetClientFunc
}

func New(asClient *aerosingle.Client, ns string) *Store {
	return &Store{asClient, nil, nil, nil}
}

func (s *Store) RegisterGetUserFunc(fn GetUserFunc) {
	s.getUserFunc = fn
}

func (s *Store) RegisterGetClientFunc(fn GetClientFunc) {
	s.getClientFunc = fn
}

func (s *Store) Close() {
	s.client.Close()
}

func (s *Store) GetUser(username, password string) (string, []string, error) {
	if s.getUserFunc == nil {
		panic("Not implement GetUserFunc")
	}

	id, scopes, err := s.getUserFunc(username, password)
	return id, scopes, err
}

func (s *Store) GetClient(cid string) (Client, error) {
	if s.getClientFunc == nil {
		panic("Not implement GetClientFunc")
	}

	client, err := s.getClientFunc(cid)
	return client, err
}

func (s *Store) SetAccessToken(accessToken *AccessToken) error {
	policy := aerospike.NewWritePolicy(0, 0)
	policy.RecordExistsAction = aerospike.CREATE_ONLY

	return s.client.Put(policy, "access_token", accessToken.AccessToken, accessToken)
}

func (s *Store) GetAccessToken(at string) (*AccessToken, error) {
	var data AccessToken
	if err := s.client.Get(nil, "access_token", at, &data); err != nil {
		return nil, err
	}

	return &data, nil
}

func (s *Store) SetRefreshToken(refreshToken *RefreshToken) error {
	policy := aerospike.NewWritePolicy(0, 0)
	policy.RecordExistsAction = aerospike.CREATE_ONLY

	return s.client.Put(policy, "refresh_token", refreshToken.RefreshToken, refreshToken)
}

func (s *Store) GetRefreshToken(rt string) (*RefreshToken, error) {
	var data RefreshToken
	if err := s.client.Get(nil, "refresh_token", rt, &data); err != nil {
		return nil, err
	}

	return &data, nil
}

func (s *Store) RemoveRefreshToken(rt string) error {
	return s.client.Delete(nil, "refresh_token", rt)
}

func (s *Store) SetAuthorizeCode(ac *AuthorizeCode) error {
	policy := aerospike.NewWritePolicy(0, 0)
	policy.RecordExistsAction = aerospike.CREATE_ONLY

	return s.client.Put(policy, "auth_code", ac.Code, ac)
}

func (s *Store) GetAuthorizeCode(code string) (*AuthorizeCode, error) {
	var data AuthorizeCode
	if err := s.client.Get(nil, "auth_code", code, &data); err != nil {
		return nil, err
	}

	return &data, nil
}

func (s *Store) SetPublicKey(key *PublicKey) {
	s.key = key
}

func (s *Store) GetKey(clientID string) (*PublicKey, error) {
	if s.key == nil {
		return nil, errNoPublicKey
	}

	return s.key, nil
}
<commit_msg>update aerospace<commit_after>package aerospike

import (
	"github.com/aerospike/aerospike-client-go"
	"github.com/plimble/aerosingle"
	. "github.com/plimble/clover"
	"github.com/plimble/utils/errors2"
)

var errNoPublicKey = errors2.NewInternal("No publickey in store")
var errNotFound = errors2.NewNotFound("not found")

type GetUserFunc func(username, password string) (string, []string, error)
type GetClientFunc func(clientID string) (Client, error)

type AeroStore struct {
	client        *aerosingle.Client
	key           *PublicKey
	getUserFunc   GetUserFunc
	getClientFunc GetClientFunc
}

func New(asClient *aerosingle.Client) *AeroStore {
	return &AeroStore{asClient, nil, nil, nil}
}

func (s *AeroStore) RegisterGetUserFunc(fn GetUserFunc) {
	s.getUserFunc = fn
}

func (s *AeroStore) RegisterGetClientFunc(fn GetClientFunc) {
	s.getClientFunc = fn
}

func (s *AeroStore) Close() {
	s.client.Close()
}

func (s *AeroStore) GetUser(username, password string) (string, []string, error) {
	if s.getUserFunc == nil {
		panic("Not implement GetUserFunc")
	}

	id, scopes, err := s.getUserFunc(username, password)
	return id, scopes, err
}

func (s *AeroStore) GetClient(cid string) (Client, error) {
	if s.getClientFunc == nil {
		panic("Not implement GetClientFunc")
	}

	client, err := s.getClientFunc(cid)
	return client, err
}

func (s *AeroStore) SetAccessToken(accessToken *AccessToken) error {
	policy := aerospike.NewWritePolicy(0, 0)
	policy.RecordExistsAction = aerospike.CREATE_ONLY

	return s.client.Put(policy, "access_token", accessToken.AccessToken, accessToken)
}

func (s *AeroStore) GetAccessToken(at string) (*AccessToken, error) {
	var data AccessToken
	if err := s.client.Get(nil, "access_token", at, &data); err != nil {
		return nil, err
	}

	return &data, nil
}

func (s *AeroStore) SetRefreshToken(refreshToken *RefreshToken) error {
	policy := aerospike.NewWritePolicy(0, 0)
	policy.RecordExistsAction = aerospike.CREATE_ONLY

	return s.client.Put(policy, "refresh_token", refreshToken.RefreshToken, refreshToken)
}

func (s *AeroStore) GetRefreshToken(rt string) (*RefreshToken, error) {
	var data RefreshToken
	if err := s.client.Get(nil, "refresh_token", rt, &data); err != nil {
		return nil, err
	}

	return &data, nil
}

func (s *AeroStore) RemoveRefreshToken(rt string) error {
	return s.client.Delete(nil, "refresh_token", rt)
}

func (s *AeroStore) SetAuthorizeCode(ac *AuthorizeCode) error {
	policy := aerospike.NewWritePolicy(0, 0)
	policy.RecordExistsAction = aerospike.CREATE_ONLY

	return s.client.Put(policy, "auth_code", ac.Code, ac)
}

func (s *AeroStore) GetAuthorizeCode(code string) (*AuthorizeCode, error) {
	var data AuthorizeCode
	if err := s.client.Get(nil, "auth_code", code, &data); err != nil {
		return nil, err
	}

	return &data, nil
}

func (s *AeroStore) SetPublicKey(key *PublicKey) {
	s.key = key
}

func (s *AeroStore) GetKey(clientID string) (*PublicKey, error) {
	if s.key == nil {
		return nil, errNoPublicKey
	}

	return s.key, nil
}
<|endoftext|>


<commit_before>// Package sdhook provides a logrus compatible logging hook for Google
// Stackdriver logging.
package sdhook

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/facebookgo/stack"
	"github.com/fluent/fluent-logger-golang/fluent"
	"github.com/sirupsen/logrus"

	errorReporting "google.golang.org/api/clouderrorreporting/v1beta1"
	logging "google.golang.org/api/logging/v2"
)

const (
	// DefaultName is the default name passed to LogName when using service
	// account credentials.
	DefaultName = "default"
)

// StackdriverHook provides a logrus hook to Google Stackdriver logging.
type StackdriverHook struct {
	// levels are the levels that logrus will hook to.
	levels []logrus.Level

	// projectID is the projectID
	projectID string

	// service is the logging service.
	service *logging.EntriesService

	// service is the error reporting service.
	errorService *errorReporting.Service

	// resource is the monitored resource.
	resource *logging.MonitoredResource

	// logName is the name of the log.
	logName string

	// labels are the labels to send with each log entry.
	labels map[string]string

	// partialSuccess allows partial writes of log entries if there is a badly
	// formatted log.
	partialSuccess bool

	// agentClient defines the fluentd logger object that can send data to
	// to the Google logging agent.
	agentClient *fluent.Fluent

	// errorReportingServiceName defines the value of the field <service>,
	// required for a valid error reporting payload. If this value is set,
	// messages where level/severity is higher than or equal to "error" will
	// be sent to Stackdriver error reporting.
	// See more at:
	// https://cloud.google.com/error-reporting/docs/formatting-error-messages
	errorReportingServiceName string

	// errorReportingLogName is the name of the log for error reporting.
	// It must contain the string "error"
	// If not given, the string "<logName>_error" is used.
	errorReportingLogName string
}

// New creates a StackdriverHook using the provided options that is suitible
// for using with logrus for logging to Google Stackdriver.
func New(opts ...Option) (*StackdriverHook, error) {
	var err error

	sh := &StackdriverHook{
		levels: logrus.AllLevels,
	}

	// apply opts
	for _, o := range opts {
		err = o(sh)
		if err != nil {
			return nil, err
		}
	}

	// check service, resource, logName set
	if sh.service == nil && sh.agentClient == nil {
		return nil, errors.New("no stackdriver service was provided")
	}
	if sh.resource == nil && sh.agentClient == nil {
		return nil, errors.New("the monitored resource was not provided")
	}
	if sh.projectID == "" && sh.agentClient == nil {
		return nil, errors.New("the project id was not provided")
	}

	// set default project name
	if sh.logName == "" {
		err = LogName(DefaultName)(sh)
		if err != nil {
			return nil, err
		}
	}

	// If error reporting log name not set, set it to log name
	// plus string suffix
	if sh.errorReportingLogName == "" {
		sh.errorReportingLogName = sh.logName + "_errors"
	}

	return sh, nil
}

func isError(entry *logrus.Entry) bool {
	if entry != nil {
		switch entry.Level {
		case logrus.ErrorLevel:
			return true
		case logrus.FatalLevel:
			return true
		case logrus.PanicLevel:
			return true
		}
	}
	return false
}

// Levels returns the logrus levels that this hook is applied to. This can be
// set using the Levels Option.
func (sh *StackdriverHook) Levels() []logrus.Level {
	return sh.levels
}

// Fire writes the message to the Stackdriver entry service.
func (sh *StackdriverHook) Fire(entry *logrus.Entry) error {
	go func(entry *logrus.Entry) {
		var httpReq *logging.HttpRequest

		// convert entry data to labels
		labels := make(map[string]string, len(entry.Data))
		for k, v := range entry.Data {
			switch x := v.(type) {
			case string:
				labels[k] = x

			case *http.Request:
				httpReq = &logging.HttpRequest{
					Referer:       x.Referer(),
					RemoteIp:      x.RemoteAddr,
					RequestMethod: x.Method,
					RequestUrl:    x.URL.String(),
					UserAgent:     x.UserAgent(),
				}

			case *logging.HttpRequest:
				httpReq = x

			default:
				labels[k] = fmt.Sprintf("%v", v)
			}
		}

		// write log entry
		if sh.agentClient != nil {
			sh.sendLogMessageViaAgent(entry, labels, httpReq)
		} else {
			sh.sendLogMessageViaAPI(entry, labels, httpReq)
		}
	}(sh.copyEntry(entry))

	return nil
}

func (sh *StackdriverHook) copyEntry(entry *logrus.Entry) *logrus.Entry {
	e := *entry
	e.Data = make(logrus.Fields, len(entry.Data))
	for k, v := range entry.Data {
		e.Data[k] = v
	}
	return &e
}

func (sh *StackdriverHook) sendLogMessageViaAgent(entry *logrus.Entry, labels map[string]string, httpReq *logging.HttpRequest) {
	// The log entry payload schema is defined by the Google fluentd
	// logging agent. See more at:
	// https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud
	logEntry := map[string]interface{}{
		"severity":         strings.ToUpper(entry.Level.String()),
		"timestampSeconds": strconv.FormatInt(entry.Time.Unix(), 10),
		"timestampNanos":   strconv.FormatInt(entry.Time.UnixNano()-entry.Time.Unix()*1000000000, 10),
		"message":          entry.Message,
	}
	for k, v := range labels {
		logEntry[k] = v
	}
	if httpReq != nil {
		logEntry["httpRequest"] = httpReq
	}
	// The error reporting payload JSON schema is defined in:
	// https://cloud.google.com/error-reporting/docs/formatting-error-messages
	// Which reflects the structure of the ErrorEvent type in:
	// https://godoc.org/google.golang.org/api/clouderrorreporting/v1beta1
	if sh.errorReportingServiceName != "" && isError(entry) {
		errorEvent := sh.buildErrorReportingEvent(entry, labels, httpReq)
		errorStructPayload, err := json.Marshal(errorEvent)
		if err != nil {
			log.Printf("error marshaling error reporting data: %s", err.Error())
		}
		var errorJSONPayload map[string]interface{}
		err = json.Unmarshal(errorStructPayload, &errorJSONPayload)
		if err != nil {
			log.Printf("error parsing error reporting data: %s", err.Error())
		}
		for k, v := range logEntry {
			errorJSONPayload[k] = v
		}
		if err := sh.agentClient.Post(sh.errorReportingLogName, errorJSONPayload); err != nil {
			log.Printf("error posting error reporting entries to logging agent: %s", err.Error())
		}
	} else {
		if err := sh.agentClient.Post(sh.logName, logEntry); err != nil {
			log.Printf("error posting log entries to logging agent: %s", err.Error())
		}
	}
}

func (sh *StackdriverHook) sendLogMessageViaAPI(entry *logrus.Entry, labels map[string]string, httpReq *logging.HttpRequest) {
	if sh.errorReportingServiceName != "" && isError(entry) {
		errorEvent := sh.buildErrorReportingEvent(entry, labels, httpReq)
		sh.errorService.Projects.Events.Report(sh.projectID, &errorEvent)
	} else {
		logName := sh.logName
		if sh.errorReportingLogName != "" && isError(entry) {
			logName = sh.errorReportingLogName
		}
		_, _ = sh.service.Write(&logging.WriteLogEntriesRequest{
			LogName:        logName,
			Resource:       sh.resource,
			Labels:         sh.labels,
			PartialSuccess: sh.partialSuccess,
			Entries: []*logging.LogEntry{
				{
					Severity:    strings.ToUpper(entry.Level.String()),
					Timestamp:   entry.Time.Format(time.RFC3339),
					TextPayload: entry.Message,
					Labels:      labels,
					HttpRequest: httpReq,
				},
			},
		}).Do()
	}
}

func (sh *StackdriverHook) buildErrorReportingEvent(entry *logrus.Entry, labels map[string]string, httpReq *logging.HttpRequest) errorReporting.ReportedErrorEvent {
	errorEvent := errorReporting.ReportedErrorEvent{
		EventTime: entry.Time.Format(time.RFC3339),
		Message:   entry.Message,
		ServiceContext: &errorReporting.ServiceContext{
			Service: sh.errorReportingServiceName,
			Version: labels["version"],
		},
		Context: &errorReporting.ErrorContext{
			User: labels["user"],
		},
	}
	// Assumes that caller stack frame information of type
	// github.com/facebookgo/stack.Frame has been added.
	// Possibly via a library like github.com/Gurpartap/logrus-stack
	if entry.Data["caller"] != nil {
		caller := entry.Data["caller"].(stack.Frame)
		errorEvent.Context.ReportLocation = &errorReporting.SourceLocation{
			FilePath:     caller.File,
			FunctionName: caller.Name,
			LineNumber:   int64(caller.Line),
		}
	}
	if httpReq != nil {
		errRepHttpRequest := &errorReporting.HttpRequestContext{
			Method:    httpReq.RequestMethod,
			Referrer:  httpReq.Referer,
			RemoteIp:  httpReq.RemoteIp,
			Url:       httpReq.RequestUrl,
			UserAgent: httpReq.UserAgent,
		}
		errorEvent.Context.HttpRequest = errRepHttpRequest
	}
	return errorEvent
}
<commit_msg>Adding subroutine wait group<commit_after>// Package sdhook provides a logrus compatible logging hook for Google
// Stackdriver logging.
package sdhook

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"net/http"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/facebookgo/stack"
	"github.com/fluent/fluent-logger-golang/fluent"
	"github.com/sirupsen/logrus"

	errorReporting "google.golang.org/api/clouderrorreporting/v1beta1"
	logging "google.golang.org/api/logging/v2"
)

const (
	// DefaultName is the default name passed to LogName when using service
	// account credentials.
	DefaultName = "default"
)

// StackdriverHook provides a logrus hook to Google Stackdriver logging.
type StackdriverHook struct {
	// levels are the levels that logrus will hook to.
	levels []logrus.Level

	// projectID is the projectID
	projectID string

	// service is the logging service.
	service *logging.EntriesService

	// service is the error reporting service.
	errorService *errorReporting.Service

	// resource is the monitored resource.
	resource *logging.MonitoredResource

	// logName is the name of the log.
	logName string

	// labels are the labels to send with each log entry.
	labels map[string]string

	// partialSuccess allows partial writes of log entries if there is a badly
	// formatted log.
	partialSuccess bool

	// agentClient defines the fluentd logger object that can send data to
	// to the Google logging agent.
	agentClient *fluent.Fluent

	// errorReportingServiceName defines the value of the field <service>,
	// required for a valid error reporting payload. If this value is set,
	// messages where level/severity is higher than or equal to "error" will
	// be sent to Stackdriver error reporting.
	// See more at:
	// https://cloud.google.com/error-reporting/docs/formatting-error-messages
	errorReportingServiceName string

	// errorReportingLogName is the name of the log for error reporting.
	// It must contain the string "error"
	// If not given, the string "<logName>_error" is used.
	errorReportingLogName string

	// waitGroup holds counters for each subroutine fired
	waitGroup sync.WaitGroup
}

// New creates a StackdriverHook using the provided options that is suitible
// for using with logrus for logging to Google Stackdriver.
func New(opts ...Option) (*StackdriverHook, error) {
	var err error

	sh := &StackdriverHook{
		levels: logrus.AllLevels,
	}

	// apply opts
	for _, o := range opts {
		err = o(sh)
		if err != nil {
			return nil, err
		}
	}

	// check service, resource, logName set
	if sh.service == nil && sh.agentClient == nil {
		return nil, errors.New("no stackdriver service was provided")
	}
	if sh.resource == nil && sh.agentClient == nil {
		return nil, errors.New("the monitored resource was not provided")
	}
	if sh.projectID == "" && sh.agentClient == nil {
		return nil, errors.New("the project id was not provided")
	}

	// set default project name
	if sh.logName == "" {
		err = LogName(DefaultName)(sh)
		if err != nil {
			return nil, err
		}
	}

	// If error reporting log name not set, set it to log name
	// plus string suffix
	if sh.errorReportingLogName == "" {
		sh.errorReportingLogName = sh.logName + "_errors"
	}

	return sh, nil
}

func isError(entry *logrus.Entry) bool {
	if entry != nil {
		switch entry.Level {
		case logrus.ErrorLevel:
			return true
		case logrus.FatalLevel:
			return true
		case logrus.PanicLevel:
			return true
		}
	}
	return false
}

// Levels returns the logrus levels that this hook is applied to. This can be
// set using the Levels Option.
func (sh *StackdriverHook) Levels() []logrus.Level {
	return sh.levels
}

// Fire writes the message to the Stackdriver entry service.
func (sh *StackdriverHook) Fire(entry *logrus.Entry) error {
	sh.waitGroup.Add(1)
	go func(entry *logrus.Entry) {
		defer sh.waitGroup.Done()
		var httpReq *logging.HttpRequest

		// convert entry data to labels
		labels := make(map[string]string, len(entry.Data))
		for k, v := range entry.Data {
			switch x := v.(type) {
			case string:
				labels[k] = x

			case *http.Request:
				httpReq = &logging.HttpRequest{
					Referer:       x.Referer(),
					RemoteIp:      x.RemoteAddr,
					RequestMethod: x.Method,
					RequestUrl:    x.URL.String(),
					UserAgent:     x.UserAgent(),
				}

			case *logging.HttpRequest:
				httpReq = x

			default:
				labels[k] = fmt.Sprintf("%v", v)
			}
		}

		// write log entry
		if sh.agentClient != nil {
			sh.sendLogMessageViaAgent(entry, labels, httpReq)
		} else {
			sh.sendLogMessageViaAPI(entry, labels, httpReq)
		}
	}(sh.copyEntry(entry))

	return nil
}

// Wait will return after all subroutines have returned.
// Use in conjunction with logrus return handling to ensure all of
// your logs are delivered before your program exits.
// `logrus.RegisterExitHandler(h.Wait)`
func (sh *StackdriverHook) Wait() {
	sh.waitGroup.Wait()
}

func (sh *StackdriverHook) copyEntry(entry *logrus.Entry) *logrus.Entry {
	e := *entry
	e.Data = make(logrus.Fields, len(entry.Data))
	for k, v := range entry.Data {
		e.Data[k] = v
	}
	return &e
}

func (sh *StackdriverHook) sendLogMessageViaAgent(entry *logrus.Entry, labels map[string]string, httpReq *logging.HttpRequest) {
	// The log entry payload schema is defined by the Google fluentd
	// logging agent. See more at:
	// https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud
	logEntry := map[string]interface{}{
		"severity":         strings.ToUpper(entry.Level.String()),
		"timestampSeconds": strconv.FormatInt(entry.Time.Unix(), 10),
		"timestampNanos":   strconv.FormatInt(entry.Time.UnixNano()-entry.Time.Unix()*1000000000, 10),
		"message":          entry.Message,
	}
	for k, v := range labels {
		logEntry[k] = v
	}
	if httpReq != nil {
		logEntry["httpRequest"] = httpReq
	}
	// The error reporting payload JSON schema is defined in:
	// https://cloud.google.com/error-reporting/docs/formatting-error-messages
	// Which reflects the structure of the ErrorEvent type in:
	// https://godoc.org/google.golang.org/api/clouderrorreporting/v1beta1
	if sh.errorReportingServiceName != "" && isError(entry) {
		errorEvent := sh.buildErrorReportingEvent(entry, labels, httpReq)
		errorStructPayload, err := json.Marshal(errorEvent)
		if err != nil {
			log.Printf("error marshaling error reporting data: %s", err.Error())
		}
		var errorJSONPayload map[string]interface{}
		err = json.Unmarshal(errorStructPayload, &errorJSONPayload)
		if err != nil {
			log.Printf("error parsing error reporting data: %s", err.Error())
		}
		for k, v := range logEntry {
			errorJSONPayload[k] = v
		}
		if err := sh.agentClient.Post(sh.errorReportingLogName, errorJSONPayload); err != nil {
			log.Printf("error posting error reporting entries to logging agent: %s", err.Error())
		}
	} else {
		if err := sh.agentClient.Post(sh.logName, logEntry); err != nil {
			log.Printf("error posting log entries to logging agent: %s", err.Error())
		}
	}
}

func (sh *StackdriverHook) sendLogMessageViaAPI(entry *logrus.Entry, labels map[string]string, httpReq *logging.HttpRequest) {
	if sh.errorReportingServiceName != "" && isError(entry) {
		errorEvent := sh.buildErrorReportingEvent(entry, labels, httpReq)
		sh.errorService.Projects.Events.Report(sh.projectID, &errorEvent)
	} else {
		logName := sh.logName
		if sh.errorReportingLogName != "" && isError(entry) {
			logName = sh.errorReportingLogName
		}
		_, _ = sh.service.Write(&logging.WriteLogEntriesRequest{
			LogName:        logName,
			Resource:       sh.resource,
			Labels:         sh.labels,
			PartialSuccess: sh.partialSuccess,
			Entries: []*logging.LogEntry{
				{
					Severity:    strings.ToUpper(entry.Level.String()),
					Timestamp:   entry.Time.Format(time.RFC3339),
					TextPayload: entry.Message,
					Labels:      labels,
					HttpRequest: httpReq,
				},
			},
		}).Do()
	}
}

func (sh *StackdriverHook) buildErrorReportingEvent(entry *logrus.Entry, labels map[string]string, httpReq *logging.HttpRequest) errorReporting.ReportedErrorEvent {
	errorEvent := errorReporting.ReportedErrorEvent{
		EventTime: entry.Time.Format(time.RFC3339),
		Message:   entry.Message,
		ServiceContext: &errorReporting.ServiceContext{
			Service: sh.errorReportingServiceName,
			Version: labels["version"],
		},
		Context: &errorReporting.ErrorContext{
			User: labels["user"],
		},
	}
	// Assumes that caller stack frame information of type
	// github.com/facebookgo/stack.Frame has been added.
	// Possibly via a library like github.com/Gurpartap/logrus-stack
	if entry.Data["caller"] != nil {
		caller := entry.Data["caller"].(stack.Frame)
		errorEvent.Context.ReportLocation = &errorReporting.SourceLocation{
			FilePath:     caller.File,
			FunctionName: caller.Name,
			LineNumber:   int64(caller.Line),
		}
	}
	if httpReq != nil {
		errRepHttpRequest := &errorReporting.HttpRequestContext{
			Method:    httpReq.RequestMethod,
			Referrer:  httpReq.Referer,
			RemoteIp:  httpReq.RemoteIp,
			Url:       httpReq.RequestUrl,
			UserAgent: httpReq.UserAgent,
		}
		errorEvent.Context.HttpRequest = errRepHttpRequest
	}
	return errorEvent
}
<|endoftext|>


<commit_before>// Copyright 2016 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package mocktikv

import (
	"bytes"
	"sync"

	"github.com/petar/GoLLRB/llrb"
	"github.com/pingcap/kvproto/pkg/kvrpcpb"
)

type mvccValue struct {
	startTS  uint64
	commitTS uint64
	value    []byte
}

type mvccLock struct {
	startTS uint64
	primary []byte
	value   []byte
	op      kvrpcpb.Op
}

type mvccEntry struct {
	key    []byte
	values []mvccValue
	lock   *mvccLock
}

func newEntry(key []byte) *mvccEntry {
	return &mvccEntry{
		key: key,
	}
}

func (e *mvccEntry) Less(than llrb.Item) bool {
	return bytes.Compare(e.key, than.(*mvccEntry).key) < 0
}

func (e *mvccEntry) lockErr() error {
	return &ErrLocked{
		Key:     e.key,
		Primary: e.lock.primary,
		StartTS: e.lock.startTS,
	}
}

func (e *mvccEntry) Get(ts uint64) ([]byte, error) {
	if e.lock != nil {
		if e.lock.startTS <= ts {
			return nil, e.lockErr()
		}
	}
	for _, v := range e.values {
		if v.commitTS <= ts {
			return v.value, nil
		}
	}
	return nil, nil
}

func (e *mvccEntry) Prewrite(mutation *kvrpcpb.Mutation, startTS uint64, primary []byte) error {
	if len(e.values) > 0 {
		if e.values[0].commitTS >= startTS {
			return ErrRetryable("write conflict")
		}
	}
	if e.lock != nil {
		if e.lock.startTS != startTS {
			return e.lockErr()
		}
		return nil
	}
	e.lock = &mvccLock{
		startTS: startTS,
		primary: primary,
		value:   mutation.Value,
		op:      mutation.GetOp(),
	}
	return nil
}

func (e *mvccEntry) checkTxnCommitted(startTS uint64) (uint64, bool) {
	for _, v := range e.values {
		if v.startTS == startTS {
			return v.commitTS, true
		}
	}
	return 0, false
}

func (e *mvccEntry) Commit(startTS, commitTS uint64) error {
	if e.lock == nil || e.lock.startTS != startTS {
		if _, ok := e.checkTxnCommitted(startTS); ok {
			return nil
		}
		return ErrRetryable("txn not found")
	}
	if e.lock.op != kvrpcpb.Op_Lock {
		e.values = append([]mvccValue{{
			startTS:  startTS,
			commitTS: commitTS,
			value:    e.lock.value,
		}}, e.values...)
	}
	e.lock = nil
	return nil
}

func (e *mvccEntry) Rollback(startTS uint64) error {
	if e.lock == nil || e.lock.startTS != startTS {
		if commitTS, ok := e.checkTxnCommitted(startTS); ok {
			return ErrAlreadyCommitted(commitTS)
		}
		return nil
	}
	e.lock = nil
	return nil
}

// MvccStore is an in-memory, multi-versioned, transaction-supported kv storage.
type MvccStore struct {
	mu   sync.RWMutex
	tree *llrb.LLRB
}

// NewMvccStore creates a MvccStore.
func NewMvccStore() *MvccStore {
	return &MvccStore{
		tree: llrb.New(),
	}
}

// Get reads a key by ts.
func (s *MvccStore) Get(key []byte, startTS uint64) ([]byte, error) {
	s.mu.RLock()
	defer s.mu.RUnlock()

	return s.get(key, startTS)
}

func (s *MvccStore) get(key []byte, startTS uint64) ([]byte, error) {
	entry := s.tree.Get(newEntry(key))
	if entry == nil {
		return nil, nil
	}
	return entry.(*mvccEntry).Get(startTS)
}

// A Pair is a KV pair read from MvccStore or an error if any occurs.
type Pair struct {
	Key   []byte
	Value []byte
	Err   error
}

// BatchGet gets values with keys and ts.
func (s *MvccStore) BatchGet(ks [][]byte, startTS uint64) []Pair {
	s.mu.RLock()
	defer s.mu.RUnlock()

	var pairs []Pair
	for _, k := range ks {
		val, err := s.get(k, startTS)
		if val == nil && err == nil {
			continue
		}
		pairs = append(pairs, Pair{
			Key:   k,
			Value: val,
			Err:   err,
		})
	}
	return pairs
}

// Scan reads up to limit numbers of Pairs from a key.
func (s *MvccStore) Scan(startKey, endKey []byte, limit int, startTS uint64) []Pair {
	s.mu.RLock()
	defer s.mu.RUnlock()

	var pairs []Pair
	iterator := func(item llrb.Item) bool {
		if len(pairs) >= limit {
			return false
		}
		k := item.(*mvccEntry).key
		if !regionContains(startKey, endKey, k) {
			return false
		}
		val, err := s.Get(k, startTS)
		if val != nil || err != nil {
			pairs = append(pairs, Pair{
				Key:   k,
				Value: val,
				Err:   err,
			})
		}
		return true
	}
	s.tree.AscendGreaterOrEqual(newEntry(startKey), iterator)
	return pairs
}

func (s *MvccStore) getOrNewEntry(key []byte) *mvccEntry {
	if item := s.tree.Get(newEntry(key)); item != nil {
		return item.(*mvccEntry)
	}
	return newEntry(key)
}

// submit writes entries into the rbtree.
func (s *MvccStore) submit(ents ...*mvccEntry) {
	for _, ent := range ents {
		s.tree.ReplaceOrInsert(ent)
	}
}

// Prewrite acquires a lock on a key. (1st phase of 2PC).
func (s *MvccStore) Prewrite(mutations []*kvrpcpb.Mutation, primary []byte, startTS uint64) []error {
	s.mu.Lock()
	defer s.mu.Unlock()

	var errs []error
	for _, m := range mutations {
		entry := s.getOrNewEntry(m.Key)
		err := entry.Prewrite(m, startTS, primary)
		s.submit(entry)
		errs = append(errs, err)
	}
	return errs
}

// Commit commits the lock on a key. (2nd phase of 2PC).
func (s *MvccStore) Commit(keys [][]byte, startTS, commitTS uint64) error {
	s.mu.Lock()
	defer s.mu.Unlock()

	var ents []*mvccEntry
	for _, k := range keys {
		entry := s.getOrNewEntry(k)
		err := entry.Commit(startTS, commitTS)
		if err != nil {
			return err
		}
		ents = append(ents, entry)
	}
	s.submit(ents...)
	return nil
}

// CommitThenGet is a shortcut for Commit+Get, often used when resolving lock.
func (s *MvccStore) CommitThenGet(key []byte, lockTS, commitTS, getTS uint64) ([]byte, error) {
	s.mu.Lock()
	defer s.mu.Unlock()

	entry := s.getOrNewEntry(key)
	err := entry.Commit(lockTS, commitTS)
	if err != nil {
		return nil, err
	}
	s.submit(entry)
	return entry.Get(getTS)
}

// Cleanup cleanups a lock, often used when resolving a expired lock.
func (s *MvccStore) Cleanup(key []byte, startTS uint64) error {
	s.mu.Lock()
	defer s.mu.Unlock()

	entry := s.getOrNewEntry(key)
	err := entry.Rollback(startTS)
	if err != nil {
		return err
	}
	s.submit(entry)
	return nil
}

// Rollback cleanups multiple locks, often used when rolling back a conflict txn.
func (s *MvccStore) Rollback(keys [][]byte, startTS uint64) error {
	s.mu.Lock()
	defer s.mu.Unlock()

	var ents []*mvccEntry
	for _, k := range keys {
		entry := s.getOrNewEntry(k)
		err := entry.Rollback(startTS)
		if err != nil {
			return err
		}
		ents = append(ents, entry)
	}
	s.submit(ents...)
	return nil
}

// RollbackThenGet is a shortcut for Rollback+Get, often used when resolving lock.
func (s *MvccStore) RollbackThenGet(key []byte, lockTS uint64) ([]byte, error) {
	s.mu.Lock()
	defer s.mu.Unlock()

	entry := s.getOrNewEntry(key)
	err := entry.Rollback(lockTS)
	if err != nil {
		return nil, err
	}
	s.submit(entry)
	return entry.Get(lockTS)
}
<commit_msg>store/tikv: add missed function to pass test. (#1185)<commit_after>// Copyright 2016 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package mocktikv

import (
	"bytes"
	"sync"

	"github.com/petar/GoLLRB/llrb"
	"github.com/pingcap/kvproto/pkg/kvrpcpb"
)

type mvccValue struct {
	startTS  uint64
	commitTS uint64
	value    []byte
}

type mvccLock struct {
	startTS uint64
	primary []byte
	value   []byte
	op      kvrpcpb.Op
}

type mvccEntry struct {
	key    []byte
	values []mvccValue
	lock   *mvccLock
}

func newEntry(key []byte) *mvccEntry {
	return &mvccEntry{
		key: key,
	}
}

func (e *mvccEntry) Less(than llrb.Item) bool {
	return bytes.Compare(e.key, than.(*mvccEntry).key) < 0
}

func (e *mvccEntry) lockErr() error {
	return &ErrLocked{
		Key:     e.key,
		Primary: e.lock.primary,
		StartTS: e.lock.startTS,
	}
}

func (e *mvccEntry) Get(ts uint64) ([]byte, error) {
	if e.lock != nil {
		if e.lock.startTS <= ts {
			return nil, e.lockErr()
		}
	}
	for _, v := range e.values {
		if v.commitTS <= ts {
			return v.value, nil
		}
	}
	return nil, nil
}

func (e *mvccEntry) Prewrite(mutation *kvrpcpb.Mutation, startTS uint64, primary []byte) error {
	if len(e.values) > 0 {
		if e.values[0].commitTS >= startTS {
			return ErrRetryable("write conflict")
		}
	}
	if e.lock != nil {
		if e.lock.startTS != startTS {
			return e.lockErr()
		}
		return nil
	}
	e.lock = &mvccLock{
		startTS: startTS,
		primary: primary,
		value:   mutation.Value,
		op:      mutation.GetOp(),
	}
	return nil
}

func (e *mvccEntry) checkTxnCommitted(startTS uint64) (uint64, bool) {
	for _, v := range e.values {
		if v.startTS == startTS {
			return v.commitTS, true
		}
	}
	return 0, false
}

func (e *mvccEntry) Commit(startTS, commitTS uint64) error {
	if e.lock == nil || e.lock.startTS != startTS {
		if _, ok := e.checkTxnCommitted(startTS); ok {
			return nil
		}
		return ErrRetryable("txn not found")
	}
	if e.lock.op != kvrpcpb.Op_Lock {
		e.values = append([]mvccValue{{
			startTS:  startTS,
			commitTS: commitTS,
			value:    e.lock.value,
		}}, e.values...)
	}
	e.lock = nil
	return nil
}

func (e *mvccEntry) Rollback(startTS uint64) error {
	if e.lock == nil || e.lock.startTS != startTS {
		if commitTS, ok := e.checkTxnCommitted(startTS); ok {
			return ErrAlreadyCommitted(commitTS)
		}
		return nil
	}
	e.lock = nil
	return nil
}

// MvccStore is an in-memory, multi-versioned, transaction-supported kv storage.
type MvccStore struct {
	mu   sync.RWMutex
	tree *llrb.LLRB
}

// NewMvccStore creates a MvccStore.
func NewMvccStore() *MvccStore {
	return &MvccStore{
		tree: llrb.New(),
	}
}

// Get reads a key by ts.
func (s *MvccStore) Get(key []byte, startTS uint64) ([]byte, error) {
	s.mu.RLock()
	defer s.mu.RUnlock()

	return s.get(key, startTS)
}

func (s *MvccStore) get(key []byte, startTS uint64) ([]byte, error) {
	entry := s.tree.Get(newEntry(key))
	if entry == nil {
		return nil, nil
	}
	return entry.(*mvccEntry).Get(startTS)
}

// A Pair is a KV pair read from MvccStore or an error if any occurs.
type Pair struct {
	Key   []byte
	Value []byte
	Err   error
}

// BatchGet gets values with keys and ts.
func (s *MvccStore) BatchGet(ks [][]byte, startTS uint64) []Pair {
	s.mu.RLock()
	defer s.mu.RUnlock()

	var pairs []Pair
	for _, k := range ks {
		val, err := s.get(k, startTS)
		if val == nil && err == nil {
			continue
		}
		pairs = append(pairs, Pair{
			Key:   k,
			Value: val,
			Err:   err,
		})
	}
	return pairs
}

func regionContains(startKey []byte, endKey []byte, key []byte) bool {
	return bytes.Compare(startKey, key) <= 0 &&
		(bytes.Compare(key, endKey) < 0 || len(endKey) == 0)
}

// Scan reads up to limit numbers of Pairs from a key.
func (s *MvccStore) Scan(startKey, endKey []byte, limit int, startTS uint64) []Pair {
	s.mu.RLock()
	defer s.mu.RUnlock()

	var pairs []Pair
	iterator := func(item llrb.Item) bool {
		if len(pairs) >= limit {
			return false
		}
		k := item.(*mvccEntry).key
		if !regionContains(startKey, endKey, k) {
			return false
		}
		val, err := s.Get(k, startTS)
		if val != nil || err != nil {
			pairs = append(pairs, Pair{
				Key:   k,
				Value: val,
				Err:   err,
			})
		}
		return true
	}
	s.tree.AscendGreaterOrEqual(newEntry(startKey), iterator)
	return pairs
}

func (s *MvccStore) getOrNewEntry(key []byte) *mvccEntry {
	if item := s.tree.Get(newEntry(key)); item != nil {
		return item.(*mvccEntry)
	}
	return newEntry(key)
}

// submit writes entries into the rbtree.
func (s *MvccStore) submit(ents ...*mvccEntry) {
	for _, ent := range ents {
		s.tree.ReplaceOrInsert(ent)
	}
}

// Prewrite acquires a lock on a key. (1st phase of 2PC).
func (s *MvccStore) Prewrite(mutations []*kvrpcpb.Mutation, primary []byte, startTS uint64) []error {
	s.mu.Lock()
	defer s.mu.Unlock()

	var errs []error
	for _, m := range mutations {
		entry := s.getOrNewEntry(m.Key)
		err := entry.Prewrite(m, startTS, primary)
		s.submit(entry)
		errs = append(errs, err)
	}
	return errs
}

// Commit commits the lock on a key. (2nd phase of 2PC).
func (s *MvccStore) Commit(keys [][]byte, startTS, commitTS uint64) error {
	s.mu.Lock()
	defer s.mu.Unlock()

	var ents []*mvccEntry
	for _, k := range keys {
		entry := s.getOrNewEntry(k)
		err := entry.Commit(startTS, commitTS)
		if err != nil {
			return err
		}
		ents = append(ents, entry)
	}
	s.submit(ents...)
	return nil
}

// CommitThenGet is a shortcut for Commit+Get, often used when resolving lock.
func (s *MvccStore) CommitThenGet(key []byte, lockTS, commitTS, getTS uint64) ([]byte, error) {
	s.mu.Lock()
	defer s.mu.Unlock()

	entry := s.getOrNewEntry(key)
	err := entry.Commit(lockTS, commitTS)
	if err != nil {
		return nil, err
	}
	s.submit(entry)
	return entry.Get(getTS)
}

// Cleanup cleanups a lock, often used when resolving a expired lock.
func (s *MvccStore) Cleanup(key []byte, startTS uint64) error {
	s.mu.Lock()
	defer s.mu.Unlock()

	entry := s.getOrNewEntry(key)
	err := entry.Rollback(startTS)
	if err != nil {
		return err
	}
	s.submit(entry)
	return nil
}

// Rollback cleanups multiple locks, often used when rolling back a conflict txn.
func (s *MvccStore) Rollback(keys [][]byte, startTS uint64) error {
	s.mu.Lock()
	defer s.mu.Unlock()

	var ents []*mvccEntry
	for _, k := range keys {
		entry := s.getOrNewEntry(k)
		err := entry.Rollback(startTS)
		if err != nil {
			return err
		}
		ents = append(ents, entry)
	}
	s.submit(ents...)
	return nil
}

// RollbackThenGet is a shortcut for Rollback+Get, often used when resolving lock.
func (s *MvccStore) RollbackThenGet(key []byte, lockTS uint64) ([]byte, error) {
	s.mu.Lock()
	defer s.mu.Unlock()

	entry := s.getOrNewEntry(key)
	err := entry.Rollback(lockTS)
	if err != nil {
		return nil, err
	}
	s.submit(entry)
	return entry.Get(lockTS)
}
<|endoftext|>


<commit_before>// Copyright 2016 Ross Light
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"net/http"

	"golang.org/x/text/language"
	textsearch "golang.org/x/text/search"

	"zombiezen.com/go/sandpass/pkg/keepass"
)

func handleSearch(w http.ResponseWriter, r *http.Request) error {
	mu.Lock()
	defer mu.Unlock()
	db, err := sessions.dbFromRequest(w, r)
	if err != nil {
		return err
	}
	var data struct {
		Query   string
		Results []*keepass.Entry
	}
	data.Query = r.FormValue("q")
	if data.Query != "" {
		data.Results = search(db, data.Query)
	}
	return tmpl.ExecuteTemplate(w, "search.html", data)
}

func search(db *keepass.Database, query string) []*keepass.Entry {
	pat := textsearch.New(language.Und, textsearch.Loose).CompileString(query)
	var results []*keepass.Entry
	for _, e := range db.Entries() {
		if start, _ := pat.IndexString(e.Title); start != -1 {
			results = append(results, e)
		}
	}
	return results
}
<commit_msg>Allow multi-word searches<commit_after>// Copyright 2016 Ross Light
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"net/http"
	"unicode"

	"golang.org/x/text/language"
	textsearch "golang.org/x/text/search"

	"zombiezen.com/go/sandpass/pkg/keepass"
)

func handleSearch(w http.ResponseWriter, r *http.Request) error {
	mu.Lock()
	defer mu.Unlock()
	db, err := sessions.dbFromRequest(w, r)
	if err != nil {
		return err
	}
	var data struct {
		Query   string
		Results []*keepass.Entry
	}
	data.Query = r.FormValue("q")
	pq := parseQuery(data.Query)
	if pq == nil {
		data.Query = ""
	} else {
		data.Results = search(db, pq)
	}
	return tmpl.ExecuteTemplate(w, "search.html", data)
}

func search(db *keepass.Database, q *parsedQuery) []*keepass.Entry {
	var results []*keepass.Entry
	for _, e := range db.Entries() {
		if q.matchesText(e.Title) {
			results = append(results, e)
		}
	}
	return results
}

type parsedQuery struct {
	pats []*textsearch.Pattern
}

func parseQuery(query string) *parsedQuery {
	if len(query) == 0 {
		return nil
	}
	var words []string
	start := -1
	for i, r := range query {
		space := unicode.IsSpace(r)
		if space && start != -1 {
			words = append(words, query[start:i])
			start = -1
		} else if !space && start == -1 {
			start = i
		}
	}
	if start != -1 {
		words = append(words, query[start:])
	}
	if len(words) == 0 {
		return nil
	}
	m := textsearch.New(language.Und, textsearch.Loose)
	if len(words) == 1 {
		return &parsedQuery{pats: []*textsearch.Pattern{m.CompileString(query)}}
	}
	pq := &parsedQuery{pats: make([]*textsearch.Pattern, len(words))}
	for i := range words {
		pq.pats[i] = m.CompileString(words[i])
	}
	return pq
}

func (pq *parsedQuery) matchesText(s string) bool {
	if pq == nil || len(pq.pats) == 0 {
		return false
	}
	for _, pat := range pq.pats {
		if start, _ := pat.IndexString(s); start == -1 {
			return false
		}
	}
	return true
}
<|endoftext|>


<commit_before>package guetzli_patapon

import "math"

const kButteraugliStep = 3

type ButteraugliComparator struct {
	width_               int
	height_              int
	target_distance_     float32
	rgb_orig_            []byte
	block_x_, block_y_   int
	factor_x_, factor_y_ int
	rgb_linear_pregamma_ [][]float32
	mask_xyz_            [][]float32
	per_block_pregamma_  [][][]float32
	comparator_          *ButteraugliButteraugliComparator
	distance_            float32
	distmap_             []float32
	stats_               *ProcessStats
}

func NewButteraugliComparator(width, height int,
	rgb []byte,
	target_distance float32,
	stats *ProcessStats) *ButteraugliComparator {
	bc := new(ButteraugliComparator)
	bc.width_ = width
	bc.height_ = height
	bc.target_distance_ = target_distance
	bc.rgb_orig_ = rgb
	bc.rgb_linear_pregamma_ = [][]float32{
		make([]float32, bc.width_*bc.height_),
		make([]float32, bc.width_*bc.height_),
		make([]float32, bc.width_*bc.height_),
	}
	bc.comparator_ = NewButteraugliButteraugliComparator(bc.width_, bc.height_, kButteraugliStep)
	bc.distance_ = 0.0
	bc.distmap_ = make([]float32, bc.width_)
	for i := range bc.distmap_ {
		bc.distmap_[i] = float32(bc.height_)
	}
	bc.stats_ = stats
	lut := Srgb8ToLinearTable
	for c := 0; c < 3; c++ {
		for y, ix := 0, 0; y < bc.height_; y++ {
			for x := 0; x < bc.width_; x, ix = x+1, ix+1 {
				bc.rgb_linear_pregamma_[c][ix] = float32(lut[bc.rgb_orig_[3*ix+c]])
			}
		}
	}
	OpsinDynamicsImage(bc.width_, bc.height_, bc.rgb_linear_pregamma_)
	return bc
}

func (bc *ButteraugliComparator) Compare(img OutputImage) {
	rgb := [][]float32{
		make([]float32, bc.width_*bc.height_),
		make([]float32, bc.width_*bc.height_),
		make([]float32, bc.width_*bc.height_),
	}
	img.ToLinearRGB_(rgb)
	OpsinDynamicsImage(bc.width_, bc.height_, rgb)
	bc.distmap_ = bc.comparator_.DiffmapOpsinDynamicsImage(bc.rgb_linear_pregamma_, rgb)
	bc.distance_ = float32(ButteraugliScoreFromDiffmap(bc.distmap_))
	GUETZLI_LOG(bc.stats_, " BA[100.00%%] D[%6.4f]", bc.distance_)
}

func (bc *ButteraugliComparator) StartBlockComparisons() {
	Mask(bc.rgb_linear_pregamma_, bc.rgb_linear_pregamma_,
		bc.width_, bc.height_,
		bc.mask_xyz_, nil)
}

func (bc *ButteraugliComparator) FinishBlockComparisons() {
	bc.mask_xyz_ = nil
}

func (bc *ButteraugliComparator) SwitchBlock(block_x, block_y, factor_x, factor_y int) {
	bc.block_x_ = block_x
	bc.block_y_ = block_y
	bc.factor_x_ = factor_x
	bc.factor_y_ = factor_y
	bc.per_block_pregamma_ = make([][][]float32, bc.factor_x_*bc.factor_y_)
	lut := Srgb8ToLinearTable
	for off_y, bx := 0, 0; off_y < bc.factor_y_; off_y++ {
		for off_x := 0; off_x < bc.factor_x_; off_x, bx = off_x+1, bx+1 {
			bc.per_block_pregamma_[bx] = [][]float32{
				make([]float32, kDCTBlockSize),
				make([]float32, kDCTBlockSize),
				make([]float32, kDCTBlockSize),
			}
			block_xx := bc.block_x_*bc.factor_x_ + off_x
			block_yy := bc.block_y_*bc.factor_y_ + off_y
			for iy, i := 0, 0; iy < 8; iy++ {
				for ix := 0; ix < 8; ix, i = ix+1, i+1 {
					x := std_min(8*block_xx+ix, bc.width_-1)
					y := std_min(8*block_yy+iy, bc.height_-1)
					px := y*bc.width_ + x
					for c := 0; c < 3; c++ {
						bc.per_block_pregamma_[bx][c][i] = float32(lut[bc.rgb_orig_[3*px+c]])
					}
				}
			}
			OpsinDynamicsImage(8, 8, bc.per_block_pregamma_[bx])
		}
	}
}

func (bc *ButteraugliComparator) CompareBlock(img *OutputImage, off_x, off_y int) float64 {
	block_x := bc.block_x_*bc.factor_x_ + off_x
	block_y := bc.block_y_*bc.factor_y_ + off_y
	xmin := 8 * block_x
	ymin := 8 * block_y
	block_ix := off_y*bc.factor_x_ + off_x
	rgb0_c := bc.per_block_pregamma_[block_ix]

	rgb1_c := make([][]float32, kDCTBlockSize)
	img.ToLinearRGB(xmin, ymin, 8, 8, rgb1_c)
	OpsinDynamicsImage(8, 8, rgb1_c)

	rgb0 := cloneMatrixFloat32(rgb0_c)
	rgb1 := cloneMatrixFloat32(rgb1_c)

	MaskHighIntensityChange(8, 8, rgb0_c, rgb1_c, rgb0, rgb1)

	var b0, b1 [3 * kDCTBlockSize]float64
	for c := 0; c < 3; c++ {
		for ix := 0; ix < kDCTBlockSize; ix++ {
			b0[c*kDCTBlockSize+ix] = float64(rgb0[c][ix])
			b1[c*kDCTBlockSize+ix] = float64(rgb1[c][ix])
		}
	}
	var diff_xyz_dc, diff_xyz_ac, diff_xyz_edge_dc [3]float64
	ButteraugliBlockDiff(b0[:], b1[:], diff_xyz_dc[:], diff_xyz_ac[:], diff_xyz_edge_dc[:])

	var scale [3]float64
	for c := 0; c < 3; c++ {
		scale[c] = float64(bc.mask_xyz_[c][ymin*bc.width_+xmin])
	}

	const kEdgeWeight = 0.05

	diff := 0.0
	diff_edge := 0.0
	for c := 0; c < 3; c++ {
		diff += diff_xyz_dc[c] * scale[c]
		diff += diff_xyz_ac[c] * scale[c]
		diff_edge += diff_xyz_edge_dc[c] * scale[c]
	}
	return math.Sqrt((1-kEdgeWeight)*diff + kEdgeWeight*diff_edge)
}

func (bc *ButteraugliComparator) BlockErrorLimit() float32 {
	return bc.target_distance_
}

func (bc *ButteraugliComparator) ComputeBlockErrorAdjustmentWeights(
	direction int,
	max_block_dist int,
	target_mul float64,
	factor_x, factor_y int,
	distmap []float32,
	block_weight []float32) {
	target_distance := float64(bc.target_distance_) * target_mul
	sizex := 8 * factor_x
	sizey := 8 * factor_y
	block_width := (bc.width_ + sizex - 1) / sizex
	block_height := (bc.height_ + sizey - 1) / sizey
	max_dist_per_block := make([]float32, block_width*block_height)
	for block_y := 0; block_y < block_height; block_y++ {
		for block_x := 0; block_x < block_width; block_x++ {
			block_ix := block_y*block_width + block_x
			x_max := std_min(bc.width_, sizex*(block_x+1))
			y_max := std_min(bc.height_, sizey*(block_y+1))
			max_dist := float32(0.0)
			for y := sizey * block_y; y < y_max; y++ {
				for x := sizex * block_x; x < x_max; x++ {
					max_dist = std_maxFloat32(max_dist, distmap[y*bc.width_+x])
				}
			}
			max_dist_per_block[block_ix] = max_dist
		}
	}
	for block_y := 0; block_y < block_height; block_y++ {
		for block_x := 0; block_x < block_width; block_x++ {
			block_ix := block_y*block_width + block_x
			max_local_dist := target_distance
			x_min := std_max(0, block_x-max_block_dist)
			y_min := std_max(0, block_y-max_block_dist)
			x_max := std_min(block_width, block_x+1+max_block_dist)
			y_max := std_min(block_height, block_y+1+max_block_dist)
			for y := y_min; y < y_max; y++ {
				for x := x_min; x < x_max; x++ {
					max_local_dist = std_maxFloat64(max_local_dist, float64(max_dist_per_block[y*block_width+x]))
				}
			}
			if direction > 0 {
				if float64(max_dist_per_block[block_ix]) <= target_distance && max_local_dist <= 1.1*target_distance {
					block_weight[block_ix] = 1.0
				}
			} else {
				const kLocalMaxWeight = 0.5
				if float64(max_dist_per_block[block_ix]) <= (1-kLocalMaxWeight)*target_distance+kLocalMaxWeight*max_local_dist {
					continue
				}
				for y := y_min; y < y_max; y++ {
					for x := x_min; x < x_max; x++ {
						d := std_max(std_abs(y-block_y), std_abs(x-block_x))
						ix := y*block_width + x
						block_weight[ix] = std_maxFloat32(block_weight[ix], 1/(float32(d)+1))
					}
				}
			}
		}
	}
}

func (bc *ButteraugliComparator) ScoreOutputSize(size int) float64 {
	return ScoreJPEG(float64(bc.distance_), size, float64(bc.target_distance_))
}
<commit_msg>(*ButteraugliComparator) implements Comparator.<commit_after>package guetzli_patapon

import "math"

const kButteraugliStep = 3

type ButteraugliComparator struct {
	width_               int
	height_              int
	target_distance_     float32
	rgb_orig_            []byte
	block_x_, block_y_   int
	factor_x_, factor_y_ int
	rgb_linear_pregamma_ [][]float32
	mask_xyz_            [][]float32
	per_block_pregamma_  [][][]float32
	comparator_          *ButteraugliButteraugliComparator
	distance_            float32
	distmap_             []float32
	stats_               *ProcessStats
}

func NewButteraugliComparator(width, height int,
	rgb []byte,
	target_distance float32,
	stats *ProcessStats) *ButteraugliComparator {
	bc := new(ButteraugliComparator)
	bc.width_ = width
	bc.height_ = height
	bc.target_distance_ = target_distance
	bc.rgb_orig_ = rgb
	bc.rgb_linear_pregamma_ = [][]float32{
		make([]float32, bc.width_*bc.height_),
		make([]float32, bc.width_*bc.height_),
		make([]float32, bc.width_*bc.height_),
	}
	bc.comparator_ = NewButteraugliButteraugliComparator(bc.width_, bc.height_, kButteraugliStep)
	bc.distance_ = 0.0
	bc.distmap_ = make([]float32, bc.width_)
	for i := range bc.distmap_ {
		bc.distmap_[i] = float32(bc.height_)
	}
	bc.stats_ = stats
	lut := Srgb8ToLinearTable
	for c := 0; c < 3; c++ {
		for y, ix := 0, 0; y < bc.height_; y++ {
			for x := 0; x < bc.width_; x, ix = x+1, ix+1 {
				bc.rgb_linear_pregamma_[c][ix] = float32(lut[bc.rgb_orig_[3*ix+c]])
			}
		}
	}
	OpsinDynamicsImage(bc.width_, bc.height_, bc.rgb_linear_pregamma_)
	return bc
}

func (bc *ButteraugliComparator) Compare(img *OutputImage) {
	rgb := [][]float32{
		make([]float32, bc.width_*bc.height_),
		make([]float32, bc.width_*bc.height_),
		make([]float32, bc.width_*bc.height_),
	}
	img.ToLinearRGB_(rgb)
	OpsinDynamicsImage(bc.width_, bc.height_, rgb)
	bc.distmap_ = bc.comparator_.DiffmapOpsinDynamicsImage(bc.rgb_linear_pregamma_, rgb)
	bc.distance_ = float32(ButteraugliScoreFromDiffmap(bc.distmap_))
	GUETZLI_LOG(bc.stats_, " BA[100.00%%] D[%6.4f]", bc.distance_)
}

func (bc *ButteraugliComparator) DistanceOK(target_mul float64) bool {
	return bc.distance_ <= float32(target_mul)*bc.target_distance_
}

func (bc *ButteraugliComparator) distmap() []float32 {
	return bc.distmap_
}

func (bc *ButteraugliComparator) distmap_aggregate() float32 {
	return bc.distance_
}

func (bc *ButteraugliComparator) StartBlockComparisons() {
	Mask(bc.rgb_linear_pregamma_, bc.rgb_linear_pregamma_,
		bc.width_, bc.height_,
		bc.mask_xyz_, nil)
}

func (bc *ButteraugliComparator) FinishBlockComparisons() {
	bc.mask_xyz_ = nil
}

func (bc *ButteraugliComparator) SwitchBlock(block_x, block_y, factor_x, factor_y int) {
	bc.block_x_ = block_x
	bc.block_y_ = block_y
	bc.factor_x_ = factor_x
	bc.factor_y_ = factor_y
	bc.per_block_pregamma_ = make([][][]float32, bc.factor_x_*bc.factor_y_)
	lut := Srgb8ToLinearTable
	for off_y, bx := 0, 0; off_y < bc.factor_y_; off_y++ {
		for off_x := 0; off_x < bc.factor_x_; off_x, bx = off_x+1, bx+1 {
			bc.per_block_pregamma_[bx] = [][]float32{
				make([]float32, kDCTBlockSize),
				make([]float32, kDCTBlockSize),
				make([]float32, kDCTBlockSize),
			}
			block_xx := bc.block_x_*bc.factor_x_ + off_x
			block_yy := bc.block_y_*bc.factor_y_ + off_y
			for iy, i := 0, 0; iy < 8; iy++ {
				for ix := 0; ix < 8; ix, i = ix+1, i+1 {
					x := std_min(8*block_xx+ix, bc.width_-1)
					y := std_min(8*block_yy+iy, bc.height_-1)
					px := y*bc.width_ + x
					for c := 0; c < 3; c++ {
						bc.per_block_pregamma_[bx][c][i] = float32(lut[bc.rgb_orig_[3*px+c]])
					}
				}
			}
			OpsinDynamicsImage(8, 8, bc.per_block_pregamma_[bx])
		}
	}
}

func (bc *ButteraugliComparator) CompareBlock(img *OutputImage, off_x, off_y int) float64 {
	block_x := bc.block_x_*bc.factor_x_ + off_x
	block_y := bc.block_y_*bc.factor_y_ + off_y
	xmin := 8 * block_x
	ymin := 8 * block_y
	block_ix := off_y*bc.factor_x_ + off_x
	rgb0_c := bc.per_block_pregamma_[block_ix]

	rgb1_c := make([][]float32, kDCTBlockSize)
	img.ToLinearRGB(xmin, ymin, 8, 8, rgb1_c)
	OpsinDynamicsImage(8, 8, rgb1_c)

	rgb0 := cloneMatrixFloat32(rgb0_c)
	rgb1 := cloneMatrixFloat32(rgb1_c)

	MaskHighIntensityChange(8, 8, rgb0_c, rgb1_c, rgb0, rgb1)

	var b0, b1 [3 * kDCTBlockSize]float64
	for c := 0; c < 3; c++ {
		for ix := 0; ix < kDCTBlockSize; ix++ {
			b0[c*kDCTBlockSize+ix] = float64(rgb0[c][ix])
			b1[c*kDCTBlockSize+ix] = float64(rgb1[c][ix])
		}
	}
	var diff_xyz_dc, diff_xyz_ac, diff_xyz_edge_dc [3]float64
	ButteraugliBlockDiff(b0[:], b1[:], diff_xyz_dc[:], diff_xyz_ac[:], diff_xyz_edge_dc[:])

	var scale [3]float64
	for c := 0; c < 3; c++ {
		scale[c] = float64(bc.mask_xyz_[c][ymin*bc.width_+xmin])
	}

	const kEdgeWeight = 0.05

	diff := 0.0
	diff_edge := 0.0
	for c := 0; c < 3; c++ {
		diff += diff_xyz_dc[c] * scale[c]
		diff += diff_xyz_ac[c] * scale[c]
		diff_edge += diff_xyz_edge_dc[c] * scale[c]
	}
	return math.Sqrt((1-kEdgeWeight)*diff + kEdgeWeight*diff_edge)
}

func (bc *ButteraugliComparator) BlockErrorLimit() float32 {
	return bc.target_distance_
}

func (bc *ButteraugliComparator) ComputeBlockErrorAdjustmentWeights(
	direction int,
	max_block_dist int,
	target_mul float64,
	factor_x, factor_y int,
	distmap []float32,
	block_weight []float32) {
	target_distance := float64(bc.target_distance_) * target_mul
	sizex := 8 * factor_x
	sizey := 8 * factor_y
	block_width := (bc.width_ + sizex - 1) / sizex
	block_height := (bc.height_ + sizey - 1) / sizey
	max_dist_per_block := make([]float32, block_width*block_height)
	for block_y := 0; block_y < block_height; block_y++ {
		for block_x := 0; block_x < block_width; block_x++ {
			block_ix := block_y*block_width + block_x
			x_max := std_min(bc.width_, sizex*(block_x+1))
			y_max := std_min(bc.height_, sizey*(block_y+1))
			max_dist := float32(0.0)
			for y := sizey * block_y; y < y_max; y++ {
				for x := sizex * block_x; x < x_max; x++ {
					max_dist = std_maxFloat32(max_dist, distmap[y*bc.width_+x])
				}
			}
			max_dist_per_block[block_ix] = max_dist
		}
	}
	for block_y := 0; block_y < block_height; block_y++ {
		for block_x := 0; block_x < block_width; block_x++ {
			block_ix := block_y*block_width + block_x
			max_local_dist := target_distance
			x_min := std_max(0, block_x-max_block_dist)
			y_min := std_max(0, block_y-max_block_dist)
			x_max := std_min(block_width, block_x+1+max_block_dist)
			y_max := std_min(block_height, block_y+1+max_block_dist)
			for y := y_min; y < y_max; y++ {
				for x := x_min; x < x_max; x++ {
					max_local_dist = std_maxFloat64(max_local_dist, float64(max_dist_per_block[y*block_width+x]))
				}
			}
			if direction > 0 {
				if float64(max_dist_per_block[block_ix]) <= target_distance && max_local_dist <= 1.1*target_distance {
					block_weight[block_ix] = 1.0
				}
			} else {
				const kLocalMaxWeight = 0.5
				if float64(max_dist_per_block[block_ix]) <= (1-kLocalMaxWeight)*target_distance+kLocalMaxWeight*max_local_dist {
					continue
				}
				for y := y_min; y < y_max; y++ {
					for x := x_min; x < x_max; x++ {
						d := std_max(std_abs(y-block_y), std_abs(x-block_x))
						ix := y*block_width + x
						block_weight[ix] = std_maxFloat32(block_weight[ix], 1/(float32(d)+1))
					}
				}
			}
		}
	}
}

func (bc *ButteraugliComparator) ScoreOutputSize(size int) float64 {
	return ScoreJPEG(float64(bc.distance_), size, float64(bc.target_distance_))
}
<|endoftext|>


<commit_before>package slackapi

// SearchArgs defines the data to send to the API service.
type SearchArgs struct {
	Query     string `json:"query"`
	Count     int    `json:"count"`
	Highlight bool   `json:"highlight"`
	Page      int    `json:"page"`
	Sort      string `json:"sort"`
	SortDir   string `json:"sort_dir"`
}

type SearchUsersArgs struct {
	Query               string `json:"query"`
	Count               int    `json:"count"`
	Fuzz                int    `json:"fuzz"`
	UAX29Tokenizer      bool   `json:"uax29_tokenizer"`
	SearchProfileFields bool   `json:"search_profile_fields"`
}

type ResponseSearch struct {
	Response
	Query    string         `json:"query"`
	Files    SearchFiles    `json:"files"`
	Posts    SearchPosts    `json:"posts"`
	Messages SearchMessages `json:"messages"`
}

type ResponseSearchUsers struct {
	Response
	Results           []User   `json:"results"`
	PresenceActiveIds []string `json:"presence_active_ids"`
}

type SearchMessages struct {
	Matches    []SearchMatches `json:"matches"`
	Pagination Pagination      `json:"pagination"`
	Paging     Paging          `json:"paging"`
	Total      int             `json:"total"`
}

type SearchFiles struct {
	Matches    []File     `json:"matches"`
	Pagination Pagination `json:"pagination"`
	Paging     Paging     `json:"paging"`
	Total      int        `json:"total"`
}

type SearchPosts struct {
	Matches    []interface{} `json:"matches"`
	Pagination Pagination    `json:"pagination"`
	Paging     Paging        `json:"paging"`
	Total      int           `json:"total"`
}

type SearchMatches struct {
	// TODO(cixtor): find a way to convert this into a Message. Currently, the
	// problem is that Channel is not a string, which is what we get from the
	// API when we get a Message.
	Attachments []Attachment  `json:"attachments"`
	Channel     SearchChannel `json:"channel"`
	IID         string        `json:"iid"`
	Permalink   string        `json:"permalink"`
	Team        string        `json:"team"`
	Text        string        `json:"text"`
	Timestamp   string        `json:"ts"`
	Type        string        `json:"type"`
	User        string        `json:"user"`
	Username    string        `json:"username"`
}

type SearchChannel struct {
	ID                 string        `json:"id"`
	IsChannel          bool          `json:"is_channel"`
	IsExtShared        bool          `json:"is_ext_shared"`
	IsGroup            bool          `json:"is_group"`
	IsIM               bool          `json:"is_im"`
	IsMpim             bool          `json:"is_mpim"`
	IsOrgShared        bool          `json:"is_org_shared"`
	IsPendingExtShared bool          `json:"is_pending_ext_shared"`
	IsPrivate          bool          `json:"is_private"`
	IsShared           bool          `json:"is_shared"`
	Name               string        `json:"name"`
	PendingShared      []interface{} `json:"pending_shared"`
}

func (s *SlackAPI) searchStuff(action string, data SearchArgs) ResponseSearch {
	// set default value for optional field.
	if data.Sort != "timestamp" {
		data.Sort = "score"
	}

	// set default value for optional field.
	if data.SortDir != "asc" {
		data.SortDir = "desc"
	}

	var response ResponseSearch
	s.getRequest(&response, action, data)
	return response
}

// SearchAll searches for messages and files matching a query.
func (s *SlackAPI) SearchAll(data SearchArgs) ResponseSearch {
	return s.searchStuff("search.all", data)
}

// SearchFiles searches for files matching a query.
func (s *SlackAPI) SearchFiles(data SearchArgs) ResponseSearch {
	return s.searchStuff("search.files", data)
}

// SearchMessages searches for messages matching a query.
func (s *SlackAPI) SearchMessages(data SearchArgs) ResponseSearch {
	return s.searchStuff("search.messages", data)
}

type SearchModulesInput struct {
	ExcludeMyChannels    bool   `json:"exclude_my_channels"`
	ExtraMessageData     bool   `json:"extra_message_data"`
	Highlight            bool   `json:"highlight"`
	NoUserProfile        bool   `json:"no_user_profile"`
	Sort                 string `json:"sort"`
	SortDir              string `json:"sort_dir"`
	Browse               string `json:"browse"`
	BrowseSessionID      string `json:"browse_session_id"`
	ClientReqID          string `json:"client_req_id"`
	Count                int    `json:"count"`
	MaxFilterSuggestions int    `json:"max_filter_suggestions"`
	Module               string `json:"module"`
	Page                 int    `json:"page"`
	ChannelType          string `json:"channel_type"`
	Extracts             int    `json:"extracts"`
	Query                string `json:"query"`
	Team                 string `json:"team"`
}

type SearchModulesResponse struct {
	Response
	Module     string      `json:"module"`
	Filters    interface{} `json:"filters"`
	Query      string      `json:"query"`
	Pagination Pagination  `json:"pagination"`
	Items      []Channel   `json:"items"`
}

// SearchModules is https://api.slack.com/methods/search.modules
func (s *SlackAPI) SearchModules(input SearchModulesInput) SearchModulesResponse {
	var out SearchModulesResponse
	if err := s.basePOST("/api/search.modules", input, &out); err != nil {
		return SearchModulesResponse{Response: Response{Error: err.Error()}}
	}
	return out
}

// SearchUsers searches for users matching a query.
func (s *SlackAPI) SearchUsers(input SearchUsersArgs) (ResponseSearchUsers, error) {
	owner, err := s.AuthTest()

	if err != nil {
		return ResponseSearchUsers{}, err
	}

	if owner.TeamID == "" {
		return ResponseSearchUsers{Response: owner.Response}, nil
	}

	var response ResponseSearchUsers
	if err := s.edgePOST("/cache/"+owner.TeamID+"/users/search", input, &response); err != nil {
		return ResponseSearchUsers{}, err
	}
	return response, nil
}
<commit_msg>Fix search.modules API call to use GET instead of POST<commit_after>package slackapi

import (
	"net/url"
	"strconv"
)

// SearchArgs defines the data to send to the API service.
type SearchArgs struct {
	Query     string `json:"query"`
	Count     int    `json:"count"`
	Highlight bool   `json:"highlight"`
	Page      int    `json:"page"`
	Sort      string `json:"sort"`
	SortDir   string `json:"sort_dir"`
}

type SearchUsersArgs struct {
	Query               string `json:"query"`
	Count               int    `json:"count"`
	Fuzz                int    `json:"fuzz"`
	UAX29Tokenizer      bool   `json:"uax29_tokenizer"`
	SearchProfileFields bool   `json:"search_profile_fields"`
}

type ResponseSearch struct {
	Response
	Query    string         `json:"query"`
	Files    SearchFiles    `json:"files"`
	Posts    SearchPosts    `json:"posts"`
	Messages SearchMessages `json:"messages"`
}

type ResponseSearchUsers struct {
	Response
	Results           []User   `json:"results"`
	PresenceActiveIds []string `json:"presence_active_ids"`
}

type SearchMessages struct {
	Matches    []SearchMatches `json:"matches"`
	Pagination Pagination      `json:"pagination"`
	Paging     Paging          `json:"paging"`
	Total      int             `json:"total"`
}

type SearchFiles struct {
	Matches    []File     `json:"matches"`
	Pagination Pagination `json:"pagination"`
	Paging     Paging     `json:"paging"`
	Total      int        `json:"total"`
}

type SearchPosts struct {
	Matches    []interface{} `json:"matches"`
	Pagination Pagination    `json:"pagination"`
	Paging     Paging        `json:"paging"`
	Total      int           `json:"total"`
}

type SearchMatches struct {
	// TODO(cixtor): find a way to convert this into a Message. Currently, the
	// problem is that Channel is not a string, which is what we get from the
	// API when we get a Message.
	Attachments []Attachment  `json:"attachments"`
	Channel     SearchChannel `json:"channel"`
	IID         string        `json:"iid"`
	Permalink   string        `json:"permalink"`
	Team        string        `json:"team"`
	Text        string        `json:"text"`
	Timestamp   string        `json:"ts"`
	Type        string        `json:"type"`
	User        string        `json:"user"`
	Username    string        `json:"username"`
}

type SearchChannel struct {
	ID                 string        `json:"id"`
	IsChannel          bool          `json:"is_channel"`
	IsExtShared        bool          `json:"is_ext_shared"`
	IsGroup            bool          `json:"is_group"`
	IsIM               bool          `json:"is_im"`
	IsMpim             bool          `json:"is_mpim"`
	IsOrgShared        bool          `json:"is_org_shared"`
	IsPendingExtShared bool          `json:"is_pending_ext_shared"`
	IsPrivate          bool          `json:"is_private"`
	IsShared           bool          `json:"is_shared"`
	Name               string        `json:"name"`
	PendingShared      []interface{} `json:"pending_shared"`
}

func (s *SlackAPI) searchStuff(action string, data SearchArgs) ResponseSearch {
	// set default value for optional field.
	if data.Sort != "timestamp" {
		data.Sort = "score"
	}

	// set default value for optional field.
	if data.SortDir != "asc" {
		data.SortDir = "desc"
	}

	var response ResponseSearch
	s.getRequest(&response, action, data)
	return response
}

// SearchAll searches for messages and files matching a query.
func (s *SlackAPI) SearchAll(data SearchArgs) ResponseSearch {
	return s.searchStuff("search.all", data)
}

// SearchFiles searches for files matching a query.
func (s *SlackAPI) SearchFiles(data SearchArgs) ResponseSearch {
	return s.searchStuff("search.files", data)
}

// SearchMessages searches for messages matching a query.
func (s *SlackAPI) SearchMessages(data SearchArgs) ResponseSearch {
	return s.searchStuff("search.messages", data)
}

type SearchModulesInput struct {
	ExcludeMyChannels    bool   `json:"exclude_my_channels"`
	ExtraMessageData     bool   `json:"extra_message_data"`
	Highlight            bool   `json:"highlight"`
	NoUserProfile        bool   `json:"no_user_profile"`
	Sort                 string `json:"sort"`
	SortDir              string `json:"sort_dir"`
	Browse               string `json:"browse"`
	BrowseSessionID      string `json:"browse_session_id"`
	ClientReqID          string `json:"client_req_id"`
	Count                int    `json:"count"`
	MaxFilterSuggestions int    `json:"max_filter_suggestions"`
	Module               string `json:"module"`
	Page                 int    `json:"page"`
	ChannelType          string `json:"channel_type"`
	Extracts             int    `json:"extracts"`
	Query                string `json:"query"`
	Team                 string `json:"team"`
}

type SearchModulesResponse struct {
	Response
	Module     string      `json:"module"`
	Filters    interface{} `json:"filters"`
	Query      string      `json:"query"`
	Pagination Pagination  `json:"pagination"`
	Items      []Channel   `json:"items"`
}

// SearchModules is https://api.slack.com/methods/search.modules
func (s *SlackAPI) SearchModules(input SearchModulesInput) SearchModulesResponse {
	in := url.Values{}
	if input.ExcludeMyChannels {
		in.Add("exclude_my_channels", "true")
	}
	if input.ExtraMessageData {
		in.Add("extra_message_data", "true")
	}
	if input.Highlight {
		in.Add("highlight", "true")
	}
	if input.NoUserProfile {
		in.Add("no_user_profile", "true")
	}
	if input.Sort != "" {
		in.Add("sort", input.Sort)
	}
	if input.SortDir != "" {
		in.Add("sort_dir", input.SortDir)
	}
	if input.Browse != "" {
		in.Add("browse", input.Browse)
	}
	if input.BrowseSessionID != "" {
		in.Add("browse_session_id", input.BrowseSessionID)
	}
	if input.ClientReqID != "" {
		in.Add("client_req_id", input.ClientReqID)
	}
	if input.Count > 0 {
		in.Add("count", strconv.Itoa(input.Count))
	}
	if input.MaxFilterSuggestions > 0 {
		in.Add("max_filter_suggestions", strconv.Itoa(input.MaxFilterSuggestions))
	}
	if input.Module != "" {
		in.Add("module", input.Module)
	}
	if input.Page > 0 {
		in.Add("page", strconv.Itoa(input.Page))
	}
	if input.ChannelType != "" {
		in.Add("channel_type", input.ChannelType)
	}
	if input.Extracts > 0 {
		in.Add("extracts", strconv.Itoa(input.Extracts))
	}
	if input.Query != "" {
		in.Add("query", input.Query)
	}
	if input.Team != "" {
		in.Add("team", input.Team)
	}
	var out SearchModulesResponse
	if err := s.baseGET("/api/search.modules", in, &out); err != nil {
		return SearchModulesResponse{Response: Response{Error: err.Error()}}
	}
	return out
}

// SearchUsers searches for users matching a query.
func (s *SlackAPI) SearchUsers(input SearchUsersArgs) (ResponseSearchUsers, error) {
	owner, err := s.AuthTest()

	if err != nil {
		return ResponseSearchUsers{}, err
	}

	if owner.TeamID == "" {
		return ResponseSearchUsers{Response: owner.Response}, nil
	}

	var response ResponseSearchUsers
	if err := s.edgePOST("/cache/"+owner.TeamID+"/users/search", input, &response); err != nil {
		return ResponseSearchUsers{}, err
	}
	return response, nil
}
<|endoftext|>


<commit_before>//
// DISCLAIMER
//
// Copyright 2020 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// Author Tomasz Mielech
//
package test

import (
	"context"
	"github.com/arangodb/go-driver"
	"github.com/stretchr/testify/require"
	"io/ioutil"
	"net/http"
	"os"
	"testing"
	"time"
)

func getZipFile(url, path string) error {
	respZip, err := http.Get(url)
	if err != nil {
		return err
	}
	defer respZip.Body.Close()

	zipContent, err := ioutil.ReadAll(respZip.Body)
	if err != nil {
		return err
	}

	return ioutil.WriteFile(path, zipContent, 0644)
}

func TestFoxxItzpapalotlService(t *testing.T) {

	c := createClientFromEnv(t, true)
	if os.Getenv("TEST_CONNECTION") == "vst" {
		skipBelowVersion(c, "3.6", t)
	}

	zipFilePath := "/tmp/itzpapalotl-v1.2.0.zip"
	err := getZipFile("https://github.com/arangodb-foxx/demo-itzpapalotl/archive/v1.2.0.zip", zipFilePath)
	require.NoError(t, err)

	timeoutCtx, cancel := context.WithTimeout(context.Background(), time.Minute*30)
	mountName := "test"
	options := driver.FoxxCreateOptions{
		Mount: "/" + mountName,
	}
	err = c.Foxx().InstallFoxxService(timeoutCtx, zipFilePath, options)
	cancel()
	require.NoError(t, err)

	timeoutCtx, cancel = context.WithTimeout(context.Background(), time.Second*30)
	connection := c.Connection()
	req, err := connection.NewRequest("GET", "_db/_system/"+mountName+"/random")
	require.NoError(t, err)
	resp, err := connection.Do(timeoutCtx, req)
	require.NotNil(t, resp)
	result := make(map[string]interface{}, 0)
	resp.ParseBody("", &result)
	require.NoError(t, err)
	value, ok := result["name"]
	require.Equal(t, true, ok)
	require.NotEmpty(t, value)
	cancel()

	timeoutCtx, cancel = context.WithTimeout(context.Background(), time.Second*30)
	deleteOptions := driver.FoxxDeleteOptions{
		Mount:    "/" + mountName,
		Teardown: true,
	}
	err = c.Foxx().UninstallFoxxService(timeoutCtx, deleteOptions)
	cancel()
	require.NoError(t, err)
}
<commit_msg>Fix integration test for foxx services<commit_after>//
// DISCLAIMER
//
// Copyright 2020 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// Author Tomasz Mielech
//
package test

import (
	"context"
	"crypto/sha256"
	"fmt"
	"github.com/arangodb/go-driver"
	"github.com/stretchr/testify/require"
	"io/ioutil"
	"net/http"
	"os"
	"testing"
	"time"
)

func getZipFile(url, path string) (string, error) {
	respZip, err := http.Get(url)
	if err != nil {
		return "", err
	}
	defer respZip.Body.Close()

	zipContent, err := ioutil.ReadAll(respZip.Body)
	if err != nil {
		return "", err
	}

	sha256sum := fmt.Sprintf("%x", sha256.Sum256(zipContent))
	return sha256sum, ioutil.WriteFile(path, zipContent, 0644)
}

func TestFoxxItzpapalotlService(t *testing.T) {

	c := createClientFromEnv(t, true)
	if os.Getenv("TEST_CONNECTION") == "vst" {
		skipBelowVersion(c, "3.6", t)
	}

	attempt := 0
	zipFilePath := "/tmp/itzpapalotl-v1.2.0.zip"
	for attempt < 3 {
		sha256sum, err := getZipFile("https://github.com/arangodb-foxx/demo-itzpapalotl/archive/v1.2.0.zip", zipFilePath)
		require.NoError(t, err)

		if sha256sum == "86117db897efe86cbbd20236abba127a08c2bdabbcd63683567ee5e84115d83a" {
			break
		}
		attempt++
	}

	if attempt == 3 {
		require.FailNow(t, "checksum of zip file is invalid")
	}

	timeoutCtx, cancel := context.WithTimeout(context.Background(), time.Minute*30)
	mountName := "test"
	options := driver.FoxxCreateOptions{
		Mount: "/" + mountName,
	}
	err := c.Foxx().InstallFoxxService(timeoutCtx, zipFilePath, options)
	cancel()
	require.NoError(t, err)

	timeoutCtx, cancel = context.WithTimeout(context.Background(), time.Second*30)
	connection := c.Connection()
	req, err := connection.NewRequest("GET", "_db/_system/"+mountName+"/random")
	require.NoError(t, err)
	resp, err := connection.Do(timeoutCtx, req)
	require.NotNil(t, resp)
	result := make(map[string]interface{}, 0)
	resp.ParseBody("", &result)
	require.NoError(t, err)
	value, ok := result["name"]
	require.Equal(t, true, ok)
	require.NotEmpty(t, value)
	cancel()

	timeoutCtx, cancel = context.WithTimeout(context.Background(), time.Second*30)
	deleteOptions := driver.FoxxDeleteOptions{
		Mount:    "/" + mountName,
		Teardown: true,
	}
	err = c.Foxx().UninstallFoxxService(timeoutCtx, deleteOptions)
	cancel()
	require.NoError(t, err)
}
<|endoftext|>


<commit_before>package ssdp

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"net"
	"net/http"
	"regexp"
	"strconv"
	"time"
)

// Service is discovered service.
type Service struct {
	// Type is a property of "ST"
	Type string

	// USN is a property of "USN"
	USN string

	// Location is a property of "LOCATION"
	Location string

	// Server is a property of "SERVER"
	Server string

	rawHeader http.Header
	maxAge    *int
}

var rxMaxAge = regexp.MustCompile(`\bmax-age\s*=\s*(\d+)\b`)

func extractMaxAge(s string, value int) int {
	v := value
	if m := rxMaxAge.FindStringSubmatch(s); m != nil {
		i64, err := strconv.ParseInt(m[1], 10, 32)
		if err == nil {
			v = int(i64)
		}
	}
	return v
}

// MaxAge extracts "max-age" value from "CACHE-CONTROL" property.
func (s *Service) MaxAge() int {
	if s.maxAge == nil {
		s.maxAge = new(int)
		*s.maxAge = extractMaxAge(s.rawHeader.Get("CACHE-CONTROL"), -1)
	}
	return *s.maxAge
}

// Header returns all properties in response of search.
func (s *Service) Header() http.Header {
	return s.rawHeader
}

const (
	// All is a search type to search all services and devices.
	All = "ssdp:all"

	// RootDevice is a search type to search UPnP root devices.
	RootDevice = "upnp:rootdevice"
)

// Search searchs services by SSDP.
func Search(searchType string, waitSec int, localAddr string) ([]Service, error) {
	// dial multicast UDP packet.
	conn, err := multicastListen(localAddr)
	if err != nil {
		return nil, err
	}
	defer conn.Close()
	logf("search on %s", conn.LocalAddr().String())

	// send request.
	msg, err := buildSearch(ssdpAddrIPv4, searchType, waitSec)
	if err != nil {
		return nil, err
	}
	if _, err := conn.WriteTo(msg, ssdpAddrIPv4); err != nil {
		return nil, err
	}

	// wait response.
	var list []Service
	h := func(a net.Addr, d []byte) error {
		srv, err := parseService(a, d)
		if err != nil {
			logf("invalid search response from %s: %s", a.String(), err)
			return nil
		}
		list = append(list, *srv)
		logf("search response from %s: %s", a.String(), srv.USN)
		return nil
	}
	d := time.Second * time.Duration(waitSec)
	if err := conn.readPackets(d, h); err != nil {
		return nil, err
	}

	return list, err
}

func buildSearch(raddr net.Addr, searchType string, waitSec int) ([]byte, error) {
	b := new(bytes.Buffer)
	// FIXME: error should be checked.
	b.WriteString("M-SEARCH * HTTP/1.1\r\n")
	fmt.Fprintf(b, "HOST: %s\r\n", raddr.String())
	fmt.Fprintf(b, "MAN: %q\r\n", "ssdp:discover")
	fmt.Fprintf(b, "MX: %d\r\n", waitSec)
	fmt.Fprintf(b, "ST: %s\r\n", searchType)
	b.WriteString("\r\n")
	return b.Bytes(), nil
}

var (
	errWithoutHTTPPrefix = errors.New("without HTTP prefix")
)

func parseService(addr net.Addr, data []byte) (*Service, error) {
	if !bytes.HasPrefix(data, []byte("HTTP")) {
		return nil, errWithoutHTTPPrefix
	}
	resp, err := http.ReadResponse(bufio.NewReader(bytes.NewReader(data)), nil)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	return &Service{
		Type:      resp.Header.Get("ST"),
		USN:       resp.Header.Get("USN"),
		Location:  resp.Header.Get("LOCATION"),
		Server:    resp.Header.Get("SERVER"),
		rawHeader: resp.Header,
	}, nil
}
<commit_msg>Add a final newline to workaround buggy SSDP clients<commit_after>package ssdp

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"net"
	"net/http"
	"regexp"
	"strconv"
	"time"
)

// Service is discovered service.
type Service struct {
	// Type is a property of "ST"
	Type string

	// USN is a property of "USN"
	USN string

	// Location is a property of "LOCATION"
	Location string

	// Server is a property of "SERVER"
	Server string

	rawHeader http.Header
	maxAge    *int
}

var rxMaxAge = regexp.MustCompile(`\bmax-age\s*=\s*(\d+)\b`)

func extractMaxAge(s string, value int) int {
	v := value
	if m := rxMaxAge.FindStringSubmatch(s); m != nil {
		i64, err := strconv.ParseInt(m[1], 10, 32)
		if err == nil {
			v = int(i64)
		}
	}
	return v
}

// MaxAge extracts "max-age" value from "CACHE-CONTROL" property.
func (s *Service) MaxAge() int {
	if s.maxAge == nil {
		s.maxAge = new(int)
		*s.maxAge = extractMaxAge(s.rawHeader.Get("CACHE-CONTROL"), -1)
	}
	return *s.maxAge
}

// Header returns all properties in response of search.
func (s *Service) Header() http.Header {
	return s.rawHeader
}

const (
	// All is a search type to search all services and devices.
	All = "ssdp:all"

	// RootDevice is a search type to search UPnP root devices.
	RootDevice = "upnp:rootdevice"
)

// Search searchs services by SSDP.
func Search(searchType string, waitSec int, localAddr string) ([]Service, error) {
	// dial multicast UDP packet.
	conn, err := multicastListen(localAddr)
	if err != nil {
		return nil, err
	}
	defer conn.Close()
	logf("search on %s", conn.LocalAddr().String())

	// send request.
	msg, err := buildSearch(ssdpAddrIPv4, searchType, waitSec)
	if err != nil {
		return nil, err
	}
	if _, err := conn.WriteTo(msg, ssdpAddrIPv4); err != nil {
		return nil, err
	}

	// wait response.
	var list []Service
	h := func(a net.Addr, d []byte) error {
		srv, err := parseService(a, d)
		if err != nil {
			logf("invalid search response from %s: %s", a.String(), err)
			return nil
		}
		list = append(list, *srv)
		logf("search response from %s: %s", a.String(), srv.USN)
		return nil
	}
	d := time.Second * time.Duration(waitSec)
	if err := conn.readPackets(d, h); err != nil {
		return nil, err
	}

	return list, err
}

func buildSearch(raddr net.Addr, searchType string, waitSec int) ([]byte, error) {
	b := new(bytes.Buffer)
	// FIXME: error should be checked.
	b.WriteString("M-SEARCH * HTTP/1.1\r\n")
	fmt.Fprintf(b, "HOST: %s\r\n", raddr.String())
	fmt.Fprintf(b, "MAN: %q\r\n", "ssdp:discover")
	fmt.Fprintf(b, "MX: %d\r\n", waitSec)
	fmt.Fprintf(b, "ST: %s\r\n", searchType)
	b.WriteString("\r\n")
	return b.Bytes(), nil
}

var (
	errWithoutHTTPPrefix = errors.New("without HTTP prefix")
)

func parseService(addr net.Addr, data []byte) (*Service, error) {
	if !bytes.HasPrefix(data, []byte("HTTP")) {
		return nil, errWithoutHTTPPrefix
	}
	// Add newline to workaround buggy SSDP responses
	str := append(data, "\r\n"...)
	resp, err := http.ReadResponse(bufio.NewReader(bytes.NewReader(str)), nil)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	return &Service{
		Type:      resp.Header.Get("ST"),
		USN:       resp.Header.Get("USN"),
		Location:  resp.Header.Get("LOCATION"),
		Server:    resp.Header.Get("SERVER"),
		rawHeader: resp.Header,
	}, nil
}
<|endoftext|>


<commit_before>package apiserver_test

import (
	"bytes"
	"context"
	"io/ioutil"
	"os"
	"path/filepath"
	"reflect"
	"testing"
	"time"

	"github.com/nyaxt/otaru/apiserver"
	"github.com/nyaxt/otaru/cli"
	opath "github.com/nyaxt/otaru/cli/path"
	feapiserver "github.com/nyaxt/otaru/extra/fe/apiserver"
	"github.com/nyaxt/otaru/extra/fe/pb"
	"github.com/nyaxt/otaru/filesystem"
	"github.com/nyaxt/otaru/otaruapiserver"
	opb "github.com/nyaxt/otaru/pb"
	"github.com/nyaxt/otaru/testutils"
)

type mockFsBackend struct {
	closeC chan struct{}
	joinC  chan struct{}
	fs     *filesystem.FileSystem
}

func runMockFsBackend() *mockFsBackend {
	fs := testutils.TestFileSystem()

	m := &mockFsBackend{
		closeC: make(chan struct{}),
		joinC:  make(chan struct{}),
		fs:     fs,
	}

	go func() {
		if err := apiserver.Serve(
			apiserver.ListenAddr(testBeListenAddr),
			apiserver.X509KeyPair(certFile, keyFile),
			apiserver.CloseChannel(m.closeC),
			otaruapiserver.InstallFileSystemService(fs),
			otaruapiserver.InstallFileHandler(fs),
		); err != nil {
			panic(err)
		}
		close(m.joinC)
	}()

	return m
}

func (m *mockFsBackend) Terminate() {
	close(m.closeC)
	<-m.joinC
}

func checkFileContents(t *testing.T, path string, b []byte) {
	t.Helper()
	c, err := ioutil.ReadFile(path)
	if err != nil {
		t.Errorf("Failed to read file %q: %v", path, err)
		return
	}
	if !bytes.Equal(c, b) {
		t.Errorf("Unexpected content of file %q: %q", path, c)
	}
}

func checkFileNotExist(t *testing.T, path string) {
	t.Helper()
	fi, err := os.Lstat(path)
	if err == nil {
		typ := "File"
		if fi.IsDir() {
			typ = "Dir"
		}
		t.Errorf("%s %q unexpectedly exists", typ, path)
		return
	}
	if !os.IsNotExist(err) {
		t.Errorf("os.Stat(%q) unexpected error: %v", path, err)
	}
}

func TestFeService(t *testing.T) {
	rootPath, err := ioutil.TempDir("", "felocal")
	if err != nil {
		t.Fatalf("failed to create tmpdir: %v", err)
	}
	if err := os.Mkdir(filepath.Join(rootPath, "foo"), 0755); err != nil {
		t.Fatalf("mkdir: %v", err)
	}
	if err := ioutil.WriteFile(filepath.Join(rootPath, "foo", "a.txt"), []byte("hoge\n"), 0644); err != nil {
		t.Fatalf("WriteFile: %v", err)
	}

	trashPath, err := ioutil.TempDir("", "fetrash")
	if err != nil {
		t.Fatalf("failed to create tmpdir: %v", err)
	}

	cfg := &cli.CliConfig{
		Host: map[string]*cli.Host{
			"hostfoo": &cli.Host{
				ApiEndpoint:      testBeListenAddr,
				ExpectedCertFile: certFile,
			},
		},
		LocalRootPath: rootPath,
		TrashDirPath:  trashPath,
	}

	m := runMockFsBackend()

	closeC := make(chan struct{})
	joinC := make(chan struct{})
	go func() {
		if err := apiserver.Serve(
			apiserver.ListenAddr(testFeListenAddr),
			apiserver.X509KeyPair(certFile, keyFile),
			apiserver.CloseChannel(closeC),
			feapiserver.InstallFeService(cfg),
		); err != nil {
			t.Errorf("Serve failed: %v", err)
		}
		close(joinC)
	}()

	// FIXME: wait until Serve to actually start accepting conns
	time.Sleep(100 * time.Millisecond)

	certtext, err := ioutil.ReadFile(certFile)
	if err != nil {
		t.Fatalf("Failed to read specified cert file: %s", certFile)
	}
	tc, err := cli.TLSConfigFromCertText(certtext)
	if err != nil {
		t.Fatalf("TLSConfigFromCertText: %v", err)
	}
	conn, err := cli.DialGrpc(testFeListenAddr, tc)
	if err != nil {
		t.Fatalf("DialGrpcVhost: %v", err)
	}
	fesc := pb.NewFeServiceClient(conn)

	t.Run("ListHosts", func(tt *testing.T) {
		ctx := context.TODO()
		resp, err := fesc.ListHosts(ctx, &pb.ListHostsRequest{})
		if err != nil {
			tt.Fatalf("ListHosts: %v", err)
		}
		if !reflect.DeepEqual(resp.Host, []string{"hostfoo", opath.VhostLocal}) {
			tt.Errorf("ListHosts resp: %v", resp.Host)
		}
	})
	t.Run("ListLocalDir", func(tt *testing.T) {
		ctx := context.TODO()
		resp, err := fesc.ListLocalDir(ctx, &pb.ListLocalDirRequest{
			Path: "/foo",
		})
		if err != nil {
			tt.Fatalf("ListLocalDir: %v", err)
		}
		if len(resp.Entry) != 1 {
			tt.Fatalf("resp.Entry: %+v", resp.Entry)
		}
		if resp.Entry[0].Name != "a.txt" {
			tt.Fatalf("resp.Entry: %+v", resp.Entry)
		}
	})
	t.Run("MkdirLocal", func(tt *testing.T) {
		if err := os.Mkdir(filepath.Join(rootPath, "mkdir"), 0755); err != nil {
			t.Fatalf("mkdir: %v", err)
		}

		ctx := context.TODO()
		_, err := fesc.MkdirLocal(ctx, &pb.MkdirLocalRequest{
			Path: "/mkdir/hoge",
		})
		if err != nil {
			tt.Fatalf("MkdirLocal: %v", err)
		}

		resp, err := fesc.ListLocalDir(ctx, &pb.ListLocalDirRequest{
			Path: "/mkdir",
		})
		if err != nil {
			tt.Fatalf("ListLocalDir: %v", err)
		}
		if len(resp.Entry) != 1 {
			tt.Fatalf("resp.Entry: %+v", resp.Entry)
		}
		if resp.Entry[0].Name != "hoge" {
			tt.Fatalf("resp.Entry: %+v", resp.Entry)
		}
		if resp.Entry[0].Type != opb.INodeType_DIR {
			tt.Fatalf("resp.Entry: %+v", resp.Entry)
		}
	})
	t.Run("MoveLocal", func(tt *testing.T) {
		if err := os.Mkdir(filepath.Join(rootPath, "move_src"), 0755); err != nil {
			t.Fatalf("mkdir: %v", err)
		}
		if err := ioutil.WriteFile(filepath.Join(rootPath, "move_src", "c.txt"), []byte("abcdef\n"), 0644); err != nil {
			t.Fatalf("WriteFile: %v", err)
		}
		if err := os.Mkdir(filepath.Join(rootPath, "move_dest"), 0755); err != nil {
			t.Fatalf("mkdir: %v", err)
		}

		ctx := context.TODO()
		_, err := fesc.MoveLocal(ctx, &pb.MoveLocalRequest{
			PathSrc:  "/move_src/c.txt",
			PathDest: "/move_dest/c.txt",
		})
		if err != nil {
			tt.Fatalf("MoveLocal: %v", err)
		}
		checkFileContents(t, filepath.Join(rootPath, "move_dest", "c.txt"), []byte("abcdef\n"))
		checkFileNotExist(t, filepath.Join(rootPath, "move_src", "c.txt"))
	})
	t.Run("RemoveLocal", func(tt *testing.T) {
		if err := os.Mkdir(filepath.Join(rootPath, "rm"), 0755); err != nil {
			t.Fatalf("mkdir: %v", err)
		}
		if err := ioutil.WriteFile(filepath.Join(rootPath, "rm", "tgt.txt"), []byte("foo"), 0644); err != nil {
			t.Fatalf("WriteFile: %v", err)
		}
		if err := ioutil.WriteFile(filepath.Join(rootPath, "rm", "nontgt.txt"), []byte("bar"), 0644); err != nil {
			t.Fatalf("WriteFile: %v", err)
		}

		ctx := context.TODO()
		_, err := fesc.RemoveLocal(ctx, &pb.RemoveLocalRequest{
			Path:           "rm/tgt.txt",
			RemoveChildren: false,
		})
		if err != nil {
			tt.Fatalf("RemoveLocal: %v", err)
		}
		checkFileNotExist(t, filepath.Join(rootPath, "rm", "tgt.txt"))
		checkFileContents(t, filepath.Join(rootPath, "rm", "nontgt.txt"), []byte("bar"))
		checkFileContents(t, filepath.Join(trashPath, "tgt.txt"), []byte("foo"))
	})
	t.Run("UploadDownload", func(tt *testing.T) {
		if err := os.Mkdir(filepath.Join(rootPath, "upldl"), 0755); err != nil {
			t.Fatalf("mkdir: %v", err)
		}
		if err := ioutil.WriteFile(filepath.Join(rootPath, "upldl", "src.txt"), []byte("src"), 0644); err != nil {
			t.Fatalf("WriteFile: %v", err)
		}
		if err := ioutil.WriteFile(filepath.Join(rootPath, "upldl", "existing.txt"), []byte("existing"), 0644); err != nil {
			t.Fatalf("WriteFile: %v", err)
		}

		ctx := context.TODO()
		_, err := fesc.Upload(ctx, &pb.UploadRequest{
			PathSrc:        "upldl/src.txt",
			OpathDest:      "//hostfoo/foo.txt",
			AllowOverwrite: false,
		})
		if err != nil {
			tt.Fatalf("Upload: %v", err)
		}

		_, err = fesc.Download(ctx, &pb.DownloadRequest{
			OpathSrc:       "//hostfoo/foo.txt",
			PathDest:       "upldl/dest.txt",
			AllowOverwrite: false,
		})
		if err != nil {
			tt.Fatalf("Download: %v", err)
		}

		_, err = fesc.Download(ctx, &pb.DownloadRequest{
			OpathSrc:       "//hostfoo/foo.txt",
			PathDest:       "upldl/existing.txt",
			AllowOverwrite: false,
		})
		if err == nil {
			tt.Fatalf("Download: unexpected overwrite: %v", err)
		}

		checkFileContents(t, filepath.Join(rootPath, "upldl", "src.txt"), []byte("src"))
		checkFileContents(t, filepath.Join(rootPath, "upldl", "dest.txt"), []byte("src"))
		checkFileContents(t, filepath.Join(rootPath, "upldl", "existing.txt"), []byte("existing"))
	})

	close(closeC)
	<-joinC

	m.Terminate()
}
<commit_msg>overwrite test<commit_after>package apiserver_test

import (
	"bytes"
	"context"
	"io/ioutil"
	"os"
	"path/filepath"
	"reflect"
	"testing"
	"time"

	"github.com/nyaxt/otaru/apiserver"
	"github.com/nyaxt/otaru/cli"
	opath "github.com/nyaxt/otaru/cli/path"
	feapiserver "github.com/nyaxt/otaru/extra/fe/apiserver"
	"github.com/nyaxt/otaru/extra/fe/pb"
	"github.com/nyaxt/otaru/filesystem"
	"github.com/nyaxt/otaru/otaruapiserver"
	opb "github.com/nyaxt/otaru/pb"
	"github.com/nyaxt/otaru/testutils"
)

type mockFsBackend struct {
	closeC chan struct{}
	joinC  chan struct{}
	fs     *filesystem.FileSystem
}

func runMockFsBackend() *mockFsBackend {
	fs := testutils.TestFileSystem()

	m := &mockFsBackend{
		closeC: make(chan struct{}),
		joinC:  make(chan struct{}),
		fs:     fs,
	}

	go func() {
		if err := apiserver.Serve(
			apiserver.ListenAddr(testBeListenAddr),
			apiserver.X509KeyPair(certFile, keyFile),
			apiserver.CloseChannel(m.closeC),
			otaruapiserver.InstallFileSystemService(fs),
			otaruapiserver.InstallFileHandler(fs),
		); err != nil {
			panic(err)
		}
		close(m.joinC)
	}()

	return m
}

func (m *mockFsBackend) Terminate() {
	close(m.closeC)
	<-m.joinC
}

func checkFileContents(t *testing.T, path string, b []byte) {
	t.Helper()
	c, err := ioutil.ReadFile(path)
	if err != nil {
		t.Errorf("Failed to read file %q: %v", path, err)
		return
	}
	if !bytes.Equal(c, b) {
		t.Errorf("Unexpected content of file %q: %q", path, c)
	}
}

func checkFileNotExist(t *testing.T, path string) {
	t.Helper()
	fi, err := os.Lstat(path)
	if err == nil {
		typ := "File"
		if fi.IsDir() {
			typ = "Dir"
		}
		t.Errorf("%s %q unexpectedly exists", typ, path)
		return
	}
	if !os.IsNotExist(err) {
		t.Errorf("os.Stat(%q) unexpected error: %v", path, err)
	}
}

func TestFeService(t *testing.T) {
	rootPath, err := ioutil.TempDir("", "felocal")
	if err != nil {
		t.Fatalf("failed to create tmpdir: %v", err)
	}
	if err := os.Mkdir(filepath.Join(rootPath, "foo"), 0755); err != nil {
		t.Fatalf("mkdir: %v", err)
	}
	if err := ioutil.WriteFile(filepath.Join(rootPath, "foo", "a.txt"), []byte("hoge\n"), 0644); err != nil {
		t.Fatalf("WriteFile: %v", err)
	}

	trashPath, err := ioutil.TempDir("", "fetrash")
	if err != nil {
		t.Fatalf("failed to create tmpdir: %v", err)
	}

	cfg := &cli.CliConfig{
		Host: map[string]*cli.Host{
			"hostfoo": &cli.Host{
				ApiEndpoint:      testBeListenAddr,
				ExpectedCertFile: certFile,
			},
		},
		LocalRootPath: rootPath,
		TrashDirPath:  trashPath,
	}

	m := runMockFsBackend()

	closeC := make(chan struct{})
	joinC := make(chan struct{})
	go func() {
		if err := apiserver.Serve(
			apiserver.ListenAddr(testFeListenAddr),
			apiserver.X509KeyPair(certFile, keyFile),
			apiserver.CloseChannel(closeC),
			feapiserver.InstallFeService(cfg),
		); err != nil {
			t.Errorf("Serve failed: %v", err)
		}
		close(joinC)
	}()

	// FIXME: wait until Serve to actually start accepting conns
	time.Sleep(100 * time.Millisecond)

	certtext, err := ioutil.ReadFile(certFile)
	if err != nil {
		t.Fatalf("Failed to read specified cert file: %s", certFile)
	}
	tc, err := cli.TLSConfigFromCertText(certtext)
	if err != nil {
		t.Fatalf("TLSConfigFromCertText: %v", err)
	}
	conn, err := cli.DialGrpc(testFeListenAddr, tc)
	if err != nil {
		t.Fatalf("DialGrpcVhost: %v", err)
	}
	fesc := pb.NewFeServiceClient(conn)

	t.Run("ListHosts", func(tt *testing.T) {
		ctx := context.TODO()
		resp, err := fesc.ListHosts(ctx, &pb.ListHostsRequest{})
		if err != nil {
			tt.Fatalf("ListHosts: %v", err)
		}
		if !reflect.DeepEqual(resp.Host, []string{"hostfoo", opath.VhostLocal}) {
			tt.Errorf("ListHosts resp: %v", resp.Host)
		}
	})
	t.Run("ListLocalDir", func(tt *testing.T) {
		ctx := context.TODO()
		resp, err := fesc.ListLocalDir(ctx, &pb.ListLocalDirRequest{
			Path: "/foo",
		})
		if err != nil {
			tt.Fatalf("ListLocalDir: %v", err)
		}
		if len(resp.Entry) != 1 {
			tt.Fatalf("resp.Entry: %+v", resp.Entry)
		}
		if resp.Entry[0].Name != "a.txt" {
			tt.Fatalf("resp.Entry: %+v", resp.Entry)
		}
	})
	t.Run("MkdirLocal", func(tt *testing.T) {
		if err := os.Mkdir(filepath.Join(rootPath, "mkdir"), 0755); err != nil {
			t.Fatalf("mkdir: %v", err)
		}

		ctx := context.TODO()
		_, err := fesc.MkdirLocal(ctx, &pb.MkdirLocalRequest{
			Path: "/mkdir/hoge",
		})
		if err != nil {
			tt.Fatalf("MkdirLocal: %v", err)
		}

		resp, err := fesc.ListLocalDir(ctx, &pb.ListLocalDirRequest{
			Path: "/mkdir",
		})
		if err != nil {
			tt.Fatalf("ListLocalDir: %v", err)
		}
		if len(resp.Entry) != 1 {
			tt.Fatalf("resp.Entry: %+v", resp.Entry)
		}
		if resp.Entry[0].Name != "hoge" {
			tt.Fatalf("resp.Entry: %+v", resp.Entry)
		}
		if resp.Entry[0].Type != opb.INodeType_DIR {
			tt.Fatalf("resp.Entry: %+v", resp.Entry)
		}
	})
	t.Run("MoveLocal", func(tt *testing.T) {
		if err := os.Mkdir(filepath.Join(rootPath, "move_src"), 0755); err != nil {
			t.Fatalf("mkdir: %v", err)
		}
		if err := ioutil.WriteFile(filepath.Join(rootPath, "move_src", "c.txt"), []byte("abcdef\n"), 0644); err != nil {
			t.Fatalf("WriteFile: %v", err)
		}
		if err := os.Mkdir(filepath.Join(rootPath, "move_dest"), 0755); err != nil {
			t.Fatalf("mkdir: %v", err)
		}

		ctx := context.TODO()
		_, err := fesc.MoveLocal(ctx, &pb.MoveLocalRequest{
			PathSrc:  "/move_src/c.txt",
			PathDest: "/move_dest/c.txt",
		})
		if err != nil {
			tt.Fatalf("MoveLocal: %v", err)
		}
		checkFileContents(t, filepath.Join(rootPath, "move_dest", "c.txt"), []byte("abcdef\n"))
		checkFileNotExist(t, filepath.Join(rootPath, "move_src", "c.txt"))
	})
	t.Run("RemoveLocal", func(tt *testing.T) {
		if err := os.Mkdir(filepath.Join(rootPath, "rm"), 0755); err != nil {
			t.Fatalf("mkdir: %v", err)
		}
		if err := ioutil.WriteFile(filepath.Join(rootPath, "rm", "tgt.txt"), []byte("foo"), 0644); err != nil {
			t.Fatalf("WriteFile: %v", err)
		}
		if err := ioutil.WriteFile(filepath.Join(rootPath, "rm", "nontgt.txt"), []byte("bar"), 0644); err != nil {
			t.Fatalf("WriteFile: %v", err)
		}

		ctx := context.TODO()
		_, err := fesc.RemoveLocal(ctx, &pb.RemoveLocalRequest{
			Path:           "rm/tgt.txt",
			RemoveChildren: false,
		})
		if err != nil {
			tt.Fatalf("RemoveLocal: %v", err)
		}
		checkFileNotExist(t, filepath.Join(rootPath, "rm", "tgt.txt"))
		checkFileContents(t, filepath.Join(rootPath, "rm", "nontgt.txt"), []byte("bar"))
		checkFileContents(t, filepath.Join(trashPath, "tgt.txt"), []byte("foo"))
	})
	t.Run("UploadDownload", func(tt *testing.T) {
		if err := os.Mkdir(filepath.Join(rootPath, "upldl"), 0755); err != nil {
			t.Fatalf("mkdir: %v", err)
		}
		if err := ioutil.WriteFile(filepath.Join(rootPath, "upldl", "src.txt"), []byte("src"), 0644); err != nil {
			t.Fatalf("WriteFile: %v", err)
		}
		if err := ioutil.WriteFile(filepath.Join(rootPath, "upldl", "existing.txt"), []byte("existing"), 0644); err != nil {
			t.Fatalf("WriteFile: %v", err)
		}

		ctx := context.TODO()
		_, err := fesc.Upload(ctx, &pb.UploadRequest{
			PathSrc:        "upldl/src.txt",
			OpathDest:      "//hostfoo/foo.txt",
			AllowOverwrite: false,
		})
		if err != nil {
			tt.Fatalf("Upload: %v", err)
		}

		_, err = fesc.Upload(ctx, &pb.UploadRequest{
			PathSrc:        "upldl/src.txt",
			OpathDest:      "//hostfoo/foo.txt",
			AllowOverwrite: false,
		})
		if err == nil {
			tt.Fatalf("Upload: unexpected overwrite")
		}

		_, err = fesc.Download(ctx, &pb.DownloadRequest{
			OpathSrc:       "//hostfoo/foo.txt",
			PathDest:       "upldl/dest.txt",
			AllowOverwrite: false,
		})
		if err != nil {
			tt.Fatalf("Download: %v", err)
		}

		_, err = fesc.Download(ctx, &pb.DownloadRequest{
			OpathSrc:       "//hostfoo/foo.txt",
			PathDest:       "upldl/existing.txt",
			AllowOverwrite: false,
		})
		if err == nil {
			tt.Fatalf("Download: unexpected overwrite")
		}

		checkFileContents(t, filepath.Join(rootPath, "upldl", "src.txt"), []byte("src"))
		checkFileContents(t, filepath.Join(rootPath, "upldl", "dest.txt"), []byte("src"))
		checkFileContents(t, filepath.Join(rootPath, "upldl", "existing.txt"), []byte("existing"))
	})

	close(closeC)
	<-joinC

	m.Terminate()
}
<|endoftext|>