<commit_before>package models

import (
	"errors"
	"fmt"
	"socialapi/config"
	"socialapi/request"
	"time"

	"github.com/VerbalExpressions/GoVerbalExpressions"
	"github.com/jinzhu/gorm"
	"github.com/koding/bongo"
)

var mentionRegex = verbalexpressions.New().
	Find("@").
	BeginCapture().
	Word().
	EndCapture().
	Regex()

type ChannelMessage struct {
	// unique identifier of the channel message
	Id int64 `json:"id,string"`

	// Token holds the uuid for interoperability with the bongo-client
	Token string `json:"token"`

	// Body of the mesage
	Body string `json:"body"`

	// Generated Slug for body
	Slug string `json:"slug"                               sql:"NOT NULL;TYPE:VARCHAR(100);"`

	// type of the m essage
	TypeConstant string `json:"typeConstant"               sql:"NOT NULL;TYPE:VARCHAR(100);"`

	// Creator of the channel message
	AccountId int64 `json:"accountId,string"               sql:"NOT NULL"`

	// in which channel this message is created
	InitialChannelId int64 `json:"initialChannelId,string" sql:"NOT NULL"`

	// holds troll, unsafe, etc
	MetaBits MetaBits `json:"metaBits"`

	// Creation date of the message
	CreatedAt time.Time `json:"createdAt"                  sql:"DEFAULT:CURRENT_TIMESTAMP"`

	// Modification date of the message
	UpdatedAt time.Time `json:"updatedAt"                  sql:"DEFAULT:CURRENT_TIMESTAMP"`

	// Deletion date of the channel message
	DeletedAt time.Time `json:"deletedAt"`

	// Extra data storage
	Payload gorm.Hstore `json:"payload,omitempty"`

	// is required to identify to request in client side
	RequestData string `json:"requestData,omitempty" sql:"-"`
}

const (
	ChannelMessage_TYPE_POST            = "post"
	ChannelMessage_TYPE_REPLY           = "reply"
	ChannelMessage_TYPE_JOIN            = "join"
	ChannelMessage_TYPE_LEAVE           = "leave"
	ChannelMessage_TYPE_PRIVATE_MESSAGE = "privatemessage"
)

func (c *ChannelMessage) MarkIfExempt() error {
	isExempt, err := c.isExempt()
	if err != nil {
		return err
	}

	if isExempt {
		c.MetaBits.Mark(Troll)
	}

	return nil
}

func (c *ChannelMessage) isExempt() (bool, error) {
	if c.MetaBits.Is(Troll) {
		return true, nil
	}

	accountId, err := c.getAccountId()
	if err != nil {
		return false, err
	}

	account, err := ResetAccountCache(accountId)
	if err != nil {
		return false, err
	}

	if account == nil {
		return false, fmt.Errorf("account is nil, accountId:%d", c.AccountId)
	}

	if account.IsTroll {
		return true, nil
	}

	return false, nil
}

func (c *ChannelMessage) getAccountId() (int64, error) {
	if c.AccountId != 0 {
		return c.AccountId, nil
	}

	if c.Id == 0 {
		return 0, fmt.Errorf("couldnt find accountId from content %+v", c)
	}

	cm := NewChannelMessage()
	if err := cm.ById(c.Id); err != nil {
		return 0, err
	}

	return cm.AccountId, nil
}

func bodyLenCheck(body string) error {
	if len(body) < config.MustGet().Limits.MessageBodyMinLen {
		return fmt.Errorf("message body length should be greater than %d, yours is %d ", config.MustGet().Limits.MessageBodyMinLen, len(body))
	}

	return nil
}

// todo create a new message while updating the channel_message and delete other
// cases, since deletion is a soft delete, old instances will still be there

// CreateRaw creates a new channel message without effected by auto generated createdAt
// and updatedAt values
func (c *ChannelMessage) CreateRaw() error {
	insertSql := "INSERT INTO " +
		c.TableName() +
		` ("body","slug","type_constant","account_id","initial_channel_id",` +
		`"created_at","updated_at","deleted_at","payload") ` +
		"VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) " +
		"RETURNING ID"

	return bongo.B.DB.CommonDB().QueryRow(insertSql, c.Body, c.Slug, c.TypeConstant, c.AccountId, c.InitialChannelId,
		c.CreatedAt, c.UpdatedAt, c.DeletedAt, c.Payload).Scan(&c.Id)
}

// UpdateBodyRaw updates message body without effecting createdAt/UpdatedAt
// timestamps
func (c *ChannelMessage) UpdateBodyRaw() error {
	updateSql := fmt.Sprintf("UPDATE %s SET body=? WHERE id=?", c.TableName())

	return bongo.B.DB.Exec(updateSql, c.Body, c.Id).Error
}

// TODO - remove this function
func (c *ChannelMessage) BuildMessages(query *request.Query, messages []ChannelMessage) ([]*ChannelMessageContainer, error) {
	containers := make([]*ChannelMessageContainer, len(messages))
	if len(containers) == 0 {
		return containers, nil
	}

	for i, message := range messages {
		d := NewChannelMessage()
		*d = message
		data, err := d.BuildMessage(query)
		if err != nil {
			return containers, err
		}
		containers[i] = data
	}

	return containers, nil
}

// TODO - remove this function
func (c *ChannelMessage) BuildMessage(query *request.Query) (*ChannelMessageContainer, error) {
	cmc := NewChannelMessageContainer()
	if err := cmc.Fetch(c.Id, query); err != nil {
		return nil, err
	}

	return cmc, cmc.AddIsFollowed(query).AddIsInteracted(query).Err
}

func (c *ChannelMessage) CheckIsMessageFollowed(query *request.Query) (bool, error) {
	channel := NewChannel()
	if err := channel.FetchPinnedActivityChannel(query.AccountId, query.GroupName); err != nil {
		if err == bongo.RecordNotFound {
			return false, nil
		}
		return false, err
	}

	cml := NewChannelMessageList()
	q := &bongo.Query{
		Selector: map[string]interface{}{
			"channel_id": channel.Id,
			"message_id": c.Id,
		},
	}
	if err := cml.One(q); err != nil {
		if err == bongo.RecordNotFound {
			return false, nil
		}

		return false, err
	}

	return true, nil
}

func (c *ChannelMessage) BuildEmptyMessageContainer() (*ChannelMessageContainer, error) {
	if c.Id == 0 {
		return nil, errors.New("Channel message id is not set")
	}
	container := NewChannelMessageContainer()
	container.Message = c

	oldId, err := FetchAccountOldIdByIdFromCache(c.AccountId)
	if err != nil {
		return nil, err
	}

	container.AccountOldId = oldId

	return container, nil
}

func generateMessageListQuery(channelId int64, q *request.Query) *bongo.Query {
	messageType := q.Type
	if messageType == "" {
		messageType = ChannelMessage_TYPE_POST
	}

	return &bongo.Query{
		Selector: map[string]interface{}{
			"account_id":         q.AccountId,
			"initial_channel_id": channelId,
			"type_constant":      messageType,
		},
		Pagination: *bongo.NewPagination(q.Limit, q.Skip),
		Sort: map[string]string{
			"created_at": "DESC",
		},
	}
}

func (c *ChannelMessage) FetchMessagesByChannelId(channelId int64, q *request.Query) ([]ChannelMessage, error) {
	query := generateMessageListQuery(channelId, q)

	var messages []ChannelMessage
	if err := c.Some(&messages, query); err != nil {
		return nil, err
	}

	if messages == nil {
		return make([]ChannelMessage, 0), nil
	}
	return messages, nil
}

func (c *ChannelMessage) GetMentionedUsernames() []string {
	flattened := make([]string, 0)

	res := mentionRegex.FindAllStringSubmatch(c.Body, -1)
	if len(res) == 0 {
		return flattened
	}

	participants := map[string]struct{}{}
	// remove duplicate mentions
	for _, ele := range res {
		participants[ele[1]] = struct{}{}
	}

	for participant := range participants {
		flattened = append(flattened, participant)
	}

	return flattened
}

// FetchTotalMessageCount fetch the count of all messages in the channel
func (c *ChannelMessage) FetchTotalMessageCount(q *request.Query) (int, error) {
	query := &bongo.Query{
		Selector: map[string]interface{}{
			"account_id":    q.AccountId,
			"type_constant": q.Type,
		},
		Pagination: *bongo.NewPagination(q.Limit, q.Skip),
	}

	query.AddScope(RemoveTrollContent(
		c, q.ShowExempt,
	))

	return c.CountWithQuery(query)
}

// FetchMessageIds fetch id of the messages in the channel
// sorts the messages by descending order
func (c *ChannelMessage) FetchMessageIds(q *request.Query) ([]int64, error) {
	query := &bongo.Query{
		Selector: map[string]interface{}{
			"account_id":    q.AccountId,
			"type_constant": q.Type,
		},
		Pluck:      "id",
		Pagination: *bongo.NewPagination(q.Limit, q.Skip),
		Sort: map[string]string{
			"created_at": "DESC",
		},
	}

	query.AddScope(RemoveTrollContent(c, q.ShowExempt))

	var messageIds []int64
	if err := c.Some(&messageIds, query); err != nil {
		return nil, err
	}

	if messageIds == nil {
		return make([]int64, 0), nil
	}

	return messageIds, nil
}

// BySlug fetchs channel message by its slug
// checks if message is in the channel or not
func (c *ChannelMessage) BySlug(query *request.Query) error {
	if query.Slug == "" {
		return errors.New("slug is not set")
	}

	// fetch message itself
	q := &bongo.Query{
		Selector: map[string]interface{}{
			"slug": query.Slug,
		},
	}

	q.AddScope(RemoveTrollContent(
		c, query.ShowExempt,
	))

	if err := c.One(q); err != nil {
		return err
	}

	// fetch channel by group name
	query.Name = query.GroupName
	query.Type = Channel_TYPE_GROUP
	ch := NewChannel()
	channel, err := ch.ByName(query)
	if err != nil {
		return err
	}

	if channel.Id == 0 {
		return errors.New("channel is not found")
	}

	// check if message is in the channel
	cml := NewChannelMessageList()
	res, err := cml.IsInChannel(c.Id, channel.Id)
	if err != nil {
		return err
	}

	// if message is not in the channel
	if !res {
		return bongo.RecordNotFound
	}

	return nil
}
<commit_msg>EmailNotifier: Change VerbalExpressions alias<commit_after>package models

import (
	"errors"
	"fmt"
	"socialapi/config"
	"socialapi/request"
	"time"

	ve "github.com/VerbalExpressions/GoVerbalExpressions"
	"github.com/jinzhu/gorm"
	"github.com/koding/bongo"
)

var mentionRegex = ve.New().
	Find("@").
	BeginCapture().
	Word().
	EndCapture().
	Regex()

type ChannelMessage struct {
	// unique identifier of the channel message
	Id int64 `json:"id,string"`

	// Token holds the uuid for interoperability with the bongo-client
	Token string `json:"token"`

	// Body of the mesage
	Body string `json:"body"`

	// Generated Slug for body
	Slug string `json:"slug"                               sql:"NOT NULL;TYPE:VARCHAR(100);"`

	// type of the m essage
	TypeConstant string `json:"typeConstant"               sql:"NOT NULL;TYPE:VARCHAR(100);"`

	// Creator of the channel message
	AccountId int64 `json:"accountId,string"               sql:"NOT NULL"`

	// in which channel this message is created
	InitialChannelId int64 `json:"initialChannelId,string" sql:"NOT NULL"`

	// holds troll, unsafe, etc
	MetaBits MetaBits `json:"metaBits"`

	// Creation date of the message
	CreatedAt time.Time `json:"createdAt"                  sql:"DEFAULT:CURRENT_TIMESTAMP"`

	// Modification date of the message
	UpdatedAt time.Time `json:"updatedAt"                  sql:"DEFAULT:CURRENT_TIMESTAMP"`

	// Deletion date of the channel message
	DeletedAt time.Time `json:"deletedAt"`

	// Extra data storage
	Payload gorm.Hstore `json:"payload,omitempty"`

	// is required to identify to request in client side
	RequestData string `json:"requestData,omitempty" sql:"-"`
}

const (
	ChannelMessage_TYPE_POST            = "post"
	ChannelMessage_TYPE_REPLY           = "reply"
	ChannelMessage_TYPE_JOIN            = "join"
	ChannelMessage_TYPE_LEAVE           = "leave"
	ChannelMessage_TYPE_PRIVATE_MESSAGE = "privatemessage"
)

func (c *ChannelMessage) MarkIfExempt() error {
	isExempt, err := c.isExempt()
	if err != nil {
		return err
	}

	if isExempt {
		c.MetaBits.Mark(Troll)
	}

	return nil
}

func (c *ChannelMessage) isExempt() (bool, error) {
	if c.MetaBits.Is(Troll) {
		return true, nil
	}

	accountId, err := c.getAccountId()
	if err != nil {
		return false, err
	}

	account, err := ResetAccountCache(accountId)
	if err != nil {
		return false, err
	}

	if account == nil {
		return false, fmt.Errorf("account is nil, accountId:%d", c.AccountId)
	}

	if account.IsTroll {
		return true, nil
	}

	return false, nil
}

func (c *ChannelMessage) getAccountId() (int64, error) {
	if c.AccountId != 0 {
		return c.AccountId, nil
	}

	if c.Id == 0 {
		return 0, fmt.Errorf("couldnt find accountId from content %+v", c)
	}

	cm := NewChannelMessage()
	if err := cm.ById(c.Id); err != nil {
		return 0, err
	}

	return cm.AccountId, nil
}

func bodyLenCheck(body string) error {
	if len(body) < config.MustGet().Limits.MessageBodyMinLen {
		return fmt.Errorf("message body length should be greater than %d, yours is %d ", config.MustGet().Limits.MessageBodyMinLen, len(body))
	}

	return nil
}

// todo create a new message while updating the channel_message and delete other
// cases, since deletion is a soft delete, old instances will still be there

// CreateRaw creates a new channel message without effected by auto generated createdAt
// and updatedAt values
func (c *ChannelMessage) CreateRaw() error {
	insertSql := "INSERT INTO " +
		c.TableName() +
		` ("body","slug","type_constant","account_id","initial_channel_id",` +
		`"created_at","updated_at","deleted_at","payload") ` +
		"VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) " +
		"RETURNING ID"

	return bongo.B.DB.CommonDB().QueryRow(insertSql, c.Body, c.Slug, c.TypeConstant, c.AccountId, c.InitialChannelId,
		c.CreatedAt, c.UpdatedAt, c.DeletedAt, c.Payload).Scan(&c.Id)
}

// UpdateBodyRaw updates message body without effecting createdAt/UpdatedAt
// timestamps
func (c *ChannelMessage) UpdateBodyRaw() error {
	updateSql := fmt.Sprintf("UPDATE %s SET body=? WHERE id=?", c.TableName())

	return bongo.B.DB.Exec(updateSql, c.Body, c.Id).Error
}

// TODO - remove this function
func (c *ChannelMessage) BuildMessages(query *request.Query, messages []ChannelMessage) ([]*ChannelMessageContainer, error) {
	containers := make([]*ChannelMessageContainer, len(messages))
	if len(containers) == 0 {
		return containers, nil
	}

	for i, message := range messages {
		d := NewChannelMessage()
		*d = message
		data, err := d.BuildMessage(query)
		if err != nil {
			return containers, err
		}
		containers[i] = data
	}

	return containers, nil
}

// TODO - remove this function
func (c *ChannelMessage) BuildMessage(query *request.Query) (*ChannelMessageContainer, error) {
	cmc := NewChannelMessageContainer()
	if err := cmc.Fetch(c.Id, query); err != nil {
		return nil, err
	}

	return cmc, cmc.AddIsFollowed(query).AddIsInteracted(query).Err
}

func (c *ChannelMessage) CheckIsMessageFollowed(query *request.Query) (bool, error) {
	channel := NewChannel()
	if err := channel.FetchPinnedActivityChannel(query.AccountId, query.GroupName); err != nil {
		if err == bongo.RecordNotFound {
			return false, nil
		}
		return false, err
	}

	cml := NewChannelMessageList()
	q := &bongo.Query{
		Selector: map[string]interface{}{
			"channel_id": channel.Id,
			"message_id": c.Id,
		},
	}
	if err := cml.One(q); err != nil {
		if err == bongo.RecordNotFound {
			return false, nil
		}

		return false, err
	}

	return true, nil
}

func (c *ChannelMessage) BuildEmptyMessageContainer() (*ChannelMessageContainer, error) {
	if c.Id == 0 {
		return nil, errors.New("Channel message id is not set")
	}
	container := NewChannelMessageContainer()
	container.Message = c

	oldId, err := FetchAccountOldIdByIdFromCache(c.AccountId)
	if err != nil {
		return nil, err
	}

	container.AccountOldId = oldId

	return container, nil
}

func generateMessageListQuery(channelId int64, q *request.Query) *bongo.Query {
	messageType := q.Type
	if messageType == "" {
		messageType = ChannelMessage_TYPE_POST
	}

	return &bongo.Query{
		Selector: map[string]interface{}{
			"account_id":         q.AccountId,
			"initial_channel_id": channelId,
			"type_constant":      messageType,
		},
		Pagination: *bongo.NewPagination(q.Limit, q.Skip),
		Sort: map[string]string{
			"created_at": "DESC",
		},
	}
}

func (c *ChannelMessage) FetchMessagesByChannelId(channelId int64, q *request.Query) ([]ChannelMessage, error) {
	query := generateMessageListQuery(channelId, q)

	var messages []ChannelMessage
	if err := c.Some(&messages, query); err != nil {
		return nil, err
	}

	if messages == nil {
		return make([]ChannelMessage, 0), nil
	}
	return messages, nil
}

func (c *ChannelMessage) GetMentionedUsernames() []string {
	flattened := make([]string, 0)

	res := mentionRegex.FindAllStringSubmatch(c.Body, -1)
	if len(res) == 0 {
		return flattened
	}

	participants := map[string]struct{}{}
	// remove duplicate mentions
	for _, ele := range res {
		participants[ele[1]] = struct{}{}
	}

	for participant := range participants {
		flattened = append(flattened, participant)
	}

	return flattened
}

// FetchTotalMessageCount fetch the count of all messages in the channel
func (c *ChannelMessage) FetchTotalMessageCount(q *request.Query) (int, error) {
	query := &bongo.Query{
		Selector: map[string]interface{}{
			"account_id":    q.AccountId,
			"type_constant": q.Type,
		},
		Pagination: *bongo.NewPagination(q.Limit, q.Skip),
	}

	query.AddScope(RemoveTrollContent(
		c, q.ShowExempt,
	))

	return c.CountWithQuery(query)
}

// FetchMessageIds fetch id of the messages in the channel
// sorts the messages by descending order
func (c *ChannelMessage) FetchMessageIds(q *request.Query) ([]int64, error) {
	query := &bongo.Query{
		Selector: map[string]interface{}{
			"account_id":    q.AccountId,
			"type_constant": q.Type,
		},
		Pluck:      "id",
		Pagination: *bongo.NewPagination(q.Limit, q.Skip),
		Sort: map[string]string{
			"created_at": "DESC",
		},
	}

	query.AddScope(RemoveTrollContent(c, q.ShowExempt))

	var messageIds []int64
	if err := c.Some(&messageIds, query); err != nil {
		return nil, err
	}

	if messageIds == nil {
		return make([]int64, 0), nil
	}

	return messageIds, nil
}

// BySlug fetchs channel message by its slug
// checks if message is in the channel or not
func (c *ChannelMessage) BySlug(query *request.Query) error {
	if query.Slug == "" {
		return errors.New("slug is not set")
	}

	// fetch message itself
	q := &bongo.Query{
		Selector: map[string]interface{}{
			"slug": query.Slug,
		},
	}

	q.AddScope(RemoveTrollContent(
		c, query.ShowExempt,
	))

	if err := c.One(q); err != nil {
		return err
	}

	// fetch channel by group name
	query.Name = query.GroupName
	query.Type = Channel_TYPE_GROUP
	ch := NewChannel()
	channel, err := ch.ByName(query)
	if err != nil {
		return err
	}

	if channel.Id == 0 {
		return errors.New("channel is not found")
	}

	// check if message is in the channel
	cml := NewChannelMessageList()
	res, err := cml.IsInChannel(c.Id, channel.Id)
	if err != nil {
		return err
	}

	// if message is not in the channel
	if !res {
		return bongo.RecordNotFound
	}

	return nil
}
<|endoftext|>


<commit_before>// Package redisence provides simple user presence system
package redisence

import (
	"fmt"
	"strconv"
	"time"

	gredis "github.com/garyburd/redigo/redis"
	"github.com/koding/redis"
)

// Status defines what is the current status of a user
// in presence system
type Status int

const (
	Offline Status = iota
	Online
	Closed
)

// Event is the data type for
// occuring events in the system
type Event struct {
	// Id is the given key by the application
	Id string

	// Status holds the changing type of event
	Status Status
}

// Prefix for redisence package
const RedisencePrefix = "redisence"

// Session holds the required connection data for redis
type Session struct {
	// main redis connection
	redis *redis.RedisSession

	// inactiveDuration specifies no-probe allowance time
	inactiveDuration time.Duration

	// receiving offline events pattern
	becameOfflinePattern string

	// receiving online events pattern
	becameOnlinePattern string

	// inactiveDurationAsString holds the expiration duration as string
	inactiveDurationAsString string

	// closeChan is used for giving close signal
	closeChan chan bool

	// errChan pipe all errors  the this channel
	errChan chan error

	// closed holds the status of connection
	closed bool

	//psc holds the pubsub channel if opened
	psc *gredis.PubSubConn
}

// New creates a session for any broker system that is architected to use,
// communicate, forward events to the presence system
func New(server string, db int, inactiveDuration time.Duration) (*Session, error) {
	redis, err := redis.NewRedisSession(&redis.RedisConf{Server: server, DB: db})
	if err != nil {
		return nil, err
	}
	redis.SetPrefix(RedisencePrefix)

	return &Session{
		redis:                redis,
		becameOfflinePattern: fmt.Sprintf("__keyevent@%d__:expired", db),
		becameOnlinePattern:  fmt.Sprintf("__keyevent@%d__:set", db),
		inactiveDuration:     inactiveDuration,
		// cache inactive duration as string
		inactiveDurationAsString: strconv.Itoa(int(inactiveDuration.Seconds())),
		closeChan:                make(chan bool, 1),
		errChan:                  make(chan error, 1),
	}, nil
}

// Close closes the redis connection gracefully
func (s *Session) Close() error {
	s.closeChan <- true
	return s.close()
}
// Ping resets the expiration time for any given key
// if key doesnt exists, it means user is now online and should be set as online
// Whenever application gets any prob from a client
// should call this function
func (s *Session) Online(ids ...string) error {
	if len(ids) == 1 {
		// if member exits increase ttl
		if s.redis.Expire(ids[0], s.inactiveDuration) == nil {
			return nil
		}

		// if member doesnt exist set it
		return s.redis.Setex(ids[0], s.inactiveDuration, ids[0])
	}

	existance, err := s.sendMultiExpire(ids)
	if err != nil {
		return err
	}

	return s.sendMultiSetIfRequired(ids, existance)
}

// Offline sets given ids as offline, ignores any error
// since not exist keys returned as nilErr
func (s *Session) Offline(ids ...string) error {
	if len(ids) == 1 {
		if s.redis.Expire(ids[0], time.Second*0) == nil {
			return nil
		}
	}

	s.sendMultiExpire(ids, "0")
	return nil
}

// sendMultiSetIfRequired accepts set of ids and their existtance status
// traverse over them and any key is not exists in db, set them in a multi/exec
// request
func (s *Session) sendMultiSetIfRequired(ids []string, existance []int) error {
	if len(ids) != len(existance) {
		return fmt.Errorf("Length is not same Ids: %d Existance: %d", len(ids), len(existance))
	}

	// cache inactive duration as string
	seconds := strconv.Itoa(int(s.inactiveDuration.Seconds()))

	// get one connection from pool
	c := s.redis.Pool().Get()

	// item count for non-existent members
	notExistsCount := 0

	for i, exists := range existance {
		// `0` means, member doesnt exists in presence system
		if exists != 0 {
			continue
		}

		// init multi command lazily
		if notExistsCount == 0 {
			c.Send("MULTI")
		}

		notExistsCount++
		c.Send("SETEX", s.redis.AddPrefix(ids[i]), seconds, ids[i])
	}

	// execute multi command if only we flushed some to connection
	if notExistsCount != 0 {
		// ignore values
		if _, err := c.Do("EXEC"); err != nil {
			return err
		}
	}

	// do not forget to close the connection
	if err := c.Close(); err != nil {
		return err
	}

	return nil
}

// sendMultiExpire if the system tries to update more than one key at a time
// inorder to leverage rtt, send multi expire
func (s *Session) sendMultiExpire(ids []string, duration string) ([]int, error) {
	// get one connection from pool
	c := s.redis.Pool().Get()

	// init multi command
	c.Send("MULTI")

	// send expire command for all members
	for _, id := range ids {
		c.Send("EXPIRE", s.redis.AddPrefix(id), seconds)
	}

	// execute command
	r, err := c.Do("EXEC")
	if err != nil {
		return make([]int, 0), err
	}

	// close connection
	if err := c.Close(); err != nil {
		return make([]int, 0), err
	}

	values, err := s.redis.Values(r)
	if err != nil {
		return make([]int, 0), err
	}

	res := make([]int, len(values))
	for i, value := range values {
		res[i], err = s.redis.Int(value)
		if err != nil {
			// what about returning half-generated slice?
			// instead of an empty one
			return make([]int, 0), err
		}

	}

	return res, nil
}

// MultipleStatus returns the current status multiple keys from system
func (s *Session) MultipleStatus(ids []string) ([]Event, error) {
	// get one connection from pool
	c := s.redis.Pool().Get()

	// init multi command
	c.Send("MULTI")

	// send expire command for all members
	for _, id := range ids {
		c.Send("EXISTS", s.redis.AddPrefix(id))
	}

	// execute command
	r, err := c.Do("EXEC")
	if err != nil {
		return make([]Event, 0), err
	}

	// close connection
	if err := c.Close(); err != nil {
		return make([]Event, 0), err
	}

	values, err := s.redis.Values(r)
	if err != nil {
		return make([]Event, 0), err
	}

	res := make([]Event, len(values))
	for i, value := range values {
		status, err := s.redis.Int(value)
		if err != nil {
			return make([]Event, 0), err
		}

		res[i] = Event{
			Id: ids[i],
			// cast redis response to Status
			Status: Status(status),
		}
	}

	return res, nil
}

// Status returns the current status a key from system
func (s *Session) Status(id string) (Event, error) {
	res := Event{
		Id:     id,
		Status: Offline,
	}

	if s.redis.Exists(id) {
		res.Status = Online
	}

	return res, nil
}

// ListenStatusChanges subscribes with a pattern to the redis and
// gets online and offline status changes from it
func (s *Session) ListenStatusChanges(events chan Event) {
	s.psc = s.redis.CreatePubSubConn()
	s.psc.PSubscribe(s.becameOnlinePattern, s.becameOfflinePattern)

	go s.listenEvents(events)

	<-s.closeChan
	events <- Event{Status: Closed}
}

func (s *Session) close() error {
	s.closed = true
	if s.psc != nil {
		s.psc.PUnsubscribe()
	}

	return s.redis.Close()
}

// createEvent Creates the event with the required properties
func (s *Session) listenEvents(events chan Event) {
	for {
		if s.closed {
			break
		}
		switch n := s.psc.Receive().(type) {
		case gredis.PMessage:
			events <- s.createEvent(n)
		case error:
			s.errChan <- n
			return
		}
	}
}

// createEvent Creates the event with the required properties
func (s *Session) createEvent(n gredis.PMessage) Event {
	e := Event{}

	switch n.Pattern {
	case s.becameOfflinePattern:
		e.Id = string(n.Data[len(RedisencePrefix)+1:])
		e.Status = Offline
	case s.becameOnlinePattern:
		e.Id = string(n.Data[len(RedisencePrefix)+1:])
		e.Status = Online
	default:
		//ignore other events
	}

	return e
}
<commit_msg>Redisence: add a function for listening errors<commit_after>// Package redisence provides simple user presence system
package redisence

import (
	"fmt"
	"strconv"
	"time"

	gredis "github.com/garyburd/redigo/redis"
	"github.com/koding/redis"
)

// Status defines what is the current status of a user
// in presence system
type Status int

const (
	Offline Status = iota
	Online
	Closed
)

// Event is the data type for
// occuring events in the system
type Event struct {
	// Id is the given key by the application
	Id string

	// Status holds the changing type of event
	Status Status
}

// Prefix for redisence package
const RedisencePrefix = "redisence"

// Session holds the required connection data for redis
type Session struct {
	// main redis connection
	redis *redis.RedisSession

	// inactiveDuration specifies no-probe allowance time
	inactiveDuration time.Duration

	// receiving offline events pattern
	becameOfflinePattern string

	// receiving online events pattern
	becameOnlinePattern string

	// inactiveDurationAsString holds the expiration duration as string
	inactiveDurationAsString string

	// closeChan is used for giving close signal
	closeChan chan bool

	// errChan pipe all errors  the this channel
	errChan chan error

	// closed holds the status of connection
	closed bool

	//psc holds the pubsub channel if opened
	psc *gredis.PubSubConn
}

// New creates a session for any broker system that is architected to use,
// communicate, forward events to the presence system
func New(server string, db int, inactiveDuration time.Duration) (*Session, error) {
	redis, err := redis.NewRedisSession(&redis.RedisConf{Server: server, DB: db})
	if err != nil {
		return nil, err
	}
	redis.SetPrefix(RedisencePrefix)

	return &Session{
		redis:                redis,
		becameOfflinePattern: fmt.Sprintf("__keyevent@%d__:expired", db),
		becameOnlinePattern:  fmt.Sprintf("__keyevent@%d__:set", db),
		inactiveDuration:     inactiveDuration,
		// cache inactive duration as string
		inactiveDurationAsString: strconv.Itoa(int(inactiveDuration.Seconds())),
		closeChan:                make(chan bool, 1),
		errChan:                  make(chan error, 1),
	}, nil
}

// Close closes the redis connection gracefully
func (s *Session) Close() error {
	s.closeChan <- true
	return s.close()
}

// Error returns error if it happens while listening  to status changes
func (s *Session) Error() error {
	return <-s.errChan
}

// Ping resets the expiration time for any given key
// if key doesnt exists, it means user is now online and should be set as online
// Whenever application gets any prob from a client
// should call this function
func (s *Session) Online(ids ...string) error {
	if len(ids) == 1 {
		// if member exits increase ttl
		if s.redis.Expire(ids[0], s.inactiveDuration) == nil {
			return nil
		}

		// if member doesnt exist set it
		return s.redis.Setex(ids[0], s.inactiveDuration, ids[0])
	}

	existance, err := s.sendMultiExpire(ids)
	if err != nil {
		return err
	}

	return s.sendMultiSetIfRequired(ids, existance)
}

// Offline sets given ids as offline, ignores any error
// since not exist keys returned as nilErr
func (s *Session) Offline(ids ...string) error {
	if len(ids) == 1 {
		if s.redis.Expire(ids[0], time.Second*0) == nil {
			return nil
		}
	}

	s.sendMultiExpire(ids, "0")
	return nil
}

// sendMultiSetIfRequired accepts set of ids and their existtance status
// traverse over them and any key is not exists in db, set them in a multi/exec
// request
func (s *Session) sendMultiSetIfRequired(ids []string, existance []int) error {
	if len(ids) != len(existance) {
		return fmt.Errorf("Length is not same Ids: %d Existance: %d", len(ids), len(existance))
	}

	// cache inactive duration as string
	seconds := strconv.Itoa(int(s.inactiveDuration.Seconds()))

	// get one connection from pool
	c := s.redis.Pool().Get()

	// item count for non-existent members
	notExistsCount := 0

	for i, exists := range existance {
		// `0` means, member doesnt exists in presence system
		if exists != 0 {
			continue
		}

		// init multi command lazily
		if notExistsCount == 0 {
			c.Send("MULTI")
		}

		notExistsCount++
		c.Send("SETEX", s.redis.AddPrefix(ids[i]), seconds, ids[i])
	}

	// execute multi command if only we flushed some to connection
	if notExistsCount != 0 {
		// ignore values
		if _, err := c.Do("EXEC"); err != nil {
			return err
		}
	}

	// do not forget to close the connection
	if err := c.Close(); err != nil {
		return err
	}

	return nil
}

// sendMultiExpire if the system tries to update more than one key at a time
// inorder to leverage rtt, send multi expire
func (s *Session) sendMultiExpire(ids []string, duration string) ([]int, error) {
	// get one connection from pool
	c := s.redis.Pool().Get()

	// init multi command
	c.Send("MULTI")

	// send expire command for all members
	for _, id := range ids {
		c.Send("EXPIRE", s.redis.AddPrefix(id), seconds)
	}

	// execute command
	r, err := c.Do("EXEC")
	if err != nil {
		return make([]int, 0), err
	}

	// close connection
	if err := c.Close(); err != nil {
		return make([]int, 0), err
	}

	values, err := s.redis.Values(r)
	if err != nil {
		return make([]int, 0), err
	}

	res := make([]int, len(values))
	for i, value := range values {
		res[i], err = s.redis.Int(value)
		if err != nil {
			// what about returning half-generated slice?
			// instead of an empty one
			return make([]int, 0), err
		}

	}

	return res, nil
}

// MultipleStatus returns the current status multiple keys from system
func (s *Session) MultipleStatus(ids []string) ([]Event, error) {
	// get one connection from pool
	c := s.redis.Pool().Get()

	// init multi command
	c.Send("MULTI")

	// send expire command for all members
	for _, id := range ids {
		c.Send("EXISTS", s.redis.AddPrefix(id))
	}

	// execute command
	r, err := c.Do("EXEC")
	if err != nil {
		return make([]Event, 0), err
	}

	// close connection
	if err := c.Close(); err != nil {
		return make([]Event, 0), err
	}

	values, err := s.redis.Values(r)
	if err != nil {
		return make([]Event, 0), err
	}

	res := make([]Event, len(values))
	for i, value := range values {
		status, err := s.redis.Int(value)
		if err != nil {
			return make([]Event, 0), err
		}

		res[i] = Event{
			Id: ids[i],
			// cast redis response to Status
			Status: Status(status),
		}
	}

	return res, nil
}

// Status returns the current status a key from system
func (s *Session) Status(id string) (Event, error) {
	res := Event{
		Id:     id,
		Status: Offline,
	}

	if s.redis.Exists(id) {
		res.Status = Online
	}

	return res, nil
}

// ListenStatusChanges subscribes with a pattern to the redis and
// gets online and offline status changes from it
func (s *Session) ListenStatusChanges(events chan Event) {
	s.psc = s.redis.CreatePubSubConn()
	s.psc.PSubscribe(s.becameOnlinePattern, s.becameOfflinePattern)

	go s.listenEvents(events)

	<-s.closeChan
	events <- Event{Status: Closed}
}

func (s *Session) close() error {
	s.closed = true
	if s.psc != nil {
		s.psc.PUnsubscribe()
	}

	return s.redis.Close()
}

// createEvent Creates the event with the required properties
func (s *Session) listenEvents(events chan Event) {
	for {
		if s.closed {
			break
		}
		switch n := s.psc.Receive().(type) {
		case gredis.PMessage:
			events <- s.createEvent(n)
		case error:
			s.errChan <- n
			return
		}
	}
}

// createEvent Creates the event with the required properties
func (s *Session) createEvent(n gredis.PMessage) Event {
	e := Event{}

	switch n.Pattern {
	case s.becameOfflinePattern:
		e.Id = string(n.Data[len(RedisencePrefix)+1:])
		e.Status = Offline
	case s.becameOnlinePattern:
		e.Id = string(n.Data[len(RedisencePrefix)+1:])
		e.Status = Online
	default:
		//ignore other events
	}

	return e
}
<|endoftext|>


<commit_before>package dnsr

import (
	"flag"
	"fmt"
	"os"
	"testing"

	"github.com/nbio/st"
)

var testResolver *Resolver

func init() {
	flag.Parse()
	if testing.Verbose() {
		DebugLogger = os.Stderr
	}
	testResolver = New(0)
	testResolve()
}

func TestSimple(t *testing.T) {
	r := New(0)
	_, err := r.ResolveErr("1.com", "")
	st.Expect(t, err, NXDOMAIN)
}

func TestResolverCache(t *testing.T) {
	r := New(0)
	r.cache.capacity = 10
	st.Expect(t, len(r.cache.entries), 0)
	for i := 0; i < 10; i++ {
		r.Resolve(fmt.Sprintf("%d.com", i), "")
	}
	st.Expect(t, len(r.cache.entries), 10)
	rrs, err := r.ResolveErr("a.com", "")
	st.Expect(t, err, NXDOMAIN)
	st.Expect(t, rrs, (RRs)(nil))
	st.Expect(t, r.cache.entries["a.com"], entry(nil))
	st.Expect(t, len(rrs), 0)
	st.Expect(t, len(r.cache.entries), 10)
}

func TestGoogleA(t *testing.T) {
	r := New(0)
	rrs, err := r.ResolveErr("google.com", "A")
	st.Expect(t, err, nil)
	st.Expect(t, len(rrs) >= 4, true)
	st.Expect(t, count(rrs, func(rr RR) bool { return rr.Type == "NS" }) >= 2, true)
	st.Expect(t, count(rrs, func(rr RR) bool { return rr.Type == "A" }) >= 1, true)
}

func TestGoogleAny(t *testing.T) {
	r := New(0)
	rrs, err := r.ResolveErr("google.com", "")
	st.Expect(t, err, nil)
	st.Expect(t, len(rrs) >= 1, true)
	st.Expect(t, count(rrs, func(rr RR) bool { return rr.Type == "NS" }) >= 2, true)
	st.Expect(t, count(rrs, func(rr RR) bool { return rr.Type == "A" }) >= 1, true)
}

func TestGoogleMulti(t *testing.T) {
	r := New(0)
	_, err := r.ResolveErr("google.com", "A")
	st.Expect(t, err, nil)
	rrs, err := r.ResolveErr("google.com", "TXT")
	st.Expect(t, err, nil)
	st.Expect(t, len(rrs) >= 1, true)
	st.Expect(t, count(rrs, func(rr RR) bool { return rr.Type == "TXT" }), 1)
	st.Expect(t, count(rrs, func(rr RR) bool { return rr.Type == "A" }), 0)
}

func TestGoogleTXT(t *testing.T) {
	r := New(0)
	rrs, err := r.ResolveErr("google.com", "TXT")
	st.Expect(t, err, nil)
	st.Expect(t, len(rrs) >= 4, true)
	st.Expect(t, count(rrs, func(rr RR) bool { return rr.Type == "TXT" }), 1)
}

func TestHerokuA(t *testing.T) {
	r := New(0)
	rrs, err := r.ResolveErr("us-east-1-a.route.herokuapp.com", "A")
	st.Expect(t, err, nil)
	st.Expect(t, count(rrs, func(rr RR) bool { return rr.Type == "A" }) >= 1, true)
}

func TestHerokuTXT(t *testing.T) {
	r := New(0)
	rrs, err := r.ResolveErr("us-east-1-a.route.herokuapp.com", "TXT")
	st.Expect(t, err, nil)
	st.Expect(t, count(rrs, func(rr RR) bool { return rr.Type == "TXT" }), 0)
}

func TestHerokuMulti(t *testing.T) {
	r := New(0)
	_, err := r.ResolveErr("us-east-1-a.route.herokuapp.com", "A")
	st.Expect(t, err, nil)
	rrs, err := r.ResolveErr("us-east-1-a.route.herokuapp.com", "TXT")
	st.Expect(t, err, nil)
	st.Expect(t, count(rrs, func(rr RR) bool { return rr.Type == "TXT" }), 0)
	st.Expect(t, count(rrs, func(rr RR) bool { return rr.Type == "A" }), 0)
}

func TestBlueOvenA(t *testing.T) {
	r := New(0)
	rrs, err := r.ResolveErr("blueoven.com", "A")
	st.Expect(t, err, nil)
	st.Expect(t, len(rrs), 2)
	st.Expect(t, count(rrs, func(rr RR) bool { return rr.Type == "NS" && rr.Name == "blueoven.com" }), 2)
}

func TestBlueOvenAny(t *testing.T) {
	r := New(0)
	rrs, err := r.ResolveErr("blueoven.com", "")
	st.Expect(t, err, nil)
	st.Expect(t, len(rrs), 2)
	st.Expect(t, count(rrs, func(rr RR) bool { return rr.Type == "NS" && rr.Name == "blueoven.com" }), 2)
}

func TestBlueOvenMulti(t *testing.T) {
	r := New(0)
	_, err := r.ResolveErr("blueoven.com", "A")
	st.Expect(t, err, nil)
	_, err = r.ResolveErr("blueoven.com", "TXT")
	st.Expect(t, err, nil)
	rrs, err := r.ResolveErr("blueoven.com", "")
	st.Expect(t, err, nil)
	st.Expect(t, len(rrs), 2)
	st.Expect(t, all(rrs, func(rr RR) bool { return rr.Type == "NS" }), true)
}

func TestBazCoUKAny(t *testing.T) {
	r := New(0)
	rrs, err := r.ResolveErr("baz.co.uk", "")
	st.Expect(t, err, nil)
	st.Expect(t, len(rrs) >= 2, true)
	st.Expect(t, count(rrs, func(rr RR) bool { return rr.Type == "NS" }) >= 2, true)
}

func BenchmarkResolve(b *testing.B) {
	for i := 0; i < b.N; i++ {
		testResolve()
	}
}

func BenchmarkResolveErr(b *testing.B) {
	for i := 0; i < b.N; i++ {
		testResolveErr()
	}
}

func testResolve() {
	testResolver.Resolve("google.com", "")
	testResolver.Resolve("blueoven.com", "")
	testResolver.Resolve("baz.co.uk", "")
	testResolver.Resolve("us-east-1-a.route.herokuapp.com", "")
}

func testResolveErr() {
	testResolver.ResolveErr("google.com", "")
	testResolver.ResolveErr("blueoven.com", "")
	testResolver.ResolveErr("baz.co.uk", "")
	testResolver.ResolveErr("us-east-1-a.route.herokuapp.com", "")
}

func count(rrs RRs, f func(RR) bool) (out int) {
	for _, rr := range rrs {
		if f(rr) {
			out++
		}
	}
	return
}

func sum(rrs RRs, f func(RR) int) (out int) {
	for _, rr := range rrs {
		out += f(rr)
	}
	return
}

func all(rrs RRs, f func(RR) bool) (out bool) {
	for _, rr := range rrs {
		if !f(rr) {
			return false
		}
	}
	return true
}
<commit_msg>Dot suffix.<commit_after>package dnsr

import (
	"flag"
	"fmt"
	"os"
	"testing"

	"github.com/nbio/st"
)

var testResolver *Resolver

func init() {
	flag.Parse()
	if testing.Verbose() {
		DebugLogger = os.Stderr
	}
	testResolver = New(0)
	testResolve()
}

func TestSimple(t *testing.T) {
	r := New(0)
	_, err := r.ResolveErr("1.com", "")
	st.Expect(t, err, NXDOMAIN)
}

func TestResolverCache(t *testing.T) {
	r := New(0)
	r.cache.capacity = 10
	st.Expect(t, len(r.cache.entries), 0)
	for i := 0; i < 10; i++ {
		r.Resolve(fmt.Sprintf("%d.com", i), "")
	}
	st.Expect(t, len(r.cache.entries), 10)
	rrs, err := r.ResolveErr("a.com", "")
	st.Expect(t, err, NXDOMAIN)
	st.Expect(t, rrs, (RRs)(nil))
	st.Expect(t, r.cache.entries["a.com"], entry(nil))
	st.Expect(t, len(rrs), 0)
	st.Expect(t, len(r.cache.entries), 10)
}

func TestGoogleA(t *testing.T) {
	r := New(0)
	rrs, err := r.ResolveErr("google.com", "A")
	st.Expect(t, err, nil)
	st.Expect(t, len(rrs) >= 4, true)
	st.Expect(t, count(rrs, func(rr RR) bool { return rr.Type == "NS" }) >= 2, true)
	st.Expect(t, count(rrs, func(rr RR) bool { return rr.Type == "A" }) >= 1, true)
}

func TestGoogleAny(t *testing.T) {
	r := New(0)
	rrs, err := r.ResolveErr("google.com", "")
	st.Expect(t, err, nil)
	st.Expect(t, len(rrs) >= 1, true)
	st.Expect(t, count(rrs, func(rr RR) bool { return rr.Type == "NS" }) >= 2, true)
	st.Expect(t, count(rrs, func(rr RR) bool { return rr.Type == "A" }) >= 1, true)
}

func TestGoogleMulti(t *testing.T) {
	r := New(0)
	_, err := r.ResolveErr("google.com", "A")
	st.Expect(t, err, nil)
	rrs, err := r.ResolveErr("google.com", "TXT")
	st.Expect(t, err, nil)
	st.Expect(t, len(rrs) >= 1, true)
	st.Expect(t, count(rrs, func(rr RR) bool { return rr.Type == "TXT" }), 1)
	st.Expect(t, count(rrs, func(rr RR) bool { return rr.Type == "A" }), 0)
}

func TestGoogleTXT(t *testing.T) {
	r := New(0)
	rrs, err := r.ResolveErr("google.com", "TXT")
	st.Expect(t, err, nil)
	st.Expect(t, len(rrs) >= 4, true)
	st.Expect(t, count(rrs, func(rr RR) bool { return rr.Type == "TXT" }), 1)
}

func TestHerokuA(t *testing.T) {
	r := New(0)
	rrs, err := r.ResolveErr("us-east-1-a.route.herokuapp.com", "A")
	st.Expect(t, err, nil)
	st.Expect(t, count(rrs, func(rr RR) bool { return rr.Type == "A" }) >= 1, true)
}

func TestHerokuTXT(t *testing.T) {
	r := New(0)
	rrs, err := r.ResolveErr("us-east-1-a.route.herokuapp.com", "TXT")
	st.Expect(t, err, nil)
	st.Expect(t, count(rrs, func(rr RR) bool { return rr.Type == "TXT" }), 0)
}

func TestHerokuMulti(t *testing.T) {
	r := New(0)
	_, err := r.ResolveErr("us-east-1-a.route.herokuapp.com", "A")
	st.Expect(t, err, nil)
	rrs, err := r.ResolveErr("us-east-1-a.route.herokuapp.com", "TXT")
	st.Expect(t, err, nil)
	st.Expect(t, count(rrs, func(rr RR) bool { return rr.Type == "TXT" }), 0)
	st.Expect(t, count(rrs, func(rr RR) bool { return rr.Type == "A" }), 0)
}

func TestBlueOvenA(t *testing.T) {
	r := New(0)
	rrs, err := r.ResolveErr("blueoven.com", "A")
	st.Expect(t, err, nil)
	st.Expect(t, len(rrs), 2)
	st.Expect(t, count(rrs, func(rr RR) bool { return rr.Type == "NS" && rr.Name == "blueoven.com." }), 2)
}

func TestBlueOvenAny(t *testing.T) {
	r := New(0)
	rrs, err := r.ResolveErr("blueoven.com", "")
	st.Expect(t, err, nil)
	st.Expect(t, len(rrs), 2)
	st.Expect(t, count(rrs, func(rr RR) bool { return rr.Type == "NS" && rr.Name == "blueoven.com." }), 2)
}

func TestBlueOvenMulti(t *testing.T) {
	r := New(0)
	_, err := r.ResolveErr("blueoven.com", "A")
	st.Expect(t, err, nil)
	_, err = r.ResolveErr("blueoven.com", "TXT")
	st.Expect(t, err, nil)
	rrs, err := r.ResolveErr("blueoven.com", "")
	st.Expect(t, err, nil)
	st.Expect(t, len(rrs), 2)
	st.Expect(t, all(rrs, func(rr RR) bool { return rr.Type == "NS" }), true)
}

func TestBazCoUKAny(t *testing.T) {
	r := New(0)
	rrs, err := r.ResolveErr("baz.co.uk", "")
	st.Expect(t, err, nil)
	st.Expect(t, len(rrs) >= 2, true)
	st.Expect(t, count(rrs, func(rr RR) bool { return rr.Type == "NS" }) >= 2, true)
}

func BenchmarkResolve(b *testing.B) {
	for i := 0; i < b.N; i++ {
		testResolve()
	}
}

func BenchmarkResolveErr(b *testing.B) {
	for i := 0; i < b.N; i++ {
		testResolveErr()
	}
}

func testResolve() {
	testResolver.Resolve("google.com", "")
	testResolver.Resolve("blueoven.com", "")
	testResolver.Resolve("baz.co.uk", "")
	testResolver.Resolve("us-east-1-a.route.herokuapp.com", "")
}

func testResolveErr() {
	testResolver.ResolveErr("google.com", "")
	testResolver.ResolveErr("blueoven.com", "")
	testResolver.ResolveErr("baz.co.uk", "")
	testResolver.ResolveErr("us-east-1-a.route.herokuapp.com", "")
}

func count(rrs RRs, f func(RR) bool) (out int) {
	for _, rr := range rrs {
		if f(rr) {
			out++
		}
	}
	return
}

func sum(rrs RRs, f func(RR) int) (out int) {
	for _, rr := range rrs {
		out += f(rr)
	}
	return
}

func all(rrs RRs, f func(RR) bool) (out bool) {
	for _, rr := range rrs {
		if !f(rr) {
			return false
		}
	}
	return true
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"socialapi/config"
	"socialapi/workers/common/handler"
	"socialapi/workers/common/mux"

	"github.com/koding/integration"
	"github.com/koding/runner"
)

const (
	Name = "WebhookMiddleware"
)

func main() {
	r := runner.New(Name)
	if err := r.Init(); err != nil {
		fmt.Println(err)
		return
	}

	appConfig := config.MustRead(r.Conf.Path)
	workerConfig := appConfig.WebhookMiddleware
	mc := mux.NewConfig(Name, workerConfig.Host, workerConfig.Port)
	m := mux.New(mc, r.Log, r.Metrics)
	path := appConfig.CustomDomain.Local

	h := integration.NewHandler(r.Log, path)

	if r.Conf.Environment == "dev" || r.Conf.Environment == "test" {
		path =
			fmt.Sprintf("http://%s:%s", appConfig.Integration.Host,
				appConfig.Integration.Port)
	}

	addHandlers(m, h)

	m.Listen()
	r.ShutdownHandler = m.Close

	r.Wait()
}

func addHandlers(m *mux.Mux, h *integration.Handler) {
	m.AddSessionlessHandler(
		handler.Request{
			Handler:  h.Push,
			Name:     "webhook-middleware-push",
			Type:     handler.PostRequest,
			Endpoint: "/push/{name}/{token}",
		},
	)
}
<commit_msg>webhook: update rootpath for dev and test environments<commit_after>package main

import (
	"fmt"
	"socialapi/config"
	"socialapi/workers/common/handler"
	"socialapi/workers/common/mux"

	"github.com/koding/integration"
	"github.com/koding/runner"
)

const (
	Name = "WebhookMiddleware"
)

func main() {
	r := runner.New(Name)
	if err := r.Init(); err != nil {
		fmt.Println(err)
		return
	}

	appConfig := config.MustRead(r.Conf.Path)
	workerConfig := appConfig.WebhookMiddleware
	mc := mux.NewConfig(Name, workerConfig.Host, workerConfig.Port)
	m := mux.New(mc, r.Log, r.Metrics)
	path := appConfig.CustomDomain.Local

	h := integration.NewHandler(r.Log, path)

	if r.Conf.Environment == "dev" || r.Conf.Environment == "test" {
		h.RootPath =
			fmt.Sprintf("http://%s:%s", appConfig.Integration.Host,
				appConfig.Integration.Port)
	}

	addHandlers(m, h)

	m.Listen()
	r.ShutdownHandler = m.Close

	r.Wait()
}

func addHandlers(m *mux.Mux, h *integration.Handler) {
	m.AddSessionlessHandler(
		handler.Request{
			Handler:  h.Push,
			Name:     "webhook-middleware-push",
			Type:     handler.PostRequest,
			Endpoint: "/push/{name}/{token}",
		},
	)
}
<|endoftext|>


<commit_before>//                           _       _
// __      _____  __ ___   ___  __ _| |_ ___
// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
//  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
//   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
//
//  Copyright © 2016 - 2019 SeMI Holding B.V. (registered @ Dutch Chamber of Commerce no 75221632). All rights reserved.
//  LICENSE WEAVIATE OPEN SOURCE: https://www.semi.technology/playbook/playbook/contract-weaviate-OSS.html
//  LICENSE WEAVIATE ENTERPRISE: https://www.semi.technology/playbook/contract-weaviate-enterprise.html
//  CONCEPT: Bob van Luijt (@bobvanluijt)
//  CONTACT: hello@semi.technology
//

package esvector

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"

	"github.com/elastic/go-elasticsearch/v5/esapi"
	"github.com/go-openapi/strfmt"
	"github.com/semi-technologies/weaviate/entities/schema/kind"
	"github.com/semi-technologies/weaviate/entities/search"
	"github.com/semi-technologies/weaviate/usecases/classification"
	"github.com/semi-technologies/weaviate/usecases/traverser"
)

func (r *Repo) GetUnclassified(ctx context.Context, kind kind.Kind,
	class string, properties []string) ([]search.Result, error) {

	mustNot := []map[string]interface{}{}
	for _, prop := range properties {
		mustNot = append(mustNot, map[string]interface{}{
			"exists": map[string]interface{}{
				"field": prop,
			},
		})

	}

	body := map[string]interface{}{
		"query": map[string]interface{}{
			"bool": map[string]interface{}{
				"must_not": mustNot,
			},
		},
		"size": 9999,
		"aggregations": map[string]interface{}{
			"count": map[string]interface{}{
				"value_count": map[string]interface{}{
					"field": "_id",
				},
			},
		},
	}

	var buf bytes.Buffer
	err := json.NewEncoder(&buf).Encode(body)
	if err != nil {
		return nil, fmt.Errorf("vector search: encode json: %v", err)
	}
	res, err := r.client.Search(
		r.client.Search.WithContext(ctx),
		r.client.Search.WithIndex(classIndexFromClassName(kind, class)),
		r.client.Search.WithBody(&buf),
	)
	if err != nil {
		return nil, fmt.Errorf("vector search: %v", err)
	}

	return r.unclassifiedSearchResponse(res, nil)
}

func (r *Repo) unclassifiedSearchResponse(res *esapi.Response, properties traverser.SelectProperties) ([]search.Result,
	error) {
	if err := errorResToErr(res, r.logger); err != nil {
		return nil, fmt.Errorf("vector search: %v", err)
	}

	var sr searchResponse
	defer res.Body.Close()
	err := json.NewDecoder(res.Body).Decode(&sr)
	if err != nil {
		return nil, fmt.Errorf("vector search: decode json: %v", err)
	}

	if err := checkClassificationCount(sr.Aggregations); err != nil {
		return nil, err
	}

	return sr.toResults(r, properties)
}

func checkClassificationCount(res map[string]interface{}) error {

	count, ok := res["count"]
	if !ok {
		return fmt.Errorf("get unclassified: expected 'count' aggregation, but got %v", res)
	}

	asMap, ok := count.(map[string]interface{})
	if !ok {
		return fmt.Errorf("get unclassified: expected 'count' to be map, got %T", count)
	}

	value, ok := asMap["value"]
	if !ok {
		return fmt.Errorf("get unclassified: expected 'count' to have key 'value', but got %v", count)
	}

	if int(value.(float64)) > 9999 {
		return fmt.Errorf("found more than 9999 unclassified items (%d), current supported maximum is 9999", int(value.(float64)))
	}

	return nil
}

func (r *Repo) AggregateNeighbors(ctx context.Context, vector []float32, kind kind.Kind, class string,
	properties []string, k int) ([]classification.NeighborRef, error) {

	propertyAggregations := map[string]interface{}{}
	mustExist := []map[string]interface{}{}
	for _, prop := range properties {
		propertyAggregations[prop] = map[string]interface{}{
			"terms": map[string]interface{}{
				"size":  1,
				"field": fmt.Sprintf("%s.beacon", prop),
			},
		}
		mustExist = append(mustExist, map[string]interface{}{
			"exists": map[string]interface{}{
				"field": prop,
			},
		})
	}

	aggregations := map[string]interface{}{
		"sample": map[string]interface{}{
			"sampler": map[string]interface{}{
				"shard_size": k,
			},
			"aggregations": propertyAggregations,
		},
	}

	query := map[string]interface{}{
		"function_score": map[string]interface{}{
			"boost_mode": "replace",
			"query": map[string]interface{}{
				"bool": map[string]interface{}{
					"must": mustExist,
				},
			},
			"functions": []interface{}{
				map[string]interface{}{
					"script_score": map[string]interface{}{
						"script": map[string]interface{}{
							"inline": "binary_vector_score",
							"lang":   "knn",
							"params": map[string]interface{}{
								"cosine": true,
								"field":  keyVector,
								"vector": vector,
							},
						},
					},
				},
			},
		},
	}

	body := map[string]interface{}{
		"query":        query,
		"aggregations": aggregations,
		"size":         0,
	}

	var buf bytes.Buffer
	err := json.NewEncoder(&buf).Encode(body)
	if err != nil {
		return nil, fmt.Errorf("vector search: encode json: %v", err)
	}

	res, err := r.client.Search(
		r.client.Search.WithContext(ctx),
		r.client.Search.WithIndex(classIndexFromClassName(kind, class)),
		r.client.Search.WithBody(&buf),
	)
	if err != nil {
		return nil, fmt.Errorf("vector search: %v", err)
	}

	return r.aggregateNeighborsResponse(res)
}

func (r *Repo) aggregateNeighborsResponse(res *esapi.Response) ([]classification.NeighborRef, error) {
	if err := errorResToErr(res, r.logger); err != nil {
		return nil, fmt.Errorf("neighbor aggregation: %v", err)
	}

	var sr searchResponse

	defer res.Body.Close()
	err := json.NewDecoder(res.Body).Decode(&sr)
	if err != nil {
		return nil, fmt.Errorf("neighbor aggregation: decode json: %v", err)
	}

	out, err := r.aggregationsToClassificationNeighborRefs(sr)
	if err != nil {
		return nil, fmt.Errorf("aggregate neighbors: %v", err)
	}
	return out, nil
}

func (r *Repo) aggregationsToClassificationNeighborRefs(input searchResponse) ([]classification.NeighborRef, error) {
	sample, ok := input.Aggregations["sample"]
	if !ok {
		return nil, fmt.Errorf("expected aggregation response to contain agg 'sample'")
	}

	asMap, ok := sample.(map[string]interface{})
	if !ok {
		return nil, fmt.Errorf("expected 'sample' to be map, got %T", sample)
	}

	var out []classification.NeighborRef
	for key, value := range asMap {
		if key == "doc_count" {
			continue
		}

		inner, err := extractInnerNeighborAgg(key, value)
		if err != nil {
			return nil, fmt.Errorf("for prop %s: %v", key, err)
		}

		out = append(out, inner)
	}

	return out, nil
}

func extractInnerNeighborAgg(prop string, agg interface{}) (classification.NeighborRef, error) {
	var out classification.NeighborRef
	asMap, ok := agg.(map[string]interface{})
	if !ok {
		return out, fmt.Errorf("expected inner agg to be map, got %T", agg)
	}

	buckets, ok := asMap["buckets"]
	if !ok {
		return out, fmt.Errorf("expected key 'buckets', got %v", asMap)
	}

	bucketsSlice, ok := buckets.([]interface{})
	if !ok {
		return out, fmt.Errorf("expected buckets to be a slice, got %T", buckets)
	}

	if len(bucketsSlice) != 1 {
		return out, fmt.Errorf("expected buckets to have len=1, got %#v", bucketsSlice)
	}

	bucketMap, ok := bucketsSlice[0].(map[string]interface{})
	if !ok {
		return out, fmt.Errorf("expected key 'buckets' to be map, got %T", bucketsSlice[0])
	}

	beacon, ok := bucketMap["key"]
	if !ok {
		return out, fmt.Errorf("expected bucket to have key 'key', got %v", bucketMap)
	}

	count, ok := bucketMap["doc_count"]
	if !ok {
		return out, fmt.Errorf("expected bucket to have key 'doc_count', got %v", bucketMap)
	}

	out.Beacon = strfmt.URI(beacon.(string))
	out.Count = int(count.(float64))
	out.Property = prop

	return out, nil
}
<commit_msg>gh-979 aggregate knn neighbors in classification in weaviate<commit_after>//                           _       _
// __      _____  __ ___   ___  __ _| |_ ___
// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
//  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
//   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
//
//  Copyright © 2016 - 2019 SeMI Holding B.V. (registered @ Dutch Chamber of Commerce no 75221632). All rights reserved.
//  LICENSE WEAVIATE OPEN SOURCE: https://www.semi.technology/playbook/playbook/contract-weaviate-OSS.html
//  LICENSE WEAVIATE ENTERPRISE: https://www.semi.technology/playbook/contract-weaviate-enterprise.html
//  CONCEPT: Bob van Luijt (@bobvanluijt)
//  CONTACT: hello@semi.technology
//

package esvector

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"

	"github.com/elastic/go-elasticsearch/v5/esapi"
	"github.com/go-openapi/strfmt"
	"github.com/semi-technologies/weaviate/entities/schema/kind"
	"github.com/semi-technologies/weaviate/entities/search"
	"github.com/semi-technologies/weaviate/usecases/classification"
	"github.com/semi-technologies/weaviate/usecases/traverser"
)

func (r *Repo) GetUnclassified(ctx context.Context, kind kind.Kind,
	class string, properties []string) ([]search.Result, error) {

	mustNot := []map[string]interface{}{}
	for _, prop := range properties {
		mustNot = append(mustNot, map[string]interface{}{
			"exists": map[string]interface{}{
				"field": prop,
			},
		})

	}

	body := map[string]interface{}{
		"query": map[string]interface{}{
			"bool": map[string]interface{}{
				"must_not": mustNot,
			},
		},
		"size": 9999,
		"aggregations": map[string]interface{}{
			"count": map[string]interface{}{
				"value_count": map[string]interface{}{
					"field": "_id",
				},
			},
		},
	}

	var buf bytes.Buffer
	err := json.NewEncoder(&buf).Encode(body)
	if err != nil {
		return nil, fmt.Errorf("vector search: encode json: %v", err)
	}
	res, err := r.client.Search(
		r.client.Search.WithContext(ctx),
		r.client.Search.WithIndex(classIndexFromClassName(kind, class)),
		r.client.Search.WithBody(&buf),
	)
	if err != nil {
		return nil, fmt.Errorf("vector search: %v", err)
	}

	return r.unclassifiedSearchResponse(res, nil)
}

func (r *Repo) unclassifiedSearchResponse(res *esapi.Response, properties traverser.SelectProperties) ([]search.Result,
	error) {
	if err := errorResToErr(res, r.logger); err != nil {
		return nil, fmt.Errorf("vector search: %v", err)
	}

	var sr searchResponse
	defer res.Body.Close()
	err := json.NewDecoder(res.Body).Decode(&sr)
	if err != nil {
		return nil, fmt.Errorf("vector search: decode json: %v", err)
	}

	if err := checkClassificationCount(sr.Aggregations); err != nil {
		return nil, err
	}

	return sr.toResults(r, properties)
}

func checkClassificationCount(res map[string]interface{}) error {

	count, ok := res["count"]
	if !ok {
		return fmt.Errorf("get unclassified: expected 'count' aggregation, but got %v", res)
	}

	asMap, ok := count.(map[string]interface{})
	if !ok {
		return fmt.Errorf("get unclassified: expected 'count' to be map, got %T", count)
	}

	value, ok := asMap["value"]
	if !ok {
		return fmt.Errorf("get unclassified: expected 'count' to have key 'value', but got %v", count)
	}

	if int(value.(float64)) > 9999 {
		return fmt.Errorf("found more than 9999 unclassified items (%d), current supported maximum is 9999", int(value.(float64)))
	}

	return nil
}

func (r *Repo) AggregateNeighbors(ctx context.Context, vector []float32, kind kind.Kind, class string,
	properties []string, k int) ([]classification.NeighborRef, error) {

	mustExist := []map[string]interface{}{}
	var propNames []string
	for _, prop := range properties {
		propNames = append(propNames, prop)
		mustExist = append(mustExist, map[string]interface{}{
			"exists": map[string]interface{}{
				"field": prop,
			},
		})
	}

	query := map[string]interface{}{
		"function_score": map[string]interface{}{
			"boost_mode": "replace",
			"query": map[string]interface{}{
				"bool": map[string]interface{}{
					"must": mustExist,
				},
			},
			"functions": []interface{}{
				map[string]interface{}{
					"script_score": map[string]interface{}{
						"script": map[string]interface{}{
							"inline": "binary_vector_score",
							"lang":   "knn",
							"params": map[string]interface{}{
								"cosine": true,
								"field":  keyVector,
								"vector": vector,
							},
						},
					},
				},
			},
		},
	}

	body := map[string]interface{}{
		"query":   query,
		"size":    k,
		"_source": append(propNames, keyVector.String()),
	}

	var buf bytes.Buffer
	err := json.NewEncoder(&buf).Encode(body)
	if err != nil {
		return nil, fmt.Errorf("vector search: encode json: %v", err)
	}

	res, err := r.client.Search(
		r.client.Search.WithContext(ctx),
		r.client.Search.WithIndex(classIndexFromClassName(kind, class)),
		r.client.Search.WithBody(&buf),
	)
	if err != nil {
		return nil, fmt.Errorf("vector search: %v", err)
	}

	return r.aggregateNeighborsResponse(res)
}

func (r *Repo) aggregateNeighborsResponse(res *esapi.Response) ([]classification.NeighborRef, error) {
	if err := errorResToErr(res, r.logger); err != nil {
		return nil, fmt.Errorf("neighbor aggregation: %v", err)
	}

	var sr searchResponse

	defer res.Body.Close()
	err := json.NewDecoder(res.Body).Decode(&sr)
	if err != nil {
		return nil, fmt.Errorf("neighbor aggregation: decode json: %v", err)
	}

	out, err := r.aggregationsToClassificationNeighborRefs(sr)
	if err != nil {
		return nil, fmt.Errorf("aggregate neighbors: %v", err)
	}
	return out, nil
}

func (r *Repo) aggregationsToClassificationNeighborRefs(input searchResponse) ([]classification.NeighborRef, error) {
	hits := input.Hits.Hits

	aggregations, err := extractRefNeighborsFromHits(hits)
	if err != nil {
		return nil, err
	}

	return aggregateRefNeighbors(aggregations)

}

func aggregateRefNeighbors(props map[string]map[string][]float64) ([]classification.NeighborRef, error) {
	var out []classification.NeighborRef
	for prop, beacons := range props {
		var winningBeacon string
		var winningCount int

		for beacon, distances := range beacons {
			if len(distances) > winningCount {
				winningBeacon = beacon
				winningCount = len(distances)
			}
		}

		out = append(out, classification.NeighborRef{
			Beacon:   strfmt.URI(winningBeacon),
			Count:    winningCount,
			Property: prop,
		})

	}

	return out, nil
}

func extractRefNeighborsFromHits(hits []hit) (map[string]map[string][]float64, error) {
	// structure is [prop][beacon][[]distance]
	aggregations := map[string]map[string][]float64{}

	for _, hit := range hits {

		for key, value := range hit.Source {
			if key == keyVector.String() {
				// ignore for now
				continue
			}

			// assume is a ref
			prop, ok := aggregations[key]
			if !ok {
				prop = map[string][]float64{}
			}

			beacon, err := extractBeaconFromProp(value)
			if err != nil {
				return nil, fmt.Errorf("prop %s: %v", key, err)
			}

			prop[beacon] = append(prop[beacon], 100)
			aggregations[key] = prop

		}
	}

	return aggregations, nil
}

func extractBeaconFromProp(prop interface{}) (string, error) {
	propSlice, ok := prop.([]interface{})
	if !ok {
		return "", fmt.Errorf("expected refs to be slice, got %T", prop)
	}

	if len(propSlice) != 1 {
		return "", fmt.Errorf("expected refs to have len 1, got %d", len(propSlice))
	}

	ref := propSlice[0]
	refMap, ok := ref.(map[string]interface{})
	if !ok {
		return "", fmt.Errorf("expected ref to be map, got %T", ref)
	}

	beacon, ok := refMap["beacon"]
	if !ok {
		return "", fmt.Errorf("expected ref (map) to have field 'beacon', got %v", refMap)
	}

	return beacon.(string), nil
}
<|endoftext|>


<commit_before>package gosseract

import (
	"io/ioutil"
	"os"
	"strings"
	"testing"

	"golang.org/x/net/html"

	. "github.com/otiai10/mint"
)

func TestVersion(t *testing.T) {
	version := Version()
	Expect(t, version).Match("[0-9]{1}.[0-9]{2}(.[0-9a-z]*)?")
}

func TestClearPersistentCache(t *testing.T) {
	client := NewClient()
	defer client.Close()
	client.init()
	ClearPersistentCache()
}

func TestNewClient(t *testing.T) {
	client := NewClient()
	defer client.Close()

	Expect(t, client).TypeOf("*gosseract.Client")
}

func TestClient_SetImage(t *testing.T) {
	client := NewClient()
	defer client.Close()

	client.Trim = true
	client.SetImage("./test/data/001-helloworld.png")

	client.SetPageSegMode(PSM_SINGLE_BLOCK)

	text, err := client.Text()
	if client.pixImage == nil {
		t.Errorf("could not set image")
	}
	Expect(t, err).ToBe(nil)
	Expect(t, text).ToBe("Hello, World!")

	err = client.SetImage("./test/data/001-helloworld.png")
	Expect(t, err).ToBe(nil)

	err = client.SetImage("")
	Expect(t, err).Not().ToBe(nil)

	err = client.SetImage("somewhere/fake/fakeimage.png")
	Expect(t, err).Not().ToBe(nil)

	_, err = client.Text()
	Expect(t, err).ToBe(nil)

	Because(t, "api must be initialized beforehand", func(t *testing.T) {
		client := &Client{}
		err := client.SetImage("./test/data/001-helloworld.png")
		Expect(t, err).Not().ToBe(nil)
	})
}

func TestClient_SetImageFromBytes(t *testing.T) {
	client := NewClient()
	defer client.Close()

	content, err := ioutil.ReadFile("./test/data/001-helloworld.png")
	if err != nil {
		t.Fatalf("could not read test file")
	}

	client.Trim = true
	client.SetImageFromBytes(content)

	client.SetPageSegMode(PSM_SINGLE_BLOCK)

	text, err := client.Text()
	if client.pixImage == nil {
		t.Errorf("could not set image")
	}
	Expect(t, err).ToBe(nil)
	Expect(t, text).ToBe("Hello, World!")
	err = client.SetImageFromBytes(content)
	Expect(t, err).ToBe(nil)

	err = client.SetImageFromBytes(nil)
	Expect(t, err).Not().ToBe(nil)

	Because(t, "api must be initialized beforehand", func(t *testing.T) {
		client := &Client{}
		err := client.SetImageFromBytes(content)
		Expect(t, err).Not().ToBe(nil)
	})
}

func TestClient_SetWhitelist(t *testing.T) {

	if os.Getenv("TESS_LSTM") == "1" {
		t.Skip("Whitelist with LSTM is not working for now. Please check https://github.com/tesseract-ocr/tesseract/issues/751")
	}

	client := NewClient()
	defer client.Close()

	client.Trim = true
	client.SetImage("./test/data/001-helloworld.png")
	client.Languages = []string{"eng"}
	client.SetWhitelist("HeloWrd,")
	text, err := client.Text()
	Expect(t, err).ToBe(nil)
	Expect(t, text).ToBe("Hello, Worldl")
}

func TestClient_SetBlacklist(t *testing.T) {

	if os.Getenv("TESS_LSTM") == "1" {
		t.Skip("Blacklist with LSTM is not working for now. Please check https://github.com/tesseract-ocr/tesseract/issues/751")
	}

	client := NewClient()
	defer client.Close()

	client.Trim = true
	err := client.SetImage("./test/data/001-helloworld.png")
	Expect(t, err).ToBe(nil)
	client.Languages = []string{"eng"}
	err = client.SetBlacklist("l")
	Expect(t, err).ToBe(nil)
	text, err := client.Text()
	Expect(t, err).ToBe(nil)
	Expect(t, text).ToBe("He110, WorId!")
}

func TestClient_SetLanguage(t *testing.T) {
	client := NewClient()
	defer client.Close()
	err := client.SetLanguage("deu")
	Expect(t, err).ToBe(nil)
	err = client.SetLanguage()
	Expect(t, err).Not().ToBe(nil)
	client.SetImage("./test/data/001-helloworld.png")
	_, err = client.Text()
	Expect(t, err).Not().ToBe(nil)
}

func TestClient_ConfigFilePath(t *testing.T) {

	if os.Getenv("TESS_LSTM") == "1" {
		t.Skip("Whitelist with LSTM is not working for now. Please check https://github.com/tesseract-ocr/tesseract/issues/751")
	}

	client := NewClient()
	defer client.Close()

	err := client.SetConfigFile("./test/config/01.config")
	Expect(t, err).ToBe(nil)
	client.SetImage("./test/data/001-helloworld.png")
	text, err := client.Text()
	Expect(t, err).ToBe(nil)

	Expect(t, text).ToBe("H    W   ")

	When(t, "the config file is not found", func(t *testing.T) {
		err := client.SetConfigFile("./test/config/not-existing")
		Expect(t, err).Not().ToBe(nil)
	})

	When(t, "the config file path is a directory", func(t *testing.T) {
		err := client.SetConfigFile("./test/config/02.config")
		Expect(t, err).Not().ToBe(nil)
	})

}

func TestClient_HTML(t *testing.T) {
	client := NewClient()
	defer client.Close()
	client.SetImage("./test/data/001-helloworld.png")
	client.SetWhitelist("Hello,World!")
	out, err := client.HOCRText()
	Expect(t, err).ToBe(nil)

	tokenizer := html.NewTokenizer(strings.NewReader(out))

	texts := []string{}
	for ttype := tokenizer.Next(); ttype != html.ErrorToken; ttype = tokenizer.Next() {
		token := tokenizer.Token()
		if token.Type == html.TextToken && strings.TrimSpace(token.Data) != "" {
			texts = append(texts, strings.Trim(token.Data, "\n"))
		}
	}
	Expect(t, texts).ToBe([]string{"Hello,", "World!"})

	When(t, "only invalid languages are given", func(t *testing.T) {
		client := NewClient()
		defer client.Close()
		client.SetLanguage("foo")
		client.SetImage("./test/data/001-helloworld.png")
		_, err := client.HOCRText()
		Expect(t, err).Not().ToBe(nil)
	})
	Because(t, "unknown key is validated when `init` is called", func(t *testing.T) {
		client := NewClient()
		defer client.Close()
		err := client.SetVariable("foobar", "hoge")
		Expect(t, err).ToBe(nil)
		client.SetImage("./test/data/001-helloworld.png")
		_, err = client.Text()
		Expect(t, err).Not().ToBe(nil)
	})
}
<commit_msg>test: add bounding_box test<commit_after>package gosseract

import (
	"image"
	"io/ioutil"
	"os"
	"strings"
	"testing"

	"golang.org/x/net/html"

	. "github.com/otiai10/mint"
)

func TestVersion(t *testing.T) {
	version := Version()
	Expect(t, version).Match("[0-9]{1}.[0-9]{2}(.[0-9a-z]*)?")
}

func TestClearPersistentCache(t *testing.T) {
	client := NewClient()
	defer client.Close()
	client.init()
	ClearPersistentCache()
}

func TestNewClient(t *testing.T) {
	client := NewClient()
	defer client.Close()

	Expect(t, client).TypeOf("*gosseract.Client")
}

func TestClient_SetImage(t *testing.T) {
	client := NewClient()
	defer client.Close()

	client.Trim = true
	client.SetImage("./test/data/001-helloworld.png")

	client.SetPageSegMode(PSM_SINGLE_BLOCK)

	text, err := client.Text()
	if client.pixImage == nil {
		t.Errorf("could not set image")
	}
	Expect(t, err).ToBe(nil)
	Expect(t, text).ToBe("Hello, World!")

	err = client.SetImage("./test/data/001-helloworld.png")
	Expect(t, err).ToBe(nil)

	err = client.SetImage("")
	Expect(t, err).Not().ToBe(nil)

	err = client.SetImage("somewhere/fake/fakeimage.png")
	Expect(t, err).Not().ToBe(nil)

	_, err = client.Text()
	Expect(t, err).ToBe(nil)

	Because(t, "api must be initialized beforehand", func(t *testing.T) {
		client := &Client{}
		err := client.SetImage("./test/data/001-helloworld.png")
		Expect(t, err).Not().ToBe(nil)
	})
}

func TestClient_SetImageFromBytes(t *testing.T) {
	client := NewClient()
	defer client.Close()

	content, err := ioutil.ReadFile("./test/data/001-helloworld.png")
	if err != nil {
		t.Fatalf("could not read test file")
	}

	client.Trim = true
	client.SetImageFromBytes(content)

	client.SetPageSegMode(PSM_SINGLE_BLOCK)

	text, err := client.Text()
	if client.pixImage == nil {
		t.Errorf("could not set image")
	}
	Expect(t, err).ToBe(nil)
	Expect(t, text).ToBe("Hello, World!")
	err = client.SetImageFromBytes(content)
	Expect(t, err).ToBe(nil)

	err = client.SetImageFromBytes(nil)
	Expect(t, err).Not().ToBe(nil)

	Because(t, "api must be initialized beforehand", func(t *testing.T) {
		client := &Client{}
		err := client.SetImageFromBytes(content)
		Expect(t, err).Not().ToBe(nil)
	})
}

func TestClient_SetWhitelist(t *testing.T) {

	if os.Getenv("TESS_LSTM") == "1" {
		t.Skip("Whitelist with LSTM is not working for now. Please check https://github.com/tesseract-ocr/tesseract/issues/751")
	}

	client := NewClient()
	defer client.Close()

	client.Trim = true
	client.SetImage("./test/data/001-helloworld.png")
	client.Languages = []string{"eng"}
	client.SetWhitelist("HeloWrd,")
	text, err := client.Text()
	Expect(t, err).ToBe(nil)
	Expect(t, text).ToBe("Hello, Worldl")
}

func TestClient_SetBlacklist(t *testing.T) {

	if os.Getenv("TESS_LSTM") == "1" {
		t.Skip("Blacklist with LSTM is not working for now. Please check https://github.com/tesseract-ocr/tesseract/issues/751")
	}

	client := NewClient()
	defer client.Close()

	client.Trim = true
	err := client.SetImage("./test/data/001-helloworld.png")
	Expect(t, err).ToBe(nil)
	client.Languages = []string{"eng"}
	err = client.SetBlacklist("l")
	Expect(t, err).ToBe(nil)
	text, err := client.Text()
	Expect(t, err).ToBe(nil)
	Expect(t, text).ToBe("He110, WorId!")
}

func TestClient_SetLanguage(t *testing.T) {
	client := NewClient()
	defer client.Close()
	err := client.SetLanguage("deu")
	Expect(t, err).ToBe(nil)
	err = client.SetLanguage()
	Expect(t, err).Not().ToBe(nil)
	client.SetImage("./test/data/001-helloworld.png")
	_, err = client.Text()
	Expect(t, err).Not().ToBe(nil)
}

func TestClient_ConfigFilePath(t *testing.T) {

	if os.Getenv("TESS_LSTM") == "1" {
		t.Skip("Whitelist with LSTM is not working for now. Please check https://github.com/tesseract-ocr/tesseract/issues/751")
	}

	client := NewClient()
	defer client.Close()

	err := client.SetConfigFile("./test/config/01.config")
	Expect(t, err).ToBe(nil)
	client.SetImage("./test/data/001-helloworld.png")
	text, err := client.Text()
	Expect(t, err).ToBe(nil)

	Expect(t, text).ToBe("H    W   ")

	When(t, "the config file is not found", func(t *testing.T) {
		err := client.SetConfigFile("./test/config/not-existing")
		Expect(t, err).Not().ToBe(nil)
	})

	When(t, "the config file path is a directory", func(t *testing.T) {
		err := client.SetConfigFile("./test/config/02.config")
		Expect(t, err).Not().ToBe(nil)
	})

}

func TestClientBoundingBox(t *testing.T) {
	client := NewClient()
	defer client.Close()
	client.SetImage("./test/data/001-helloworld.png")
	client.SetWhitelist("Hello,World!")
	boxes, err := client.GetBoundingBoxes()
	Expect(t, err).ToBe(nil)
	words := []string{"Hello,", "World!"}
	coords := []image.Rectangle{
		image.Rect(74, 64, 524, 190),
		image.Rect(638, 64, 1099, 170),
	}

	for i, box := range boxes {
		Expect(t, box.Word).ToBe(words[i])
		Expect(t, box.Box).ToBe(coords[i])
	}
}

func TestClient_HTML(t *testing.T) {
	client := NewClient()
	defer client.Close()
	client.SetImage("./test/data/001-helloworld.png")
	client.SetWhitelist("Hello,World!")
	out, err := client.HOCRText()
	Expect(t, err).ToBe(nil)

	tokenizer := html.NewTokenizer(strings.NewReader(out))

	texts := []string{}
	for ttype := tokenizer.Next(); ttype != html.ErrorToken; ttype = tokenizer.Next() {
		token := tokenizer.Token()
		if token.Type == html.TextToken && strings.TrimSpace(token.Data) != "" {
			texts = append(texts, strings.Trim(token.Data, "\n"))
		}
	}
	Expect(t, texts).ToBe([]string{"Hello,", "World!"})

	When(t, "only invalid languages are given", func(t *testing.T) {
		client := NewClient()
		defer client.Close()
		client.SetLanguage("foo")
		client.SetImage("./test/data/001-helloworld.png")
		_, err := client.HOCRText()
		Expect(t, err).Not().ToBe(nil)
	})
	Because(t, "unknown key is validated when `init` is called", func(t *testing.T) {
		client := NewClient()
		defer client.Close()
		err := client.SetVariable("foobar", "hoge")
		Expect(t, err).ToBe(nil)
		client.SetImage("./test/data/001-helloworld.png")
		_, err = client.Text()
		Expect(t, err).Not().ToBe(nil)
	})
}
<|endoftext|>


<commit_before>package acceptance

import (
	"encoding/json"
	"io"
	"io/ioutil"
	"os"
	"os/exec"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
	"github.com/pivotal-cf-experimental/pivnet-resource/concourse"
)

var _ = Describe("Acceptance", func() {
	var (
		productName    string
		productVersion string
		destDirectory  string
	)

	BeforeEach(func() {
		var err error
		productName = "push-notification-service"
		productVersion = "1.4.0"
		destDirectory, err = ioutil.TempDir("", "pivnet-resource")
		Expect(err).NotTo(HaveOccurred())
	})

	AfterEach(func() {
		os.RemoveAll(destDirectory)
	})

	Context("In", func() {
		It("successfully downloads all of the files in the specified release", func() {
			command := exec.Command(inPath, destDirectory)
			writer, err := command.StdinPipe()
			Expect(err).ShouldNot(HaveOccurred())

			raw, err := json.Marshal(concourse.Request{
				Source: concourse.Source{
					APIToken:    os.Getenv("API_TOKEN"),
					ProductName: productName,
				},
				Version: map[string]string{
					"product_version": productVersion,
				},
			})
			Expect(err).ShouldNot(HaveOccurred())

			session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
			Expect(err).NotTo(HaveOccurred())

			_, err = io.WriteString(writer, string(raw))
			Expect(err).ShouldNot(HaveOccurred())

			Eventually(session, "10s").Should(gexec.Exit(0))

			dataDir, err := os.Open(destDirectory)
			Expect(err).ShouldNot(HaveOccurred())

			files, err := dataDir.Readdir(2)
			Expect(err).ShouldNot(HaveOccurred())

			var fileNames []string
			for _, f := range files {
				fileNames = append(fileNames, f.Name())
				Expect(f.Size()).ToNot(BeZero())
			}

			Expect(fileNames).To(ConsistOf([]string{"PCFPush-1.4.0.aar", "PCFPush-1.4.0.framework.zip"}))
		})
	})
})
<commit_msg>Add acceptance test to validate in returns correct response.<commit_after>package acceptance

import (
	"encoding/json"
	"io"
	"io/ioutil"
	"os"
	"os/exec"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
	"github.com/pivotal-cf-experimental/pivnet-resource/concourse"
)

var _ = Describe("Acceptance", func() {
	var (
		productName    string
		productVersion string
		destDirectory  string
	)

	BeforeEach(func() {
		var err error
		productName = "push-notification-service"
		productVersion = "1.4.0"
		destDirectory, err = ioutil.TempDir("", "pivnet-resource")
		Expect(err).NotTo(HaveOccurred())
	})

	AfterEach(func() {
		os.RemoveAll(destDirectory)
	})

	Context("In", func() {
		It("returns valid json", func() {
			command := exec.Command(inPath, destDirectory)
			writer, err := command.StdinPipe()
			Expect(err).ShouldNot(HaveOccurred())

			raw, err := json.Marshal(concourse.Request{
				Source: concourse.Source{
					APIToken:    os.Getenv("API_TOKEN"),
					ProductName: productName,
				},
				Version: map[string]string{
					"product_version": productVersion,
				},
			})
			Expect(err).ShouldNot(HaveOccurred())

			session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
			Expect(err).NotTo(HaveOccurred())

			_, err = io.WriteString(writer, string(raw))
			Expect(err).ShouldNot(HaveOccurred())

			Eventually(session, "10s").Should(gexec.Exit(0))

			By("Outputting a valid json response")
			response := concourse.InResponse{}
			err = json.Unmarshal(session.Out.Contents(), &response)
			Expect(err).ShouldNot(HaveOccurred())

			Expect(response.Version.ProductVersion).To(Equal(productVersion))
		})

		It("successfully downloads all of the files in the specified release", func() {
			command := exec.Command(inPath, destDirectory)
			writer, err := command.StdinPipe()
			Expect(err).ShouldNot(HaveOccurred())

			raw, err := json.Marshal(concourse.Request{
				Source: concourse.Source{
					APIToken:    os.Getenv("API_TOKEN"),
					ProductName: productName,
				},
				Version: map[string]string{
					"product_version": productVersion,
				},
			})
			Expect(err).ShouldNot(HaveOccurred())

			session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
			Expect(err).NotTo(HaveOccurred())

			_, err = io.WriteString(writer, string(raw))
			Expect(err).ShouldNot(HaveOccurred())

			Eventually(session, "10s").Should(gexec.Exit(0))

			dataDir, err := os.Open(destDirectory)
			Expect(err).ShouldNot(HaveOccurred())

			files, err := dataDir.Readdir(2)
			Expect(err).ShouldNot(HaveOccurred())

			var fileNames []string
			for _, f := range files {
				fileNames = append(fileNames, f.Name())
				Expect(f.Size()).ToNot(BeZero())
			}

			Expect(fileNames).To(ConsistOf([]string{"PCFPush-1.4.0.aar", "PCFPush-1.4.0.framework.zip"}))
		})
	})
})
<|endoftext|>


<commit_before>package reporting

import (
	"github.com/smartystreets/goconvey/gotest"
	"runtime"
	"strings"
)

type AssertionResult struct {
	File       string
	Line       int
	Failure    string
	Error      interface{}
	StackTrace string
	Skipped    bool
}

func NewFailureReport(failure string) *AssertionResult {
	file, line := caller()
	stack := stackTrace()
	report := AssertionResult{file, line, failure, nil, stack, false}
	return &report
}
func NewErrorReport(err interface{}) *AssertionResult {
	file, line := caller()
	stack := fullStackTrace()
	report := AssertionResult{file, line, "", err, stack, false}
	return &report
}
func NewSuccessReport() *AssertionResult {
	file, line := caller()
	stack := stackTrace()
	report := AssertionResult{file, line, "", nil, stack, false}
	return &report
}
func NewSkipReport() *AssertionResult {
	file, line := caller()
	stack := stackTrace()
	report := AssertionResult{file, line, "", nil, stack, true}
	return &report
}

func caller() (file string, line int) {
	file, line, _ = gotest.ResolveExternalCaller()
	return
}
func stackTrace() string {
	buffer := make([]byte, 1024*64)
	runtime.Stack(buffer, false)
	formatted := strings.Trim(string(buffer), string([]byte{0}))
	return removeInternalEntries(formatted)
}
func fullStackTrace() string {
	buffer := make([]byte, 1024*64)
	runtime.Stack(buffer, true)
	formatted := strings.Trim(string(buffer), string([]byte{0}))
	return removeInternalEntries(formatted)
}
func removeInternalEntries(stack string) string {
	lines := strings.Split(stack, newline)
	filtered := []string{}
	for _, line := range lines {
		if !isExternal(line) {
			filtered = append(filtered, line)
		}
	}
	return strings.Join(filtered, newline)
}
func isExternal(line string) bool {
	for _, p := range internalPackages {
		if strings.Contains(line, p) {
			return true
		}
	}
	return false
}

// NOTE: any new packages that host goconvey packages will need to be added here!
// An alternative is to scan the goconvey directory and then exclude stuff like
// the examples package but that's nasty too.
var internalPackages = []string{
	"goconvey/assertions",
	"goconvey/convey",
	"goconvey/execution",
	"goconvey/gotest",
	"goconvey/printing",
	"goconvey/reporting",
}
<commit_msg>Refactored report creation methods to allow for more flexible addition/deletion of AssertionResult fields.<commit_after>package reporting

import (
	"github.com/smartystreets/goconvey/gotest"
	"runtime"
	"strings"
)

type AssertionResult struct {
	File string
	Line int
	// Expected   string // TODO: unmarshall
	// Actual     string // TODO: unmarshall
	Failure    string
	Error      interface{}
	StackTrace string
	Skipped    bool
}

func NewFailureReport(failure string) *AssertionResult {
	report := &AssertionResult{}
	report.File, report.Line = caller()
	report.StackTrace = stackTrace()
	report.Failure = failure
	// TODO: unmarshall string if possible, assign to Expected, Actual, and Failure
	return report
}
func NewErrorReport(err interface{}) *AssertionResult {
	report := &AssertionResult{}
	report.File, report.Line = caller()
	report.StackTrace = fullStackTrace()
	report.Error = err
	return report
}
func NewSuccessReport() *AssertionResult {
	report := &AssertionResult{}
	report.File, report.Line = caller()
	report.StackTrace = fullStackTrace()
	return report
}
func NewSkipReport() *AssertionResult {
	report := &AssertionResult{}
	report.File, report.Line = caller()
	report.StackTrace = fullStackTrace()
	report.Skipped = true
	return report
}

func caller() (file string, line int) {
	file, line, _ = gotest.ResolveExternalCaller()
	return
}
func stackTrace() string {
	buffer := make([]byte, 1024*64)
	runtime.Stack(buffer, false)
	formatted := strings.Trim(string(buffer), string([]byte{0}))
	return removeInternalEntries(formatted)
}
func fullStackTrace() string {
	buffer := make([]byte, 1024*64)
	runtime.Stack(buffer, true)
	formatted := strings.Trim(string(buffer), string([]byte{0}))
	return removeInternalEntries(formatted)
}
func removeInternalEntries(stack string) string {
	lines := strings.Split(stack, newline)
	filtered := []string{}
	for _, line := range lines {
		if !isExternal(line) {
			filtered = append(filtered, line)
		}
	}
	return strings.Join(filtered, newline)
}
func isExternal(line string) bool {
	for _, p := range internalPackages {
		if strings.Contains(line, p) {
			return true
		}
	}
	return false
}

// NOTE: any new packages that host goconvey packages will need to be added here!
// An alternative is to scan the goconvey directory and then exclude stuff like
// the examples package but that's nasty too.
var internalPackages = []string{
	"goconvey/assertions",
	"goconvey/convey",
	"goconvey/execution",
	"goconvey/gotest",
	"goconvey/printing",
	"goconvey/reporting",
}
<|endoftext|>


<commit_before>package httputilmore

const (
	HeaderAccept                       = "Accept"
	HeaderAuthorization                = "Authorization"
	HeaderCacheControl                 = "Cache-Control"
	HeaderContentDisposition           = "Content-Disposition"
	HeaderContentEncoding              = "Content-Encoding"
	HeaderContentLanguage              = "Content-Language"
	HeaderContentLength                = "Content-Length"
	HeaderContentMD5                   = "Content-MD5"
	HeaderContentTransferEncoding      = "Content-Transfer-Encoding"
	HeaderContentType                  = "Content-Type"
	HeaderDate                         = "Date"
	HeaderIfMatch                      = "If-Match"
	headerIfModifiedSince              = "If-Modified-Since"
	HeaderIfNoneMatch                  = "If-None-Match"
	HeaderIfUnmodifiedSince            = "If-Unmodified-Since"
	HeaderLocation                     = "Location"
	HeaderRange                        = "Range"
	HeaderUserAgent                    = "User-Agent"
	HeaderXContentTypeOptions          = "X-Content-Type-Options"
	ContentTypeAppJsonUtf8             = "application/json; charset=utf-8"
	ContentTypeAppFormUrlEncoded       = "application/x-www-form-urlencoded"
	ContentTypeAppXml                  = "application/xml"
	ContentTypeAppXmlUtf8              = "application/xml; charset=utf-8"
	ContentTypeTextCalendarUtf8Request = `text/calendar; charset="utf-8"; method=REQUEST`
	ContentTypeTextHtmlUtf8            = "text/html; charset=utf-8"
	ContentTypeTextPlainUsAscii        = "text/plain; charset=us-ascii"
	ContentTypeTextPlainUtf8           = "text/plain; charset=utf-8"
	ContentTypeTextXmlUtf8             = "text/xml; charset=utf-8"
	SchemeHTTPS                        = "https"
)
<commit_msg>add comment<commit_after>package httputilmore

// Constants ensuring that header names are correctly spelled and consistently cased.
const (
	HeaderAccept                       = "Accept"
	HeaderAuthorization                = "Authorization"
	HeaderCacheControl                 = "Cache-Control"
	HeaderContentDisposition           = "Content-Disposition"
	HeaderContentEncoding              = "Content-Encoding"
	HeaderContentLanguage              = "Content-Language"
	HeaderContentLength                = "Content-Length"
	HeaderContentMD5                   = "Content-MD5"
	HeaderContentTransferEncoding      = "Content-Transfer-Encoding"
	HeaderContentType                  = "Content-Type"
	HeaderDate                         = "Date"
	HeaderIfMatch                      = "If-Match"
	headerIfModifiedSince              = "If-Modified-Since"
	HeaderIfNoneMatch                  = "If-None-Match"
	HeaderIfUnmodifiedSince            = "If-Unmodified-Since"
	HeaderLocation                     = "Location"
	HeaderRange                        = "Range"
	HeaderUserAgent                    = "User-Agent"
	HeaderXContentTypeOptions          = "X-Content-Type-Options"
	ContentTypeAppJsonUtf8             = "application/json; charset=utf-8"
	ContentTypeAppFormUrlEncoded       = "application/x-www-form-urlencoded"
	ContentTypeAppXml                  = "application/xml"
	ContentTypeAppXmlUtf8              = "application/xml; charset=utf-8"
	ContentTypeTextCalendarUtf8Request = `text/calendar; charset="utf-8"; method=REQUEST`
	ContentTypeTextHtmlUtf8            = "text/html; charset=utf-8"
	ContentTypeTextPlainUsAscii        = "text/plain; charset=us-ascii"
	ContentTypeTextPlainUtf8           = "text/plain; charset=utf-8"
	ContentTypeTextXmlUtf8             = "text/xml; charset=utf-8"
	SchemeHTTPS                        = "https"
)
<|endoftext|>