<commit_before>// Copyright (c) Liam Stanley <me@liamstanley.io>. All rights reserved. Use
// of this source code is governed by the MIT license that can be found in
// the LICENSE file.

package girc

import (
	"fmt"
	"strings"
	"sync"
	"time"
)

// state represents the actively-changing variables within the client
// runtime.
type state struct {
	// m is a RW mutex lock, used to guard the state from goroutines causing
	// corruption.
	mu sync.RWMutex
	// nick, ident, and host are the internal trackers for our user.
	nick, ident, host string
	// channels represents all channels we're active in.
	channels map[string]*Channel
	// enabledCap are the capabilities which are enabled for this connection.
	enabledCap []string
	// tmpCap are the capabilties which we share with the server during the
	// last capability check. These will get sent once we have received the
	// last capability list command from the server.
	tmpCap []string
	// serverOptions are the standard capabilities and configurations
	// supported by the server at connection time. This also includes
	// RPL_ISUPPORT entries.
	serverOptions map[string]string
	// motd is the servers message of the day.
	motd string
}

func (s *state) clean() {
	s.mu.Lock()
	s.nick = ""
	s.ident = ""
	s.host = ""
	s.channels = make(map[string]*Channel)
	s.serverOptions = make(map[string]string)
	s.enabledCap = []string{}
	s.motd = ""
	s.mu.Unlock()
}

// User represents an IRC user and the state attached to them.
type User struct {
	// Nick is the users current nickname.
	Nick string
	// Ident is the users username/ident. Ident is commonly prefixed with a
	// "~", which indicates that they do not have a identd server setup for
	// authentication.
	Ident string
	// Host is the visible host of the users connection that the server has
	// provided to us for their connection. May not always be accurate due to
	// many networks spoofing/hiding parts of the hostname for privacy
	// reasons.
	Host string

	// FirstSeen represents the first time that the user was seen by the
	// client for the given channel. Only usable if from state, not in past.
	FirstSeen time.Time
	// LastActive represents the last time that we saw the user active,
	// which could be during nickname change, message, channel join, etc.
	// Only usable if from state, not in past.
	LastActive time.Time

	// Perms are the user permissions applied to this user that affect the given
	// channel. This supports non-rfc style modes like Admin, Owner, and HalfOp.
	// If you want to easily check if a user has permissions equal or greater
	// than OP, use Perms.IsAdmin().
	Perms UserPerms

	// Extras are things added on by additional tracking methods, which may
	// or may not work on the IRC server in mention.
	Extras struct {
		// Name is the users "realname" or full name. Commonly contains links
		// to the IRC client being used, or something of non-importance. May
		// also be empty if unsupported by the server/tracking is disabled.
		Name string
		// Account refers to the account which the user is authenticated as.
		// This differs between each network (e.g. usually Nickserv, but
		// could also be something like Undernet). May also be empty if
		// unsupported by the server/tracking is disabled.
		Account string
		// Away refers to the away status of the user. An empty string
		// indicates that they are active, otherwise the string is what they
		// set as their away message. May also be empty if unsupported by the
		// server/tracking is disabled.
		Away string
	}
}

// Message returns an event which can be used to send a response to the user
// as a private message.
func (u *User) Message(message string) *Event {
	return &Event{Command: PRIVMSG, Params: []string{u.Nick}, Trailing: message}
}

// Messagef returns an event which can be used to send a response to the user
// as a private message. format is a printf format string, which a's
// arbitrary arguments will be passed to.
func (u *User) Messagef(format string, a ...interface{}) *Event {
	return u.Message(fmt.Sprintf(format, a...))
}

// MessageTo returns an event which can be used to send a response to the
// user in a channel as a private message.
func (u *User) MessageTo(channel, message string) *Event {
	return &Event{Command: PRIVMSG, Params: []string{u.Nick}, Trailing: channel + ": " + message}
}

// MessageTof returns an event which can be used to send a response to the
// channel. format is a printf format string, which a's arbitrary arguments
// will be passed to.
func (u *User) MessageTof(channel, format string, a ...interface{}) *Event {
	return u.MessageTo(channel, fmt.Sprintf(format, a...))
}

// Lifetime represents the amount of time that has passed since we have first
// seen the user.
func (u *User) Lifetime() time.Duration {
	return time.Since(u.FirstSeen)
}

// Active represents the the amount of time that has passed since we have
// last seen the user.
func (u *User) Active() time.Duration {
	return time.Since(u.LastActive)
}

// IsActive returns true if they were active within the last 30 minutes.
func (u *User) IsActive() bool {
	return u.Active() < (time.Minute * 30)
}

// Channel represents an IRC channel and the state attached to it.
type Channel struct {
	// Name of the channel. Must be rfc compliant. Always represented as
	// lower-case, to ensure that the channel is only being tracked once.
	Name string
	// Topic of the channel.
	Topic string
	// users represents the users that we can currently see within the
	// channel.
	users map[string]*User
	// Joined represents the first time that the client joined the channel.
	Joined time.Time
	// Modes are the known channel modes that the bot has captured.
	Modes CModes
}

// Copy returns a deep copy of a given channel.
func (c *Channel) Copy() *Channel {
	nc := &Channel{}
	*nc = *c

	// Copy the users.
	nc.users = make(map[string]*User)
	for k, v := range c.users {
		nc.users[k] = v
	}

	// And modes.
	nc.Modes = c.Modes.Copy()

	return nc
}

// Users returns a list of users in a given channel.
func (c *Channel) Users() []*User {
	out := make([]*User, len(c.users))

	var index int
	for _, u := range c.users {
		out[index] = u

		index++
	}

	return out
}

// NickList returns a list of nicknames in a given channel.
func (c *Channel) NickList() []string {
	out := make([]string, len(c.users))

	var index int
	for k := range c.users {
		out[index] = k

		index++
	}

	return out
}

// Len returns the count of users in a given channel.
func (c *Channel) Len() int {
	return len(c.users)
}

// Lookup looks up a user in a channel based on a given nickname. If the
// user wasn't found, user is nil.
func (c *Channel) Lookup(nick string) *User {
	for k, v := range c.users {
		if ToRFC1459(k) == ToRFC1459(nick) {
			// No need to have a copy, as if one has access to a channel,
			// should already have a full copy.
			return v
		}
	}

	return nil
}

// Message returns an event which can be used to send a response to the channel.
func (c *Channel) Message(message string) *Event {
	return &Event{Command: PRIVMSG, Params: []string{c.Name}, Trailing: message}
}

// Messagef returns an event which can be used to send a response to the
// channel. format is a printf format string, which a's arbitrary arguments
// will be passed to.
func (c *Channel) Messagef(format string, a ...interface{}) *Event {
	return c.Message(fmt.Sprintf(format, a...))
}

// Lifetime represents the amount of time that has passed since we have first
// joined the channel.
func (c *Channel) Lifetime() time.Duration {
	return time.Since(c.Joined)
}

// createChanIfNotExists creates the channel in state, if not already done.
// Always use state.mu for transaction.
func (s *state) createChanIfNotExists(name string) (channel *Channel) {
	// Not a valid channel.
	if !IsValidChannel(name) {
		return nil
	}

	supported := s.chanModes()
	prefixes, _ := parsePrefixes(s.userPrefixes())

	name = strings.ToLower(name)
	if _, ok := s.channels[name]; !ok {
		channel = &Channel{
			Name:   name,
			users:  make(map[string]*User),
			Joined: time.Now(),
			Modes:  NewCModes(supported, prefixes),
		}
		s.channels[name] = channel
	} else {
		channel = s.channels[name]
	}

	return channel
}

// deleteChannel removes the channel from state, if not already done. Always
// use state.mu for transaction.
func (s *state) deleteChannel(name string) {
	channel := s.createChanIfNotExists(name)
	if channel == nil {
		return
	}

	if _, ok := s.channels[channel.Name]; ok {
		delete(s.channels, channel.Name)
	}
}

// lookupChannel returns a reference to a channel with a given case-insensitive
// name. nil returned if no results found.
func (s *state) lookupChannel(name string) *Channel {
	if !IsValidChannel(name) {
		return nil
	}

	return s.channels[strings.ToLower(name)]
}

// createUserIfNotExists creates the channel and user in state, if not already
// done. Always use state.mu for transaction.
func (s *state) createUserIfNotExists(channelName, nick string) (user *User) {
	if !IsValidNick(nick) {
		return nil
	}

	channel := s.createChanIfNotExists(channelName)
	if channel == nil {
		return nil
	}

	if _, ok := channel.users[nick]; ok {
		channel.users[nick].LastActive = time.Now()
		return channel.users[nick]
	}

	user = &User{Nick: nick, FirstSeen: time.Now(), LastActive: time.Now()}
	channel.users[nick] = user

	return user
}

// deleteUser removes the user from channel state. Always use state.mu for
// transaction.
func (s *state) deleteUser(nick string) {
	if !IsValidNick(nick) {
		return
	}

	for k := range s.channels {
		// Check to see if they're in this channel.
		if _, ok := s.channels[k].users[nick]; !ok {
			continue
		}

		delete(s.channels[k].users, nick)
	}
}

// renameUser renames the user in state, in all locations where relevant.
// Always use state.mu for transaction.
func (s *state) renameUser(from, to string) {
	if !IsValidNick(from) || !IsValidNick(to) {
		return
	}

	// Update our nickname.
	if from == s.nick {
		s.nick = to
	}

	for k := range s.channels {
		// Check to see if they're in this channel.
		if _, ok := s.channels[k].users[from]; !ok {
			continue
		}

		// Take the actual reference to the pointer.
		source := *s.channels[k].users[from]

		// Update the nick field (as we not only have a key, but a matching
		// struct field).
		source.Nick = to
		source.LastActive = time.Now()

		// Delete the old reference.
		delete(s.channels[k].users, from)

		// In with the new.
		s.channels[k].users[to] = &source
	}
}

// lookupUsers returns a slice of references to users matching a given
// query. mathType is of "nick", "name", "ident" or "account".
func (s *state) lookupUsers(matchType, toMatch string) []*User {
	var users []*User

	for c := range s.channels {
		for u := range s.channels[c].users {
			switch matchType {
			case "nick":
				if ToRFC1459(s.channels[c].users[u].Nick) == ToRFC1459(toMatch) {
					users = append(users, s.channels[c].users[u])
					continue
				}
			case "ident":
				if ToRFC1459(s.channels[c].users[u].Ident) == ToRFC1459(toMatch) {
					users = append(users, s.channels[c].users[u])
					continue
				}
			case "account":
				if ToRFC1459(s.channels[c].users[u].Extras.Account) == ToRFC1459(toMatch) {
					users = append(users, s.channels[c].users[u])
					continue
				}
			}
		}
	}

	return users
}
<commit_msg>remove message and messagef related functions from events<commit_after>// Copyright (c) Liam Stanley <me@liamstanley.io>. All rights reserved. Use
// of this source code is governed by the MIT license that can be found in
// the LICENSE file.

package girc

import (
	"strings"
	"sync"
	"time"
)

// state represents the actively-changing variables within the client
// runtime.
type state struct {
	// m is a RW mutex lock, used to guard the state from goroutines causing
	// corruption.
	mu sync.RWMutex
	// nick, ident, and host are the internal trackers for our user.
	nick, ident, host string
	// channels represents all channels we're active in.
	channels map[string]*Channel
	// enabledCap are the capabilities which are enabled for this connection.
	enabledCap []string
	// tmpCap are the capabilties which we share with the server during the
	// last capability check. These will get sent once we have received the
	// last capability list command from the server.
	tmpCap []string
	// serverOptions are the standard capabilities and configurations
	// supported by the server at connection time. This also includes
	// RPL_ISUPPORT entries.
	serverOptions map[string]string
	// motd is the servers message of the day.
	motd string
}

func (s *state) clean() {
	s.mu.Lock()
	s.nick = ""
	s.ident = ""
	s.host = ""
	s.channels = make(map[string]*Channel)
	s.serverOptions = make(map[string]string)
	s.enabledCap = []string{}
	s.motd = ""
	s.mu.Unlock()
}

// User represents an IRC user and the state attached to them.
type User struct {
	// Nick is the users current nickname.
	Nick string
	// Ident is the users username/ident. Ident is commonly prefixed with a
	// "~", which indicates that they do not have a identd server setup for
	// authentication.
	Ident string
	// Host is the visible host of the users connection that the server has
	// provided to us for their connection. May not always be accurate due to
	// many networks spoofing/hiding parts of the hostname for privacy
	// reasons.
	Host string

	// FirstSeen represents the first time that the user was seen by the
	// client for the given channel. Only usable if from state, not in past.
	FirstSeen time.Time
	// LastActive represents the last time that we saw the user active,
	// which could be during nickname change, message, channel join, etc.
	// Only usable if from state, not in past.
	LastActive time.Time

	// Perms are the user permissions applied to this user that affect the given
	// channel. This supports non-rfc style modes like Admin, Owner, and HalfOp.
	// If you want to easily check if a user has permissions equal or greater
	// than OP, use Perms.IsAdmin().
	Perms UserPerms

	// Extras are things added on by additional tracking methods, which may
	// or may not work on the IRC server in mention.
	Extras struct {
		// Name is the users "realname" or full name. Commonly contains links
		// to the IRC client being used, or something of non-importance. May
		// also be empty if unsupported by the server/tracking is disabled.
		Name string
		// Account refers to the account which the user is authenticated as.
		// This differs between each network (e.g. usually Nickserv, but
		// could also be something like Undernet). May also be empty if
		// unsupported by the server/tracking is disabled.
		Account string
		// Away refers to the away status of the user. An empty string
		// indicates that they are active, otherwise the string is what they
		// set as their away message. May also be empty if unsupported by the
		// server/tracking is disabled.
		Away string
	}
}

// Lifetime represents the amount of time that has passed since we have first
// seen the user.
func (u *User) Lifetime() time.Duration {
	return time.Since(u.FirstSeen)
}

// Active represents the the amount of time that has passed since we have
// last seen the user.
func (u *User) Active() time.Duration {
	return time.Since(u.LastActive)
}

// IsActive returns true if they were active within the last 30 minutes.
func (u *User) IsActive() bool {
	return u.Active() < (time.Minute * 30)
}

// Channel represents an IRC channel and the state attached to it.
type Channel struct {
	// Name of the channel. Must be rfc compliant. Always represented as
	// lower-case, to ensure that the channel is only being tracked once.
	Name string
	// Topic of the channel.
	Topic string
	// users represents the users that we can currently see within the
	// channel.
	users map[string]*User
	// Joined represents the first time that the client joined the channel.
	Joined time.Time
	// Modes are the known channel modes that the bot has captured.
	Modes CModes
}

// Copy returns a deep copy of a given channel.
func (c *Channel) Copy() *Channel {
	nc := &Channel{}
	*nc = *c

	// Copy the users.
	nc.users = make(map[string]*User)
	for k, v := range c.users {
		nc.users[k] = v
	}

	// And modes.
	nc.Modes = c.Modes.Copy()

	return nc
}

// Users returns a list of users in a given channel.
func (c *Channel) Users() []*User {
	out := make([]*User, len(c.users))

	var index int
	for _, u := range c.users {
		out[index] = u

		index++
	}

	return out
}

// NickList returns a list of nicknames in a given channel.
func (c *Channel) NickList() []string {
	out := make([]string, len(c.users))

	var index int
	for k := range c.users {
		out[index] = k

		index++
	}

	return out
}

// Len returns the count of users in a given channel.
func (c *Channel) Len() int {
	return len(c.users)
}

// Lookup looks up a user in a channel based on a given nickname. If the
// user wasn't found, user is nil.
func (c *Channel) Lookup(nick string) *User {
	for k, v := range c.users {
		if ToRFC1459(k) == ToRFC1459(nick) {
			// No need to have a copy, as if one has access to a channel,
			// should already have a full copy.
			return v
		}
	}

	return nil
}

// Lifetime represents the amount of time that has passed since we have first
// joined the channel.
func (c *Channel) Lifetime() time.Duration {
	return time.Since(c.Joined)
}

// createChanIfNotExists creates the channel in state, if not already done.
// Always use state.mu for transaction.
func (s *state) createChanIfNotExists(name string) (channel *Channel) {
	// Not a valid channel.
	if !IsValidChannel(name) {
		return nil
	}

	supported := s.chanModes()
	prefixes, _ := parsePrefixes(s.userPrefixes())

	name = strings.ToLower(name)
	if _, ok := s.channels[name]; !ok {
		channel = &Channel{
			Name:   name,
			users:  make(map[string]*User),
			Joined: time.Now(),
			Modes:  NewCModes(supported, prefixes),
		}
		s.channels[name] = channel
	} else {
		channel = s.channels[name]
	}

	return channel
}

// deleteChannel removes the channel from state, if not already done. Always
// use state.mu for transaction.
func (s *state) deleteChannel(name string) {
	channel := s.createChanIfNotExists(name)
	if channel == nil {
		return
	}

	if _, ok := s.channels[channel.Name]; ok {
		delete(s.channels, channel.Name)
	}
}

// lookupChannel returns a reference to a channel with a given case-insensitive
// name. nil returned if no results found.
func (s *state) lookupChannel(name string) *Channel {
	if !IsValidChannel(name) {
		return nil
	}

	return s.channels[strings.ToLower(name)]
}

// createUserIfNotExists creates the channel and user in state, if not already
// done. Always use state.mu for transaction.
func (s *state) createUserIfNotExists(channelName, nick string) (user *User) {
	if !IsValidNick(nick) {
		return nil
	}

	channel := s.createChanIfNotExists(channelName)
	if channel == nil {
		return nil
	}

	if _, ok := channel.users[nick]; ok {
		channel.users[nick].LastActive = time.Now()
		return channel.users[nick]
	}

	user = &User{Nick: nick, FirstSeen: time.Now(), LastActive: time.Now()}
	channel.users[nick] = user

	return user
}

// deleteUser removes the user from channel state. Always use state.mu for
// transaction.
func (s *state) deleteUser(nick string) {
	if !IsValidNick(nick) {
		return
	}

	for k := range s.channels {
		// Check to see if they're in this channel.
		if _, ok := s.channels[k].users[nick]; !ok {
			continue
		}

		delete(s.channels[k].users, nick)
	}
}

// renameUser renames the user in state, in all locations where relevant.
// Always use state.mu for transaction.
func (s *state) renameUser(from, to string) {
	if !IsValidNick(from) || !IsValidNick(to) {
		return
	}

	// Update our nickname.
	if from == s.nick {
		s.nick = to
	}

	for k := range s.channels {
		// Check to see if they're in this channel.
		if _, ok := s.channels[k].users[from]; !ok {
			continue
		}

		// Take the actual reference to the pointer.
		source := *s.channels[k].users[from]

		// Update the nick field (as we not only have a key, but a matching
		// struct field).
		source.Nick = to
		source.LastActive = time.Now()

		// Delete the old reference.
		delete(s.channels[k].users, from)

		// In with the new.
		s.channels[k].users[to] = &source
	}
}

// lookupUsers returns a slice of references to users matching a given
// query. mathType is of "nick", "name", "ident" or "account".
func (s *state) lookupUsers(matchType, toMatch string) []*User {
	var users []*User

	for c := range s.channels {
		for u := range s.channels[c].users {
			switch matchType {
			case "nick":
				if ToRFC1459(s.channels[c].users[u].Nick) == ToRFC1459(toMatch) {
					users = append(users, s.channels[c].users[u])
					continue
				}
			case "ident":
				if ToRFC1459(s.channels[c].users[u].Ident) == ToRFC1459(toMatch) {
					users = append(users, s.channels[c].users[u])
					continue
				}
			case "account":
				if ToRFC1459(s.channels[c].users[u].Extras.Account) == ToRFC1459(toMatch) {
					users = append(users, s.channels[c].users[u])
					continue
				}
			}
		}
	}

	return users
}
<|endoftext|>


<commit_before>package pymlstate

import (
	"encoding/binary"
	"errors"
	"fmt"
	"github.com/ugorji/go/codec"
	"io"
	"pfi/sensorbee/py/pystate"
	"pfi/sensorbee/sensorbee/core"
	"pfi/sensorbee/sensorbee/data"
	"sync"
)

var (
	lossPath = data.MustCompilePath("loss")
	accPath  = data.MustCompilePath("accuracy")
)

// State is python instance specialized to multiple layer classification.
// The python instance and this struct must not be coppied directly by assignment
// statement because it doesn't increase reference count of instance.
type State struct {
	base   *pystate.Base
	params MLParams
	bucket []data.Value
	rwm    sync.RWMutex
}

// MLParams is parameters pymlstate defines in addition to those pystate does.
// These parameters come from a WITH clause of a CREATE STATE statement.
type MLParams struct {
	// BatchSize is number of tuples in a single batch training. Write method,
	// which is usually called by an INSERT INTOT statement via uds Sink, stores
	// tuples without training until it has tuples as many as batch_train_size.
	// This is an optional parameter and its default value is 10.
	BatchSize int `codec:"batch_train_size"`
}

// New creates `core.SharedState` for multiple layer classification.
func New(baseParams *pystate.BaseParams, mlParams *MLParams, params data.Map) (*State, error) {
	b, err := pystate.NewBase(baseParams, params)
	if err != nil {
		return nil, err
	}

	s := &State{
		base:   b,
		params: *mlParams,
		bucket: make([]data.Value, 0, mlParams.BatchSize),
	}
	return s, nil
}

// Terminate terminates this state.
func (s *State) Terminate(ctx *core.Context) error {
	s.rwm.Lock()
	defer s.rwm.Unlock()
	if err := s.base.Terminate(ctx); err != nil {
		return err
	}
	// Don't set s.base = nil because it's used for the termination detection.
	s.bucket = nil
	return nil
}

// Write stores a tuple to its bucket and calls "fit" function every
// "batch_train_size" times.
func (s *State) Write(ctx *core.Context, t *core.Tuple) error {
	s.rwm.Lock()
	defer s.rwm.Unlock()
	if err := s.base.CheckTermination(); err != nil {
		return err
	}

	s.bucket = append(s.bucket, t.Data)
	if len(s.bucket) < s.params.BatchSize {
		return nil
	}

	m, err := s.fit(ctx, s.bucket)
	prevBucketSize := len(s.bucket)
	s.bucket = s.bucket[:0] // clear slice but keep capacity
	if err != nil {
		ctx.ErrLog(err).WithField("bucket_size", prevBucketSize).
			Error("pymlstate's training via Write (INSERT INTO) failed")
		return err
	}

	// TODO: add option to toggle the following logging

	ret, err := data.AsMap(m)
	if err != nil {
		// The following log is optional. So, it isn't a error even if the
		// result doesn't have accuracy and loss fields.
		// TODO: write a warning log after the logging option is added.
		return nil
	}

	var loss float64
	if l, e := ret.Get(lossPath); e != nil {
		// TODO: add warning
		return nil
	} else if loss, e = data.ToFloat(l); e != nil {
		// TODO: add warning
		return nil
	}

	var acc float64
	if a, e := ret.Get(accPath); e != nil {
		// TODO: add warning
		return nil
	} else if acc, e = data.ToFloat(a); e != nil {
		// TODO: add warning
		return nil
	}
	ctx.Log().Debugf("loss=%.3f acc=%.3f", loss/float64(s.params.BatchSize),
		acc/float64(s.params.BatchSize))
	return nil
}

// Fit receives `data.Array` type but it assumes `[]data.Map` type
// for passing arguments to `fit` method.
func (s *State) Fit(ctx *core.Context, bucket []data.Value, args ...data.Value) (
	data.Value, error) {
	s.rwm.RLock()
	defer s.rwm.RUnlock()
	return s.fit(ctx, bucket, args...)
}

// fit is the internal implementation of Fit. fit doesn't acquire the lock nor
// check s.ins == nil. RLock is sufficient when calling this method because
// this method itself doesn't change any field of State. Although the model
// will be updated by the data, the model is protected by Python's GIL. So,
// this method doesn't require a write lock.
func (s *State) fit(ctx *core.Context, bucket []data.Value, args ...data.Value) (
	data.Value, error) {
	aggArg := make([]data.Value, 1+len(args))
	aggArg[0] = data.Array(bucket)
	for i, v := range args {
		aggArg[i+1] = v
	}
	return s.base.Call("fit", aggArg...)
}

// Predict applies the model to the data. It returns a result returned from
// Python script.
func (s *State) Predict(ctx *core.Context, dt data.Value) (data.Value, error) {
	s.rwm.RLock()
	defer s.rwm.RUnlock()
	return s.base.Call("predict", dt)
}

// Save saves the model of the state. pystate calls `save` method and
// use its return value as dumped model.
func (s *State) Save(ctx *core.Context, w io.Writer, params data.Map) error {
	s.rwm.RLock()
	defer s.rwm.RUnlock()
	if err := s.base.CheckTermination(); err != nil {
		return err
	}

	if err := s.saveState(w); err != nil {
		return err
	}
	return s.base.Save(ctx, w, params)
}

const (
	pyMLStateFormatVersion uint8 = 1
)

func (s *State) saveState(w io.Writer) error {
	if _, err := w.Write([]byte{pyMLStateFormatVersion}); err != nil {
		return err
	}

	// Save parameter of State before save python's model
	msgpackHandle := &codec.MsgpackHandle{}
	var out []byte
	enc := codec.NewEncoderBytes(&out, msgpackHandle)
	if err := enc.Encode(&s.params); err != nil {
		return err
	}

	// Write size of MLParams
	dataSize := uint32(len(out))
	err := binary.Write(w, binary.LittleEndian, dataSize)
	if err != nil {
		return err
	}

	// Write MLParams in msgpack
	n, err := w.Write(out)
	if err != nil {
		return err
	}

	if n < len(out) {
		return errors.New("cannot save the MLParams data")
	}

	return nil
}

// Load loads the model of the state. pystate calls `load` method and
// pass to the model data by using method parameter.
func (s *State) Load(ctx *core.Context, r io.Reader, params data.Map) error {
	s.rwm.Lock()
	defer s.rwm.Unlock()
	if err := s.base.CheckTermination(); err != nil {
		return err
	}

	var formatVersion uint8
	if err := binary.Read(r, binary.LittleEndian, &formatVersion); err != nil {
		return err
	}

	// TODO: remove MLParams specific parameters from params

	switch formatVersion {
	case 1:
		return s.loadMLParamsAndDataV1(ctx, r, params)
	default:
		return fmt.Errorf("unsupported format version of State container: %v", formatVersion)
	}
}

func (s *State) loadMLParamsAndDataV1(ctx *core.Context, r io.Reader, params data.Map) error {
	var dataSize uint32
	if err := binary.Read(r, binary.LittleEndian, &dataSize); err != nil {
		return err
	}
	if dataSize == 0 {
		return errors.New("size of MLParams must be greater than 0")
	}

	// Read MLParams from reader
	buf := make([]byte, dataSize)
	n, err := r.Read(buf)
	if err != nil {
		return err
	}
	if n != int(dataSize) {
		return errors.New("read size is different from the size of MLParams")
	}

	// Desirialize MLParams
	var saved MLParams
	msgpackHandle := &codec.MsgpackHandle{}
	dec := codec.NewDecoderBytes(buf, msgpackHandle)
	if err := dec.Decode(&saved); err != nil {
		return err
	}
	if err := s.base.Load(ctx, r, params); err != nil {
		return err
	}
	s.params = saved
	return nil
}

// Fit trains the model. It applies tuples that bucket has in a batch manner.
// The return value of this function depends on the implementation of Python
// UDS.
func Fit(ctx *core.Context, stateName string, bucket []data.Value, args ...data.Value) (
	data.Value, error) {
	s, err := lookupState(ctx, stateName)
	if err != nil {
		return nil, err
	}

	return s.Fit(ctx, bucket, args...)
}

// Predict applies the model to the given data and returns estimated values.
// The format of the return value depends on each Python UDS.
func Predict(ctx *core.Context, stateName string, dt data.Value) (data.Value, error) {
	s, err := lookupState(ctx, stateName)
	if err != nil {
		return nil, err
	}

	return s.Predict(ctx, dt)
}

func lookupState(ctx *core.Context, stateName string) (*State, error) {
	st, err := ctx.SharedStates.Get(stateName)
	if err != nil {
		return nil, err
	}

	if s, ok := st.(*State); ok {
		return s, nil
	}

	return nil, fmt.Errorf("state '%v' isn't a State", stateName)
}
<commit_msg>support variable length arguments on pymlstate_predict UDF<commit_after>package pymlstate

import (
	"encoding/binary"
	"errors"
	"fmt"
	"github.com/ugorji/go/codec"
	"io"
	"pfi/sensorbee/py/pystate"
	"pfi/sensorbee/sensorbee/core"
	"pfi/sensorbee/sensorbee/data"
	"sync"
)

var (
	lossPath = data.MustCompilePath("loss")
	accPath  = data.MustCompilePath("accuracy")
)

// State is python instance specialized to multiple layer classification.
// The python instance and this struct must not be coppied directly by assignment
// statement because it doesn't increase reference count of instance.
type State struct {
	base   *pystate.Base
	params MLParams
	bucket []data.Value
	rwm    sync.RWMutex
}

// MLParams is parameters pymlstate defines in addition to those pystate does.
// These parameters come from a WITH clause of a CREATE STATE statement.
type MLParams struct {
	// BatchSize is number of tuples in a single batch training. Write method,
	// which is usually called by an INSERT INTOT statement via uds Sink, stores
	// tuples without training until it has tuples as many as batch_train_size.
	// This is an optional parameter and its default value is 10.
	BatchSize int `codec:"batch_train_size"`
}

// New creates `core.SharedState` for multiple layer classification.
func New(baseParams *pystate.BaseParams, mlParams *MLParams, params data.Map) (*State, error) {
	b, err := pystate.NewBase(baseParams, params)
	if err != nil {
		return nil, err
	}

	s := &State{
		base:   b,
		params: *mlParams,
		bucket: make([]data.Value, 0, mlParams.BatchSize),
	}
	return s, nil
}

// Terminate terminates this state.
func (s *State) Terminate(ctx *core.Context) error {
	s.rwm.Lock()
	defer s.rwm.Unlock()
	if err := s.base.Terminate(ctx); err != nil {
		return err
	}
	// Don't set s.base = nil because it's used for the termination detection.
	s.bucket = nil
	return nil
}

// Write stores a tuple to its bucket and calls "fit" function every
// "batch_train_size" times.
func (s *State) Write(ctx *core.Context, t *core.Tuple) error {
	s.rwm.Lock()
	defer s.rwm.Unlock()
	if err := s.base.CheckTermination(); err != nil {
		return err
	}

	s.bucket = append(s.bucket, t.Data)
	if len(s.bucket) < s.params.BatchSize {
		return nil
	}

	m, err := s.fit(ctx, s.bucket)
	prevBucketSize := len(s.bucket)
	s.bucket = s.bucket[:0] // clear slice but keep capacity
	if err != nil {
		ctx.ErrLog(err).WithField("bucket_size", prevBucketSize).
			Error("pymlstate's training via Write (INSERT INTO) failed")
		return err
	}

	// TODO: add option to toggle the following logging

	ret, err := data.AsMap(m)
	if err != nil {
		// The following log is optional. So, it isn't a error even if the
		// result doesn't have accuracy and loss fields.
		// TODO: write a warning log after the logging option is added.
		return nil
	}

	var loss float64
	if l, e := ret.Get(lossPath); e != nil {
		// TODO: add warning
		return nil
	} else if loss, e = data.ToFloat(l); e != nil {
		// TODO: add warning
		return nil
	}

	var acc float64
	if a, e := ret.Get(accPath); e != nil {
		// TODO: add warning
		return nil
	} else if acc, e = data.ToFloat(a); e != nil {
		// TODO: add warning
		return nil
	}
	ctx.Log().Debugf("loss=%.3f acc=%.3f", loss/float64(s.params.BatchSize),
		acc/float64(s.params.BatchSize))
	return nil
}

// Fit receives `data.Array` type but it assumes `[]data.Map` type
// for passing arguments to `fit` method.
func (s *State) Fit(ctx *core.Context, bucket []data.Value, args ...data.Value) (
	data.Value, error) {
	s.rwm.RLock()
	defer s.rwm.RUnlock()
	return s.fit(ctx, bucket, args...)
}

// fit is the internal implementation of Fit. fit doesn't acquire the lock nor
// check s.ins == nil. RLock is sufficient when calling this method because
// this method itself doesn't change any field of State. Although the model
// will be updated by the data, the model is protected by Python's GIL. So,
// this method doesn't require a write lock.
func (s *State) fit(ctx *core.Context, bucket []data.Value, args ...data.Value) (
	data.Value, error) {
	aggArg := make([]data.Value, 1+len(args))
	aggArg[0] = data.Array(bucket)
	for i, v := range args {
		aggArg[i+1] = v
	}
	return s.base.Call("fit", aggArg...)
}

// Predict applies the model to the data. It returns a result returned from
// Python script.
func (s *State) Predict(ctx *core.Context, dt data.Value, args ...data.Value) (
	data.Value, error) {
	s.rwm.RLock()
	defer s.rwm.RUnlock()
	aggArg := make([]data.Value, 1+len(args))
	aggArg[0] = dt
	for i, v := range args {
		aggArg[i+1] = v
	}
	return s.base.Call("predict", aggArg...)
}

// Save saves the model of the state. pystate calls `save` method and
// use its return value as dumped model.
func (s *State) Save(ctx *core.Context, w io.Writer, params data.Map) error {
	s.rwm.RLock()
	defer s.rwm.RUnlock()
	if err := s.base.CheckTermination(); err != nil {
		return err
	}

	if err := s.saveState(w); err != nil {
		return err
	}
	return s.base.Save(ctx, w, params)
}

const (
	pyMLStateFormatVersion uint8 = 1
)

func (s *State) saveState(w io.Writer) error {
	if _, err := w.Write([]byte{pyMLStateFormatVersion}); err != nil {
		return err
	}

	// Save parameter of State before save python's model
	msgpackHandle := &codec.MsgpackHandle{}
	var out []byte
	enc := codec.NewEncoderBytes(&out, msgpackHandle)
	if err := enc.Encode(&s.params); err != nil {
		return err
	}

	// Write size of MLParams
	dataSize := uint32(len(out))
	err := binary.Write(w, binary.LittleEndian, dataSize)
	if err != nil {
		return err
	}

	// Write MLParams in msgpack
	n, err := w.Write(out)
	if err != nil {
		return err
	}

	if n < len(out) {
		return errors.New("cannot save the MLParams data")
	}

	return nil
}

// Load loads the model of the state. pystate calls `load` method and
// pass to the model data by using method parameter.
func (s *State) Load(ctx *core.Context, r io.Reader, params data.Map) error {
	s.rwm.Lock()
	defer s.rwm.Unlock()
	if err := s.base.CheckTermination(); err != nil {
		return err
	}

	var formatVersion uint8
	if err := binary.Read(r, binary.LittleEndian, &formatVersion); err != nil {
		return err
	}

	// TODO: remove MLParams specific parameters from params

	switch formatVersion {
	case 1:
		return s.loadMLParamsAndDataV1(ctx, r, params)
	default:
		return fmt.Errorf("unsupported format version of State container: %v", formatVersion)
	}
}

func (s *State) loadMLParamsAndDataV1(ctx *core.Context, r io.Reader, params data.Map) error {
	var dataSize uint32
	if err := binary.Read(r, binary.LittleEndian, &dataSize); err != nil {
		return err
	}
	if dataSize == 0 {
		return errors.New("size of MLParams must be greater than 0")
	}

	// Read MLParams from reader
	buf := make([]byte, dataSize)
	n, err := r.Read(buf)
	if err != nil {
		return err
	}
	if n != int(dataSize) {
		return errors.New("read size is different from the size of MLParams")
	}

	// Desirialize MLParams
	var saved MLParams
	msgpackHandle := &codec.MsgpackHandle{}
	dec := codec.NewDecoderBytes(buf, msgpackHandle)
	if err := dec.Decode(&saved); err != nil {
		return err
	}
	if err := s.base.Load(ctx, r, params); err != nil {
		return err
	}
	s.params = saved
	return nil
}

// Fit trains the model. It applies tuples that bucket has in a batch manner.
// The return value of this function depends on the implementation of Python
// UDS.
func Fit(ctx *core.Context, stateName string, bucket []data.Value, args ...data.Value) (
	data.Value, error) {
	s, err := lookupState(ctx, stateName)
	if err != nil {
		return nil, err
	}

	return s.Fit(ctx, bucket, args...)
}

// Predict applies the model to the given data and returns estimated values.
// The format of the return value depends on each Python UDS.
func Predict(ctx *core.Context, stateName string, dt data.Value, args ...data.Value) (
	data.Value, error) {
	s, err := lookupState(ctx, stateName)
	if err != nil {
		return nil, err
	}

	return s.Predict(ctx, dt, args...)
}

func lookupState(ctx *core.Context, stateName string) (*State, error) {
	st, err := ctx.SharedStates.Get(stateName)
	if err != nil {
		return nil, err
	}

	if s, ok := st.(*State); ok {
		return s, nil
	}

	return nil, fmt.Errorf("state '%v' isn't a State", stateName)
}
<|endoftext|>


<commit_before>package nicehash

import (
	"encoding/json"
	"errors"
	"strconv"
	"time"
)

type GlobalStats struct {
	Algo                  AlgoType `json:"algo"`
	ProfitabilityAboveBtc float32  `json:"profitability_above_btc,string"`
	ProfitabilityAboveLtc float32  `json:"profitability_above_ltc,string"`
	Price                 float64  `json:"price,string"`
	ProfitabilityBtc      float32  `json:"profitability_btc,string"`
	ProfitabilityLtc      float32  `json:"profitability_ltc,string"`
	Speed                 float64  `json:"speed,string"`
}

func (client *NicehashClient) GetStatsGlobalCurrent() ([]GlobalStats, error) {
	stats := &struct {
		Result struct {
			Error string        `json:"error"`
			Stats []GlobalStats `json:"stats"`
		} `json:"result"`
	}{}
	params := &Params{Method: "stats.global.current", Algo: AlgoTypeMAX, Location: LocationMAX}
	_, err := client.sling.New().Get("").QueryStruct(params).ReceiveSuccess(&stats)
	if err != nil {
		return nil, err
	}
	if stats.Result.Error != "" {
		return nil, errors.New(stats.Result.Error)
	}
	return stats.Result.Stats, nil
}

func (client *NicehashClient) GetStatsGlobalDay() ([]GlobalStats, error) {
	stats := &struct {
		Result struct {
			Error string        `json:"error"`
			Stats []GlobalStats `json:"stats"`
		} `json:"result"`
	}{}
	params := &Params{Method: "stats.global.24h", Algo: AlgoTypeMAX, Location: LocationMAX}
	_, err := client.sling.New().Get("").QueryStruct(params).ReceiveSuccess(&stats)
	if err != nil {
		return nil, err
	}
	if stats.Result.Error != "" {
		return nil, errors.New(stats.Result.Error)
	}
	return stats.Result.Stats, nil
}

type ProviderStats struct {
	Algo          AlgoType `json:"algo"`
	Balance       float64  `json:"balance,string"`
	AcceptedSpeed float64  `json:"accepted_speed,string"`
	RejectedSpeed float64  `json:"rejected_speed,string"`
}

type ProviderPayments struct {
	Amount float64   `json:"amount,string"`
	Fee    float64   `json:"fee,string"`
	TxID   string    `json:"TXID"`
	Time   time.Time `json:"time"`
}

func (t *ProviderPayments) UnmarshalJSON(data []byte) error {
	var err error
	type Alias ProviderPayments
	aux := &struct {
		Time string `json:"time"`
		*Alias
	}{
		Alias: (*Alias)(t),
	}
	if err = json.Unmarshal(data, &aux); err != nil {
		return err
	}
	t.Time, err = time.Parse("2006-01-02 15:04:05", aux.Time)
	return err
}

func (client *NicehashClient) GetStatsProvider(addr string) ([]ProviderStats, []ProviderPayments, error) {
	stats := &struct {
		Result struct {
			Error    string             `json:"error"`
			Stats    []ProviderStats    `json:"stats"`
			Payments []ProviderPayments `json:"payments"`
		} `json:"result"`
	}{}
	params := &Params{Method: "stats.provider", Algo: AlgoTypeMAX, Location: LocationMAX, Addr: addr}
	_, err := client.sling.New().Get("").QueryStruct(params).ReceiveSuccess(&stats)
	if err != nil {
		return nil, nil, err
	}
	if stats.Result.Error != "" {
		return nil, nil, errors.New(stats.Result.Error)
	}
	return stats.Result.Stats, stats.Result.Payments, nil
}

type ProviderExStats struct {
	Algo          AlgoType `json:"algo"`
	Suffix        string   `json:"suffix"`
	Name          string   `json:"name"`
	Profitability float64  `json:"profitability,string"`
	Unpaid        float64  `json:"balance,string"`
	AcceptedSpeed float64  `json:"accepted_speed,string"`
	RejectedSpeed float64  `json:"rejected_speed,string"`
}

func (t *ProviderExStats) UnmarshalJSON(data []byte) error {
	var err error
	type Alias ProviderExStats
	aux := &struct {
		Data []interface{} `json:"data"`
		*Alias
	}{
		Alias: (*Alias)(t),
	}
	if err = json.Unmarshal(data, &aux); err != nil {
		return err
	}
	speeds := aux.Data[0].(map[string]interface{})
	if val, ok := speeds["a"]; ok {
		t.AcceptedSpeed, err = strconv.ParseFloat(val.(string), 64)
		if err != nil {
			return err
		}
	}
	if val, ok := speeds["rs"]; ok {
		t.RejectedSpeed, err = strconv.ParseFloat(val.(string), 64)
		if err != nil {
			return err
		}
	}
	t.Unpaid, err = strconv.ParseFloat(aux.Data[1].(string), 64)
	return err
}

type ProviderExHistory struct {
	Algo AlgoType                            `json:"algo"`
	Data map[time.Time]ProviderExHistoryItem `json:"data"`
}

type ProviderExHistoryItem struct {
	Unpaid        float64 `json:"balance,string"`
	AcceptedSpeed float64 `json:"accepted_speed,string"`
	RejectedSpeed float64 `json:"rejected_speed,string"`
}

func (t *ProviderExHistory) UnmarshalJSON(data []byte) error {
	var err error
	type Alias ProviderExHistory
	aux := &struct {
		Data [][]interface{} `json:"data"`
		*Alias
	}{
		Alias: (*Alias)(t),
	}
	if err = json.Unmarshal(data, &aux); err != nil {
		return err
	}
	t.Data = make(map[time.Time]ProviderExHistoryItem)
	for _, d := range aux.Data {
		var item ProviderExHistoryItem
		date := time.Unix(int64(d[0].(float64))*300, 0)
		speeds := d[1].(map[string]interface{})
		if val, ok := speeds["a"]; ok {
			item.AcceptedSpeed, err = strconv.ParseFloat(val.(string), 64)
			if err != nil {
				return err
			}
		}
		if val, ok := speeds["rs"]; ok {
			item.RejectedSpeed, err = strconv.ParseFloat(val.(string), 64)
			if err != nil {
				return err
			}
		}
		item.Unpaid, err = strconv.ParseFloat(d[2].(string), 64)
		if err != nil {
			return err
		}
		if item.AcceptedSpeed > 0 || item.RejectedSpeed > 0 {
			t.Data[date] = item
		}
	}
	return nil
}

type ProviderExPayments struct {
	Amount float64   `json:"amount,string"`
	Fee    float64   `json:"fee,string"`
	TxID   string    `json:"TXID"`
	Time   time.Time `json:"time"`
}

func (t *ProviderExPayments) UnmarshalJSON(data []byte) error {
	var err error
	type Alias ProviderPayments
	aux := &struct {
		Time int64 `json:"time"`
		*Alias
	}{
		Alias: (*Alias)(t),
	}
	if err = json.Unmarshal(data, &aux); err != nil {
		return err
	}
	t.Time = time.Unix(aux.Time, 0)
	return err
}

type StatsProviderEx struct {
	Error    string               `json:"error"`
	Current  []ProviderExStats    `json:"current"`
	Past     []ProviderExHistory  `json:"past"`
	Payments []ProviderExPayments `json:"payments"`
}

func (client *NicehashClient) GetStatsProviderEx(addr string) (StatsProviderEx, error) {
	stats := &struct {
		Result StatsProviderEx `json:"result"`
	}{}
	params := &Params{Method: "stats.provider.ex", Algo: AlgoTypeMAX, Location: LocationMAX, Addr: addr}
	_, err := client.sling.New().Get("").QueryStruct(params).ReceiveSuccess(&stats)
	if err != nil {
		return StatsProviderEx{}, err
	}
	if stats.Result.Error != "" {
		return StatsProviderEx{}, errors.New(stats.Result.Error)
	}
	return stats.Result, nil
}

type ProviderWorker struct {
	Name          string   `json:"name"`
	AcceptedSpeed float64  `json:"accepted_speed,string"`
	RejectedSpeed float64  `json:"rejected_speed,string"`
	Connected     uint64   `json:"connected"`
	XnSubEnabled  bool     `json:"xnsub"`
	Difficulty    float64  `json:"difficulty"`
	Location      Location `json:"location"`
}

func (t *ProviderWorker) UnmarshalJSON(data []byte) error {
	var err error
	var aux []interface{}
	if err = json.Unmarshal(data, &aux); err != nil {
		return err
	}
	t.Name = aux[0].(string)
	speeds := aux[1].(map[string]interface{})
	if val, ok := speeds["a"]; ok {
		t.AcceptedSpeed, err = strconv.ParseFloat(val.(string), 64)
		if err != nil {
			return err
		}
	}
	if val, ok := speeds["rs"]; ok {
		t.RejectedSpeed, err = strconv.ParseFloat(val.(string), 64)
		if err != nil {
			return err
		}
	}
	t.Connected = uint64(aux[2].(float64))
	t.XnSubEnabled = aux[3].(float64) == 1
	t.Difficulty, err = strconv.ParseFloat(aux[4].(string), 64)
	if err != nil {
		return err
	}
	t.Location = Location(uint64(aux[5].(float64)))
	return nil
}

func (client *NicehashClient) GetStatsProviderWorkers(addr string, algo AlgoType) ([]ProviderWorker, error) {
	stats := &struct {
		Result struct {
			Error   string           `json:"error"`
			Address string           `json:"addr"`
			Algo    AlgoType         `json:"algo"`
			Workers []ProviderWorker `json:"workers"`
		} `json:"result"`
	}{}
	params := &Params{Method: "stats.provider.workers", Algo: algo, Location: LocationMAX, Addr: addr}
	_, err := client.sling.New().Get("").QueryStruct(params).ReceiveSuccess(&stats)
	if err != nil {
		return nil, err
	}
	if stats.Result.Error != "" {
		return nil, errors.New(stats.Result.Error)
	}
	return stats.Result.Workers, nil
}
<commit_msg>Handle http status codes<commit_after>package nicehash

import (
	"encoding/json"
	"errors"
	"strconv"
	"time"
)

type GlobalStats struct {
	Algo                  AlgoType `json:"algo"`
	ProfitabilityAboveBtc float32  `json:"profitability_above_btc,string"`
	ProfitabilityAboveLtc float32  `json:"profitability_above_ltc,string"`
	Price                 float64  `json:"price,string"`
	ProfitabilityBtc      float32  `json:"profitability_btc,string"`
	ProfitabilityLtc      float32  `json:"profitability_ltc,string"`
	Speed                 float64  `json:"speed,string"`
}

func (client *NicehashClient) GetStatsGlobalCurrent() ([]GlobalStats, error) {
	stats := &struct {
		Result struct {
			Error string        `json:"error"`
			Stats []GlobalStats `json:"stats"`
		} `json:"result"`
	}{}
	params := &Params{Method: "stats.global.current", Algo: AlgoTypeMAX, Location: LocationMAX}
	resp, err := client.sling.New().Get("").QueryStruct(params).ReceiveSuccess(&stats)
	if err != nil {
		return nil, err
	}
	if code := resp.StatusCode; code < 200 || 299 < code {
		return nil, errors.New("Http response: " + resp.Status)
	}
	if stats.Result.Error != "" {
		return nil, errors.New(stats.Result.Error)
	}
	return stats.Result.Stats, nil
}

func (client *NicehashClient) GetStatsGlobalDay() ([]GlobalStats, error) {
	stats := &struct {
		Result struct {
			Error string        `json:"error"`
			Stats []GlobalStats `json:"stats"`
		} `json:"result"`
	}{}
	params := &Params{Method: "stats.global.24h", Algo: AlgoTypeMAX, Location: LocationMAX}
	resp, err := client.sling.New().Get("").QueryStruct(params).ReceiveSuccess(&stats)
	if err != nil {
		return nil, err
	}
	if code := resp.StatusCode; code < 200 || 299 < code {
		return nil, errors.New("Http response: " + resp.Status)
	}
	if stats.Result.Error != "" {
		return nil, errors.New(stats.Result.Error)
	}
	return stats.Result.Stats, nil
}

type ProviderStats struct {
	Algo          AlgoType `json:"algo"`
	Balance       float64  `json:"balance,string"`
	AcceptedSpeed float64  `json:"accepted_speed,string"`
	RejectedSpeed float64  `json:"rejected_speed,string"`
}

type ProviderPayments struct {
	Amount float64   `json:"amount,string"`
	Fee    float64   `json:"fee,string"`
	TxID   string    `json:"TXID"`
	Time   time.Time `json:"time"`
}

func (t *ProviderPayments) UnmarshalJSON(data []byte) error {
	var err error
	type Alias ProviderPayments
	aux := &struct {
		Time string `json:"time"`
		*Alias
	}{
		Alias: (*Alias)(t),
	}
	if err = json.Unmarshal(data, &aux); err != nil {
		return err
	}
	t.Time, err = time.Parse("2006-01-02 15:04:05", aux.Time)
	return err
}

func (client *NicehashClient) GetStatsProvider(addr string) ([]ProviderStats, []ProviderPayments, error) {
	stats := &struct {
		Result struct {
			Error    string             `json:"error"`
			Stats    []ProviderStats    `json:"stats"`
			Payments []ProviderPayments `json:"payments"`
		} `json:"result"`
	}{}
	params := &Params{Method: "stats.provider", Algo: AlgoTypeMAX, Location: LocationMAX, Addr: addr}
	resp, err := client.sling.New().Get("").QueryStruct(params).ReceiveSuccess(&stats)
	if err != nil {
		return nil, nil, err
	}
	if code := resp.StatusCode; code < 200 || 299 < code {
		return nil, nil, errors.New("Http response: " + resp.Status)
	}
	if stats.Result.Error != "" {
		return nil, nil, errors.New(stats.Result.Error)
	}
	return stats.Result.Stats, stats.Result.Payments, nil
}

type ProviderExStats struct {
	Algo          AlgoType `json:"algo"`
	Suffix        string   `json:"suffix"`
	Name          string   `json:"name"`
	Profitability float64  `json:"profitability,string"`
	Unpaid        float64  `json:"balance,string"`
	AcceptedSpeed float64  `json:"accepted_speed,string"`
	RejectedSpeed float64  `json:"rejected_speed,string"`
}

func (t *ProviderExStats) UnmarshalJSON(data []byte) error {
	var err error
	type Alias ProviderExStats
	aux := &struct {
		Data []interface{} `json:"data"`
		*Alias
	}{
		Alias: (*Alias)(t),
	}
	if err = json.Unmarshal(data, &aux); err != nil {
		return err
	}
	speeds := aux.Data[0].(map[string]interface{})
	if val, ok := speeds["a"]; ok {
		t.AcceptedSpeed, err = strconv.ParseFloat(val.(string), 64)
		if err != nil {
			return err
		}
	}
	if val, ok := speeds["rs"]; ok {
		t.RejectedSpeed, err = strconv.ParseFloat(val.(string), 64)
		if err != nil {
			return err
		}
	}
	t.Unpaid, err = strconv.ParseFloat(aux.Data[1].(string), 64)
	return err
}

type ProviderExHistory struct {
	Algo AlgoType                            `json:"algo"`
	Data map[time.Time]ProviderExHistoryItem `json:"data"`
}

type ProviderExHistoryItem struct {
	Unpaid        float64 `json:"balance,string"`
	AcceptedSpeed float64 `json:"accepted_speed,string"`
	RejectedSpeed float64 `json:"rejected_speed,string"`
}

func (t *ProviderExHistory) UnmarshalJSON(data []byte) error {
	var err error
	type Alias ProviderExHistory
	aux := &struct {
		Data [][]interface{} `json:"data"`
		*Alias
	}{
		Alias: (*Alias)(t),
	}
	if err = json.Unmarshal(data, &aux); err != nil {
		return err
	}
	t.Data = make(map[time.Time]ProviderExHistoryItem)
	for _, d := range aux.Data {
		var item ProviderExHistoryItem
		date := time.Unix(int64(d[0].(float64))*300, 0)
		speeds := d[1].(map[string]interface{})
		if val, ok := speeds["a"]; ok {
			item.AcceptedSpeed, err = strconv.ParseFloat(val.(string), 64)
			if err != nil {
				return err
			}
		}
		if val, ok := speeds["rs"]; ok {
			item.RejectedSpeed, err = strconv.ParseFloat(val.(string), 64)
			if err != nil {
				return err
			}
		}
		item.Unpaid, err = strconv.ParseFloat(d[2].(string), 64)
		if err != nil {
			return err
		}
		if item.AcceptedSpeed > 0 || item.RejectedSpeed > 0 {
			t.Data[date] = item
		}
	}
	return nil
}

type ProviderExPayments struct {
	Amount float64   `json:"amount,string"`
	Fee    float64   `json:"fee,string"`
	TxID   string    `json:"TXID"`
	Time   time.Time `json:"time"`
}

func (t *ProviderExPayments) UnmarshalJSON(data []byte) error {
	var err error
	type Alias ProviderPayments
	aux := &struct {
		Time int64 `json:"time"`
		*Alias
	}{
		Alias: (*Alias)(t),
	}
	if err = json.Unmarshal(data, &aux); err != nil {
		return err
	}
	t.Time = time.Unix(aux.Time, 0)
	return err
}

type StatsProviderEx struct {
	Error    string               `json:"error"`
	Current  []ProviderExStats    `json:"current"`
	Past     []ProviderExHistory  `json:"past"`
	Payments []ProviderExPayments `json:"payments"`
}

func (client *NicehashClient) GetStatsProviderEx(addr string) (StatsProviderEx, error) {
	stats := &struct {
		Result StatsProviderEx `json:"result"`
	}{}
	params := &Params{Method: "stats.provider.ex", Algo: AlgoTypeMAX, Location: LocationMAX, Addr: addr}
	resp, err := client.sling.New().Get("").QueryStruct(params).ReceiveSuccess(&stats)
	if err != nil {
		return StatsProviderEx{}, err
	}
	if code := resp.StatusCode; code < 200 || 299 < code {
		return StatsProviderEx{}, errors.New("Http response: " + resp.Status)
	}
	if stats.Result.Error != "" {
		return StatsProviderEx{}, errors.New(stats.Result.Error)
	}
	return stats.Result, nil
}

type ProviderWorker struct {
	Name          string   `json:"name"`
	AcceptedSpeed float64  `json:"accepted_speed,string"`
	RejectedSpeed float64  `json:"rejected_speed,string"`
	Connected     uint64   `json:"connected"`
	XnSubEnabled  bool     `json:"xnsub"`
	Difficulty    float64  `json:"difficulty"`
	Location      Location `json:"location"`
}

func (t *ProviderWorker) UnmarshalJSON(data []byte) error {
	var err error
	var aux []interface{}
	if err = json.Unmarshal(data, &aux); err != nil {
		return err
	}
	t.Name = aux[0].(string)
	speeds := aux[1].(map[string]interface{})
	if val, ok := speeds["a"]; ok {
		t.AcceptedSpeed, err = strconv.ParseFloat(val.(string), 64)
		if err != nil {
			return err
		}
	}
	if val, ok := speeds["rs"]; ok {
		t.RejectedSpeed, err = strconv.ParseFloat(val.(string), 64)
		if err != nil {
			return err
		}
	}
	t.Connected = uint64(aux[2].(float64))
	t.XnSubEnabled = aux[3].(float64) == 1
	t.Difficulty, err = strconv.ParseFloat(aux[4].(string), 64)
	if err != nil {
		return err
	}
	t.Location = Location(uint64(aux[5].(float64)))
	return nil
}

func (client *NicehashClient) GetStatsProviderWorkers(addr string, algo AlgoType) ([]ProviderWorker, error) {
	stats := &struct {
		Result struct {
			Error   string           `json:"error"`
			Address string           `json:"addr"`
			Algo    AlgoType         `json:"algo"`
			Workers []ProviderWorker `json:"workers"`
		} `json:"result"`
	}{}
	params := &Params{Method: "stats.provider.workers", Algo: algo, Location: LocationMAX, Addr: addr}
	_, err := client.sling.New().Get("").QueryStruct(params).ReceiveSuccess(&stats)
	if err != nil {
		return nil, err
	}
	if stats.Result.Error != "" {
		return nil, errors.New(stats.Result.Error)
	}
	return stats.Result.Workers, nil
}
<|endoftext|>


<commit_before>package honu

import (
	"fmt"
	"sync"
)

//===========================================================================
// Store is an interface for any key/value store and is created with NewStore
//===========================================================================

// NewStore creates and initializes a key value store
func NewStore(pid uint64, sequential bool) Store {
	var store Store

	// Create the type-specific data structures
	if sequential {
		// Create a sequential store on demand.
		store = new(SequentialStore)
		info("created sequential consistency storage")
	} else {
		// The default is a linearizable store.
		store = new(LinearizableStore)
		info("created linearizable consistency storage")
	}

	// Initialize the store and return
	store.Init(pid)
	return store

}

// Locker is an interface for defining the sync.RWMutex methods including
// Lock and Unlock for write protection from sync.Locker and RLock and RUnlock
// for read protection.
type Locker interface {
	sync.Locker
	RLock()
	RUnlock()
}

// Store is an interface for multiple in-memory storage types under the hood.
type Store interface {
	Locker
	Init(pid uint64)                                                                // Initialize the store
	Get(key string) (value []byte, version string, err error)                       // Get a value and version for a given key
	GetEntry(key string) *Entry                                                     // Get the entire entry without a lock
	Put(key string, value []byte, trackVisibility bool) (version string, err error) // Put a value for a given key and get associated version
	PutEntry(key string, entry *Entry) (modified bool)                              // Put the entry without modifying the version
	View() map[string]Version                                                       // Returns a map containing the latest version of all keys
	Update(key string, version *Version)                                            // Update the version scalar from a remote source
	Snapshot(path string) error                                                     // Write a snapshot of the version history to disk
	Length() int                                                                    // Returns the number of items in the store (number of keys)

}

//===========================================================================
// Storage with Linearizable Consistency
//===========================================================================

// LinearizableStore implements a versioned, in-memory key-value store that
// keeps a single montonically increasing counter across all objects such
// that a single ordering for all writes (and associated reads) exists. All
// accesses are guarded by read and write locks to ensure linearizable
// consistency and version parents are the last written object no matter the
// key to create a cross-object version history.
type LinearizableStore struct {
	sync.RWMutex
	pid       uint64            // the local process id
	current   uint64            // the current version scalar
	lastWrite *Version          // the version of the last write
	namespace map[string]*Entry // maps keys to the latest entry
	history   *History          // tracks the verion history chain
}

// Init the store creating the internal data structures.
func (s *LinearizableStore) Init(pid uint64) {
	s.pid = pid
	s.namespace = make(map[string]*Entry)
	s.lastWrite = &NullVersion

	// Create, initialize and run the history.
	s.history = new(History)
	s.history.Init()
	s.history.Run()
}

// Get the most recently seen value and version pair for a specific key.
// Returns a not found error if the key has not been written to the namespace.
//
// This operation wraps the entire store in a read lock, meaning that other
// values can be read but no values can be written on Get.
func (s *LinearizableStore) Get(key string) (value []byte, version string, err error) {
	s.RLock()
	defer s.RUnlock()

	entry, ok := s.namespace[key]
	if !ok {
		err = fmt.Errorf("key '%s' not found in namespace", key)
		return value, version, err
	}

	version = entry.Version.String()
	value = entry.Value
	return value, version, err
}

// GetEntry returns the entire entry from the namespace without a lock.
// Returns nil if the given key is not in the store.
func (s *LinearizableStore) GetEntry(key string) *Entry {
	entry, ok := s.namespace[key]
	if !ok {
		return nil
	}
	return entry
}

// Put a value into the namespace, incrementing the version across all
// objects. This operation creates an entry whose parent is the last written
// version of any object. Put also stores all versions and associated entries,
// maintaining a complete version history.
//
// This operation locks the entire store, waiting for all read locks to be
// released and not allowing any other read or write locks until complete.
func (s *LinearizableStore) Put(key string, value []byte, trackVisibility bool) (string, error) {
	s.Lock()
	defer s.Unlock()

	// Create the new version
	s.current++
	version := &Version{s.current, s.pid}

	// Create the new entry
	entry := &Entry{
		Key:             &key,
		Version:         version,
		Parent:          s.lastWrite,
		Value:           value,
		TrackVisibility: trackVisibility,
	}

	// Update the namespace, versions, and last write
	s.namespace[key] = entry
	s.history.Append(entry.Key, entry.Parent, entry.Version)
	s.lastWrite = version

	// Return the version and no error for this method
	return version.String(), nil
}

// PutEntry without modifying version information. Returns true if the entry
// is modified or not -- will only put an entry that has a greater version
// than the current entry.
//
// This method is also responsible for updating the lamport clock.
func (s *LinearizableStore) PutEntry(key string, entry *Entry) bool {
	s.Lock()
	defer s.Unlock()

	// Get the current version
	current, ok := s.namespace[key]
	if !ok {
		current = &Entry{Key: &key, Version: &NullVersion, Parent: &NullVersion}
	}

	// If entry is less than or equal to current version, do not put.
	if entry.Version.LesserEqual(current.Version) {
		return false
	}

	// Update the version scalar
	if entry.Version.Scalar > s.current {
		s.current = entry.Version.Scalar
	}

	// Update the entry
	current.Version = entry.Version
	current.Parent = entry.Parent
	current.Value = entry.Value

	// Update the namespace, versions, and last write
	s.namespace[key] = current
	s.history.Append(current.Key, current.Parent, current.Version)
	s.lastWrite = current.Version
	return true
}

// Update the current version counter with the global value.
func (s *LinearizableStore) Update(key string, version *Version) {
	if version.Scalar > s.current {
		s.current = version.Scalar
	}
}

// View returns the current version for every key in the namespace.
func (s *LinearizableStore) View() map[string]Version {
	s.RLock()
	defer s.RUnlock()

	view := make(map[string]Version)
	for key, entry := range s.namespace {
		view[key] = *entry.Version
	}

	return view
}

// Snapshot the current version history to disk, writing the version data to
// the specified path. Returns any I/O errors if snapshotting is unsuccessful.
func (s *LinearizableStore) Snapshot(path string) error {
	return fmt.Errorf("version history snapshot not implemented yet")
}

// Length returns the number of items in the Store, namely the number of keys
// in the namespace. This does not reflect the number of versions.
func (s *LinearizableStore) Length() int {
	return len(s.namespace)
}

//===========================================================================
// Storage wtih Sequential Consistency
//===========================================================================

// SequentialStore implements a key/value store where each key is versioned
// independently of all other keys. The Store is only locked when a new key is
// added, but readers and writers take locks on individual keys afterward.
// A sequential store therefore allows multi-thread access to different
// objects simultaneously.
//
// The version history for objects in a sequential store is therefore relative
// to the object itself. Parent versions of entries are simply the previous
// entry in the store. Each object has its own independent scalar component.
type SequentialStore struct {
	sync.RWMutex
	pid       uint64            // the local process id
	namespace map[string]*Entry // maps keys to the latest entry
	history   *History          // tracks the verion history chain

}

// Init the store creating the internal data structures.
func (s *SequentialStore) Init(pid uint64) {
	s.pid = pid
	s.namespace = make(map[string]*Entry)

	// Create, initialize and run the history.
	s.history = new(History)
	s.history.Init()
	s.history.Run()
}

// get is an internal method surrounded by a read lock that fetches the
// given value for a specific key. It returns a locked entry, if the mutable
// flag is true, it is write locked, otherwise it is read locked.
//
// NOTE: callers must unlock the entry themselves!
func (s *SequentialStore) get(key string, mutable bool) *Entry {
	s.RLock()
	defer s.RUnlock()

	// Get the entry from the namespace
	entry, ok := s.namespace[key]
	if !ok {
		return nil
	}

	// Lock the entry according the mutable flag
	if mutable {
		entry.Lock()
	} else {
		entry.RLock()
	}

	// Return the locked entry
	return entry
}

// Get the most recently seen value and version pair for a specific key.
// Returns a not found error if the key has not been written to the namespace.
//
// This operation only locks the store with a read-lock on fetch but also adds
// a read-lock to the entry so that it cannot be modified in flight.
func (s *SequentialStore) Get(key string) (value []byte, version string, err error) {
	// Fetch the value, read-locking the entire store
	entry := s.get(key, false)

	// Handle not found error
	if entry == nil {
		err = fmt.Errorf("key '%s' not found in namespace", key)
		return nil, "", err
	}

	// Ensure that the entry is unlocked before we're done
	defer entry.RUnlock()

	// Extract the data required from the entry.
	return entry.Value, entry.Version.String(), nil
}

// GetEntry returns the entire entry from the namespace without a lock.
// Returns nil if the given key is not in the store.
func (s *SequentialStore) GetEntry(key string) *Entry {
	entry, ok := s.namespace[key]
	if !ok {
		return nil
	}
	return entry
}

// make is an internal method that surrounds the store in a write lock to
// create an empty entry for the given key. It returns a write locked entry to
// ensure that the caller can update the entry with values before unlock but
// releases the store as soon as possible to prevent write delays.
//
// NOTE: this method should not be called if the key already exists!
// NOTE: callers must unlock the entry themselves!
func (s *SequentialStore) make(key string) *Entry {
	// Acquire a write lock
	s.Lock()
	defer s.Unlock()

	// Create a write locked entry
	entry := &Entry{Key: &key, Version: &NullVersion, Parent: &NullVersion}
	entry.Lock()

	// Insert the entry into the namespace and return it
	s.namespace[key] = entry
	return entry
}

// Put a value into the namespace and increment the version. Returns the
// version for the given key and any error that might occur.
func (s *SequentialStore) Put(key string, value []byte, trackVisibility bool) (string, error) {
	// Attempt to get the write-locked version from the store
	entry := s.get(key, true)

	// Make an empty entry if there was no entry already in the store
	if entry == nil {
		entry = s.make(key)
	} else {
		// Update the parent of the entry to the old entry
		entry.Parent = entry.Version
	}

	// Ensure that the entry is unlocked when done
	defer entry.Unlock()

	// Create the version for the new entry
	entry.Current++
	entry.Version = &Version{entry.Current, s.pid}

	// Update the value
	entry.Value = value
	entry.TrackVisibility = trackVisibility

	// Store the version in the version history and return it
	s.history.Append(entry.Key, entry.Parent, entry.Version)
	return entry.Version.String(), nil
}

// PutEntry without modifying version information. Returns true if the entry
// is modified or not -- will only put an entry that has a greater version
// than the current entry.
//
// This method is also responsible for updating the lamport clock.
func (s *SequentialStore) PutEntry(key string, entry *Entry) bool {
	// Attempt to get the write-locked version from the store
	current := s.get(key, true)

	// Make an empty entry if there was no entry already in the store.
	if current == nil {
		current = s.make(key)
	}

	// Ensure the entry is unlocked when done
	defer current.Unlock()

	// If entry is less than or equal to current version, do not put.
	if entry.Version.LesserEqual(current.Version) {
		return false
	}

	// Update the scalar with the new information.
	if entry.Version.Scalar > current.Current {
		current.Current = entry.Version.Scalar
	}

	// Replace current entry with new information.
	current.Version = entry.Version
	current.Parent = entry.Parent
	current.Value = entry.Value

	// Store the version in the version history and return true.
	s.history.Append(current.Key, current.Parent, current.Version)
	return true
}

// Update the current version counter with the global value.
func (s *SequentialStore) Update(key string, version *Version) {
	entry := s.get(key, true)
	if entry == nil {
		entry = s.make(key)
	}

	defer entry.Unlock()

	if version.Scalar > entry.Current {
		entry.Current = version.Scalar
	}
}

// View returns the current version for every key in the namespace.
func (s *SequentialStore) View() map[string]Version {
	s.RLock()
	defer s.RUnlock()

	view := make(map[string]Version)
	for key, entry := range s.namespace {
		view[key] = *entry.Version
	}

	return view
}

// Snapshot the current version history to disk, writing the version data to
// the specified path. Returns any I/O errors if snapshotting is unsuccessful.
func (s *SequentialStore) Snapshot(path string) error {
	return fmt.Errorf("version history snapshot not implemented yet")
}

// Length returns the number of items in the Store, namely the number of keys
// in the namespace. This does not reflect the number of versions.
func (s *SequentialStore) Length() int {
	return len(s.namespace)
}
<commit_msg>track visibility on current entry<commit_after>package honu

import (
	"fmt"
	"sync"
)

//===========================================================================
// Store is an interface for any key/value store and is created with NewStore
//===========================================================================

// NewStore creates and initializes a key value store
func NewStore(pid uint64, sequential bool) Store {
	var store Store

	// Create the type-specific data structures
	if sequential {
		// Create a sequential store on demand.
		store = new(SequentialStore)
		info("created sequential consistency storage")
	} else {
		// The default is a linearizable store.
		store = new(LinearizableStore)
		info("created linearizable consistency storage")
	}

	// Initialize the store and return
	store.Init(pid)
	return store

}

// Locker is an interface for defining the sync.RWMutex methods including
// Lock and Unlock for write protection from sync.Locker and RLock and RUnlock
// for read protection.
type Locker interface {
	sync.Locker
	RLock()
	RUnlock()
}

// Store is an interface for multiple in-memory storage types under the hood.
type Store interface {
	Locker
	Init(pid uint64)                                                                // Initialize the store
	Get(key string) (value []byte, version string, err error)                       // Get a value and version for a given key
	GetEntry(key string) *Entry                                                     // Get the entire entry without a lock
	Put(key string, value []byte, trackVisibility bool) (version string, err error) // Put a value for a given key and get associated version
	PutEntry(key string, entry *Entry) (modified bool)                              // Put the entry without modifying the version
	View() map[string]Version                                                       // Returns a map containing the latest version of all keys
	Update(key string, version *Version)                                            // Update the version scalar from a remote source
	Snapshot(path string) error                                                     // Write a snapshot of the version history to disk
	Length() int                                                                    // Returns the number of items in the store (number of keys)

}

//===========================================================================
// Storage with Linearizable Consistency
//===========================================================================

// LinearizableStore implements a versioned, in-memory key-value store that
// keeps a single montonically increasing counter across all objects such
// that a single ordering for all writes (and associated reads) exists. All
// accesses are guarded by read and write locks to ensure linearizable
// consistency and version parents are the last written object no matter the
// key to create a cross-object version history.
type LinearizableStore struct {
	sync.RWMutex
	pid       uint64            // the local process id
	current   uint64            // the current version scalar
	lastWrite *Version          // the version of the last write
	namespace map[string]*Entry // maps keys to the latest entry
	history   *History          // tracks the verion history chain
}

// Init the store creating the internal data structures.
func (s *LinearizableStore) Init(pid uint64) {
	s.pid = pid
	s.namespace = make(map[string]*Entry)
	s.lastWrite = &NullVersion

	// Create, initialize and run the history.
	s.history = new(History)
	s.history.Init()
	s.history.Run()
}

// Get the most recently seen value and version pair for a specific key.
// Returns a not found error if the key has not been written to the namespace.
//
// This operation wraps the entire store in a read lock, meaning that other
// values can be read but no values can be written on Get.
func (s *LinearizableStore) Get(key string) (value []byte, version string, err error) {
	s.RLock()
	defer s.RUnlock()

	entry, ok := s.namespace[key]
	if !ok {
		err = fmt.Errorf("key '%s' not found in namespace", key)
		return value, version, err
	}

	version = entry.Version.String()
	value = entry.Value
	return value, version, err
}

// GetEntry returns the entire entry from the namespace without a lock.
// Returns nil if the given key is not in the store.
func (s *LinearizableStore) GetEntry(key string) *Entry {
	entry, ok := s.namespace[key]
	if !ok {
		return nil
	}
	return entry
}

// Put a value into the namespace, incrementing the version across all
// objects. This operation creates an entry whose parent is the last written
// version of any object. Put also stores all versions and associated entries,
// maintaining a complete version history.
//
// This operation locks the entire store, waiting for all read locks to be
// released and not allowing any other read or write locks until complete.
func (s *LinearizableStore) Put(key string, value []byte, trackVisibility bool) (string, error) {
	s.Lock()
	defer s.Unlock()

	// Create the new version
	s.current++
	version := &Version{s.current, s.pid}

	// Create the new entry
	entry := &Entry{
		Key:             &key,
		Version:         version,
		Parent:          s.lastWrite,
		Value:           value,
		TrackVisibility: trackVisibility,
	}

	// Update the namespace, versions, and last write
	s.namespace[key] = entry
	s.history.Append(entry.Key, entry.Parent, entry.Version)
	s.lastWrite = version

	// Return the version and no error for this method
	return version.String(), nil
}

// PutEntry without modifying version information. Returns true if the entry
// is modified or not -- will only put an entry that has a greater version
// than the current entry.
//
// This method is also responsible for updating the lamport clock.
func (s *LinearizableStore) PutEntry(key string, entry *Entry) bool {
	s.Lock()
	defer s.Unlock()

	// Get the current version
	current, ok := s.namespace[key]
	if !ok {
		current = &Entry{Key: &key, Version: &NullVersion, Parent: &NullVersion}
	}

	// If entry is less than or equal to current version, do not put.
	if entry.Version.LesserEqual(current.Version) {
		return false
	}

	// Update the version scalar
	if entry.Version.Scalar > s.current {
		s.current = entry.Version.Scalar
	}

	// Update the entry
	current.Version = entry.Version
	current.Parent = entry.Parent
	current.Value = entry.Value
	current.TrackVisibility = entry.TrackVisibility

	// Update the namespace, versions, and last write
	s.namespace[key] = current
	s.history.Append(current.Key, current.Parent, current.Version)
	s.lastWrite = current.Version
	return true
}

// Update the current version counter with the global value.
func (s *LinearizableStore) Update(key string, version *Version) {
	if version.Scalar > s.current {
		s.current = version.Scalar
	}
}

// View returns the current version for every key in the namespace.
func (s *LinearizableStore) View() map[string]Version {
	s.RLock()
	defer s.RUnlock()

	view := make(map[string]Version)
	for key, entry := range s.namespace {
		view[key] = *entry.Version
	}

	return view
}

// Snapshot the current version history to disk, writing the version data to
// the specified path. Returns any I/O errors if snapshotting is unsuccessful.
func (s *LinearizableStore) Snapshot(path string) error {
	return fmt.Errorf("version history snapshot not implemented yet")
}

// Length returns the number of items in the Store, namely the number of keys
// in the namespace. This does not reflect the number of versions.
func (s *LinearizableStore) Length() int {
	return len(s.namespace)
}

//===========================================================================
// Storage wtih Sequential Consistency
//===========================================================================

// SequentialStore implements a key/value store where each key is versioned
// independently of all other keys. The Store is only locked when a new key is
// added, but readers and writers take locks on individual keys afterward.
// A sequential store therefore allows multi-thread access to different
// objects simultaneously.
//
// The version history for objects in a sequential store is therefore relative
// to the object itself. Parent versions of entries are simply the previous
// entry in the store. Each object has its own independent scalar component.
type SequentialStore struct {
	sync.RWMutex
	pid       uint64            // the local process id
	namespace map[string]*Entry // maps keys to the latest entry
	history   *History          // tracks the verion history chain

}

// Init the store creating the internal data structures.
func (s *SequentialStore) Init(pid uint64) {
	s.pid = pid
	s.namespace = make(map[string]*Entry)

	// Create, initialize and run the history.
	s.history = new(History)
	s.history.Init()
	s.history.Run()
}

// get is an internal method surrounded by a read lock that fetches the
// given value for a specific key. It returns a locked entry, if the mutable
// flag is true, it is write locked, otherwise it is read locked.
//
// NOTE: callers must unlock the entry themselves!
func (s *SequentialStore) get(key string, mutable bool) *Entry {
	s.RLock()
	defer s.RUnlock()

	// Get the entry from the namespace
	entry, ok := s.namespace[key]
	if !ok {
		return nil
	}

	// Lock the entry according the mutable flag
	if mutable {
		entry.Lock()
	} else {
		entry.RLock()
	}

	// Return the locked entry
	return entry
}

// Get the most recently seen value and version pair for a specific key.
// Returns a not found error if the key has not been written to the namespace.
//
// This operation only locks the store with a read-lock on fetch but also adds
// a read-lock to the entry so that it cannot be modified in flight.
func (s *SequentialStore) Get(key string) (value []byte, version string, err error) {
	// Fetch the value, read-locking the entire store
	entry := s.get(key, false)

	// Handle not found error
	if entry == nil {
		err = fmt.Errorf("key '%s' not found in namespace", key)
		return nil, "", err
	}

	// Ensure that the entry is unlocked before we're done
	defer entry.RUnlock()

	// Extract the data required from the entry.
	return entry.Value, entry.Version.String(), nil
}

// GetEntry returns the entire entry from the namespace without a lock.
// Returns nil if the given key is not in the store.
func (s *SequentialStore) GetEntry(key string) *Entry {
	entry, ok := s.namespace[key]
	if !ok {
		return nil
	}
	return entry
}

// make is an internal method that surrounds the store in a write lock to
// create an empty entry for the given key. It returns a write locked entry to
// ensure that the caller can update the entry with values before unlock but
// releases the store as soon as possible to prevent write delays.
//
// NOTE: this method should not be called if the key already exists!
// NOTE: callers must unlock the entry themselves!
func (s *SequentialStore) make(key string) *Entry {
	// Acquire a write lock
	s.Lock()
	defer s.Unlock()

	// Create a write locked entry
	entry := &Entry{Key: &key, Version: &NullVersion, Parent: &NullVersion}
	entry.Lock()

	// Insert the entry into the namespace and return it
	s.namespace[key] = entry
	return entry
}

// Put a value into the namespace and increment the version. Returns the
// version for the given key and any error that might occur.
func (s *SequentialStore) Put(key string, value []byte, trackVisibility bool) (string, error) {
	// Attempt to get the write-locked version from the store
	entry := s.get(key, true)

	// Make an empty entry if there was no entry already in the store
	if entry == nil {
		entry = s.make(key)
	} else {
		// Update the parent of the entry to the old entry
		entry.Parent = entry.Version
	}

	// Ensure that the entry is unlocked when done
	defer entry.Unlock()

	// Create the version for the new entry
	entry.Current++
	entry.Version = &Version{entry.Current, s.pid}

	// Update the value
	entry.Value = value
	entry.TrackVisibility = trackVisibility

	// Store the version in the version history and return it
	s.history.Append(entry.Key, entry.Parent, entry.Version)
	return entry.Version.String(), nil
}

// PutEntry without modifying version information. Returns true if the entry
// is modified or not -- will only put an entry that has a greater version
// than the current entry.
//
// This method is also responsible for updating the lamport clock.
func (s *SequentialStore) PutEntry(key string, entry *Entry) bool {
	// Attempt to get the write-locked version from the store
	current := s.get(key, true)

	// Make an empty entry if there was no entry already in the store.
	if current == nil {
		current = s.make(key)
	}

	// Ensure the entry is unlocked when done
	defer current.Unlock()

	// If entry is less than or equal to current version, do not put.
	if entry.Version.LesserEqual(current.Version) {
		return false
	}

	// Update the scalar with the new information.
	if entry.Version.Scalar > current.Current {
		current.Current = entry.Version.Scalar
	}

	// Replace current entry with new information.
	current.Version = entry.Version
	current.Parent = entry.Parent
	current.Value = entry.Value
	current.TrackVisibility = entry.TrackVisibility

	// Store the version in the version history and return true.
	s.history.Append(current.Key, current.Parent, current.Version)
	return true
}

// Update the current version counter with the global value.
func (s *SequentialStore) Update(key string, version *Version) {
	entry := s.get(key, true)
	if entry == nil {
		entry = s.make(key)
	}

	defer entry.Unlock()

	if version.Scalar > entry.Current {
		entry.Current = version.Scalar
	}
}

// View returns the current version for every key in the namespace.
func (s *SequentialStore) View() map[string]Version {
	s.RLock()
	defer s.RUnlock()

	view := make(map[string]Version)
	for key, entry := range s.namespace {
		view[key] = *entry.Version
	}

	return view
}

// Snapshot the current version history to disk, writing the version data to
// the specified path. Returns any I/O errors if snapshotting is unsuccessful.
func (s *SequentialStore) Snapshot(path string) error {
	return fmt.Errorf("version history snapshot not implemented yet")
}

// Length returns the number of items in the Store, namely the number of keys
// in the namespace. This does not reflect the number of versions.
func (s *SequentialStore) Length() int {
	return len(s.namespace)
}
<|endoftext|>


<commit_before>package stocks

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"time"
)

const (
	timeout = time.Duration(time.Second * 10)
)

// get full stock details into a struct
func GetQuote(symbol string) (Stock, error) {
	client := http.Client{Timeout: timeout}

	url := fmt.Sprintf("http://finance.yahoo.com/webservice/v1/symbols/%s/quote?format=json", symbol)
	res, err := client.Get(url)
	if err != nil {
		return Stock{}, fmt.Errorf("Stocks cannot access yahoo finance API: %v", err)
	}
	defer res.Body.Close()

	content, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return Stock{}, fmt.Errorf("Stocks cannot read json body: %v", err)
	}

	var stock Stock

	err = json.Unmarshal(content, &stock)
	if err != nil {
		return Stock{}, fmt.Errorf("Stocks cannot parse json data: %v", err)
	}

	return stock, nil
}

// return the stock name
func (stock Stock) GetName() string {
	return stock.
		List.
		Resources[0].
		Resource.
		Fields.
		Name
}

// return the stock symbol
func (stock Stock) GetSymbol() string {
	return stock.List.Resources[0].Resource.Fields.Symbol
}

// return the stock price
func (stock Stock) GetPrice() (float64, error) {
	price, err := strconv.ParseFloat(stock.List.Resources[0].Resource.Fields.Price, 64)
	if err != nil {
		return 1.0, fmt.Errorf("Stock price: %v", err)
	}

	return price, nil
}

// just print all details nicely
func (stock Stock) PrettyPrint() {
	name := stock.GetName()
	sym := stock.GetSymbol()
	price, err := stock.GetPrice()
	if err != nil {
		fmt.Errorf("Error getting price: %v", err)
	}

	fmt.Println("-------------------------------")
	fmt.Printf("Name:\t%s\nSymbol:\t%s\nPrice:\t%f\n", name, sym, price)
	fmt.Println("-------------------------------")
}
<commit_msg>cleanup<commit_after>package stocks

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"time"
)

const (
	timeout = time.Duration(time.Second * 10)
)

// get full stock details into a struct
func GetQuote(symbol string) (Stock, error) {
	// set http client timeout
	client := http.Client{Timeout: timeout}

	url := fmt.Sprintf("http://finance.yahoo.com/webservice/v1/symbols/%s/quote?format=json", symbol)
	res, err := client.Get(url)
	if err != nil {
		return Stock{}, fmt.Errorf("Stocks cannot access yahoo finance API: %v", err)
	}
	defer res.Body.Close()

	content, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return Stock{}, fmt.Errorf("Stocks cannot read json body: %v", err)
	}

	var stock Stock

	err = json.Unmarshal(content, &stock)
	if err != nil {
		return Stock{}, fmt.Errorf("Stocks cannot parse json data: %v", err)
	}

	return stock, nil
}

// return the stock name
func (stock Stock) GetName() string {
	return stock.List.Resources[0].Resource.Fields.Name
}

// return the stock symbol
func (stock Stock) GetSymbol() string {
	return stock.List.Resources[0].Resource.Fields.Symbol
}

// return the stock price
func (stock Stock) GetPrice() (float64, error) {
	price, err := strconv.ParseFloat(stock.List.Resources[0].Resource.Fields.Price, 64)
	if err != nil {
		return 1.0, fmt.Errorf("Stock price: %v", err)
	}

	return price, nil
}

// just print all details nicely
func (stock Stock) PrettyPrint() {
	name := stock.GetName()
	sym := stock.GetSymbol()
	price, err := stock.GetPrice()
	if err != nil {
		fmt.Printf("Error getting price: %v", err)
	}

	fmt.Printf("Name:\t%s\nSymbol:\t%s\nPrice:\t%f\n", name, sym, price)
}
<|endoftext|>


<commit_before>package cruncy

import (
	"errors"
	"sync"
	"time"

	"github.com/boltdb/bolt"
)

type Store struct {
	db    *bolt.DB
	mutex sync.Mutex
}

var (
	ErrNotFound = errors.New("ression: key not found")
	ErrBadValue = errors.New("ression: bad value")
)

func Open(path string) (*Store, error) {
	opts := &bolt.Options{
		Timeout: 50 * time.Millisecond,
	}
	db, err := bolt.Open(path, 0640, opts)

	if err != nil {
		return nil, err
	}

	return &Store{db: db}, nil
}

func (store *Store) CreateBucket(bucket string) error {
	return store.db.Update(func(tx *bolt.Tx) error {
		_, err := tx.CreateBucketIfNotExists([]byte(bucket))
		return err
	})
}

func (store *Store) Close() error {
	return store.db.Close()
}

func (store *Store) Put(bucket string, key string, value string) error {
	store.mutex.Lock()
	defer store.mutex.Unlock()

	return store.db.Update(func(tx *bolt.Tx) error {
		return tx.Bucket([]byte(bucket)).Put([]byte(key), ([]byte(value)))
	})
}

func (store *Store) Get(bucket, key string, value *string) error {
	store.mutex.Lock()
	defer store.mutex.Unlock()

	return store.db.View(func(tx *bolt.Tx) error {
		c := tx.Bucket([]byte(bucket)).Cursor()
		if k, v := c.Seek([]byte(key)); k == nil || string(k) != key {
			return ErrNotFound
		} else if value == nil {
			return nil
		} else {
			*value = string(v[:])
			return nil
		}
	})
}

// Delete the entry with the given key. If no such key is present in the store,
// it returns ErrNotFound.
//
//	store.Delete("key42")
func (store *Store) Delete(bucket string, key string) error {
	store.mutex.Lock()
	defer store.mutex.Unlock()

	return store.db.Update(func(tx *bolt.Tx) error {
		c := tx.Bucket([]byte(bucket)).Cursor()
		if k, _ := c.Seek([]byte(key)); k == nil || string(k) != key {
			return ErrNotFound
		} else {
			return c.Delete()
		}
	})
}
<commit_msg>Work on store<commit_after>package cruncy

import (
	"errors"
	"sync"
	"time"

	bolt "go.etcd.io/bbolt"
)

// Store is db abstarction on top of bolt db
type Store struct {
	db    *bolt.DB
	mutex sync.Mutex
}

var (
	// ErrNotFound error key not found
	ErrNotFound = errors.New("store: key not found")
	// ErrBadValue error bad value
	ErrBadValue = errors.New("store: bad value")
)

// Open a database file
func Open(path string) (*Store, error) {
	opts := &bolt.Options{
		Timeout: 50 * time.Millisecond,
	}
	db, err := bolt.Open(path, 0640, opts)

	if err != nil {
		return nil, err
	}

	return &Store{db: db}, nil
}

// CreateBucket creates a buck
func (store *Store) CreateBucket(bucket string) error {
	return store.db.Update(func(tx *bolt.Tx) error {
		_, err := tx.CreateBucketIfNotExists([]byte(bucket))
		return err
	})
}

// Close the database
func (store *Store) Close() error {
	return store.db.Close()
}

// Put a key/value into a given bucket
func (store *Store) Put(bucket string, key string, value string) error {
	store.mutex.Lock()
	defer store.mutex.Unlock()

	return store.db.Update(func(tx *bolt.Tx) error {
		return tx.Bucket([]byte(bucket)).Put([]byte(key), ([]byte(value)))
	})
}

// Get a key/value from a given bucket
func (store *Store) Get(bucket, key string, value *string) error {
	store.mutex.Lock()
	defer store.mutex.Unlock()

	return store.db.View(func(tx *bolt.Tx) error {
		c := tx.Bucket([]byte(bucket)).Cursor()
		if k, v := c.Seek([]byte(key)); k == nil || string(k) != key {
			return ErrNotFound
		} else if value == nil {
			return nil
		} else {
			*value = string(v[:])
			return nil
		}
	})
}

// Delete the entry with the given key. If no such key is present in the store,
// it returns ErrNotFound.
//
//	store.Delete("key42")
func (store *Store) Delete(bucket string, key string) error {
	store.mutex.Lock()
	defer store.mutex.Unlock()

	return store.db.Update(func(tx *bolt.Tx) error {
		c := tx.Bucket([]byte(bucket)).Cursor()
		if k, _ := c.Seek([]byte(key)); k == nil || string(k) != key {
			return ErrNotFound
		} else {
			return c.Delete()
		}
	})
}
<|endoftext|>


<commit_before>package rethinkdb_session_store

import (
	"net/http"

	"github.com/dancannon/gorethink"
	"github.com/gorilla/securecookie"
	"github.com/gorilla/sessions"
)

type RethinkDBStore struct {
	Codecs  []securecookie.Codec
	Options *sessions.Options // default configuration

	term             gorethink.Term
	rethinkdbSession *gorethink.Session
}

func NewRethinkDBStore(rethinkdbSession *gorethink.Session, db, table string, keyPairs ...[]byte) *RethinkDBStore {
	return &RethinkDBStore{
		Codecs: securecookie.CodecsFromPairs(keyPairs...),
		Options: &sessions.Options{
			Path:   "/",
			MaxAge: 86400 * 30,
		},
		term:             gorethink.Db(db).Table(table),
		rethinkdbSession: rethinkdbSession,
	}
}

func (s *RethinkDBStore) Get(r *http.Request, name string) (*sessions.Session, error) {
	return sessions.GetRegistry(r).Get(s, name)
}

func (s *RethinkDBStore) New(r *http.Request, name string) (*sessions.Session, error) {
	session := sessions.NewSession(s, name)
	opts := *s.Options
	session.Options = &opts
	session.IsNew = true
	var err error
	if c, errCookie := r.Cookie(name); errCookie == nil {
		err = securecookie.DecodeMulti(name, c.Value, &session.ID, s.Codecs...)
		if err == nil {
			err = s.load(session)
			if err == nil {
				session.IsNew = false
			}
		}
	}
	return session, err
}

func (s *RethinkDBStore) Save(r *http.Request, w http.ResponseWriter, session *sessions.Session) error {
	if err := s.save(session); err != nil {
		return err
	}
	encoded, err := securecookie.EncodeMulti(session.Name(), session.ID, s.Codecs...)
	if err != nil {
		return err
	}
	http.SetCookie(w, sessions.NewCookie(session.Name(), encoded, session.Options))
	return nil
}

func (s *RethinkDBStore) save(session *sessions.Session) error {
	encoded, err := securecookie.EncodeMulti(session.Name(), session.Values, s.Codecs...)
	if err != nil {
		return err
	}

	value := map[string]interface{}{"encoded": encoded}
	if session.ID != "" {
		_, err := s.term.Get(session.ID).Update(value).Run(s.rethinkdbSession)
		if err != nil {
			return err
		}
	} else {
		write, err := s.term.Insert(value).RunWrite(s.rethinkdbSession)
		if err != nil {
			return err
		}
		session.ID = write.GeneratedKeys[0]
	}
	return nil
}

func (s *RethinkDBStore) load(session *sessions.Session) error {
	value := map[string]interface{}{}
	cursor, err := s.term.Get(session.ID).Run(s.rethinkdbSession)
	if err != nil {
		return err
	}
	err = cursor.One(&value)
	if err != nil {
		return err
	}

	if err = securecookie.DecodeMulti(session.Name(), value["encoded"].(string), &session.Values, s.Codecs...); err != nil {
		return err
	}
	return nil
}
<commit_msg>change storage to raw json<commit_after>package rethinkdb_session_store

import (
	"errors"
	"net/http"

	"github.com/dancannon/gorethink"
	"github.com/gorilla/securecookie"
	"github.com/gorilla/sessions"
)

type RethinkDBStore struct {
	Codecs  []securecookie.Codec
	Options *sessions.Options // default configuration

	term             gorethink.Term
	rethinkdbSession *gorethink.Session
}

func NewRethinkDBStore(rethinkdbSession *gorethink.Session, db, table string, keyPairs ...[]byte) *RethinkDBStore {
	return &RethinkDBStore{
		Codecs: securecookie.CodecsFromPairs(keyPairs...),
		Options: &sessions.Options{
			Path:   "/",
			MaxAge: 86400 * 30,
		},
		term:             gorethink.Db(db).Table(table),
		rethinkdbSession: rethinkdbSession,
	}
}

func (s *RethinkDBStore) Get(r *http.Request, name string) (*sessions.Session, error) {
	return sessions.GetRegistry(r).Get(s, name)
}

func (s *RethinkDBStore) New(r *http.Request, name string) (*sessions.Session, error) {
	session := sessions.NewSession(s, name)
	opts := *s.Options
	session.Options = &opts
	session.IsNew = true
	var err error
	if c, errCookie := r.Cookie(name); errCookie == nil {
		err = securecookie.DecodeMulti(name, c.Value, &session.ID, s.Codecs...)
		if err == nil {
			err = s.load(session)
			if err == nil {
				session.IsNew = false
			}
		}
	}
	return session, err
}

func (s *RethinkDBStore) Save(r *http.Request, w http.ResponseWriter, session *sessions.Session) error {
	if err := s.save(session); err != nil {
		return err
	}
	encoded, err := securecookie.EncodeMulti(session.Name(), session.ID, s.Codecs...)
	if err != nil {
		return err
	}
	http.SetCookie(w, sessions.NewCookie(session.Name(), encoded, session.Options))
	return nil
}

func (s *RethinkDBStore) save(session *sessions.Session) error {
	values := map[string]interface{}{}
	for k, v := range session.Values {
		kstr, ok := k.(string)
		if !ok {
			return errors.New("cannot serialize non-string value key")
		}

		values[kstr] = v
	}

	json := map[string]interface{}{
		"name":   session.Name(),
		"values": values,
	}

	var write gorethink.WriteResponse
	var err error
	if session.ID != "" {
		write, err = s.term.Get(session.ID).Update(json).RunWrite(s.rethinkdbSession)
		if err == nil && write.Updated == 0 {
			json["id"] = session.ID
		}
	}

	if write.Updated == 0 {
		write, err = s.term.Insert(json).RunWrite(s.rethinkdbSession)
		if err == nil && len(write.GeneratedKeys) > 0 {
			session.ID = write.GeneratedKeys[0]
		}
	}

	return err
}

func (s *RethinkDBStore) load(session *sessions.Session) error {
	if session.ID == "" {
		return errors.New("invalid session id")
	}

	json := map[string]interface{}{}
	cursor, err := s.term.Get(session.ID).Run(s.rethinkdbSession)
	if err != nil {
		return err
	}
	err = cursor.One(&json)
	if err != nil {
		return err
	}

	values, ok := json["values"].(map[string]interface{})
	if !ok {
		return errors.New("failed to decode session")
	}

	for k, v := range values {
		session.Values[k] = v
	}

	return nil
}
<|endoftext|>