<commit_before>/*
Package gopiano provides a thin wrapper library around the Pandora.com client API.

This client API has been reverse engineered and documentation is available at
https://6xq.net/pandora-apidoc/

The package provides a Client struct with a myriad of methods which interact with the
Pandora JSON API's own methods. Each method returns a struct of the parsed JSON data and an error.
All of the responses that these methods return can be found in the responses subpackage. There
is also a requests subpackage but mostly you don't need to bother with those; they get instantiated
by these client methods.
*/
package gopiano

import (
	"encoding/hex"
	"encoding/json"
	"io"
	"net/http"
	"net/url"
	"strings"
	"time"

	"golang.org/x/crypto/blowfish"

	"denniskupec.com/gopiano/coder"
	"denniskupec.com/gopiano/request"
	"denniskupec.com/gopiano/response"
)

// Describes a particular type of client to emulate.
type ClientDescription struct {
	DeviceModel string
	Username    string
	Password    string
	BaseURL     string
	EncryptKey  string
	DecryptKey  string
	Version     string
}

// Class for a Client object.
type Client struct {
	description      ClientDescription
	encrypter        *blowfish.Cipher
	decrypter        *blowfish.Cipher
	timeOffset       time.Duration
	partnerAuthToken string
	partnerID        string
	userAuthToken    string
	userID           string
}

// Create a new Client with specified ClientDescription
func NewClient(d ClientDescription) (*Client, error) {
	encrypter, err := blowfish.NewCipher([]byte(d.EncryptKey))
	if err != nil {
		return nil, err
	}
	decrypter, err := blowfish.NewCipher([]byte(d.DecryptKey))
	if err != nil {
		return nil, err
	}
	return &Client{
		description: d,
		encrypter:   encrypter,
		decrypter:   decrypter,
	}, nil
}

// Blowfish decrypts a string in ECB mode.
// Some data returned from the Pandora API is encrypted. This decrypts it.
// The key for the decryption is provided by the ClientDescription.
func (c *Client) decrypt(data string) (string, error) {
	chunks := make([]string, 0)
	for i := 0; i < len(data); i += 16 {
		var buf [16]byte
		var decoded, decrypted [8]byte
		copy(buf[:], data[i:])
		_, err := hex.Decode(decoded[:], buf[:])
		if err != nil {
			return "", err
		}
		c.decrypter.Decrypt(decrypted[:], decoded[:])
		chunks = append(chunks, strings.Trim(string(decrypted[:]), "\x00"))
	}
	return strings.Join(chunks, ""), nil
}

// Client.PandoraCall is the basic function to send an HTTP POST to pandora.com.
// Arguments: protocol is either "https://" or "http://", method is whatever must be in
// the "method" url argument and specifies the remote procedure to call, body is an io.Reader
// to be passed directly into http.Post, and data is to be passed to json.Unmarshal to parse
// the JSON response.
func PandoraCall(callUrl string, body io.Reader) (json.RawMessage, error) {
	resp, err := http.Post(callUrl, "text/plain", body)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	var wrap response.Wrapper
	if err := json.NewDecoder(resp.Body).Decode(&wrap); err != nil {
		return nil, err
	}

	if wrap.Stat == "fail" {
		if message, ok := response.ErrorCodeMap[wrap.Code]; ok {
			wrap.Message = message
		}
		return nil, wrap.ErrorResponse
	}

	return wrap.Result, nil
}

func (c *Client) formatURL(req request.Type) string {
	urlArgs := url.Values{
		"method": {req.Method()},
	}

	if c.partnerID != "" {
		urlArgs.Add("partner_id", c.partnerID)
	}
	if c.userID != "" {
		urlArgs.Add("user_id", c.userID)
	}
	if c.partnerAuthToken != "" && c.userAuthToken == "" {
		urlArgs.Add("auth_token", c.partnerAuthToken)
	} else if c.userAuthToken != "" {
		urlArgs.Add("auth_token", c.userAuthToken)
	}

	return req.Protocol().URL() + c.description.BaseURL + "?" + urlArgs.Encode()
}

func (c *Client) Call(req request.Type, data interface{}) error {
	enc := coder.New(c.encrypter)
	if err := json.NewEncoder(enc).Encode(req); err != nil {
		return err
	}

	res, err := PandoraCall(c.formatURL(req), enc)
	if err != nil {
		return err
	}

	return json.Unmarshal(res, data)
}

// Most calls require a SyncTime int argument (Unix epoch). We store our current time offset
// but must calculate the SyncTime for each call. This method does that.
func (c *Client) GetSyncTime() int {
	return int(time.Now().Add(c.timeOffset).Unix())
}

func (c *Client) Token() request.UserToken {
	return request.UserToken{
		UserAuthToken: c.userAuthToken,
		SyncTime:      c.GetSyncTime(),
	}
}
<commit_msg>gopiano: refactor call error handling<commit_after>/*
Package gopiano provides a thin wrapper library around the Pandora.com client API.

This client API has been reverse engineered and documentation is available at
https://6xq.net/pandora-apidoc/

The package provides a Client struct with a myriad of methods which interact with the
Pandora JSON API's own methods. Each method returns a struct of the parsed JSON data and an error.
All of the responses that these methods return can be found in the responses subpackage. There
is also a requests subpackage but mostly you don't need to bother with those; they get instantiated
by these client methods.
*/
package gopiano

import (
	"encoding/hex"
	"encoding/json"
	"io"
	"net/http"
	"net/url"
	"strings"
	"time"

	"golang.org/x/crypto/blowfish"

	"denniskupec.com/gopiano/coder"
	"denniskupec.com/gopiano/request"
	"denniskupec.com/gopiano/response"
)

// Describes a particular type of client to emulate.
type ClientDescription struct {
	DeviceModel string
	Username    string
	Password    string
	BaseURL     string
	EncryptKey  string
	DecryptKey  string
	Version     string
}

// Class for a Client object.
type Client struct {
	description      ClientDescription
	encrypter        *blowfish.Cipher
	decrypter        *blowfish.Cipher
	timeOffset       time.Duration
	partnerAuthToken string
	partnerID        string
	userAuthToken    string
	userID           string
}

// Create a new Client with specified ClientDescription
func NewClient(d ClientDescription) (*Client, error) {
	encrypter, err := blowfish.NewCipher([]byte(d.EncryptKey))
	if err != nil {
		return nil, err
	}
	decrypter, err := blowfish.NewCipher([]byte(d.DecryptKey))
	if err != nil {
		return nil, err
	}
	return &Client{
		description: d,
		encrypter:   encrypter,
		decrypter:   decrypter,
	}, nil
}

// Blowfish decrypts a string in ECB mode.
// Some data returned from the Pandora API is encrypted. This decrypts it.
// The key for the decryption is provided by the ClientDescription.
func (c *Client) decrypt(data string) (string, error) {
	chunks := make([]string, 0)
	for i := 0; i < len(data); i += 16 {
		var buf [16]byte
		var decoded, decrypted [8]byte
		copy(buf[:], data[i:])
		_, err := hex.Decode(decoded[:], buf[:])
		if err != nil {
			return "", err
		}
		c.decrypter.Decrypt(decrypted[:], decoded[:])
		chunks = append(chunks, strings.Trim(string(decrypted[:]), "\x00"))
	}
	return strings.Join(chunks, ""), nil
}

// Client.PandoraCall is the basic function to send an HTTP POST to pandora.com.
// Arguments: protocol is either "https://" or "http://", method is whatever must be in
// the "method" url argument and specifies the remote procedure to call, body is an io.Reader
// to be passed directly into http.Post, and data is to be passed to json.Unmarshal to parse
// the JSON response.
func PandoraCall(callUrl string, body io.Reader) (json.RawMessage, error) {
	resp, err := http.Post(callUrl, "text/plain", body)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	var wrap response.Wrapper
	if err := json.NewDecoder(resp.Body).Decode(&wrap); err != nil {
		return nil, err
	}

	if wrap.Stat == "ok" {
		return wrap.Result, nil
	}

	if message, ok := response.ErrorCodeMap[wrap.Code]; ok {
		wrap.ErrorResponse.Message = message
	}

	return nil, wrap.ErrorResponse
}

func (c *Client) formatURL(req request.Type) string {
	urlArgs := url.Values{
		"method": {req.Method()},
	}

	if c.partnerID != "" {
		urlArgs.Add("partner_id", c.partnerID)
	}
	if c.userID != "" {
		urlArgs.Add("user_id", c.userID)
	}
	if c.partnerAuthToken != "" && c.userAuthToken == "" {
		urlArgs.Add("auth_token", c.partnerAuthToken)
	} else if c.userAuthToken != "" {
		urlArgs.Add("auth_token", c.userAuthToken)
	}

	return req.Protocol().URL() + c.description.BaseURL + "?" + urlArgs.Encode()
}

func (c *Client) Call(req request.Type, data interface{}) error {
	enc := coder.New(c.encrypter)
	if err := json.NewEncoder(enc).Encode(req); err != nil {
		return err
	}

	res, err := PandoraCall(c.formatURL(req), enc)
	if err != nil {
		return err
	}

	return json.Unmarshal(res, data)
}

// Most calls require a SyncTime int argument (Unix epoch). We store our current time offset
// but must calculate the SyncTime for each call. This method does that.
func (c *Client) GetSyncTime() int {
	return int(time.Now().Add(c.timeOffset).Unix())
}

func (c *Client) Token() request.UserToken {
	return request.UserToken{
		UserAuthToken: c.userAuthToken,
		SyncTime:      c.GetSyncTime(),
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package gogs

import (
	"bytes"
	"encoding/json"
	"fmt"
)

type CreateUserOption struct {
	SourceID   int64  `json:"source_id"`
	LoginName  string `json:"login_name"`
	Username   string `json:"username" binding:"Required;AlphaDashDot;MaxSize(35)"`
	FullName   string `json:"full_name" binding:"MaxSize(100)"`
	Email      string `json:"email" binding:"Required;Email;MaxSize(254)"`
	Password   string `json:"password" binding:"MaxSize(255)"`
	SendNotify bool   `json:"send_notify"`
}

func (c *Client) AdminCreateUser(opt CreateUserOption) (*User, error) {
	body, err := json.Marshal(&opt)
	if err != nil {
		return nil, err
	}
	user := new(User)
	return user, c.getParsedResponse("POST", "/admin/users", jsonHeader, bytes.NewReader(body), user)
}

type EditUserOption struct {
	SourceID         int64  `json:"source_id"`
	LoginName        string `json:"login_name"`
	FullName         string `json:"full_name" binding:"MaxSize(100)"`
	Email            string `json:"email" binding:"Required;Email;MaxSize(254)"`
	Password         string `json:"password" binding:"MaxSize(255)"`
	Website          string `json:"website" binding:"MaxSize(50)"`
	Location         string `json:"location" binding:"MaxSize(50)"`
	Active           *bool  `json:"active"`
	Admin            *bool  `json:"admin"`
	AllowGitHook     *bool  `json:"allow_git_hook"`
	AllowImportLocal *bool  `json:"allow_import_local"`
}

func (c *Client) AdminEditUser(user string, opt EditUserOption) error {
	body, err := json.Marshal(&opt)
	if err != nil {
		return err
	}
	_, err = c.getResponse("PATCH", fmt.Sprintf("/admin/users/%s", user), jsonHeader, bytes.NewReader(body))
	return err
}

func (c *Client) AdminDeleteUser(user string) error {
	_, err := c.getResponse("DELETE", fmt.Sprintf("/admin/users/%s", user), nil, nil)
	return err
}

func (c *Client) AdminCreateUserPublicKey(user string, opt CreateKeyOption) (*PublicKey, error) {
	body, err := json.Marshal(&opt)
	if err != nil {
		return nil, err
	}
	key := new(PublicKey)
	return key, c.getParsedResponse("POST", fmt.Sprintf("/admin/users/%s/keys", user), jsonHeader, bytes.NewReader(body), key)
}
<commit_msg>Add optional MaxRepoCreation field to EditUserOption (#22)<commit_after>// Copyright 2015 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package gogs

import (
	"bytes"
	"encoding/json"
	"fmt"
)

type CreateUserOption struct {
	SourceID   int64  `json:"source_id"`
	LoginName  string `json:"login_name"`
	Username   string `json:"username" binding:"Required;AlphaDashDot;MaxSize(35)"`
	FullName   string `json:"full_name" binding:"MaxSize(100)"`
	Email      string `json:"email" binding:"Required;Email;MaxSize(254)"`
	Password   string `json:"password" binding:"MaxSize(255)"`
	SendNotify bool   `json:"send_notify"`
}

func (c *Client) AdminCreateUser(opt CreateUserOption) (*User, error) {
	body, err := json.Marshal(&opt)
	if err != nil {
		return nil, err
	}
	user := new(User)
	return user, c.getParsedResponse("POST", "/admin/users", jsonHeader, bytes.NewReader(body), user)
}

type EditUserOption struct {
	SourceID         int64  `json:"source_id"`
	LoginName        string `json:"login_name"`
	FullName         string `json:"full_name" binding:"MaxSize(100)"`
	Email            string `json:"email" binding:"Required;Email;MaxSize(254)"`
	Password         string `json:"password" binding:"MaxSize(255)"`
	Website          string `json:"website" binding:"MaxSize(50)"`
	Location         string `json:"location" binding:"MaxSize(50)"`
	Active           *bool  `json:"active"`
	Admin            *bool  `json:"admin"`
	AllowGitHook     *bool  `json:"allow_git_hook"`
	AllowImportLocal *bool  `json:"allow_import_local"`
	MaxRepoCreation  *int   `json:"max_repo_creation"`
}

func (c *Client) AdminEditUser(user string, opt EditUserOption) error {
	body, err := json.Marshal(&opt)
	if err != nil {
		return err
	}
	_, err = c.getResponse("PATCH", fmt.Sprintf("/admin/users/%s", user), jsonHeader, bytes.NewReader(body))
	return err
}

func (c *Client) AdminDeleteUser(user string) error {
	_, err := c.getResponse("DELETE", fmt.Sprintf("/admin/users/%s", user), nil, nil)
	return err
}

func (c *Client) AdminCreateUserPublicKey(user string, opt CreateKeyOption) (*PublicKey, error) {
	body, err := json.Marshal(&opt)
	if err != nil {
		return nil, err
	}
	key := new(PublicKey)
	return key, c.getParsedResponse("POST", fmt.Sprintf("/admin/users/%s/keys", user), jsonHeader, bytes.NewReader(body), key)
}
<|endoftext|>


<commit_before>// Copyright 2015 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with go-ethereum.  If not, see <http://www.gnu.org/licenses/>.

// Package implements a private key management facility.
//
// This abstracts part of a user's interaction with an account she controls.
package accounts

// Currently this is pretty much a passthrough to the KeyStore interface,
// and accounts persistence is derived from stored keys' addresses

import (
	"crypto/ecdsa"
	crand "crypto/rand"
	"errors"
	"fmt"
	"os"
	"sync"
	"time"

	"github.com/ethereum/go-ethereum/common"
	"github.com/ethereum/go-ethereum/crypto"
)

var (
	ErrLocked = errors.New("account is locked")
	ErrNoKeys = errors.New("no keys in store")
)

type Account struct {
	Address common.Address
}

type Manager struct {
	keyStore crypto.KeyStore
	unlocked map[common.Address]*unlocked
	mutex    sync.RWMutex
}

type unlocked struct {
	*crypto.Key
	abort chan struct{}
}

func NewManager(keyStore crypto.KeyStore) *Manager {
	return &Manager{
		keyStore: keyStore,
		unlocked: make(map[common.Address]*unlocked),
	}
}

func (am *Manager) HasAccount(addr common.Address) bool {
	accounts, _ := am.Accounts()
	for _, acct := range accounts {
		if acct.Address == addr {
			return true
		}
	}
	return false
}

func (am *Manager) DeleteAccount(address common.Address, auth string) error {
	return am.keyStore.DeleteKey(address, auth)
}

func (am *Manager) Sign(a Account, toSign []byte) (signature []byte, err error) {
	am.mutex.RLock()
	unlockedKey, found := am.unlocked[a.Address]
	am.mutex.RUnlock()
	if !found {
		return nil, ErrLocked
	}
	signature, err = crypto.Sign(toSign, unlockedKey.PrivateKey)
	return signature, err
}

// unlock indefinitely
func (am *Manager) Unlock(addr common.Address, keyAuth string) error {
	return am.TimedUnlock(addr, keyAuth, 0)
}

// Unlock unlocks the account with the given address. The account
// stays unlocked for the duration of timeout
// it timeout is 0 the account is unlocked for the entire session
func (am *Manager) TimedUnlock(addr common.Address, keyAuth string, timeout time.Duration) error {
	key, err := am.keyStore.GetKey(addr, keyAuth)
	if err != nil {
		return err
	}
	var u *unlocked
	am.mutex.Lock()
	defer am.mutex.Unlock()
	var found bool
	u, found = am.unlocked[addr]
	if found {
		// terminate dropLater for this key to avoid unexpected drops.
		if u.abort != nil {
			close(u.abort)
		}
	}
	if timeout > 0 {
		u = &unlocked{Key: key, abort: make(chan struct{})}
		go am.expire(addr, u, timeout)
	} else {
		u = &unlocked{Key: key}
	}
	am.unlocked[addr] = u
	return nil
}

func (am *Manager) expire(addr common.Address, u *unlocked, timeout time.Duration) {
	t := time.NewTimer(timeout)
	defer t.Stop()
	select {
	case <-u.abort:
		// just quit
	case <-t.C:
		am.mutex.Lock()
		// only drop if it's still the same key instance that dropLater
		// was launched with. we can check that using pointer equality
		// because the map stores a new pointer every time the key is
		// unlocked.
		if am.unlocked[addr] == u {
			zeroKey(u.PrivateKey)
			delete(am.unlocked, addr)
		}
		am.mutex.Unlock()
	}
}

func (am *Manager) NewAccount(auth string) (Account, error) {
	key, err := am.keyStore.GenerateNewKey(crand.Reader, auth)
	if err != nil {
		return Account{}, err
	}
	return Account{Address: key.Address}, nil
}

func (am *Manager) AddressByIndex(index int) (addr string, err error) {
	var addrs []common.Address
	addrs, err = am.keyStore.GetKeyAddresses()
	if err != nil {
		return
	}
	if index < 0 || index >= len(addrs) {
		err = fmt.Errorf("index out of range: %d (should be 0-%d)", index, len(addrs)-1)
	} else {
		addr = addrs[index].Hex()
	}
	return
}

func (am *Manager) Accounts() ([]Account, error) {
	addresses, err := am.keyStore.GetKeyAddresses()
	if os.IsNotExist(err) {
		return nil, ErrNoKeys
	} else if err != nil {
		return nil, err
	}
	accounts := make([]Account, len(addresses))
	for i, addr := range addresses {
		accounts[i] = Account{
			Address: addr,
		}
	}
	return accounts, err
}

// zeroKey zeroes a private key in memory.
func zeroKey(k *ecdsa.PrivateKey) {
	b := k.D.Bits()
	for i := range b {
		b[i] = 0
	}
}

// USE WITH CAUTION = this will save an unencrypted private key on disk
// no cli or js interface
func (am *Manager) Export(path string, addr common.Address, keyAuth string) error {
	key, err := am.keyStore.GetKey(addr, keyAuth)
	if err != nil {
		return err
	}
	return crypto.SaveECDSA(path, key.PrivateKey)
}

func (am *Manager) Import(path string, keyAuth string) (Account, error) {
	privateKeyECDSA, err := crypto.LoadECDSA(path)
	if err != nil {
		return Account{}, err
	}
	key := crypto.NewKeyFromECDSA(privateKeyECDSA)
	if err = am.keyStore.StoreKey(key, keyAuth); err != nil {
		return Account{}, err
	}
	return Account{Address: key.Address}, nil
}

func (am *Manager) Update(addr common.Address, authFrom, authTo string) (err error) {
	var key *crypto.Key
	key, err = am.keyStore.GetKey(addr, authFrom)

	if err == nil {
		err = am.keyStore.StoreKey(key, authTo)
		if err == nil {
			am.keyStore.Cleanup(addr)
		}
	}
	return
}

func (am *Manager) ImportPreSaleKey(keyJSON []byte, password string) (acc Account, err error) {
	var key *crypto.Key
	key, err = crypto.ImportPreSaleKey(am.keyStore, keyJSON, password)
	if err != nil {
		return
	}
	if err = am.keyStore.StoreKey(key, password); err != nil {
		return
	}
	return Account{Address: key.Address}, nil
}
<commit_msg>fix wallet key duplicate write - how did it get there? mystery. fixes #1411<commit_after>// Copyright 2015 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with go-ethereum.  If not, see <http://www.gnu.org/licenses/>.

// Package implements a private key management facility.
//
// This abstracts part of a user's interaction with an account she controls.
package accounts

// Currently this is pretty much a passthrough to the KeyStore interface,
// and accounts persistence is derived from stored keys' addresses

import (
	"crypto/ecdsa"
	crand "crypto/rand"
	"errors"
	"fmt"
	"os"
	"sync"
	"time"

	"github.com/ethereum/go-ethereum/common"
	"github.com/ethereum/go-ethereum/crypto"
)

var (
	ErrLocked = errors.New("account is locked")
	ErrNoKeys = errors.New("no keys in store")
)

type Account struct {
	Address common.Address
}

type Manager struct {
	keyStore crypto.KeyStore
	unlocked map[common.Address]*unlocked
	mutex    sync.RWMutex
}

type unlocked struct {
	*crypto.Key
	abort chan struct{}
}

func NewManager(keyStore crypto.KeyStore) *Manager {
	return &Manager{
		keyStore: keyStore,
		unlocked: make(map[common.Address]*unlocked),
	}
}

func (am *Manager) HasAccount(addr common.Address) bool {
	accounts, _ := am.Accounts()
	for _, acct := range accounts {
		if acct.Address == addr {
			return true
		}
	}
	return false
}

func (am *Manager) DeleteAccount(address common.Address, auth string) error {
	return am.keyStore.DeleteKey(address, auth)
}

func (am *Manager) Sign(a Account, toSign []byte) (signature []byte, err error) {
	am.mutex.RLock()
	unlockedKey, found := am.unlocked[a.Address]
	am.mutex.RUnlock()
	if !found {
		return nil, ErrLocked
	}
	signature, err = crypto.Sign(toSign, unlockedKey.PrivateKey)
	return signature, err
}

// unlock indefinitely
func (am *Manager) Unlock(addr common.Address, keyAuth string) error {
	return am.TimedUnlock(addr, keyAuth, 0)
}

// Unlock unlocks the account with the given address. The account
// stays unlocked for the duration of timeout
// it timeout is 0 the account is unlocked for the entire session
func (am *Manager) TimedUnlock(addr common.Address, keyAuth string, timeout time.Duration) error {
	key, err := am.keyStore.GetKey(addr, keyAuth)
	if err != nil {
		return err
	}
	var u *unlocked
	am.mutex.Lock()
	defer am.mutex.Unlock()
	var found bool
	u, found = am.unlocked[addr]
	if found {
		// terminate dropLater for this key to avoid unexpected drops.
		if u.abort != nil {
			close(u.abort)
		}
	}
	if timeout > 0 {
		u = &unlocked{Key: key, abort: make(chan struct{})}
		go am.expire(addr, u, timeout)
	} else {
		u = &unlocked{Key: key}
	}
	am.unlocked[addr] = u
	return nil
}

func (am *Manager) expire(addr common.Address, u *unlocked, timeout time.Duration) {
	t := time.NewTimer(timeout)
	defer t.Stop()
	select {
	case <-u.abort:
		// just quit
	case <-t.C:
		am.mutex.Lock()
		// only drop if it's still the same key instance that dropLater
		// was launched with. we can check that using pointer equality
		// because the map stores a new pointer every time the key is
		// unlocked.
		if am.unlocked[addr] == u {
			zeroKey(u.PrivateKey)
			delete(am.unlocked, addr)
		}
		am.mutex.Unlock()
	}
}

func (am *Manager) NewAccount(auth string) (Account, error) {
	key, err := am.keyStore.GenerateNewKey(crand.Reader, auth)
	if err != nil {
		return Account{}, err
	}
	return Account{Address: key.Address}, nil
}

func (am *Manager) AddressByIndex(index int) (addr string, err error) {
	var addrs []common.Address
	addrs, err = am.keyStore.GetKeyAddresses()
	if err != nil {
		return
	}
	if index < 0 || index >= len(addrs) {
		err = fmt.Errorf("index out of range: %d (should be 0-%d)", index, len(addrs)-1)
	} else {
		addr = addrs[index].Hex()
	}
	return
}

func (am *Manager) Accounts() ([]Account, error) {
	addresses, err := am.keyStore.GetKeyAddresses()
	if os.IsNotExist(err) {
		return nil, ErrNoKeys
	} else if err != nil {
		return nil, err
	}
	accounts := make([]Account, len(addresses))
	for i, addr := range addresses {
		accounts[i] = Account{
			Address: addr,
		}
	}
	return accounts, err
}

// zeroKey zeroes a private key in memory.
func zeroKey(k *ecdsa.PrivateKey) {
	b := k.D.Bits()
	for i := range b {
		b[i] = 0
	}
}

// USE WITH CAUTION = this will save an unencrypted private key on disk
// no cli or js interface
func (am *Manager) Export(path string, addr common.Address, keyAuth string) error {
	key, err := am.keyStore.GetKey(addr, keyAuth)
	if err != nil {
		return err
	}
	return crypto.SaveECDSA(path, key.PrivateKey)
}

func (am *Manager) Import(path string, keyAuth string) (Account, error) {
	privateKeyECDSA, err := crypto.LoadECDSA(path)
	if err != nil {
		return Account{}, err
	}
	key := crypto.NewKeyFromECDSA(privateKeyECDSA)
	if err = am.keyStore.StoreKey(key, keyAuth); err != nil {
		return Account{}, err
	}
	return Account{Address: key.Address}, nil
}

func (am *Manager) Update(addr common.Address, authFrom, authTo string) (err error) {
	var key *crypto.Key
	key, err = am.keyStore.GetKey(addr, authFrom)

	if err == nil {
		err = am.keyStore.StoreKey(key, authTo)
		if err == nil {
			am.keyStore.Cleanup(addr)
		}
	}
	return
}

func (am *Manager) ImportPreSaleKey(keyJSON []byte, password string) (acc Account, err error) {
	var key *crypto.Key
	key, err = crypto.ImportPreSaleKey(am.keyStore, keyJSON, password)
	if err != nil {
		return
	}
	return Account{Address: key.Address}, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"sort"

	"github.com/nsf/termbox-go"
)

type Result struct {
	contents    string
	highlighted map[int]bool
	marked      bool
	score       int
}

func (res Result) Draw(x, y, w int, selected bool) {
	const coldef = termbox.ColorDefault

	color := coldef
	if selected {
		color = coldef | termbox.AttrReverse
		// color = termbox.ColorRed | termbox.AttrReverse
	}

	line := ""
	line += fmt.Sprintf("%4d ", res.score)
	if res.marked {
		line += "*"
	} else {
		line += " "
	}
	tbprint(x, y, color, color, line)
	x += len(line)

	for idx, c := range res.contents {
		fg := color
		if res.highlighted[idx] {
			fg = termbox.ColorGreen | termbox.ColorDefault | termbox.AttrBold
		}
		termbox.SetCell(x, y, c, fg, color)
		x++
	}

	// c := termbox.Cell{Ch: ' ', Fg: color, Bg: color}
	// fill(x, y, w, 1, c)
	tclear(x, y, w, 1)
}

type ResultCollection []*Result

func (rc ResultCollection) Len() int {
	return len(rc)
}
func (rc ResultCollection) Swap(i, j int) {
	rc[i], rc[j] = rc[j], rc[i]
}
func (rc ResultCollection) Less(i, j int) bool {
	return rc[i].score > rc[j].score
}

type Results struct {
	// Array of results to be filtered
	allresults ResultCollection
	results    ResultCollection

	// Current user input
	lastuserinput string

	// Visible result lines
	top_result    int
	bottom_result int

	// Total number of results
	result_count int

	// Index of currently selected line
	result_selected int

	// View size
	x, y, h, w int
}

func (r *Results) SelectFirst() {
	r.result_selected = 0
	r.top_result = 0

	if r.result_count > r.h {
		r.bottom_result = r.h
	} else {
		r.bottom_result = r.result_count
	}
}

func (r *Results) SelectPrevious() {
	if r.result_selected > 0 {
		r.result_selected--
	}
	if r.top_result > 0 {
		r.top_result--
		r.bottom_result--
	}
}

func (r *Results) SelectNext() {
	if r.result_selected < (r.result_count - 1) {
		r.result_selected++

		if r.result_selected >= r.bottom_result {
			r.top_result++
			r.bottom_result++
		}
	}
}

func (r *Results) Insert(s string) {
	result := new(Result)
	result.contents = s
	r.allresults = append(r.allresults, result)
	r.result_count++
}

func (r *Results) Queue(s string) {
}

func tbprint(x, y int, fg, bg termbox.Attribute, msg string) {
	for _, c := range msg {
		termbox.SetCell(x, y, c, fg, bg)
		x++
	}
}

func (r *Results) Draw() {

	tclear(r.x, r.y, r.w, r.h)

	cy := r.y

	for cnt, res := range r.results[r.top_result:r.bottom_result] {
		is_selected := (cnt + r.top_result) == r.result_selected
		res.Draw(r.x, cy, r.w, is_selected)
		cy++
	}
}

func (r *Results) ToggleMark() {
	if r.result_count > 0 {
		r.results[r.result_selected].marked = !r.results[r.result_selected].marked
		r.SelectNext()
	}
}

func (r *Results) ToggleMarkAll() {
	for _, res := range r.results {
		res.marked = !res.marked
	}
}

func (r *Results) SetSize(x, y, w, h int) {
	r.x, r.y, r.w, r.h = x, y, w, h

	r.top_result = 0
	if r.result_count > r.h {
		r.bottom_result = r.h
	} else {
		r.bottom_result = r.result_count
	}
}

func (r *Results) CopyAll() {
	r.results = r.allresults
}

func (r *Results) Filter(userinput string) {
	if len(userinput) == 0 {
		r.results = r.allresults
		r.result_count = len(r.allresults)

		for _, res := range r.results {
			res.highlighted = nil
		}

		r.SelectFirst()
		return
	}

	initialset := r.allresults

	// Optimization
	// Now invalid because results are changing...
	// if len(r.lastuserinput) > 0 && strings.HasPrefix(userinput, r.lastuserinput) {
	// 	initialset = r.results
	// 	if len(r.results) == 0 {
	// 		r.result_count = 0
	// 		r.SelectFirst()
	// 		return
	// 	}
	// }
	// r.lastuserinput = userinput

	r.results = make([]*Result, 0, 100)
	r.result_count = 0

	// Filter
	for _, res := range initialset {
		best := score2(res.contents, userinput)
		res.score, res.highlighted = best.score, best.highlight
		if res.score > 0 {
			r.results = append(r.results, res)
			r.result_count++
		}
	}

	// Sort
	sort.Sort(r.results)

	// TODO: better cursor behaviouree
	r.SelectFirst()
}
<commit_msg>Stupid simple lower case matching<commit_after>package main

import (
	"fmt"
	"sort"
	"strings"

	"github.com/nsf/termbox-go"
)

type Result struct {
	contents        string       // user input is matched against lowercase version
	displayContents string       // original filename (or line) to display
	highlighted     map[int]bool // hashmap of the characters to be highlighted
	marked          bool         // true when the current line is selected
	score           int          // what is the score for this particular result?
}

func (res Result) Draw(x, y, w int, selected bool) {
	const coldef = termbox.ColorDefault

	color := coldef
	if selected {
		color = coldef | termbox.AttrReverse
	}

	line := ""
	line += fmt.Sprintf("%4d ", res.score)
	if res.marked {
		line += "*"
	} else {
		line += " "
	}
	tbprint(x, y, color, color, line)
	x += len(line)

	for idx, c := range res.displayContents {
		fg := color
		if res.highlighted[idx] {
			fg = termbox.ColorGreen | termbox.ColorDefault | termbox.AttrBold
		}
		termbox.SetCell(x, y, c, fg, color)
		x++
	}

	tclear(x, y, w, 1)
}

type ResultCollection []*Result

func (rc ResultCollection) Len() int {
	return len(rc)
}
func (rc ResultCollection) Swap(i, j int) {
	rc[i], rc[j] = rc[j], rc[i]
}
func (rc ResultCollection) Less(i, j int) bool {
	return rc[i].score > rc[j].score
}

type Results struct {
	// Array of results to be filtered
	allresults ResultCollection
	results    ResultCollection

	// Current user input
	lastuserinput string

	// Visible result lines
	top_result    int
	bottom_result int

	// Total number of results
	result_count int

	// Index of currently selected line
	result_selected int

	// View size
	x, y, h, w int
}

func (r *Results) SelectFirst() {
	r.result_selected = 0
	r.top_result = 0

	if r.result_count > r.h {
		r.bottom_result = r.h
	} else {
		r.bottom_result = r.result_count
	}
}

func (r *Results) SelectPrevious() {
	if r.result_selected > 0 {
		r.result_selected--
	}
	if r.top_result > 0 {
		r.top_result--
		r.bottom_result--
	}
}

func (r *Results) SelectNext() {
	if r.result_selected < (r.result_count - 1) {
		r.result_selected++

		if r.result_selected >= r.bottom_result {
			r.top_result++
			r.bottom_result++
		}
	}
}

func (r *Results) Insert(s string) {
	result := new(Result)
	result.contents = strings.ToLower(s)
	result.displayContents = s
	r.allresults = append(r.allresults, result)
	r.result_count++
}

func (r *Results) Queue(s string) {
}

func tbprint(x, y int, fg, bg termbox.Attribute, msg string) {
	for _, c := range msg {
		termbox.SetCell(x, y, c, fg, bg)
		x++
	}
}

func (r *Results) Draw() {

	tclear(r.x, r.y, r.w, r.h)

	cy := r.y

	for cnt, res := range r.results[r.top_result:r.bottom_result] {
		is_selected := (cnt + r.top_result) == r.result_selected
		res.Draw(r.x, cy, r.w, is_selected)
		cy++
	}
}

func (r *Results) ToggleMark() {
	if r.result_count > 0 {
		r.results[r.result_selected].marked = !r.results[r.result_selected].marked
		r.SelectNext()
	}
}

func (r *Results) ToggleMarkAll() {
	for _, res := range r.results {
		res.marked = !res.marked
	}
}

func (r *Results) SetSize(x, y, w, h int) {
	r.x, r.y, r.w, r.h = x, y, w, h

	r.top_result = 0
	if r.result_count > r.h {
		r.bottom_result = r.h
	} else {
		r.bottom_result = r.result_count
	}
}

func (r *Results) CopyAll() {
	r.results = r.allresults
}

func (r *Results) Filter(userinput string) {
	if len(userinput) == 0 {
		r.results = r.allresults
		r.result_count = len(r.allresults)

		for _, res := range r.results {
			res.highlighted = nil
		}

		r.SelectFirst()
		return
	}

	initialset := r.allresults

	// Optimization
	// Now invalid because results are changing...
	// if len(r.lastuserinput) > 0 && strings.HasPrefix(userinput, r.lastuserinput) {
	// 	initialset = r.results
	// 	if len(r.results) == 0 {
	// 		r.result_count = 0
	// 		r.SelectFirst()
	// 		return
	// 	}
	// }
	// r.lastuserinput = userinput

	r.results = make([]*Result, 0, 100)
	r.result_count = 0

	// Filter
	for _, res := range initialset {
		best := score2(res.contents, userinput)
		res.score, res.highlighted = best.score, best.highlight
		if res.score > 0 {
			r.results = append(r.results, res)
			r.result_count++
		}
	}

	// Sort
	sort.Sort(r.results)

	// TODO: better cursor behaviouree
	r.SelectFirst()
}
<|endoftext|>


<commit_before>// Copyright © 2015 Alexey Khalyapin - halyapin@gmail.com
//
// This program or package and any associated files are licensed under the
// GNU GENERAL PUBLIC LICENSE Version 2 (the "License"); you may not use these files
// except in compliance with the License. You can get a copy of the License
// at: http://www.gnu.org/licenses/gpl-2.0.html

package reverse

import (
	"errors"
	"fmt"
	"strings"
)

var Urls *urlStore

func init() {
	Urls = &urlStore{store: make(map[string]url)}
}

// Adds url to store
func Add(urlName, urlAddr string, params ...string) string {
	return Urls.MustAdd(urlName, urlAddr, params...)
}

func AddGr(urlName, group, urlAddr string, params ...string) string {
	return Urls.MustAddGr(urlName, group, urlAddr, params...)
}

// Reverse url by name
func Rev(urlName string, params ...string) string {
	return Urls.MustReverse(urlName, params...)
}

// Gets raw url by name
func Get(urlName string) string {
	return Urls.Get(urlName)
}

type url struct {
	url    string
	params []string
}

type urlStore struct {
	store map[string]url
}

// Adds a Url to the Store
func (us *urlStore) Add(urlName, urlAddr string, params ...string) (string, error) {
	return us.AddGr(urlName, "", urlAddr, params...)
}

// Adds a Url and panics if error
func (us urlStore) MustAdd(urlName, urlAddr string, params ...string) string {
	addr, err := us.Add(urlName, urlAddr, params...)
	if err != nil {
		panic(err)
	}
	return addr
}

// Adds with group refix
func (us *urlStore) AddGr(urlName, group, urlAddr string, params ...string) (string, error) {
	if _, ok := us.store[urlName]; ok {
		return "", errors.New("Url already exists. Try to use .Get() method.")
	}

	tmpUrl := url{group + urlAddr, params}
	us.store[urlName] = tmpUrl
	return urlAddr, nil
}

// Adds a Url with group prefix
func (us urlStore) MustAddGr(urlName, group, urlAddr string, params ...string) string {
	addr, err := us.AddGr(urlName, group, urlAddr, params...)
	if err != nil {
		panic(err)
	}
	return addr
}

// Gets raw url string
func (us urlStore) Get(urlName string) string {
	return us.store[urlName].url
}

// Gets reversed url
func (us urlStore) Reverse(urlName string, params ...string) (string, error) {
	if len(params) != len(us.store[urlName].params) {
		return "", errors.New("Bad Url Reverse: mismatch params")
	}
	res := us.store[urlName].url
	for i, val := range params {
		res = strings.Replace(res, us.store[urlName].params[i], val, 1)
	}
	return res, nil
}

// Gets reversed url and panics if error
func (us urlStore) MustReverse(urlName string, params ...string) string {
	res, err := us.Reverse(urlName, params...)
	if err != nil {
		panic(err)
	}
	return res
}

func (us urlStore) Sting() string {
	return fmt.Sprint(us.store)
}

// For testing
func (us urlStore) getParam(urlName string, num int) string {
	return us.store[urlName].params[num]
}

<commit_msg>add shortcut<commit_after>// Copyright © 2015 Alexey Khalyapin - halyapin@gmail.com
//
// This program or package and any associated files are licensed under the
// GNU GENERAL PUBLIC LICENSE Version 2 (the "License"); you may not use these files
// except in compliance with the License. You can get a copy of the License
// at: http://www.gnu.org/licenses/gpl-2.0.html

package reverse

import (
	"errors"
	"fmt"
	"strings"
)

var Urls *urlStore

func init() {
	Urls = &urlStore{store: make(map[string]url)}
}

// Adds url to store
func Add(urlName, urlAddr string, params ...string) string {
	return Urls.MustAdd(urlName, urlAddr, params...)
}

func AddGr(urlName, group, urlAddr string, params ...string) string {
	return Urls.MustAddGr(urlName, group, urlAddr, params...)
}

// Reverse url by name
func Rev(urlName string, params ...string) string {
	return Urls.MustReverse(urlName, params...)
}

// Gets raw url by name
func Get(urlName string) string {
	return Urls.Get(urlName)
}

type url struct {
	url    string
	params []string
}

type urlStore struct {
	store map[string]url
}

// Adds a Url to the Store
func (us *urlStore) Add(urlName, urlAddr string, params ...string) (string, error) {
	return us.AddGr(urlName, "", urlAddr, params...)
}

// Adds a Url and panics if error
func (us urlStore) MustAdd(urlName, urlAddr string, params ...string) string {
	addr, err := us.Add(urlName, urlAddr, params...)
	if err != nil {
		panic(err)
	}
	return addr
}

// Adds with group refix
func (us *urlStore) AddGr(urlName, group, urlAddr string, params ...string) (string, error) {
	if _, ok := us.store[urlName]; ok {
		return "", errors.New("Url already exists. Try to use .Get() method.")
	}

	tmpUrl := url{group + urlAddr, params}
	us.store[urlName] = tmpUrl
	return urlAddr, nil
}

// Adds a Url with group prefix
func (us urlStore) MustAddGr(urlName, group, urlAddr string, params ...string) string {
	addr, err := us.AddGr(urlName, group, urlAddr, params...)
	if err != nil {
		panic(err)
	}
	return addr
}

// Gets raw url string
func (us urlStore) Get(urlName string) string {
	return us.store[urlName].url
}

// Gets reversed url
func (us urlStore) Reverse(urlName string, params ...string) (string, error) {
	if len(params) != len(us.store[urlName].params) {
		return "", errors.New("Bad Url Reverse: mismatch params")
	}
	res := us.store[urlName].url
	for i, val := range params {
		res = strings.Replace(res, us.store[urlName].params[i], val, 1)
	}
	return res, nil
}

// Gets reversed url and panics if error
func (us urlStore) MustReverse(urlName string, params ...string) string {
	res, err := us.Reverse(urlName, params...)
	if err != nil {
		panic(err)
	}
	return res
}

func (us urlStore) Rev(urlName string, params ...string) string {
	return us.MustReverse(urlName, params...)
}

func (us urlStore) Sting() string {
	return fmt.Sprint(us.store)
}

// For testing
func (us urlStore) getParam(urlName string, num int) string {
	return us.store[urlName].params[num]
}

<|endoftext|>


<commit_before>package web

import (
	"regexp"
	"sync"
	"sync/atomic"

	ws "github.com/henrylee2cn/pholcus/common/websocket"
	"github.com/henrylee2cn/pholcus/logs"
)

// send log api
func wsLogHandle(conn *ws.Conn) {
	defer func() {
		if p := recover(); p != nil {
			logs.Log.Error("%v", p)
		}
	}()
	// var err error
	sess, _ := globalSessions.SessionStart(nil, conn.Request())
	sessID := sess.SessionID()
	connPool := Lsc.connPool.Load().(map[string]*ws.Conn)
	if connPool[sessID] == nil {
		Lsc.Add(sessID, conn)
	}
	defer func() {
		Lsc.Remove(sessID)
	}()
	for {
		if err := ws.JSON.Receive(conn, nil); err != nil {
			return
		}
	}
}

type LogSocketController struct {
	connPool atomic.Value
	lock     sync.Mutex
}

var (
	// Lsc log set
	Lsc = func() *LogSocketController {
		l := new(LogSocketController)
		l.connPool.Store(make(map[string]*ws.Conn))
		return l
	}()
	colorRegexp = regexp.MustCompile("\033\\[[0-9]{1,2}m")
)

func (self *LogSocketController) Write(p []byte) (int, error) {
	defer func() {
		recover()
	}()
	p = colorRegexp.ReplaceAll(p, []byte{})
	connPool := self.connPool.Load().(map[string]*ws.Conn)
	for sessID, conn := range connPool {
		_, err := ws.Message.Send(conn, (string(p) + "\r\n"))
		if err != nil {
			self.Remove(sessID)
		}
	}
	return len(p), nil
}

func (self *LogSocketController) Add(sessID string, conn *ws.Conn) {
	self.lock.Lock()
	defer self.lock.Unlock()

	connPool := self.connPool.Load().(map[string]*ws.Conn)
	newConnPool := make(map[string]*ws.Conn, len(connPool)+1)
	for k, v := range connPool {
		newConnPool[k] = v
	}
	newConnPool[sessID] = conn
	self.connPool.Store(newConnPool)
}

func (self *LogSocketController) Remove(sessID string) {
	self.lock.Lock()
	defer self.lock.Unlock()
	defer func() {
		recover()
	}()
	connPool := self.connPool.Load().(map[string]*ws.Conn)
	conn := connPool[sessID]
	if conn == nil {
		return
	}
	conn.Close()
	newConnPool := make(map[string]*ws.Conn, len(connPool)+1)
	for k, v := range connPool {
		if k != sessID {
			newConnPool[k] = v
		}
	}
	self.connPool.Store(newConnPool)
}
<commit_msg>web UI: Filtering log color characters<commit_after>package web

import (
	"regexp"
	"sync"
	"sync/atomic"

	ws "github.com/henrylee2cn/pholcus/common/websocket"
	"github.com/henrylee2cn/pholcus/logs"
)

// send log api
func wsLogHandle(conn *ws.Conn) {
	defer func() {
		if p := recover(); p != nil {
			logs.Log.Error("%v", p)
		}
	}()
	// var err error
	sess, _ := globalSessions.SessionStart(nil, conn.Request())
	sessID := sess.SessionID()
	connPool := Lsc.connPool.Load().(map[string]*ws.Conn)
	if connPool[sessID] == nil {
		Lsc.Add(sessID, conn)
	}
	defer func() {
		Lsc.Remove(sessID)
	}()
	for {
		if err := ws.JSON.Receive(conn, nil); err != nil {
			return
		}
	}
}

type LogSocketController struct {
	connPool atomic.Value
	lock     sync.Mutex
}

var (
	// Lsc log set
	Lsc = func() *LogSocketController {
		l := new(LogSocketController)
		l.connPool.Store(make(map[string]*ws.Conn))
		return l
	}()
	colorRegexp = regexp.MustCompile("\033\\[[0-9;]{1,4}m")
)

func (self *LogSocketController) Write(p []byte) (int, error) {
	defer func() {
		recover()
	}()
	p = colorRegexp.ReplaceAll(p, []byte{})
	connPool := self.connPool.Load().(map[string]*ws.Conn)
	for sessID, conn := range connPool {
		_, err := ws.Message.Send(conn, (string(p) + "\r\n"))
		if err != nil {
			self.Remove(sessID)
		}
	}
	return len(p), nil
}

func (self *LogSocketController) Add(sessID string, conn *ws.Conn) {
	self.lock.Lock()
	defer self.lock.Unlock()

	connPool := self.connPool.Load().(map[string]*ws.Conn)
	newConnPool := make(map[string]*ws.Conn, len(connPool)+1)
	for k, v := range connPool {
		newConnPool[k] = v
	}
	newConnPool[sessID] = conn
	self.connPool.Store(newConnPool)
}

func (self *LogSocketController) Remove(sessID string) {
	self.lock.Lock()
	defer self.lock.Unlock()
	defer func() {
		recover()
	}()
	connPool := self.connPool.Load().(map[string]*ws.Conn)
	conn := connPool[sessID]
	if conn == nil {
		return
	}
	conn.Close()
	newConnPool := make(map[string]*ws.Conn, len(connPool)+1)
	for k, v := range connPool {
		if k != sessID {
			newConnPool[k] = v
		}
	}
	self.connPool.Store(newConnPool)
}
<|endoftext|>


<commit_before>package anomalyzer

import (
	"fmt"
	"github.com/drewlanenga/govector"
	"math"
	"sort"
)

type Algorithm func(govector.Vector, AnomalyzerConf) float64

var (
	Algorithms = map[string]Algorithm{
		"magnitude": MagnitudeTest,
		"diff":      DiffTest,
		"rank":      RankTest,
		"cdf":       CDFTest,
		"fence":     FenceTest,
		"ks":        BootstrapKSTest,
	}
)

// Identity function
func identity(anything interface{}) interface{} {
	return anything
}

// Returns a value within a given window (xmin and xmax).
func cap(x, min, max float64) float64 {
	return math.Max(math.Min(x, max), min)
}

// Returns a contant
func constant(x float64) float64 {
	return 0.2
}

// Return integer math comparisons
func max(x, y int) int {
	if x > y {
		return x
	}
	return y
}

func min(x, y int) int {
	if y < x {
		return y
	}
	return x
}

// Return a vector slice for the active window and reference window.
// Some tests require different minimum thresholds for sizes of reference windows.
// This can be specified in the minRefSize parameter. If size isn't important, use -1
func extractWindows(vector govector.Vector, refSize, activeSize, minRefSize int) (govector.Vector, govector.Vector, error) {
	n := len(vector)
	activeSize = min(activeSize, n)
	refSize = min(refSize, n-activeSize)

	// make sure the reference size is at least as big as the active size
	// note that this penalty might be overly severe for some tests
	if refSize < minRefSize {
		return nil, nil, fmt.Errorf("Reference size must be at least as big as active size")
	}

	// return reference and active windows
	return vector[n-activeSize-refSize : n-activeSize], vector[n-activeSize:], nil
}

// This function can be used to test whether or not data is getting close to a
// specified upper or lower bound.
func FenceTest(vector govector.Vector, conf AnomalyzerConf) float64 {
	// we don't really care about a reference window for this one
	_, active, _ := extractWindows(vector, conf.referenceSize, conf.ActiveSize, -1)

	x := active.Mean()

	distance := 0.0
	if conf.LowerBound == NA {
		// we only care about distance from the upper bound
		distance = x / conf.UpperBound
	} else {
		// we care about both bounds, so measure distance
		// from midpoint

		bound := (conf.UpperBound - conf.LowerBound) / 2
		mid := conf.LowerBound + bound

		distance = (math.Abs(x - mid)) / bound
	}
	return weightExp(cap(distance, 0, 1), 10)
}

// This is a function will sharply scale values between 0 and 1 such that
// smaller values are weighted more towards 0. A larger base value means a
// more horshoe type function.
func weightExp(x, base float64) float64 {
	return (math.Pow(base, x) - 1) / (math.Pow(base, 1) - 1)
}

// Generates permutations of reference and active window values to determine
// whether or not data is anomalous. The number of permutations desired has
// been set to 500 but can be increased for more precision.
func DiffTest(vector govector.Vector, conf AnomalyzerConf) float64 {
	// Find the differences between neighboring elements and rank those differences.
	ranks := vector.Diff().Apply(math.Abs).Rank()

	// The indexing runs to length-1 because after applying .Diff(), We have
	// decreased the length of out vector by 1.
	_, active, err := extractWindows(ranks, conf.referenceSize-1, conf.ActiveSize, conf.ActiveSize)
	if err != nil {
		return NA
	}

	// Consider the sum of the ranks across the active data. This is the sum that
	// we will compare our permutations to.
	activeSum := active.Sum()

	i := 0
	significant := 0

	// Permute the active and reference data and compute the sums across the tail
	// (from the length of the reference data to the full length).
	for i < conf.PermCount {
		permRanks := vector.Shuffle().Diff().Apply(math.Abs).Rank()
		_, permActive, _ := extractWindows(permRanks, conf.referenceSize-1, conf.ActiveSize, conf.ActiveSize)

		// If we find a sum that is less than the initial sum across the active data,
		// this implies our initial sum might be uncharacteristically high. We increment
		// our count.
		if permActive.Sum() < activeSum {
			significant++
		}
		i++
	}
	// We return the percentage of the number of iterations where we found our initial
	// sum to be high.
	return float64(significant) / float64(conf.PermCount)
}

// Very similar to the above.
func RankTest(vector govector.Vector, conf AnomalyzerConf) float64 {
	// Find the differences between neighboring elements and rank those differences.
	ranks := vector.Rank()

	_, active, err := extractWindows(ranks, conf.referenceSize, conf.ActiveSize, conf.ActiveSize)
	if err != nil {
		return NA
	}

	// Consider the sum of the ranks across the active data. This is the sum that
	// we will compare our permutations to.
	activeSum := active.Sum()

	i := 0
	significant := 0

	// Permute the active and reference data and compute the sums across the tail
	// (from the length of the reference data to the full length).
	for i < conf.PermCount {
		permRanks := vector.Shuffle().Rank()
		_, permActive, _ := extractWindows(permRanks, conf.referenceSize, conf.ActiveSize, conf.ActiveSize)

		// If we find a sum that is less than the initial sum across the active data,
		// this implies our initial sum might be uncharacteristically high. We increment
		// our count.
		if permActive.Sum() < activeSum {
			significant++
		}
		i++
	}
	// We return the percentage of the number of iterations where we found our initial
	// sum to be high.
	return float64(significant) / float64(conf.PermCount)
}

// Generates the cumulative distribution function using the difference in the means
// for the data.
func CDFTest(vector govector.Vector, conf AnomalyzerConf) float64 {
	diffs := vector.Diff().Apply(math.Abs)
	reference, active, err := extractWindows(diffs, conf.referenceSize-1, conf.ActiveSize, conf.ActiveSize)
	if err != nil {
		return NA
	}

	// Find the empircal distribution function using the reference window.
	refEcdf := reference.Ecdf()

	// Difference between the active and reference means.
	activeDiff := active.Mean() - reference.Mean()

	// Apply the empirical distribution function to that difference.
	percentile := refEcdf(activeDiff)

	// Scale so max probability is in tails and prob at 0.5 is 0.
	return (2 * math.Abs(0.5-percentile))
}

// Generates the percent difference between the means of the reference and active
// data. Returns a value scaled such that it lies between 0 and 1.
func MagnitudeTest(vector govector.Vector, conf AnomalyzerConf) float64 {
	reference, active, err := extractWindows(vector, conf.referenceSize, conf.ActiveSize, 1)
	if err != nil {
		return NA
	}

	activeMean := active.Mean()
	refMean := reference.Mean()

	// If the baseline is 0, then the magnitude should be Inf, but we'll
	// round to 1.
	if refMean == 0 {
		return 1
	}

	pdiff := math.Abs(activeMean-refMean) / refMean
	return weightExp(pdiff, 10)
}

// Calculate a Kolmogorov-Smirnov test statistic.
func KSTest(vector govector.Vector, conf AnomalyzerConf) float64 {
	reference, active, err := extractWindows(vector, conf.referenceSize, conf.ActiveSize, conf.ActiveSize)
	if err != nil {
		return NA
	}

	n1 := len(reference)
	n2 := len(active)

	// First sort the active data and generate a cummulative distribution function
	// using that data. Do the same for the reference data.
	sort.Sort(active)
	activeEcdf := active.Ecdf()
	sort.Sort(reference)
	refEcdf := reference.Ecdf()

	// We want the reference and active vectors to have the same length n, so we
	// consider the min and max for each and interpolated the points between.
	min := math.Min(reference[0], active[0])
	max := math.Max(reference[n1-1], active[n2-1])

	interpolated := interpolate(min, max, n1+n2)

	// Then we apply the distribution function over the interpolated data.
	activeDist := interpolated.Apply(activeEcdf)
	refDist := interpolated.Apply(refEcdf)

	// Find the maximum displacement between both distributions. Use this value
	// to calculate the KS test score.
	d := 0.0
	for i := 0; i < n1+n2; i++ {
		d = math.Max(d, math.Abs(activeDist[i]-refDist[i]))
	}

	return d
}

func BootstrapKSTest(vector govector.Vector, conf AnomalyzerConf) float64 {
	dist := KSTest(vector, conf)
	if dist == NA {
		return NA
	}

	i := 0
	significant := 0

	for i < conf.PermCount {
		permVector := vector.Shuffle()
		permDist := KSTest(permVector, conf)

		if permDist < dist {
			significant++
		}
		i++
	}
	return float64(significant) / float64(conf.PermCount)
}

// A helper function for KS that rescales a vector to the desired length npoints.
func interpolate(min, max float64, npoints int) govector.Vector {
	interp := make(govector.Vector, npoints)

	step := (max - min) / (float64(npoints) - 1)
	interp[0] = min
	i := 1
	for i < npoints {
		interp[i] = interp[i-1] + step
		i++
	}
	return interp
}
<commit_msg>rename KSTest to KsStat<commit_after>package anomalyzer

import (
	"fmt"
	"github.com/drewlanenga/govector"
	"math"
	"sort"
)

type Algorithm func(govector.Vector, AnomalyzerConf) float64

var (
	Algorithms = map[string]Algorithm{
		"magnitude": MagnitudeTest,
		"diff":      DiffTest,
		"rank":      RankTest,
		"cdf":       CDFTest,
		"fence":     FenceTest,
		"ks":        BootstrapKsTest,
	}
)

// Identity function
func identity(anything interface{}) interface{} {
	return anything
}

// Returns a value within a given window (xmin and xmax).
func cap(x, min, max float64) float64 {
	return math.Max(math.Min(x, max), min)
}

// Returns a contant
func constant(x float64) float64 {
	return 0.2
}

// Return integer math comparisons
func max(x, y int) int {
	if x > y {
		return x
	}
	return y
}

func min(x, y int) int {
	if y < x {
		return y
	}
	return x
}

// Return a vector slice for the active window and reference window.
// Some tests require different minimum thresholds for sizes of reference windows.
// This can be specified in the minRefSize parameter. If size isn't important, use -1
func extractWindows(vector govector.Vector, refSize, activeSize, minRefSize int) (govector.Vector, govector.Vector, error) {
	n := len(vector)
	activeSize = min(activeSize, n)
	refSize = min(refSize, n-activeSize)

	// make sure the reference size is at least as big as the active size
	// note that this penalty might be overly severe for some tests
	if refSize < minRefSize {
		return nil, nil, fmt.Errorf("Reference size must be at least as big as active size")
	}

	// return reference and active windows
	return vector[n-activeSize-refSize : n-activeSize], vector[n-activeSize:], nil
}

// This function can be used to test whether or not data is getting close to a
// specified upper or lower bound.
func FenceTest(vector govector.Vector, conf AnomalyzerConf) float64 {
	// we don't really care about a reference window for this one
	_, active, _ := extractWindows(vector, conf.referenceSize, conf.ActiveSize, -1)

	x := active.Mean()

	distance := 0.0
	if conf.LowerBound == NA {
		// we only care about distance from the upper bound
		distance = x / conf.UpperBound
	} else {
		// we care about both bounds, so measure distance
		// from midpoint

		bound := (conf.UpperBound - conf.LowerBound) / 2
		mid := conf.LowerBound + bound

		distance = (math.Abs(x - mid)) / bound
	}
	return weightExp(cap(distance, 0, 1), 10)
}

// This is a function will sharply scale values between 0 and 1 such that
// smaller values are weighted more towards 0. A larger base value means a
// more horshoe type function.
func weightExp(x, base float64) float64 {
	return (math.Pow(base, x) - 1) / (math.Pow(base, 1) - 1)
}

// Generates permutations of reference and active window values to determine
// whether or not data is anomalous. The number of permutations desired has
// been set to 500 but can be increased for more precision.
func DiffTest(vector govector.Vector, conf AnomalyzerConf) float64 {
	// Find the differences between neighboring elements and rank those differences.
	ranks := vector.Diff().Apply(math.Abs).Rank()

	// The indexing runs to length-1 because after applying .Diff(), We have
	// decreased the length of out vector by 1.
	_, active, err := extractWindows(ranks, conf.referenceSize-1, conf.ActiveSize, conf.ActiveSize)
	if err != nil {
		return NA
	}

	// Consider the sum of the ranks across the active data. This is the sum that
	// we will compare our permutations to.
	activeSum := active.Sum()

	i := 0
	significant := 0

	// Permute the active and reference data and compute the sums across the tail
	// (from the length of the reference data to the full length).
	for i < conf.PermCount {
		permRanks := vector.Shuffle().Diff().Apply(math.Abs).Rank()
		_, permActive, _ := extractWindows(permRanks, conf.referenceSize-1, conf.ActiveSize, conf.ActiveSize)

		// If we find a sum that is less than the initial sum across the active data,
		// this implies our initial sum might be uncharacteristically high. We increment
		// our count.
		if permActive.Sum() < activeSum {
			significant++
		}
		i++
	}
	// We return the percentage of the number of iterations where we found our initial
	// sum to be high.
	return float64(significant) / float64(conf.PermCount)
}

// Very similar to the above.
func RankTest(vector govector.Vector, conf AnomalyzerConf) float64 {
	// Find the differences between neighboring elements and rank those differences.
	ranks := vector.Rank()

	_, active, err := extractWindows(ranks, conf.referenceSize, conf.ActiveSize, conf.ActiveSize)
	if err != nil {
		return NA
	}

	// Consider the sum of the ranks across the active data. This is the sum that
	// we will compare our permutations to.
	activeSum := active.Sum()

	i := 0
	significant := 0

	// Permute the active and reference data and compute the sums across the tail
	// (from the length of the reference data to the full length).
	for i < conf.PermCount {
		permRanks := vector.Shuffle().Rank()
		_, permActive, _ := extractWindows(permRanks, conf.referenceSize, conf.ActiveSize, conf.ActiveSize)

		// If we find a sum that is less than the initial sum across the active data,
		// this implies our initial sum might be uncharacteristically high. We increment
		// our count.
		if permActive.Sum() < activeSum {
			significant++
		}
		i++
	}
	// We return the percentage of the number of iterations where we found our initial
	// sum to be high.
	return float64(significant) / float64(conf.PermCount)
}

// Generates the cumulative distribution function using the difference in the means
// for the data.
func CDFTest(vector govector.Vector, conf AnomalyzerConf) float64 {
	diffs := vector.Diff().Apply(math.Abs)
	reference, active, err := extractWindows(diffs, conf.referenceSize-1, conf.ActiveSize, conf.ActiveSize)
	if err != nil {
		return NA
	}

	// Find the empircal distribution function using the reference window.
	refEcdf := reference.Ecdf()

	// Difference between the active and reference means.
	activeDiff := active.Mean() - reference.Mean()

	// Apply the empirical distribution function to that difference.
	percentile := refEcdf(activeDiff)

	// Scale so max probability is in tails and prob at 0.5 is 0.
	return (2 * math.Abs(0.5-percentile))
}

// Generates the percent difference between the means of the reference and active
// data. Returns a value scaled such that it lies between 0 and 1.
func MagnitudeTest(vector govector.Vector, conf AnomalyzerConf) float64 {
	reference, active, err := extractWindows(vector, conf.referenceSize, conf.ActiveSize, 1)
	if err != nil {
		return NA
	}

	activeMean := active.Mean()
	refMean := reference.Mean()

	// If the baseline is 0, then the magnitude should be Inf, but we'll
	// round to 1.
	if refMean == 0 {
		return 1
	}

	pdiff := math.Abs(activeMean-refMean) / refMean
	return weightExp(pdiff, 10)
}

// Calculate a Kolmogorov-Smirnov test statistic.
func KsStat(vector govector.Vector, conf AnomalyzerConf) float64 {
	reference, active, err := extractWindows(vector, conf.referenceSize, conf.ActiveSize, conf.ActiveSize)
	if err != nil {
		return NA
	}

	n1 := len(reference)
	n2 := len(active)

	// First sort the active data and generate a cummulative distribution function
	// using that data. Do the same for the reference data.
	sort.Sort(active)
	activeEcdf := active.Ecdf()
	sort.Sort(reference)
	refEcdf := reference.Ecdf()

	// We want the reference and active vectors to have the same length n, so we
	// consider the min and max for each and interpolated the points between.
	min := math.Min(reference[0], active[0])
	max := math.Max(reference[n1-1], active[n2-1])

	interpolated := interpolate(min, max, n1+n2)

	// Then we apply the distribution function over the interpolated data.
	activeDist := interpolated.Apply(activeEcdf)
	refDist := interpolated.Apply(refEcdf)

	// Find the maximum displacement between both distributions. Use this value
	// to calculate the KS test score.
	d := 0.0
	for i := 0; i < n1+n2; i++ {
		d = math.Max(d, math.Abs(activeDist[i]-refDist[i]))
	}

	return d
}

func BootstrapKsTest(vector govector.Vector, conf AnomalyzerConf) float64 {
	dist := KsStat(vector, conf)
	if dist == NA {
		return NA
	}

	i := 0
	significant := 0

	for i < conf.PermCount {
		permVector := vector.Shuffle()
		permDist := KsStat(permVector, conf)

		if permDist < dist {
			significant++
		}
		i++
	}
	return float64(significant) / float64(conf.PermCount)
}

// A helper function for KS that rescales a vector to the desired length npoints.
func interpolate(min, max float64, npoints int) govector.Vector {
	interp := make(govector.Vector, npoints)

	step := (max - min) / (float64(npoints) - 1)
	interp[0] = min
	i := 1
	for i < npoints {
		interp[i] = interp[i-1] + step
		i++
	}
	return interp
}
<|endoftext|>


<commit_before>package main

import (
	//"flag"
	"fmt"
	"io/ioutil"
	//"github.com/go-yaml/yaml"
)

func readFile() string {
	fmt.Printf("Readin input.yml!\n")
	data, err := ioutil.ReadFile("input.yml")
	if err != nil {
		panic(err)
	}
	return string(data)
}

func parse(data string) {

}

func filter() {

}

func convert() {

}

func writeFile(data string) {
	err := ioutil.WriteFile("output.yml", []byte(data), 0644)
	if err != nil {
		panic(err)
	}
}

func main() {
	data := readFile()
	fmt.Print(data)

	writeFile(data)
}
<commit_msg>Use utility function for checking errors<commit_after>package main

import (
	//"flag"
	"fmt"
	"io/ioutil"
	//"github.com/go-yaml/yaml"
)

func readFile() string {
	fmt.Printf("Readin input.yml!\n")
	data, err := ioutil.ReadFile("input.yml")
	checkError(err)
	return string(data)
}

func parse(data string) {

}

func filter() {

}

func convert() {

}

func writeFile(data string) {
	err := ioutil.WriteFile("output.yml", []byte(data), 0644)
	checkError(err)
}

func checkError(err error) {
	if err != nil {
		panic(err)
	}
}

func main() {
	data := readFile()
	fmt.Print(data)

	writeFile(data)
}
<|endoftext|>


<commit_before>package main

import (
	//"flag"
	"fmt"
	"io/ioutil"
	//"github.com/go-yaml/yaml"
)

func readFile() string {
	fmt.Printf("Readin input.yml!\n")
	data, err := ioutil.ReadFile("input.yml")
	checkError(err)
	return string(data)
}

func parse(data string) {

}

func filter() {

}

func convert() {

}

func writeFile(data string) {
	err := ioutil.WriteFile("output.yml", []byte(data), 0644)
	checkError(err)
}

func checkError(err error) {
	if err != nil {
		panic(err)
	}
}

func main() {
	data := readFile()
	fmt.Print(data)

	writeFile(data)
}
<commit_msg>Use Printen<commit_after>package main

import (
	//"flag"
	"fmt"
	"io/ioutil"
	//"github.com/go-yaml/yaml"
)

func readFile() string {
	fmt.Println("Reading input.yml!")
	data, err := ioutil.ReadFile("input.yml")
	checkError(err)
	return string(data)
}

func parse(data string) {

}

func filter() {

}

func convert() {

}

func writeFile(data string) {
	err := ioutil.WriteFile("output.yml", []byte(data), 0644)
	checkError(err)
}

func checkError(err error) {
	if err != nil {
		panic(err)
	}
}

func main() {
	data := readFile()
	fmt.Print(data)

	writeFile(data)
}
<|endoftext|>


<commit_before>package network

import (
	"encoding/binary"
	"encoding/json"
	"fmt"
	"math/rand"
	"net"
	"sort"
	"time"

	"github.com/NetSys/di/db"
	"github.com/NetSys/di/join"
	"github.com/NetSys/di/minion/consensus"

	log "github.com/Sirupsen/logrus"
)

const labelDir = "/minion/labels"
const containerDir = "/minion/containers"

// We store rand.Uint32() in a variable so it's easily mocked out by the unit tests.
// Nondeterminism is hard to test.
var rand32 = rand.Uint32

// A directory containers the first and seceond level of a Tree requested from the
// consensus store.
type directory map[string]map[string]string

// wakeChan collapses the various channels these functions wait on into a single channel.
// Multiple redundant pings will be coalesced into a single message.
func wakeChan(conn db.Conn, store consensus.Store) chan struct{} {
	labelWatch := store.Watch(labelDir, 1*time.Second)
	containerWatch := store.Watch(labelDir, 1*time.Second)
	trigg := conn.TriggerTick(30, db.MinionTable, db.ContainerTable, db.LabelTable,
		db.EtcdTable).C

	c := make(chan struct{}, 1)
	go func() {
		for {
			select {
			case <-labelWatch:
			case <-containerWatch:
			case <-trigg:
			}

			select {
			case c <- struct{}{}:
			default: // There's a notification in queue no need for another.
			}
		}
	}()

	return c
}

func readStoreRun(conn db.Conn, store consensus.Store) {
	for range wakeChan(conn, store) {
		labelDir, err := getDirectory(store, labelDir)
		containerDir, err2 := getDirectory(store, containerDir)
		if err2 != nil {
			err = err2
		}

		if err != nil {
			log.WithError(err).Warn("Failed to read from cluster store.")
			continue
		}

		conn.Transact(func(view db.Database) error {
			readContainerTransact(view, containerDir)
			readLabelTransact(view, labelDir)
			return nil
		})
	}
}

func readContainerTransact(view db.Database, dir directory) {
	minions := view.SelectFromMinion(nil)
	worker := len(minions) == 1 && minions[0].Role == db.Worker

	for _, container := range view.SelectFromContainer(nil) {
		container.IP = ""
		var labels []string
		if children, ok := dir[container.SchedID]; ok {
			json.Unmarshal([]byte(children["Labels"]), &labels)

			container.IP = children["IP"]
			ip := net.ParseIP(container.IP).To4()
			if ip != nil {
				container.Mac = fmt.Sprintf("02:00:%02x:%02x:%02x:%02x",
					ip[0], ip[1], ip[2], ip[3])
			}
		}

		if worker {
			// Masters get their labels from the policy, workers from the
			// consensus store.
			container.Labels = labels
		}

		view.Commit(container)
	}
}

func readLabelTransact(view db.Database, dir directory) {
	pairs, dbls, dirKeys := join.Join(view.SelectFromLabel(nil), dir.keys(),
		func(left, right interface{}) int {
			if left.(db.Label).Label == right.(string) {
				return 0
			}
			return 1
		})

	for _, dbl := range dbls {
		view.Remove(dbl.(db.Label))
	}

	for _, key := range dirKeys {
		pairs = append(pairs, join.Pair{L: view.InsertLabel(), R: key})
	}

	for _, pair := range pairs {
		dbl := pair.L.(db.Label)
		dbl.Label = pair.R.(string)
		dbl.IP = dir[dbl.Label]["IP"]
		_, dbl.MultiHost = dir[dbl.Label]["MultiHost"]
		view.Commit(dbl)
	}
}

func writeStoreRun(conn db.Conn, store consensus.Store) {
	for range wakeChan(conn, store) {
		leader := false
		var containers []db.Container
		conn.Transact(func(view db.Database) error {
			etcds := view.SelectFromEtcd(nil)
			leader = len(etcds) == 1 && etcds[0].Leader
			containers = view.SelectFromContainer(nil)
			return nil
		})

		if !leader {
			continue
		}

		if err := writeStoreContainers(store, containers); err != nil {
			log.WithError(err).Warning("Failed to update containers in ETCD")
		}

		writeStoreLabels(store, containers)
	}
}

func writeStoreContainers(store consensus.Store, containers []db.Container) error {
	var ids []string
	for _, container := range containers {
		if container.SchedID != "" {
			ids = append(ids, container.SchedID)
		}
	}

	store.Mkdir(containerDir)
	dir, err := getDirectory(store, containerDir)
	if err != nil {
		return err
	}

	syncDir(store, dir, containerDir, ids)
	syncIPs(store, dir, containerDir, net.IPv4(10, 0, 0, 0))
	syncLabels(store, dir, containerDir, containers)

	return nil
}

func writeStoreLabels(store consensus.Store, containers []db.Container) error {
	store.Mkdir(labelDir)
	dir, err := getDirectory(store, labelDir)
	if err != nil {
		return err
	}

	var ids []string
	for _, c := range containers {
		for _, l := range c.Labels {
			ids = append(ids, l)
		}
	}

	syncDir(store, dir, labelDir, ids)

	// Labels that point to a single container don't need IPs separate from their
	// constituent container IP.  Thus the following code marks those labels, by the
	// absence of the `MultiHost` file in the consensus store, and sets their IP to
	// whatever is found in the container table.

	// Map from each label to the containers that implement it.
	labelMap := map[string][]db.Container{}
	for _, dbc := range containers {
		for _, label := range dbc.Labels {
			labelMap[label] = append(labelMap[label], dbc)
		}
	}

	// Mark labels as MultiHost if they have more than one container.
	for label, dbcs := range labelMap {
		isMH := len(dbcs) > 1
		if _, ok := dir[label]["MultiHost"]; ok == isMH {
			continue
		}

		var err error
		path := fmt.Sprintf("%s/%s/MultiHost", labelDir, label)
		if isMH {
			dir[label]["MultiHost"] = "true"
			err = store.Set(path, "true")
		} else {
			delete(dir[label], "MultiHost")
			err = store.Delete(path)
		}

		if err != nil {
			log.WithFields(log.Fields{
				"label": label,
				"error": err,
			}).Warn("Failed to change MultiHost key.")
		}
	}

	// Set the IPs of the non-MultiHost containers.
	for label, dbcs := range labelMap {
		if len(dbcs) != 1 {
			continue
		}

		dbc := dbcs[0]
		if dbc.IP == "" || dir[label]["IP"] == dbc.IP {
			continue
		}

		path := fmt.Sprintf("%s/%s/IP", labelDir, label)
		if err := store.Set(path, dbc.IP); err != nil {
			log.WithFields(log.Fields{
				"label": label,
				"IP":    dbc.IP,
				"error": err,
			}).Warn("Consensus store failed set label IP.")
		}
	}

	// Remove the non-MultiHost containers from `dir` to avoid confusing syncIPs.
	for label, mp := range dir {
		if _, ok := mp["MultiHost"]; !ok {
			delete(dir, label)
		}
	}

	syncIPs(store, dir, labelDir, net.IPv4(10, 1, 0, 0))
	return nil
}

func syncDir(store consensus.Store, dir directory, path string, idsArg []string) {
	_, dirKeys, ids := join.Join(dir.keys(), idsArg,
		func(left, right interface{}) int {
			if left.(string) != right.(string) {
				return -1
			}
			return 0
		})

	var etcdLog string
	for _, dirKey := range dirKeys {
		id := dirKey.(string)
		keyPath := fmt.Sprintf("%s/%s", path, id)
		err := store.Delete(keyPath)
		if err != nil {
			etcdLog = fmt.Sprintf("Failed to delete %s: %s", keyPath, err)
		}
		delete(dir, id)
	}

	for _, idElem := range ids {
		id := idElem.(string)
		if _, ok := dir[id]; ok {
			continue
		}

		key := fmt.Sprintf("%s/%s", path, id)
		if err := store.Mkdir(key); err != nil {
			etcdLog = fmt.Sprintf("Failed to create dir %s: %s", key, err)
			continue
		}
		dir[id] = map[string]string{}
	}

	// Etcd failure leads to a bunch of useless errors.  Therefore we only log once.
	if etcdLog != "" {
		log.Error(etcdLog)
	}
}

// syncIPs() takes a directory and creates an IP node for every entry that's missing one.
func syncIPs(store consensus.Store, dir directory, path string, prefixIP net.IP) {
	prefix := binary.BigEndian.Uint32(prefixIP.To4())
	mask := uint32(0xffff0000)

	var unassigned []string
	ipSet := map[uint32]struct{}{}
	for k, m := range dir {
		ip := parseIP(m["IP"], prefix, mask)
		if ip != 0 {
			ipSet[ip] = struct{}{}
		} else {
			unassigned = append(unassigned, k)
		}
	}

	// Don't assign the IP of the default gateway
	ipSet[parseIP(gatewayIP, prefix, mask)] = struct{}{}
	var etcdLog string
	for _, k := range unassigned {
		ip32 := randomIP(ipSet, prefix, mask)
		ipPath := fmt.Sprintf("%s/%s/IP", path, k)
		if ip32 == 0 {
			log.Errorf("Failed to allocate IP for %s.", k)
			store.Delete(ipPath)
			delete(dir[k], "IP")
			continue
		}

		b := make([]byte, 4)
		binary.BigEndian.PutUint32(b, ip32)

		ipStr := net.IP(b).String()
		if err := store.Set(ipPath, ipStr); err != nil {
			etcdLog = fmt.Sprintf("Failed to set key %s: %s", ipPath, err)
			continue
		}

		dir[k]["IP"] = ipStr
		ipSet[ip32] = struct{}{}
	}

	// Etcd failure leads to a bunch of useless errors.  Therefore we only log once.
	if etcdLog != "" {
		log.Error(etcdLog)
	}
}

func syncLabels(store consensus.Store, dir directory, path string,
	containers []db.Container) {

	idLabelMap := map[string][]string{}
	for _, container := range containers {
		if container.SchedID != "" {
			idLabelMap[container.SchedID] = container.Labels
		}
	}

	for id, children := range dir {
		labels := idLabelMap[id]
		if labels == nil {
			labels = []string{}
		}
		sort.Sort(sort.StringSlice(labels))

		jsByte, err := json.Marshal(labels)
		if err != nil {
			panic("Not Reached")
		}
		js := string(jsByte)

		if js == children["Labels"] {
			continue
		}

		key := fmt.Sprintf("%s/%s/Labels", path, id)
		if err := store.Set(key, js); err != nil {
			log.WithField("path", path).Error("Failed to set label key.")
			continue
		}
		dir[id]["Labels"] = js
	}
}

func getDirectory(store consensus.Store, path string) (directory, error) {
	tree, err := store.GetTree(path)
	if err != nil {
		return nil, err
	}

	dir := make(directory)
	for _, l1 := range tree.Children {
		childMap := make(map[string]string)
		for _, l2 := range l1.Children {
			childMap[l2.Key] = l2.Value
		}
		dir[l1.Key] = childMap
	}

	return dir, nil
}

func parseIP(ipStr string, prefix, mask uint32) uint32 {
	ip := net.ParseIP(ipStr).To4()
	if ip == nil {
		return 0
	}

	ip32 := binary.BigEndian.Uint32(ip)
	if ip32&mask != prefix {
		return 0
	}

	return ip32
}

// Choose a random IP address in prefix/mask that isn't in 'conflicts'.
// Returns 0 on failure.
func randomIP(conflicts map[uint32]struct{}, prefix, mask uint32) uint32 {
	for i := 0; i < 256; i++ {
		ip32 := (rand32() & ^mask) | (prefix & mask)
		if _, ok := conflicts[ip32]; !ok {
			return ip32
		}
	}

	return 0
}

func (dir directory) keys() []string {
	var keys []string
	for key := range dir {
		keys = append(keys, key)
	}
	return keys
}
<commit_msg>network: Fix warning when reading container and label directories<commit_after>package network

import (
	"encoding/binary"
	"encoding/json"
	"fmt"
	"math/rand"
	"net"
	"sort"
	"time"

	"github.com/NetSys/di/db"
	"github.com/NetSys/di/join"
	"github.com/NetSys/di/minion/consensus"

	log "github.com/Sirupsen/logrus"
)

const labelDir = "/minion/labels"
const containerDir = "/minion/containers"

// We store rand.Uint32() in a variable so it's easily mocked out by the unit tests.
// Nondeterminism is hard to test.
var rand32 = rand.Uint32

// A directory containers the first and seceond level of a Tree requested from the
// consensus store.
type directory map[string]map[string]string

// wakeChan collapses the various channels these functions wait on into a single channel.
// Multiple redundant pings will be coalesced into a single message.
func wakeChan(conn db.Conn, store consensus.Store) chan struct{} {
	labelWatch := store.Watch(labelDir, 1*time.Second)
	containerWatch := store.Watch(labelDir, 1*time.Second)
	trigg := conn.TriggerTick(30, db.MinionTable, db.ContainerTable, db.LabelTable,
		db.EtcdTable).C

	c := make(chan struct{}, 1)
	go func() {
		for {
			select {
			case <-labelWatch:
			case <-containerWatch:
			case <-trigg:
			}

			select {
			case c <- struct{}{}:
			default: // There's a notification in queue no need for another.
			}
		}
	}()

	return c
}

func readStoreRun(conn db.Conn, store consensus.Store) {
	// If the directories don't exist, create them so we may watch them.  If they
	// exist already these will return an error that we won't log, but that's ok
	// cause the loop will error too.
	store.Mkdir(labelDir)
	store.Mkdir(containerDir)

	for range wakeChan(conn, store) {
		labelDir, err := getDirectory(store, labelDir)
		containerDir, err2 := getDirectory(store, containerDir)
		if err2 != nil {
			err = err2
		}

		if err != nil {
			log.WithError(err).Warn("Failed to read from cluster store.")
			continue
		}

		conn.Transact(func(view db.Database) error {
			readContainerTransact(view, containerDir)
			readLabelTransact(view, labelDir)
			return nil
		})
	}
}

func readContainerTransact(view db.Database, dir directory) {
	minions := view.SelectFromMinion(nil)
	worker := len(minions) == 1 && minions[0].Role == db.Worker

	for _, container := range view.SelectFromContainer(nil) {
		container.IP = ""
		var labels []string
		if children, ok := dir[container.SchedID]; ok {
			json.Unmarshal([]byte(children["Labels"]), &labels)

			container.IP = children["IP"]
			ip := net.ParseIP(container.IP).To4()
			if ip != nil {
				container.Mac = fmt.Sprintf("02:00:%02x:%02x:%02x:%02x",
					ip[0], ip[1], ip[2], ip[3])
			}
		}

		if worker {
			// Masters get their labels from the policy, workers from the
			// consensus store.
			container.Labels = labels
		}

		view.Commit(container)
	}
}

func readLabelTransact(view db.Database, dir directory) {
	pairs, dbls, dirKeys := join.Join(view.SelectFromLabel(nil), dir.keys(),
		func(left, right interface{}) int {
			if left.(db.Label).Label == right.(string) {
				return 0
			}
			return 1
		})

	for _, dbl := range dbls {
		view.Remove(dbl.(db.Label))
	}

	for _, key := range dirKeys {
		pairs = append(pairs, join.Pair{L: view.InsertLabel(), R: key})
	}

	for _, pair := range pairs {
		dbl := pair.L.(db.Label)
		dbl.Label = pair.R.(string)
		dbl.IP = dir[dbl.Label]["IP"]
		_, dbl.MultiHost = dir[dbl.Label]["MultiHost"]
		view.Commit(dbl)
	}
}

func writeStoreRun(conn db.Conn, store consensus.Store) {
	for range wakeChan(conn, store) {
		leader := false
		var containers []db.Container
		conn.Transact(func(view db.Database) error {
			etcds := view.SelectFromEtcd(nil)
			leader = len(etcds) == 1 && etcds[0].Leader
			containers = view.SelectFromContainer(nil)
			return nil
		})

		if !leader {
			continue
		}

		if err := writeStoreContainers(store, containers); err != nil {
			log.WithError(err).Warning("Failed to update containers in ETCD")
		}

		writeStoreLabels(store, containers)
	}
}

func writeStoreContainers(store consensus.Store, containers []db.Container) error {
	var ids []string
	for _, container := range containers {
		if container.SchedID != "" {
			ids = append(ids, container.SchedID)
		}
	}

	store.Mkdir(containerDir)
	dir, err := getDirectory(store, containerDir)
	if err != nil {
		return err
	}

	syncDir(store, dir, containerDir, ids)
	syncIPs(store, dir, containerDir, net.IPv4(10, 0, 0, 0))
	syncLabels(store, dir, containerDir, containers)

	return nil
}

func writeStoreLabels(store consensus.Store, containers []db.Container) error {
	store.Mkdir(labelDir)
	dir, err := getDirectory(store, labelDir)
	if err != nil {
		return err
	}

	var ids []string
	for _, c := range containers {
		for _, l := range c.Labels {
			ids = append(ids, l)
		}
	}

	syncDir(store, dir, labelDir, ids)

	// Labels that point to a single container don't need IPs separate from their
	// constituent container IP.  Thus the following code marks those labels, by the
	// absence of the `MultiHost` file in the consensus store, and sets their IP to
	// whatever is found in the container table.

	// Map from each label to the containers that implement it.
	labelMap := map[string][]db.Container{}
	for _, dbc := range containers {
		for _, label := range dbc.Labels {
			labelMap[label] = append(labelMap[label], dbc)
		}
	}

	// Mark labels as MultiHost if they have more than one container.
	for label, dbcs := range labelMap {
		isMH := len(dbcs) > 1
		if _, ok := dir[label]["MultiHost"]; ok == isMH {
			continue
		}

		var err error
		path := fmt.Sprintf("%s/%s/MultiHost", labelDir, label)
		if isMH {
			dir[label]["MultiHost"] = "true"
			err = store.Set(path, "true")
		} else {
			delete(dir[label], "MultiHost")
			err = store.Delete(path)
		}

		if err != nil {
			log.WithFields(log.Fields{
				"label": label,
				"error": err,
			}).Warn("Failed to change MultiHost key.")
		}
	}

	// Set the IPs of the non-MultiHost containers.
	for label, dbcs := range labelMap {
		if len(dbcs) != 1 {
			continue
		}

		dbc := dbcs[0]
		if dbc.IP == "" || dir[label]["IP"] == dbc.IP {
			continue
		}

		path := fmt.Sprintf("%s/%s/IP", labelDir, label)
		if err := store.Set(path, dbc.IP); err != nil {
			log.WithFields(log.Fields{
				"label": label,
				"IP":    dbc.IP,
				"error": err,
			}).Warn("Consensus store failed set label IP.")
		}
	}

	// Remove the non-MultiHost containers from `dir` to avoid confusing syncIPs.
	for label, mp := range dir {
		if _, ok := mp["MultiHost"]; !ok {
			delete(dir, label)
		}
	}

	syncIPs(store, dir, labelDir, net.IPv4(10, 1, 0, 0))
	return nil
}

func syncDir(store consensus.Store, dir directory, path string, idsArg []string) {
	_, dirKeys, ids := join.Join(dir.keys(), idsArg,
		func(left, right interface{}) int {
			if left.(string) != right.(string) {
				return -1
			}
			return 0
		})

	var etcdLog string
	for _, dirKey := range dirKeys {
		id := dirKey.(string)
		keyPath := fmt.Sprintf("%s/%s", path, id)
		err := store.Delete(keyPath)
		if err != nil {
			etcdLog = fmt.Sprintf("Failed to delete %s: %s", keyPath, err)
		}
		delete(dir, id)
	}

	for _, idElem := range ids {
		id := idElem.(string)
		if _, ok := dir[id]; ok {
			continue
		}

		key := fmt.Sprintf("%s/%s", path, id)
		if err := store.Mkdir(key); err != nil {
			etcdLog = fmt.Sprintf("Failed to create dir %s: %s", key, err)
			continue
		}
		dir[id] = map[string]string{}
	}

	// Etcd failure leads to a bunch of useless errors.  Therefore we only log once.
	if etcdLog != "" {
		log.Error(etcdLog)
	}
}

// syncIPs() takes a directory and creates an IP node for every entry that's missing one.
func syncIPs(store consensus.Store, dir directory, path string, prefixIP net.IP) {
	prefix := binary.BigEndian.Uint32(prefixIP.To4())
	mask := uint32(0xffff0000)

	var unassigned []string
	ipSet := map[uint32]struct{}{}
	for k, m := range dir {
		ip := parseIP(m["IP"], prefix, mask)
		if ip != 0 {
			ipSet[ip] = struct{}{}
		} else {
			unassigned = append(unassigned, k)
		}
	}

	// Don't assign the IP of the default gateway
	ipSet[parseIP(gatewayIP, prefix, mask)] = struct{}{}
	var etcdLog string
	for _, k := range unassigned {
		ip32 := randomIP(ipSet, prefix, mask)
		ipPath := fmt.Sprintf("%s/%s/IP", path, k)
		if ip32 == 0 {
			log.Errorf("Failed to allocate IP for %s.", k)
			store.Delete(ipPath)
			delete(dir[k], "IP")
			continue
		}

		b := make([]byte, 4)
		binary.BigEndian.PutUint32(b, ip32)

		ipStr := net.IP(b).String()
		if err := store.Set(ipPath, ipStr); err != nil {
			etcdLog = fmt.Sprintf("Failed to set key %s: %s", ipPath, err)
			continue
		}

		dir[k]["IP"] = ipStr
		ipSet[ip32] = struct{}{}
	}

	// Etcd failure leads to a bunch of useless errors.  Therefore we only log once.
	if etcdLog != "" {
		log.Error(etcdLog)
	}
}

func syncLabels(store consensus.Store, dir directory, path string,
	containers []db.Container) {

	idLabelMap := map[string][]string{}
	for _, container := range containers {
		if container.SchedID != "" {
			idLabelMap[container.SchedID] = container.Labels
		}
	}

	for id, children := range dir {
		labels := idLabelMap[id]
		if labels == nil {
			labels = []string{}
		}
		sort.Sort(sort.StringSlice(labels))

		jsByte, err := json.Marshal(labels)
		if err != nil {
			panic("Not Reached")
		}
		js := string(jsByte)

		if js == children["Labels"] {
			continue
		}

		key := fmt.Sprintf("%s/%s/Labels", path, id)
		if err := store.Set(key, js); err != nil {
			log.WithField("path", path).Error("Failed to set label key.")
			continue
		}
		dir[id]["Labels"] = js
	}
}

func getDirectory(store consensus.Store, path string) (directory, error) {
	tree, err := store.GetTree(path)
	if err != nil {
		return nil, err
	}

	dir := make(directory)
	for _, l1 := range tree.Children {
		childMap := make(map[string]string)
		for _, l2 := range l1.Children {
			childMap[l2.Key] = l2.Value
		}
		dir[l1.Key] = childMap
	}

	return dir, nil
}

func parseIP(ipStr string, prefix, mask uint32) uint32 {
	ip := net.ParseIP(ipStr).To4()
	if ip == nil {
		return 0
	}

	ip32 := binary.BigEndian.Uint32(ip)
	if ip32&mask != prefix {
		return 0
	}

	return ip32
}

// Choose a random IP address in prefix/mask that isn't in 'conflicts'.
// Returns 0 on failure.
func randomIP(conflicts map[uint32]struct{}, prefix, mask uint32) uint32 {
	for i := 0; i < 256; i++ {
		ip32 := (rand32() & ^mask) | (prefix & mask)
		if _, ok := conflicts[ip32]; !ok {
			return ip32
		}
	}

	return 0
}

func (dir directory) keys() []string {
	var keys []string
	for key := range dir {
		keys = append(keys, key)
	}
	return keys
}
<|endoftext|>


<commit_before>package main

import ("fmt"
	"os"
	"strconv"
	"github.com/jackyb/go-sdl2/sdl"
	"github.com/op/go-nanomsg"
	"github.com/fire/go-ogre3d")

func main() {
	fmt.Printf("Hello, game!\n")

	sdl.Init(sdl.INIT_EVERYTHING)
	window := sdl.CreateWindow("es_core::SDL",
		sdl.WINDOWPOS_UNDEFINED,
		sdl.WINDOWPOS_UNDEFINED,
		800,
		600,
		sdl.WINDOW_OPENGL | sdl.WINDOW_SHOWN)
	if window == nil {
		panic(fmt.Sprintf("sdl.CreateWindow failed: %s\n", sdl.GetError()))
	}
	glcontext := sdl.GL_CreateContext(window)
	if glcontext == nil {
		panic(fmt.Sprintf("sdl.CreateContext failed: %s\n", sdl.GetError()))
	}
	var info sdl.SysWMInfo
	if !window.GetWMInfo(&info) {
		panic(fmt.Sprintf("window.GetWMInfo failed.\n"))
	}
	// Parse and print info's version
	// Parse and print info's SYSWM_TYPE
	root := ogre.NewRoot("", "", "ogre.log")
	wd, err := os.Getwd()
	if err != nil {
		panic(err)
	}
	root.LoadPlugin(wd  + "/../frameworks/RenderSystem_GL.framework")
	renderers := root.GetAvailableRenderers()
	if renderers.RenderSystemListSize() != 1 {
		panic(fmt.Sprintf("Failed to initalize RendererRenderSystem_GL"))
	}
	root.SetRenderSystem(renderers.RenderSystemListGet(0))
	root.Initialise(false, "es_core::ogre")
	params := ogre.CreateNameValuePairList()
	params.AddPair("externalGLControl", "1")
	osxWindow := info.GetCocoaInfo()
	windowString := strconv.FormatUint(uint64(*(*uint32)(osxWindow.Window)), 10)
	params.AddPair("externalWindowHandle", windowString)
	params.AddPair("macAPI", "cocoa")
	params.AddPair("macAPICocoaUseNSView", "true")
	renderWindow := root.CreateRenderWindow("es_core::ogre", 800, 600, false, params)
	renderWindow.IsClosed() // Delete me

	game_socket, err := nanomsg.NewSocket(nanomsg.AF_SP, nanomsg.BUS)
        if err != nil {
                panic(err)
        }
        _, err = game_socket.Connect("tcp://127.0.0.1:60206")
        if err != nil {
                panic(err)
        }
}
<commit_msg>Ogre3d window is initialized.<commit_after>package main

import ("fmt"
	"os"
	"strconv"
	"github.com/jackyb/go-sdl2/sdl"
	"github.com/op/go-nanomsg"
	"github.com/fire/go-ogre3d")

func main() {
	fmt.Printf("Hello, game!\n")

	sdl.Init(sdl.INIT_EVERYTHING)
	window := sdl.CreateWindow("es_core::SDL",
		sdl.WINDOWPOS_UNDEFINED,
		sdl.WINDOWPOS_UNDEFINED,
		800,
		600,
		sdl.WINDOW_SHOWN)
	if window == nil {
		panic(fmt.Sprintf("sdl.CreateWindow failed: %s\n", sdl.GetError()))
	}
	var info sdl.SysWMInfo
	if !window.GetWMInfo(&info) {
		panic(fmt.Sprintf("window.GetWMInfo failed.\n"))
	}
	// Parse and print info's version
	// Parse and print info's SYSWM_TYPE
	root := ogre.NewRoot("", "", "ogre.log")
	wd, err := os.Getwd()
	if err != nil {
		panic(err)
	}
	root.LoadPlugin(wd  + "/../frameworks/RenderSystem_GL.framework")
	renderers := root.GetAvailableRenderers()
	if renderers.RenderSystemListSize() != 1 {
		panic(fmt.Sprintf("Failed to initalize RendererRenderSystem_GL"))
	}
	root.SetRenderSystem(renderers.RenderSystemListGet(0))
	root.Initialise(false, "es_core::ogre")
	params := ogre.CreateNameValuePairList()
	params.AddPair("macAPI", "cocoa")
	osxWindow := info.GetCocoaInfo()
	windowString := strconv.FormatUint(uint64(*(*uint32)(osxWindow.Window)), 10)
	params.AddPair("parentWindowHandle", windowString)

	renderWindow := root.CreateRenderWindow("es_core::ogre", 800, 600, false, params)
	renderWindow.IsClosed() // Delete me

	game_socket, err := nanomsg.NewSocket(nanomsg.AF_SP, nanomsg.BUS)
        if err != nil {
                panic(err)
        }
        _, err = game_socket.Connect("tcp://127.0.0.1:60206")
        if err != nil {
                panic(err)
        }
}
<|endoftext|>