<commit_before>// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package notifiers implements several kinds of notifier.Notifier
package notifiers

import (
	"bytes"
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"time"

	"gopkg.in/yaml.v2"

	"github.com/coreos/clair/config"
	"github.com/coreos/clair/database"
	"github.com/coreos/clair/notifier"
)

const timeout = 5 * time.Second

// A WebhookNotifier dispatches notifications to a webhook endpoint.
type WebhookNotifier struct {
	endpoint string
	client   *http.Client
}

// A WebhookNotifierConfiguration represents the configuration of a WebhookNotifier.
type WebhookNotifierConfiguration struct {
	Endpoint   string
	ServerName string
	CertFile   string
	KeyFile    string
	CAFile     string
}

func init() {
	notifier.RegisterNotifier("webhook", &WebhookNotifier{})
}

func (h *WebhookNotifier) Configure(config *config.NotifierConfig) (bool, error) {
	// Get configuration
	var httpConfig WebhookNotifierConfiguration
	if config == nil {
		return false, nil
	}
	if _, ok := config.Params["http"]; !ok {
		return false, nil
	}
	yamlConfig, err := yaml.Marshal(config.Params["http"])
	if err != nil {
		return false, errors.New("invalid configuration")
	}
	err = yaml.Unmarshal(yamlConfig, &httpConfig)
	if err != nil {
		return false, errors.New("invalid configuration")
	}

	// Validate endpoint URL.
	if httpConfig.Endpoint == "" {
		return false, nil
	}
	if _, err := url.Parse(httpConfig.Endpoint); err != nil {
		return false, errors.New("invalid endpoint URL")
	}
	h.endpoint = httpConfig.Endpoint

	// Initialize TLS.
	tlsConfig, err := loadTLSClientConfig(&httpConfig)
	if err != nil {
		return false, fmt.Errorf("could not initialize client cert auth: %s\n", err)
	}

	h.client = &http.Client{
		Transport: &http.Transport{
			TLSClientConfig: tlsConfig,
		},
		Timeout: timeout,
	}
	return true, nil
}

func (h *WebhookNotifier) Send(notification database.VulnerabilityNotification) error {
	// Marshal notification.
	jsonNotification, err := json.Marshal(notification.Name)
	if err != nil {
		return fmt.Errorf("could not marshal: %s", err)
	}

	// Send notification via HTTP POST.
	resp, err := h.client.Post(h.endpoint, "application/json", bytes.NewBuffer(jsonNotification))
	if err != nil || resp == nil || (resp.StatusCode != 200 && resp.StatusCode != 201) {
		if resp != nil {
			return fmt.Errorf("got status %d, expected 200/201", resp.StatusCode)
		}
		return err
	}
	defer resp.Body.Close()

	return nil
}

// loadTLSClientConfig initializes a *tls.Config using the given WebhookNotifierConfiguration.
//
// If no certificates are given, (nil, nil) is returned.
// The CA certificate is optional and falls back to the system default.
func loadTLSClientConfig(cfg *WebhookNotifierConfiguration) (*tls.Config, error) {
	if cfg.CertFile == "" || cfg.KeyFile == "" {
		return nil, nil
	}

	cert, err := tls.LoadX509KeyPair(cfg.CertFile, cfg.KeyFile)
	if err != nil {
		return nil, err
	}

	var caCertPool *x509.CertPool
	if cfg.CAFile != "" {
		caCert, err := ioutil.ReadFile(cfg.CAFile)
		if err != nil {
			return nil, err
		}
		caCertPool = x509.NewCertPool()
		caCertPool.AppendCertsFromPEM(caCert)
	}

	tlsConfig := &tls.Config{
		ServerName:   cfg.ServerName,
		Certificates: []tls.Certificate{cert},
		RootCAs:      caCertPool,
	}

	return tlsConfig, nil
}
<commit_msg>webhook: add JSON envelope<commit_after>// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package notifiers implements several kinds of notifier.Notifier
package notifiers

import (
	"bytes"
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"time"

	"gopkg.in/yaml.v2"

	"github.com/coreos/clair/config"
	"github.com/coreos/clair/database"
	"github.com/coreos/clair/notifier"
)

const timeout = 5 * time.Second

// A WebhookNotifier dispatches notifications to a webhook endpoint.
type WebhookNotifier struct {
	endpoint string
	client   *http.Client
}

// A WebhookNotifierConfiguration represents the configuration of a WebhookNotifier.
type WebhookNotifierConfiguration struct {
	Endpoint   string
	ServerName string
	CertFile   string
	KeyFile    string
	CAFile     string
}

func init() {
	notifier.RegisterNotifier("webhook", &WebhookNotifier{})
}

func (h *WebhookNotifier) Configure(config *config.NotifierConfig) (bool, error) {
	// Get configuration
	var httpConfig WebhookNotifierConfiguration
	if config == nil {
		return false, nil
	}
	if _, ok := config.Params["http"]; !ok {
		return false, nil
	}
	yamlConfig, err := yaml.Marshal(config.Params["http"])
	if err != nil {
		return false, errors.New("invalid configuration")
	}
	err = yaml.Unmarshal(yamlConfig, &httpConfig)
	if err != nil {
		return false, errors.New("invalid configuration")
	}

	// Validate endpoint URL.
	if httpConfig.Endpoint == "" {
		return false, nil
	}
	if _, err := url.Parse(httpConfig.Endpoint); err != nil {
		return false, errors.New("invalid endpoint URL")
	}
	h.endpoint = httpConfig.Endpoint

	// Initialize TLS.
	tlsConfig, err := loadTLSClientConfig(&httpConfig)
	if err != nil {
		return false, fmt.Errorf("could not initialize client cert auth: %s\n", err)
	}

	h.client = &http.Client{
		Transport: &http.Transport{
			TLSClientConfig: tlsConfig,
		},
		Timeout: timeout,
	}
	return true, nil
}

type notificationEnvelope struct {
	Notification struct {
		Name string
	}
}

func (h *WebhookNotifier) Send(notification database.VulnerabilityNotification) error {
	// Marshal notification.
	jsonNotification, err := json.Marshal(notificationEnvelope{struct{ Name string }{notification.Name}})
	if err != nil {
		return fmt.Errorf("could not marshal: %s", err)
	}

	// Send notification via HTTP POST.
	resp, err := h.client.Post(h.endpoint, "application/json", bytes.NewBuffer(jsonNotification))
	if err != nil || resp == nil || (resp.StatusCode != 200 && resp.StatusCode != 201) {
		if resp != nil {
			return fmt.Errorf("got status %d, expected 200/201", resp.StatusCode)
		}
		return err
	}
	defer resp.Body.Close()

	return nil
}

// loadTLSClientConfig initializes a *tls.Config using the given WebhookNotifierConfiguration.
//
// If no certificates are given, (nil, nil) is returned.
// The CA certificate is optional and falls back to the system default.
func loadTLSClientConfig(cfg *WebhookNotifierConfiguration) (*tls.Config, error) {
	if cfg.CertFile == "" || cfg.KeyFile == "" {
		return nil, nil
	}

	cert, err := tls.LoadX509KeyPair(cfg.CertFile, cfg.KeyFile)
	if err != nil {
		return nil, err
	}

	var caCertPool *x509.CertPool
	if cfg.CAFile != "" {
		caCert, err := ioutil.ReadFile(cfg.CAFile)
		if err != nil {
			return nil, err
		}
		caCertPool = x509.NewCertPool()
		caCertPool.AppendCertsFromPEM(caCert)
	}

	tlsConfig := &tls.Config{
		ServerName:   cfg.ServerName,
		Certificates: []tls.Certificate{cert},
		RootCAs:      caCertPool,
	}

	return tlsConfig, nil
}
<|endoftext|>


<commit_before>// Package rwcount implements a simple counter that wraps an io.Reader or io.Writer.
// Useful for functions (like binary.Read/Write) which do not return read/write counts.
package rwcount

import "io"

// CountReader is used to wrap a io.Reader for counting
type CountReader struct {
	io.Reader
	bytesRead int64
}

func (c *CountReader) Read(d []byte) (total int, err error) {
	total, err = c.Reader.Read(d)
	c.bytesRead += int64(total)
	return
}

// BytesRead returns the number of bytes read
func (c CountReader) BytesRead() int64 {
	return c.bytesRead
}

// CountWriter is used to wrap a io.Writer for counting
type CountWriter struct {
	io.Writer
	bytesWritten int64
}

func (c *CountWriter) Write(d []byte) (total int, err error) {
	total, err = c.Writer.Write(d)
	c.bytesWritten += int64(total)
	return
}

// BytesWritten returns the number of bytes written
func (c CountWriter) BytesWritten() int64 {
	return c.bytesWritten
}
<commit_msg>Now allows errors to be sticky<commit_after>// Package rwcount implements a simple counter that wraps an io.Reader or io.Writer.
// Useful for functions (like binary.Read/Write) which do not return read/write counts.
package rwcount

import "io"

// CountReader is used to wrap a io.Reader for counting
type CountReader struct {
	io.Reader
	bytesRead int64
	StickyErr bool
	Err       error
}

// Read implements the io.Reader interface
func (c *CountReader) Read(d []byte) (int, error) {
	if c.StickyErr && c.Err != nil {
		return 0, c.Err
	}
	total, err := c.Reader.Read(d)
	c.bytesRead += int64(total)
	c.Err = err
	return total, err
}

// BytesRead returns the number of bytes read
func (c CountReader) BytesRead() int64 {
	return c.bytesRead
}

// CountWriter is used to wrap a io.Writer for counting
type CountWriter struct {
	io.Writer
	bytesWritten int64
	StickyErr    bool
	Err          error
}

// Write implements the io.Writer interface
func (c *CountWriter) Write(d []byte) (int, error) {
	if c.StickyErr && c.Err != nil {
		return 0, c.Err
	}
	total, err := c.Writer.Write(d)
	c.bytesWritten += int64(total)
	c.Err = err
	return
}

// BytesWritten returns the number of bytes written
func (c CountWriter) BytesWritten() int64 {
	return c.bytesWritten
}
<|endoftext|>


<commit_before>package daemon

import (
	"encoding/json"
	"runtime"
	"time"

	"github.com/gorilla/websocket"
	"github.com/ondevice/ondevice/api"
	"github.com/ondevice/ondevice/config"
	"github.com/ondevice/ondevice/logg"
	"github.com/ondevice/ondevice/service"
	"github.com/ondevice/ondevice/tunnel"
	"github.com/ondevice/ondevice/util"
)

// DeviceSocket -- represents a device's connection to the ondevice.io API server
type DeviceSocket struct {
	tunnel.Connection

	lastPing time.Time
	wdog     *util.Watchdog

	OnConnection func(tunnelID string, service string, protocol string)
	OnError      func(error)
}

type pingMsg struct {
	Type string `json:"_type"`
	Ts   int    `json:"ts"`
}

// Connect -- Go online
func Connect(auths ...api.Authentication) (*DeviceSocket, error) {
	// TODO use the new 'key' param instead of 'id'
	params := map[string]string{"id": config.GetDeviceKey()}
	rc := DeviceSocket{}

	err := tunnel.OpenWebsocket(&rc.Connection, "/serve", params, rc.onMessage, auths...)

	if err != nil {
		return nil, err
	}

	rc.wdog = util.NewWatchdog(180*time.Second, rc.onPingTimeout)

	return &rc, nil
}

// SendConnectionError -- Send an connection error message to the API server)
func (d *DeviceSocket) SendConnectionError(code int, msg string, tunnelID string) {
	logg.Debugf("Sending connection error: %s (code %d)", msg, code)
	data := make(map[string]interface{})
	data["_type"] = "connectError"
	data["tunnelId"] = tunnelID
	data["code"] = code
	data["msg"] = msg
	d.SendJSON(data)
}

func (d *DeviceSocket) announce(service string, protocol string) {
	var data = map[string]string{"_type": "announce", "name": service, "protocol": protocol}
	d.SendJSON(data)
}

func (d *DeviceSocket) onConnect(msg *map[string]interface{}) {
	clientIP := _getString(msg, "clientIp")
	clientUser := _getString(msg, "clientUser")
	protocol := _getString(msg, "protocol")
	svc := _getString(msg, "service")
	brokerURL := _getString(msg, "broker")
	tunnelID := _getString(msg, "tunnelId")

	logg.Infof("Connection request for %s@%s from user %s@%s", protocol, svc, clientUser, clientIP)

	handler := service.GetServiceHandler(svc, protocol)
	if handler == nil {
		// TODO send the error back to the API server
		logg.Error("Coudln't find protocol handler: ", protocol)
		return
	}

	handler.Start(tunnelID, brokerURL)
}

func (d *DeviceSocket) onError(msg *map[string]interface{}) {
	code := _getInt(msg, "code")
	message := _getString(msg, "msg")
	var codeName = tunnel.GetErrorCodeName(int(code))

	logg.Errorf("Device ERROR: %s - %s ", codeName, message)
}

func (d *DeviceSocket) onHello(msg *map[string]interface{}) {
	logg.Debug("Got hello message: ", msg)
	var devID, key string
	if _contains(msg, "name") {
		// deprecated hello message format (for backwards compatibility) -- 2017-01-19
		devID = _getString(msg, "name")
		key = _getString(msg, "devId")
	} else {
		devID = _getString(msg, "devId")
		key = _getString(msg, "key")
	}

	logg.Infof("Connection established, online as '%s'", devID)

	// update the key if changed
	if config.GetDeviceKey() != key {
		logg.Debug("Updating device key: ", key)
		config.SetValue("device", "key", key)
	}

	// update devID
	config.SetValue("device", "dev-id", devID)

	// TODO announce configured services
	d.announce("ssh", "ssh")
}

func (d *DeviceSocket) onMessage(_type int, data []byte) {
	if _type == websocket.BinaryMessage {
		logg.Error("Got a binary message over the device websocket: ", string(data))
		return
	}

	msg := new(map[string]interface{})

	json.Unmarshal(data, msg)

	msgType := _getString(msg, "_type")
	switch msgType {
	case "hello":
		d.onHello(msg)
		break
	case "ping":
		var ping pingMsg
		json.Unmarshal(data, &ping)
		d.onPing(ping)
		break
	case "connect":
		d.onConnect(msg)
	case "error":
		d.onError(msg)
	default:
		logg.Error("Unsupported WS message: ", data)
		break
	}
}

func (d *DeviceSocket) onPing(msg pingMsg) {
	// quick'n'dirty way to see if we're leaking goroutines (e.g. with stray bloking reads)
	logg.Debugf("Got ping message: %+v (active goroutines: %d)", msg, runtime.NumGoroutine())
	d.lastPing = time.Now()
	d.wdog.Kick()
	resp := make(map[string]interface{}, 1)
	resp["_type"] = "pong"
	resp["ts"] = msg.Ts
	d.SendJSON(resp)
}

func (d *DeviceSocket) onPingTimeout() {
	logg.Warning("Haven't got a ping from the API server in a while, closing connection...")
	d.Close()
	d.wdog.Stop()
}

func _contains(m *map[string]interface{}, key string) bool {
	_, ok := (*m)[key]
	return ok
}

func _getInt(m *map[string]interface{}, key string) int64 {
	return (*m)[key].(int64)
}

func _getString(m *map[string]interface{}, key string) string {
	//logg.Debugf("-- %s: %s", key, (*m)[key])
	return (*m)[key].(string)
}
<commit_msg>minor update to daemon.Connect() (using the properly named ‘key’ param instead of ‘id’)<commit_after>package daemon

import (
	"encoding/json"
	"runtime"
	"time"

	"github.com/gorilla/websocket"
	"github.com/ondevice/ondevice/api"
	"github.com/ondevice/ondevice/config"
	"github.com/ondevice/ondevice/logg"
	"github.com/ondevice/ondevice/service"
	"github.com/ondevice/ondevice/tunnel"
	"github.com/ondevice/ondevice/util"
)

// DeviceSocket -- represents a device's connection to the ondevice.io API server
type DeviceSocket struct {
	tunnel.Connection

	lastPing time.Time
	wdog     *util.Watchdog

	OnConnection func(tunnelID string, service string, protocol string)
	OnError      func(error)
}

type pingMsg struct {
	Type string `json:"_type"`
	Ts   int    `json:"ts"`
}

// Connect -- Go online
func Connect(auths ...api.Authentication) (*DeviceSocket, error) {
	// TODO use the new 'key' param instead of 'id'
	params := map[string]string{"key": config.GetDeviceKey()}
	rc := DeviceSocket{}

	err := tunnel.OpenWebsocket(&rc.Connection, "/serve", params, rc.onMessage, auths...)

	if err != nil {
		return nil, err
	}

	rc.wdog = util.NewWatchdog(180*time.Second, rc.onPingTimeout)

	return &rc, nil
}

// SendConnectionError -- Send an connection error message to the API server)
func (d *DeviceSocket) SendConnectionError(code int, msg string, tunnelID string) {
	logg.Debugf("Sending connection error: %s (code %d)", msg, code)
	data := make(map[string]interface{})
	data["_type"] = "connectError"
	data["tunnelId"] = tunnelID
	data["code"] = code
	data["msg"] = msg
	d.SendJSON(data)
}

func (d *DeviceSocket) announce(service string, protocol string) {
	var data = map[string]string{"_type": "announce", "name": service, "protocol": protocol}
	d.SendJSON(data)
}

func (d *DeviceSocket) onConnect(msg *map[string]interface{}) {
	clientIP := _getString(msg, "clientIp")
	clientUser := _getString(msg, "clientUser")
	protocol := _getString(msg, "protocol")
	svc := _getString(msg, "service")
	brokerURL := _getString(msg, "broker")
	tunnelID := _getString(msg, "tunnelId")

	logg.Infof("Connection request for %s@%s from user %s@%s", protocol, svc, clientUser, clientIP)

	handler := service.GetServiceHandler(svc, protocol)
	if handler == nil {
		// TODO send the error back to the API server
		logg.Error("Coudln't find protocol handler: ", protocol)
		return
	}

	handler.Start(tunnelID, brokerURL)
}

func (d *DeviceSocket) onError(msg *map[string]interface{}) {
	code := _getInt(msg, "code")
	message := _getString(msg, "msg")
	var codeName = tunnel.GetErrorCodeName(int(code))

	logg.Errorf("Device ERROR: %s - %s ", codeName, message)
}

func (d *DeviceSocket) onHello(msg *map[string]interface{}) {
	logg.Debug("Got hello message: ", msg)
	var devID, key string
	if _contains(msg, "name") {
		// deprecated hello message format (for backwards compatibility) -- 2017-01-19
		devID = _getString(msg, "name")
		key = _getString(msg, "devId")
	} else {
		devID = _getString(msg, "devId")
		key = _getString(msg, "key")
	}

	logg.Infof("Connection established, online as '%s'", devID)

	// update the key if changed
	if config.GetDeviceKey() != key {
		logg.Debug("Updating device key: ", key)
		config.SetValue("device", "key", key)
	}

	// update devID
	config.SetValue("device", "dev-id", devID)

	// TODO announce configured services
	d.announce("ssh", "ssh")
}

func (d *DeviceSocket) onMessage(_type int, data []byte) {
	if _type == websocket.BinaryMessage {
		logg.Error("Got a binary message over the device websocket: ", string(data))
		return
	}

	msg := new(map[string]interface{})

	json.Unmarshal(data, msg)

	msgType := _getString(msg, "_type")
	switch msgType {
	case "hello":
		d.onHello(msg)
		break
	case "ping":
		var ping pingMsg
		json.Unmarshal(data, &ping)
		d.onPing(ping)
		break
	case "connect":
		d.onConnect(msg)
	case "error":
		d.onError(msg)
	default:
		logg.Error("Unsupported WS message: ", data)
		break
	}
}

func (d *DeviceSocket) onPing(msg pingMsg) {
	// quick'n'dirty way to see if we're leaking goroutines (e.g. with stray bloking reads)
	logg.Debugf("Got ping message: %+v (active goroutines: %d)", msg, runtime.NumGoroutine())
	d.lastPing = time.Now()
	d.wdog.Kick()
	resp := make(map[string]interface{}, 1)
	resp["_type"] = "pong"
	resp["ts"] = msg.Ts
	d.SendJSON(resp)
}

func (d *DeviceSocket) onPingTimeout() {
	logg.Warning("Haven't got a ping from the API server in a while, closing connection...")
	d.Close()
	d.wdog.Stop()
}

func _contains(m *map[string]interface{}, key string) bool {
	_, ok := (*m)[key]
	return ok
}

func _getInt(m *map[string]interface{}, key string) int64 {
	return (*m)[key].(int64)
}

func _getString(m *map[string]interface{}, key string) string {
	//logg.Debugf("-- %s: %s", key, (*m)[key])
	return (*m)[key].(string)
}
<|endoftext|>


<commit_before>package daemon

import (
	"fmt"
	"os"
	"path"

	"github.com/Sirupsen/logrus"
)

// ContainerRmConfig is a holder for passing in runtime config.
type ContainerRmConfig struct {
	ForceRemove, RemoveVolume, RemoveLink bool
}

// ContainerRm removes the container id from the filesystem. An error
// is returned if the container is not found, or if the remove
// fails. If the remove succeeds, the container name is released, and
// network links are removed.
func (daemon *Daemon) ContainerRm(name string, config *ContainerRmConfig) error {
	container, err := daemon.Get(name)
	if err != nil {
		return err
	}

	if config.RemoveLink {
		name, err := GetFullContainerName(name)
		if err != nil {
			return err
		}
		parent, n := path.Split(name)
		if parent == "/" {
			return fmt.Errorf("Conflict, cannot remove the default name of the container")
		}
		pe := daemon.containerGraph().Get(parent)
		if pe == nil {
			return fmt.Errorf("Cannot get parent %s for name %s", parent, name)
		}

		if err := daemon.containerGraph().Delete(name); err != nil {
			return err
		}

		parentContainer, _ := daemon.Get(pe.ID())
		if parentContainer != nil {
			if err := parentContainer.updateNetwork(); err != nil {
				logrus.Debugf("Could not update network to remove link %s: %v", n, err)
			}
		}

		return nil
	}

	if err := daemon.rm(container, config.ForceRemove); err != nil {
		return fmt.Errorf("Cannot destroy container %s: %v", name, err)
	}

	container.removeMountPoints(config.RemoveVolume)
	return nil
}

// Destroy unregisters a container from the daemon and cleanly removes its contents from the filesystem.
func (daemon *Daemon) rm(container *Container, forceRemove bool) (err error) {
	if container.IsRunning() {
		if !forceRemove {
			return fmt.Errorf("Conflict, You cannot remove a running container. Stop the container before attempting removal or use -f")
		}
		if err := container.Kill(); err != nil {
			return fmt.Errorf("Could not kill running container, cannot remove - %v", err)
		}
	}

	// stop collection of stats for the container regardless
	// if stats are currently getting collected.
	daemon.statsCollector.stopCollection(container)

	element := daemon.containers.Get(container.ID)
	if element == nil {
		return fmt.Errorf("Container %v not found - maybe it was already destroyed?", container.ID)
	}

	// Container state RemovalInProgress should be used to avoid races.
	if err = container.setRemovalInProgress(); err != nil {
		return fmt.Errorf("Failed to set container state to RemovalInProgress: %s", err)
	}

	defer container.resetRemovalInProgress()

	if err = container.Stop(3); err != nil {
		return err
	}

	// Mark container dead. We don't want anybody to be restarting it.
	container.setDead()

	// Save container state to disk. So that if error happens before
	// container meta file got removed from disk, then a restart of
	// docker should not make a dead container alive.
	if err := container.toDiskLocking(); err != nil {
		logrus.Errorf("Error saving dying container to disk: %v", err)
	}

	// If force removal is required, delete container from various
	// indexes even if removal failed.
	defer func() {
		if err != nil && forceRemove {
			daemon.idIndex.Delete(container.ID)
			daemon.containers.Delete(container.ID)
			os.RemoveAll(container.root)
			container.logEvent("destroy")
		}
	}()

	if _, err := daemon.containerGraphDB.Purge(container.ID); err != nil {
		logrus.Debugf("Unable to remove container from link graph: %s", err)
	}

	if err = daemon.driver.Remove(container.ID); err != nil {
		return fmt.Errorf("Driver %s failed to remove root filesystem %s: %s", daemon.driver, container.ID, err)
	}

	initID := fmt.Sprintf("%s-init", container.ID)
	if err := daemon.driver.Remove(initID); err != nil {
		return fmt.Errorf("Driver %s failed to remove init filesystem %s: %s", daemon.driver, initID, err)
	}

	if err = os.RemoveAll(container.root); err != nil {
		return fmt.Errorf("Unable to remove filesystem for %v: %v", container.ID, err)
	}

	if err = daemon.execDriver.Clean(container.ID); err != nil {
		return fmt.Errorf("Unable to remove execdriver data for %s: %s", container.ID, err)
	}

	selinuxFreeLxcContexts(container.ProcessLabel)
	daemon.idIndex.Delete(container.ID)
	daemon.containers.Delete(container.ID)

	container.logEvent("destroy")
	return nil
}

// VolumeRm removes the volume with the given name.
// If the volume is referenced by a container it is not removed
// This is called directly from the remote API
func (daemon *Daemon) VolumeRm(name string) error {
	v, err := daemon.volumes.Get(name)
	if err != nil {
		return err
	}
	if err := daemon.volumes.Remove(v); err != nil {
		if err == ErrVolumeInUse {
			return fmt.Errorf("Conflict: %v", err)
		}
		return err
	}
	return nil
}
<commit_msg>record the error of removing volumes<commit_after>package daemon

import (
	"fmt"
	"os"
	"path"

	"github.com/Sirupsen/logrus"
)

// ContainerRmConfig is a holder for passing in runtime config.
type ContainerRmConfig struct {
	ForceRemove, RemoveVolume, RemoveLink bool
}

// ContainerRm removes the container id from the filesystem. An error
// is returned if the container is not found, or if the remove
// fails. If the remove succeeds, the container name is released, and
// network links are removed.
func (daemon *Daemon) ContainerRm(name string, config *ContainerRmConfig) error {
	container, err := daemon.Get(name)
	if err != nil {
		return err
	}

	if config.RemoveLink {
		name, err := GetFullContainerName(name)
		if err != nil {
			return err
		}
		parent, n := path.Split(name)
		if parent == "/" {
			return fmt.Errorf("Conflict, cannot remove the default name of the container")
		}
		pe := daemon.containerGraph().Get(parent)
		if pe == nil {
			return fmt.Errorf("Cannot get parent %s for name %s", parent, name)
		}

		if err := daemon.containerGraph().Delete(name); err != nil {
			return err
		}

		parentContainer, _ := daemon.Get(pe.ID())
		if parentContainer != nil {
			if err := parentContainer.updateNetwork(); err != nil {
				logrus.Debugf("Could not update network to remove link %s: %v", n, err)
			}
		}

		return nil
	}

	if err := daemon.rm(container, config.ForceRemove); err != nil {
		return fmt.Errorf("Cannot destroy container %s: %v", name, err)
	}

	if err := container.removeMountPoints(config.RemoveVolume); err != nil {
		logrus.Errorf("%v", err)
	}

	return nil
}

// Destroy unregisters a container from the daemon and cleanly removes its contents from the filesystem.
func (daemon *Daemon) rm(container *Container, forceRemove bool) (err error) {
	if container.IsRunning() {
		if !forceRemove {
			return fmt.Errorf("Conflict, You cannot remove a running container. Stop the container before attempting removal or use -f")
		}
		if err := container.Kill(); err != nil {
			return fmt.Errorf("Could not kill running container, cannot remove - %v", err)
		}
	}

	// stop collection of stats for the container regardless
	// if stats are currently getting collected.
	daemon.statsCollector.stopCollection(container)

	element := daemon.containers.Get(container.ID)
	if element == nil {
		return fmt.Errorf("Container %v not found - maybe it was already destroyed?", container.ID)
	}

	// Container state RemovalInProgress should be used to avoid races.
	if err = container.setRemovalInProgress(); err != nil {
		return fmt.Errorf("Failed to set container state to RemovalInProgress: %s", err)
	}

	defer container.resetRemovalInProgress()

	if err = container.Stop(3); err != nil {
		return err
	}

	// Mark container dead. We don't want anybody to be restarting it.
	container.setDead()

	// Save container state to disk. So that if error happens before
	// container meta file got removed from disk, then a restart of
	// docker should not make a dead container alive.
	if err := container.toDiskLocking(); err != nil {
		logrus.Errorf("Error saving dying container to disk: %v", err)
	}

	// If force removal is required, delete container from various
	// indexes even if removal failed.
	defer func() {
		if err != nil && forceRemove {
			daemon.idIndex.Delete(container.ID)
			daemon.containers.Delete(container.ID)
			os.RemoveAll(container.root)
			container.logEvent("destroy")
		}
	}()

	if _, err := daemon.containerGraphDB.Purge(container.ID); err != nil {
		logrus.Debugf("Unable to remove container from link graph: %s", err)
	}

	if err = daemon.driver.Remove(container.ID); err != nil {
		return fmt.Errorf("Driver %s failed to remove root filesystem %s: %s", daemon.driver, container.ID, err)
	}

	initID := fmt.Sprintf("%s-init", container.ID)
	if err := daemon.driver.Remove(initID); err != nil {
		return fmt.Errorf("Driver %s failed to remove init filesystem %s: %s", daemon.driver, initID, err)
	}

	if err = os.RemoveAll(container.root); err != nil {
		return fmt.Errorf("Unable to remove filesystem for %v: %v", container.ID, err)
	}

	if err = daemon.execDriver.Clean(container.ID); err != nil {
		return fmt.Errorf("Unable to remove execdriver data for %s: %s", container.ID, err)
	}

	selinuxFreeLxcContexts(container.ProcessLabel)
	daemon.idIndex.Delete(container.ID)
	daemon.containers.Delete(container.ID)

	container.logEvent("destroy")
	return nil
}

// VolumeRm removes the volume with the given name.
// If the volume is referenced by a container it is not removed
// This is called directly from the remote API
func (daemon *Daemon) VolumeRm(name string) error {
	v, err := daemon.volumes.Get(name)
	if err != nil {
		return err
	}
	if err := daemon.volumes.Remove(v); err != nil {
		if err == ErrVolumeInUse {
			return fmt.Errorf("Conflict: %v", err)
		}
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"testing"
	"time"
	"net/http"
)

func TestSanity(t *testing.T) {
	if 1 != 1 {
		t.Error("Failed sanity check.")
	}
}

func TestStatus(t *testing.T) {
	s := status()
	if s != "" {
		t.Error("Unexpected status output.")
	}
}

func TestNewEvent(t *testing.T) {
	e := newEvent("test_toast")
	if e.name != "test_toast" {
		t.Error("Failed to set event name.")
	}
	// This should not take even close to a second.
	if time.Since(e.timestamp) > time.Second {
		t.Error("Failed to set event timestamp.")
	}
}

func TestNewCallback(t *testing.T) {
	c := newCallback("test_toast", "http://localhost:1339/ev/")
	if c.event != "test_toast" {
		t.Error("Failed to set callback event.")
	}
	if c.addr != "http://localhost:1339/ev/" {
		t.Error("Failed to set callback address.")
	}
}

func TestRegisterCallback(t *testing.T) {
	registerCallback("test_bread", "http://localhost:1339/ev/")
	tev := callbacks["test_bread"][0]
	if tev.event != "test_bread" {
		t.Error("Failed to set callback event.")
	}
	if tev.addr != "http://localhost:1339/ev/" {
		t.Error("Failed to set callback address.")
	}
}

func TestApiV1Access(t *testing.T) {
	go main() // Start the actual webserver

	// Check general connectivity.
	_, err := http.Get("http://localhost:8080/api/v1/")
	if err != nil {
		t.Error(err.Error())
	}

	// Register some callback.
	var tpl1 = []byte(`
		{"action": "register", "event": "test_apiv1"}
	`)

	resp, err := http.Post("http://localhost:8080/api/v1/",
	                       "application/json", bytes.NewBuffer(tpl1))
	if err != nil {
		t.Error(err.Error())
	}
	if resp.StatusCode != 200 {
		t.Error("Server says we failed to register a callback.")
	}
	if len(callbacks["test_apiv1"]) != 1 {
		t.Error("Failed to register a callback..")
	}
}

<commit_msg>add a test for the just fixed bug<commit_after>package main

import (
	"bytes"
	"strings"
	"testing"
	"time"
	"net/http"
)

func TestSanity(t *testing.T) {
	if 1 != 1 {
		t.Error("Failed sanity check.")
	}
}

func TestStatus(t *testing.T) {
	s := status()
	if s != "" {
		t.Error("Unexpected status output.")
	}
}

func TestNewEvent(t *testing.T) {
	e := newEvent("test_toast")
	if e.name != "test_toast" {
		t.Error("Failed to set event name.")
	}
	// This should not take even close to a second.
	if time.Since(e.timestamp) > time.Second {
		t.Error("Failed to set event timestamp.")
	}
}

func TestNewCallback(t *testing.T) {
	c := newCallback("test_toast", "http://localhost:1339/ev/")
	if c.event != "test_toast" {
		t.Error("Failed to set callback event.")
	}
	if c.addr != "http://localhost:1339/ev/" {
		t.Error("Failed to set callback address.")
	}
}

func TestRegisterCallback(t *testing.T) {
	registerCallback("test_bread", "http://localhost:1339/ev/")
	tev := callbacks["test_bread"][0]
	if tev.event != "test_bread" {
		t.Error("Failed to set callback event.")
	}
	if tev.addr != "http://localhost:1339/ev/" {
		t.Error("Failed to set callback address.")
	}
}

func TestApiV1Access(t *testing.T) {
	go main() // Start the actual webserver

	// Check general connectivity.
	_, err := http.Get("http://localhost:8080/api/v1/")
	if err != nil {
		t.Error(err.Error())
	}

	// Register some callback.
	var tpl1 = []byte(`
		{"action": "register", "event": "test_apiv1"}
	`)

	resp, err := http.Post("http://localhost:8080/api/v1/",
	                       "application/json", bytes.NewBuffer(tpl1))
	if err != nil {
		t.Error(err.Error())
	}
	if resp.StatusCode != 200 {
		t.Error("Server says we failed to register a callback.")
	}
	if len(callbacks["test_apiv1"]) != 1 {
		t.Error("Failed to register a callback.")
	}
	if !strings.Contains(callbacks["test_apiv1"][0].addr, "127.0.0.1:") {
		t.Error("Failed to register a callback address correctly.")
	}
	}
}

<|endoftext|>


<commit_before>// +build freebsd netbsd openbsd dragonfly darwin

package immortal

import (
	"fmt"
	"log"
	"math/rand"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"
)

// ScanDir struct
type ScanDir struct {
	scandir  string
	sdir     string
	services map[string]string
	watch    chan string
	sync.Mutex
}

// NewScanDir returns ScanDir struct
func NewScanDir(path string) (*ScanDir, error) {
	if !isDir(path) {
		return nil, fmt.Errorf("%q is not a directory", path)
	}

	dir, err := filepath.EvalSymlinks(path)
	if err != nil {
		return nil, err
	}

	dir, err = filepath.Abs(dir)
	if err != nil {
		return nil, err
	}

	d, err := os.Open(dir)
	if err != nil {
		if os.IsPermission(err) {
			return nil, os.ErrPermission
		}
		return nil, err
	}
	defer d.Close()

	return &ScanDir{
		scandir:  dir,
		sdir:     GetSdir(),
		services: map[string]string{},
		watch:    make(chan string, 1),
	}, nil
}

// Start check for changes on directory
func (s *ScanDir) Start(ctl Control) {
	log.Printf("immortal scandir: %s", s.scandir)

	// create supervise directory (/var/run/immortal) if doesn't exists
	// IMMORTAL_SDIR
	if !isDir(s.sdir) {
		if err := os.MkdirAll(s.sdir, os.ModePerm); err != nil {
			log.Fatalf("Could not create supervise dir: %q, %v", s.sdir, err)
		}
	}

	// check for changes on sdir helps to restart stoped services
	go WatchDir(s.sdir, s.watch)

	// check for new services on scandir
	go WatchDir(s.scandir, s.watch)
	s.watch <- s.scandir

	for {
		select {
		case watch := <-s.watch:
			switch watch {
			case s.sdir:
				// RESTART, after halting a service, this will
				// start stopped services after 1 second of receiving the signal
				time.Sleep(time.Second)
				if err := s.Scandir(ctl); err != nil && !os.IsPermission(err) {
					log.Printf("Scandir error: %s", err)
				}
			case s.scandir:
				if err := s.Scandir(ctl); err != nil && !os.IsPermission(err) {
					log.Printf("Scandir error: %s", err)
				}
			default:
				serviceFile := filepath.Base(watch)
				serviceName := strings.TrimSuffix(serviceFile, filepath.Ext(serviceFile))
				if isFile(watch) {
					md5, err := md5sum(watch)
					if err != nil {
						log.Fatalf("Error getting the md5sum: %s", err)
					}
					s.Lock()
					// restart if file changed
					if md5 != s.services[serviceName] {
						s.services[serviceName] = md5
						log.Printf("Stopping: %s\n", serviceName)
						ctl.SendSignal(filepath.Join(s.sdir, serviceName, "immortal.sock"), "halt")
					}
					s.Unlock()
					log.Printf("Starting: %s\n", serviceName)
					// try to start before via socket
					if _, err := ctl.SendSignal(filepath.Join(s.sdir, serviceName, "immortal.sock"), "start"); err != nil {
						if out, err := ctl.Run(fmt.Sprintf("immortal -c %s -ctl %s", watch, serviceName)); err != nil {
							// keep retrying
							delete(s.services, serviceName)
							log.Println(err)
						} else {
							log.Printf("%s\n", out)
						}
					}
					go func() {
						if err := WatchFile(watch, s.watch); err != nil {
							log.Printf("WatchFile error: %s", err)
							// try 3 times sleeping i*100ms between retries
							for i := int32(100); i <= 300; i += 100 {
								time.Sleep(time.Duration(rand.Int31n(i)) * time.Millisecond)
								err := WatchFile(watch, s.watch)
								if err == nil {
									return
								}
							}
							log.Printf("Could not watch file %q error: %s", watch, err)
						}
					}()
				} else {
					// remove service
					s.Lock()
					delete(s.services, serviceName)
					s.Unlock()
					ctl.SendSignal(filepath.Join(s.sdir, serviceName, "immortal.sock"), "halt")
					log.Printf("Exiting: %s\n", serviceName)
				}
			}
		}
	}
}

// Scaner searches for *.yml if file changes it will reload(stop-start)
func (s *ScanDir) Scandir(ctl Control) error {
	s.Lock()
	defer s.Unlock()
	find := func(path string, f os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if f.Mode().IsRegular() {
			if filepath.Ext(f.Name()) == ".yml" {
				name := strings.TrimSuffix(f.Name(), filepath.Ext(f.Name()))
				md5, err := md5sum(path)
				if err != nil {
					return fmt.Errorf("Error getting the md5sum: %s", err)
				}
				// start or restart if service is not in map or file lock don't exists
				if _, ok := s.services[name]; !ok || !isFile(filepath.Join(s.sdir, name, "lock")) {
					s.services[name] = md5
					log.Printf("Starting: %s\n", name)
					if out, err := ctl.Run(fmt.Sprintf("immortal -c %s -ctl %s", path, name)); err != nil {
						log.Println(err)
					} else {
						log.Printf("%s\n", out)
					}
					go func() {
						if err := WatchFile(path, s.watch); err != nil {
							log.Printf("WatchFile error: %s", err)
							// try 3 times sleeping i*100ms between retries
							for i := int32(100); i <= 300; i += 100 {
								time.Sleep(time.Duration(rand.Int31n(i)) * time.Millisecond)
								err := WatchFile(path, s.watch)
								if err == nil {
									return
								}
							}
							log.Printf("Could not watch file %q error: %s", path, err)
						}
					}()
				}
			}
		}

		// Block for 100 ms on each call to kevent (WatchFile)
		time.Sleep(100 * time.Millisecond)

		return err
	}
	return filepath.Walk(s.scandir, find)
}
<commit_msg>log only if can't watch file<commit_after>// +build freebsd netbsd openbsd dragonfly darwin

package immortal

import (
	"fmt"
	"log"
	"math/rand"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"
)

// ScanDir struct
type ScanDir struct {
	scandir  string
	sdir     string
	services map[string]string
	watch    chan string
	sync.Mutex
}

// NewScanDir returns ScanDir struct
func NewScanDir(path string) (*ScanDir, error) {
	if !isDir(path) {
		return nil, fmt.Errorf("%q is not a directory", path)
	}

	dir, err := filepath.EvalSymlinks(path)
	if err != nil {
		return nil, err
	}

	dir, err = filepath.Abs(dir)
	if err != nil {
		return nil, err
	}

	d, err := os.Open(dir)
	if err != nil {
		if os.IsPermission(err) {
			return nil, os.ErrPermission
		}
		return nil, err
	}
	defer d.Close()

	return &ScanDir{
		scandir:  dir,
		sdir:     GetSdir(),
		services: map[string]string{},
		watch:    make(chan string, 1),
	}, nil
}

// Start check for changes on directory
func (s *ScanDir) Start(ctl Control) {
	log.Printf("immortal scandir: %s", s.scandir)

	// create supervise directory (/var/run/immortal) if doesn't exists
	// IMMORTAL_SDIR
	if !isDir(s.sdir) {
		if err := os.MkdirAll(s.sdir, os.ModePerm); err != nil {
			log.Fatalf("Could not create supervise dir: %q, %v", s.sdir, err)
		}
	}

	// check for changes on sdir helps to restart stoped services
	go WatchDir(s.sdir, s.watch)

	// check for new services on scandir
	go WatchDir(s.scandir, s.watch)
	s.watch <- s.scandir

	for {
		select {
		case watch := <-s.watch:
			switch watch {
			case s.sdir:
				// RESTART, after halting a service, this will
				// start stopped services after 1 second of receiving the signal
				time.Sleep(time.Second)
				if err := s.Scandir(ctl); err != nil && !os.IsPermission(err) {
					log.Printf("Scandir error: %s", err)
				}
			case s.scandir:
				if err := s.Scandir(ctl); err != nil && !os.IsPermission(err) {
					log.Printf("Scandir error: %s", err)
				}
			default:
				serviceFile := filepath.Base(watch)
				serviceName := strings.TrimSuffix(serviceFile, filepath.Ext(serviceFile))
				if isFile(watch) {
					md5, err := md5sum(watch)
					if err != nil {
						log.Fatalf("Error getting the md5sum: %s", err)
					}
					s.Lock()
					// restart if file changed
					if md5 != s.services[serviceName] {
						s.services[serviceName] = md5
						log.Printf("Stopping: %s\n", serviceName)
						ctl.SendSignal(filepath.Join(s.sdir, serviceName, "immortal.sock"), "halt")
					}
					s.Unlock()
					log.Printf("Starting: %s\n", serviceName)
					// try to start before via socket
					if _, err := ctl.SendSignal(filepath.Join(s.sdir, serviceName, "immortal.sock"), "start"); err != nil {
						if out, err := ctl.Run(fmt.Sprintf("immortal -c %s -ctl %s", watch, serviceName)); err != nil {
							// keep retrying
							delete(s.services, serviceName)
							log.Println(err)
						} else {
							log.Printf("%s\n", out)
						}
					}
					go func() {
						if err := WatchFile(watch, s.watch); err != nil {
							log.Printf("WatchFile error: %s", err)
							// try 3 times sleeping i*100ms between retries
							for i := int32(100); i <= 300; i += 100 {
								time.Sleep(time.Duration(rand.Int31n(i)) * time.Millisecond)
								err := WatchFile(watch, s.watch)
								if err == nil {
									return
								}
							}
							log.Printf("Could not watch file %q error: %s", watch, err)
						}
					}()
				} else {
					// remove service
					s.Lock()
					delete(s.services, serviceName)
					s.Unlock()
					ctl.SendSignal(filepath.Join(s.sdir, serviceName, "immortal.sock"), "halt")
					log.Printf("Exiting: %s\n", serviceName)
				}
			}
		}
	}
}

// Scaner searches for *.yml if file changes it will reload(stop-start)
func (s *ScanDir) Scandir(ctl Control) error {
	s.Lock()
	defer s.Unlock()
	find := func(path string, f os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if f.Mode().IsRegular() {
			if filepath.Ext(f.Name()) == ".yml" {
				name := strings.TrimSuffix(f.Name(), filepath.Ext(f.Name()))
				md5, err := md5sum(path)
				if err != nil {
					return fmt.Errorf("Error getting the md5sum: %s", err)
				}
				// start or restart if service is not in map or file lock don't exists
				if _, ok := s.services[name]; !ok || !isFile(filepath.Join(s.sdir, name, "lock")) {
					s.services[name] = md5
					log.Printf("Starting: %s\n", name)
					if out, err := ctl.Run(fmt.Sprintf("immortal -c %s -ctl %s", path, name)); err != nil {
						log.Println(err)
					} else {
						log.Printf("%s\n", out)
					}
					go func() {
						if err := WatchFile(path, s.watch); err != nil {
							// try 3 times sleeping i*100ms between retries
							for i := int32(100); i <= 300; i += 100 {
								time.Sleep(time.Duration(rand.Int31n(i)) * time.Millisecond)
								err := WatchFile(path, s.watch)
								if err == nil {
									return
								}
							}
							log.Printf("Could not watch file %q error: %s", path, err)
						}
					}()
				}
			}
		}

		// Block for 100 ms on each call to kevent (WatchFile)
		time.Sleep(100 * time.Millisecond)

		return err
	}
	return filepath.Walk(s.scandir, find)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/binary"
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"time"

	"github.com/conformal/btcwire"
)

var (
	blockdir = flag.String("blockdir", "/home/ubuntu/.bitcoin/testnet3/blocks", "The directory containing bitcoin blocks")
	logger   = log.New(os.Stdout, "", log.Llongfile)
	empt     = [32]byte{}
	//_genesisHash, _ = hex.DecodeString("43497fd7f826957108f4a30fd9cec3aeba79972084e90ead01ea330900000000")
	genesisHash = [32]byte{}
	maxBlocks   = 100000
)

type BlockHead struct {
	// A struct that matches the exact format of blocks stored in blk*.dat files
	Magic      [4]byte
	Length     uint32
	Version    int32
	PrevHash   [32]byte
	MerkleRoot [32]byte
	Timestamp  uint32
	Difficulty uint32
	Nonce      uint32
}

type Block struct {
	// A custom block object for processing
	PrevBlock *Block
	Head      *BlockHead
	RelTxs    []*btcwire.MsgTx
	Hash      [32]byte
	depth     int
}

func check(err error) {
	if err != nil {
		logger.Fatal(err)
	}
}

func btcBHFromBH(bh BlockHead) *btcwire.BlockHeader {
	// utility function to convert custom BlockHead type to btcwire BlockHeader
	prevhash, _ := btcwire.NewShaHash(bh.PrevHash[:])
	merkle, _ := btcwire.NewShaHash(bh.MerkleRoot[:])
	timestamp := time.Unix(int64(bh.Timestamp), 0)

	btcbh := btcwire.BlockHeader{
		Version:    bh.Version,
		PrevBlock:  *prevhash,
		MerkleRoot: *merkle,
		Timestamp:  timestamp,
		Bits:       bh.Difficulty,
		Nonce:      bh.Nonce,
	}
	return &btcbh
}

func blockHash(bh BlockHead) [32]byte {
	// Print the hash of the block from the headers in the block
	btcbh := btcBHFromBH(bh)
	hash, _ := btcbh.BlockSha()
	return [32]byte(hash)
}

func proceed(f *os.File) bool {
	// finds the start of the next block and places the cursor on it
	for {
		var b [4]byte
		_, err := io.ReadFull(f, b[:])
		if err != nil {
			return false
		}
		discrim := binary.BigEndian.Uint32(b[:])
		if discrim != 0x00000000 {
			// seek backwards to start of block
			f.Seek(-4, 1)
			return true
		}
	}
}

func playWithFile(fname string, blkList []*Block, blkMap map[[32]byte]*Block) ([]*Block, map[[32]byte]*Block) {
	// given a blk file attempts to parse every block within it. Adding the block
	// to a global list of seen blocks. Additionally we strip out the interesting
	// transactions at this stage.
	file, err := os.Open(fname)
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	genesis := false
	if len(blkList) == 0 {
		genesis = true
	}
	bad := 0
	for i := 0; i < maxBlocks; i++ {
		var blk Block
		var bh BlockHead

		ok := proceed(file)
		if !ok {
			fmt.Println("Hit end of file: ", fname)
		}
		err = binary.Read(file, binary.LittleEndian, &bh)
		if err == io.EOF || err == io.ErrUnexpectedEOF {
			fmt.Println("At the end of file: ", fname)
			break
		}
		if err != nil {
			logger.Fatal(err)
		}

		tx_num, err := readVarInt(file, 0)
		if err != nil {
			logger.Fatal(err)
		}

		//printBlockHead(bh)
		//fmt.Printf("num_tx:\t\t%d\n", tx_num)

		hash := blockHash(bh)

		for i := uint64(0); i < tx_num; i++ {
			tx := btcwire.MsgTx{}
			err := tx.Deserialize(file)
			if err != nil {
				logger.Fatal(err)
			}
		}

		if genesis {
			blk = Block{
				PrevBlock: nil,
				Head:      &bh,
				RelTxs:    make([]*btcwire.MsgTx, 0),
				Hash:      hash,
				depth:     0,
			}
			genesis = false
			fmt.Printf("The hash of the genesis block:\n%x\n", hash)
		} else {
			prevBlock, ok := blkMap[bh.PrevHash]
			if !ok {
				logger.Fatal("No prevhash in older block files")
			}
			// use last blk to find prevhash
			blk = Block{
				PrevBlock: prevBlock,
				Head:      &bh,
				RelTxs:    make([]*btcwire.MsgTx, 0),
				Hash:      hash,
			}
		}
		blkMap[hash] = &blk
		blkList = append(blkList, &blk)
	}

	return blkList, blkMap
}

func calcHeight(blkList []*Block) int {
	// Computes the best chain's total height by starting from the latest blocks
	// and working pack to the genesis block.
	var blk *Block
	for j := len(blkList) - 1; j >= 0; j-- {
		blk = blkList[j]
		if blk.depth == 0 {
			blk.depth = 1
		}
		//if blk.PrevBlock == nil && blk.Hash == genesisHash {
		if blk.Hash == genesisHash {
			println("found genesis hash", j)
			break
		}
		nextD := blk.depth + 1
		if blk.PrevBlock != nil && blk.PrevBlock.depth < nextD {
			blk.PrevBlock.depth = nextD
		}
	}
	return blk.depth
}

func main() {
	flag.Parse()

	//copy(genesisHash[:], _genesisHash)

	glob := "/blk*.dat"

	blockfiles, err := filepath.Glob(*blockdir + glob)
	if err != nil {
		log.Fatal(err)
	}
	if len(blockfiles) < 1 {
		log.Fatal(errors.New("Could not find any blockfiles at " + *blockdir))
	}

	var blkList []*Block
	var blkMap map[[32]byte]*Block
	blkList = make([]*Block, 0, maxBlocks)
	blkMap = make(map[[32]byte]*Block)
	for _, filename := range blockfiles {
		println(filename)
		blkList, blkMap = playWithFile(filename, blkList, blkMap)
		fmt.Println("Processed:", len(blkList))
	}

	println("Finding blockchain height")
	h := calcHeight(blkList)
	println("Height: ", h)
}

// From btcwire common.go
// readVarInt reads a variable length integer from r and returns it as a uint64.
func readVarInt(r io.Reader, pver uint32) (uint64, error) {
	var b [8]byte
	_, err := io.ReadFull(r, b[0:1])
	if err != nil {
		return 0, err
	}

	var rv uint64
	discriminant := uint8(b[0])
	switch discriminant {
	case 0xff:
		_, err := io.ReadFull(r, b[:])
		if err != nil {
			return 0, err
		}
		rv = binary.LittleEndian.Uint64(b[:])

	case 0xfe:
		_, err := io.ReadFull(r, b[0:4])
		if err != nil {
			return 0, err
		}
		rv = uint64(binary.LittleEndian.Uint32(b[:]))

	case 0xfd:
		_, err := io.ReadFull(r, b[0:2])
		if err != nil {
			return 0, err
		}
		rv = uint64(binary.LittleEndian.Uint16(b[:]))

	default:
		rv = uint64(discriminant)
	}

	return rv, nil
}

func printBlockHead(blk BlockHead) {
	// Prints out header from a given block

	prevhash, _ := btcwire.NewShaHash(blk.PrevHash[:])
	merkle, _ := btcwire.NewShaHash(blk.MerkleRoot[:])
	timestamp := time.Unix(int64(blk.Timestamp), 0)

	bh := btcwire.BlockHeader{
		Version:    blk.Version,
		PrevBlock:  *prevhash,
		MerkleRoot: *merkle,
		Timestamp:  timestamp,
		Bits:       blk.Difficulty,
		Nonce:      blk.Nonce,
	}
	hash, err := bh.BlockSha()
	check(err)
	fmt.Printf("Hash:\t\t%s", hash)
	fmt.Printf(`
prevHash:	%s
merkle root:	%s
timestamp:	%s
difficulty:	%d
nonce:		%d
bit len:	%d
`,
		prevhash.String(), merkle.String(), timestamp, blk.Difficulty, blk.Nonce, blk.Length)
}
<commit_msg>added chainTip algorthim. Scanning is being slowed down by disk seeks. This is being ignored by me<commit_after>package main

import (
	"encoding/binary"
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"time"

	"github.com/conformal/btcwire"
)

var (
	blockdir = flag.String("blockdir", "/home/ubuntu/.bitcoin/testnet3/blocks", "The directory containing bitcoin blocks")
	logger   = log.New(os.Stdout, "", log.Llongfile)
	empt     = [32]byte{}
	//_genesisHash, _ = hex.DecodeString("43497fd7f826957108f4a30fd9cec3aeba79972084e90ead01ea330900000000")
	genesisHash = [32]byte{}
	maxBlocks   = 500000
)

type BlockHead struct {
	// A struct that matches the exact format of blocks stored in blk*.dat files
	Magic      [4]byte
	Length     uint32
	Version    int32
	PrevHash   [32]byte
	MerkleRoot [32]byte
	Timestamp  uint32
	Difficulty uint32
	Nonce      uint32
}

type Block struct {
	// A custom block object for processing
	PrevBlock *Block
	NextBlock *Block
	Head      *BlockHead
	RelTxs    []*btcwire.MsgTx
	Hash      [32]byte
	depth     int
}

func check(err error) {
	if err != nil {
		logger.Fatal(err)
	}
}

func btcBHFromBH(bh BlockHead) *btcwire.BlockHeader {
	// utility function to convert custom BlockHead type to btcwire BlockHeader
	prevhash, _ := btcwire.NewShaHash(bh.PrevHash[:])
	merkle, _ := btcwire.NewShaHash(bh.MerkleRoot[:])
	timestamp := time.Unix(int64(bh.Timestamp), 0)

	btcbh := btcwire.BlockHeader{
		Version:    bh.Version,
		PrevBlock:  *prevhash,
		MerkleRoot: *merkle,
		Timestamp:  timestamp,
		Bits:       bh.Difficulty,
		Nonce:      bh.Nonce,
	}
	return &btcbh
}

func blockHash(bh BlockHead) [32]byte {
	// Print the hash of the block from the headers in the block
	btcbh := btcBHFromBH(bh)
	hash, _ := btcbh.BlockSha()
	return [32]byte(hash)
}

func proceed(f *os.File) bool {
	// finds the start of the next block and places the cursor on it
	for {
		var b [4]byte
		_, err := io.ReadFull(f, b[:])
		if err != nil {
			return false
		}
		discrim := binary.BigEndian.Uint32(b[:])
		if discrim != 0x00000000 {
			// seek backwards to start of block
			// TODO make more effecient
			f.Seek(-4, 1)
			return true
		}
	}
}

func playWithFile(fname string, blkList []*Block, blkMap map[[32]byte]*Block) ([]*Block, map[[32]byte]*Block) {
	// given a blk file attempts to parse every block within it. Adding the block
	// to a global list of seen blocks. Additionally we strip out the interesting
	// transactions at this stage.
	file, err := os.Open(fname)
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	seenGenesis := true
	if len(blkList) == 0 {
		seenGenesis = false
	}
	for {
		var blk Block
		var bh BlockHead

		ok := proceed(file)
		if !ok {
			fmt.Println("Hit end of file: ", fname)
			break
		}
		err = binary.Read(file, binary.LittleEndian, &bh)
		if err == io.EOF || err == io.ErrUnexpectedEOF {
			fmt.Println("At the end of file: ", fname)
			break
		}
		if err != nil {
			logger.Fatal(err)
		}

		tx_num, err := readVarInt(file, 0)
		if err != nil {
			logger.Fatal(err)
		}

		hash := blockHash(bh)
		for i := uint64(0); i < tx_num; i++ {
			tx := btcwire.MsgTx{}
			err := tx.Deserialize(file)
			if err != nil {
				logger.Fatal(err)
			}
		}

		blk = Block{
			//	PrevBlock: nil,
			Head:   &bh,
			RelTxs: make([]*btcwire.MsgTx, 0),
			Hash:   hash,
			depth:  1,
		}
		if !seenGenesis {
			seenGenesis = true
			genesisHash = hash
			// Make the hash of the genesis block useful
			var s [32]byte
			copy(s[:], hash[:])
			for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 {
				s[i], s[j] = s[j], s[i]
			}
			fmt.Printf("The hash of the genesis block:\n%x\n", s)
		}
		blkMap[hash] = &blk
		blkList = append(blkList, &blk)
	}

	return blkList, blkMap
}

func calcHeight(blkList []*Block, blkMap map[[32]byte]*Block) int {
	// Computes the best chain's total height by starting from the latest blocks
	// and working pack to the genesis block.
	for j := len(blkList) - 1; j >= 0; j-- {
		blk := blkList[j]
		//if blk.PrevBlock == nil && blk.Hash == genesisHash {
		if blk.Hash == genesisHash {
			println("Found Genesis Hash")
			return blk.depth
		}
		nextD := blk.depth + 1
		prevBlock, ok := blkMap[blk.Head.PrevHash]
		if ok {
			if prevBlock.depth < nextD {
				prevBlock.depth = nextD
			}
		}
	}
	return -1
}

func main() {
	flag.Parse()

	//copy(genesisHash[:], _genesisHash)
	glob := "/blk*.dat"
	blockfiles, err := filepath.Glob(*blockdir + glob)
	if err != nil {
		log.Fatal(err)
	}
	if len(blockfiles) < 1 {
		log.Fatal(errors.New("Could not find any blockfiles at " + *blockdir))
	}

	blkList := make([]*Block, 0, maxBlocks)
	blkMap := make(map[[32]byte]*Block)
	for _, filename := range blockfiles {
		println(filename)
		blkList, blkMap = playWithFile(filename, blkList, blkMap)
		fmt.Println("Processed:", len(blkList))
	}

	println("Finding blockchain tip")
	println(len(blkList))
	genesisBlk := linkChain(blkList, blkMap)
	tip, h := chainTip(genesisBlk)
	println("Height: ", h)
	printBlockHead(*tip.Head)
}

func linkChain(blkList []*Block, blkMap map[[32]byte]*Block) *Block {
	// Walks the block list backwards & builds out the linked list so that on a
	// walk back up we can return the block at the end of the longest chain
	absents := 0
	for j := len(blkList) - 1; j >= 0; j-- {
		blk := blkList[j]
		if blk.Hash == genesisHash {
			fmt.Println("Found Genesis Hash")
			break
		}

		prevBlk, ok := blkMap[blk.Head.PrevHash]
		if !ok {
			absents++
		} else {
			// this block points back to another block that we have in memory
			currentD := blk.depth + 1
			if prevBlk.depth < currentD {
				prevBlk.depth = currentD
				prevBlk.NextBlock = blk
				blk.PrevBlock = prevBlk
			}
		}
	}
	fmt.Printf("blocks with absent parents: %d\n", absents)
	genesisBlk, ok := blkMap[genesisHash]
	if !ok {
		logger.Fatal("Could not find the genesis block. Big problem!")
	}
	return genesisBlk
}

func chainTip(blk *Block) (*Block, int) {
	return recurseTip(blk, 0)
}

func recurseTip(blk *Block, confs int) (*Block, int) {
	if blk.NextBlock == nil {
		return blk, confs
	} else {
		//		println(blk.Head.Nonce)
		return recurseTip(blk.NextBlock, confs+1)
	}
}

// From btcwire common.go
func readVarInt(r io.Reader, pver uint32) (uint64, error) {
	// readVarInt reads a variable length integer from r and returns it as a uint64.
	var b [8]byte
	_, err := io.ReadFull(r, b[0:1])
	if err != nil {
		return 0, err
	}

	var rv uint64
	discriminant := uint8(b[0])
	switch discriminant {
	case 0xff:
		_, err := io.ReadFull(r, b[:])
		if err != nil {
			return 0, err
		}
		rv = binary.LittleEndian.Uint64(b[:])

	case 0xfe:
		_, err := io.ReadFull(r, b[0:4])
		if err != nil {
			return 0, err
		}
		rv = uint64(binary.LittleEndian.Uint32(b[:]))

	case 0xfd:
		_, err := io.ReadFull(r, b[0:2])
		if err != nil {
			return 0, err
		}
		rv = uint64(binary.LittleEndian.Uint16(b[:]))

	default:
		rv = uint64(discriminant)
	}

	return rv, nil
}

func printBlockHead(blk BlockHead) {
	// Prints out header from a given block

	prevhash, _ := btcwire.NewShaHash(blk.PrevHash[:])
	merkle, _ := btcwire.NewShaHash(blk.MerkleRoot[:])
	timestamp := time.Unix(int64(blk.Timestamp), 0)

	bh := btcwire.BlockHeader{
		Version:    blk.Version,
		PrevBlock:  *prevhash,
		MerkleRoot: *merkle,
		Timestamp:  timestamp,
		Bits:       blk.Difficulty,
		Nonce:      blk.Nonce,
	}
	hash, err := bh.BlockSha()
	check(err)
	fmt.Printf(`
Hash:		%s
prevHash:	%s
merkle root:	%s
timestamp:	%s
difficulty:	%d
nonce:		%d
bit len:	%d
==========-
`,
		hash, prevhash.String(), merkle.String(), timestamp, blk.Difficulty, blk.Nonce, blk.Length)
}
<|endoftext|>


<commit_before>package sdl

// #cgo darwin CFLAGS: -mmacosx-version-min=10.7
// #cgo linux freebsd darwin windows pkg-config: sdl2
// #cgo linux freebsd darwin windows LDFLAGS: -lSDL2_ttf -lSDL2_mixer -lSDL2_image -mmacosx-version-min=10.7
// #include "wrapper.h"
import "C"
import (
	"errors"
	"runtime"
	"unsafe"
)

func init() {
	initTTF()
	initMix()
	initImg()
}

// Point is a structure that defines a two demensional point.
// (https://wiki.libsdl.org/SDL_Point)
type point struct {
	X int32
	Y int32
}

func (p *point) cptr() *C.SDL_Point {
	return (*C.SDL_Point)(unsafe.Pointer(p))
}

// Rect is a structure that defines a rectangle, with the origin at the upper
// left.
// (https://wiki.libsdl.org/SDL_Rect)
type rect struct {
	X int32
	Y int32
	W int32
	H int32
}

func (a *rect) cptr() *C.SDL_Rect {
	return (*C.SDL_Rect)(unsafe.Pointer(a))
}

// Color defines a color using r, g, b, a values from 0-255
// (https://wiki.libsdl.org/SDL_Color)
type color struct {
	R uint8
	G uint8
	B uint8
	A uint8
}

func (a *color) cptr() *C.SDL_Color {
	return (*C.SDL_Color)(unsafe.Pointer(a))
}

// Surface SDL Surface (https://wiki.libsdl.org/SDL_Surface)
type surface C.SDL_Surface

func (a *surface) cptr() *C.SDL_Surface {
	return (*C.SDL_Surface)(unsafe.Pointer(a))
}

// mainThread manages the thread that SDL calls execute on
var mainThread = make(chan func())
var mainDone = make(chan struct{})

type callback func()

// lockMain calls runtime.LockOSThread on the calling thread.  This is intended to be the main thread since SDL on some platforms requires the main thread.  Use the MainThread channel to execute SDL calls.
// https://github.com/golang/go/wiki/LockOSThread
func lockMain(cb callback) {
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	go cb()
	for {
		select {
		case f := <-mainThread:
			f()
		case <-mainDone:
			return
		}
	}
}

// GetTicks (https://wiki.libsdl.org/SDL_GetTicks)
func getTicks() uint32 {
	return uint32(C.SDL_GetTicks())
}

// Delay (https://wiki.libsdl.org/SDL_Delay)
func delay(ms uint32) {
	C.SDL_Delay(C.Uint32(ms))
}

// GetError (https://wiki.libsdl.org/SDL_GetError)
func getError() error {
	if err := C.SDL_GetError(); err != nil {
		return errors.New(C.GoString(err))
	}
	return nil
}

// getVersion gets the current SDL version (https://wiki.libsdl.org/SDL_GetVersion) return values are major, minor, patch
func getVersion() (int, int, int) {
	var version C.SDL_version
	C.SDL_GetVersion(&version)

	return int(version.major), int(version.minor), int(version.patch)
}

// ClearError (https://wiki.libsdl.org/SDL_ClearError)
func clearError() {
	C.SDL_ClearError()
}

// FreeSurface (https://wiki.libsdl.org/SDL_FreeSurface)
func freeSurface(s *surface) {
	C.SDL_FreeSurface(s.cptr())
}

func quit() {
	C.Mix_Quit()
	C.TTF_Quit()
	C.IMG_Quit()
	C.SDL_Quit()
}
<commit_msg>Fixed the cgo flags for the macosx version<commit_after>package sdl

// #cgo darwin CFLAGS: -mmacosx-version-min=10.7
// #cgo linux freebsd darwin windows pkg-config: sdl2
// #cgo darwin LDFLAGS: -lSDL2_ttf -lSDL2_mixer -lSDL2_image -mmacosx-version-min=10.7
// #cgo linux freebsd windows LDFLAGS: -lSDL2_ttf -lSDL2_mixer -lSDL2_image
// #include "wrapper.h"
import "C"
import (
	"errors"
	"runtime"
	"unsafe"
)

func init() {
	initTTF()
	initMix()
	initImg()
}

// Point is a structure that defines a two demensional point.
// (https://wiki.libsdl.org/SDL_Point)
type point struct {
	X int32
	Y int32
}

func (p *point) cptr() *C.SDL_Point {
	return (*C.SDL_Point)(unsafe.Pointer(p))
}

// Rect is a structure that defines a rectangle, with the origin at the upper
// left.
// (https://wiki.libsdl.org/SDL_Rect)
type rect struct {
	X int32
	Y int32
	W int32
	H int32
}

func (a *rect) cptr() *C.SDL_Rect {
	return (*C.SDL_Rect)(unsafe.Pointer(a))
}

// Color defines a color using r, g, b, a values from 0-255
// (https://wiki.libsdl.org/SDL_Color)
type color struct {
	R uint8
	G uint8
	B uint8
	A uint8
}

func (a *color) cptr() *C.SDL_Color {
	return (*C.SDL_Color)(unsafe.Pointer(a))
}

// Surface SDL Surface (https://wiki.libsdl.org/SDL_Surface)
type surface C.SDL_Surface

func (a *surface) cptr() *C.SDL_Surface {
	return (*C.SDL_Surface)(unsafe.Pointer(a))
}

// mainThread manages the thread that SDL calls execute on
var mainThread = make(chan func())
var mainDone = make(chan struct{})

type callback func()

// lockMain calls runtime.LockOSThread on the calling thread.  This is intended to be the main thread since SDL on some platforms requires the main thread.  Use the MainThread channel to execute SDL calls.
// https://github.com/golang/go/wiki/LockOSThread
func lockMain(cb callback) {
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	go cb()
	for {
		select {
		case f := <-mainThread:
			f()
		case <-mainDone:
			return
		}
	}
}

// GetTicks (https://wiki.libsdl.org/SDL_GetTicks)
func getTicks() uint32 {
	return uint32(C.SDL_GetTicks())
}

// Delay (https://wiki.libsdl.org/SDL_Delay)
func delay(ms uint32) {
	C.SDL_Delay(C.Uint32(ms))
}

// GetError (https://wiki.libsdl.org/SDL_GetError)
func getError() error {
	if err := C.SDL_GetError(); err != nil {
		return errors.New(C.GoString(err))
	}
	return nil
}

// getVersion gets the current SDL version (https://wiki.libsdl.org/SDL_GetVersion) return values are major, minor, patch
func getVersion() (int, int, int) {
	var version C.SDL_version
	C.SDL_GetVersion(&version)

	return int(version.major), int(version.minor), int(version.patch)
}

// ClearError (https://wiki.libsdl.org/SDL_ClearError)
func clearError() {
	C.SDL_ClearError()
}

// FreeSurface (https://wiki.libsdl.org/SDL_FreeSurface)
func freeSurface(s *surface) {
	C.SDL_FreeSurface(s.cptr())
}

func quit() {
	C.Mix_Quit()
	C.TTF_Quit()
	C.IMG_Quit()
	C.SDL_Quit()
}
<|endoftext|>


<commit_before>package actor

import "sync/atomic"
import "strconv"

type HostResolver func(*PID) (ActorRef, bool)
type ProcessRegistryValue struct {
	Host           string
	LocalPids      map[string]ActorRef
	RemoteHandlers []HostResolver
	SequenceID     uint64
}

var ProcessRegistry = &ProcessRegistryValue{
	Host:           "nonhost",
	LocalPids:      make(map[string]ActorRef),
	RemoteHandlers: make([]HostResolver, 0),
}

func (pr *ProcessRegistryValue) RegisterHostResolver(handler HostResolver) {
	pr.RemoteHandlers = append(pr.RemoteHandlers, handler)
}

func (pr *ProcessRegistryValue) registerPID(actorRef ActorRef) *PID {
	id := atomic.AddUint64(&pr.SequenceID, 1)

	pid := PID{
		Host: pr.Host,
		Id:   strconv.FormatUint(id, 16),
	}

	pr.LocalPids[pid.Id] = actorRef
	return &pid
}

func (pr *ProcessRegistryValue) fromPID(pid *PID) (ActorRef, bool) {
	if pid.Host != pr.Host {
		for _, handler := range pr.RemoteHandlers {
			ref, ok := handler(pid)
			if ok {
				return ref, true
			}
		}
		panic("Unknown host or node")
		return deadLetter, false
	}
	ref, ok := pr.LocalPids[pid.Id]
	if !ok {
		panic("Unknown PID")
		return deadLetter, false
	}
	return ref, true
}

func (pr *ProcessRegistryValue) Register(name string, pid *PID) {
	ref, _ := pr.fromPID(pid)
	pr.LocalPids[name] = ref
}
<commit_msg>rw lock on processes<commit_after>package actor

import (
	"sync"
	"sync/atomic"
)
import "strconv"

type HostResolver func(*PID) (ActorRef, bool)
type ProcessRegistryValue struct {
	Host           string
	LocalPids      map[string]ActorRef  //maybe this should be replaced with something lockfree like ctrie instead
	RemoteHandlers []HostResolver
	SequenceID     uint64
	rw             sync.RWMutex
}

var ProcessRegistry = &ProcessRegistryValue{
	Host:           "nonhost",
	LocalPids:      make(map[string]ActorRef),
	RemoteHandlers: make([]HostResolver, 0),
}

func (pr *ProcessRegistryValue) RegisterHostResolver(handler HostResolver) {
	pr.RemoteHandlers = append(pr.RemoteHandlers, handler)
}

func (pr *ProcessRegistryValue) registerPID(actorRef ActorRef) *PID {
	id := atomic.AddUint64(&pr.SequenceID, 1)

	pid := PID{
		Host: pr.Host,
		Id:   strconv.FormatUint(id, 16),
	}

	pr.rw.Lock()
	defer pr.rw.Unlock()
	pr.LocalPids[pid.Id] = actorRef
	return &pid
}

func (pr *ProcessRegistryValue) fromPID(pid *PID) (ActorRef, bool) {
	if pid.Host != pr.Host {
		for _, handler := range pr.RemoteHandlers {
			ref, ok := handler(pid)
			if ok {
				return ref, true
			}
		}
		panic("Unknown host or node")
		return deadLetter, false
	}
	pr.rw.RLock()
	defer pr.rw.RUnlock()
	ref, ok := pr.LocalPids[pid.Id]
	if !ok {
		panic("Unknown PID")
		return deadLetter, false
	}
	return ref, true
}

func (pr *ProcessRegistryValue) Register(name string, pid *PID) {
	ref, _ := pr.fromPID(pid)
	pr.LocalPids[name] = ref
}
<|endoftext|>


<commit_before>package lytics

import (
	"fmt"
	"net/url"
	"strconv"
	"strings"
	"time"
)

const (
	segmentEndpoint               = "segment/:id"
	segmentListEndpoint           = "segment"
	segmentSizeEndpoint           = "segment/:id/sizes"
	segmentSizesEndpoint          = "segment/sizes"       // ids
	segmentAttributionEndpoint    = "segment/attribution" // ids
	segmentScanEndpoint           = "segment/:id/scan"
	segmentCollectionListEndpoint = "segmentcollection"
	segmentCollectionEndpoint     = "segmentcollection/:id"
)

type Segment struct {
	Id          string    `json:"id"`
	Aid         int       `json:"aid"` // Deprecated; use AccountId
	AccountId   string    `json:"account_id"`
	ShortId     string    `json:"short_id,omitempty"`
	Name        string    `json:"name"`
	IsPublic    bool      `json:"is_public"`
	PublicName  string    `json:"public_name,omitempty"`
	SlugName    string    `json:"slug_name"`
	Description string    `json:"description,omitempty"`
	Table       string    `json:"table,omitempty"`
	AuthorId    string    `json:"author_id"`
	Updated     time.Time `json:"updated" bson:"updated"`
	Created     time.Time `json:"created" bson:"created"`
	SegType     string    `json:"op"`
	Negate      bool      `json:"negate"`
	Tags        []string  `json:"tags"`
}

type SegmentSize struct {
	Id       string  `json:"id"`
	Name     string  `json:"name"`
	SlugName string  `json:"slug_name"`
	Size     float64 `json:"size"`
}

type SegmentAttribution struct {
	Id      string                      `json:"id"`
	Metrics []SegmentAttributionMetrics `json:"metrics"`
}

type SegmentAttributionMetrics struct {
	Value   int64   `json:"value"`
	Ts      string  `json:"ts"`
	Anomaly float64 `json:"anomaly"`
}

type SegmentCollection struct {
	Aid           int               `json:"aid"`
	AccountId     string            `json:"account_id"`
	Id            string            `json:"id"`
	Name          string            `json:"name"`
	Slug          string            `json:"slug_name"`
	Description   string            `json:"description,omitempty"`
	Table         string            `json:"table,omitempty"`
	AuthorId      string            `json:"author_id"`
	Updated       time.Time         `json:"updated"`
	Created       time.Time         `json:"created"`
	Internal      bool              `json:"internal"`
	Collection    []*SegColRelation `json:"collection""`
	ParentSegment string            `json:"parent_segment"`
}

type SegColRelation struct {
	Id    string `json:"id"`
	Order int    `json:"order"`
}

type SegmentScanner struct {
	SegmentID string
	Next      string
	Previous  string
	Loader    chan []Entity
	Shutdown  chan bool
	Total     int
	Batches   []int
}

// Created is a helper method to convert the timestamp into human readable format for metrics
func (s *SegmentAttributionMetrics) Created() (time.Time, error) {
	return parseLyticsTime(s.Ts)
}

// GetSegment returns the details for a single segment based on id
// https://www.getlytics.com/developers/rest-api#segment
func (l *Client) GetSegment(id string) (Segment, error) {
	res := ApiResp{}
	data := Segment{}

	// make the request
	err := l.Get(parseLyticsURL(segmentEndpoint, map[string]string{"id": id}), nil, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// GetSegments returns a list of all segments for an account
// https://www.getlytics.com/developers/rest-api#segment-list
func (l *Client) GetSegments() ([]Segment, error) {
	res := ApiResp{}
	data := []Segment{}

	// make the request
	err := l.Get(segmentListEndpoint, nil, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// GetSegmentSize returns the segment size information for a single segment
// https://www.getlytics.com/developers/rest-api#segment-sizes
func (l *Client) GetSegmentSize(id string) (SegmentSize, error) {
	res := ApiResp{}
	data := SegmentSize{}

	// make the request
	err := l.Get(parseLyticsURL(segmentSizeEndpoint, map[string]string{"id": id}), nil, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// GetSegmentSizes returns the segment sizes for all segments on an account
// https://www.getlytics.com/developers/rest-api#segment-sizes
func (l *Client) GetSegmentSizes(segments []string) ([]SegmentSize, error) {
	params := url.Values{}
	res := ApiResp{}
	data := []SegmentSize{}

	// if we have specific segments to filter by add those to the params as comma separated string
	if len(segments) > 0 {
		params.Add("ids", strings.Join(segments, ","))
	}

	// make the request
	err := l.Get(segmentSizesEndpoint, params, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// GetSegmentAttribution returns the attribution (change over time) for segments
// method accepts a string slice of 1 or more segments to query.
// NOT CURRENTLY DOCUMENTED
func (l *Client) GetSegmentAttribution(segments []string) ([]SegmentAttribution, error) {
	params := url.Values{}

	res := ApiResp{}
	data := []SegmentAttribution{}

	// if the request is for a specific set of segments add that as comma separated param
	if len(segments) > 0 {
		params.Add("ids", strings.Join(segments, ","))
	}

	// make the request
	err := l.Get(parseLyticsURL(segmentAttributionEndpoint, nil), params, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// GetSegmentCollection returns a single collection of segments
// (a grouped/named lists of segments)
func (l *Client) GetSegmentCollection(id string) (SegmentCollection, error) {
	res := ApiResp{}
	data := SegmentCollection{}

	err := l.Get(parseLyticsURL(segmentCollectionEndpoint, map[string]string{"id": id}), nil, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// GetSegmentCollectionList returns a list of all segment
// collections for an account
func (l *Client) GetSegmentCollectionList() ([]SegmentCollection, error) {
	res := ApiResp{}
	data := []SegmentCollection{}

	err := l.Get(parseLyticsURL(segmentCollectionListEndpoint, nil), nil, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// Other Available Endpoints
// * POST    create segment
// * PUT     update segment
// * DELETE  remove segment

// **************************** START OF SEGMENT SCAN METHODS ****************************

// GetSegmentEntities returns a single page of entities (20max) for the given segment
// also returns the next value if there are more than 20 entities in the segment
// https://www.getlytics.com/developers/rest-api#segment-scan
func (l *Client) GetSegmentEntities(segment, next string, limit int) (interface{}, string, []Entity, error) {
	res := ApiResp{}
	data := []Entity{}
	params := url.Values{}

	params.Add("start", next)
	params.Add("limit", strconv.Itoa(limit))

	// make the request
	err := l.Get(parseLyticsURL(segmentScanEndpoint, map[string]string{"id": segment}), params, nil, &res, &data)
	if err != nil {
		return "", "", data, err
	}

	return res.Status, res.Next, data, nil
}

// CreateScanner generates a segment scanner so that we can process entities as they are loaded
// reduces the load as some segments can have hundreds of thousands of users and it is impractical
// to return a single result
func (l *Client) CreateScanner() error {
	scanner := SegmentScanner{}

	// create loader and shutdown
	loader := make(chan []Entity)
	shutdown := make(chan bool)

	// save to scanner
	scanner.Loader = loader
	scanner.Shutdown = shutdown

	// add the scanner to the client
	l.Scan = &scanner

	return nil
}

// LoadEntity does the heavy lifting when it comes to paging. It loops through all available pages
// and emits the batch of entities to be processed along the way. Also maintains a slice of batch
// counts and total count to help with debugging and reporting.
func (l *Client) LoadEntity() {
	var (
		entities []Entity
		err      error
		fails    int
		maxTries int
	)

	maxTries = 10

	// make calls for next batch of segments until we run out of next params
	for {
		_, l.Scan.Next, entities, err = l.GetSegmentEntities(l.Scan.SegmentID, l.Scan.Next, 100)
		if err != nil {
			fails++

			if fails > maxTries {
				panic(fmt.Sprintf("Failed to get entities, exceeded max tries(%d): %v", maxTries, err))
			}

			// if we fail and have not exceeded the limit, try again
			continue
		}

		// for logging add the batch details to the scanner
		l.Scan.Batches = append(l.Scan.Batches, len(entities))

		// for logging add the total entites returned to the scanner
		l.Scan.Total = l.Scan.Total + len(entities)

		// emit the entities to the loader for processing
		l.Scan.Loader <- entities

		// if there are no more pages we will have a blank next, just break and return
		if l.Scan.Next == "" {
			break
		}
	}

	// everything worked, shutdown safely
	l.Scan.Shutdown <- true
}

// PageMembers sets the segment on the master scanner and initiates the main go routine
// for paging
func (l *Client) PageMembers(segment string) error {
	// save the target segment on the scanner
	l.Scan.SegmentID = segment

	// fire up the go routine for paging entities
	go l.LoadEntity()
	return nil
}
<commit_msg>Add segment kind field.<commit_after>package lytics

import (
	"fmt"
	"net/url"
	"strconv"
	"strings"
	"time"
)

const (
	segmentEndpoint               = "segment/:id"
	segmentListEndpoint           = "segment"
	segmentSizeEndpoint           = "segment/:id/sizes"
	segmentSizesEndpoint          = "segment/sizes"       // ids
	segmentAttributionEndpoint    = "segment/attribution" // ids
	segmentScanEndpoint           = "segment/:id/scan"
	segmentCollectionListEndpoint = "segmentcollection"
	segmentCollectionEndpoint     = "segmentcollection/:id"
)

type Segment struct {
	Id          string    `json:"id"`
	Aid         int       `json:"aid"` // Deprecated; use AccountId
	AccountId   string    `json:"account_id"`
	ShortId     string    `json:"short_id,omitempty"`
	Name        string    `json:"name"`
	IsPublic    bool      `json:"is_public"`
	PublicName  string    `json:"public_name,omitempty"`
	SlugName    string    `json:"slug_name"`
	Description string    `json:"description,omitempty"`
	Kind        string    `json:"kind,omitempty"`
	Table       string    `json:"table,omitempty"`
	AuthorId    string    `json:"author_id"`
	Updated     time.Time `json:"updated" bson:"updated"`
	Created     time.Time `json:"created" bson:"created"`
	SegType     string    `json:"op"`
	Negate      bool      `json:"negate"`
	Tags        []string  `json:"tags"`
}

type SegmentSize struct {
	Id       string  `json:"id"`
	Name     string  `json:"name"`
	SlugName string  `json:"slug_name"`
	Size     float64 `json:"size"`
}

type SegmentAttribution struct {
	Id      string                      `json:"id"`
	Metrics []SegmentAttributionMetrics `json:"metrics"`
}

type SegmentAttributionMetrics struct {
	Value   int64   `json:"value"`
	Ts      string  `json:"ts"`
	Anomaly float64 `json:"anomaly"`
}

type SegmentCollection struct {
	Aid           int               `json:"aid"`
	AccountId     string            `json:"account_id"`
	Id            string            `json:"id"`
	Name          string            `json:"name"`
	Slug          string            `json:"slug_name"`
	Description   string            `json:"description,omitempty"`
	Table         string            `json:"table,omitempty"`
	AuthorId      string            `json:"author_id"`
	Updated       time.Time         `json:"updated"`
	Created       time.Time         `json:"created"`
	Internal      bool              `json:"internal"`
	Collection    []*SegColRelation `json:"collection""`
	ParentSegment string            `json:"parent_segment"`
}

type SegColRelation struct {
	Id    string `json:"id"`
	Order int    `json:"order"`
}

type SegmentScanner struct {
	SegmentID string
	Next      string
	Previous  string
	Loader    chan []Entity
	Shutdown  chan bool
	Total     int
	Batches   []int
}

// Created is a helper method to convert the timestamp into human readable format for metrics
func (s *SegmentAttributionMetrics) Created() (time.Time, error) {
	return parseLyticsTime(s.Ts)
}

// GetSegment returns the details for a single segment based on id
// https://www.getlytics.com/developers/rest-api#segment
func (l *Client) GetSegment(id string) (Segment, error) {
	res := ApiResp{}
	data := Segment{}

	// make the request
	err := l.Get(parseLyticsURL(segmentEndpoint, map[string]string{"id": id}), nil, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// GetSegments returns a list of all segments for an account
// https://www.getlytics.com/developers/rest-api#segment-list
func (l *Client) GetSegments() ([]Segment, error) {
	res := ApiResp{}
	data := []Segment{}

	// make the request
	err := l.Get(segmentListEndpoint, nil, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// GetSegmentSize returns the segment size information for a single segment
// https://www.getlytics.com/developers/rest-api#segment-sizes
func (l *Client) GetSegmentSize(id string) (SegmentSize, error) {
	res := ApiResp{}
	data := SegmentSize{}

	// make the request
	err := l.Get(parseLyticsURL(segmentSizeEndpoint, map[string]string{"id": id}), nil, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// GetSegmentSizes returns the segment sizes for all segments on an account
// https://www.getlytics.com/developers/rest-api#segment-sizes
func (l *Client) GetSegmentSizes(segments []string) ([]SegmentSize, error) {
	params := url.Values{}
	res := ApiResp{}
	data := []SegmentSize{}

	// if we have specific segments to filter by add those to the params as comma separated string
	if len(segments) > 0 {
		params.Add("ids", strings.Join(segments, ","))
	}

	// make the request
	err := l.Get(segmentSizesEndpoint, params, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// GetSegmentAttribution returns the attribution (change over time) for segments
// method accepts a string slice of 1 or more segments to query.
// NOT CURRENTLY DOCUMENTED
func (l *Client) GetSegmentAttribution(segments []string) ([]SegmentAttribution, error) {
	params := url.Values{}

	res := ApiResp{}
	data := []SegmentAttribution{}

	// if the request is for a specific set of segments add that as comma separated param
	if len(segments) > 0 {
		params.Add("ids", strings.Join(segments, ","))
	}

	// make the request
	err := l.Get(parseLyticsURL(segmentAttributionEndpoint, nil), params, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// GetSegmentCollection returns a single collection of segments
// (a grouped/named lists of segments)
func (l *Client) GetSegmentCollection(id string) (SegmentCollection, error) {
	res := ApiResp{}
	data := SegmentCollection{}

	err := l.Get(parseLyticsURL(segmentCollectionEndpoint, map[string]string{"id": id}), nil, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// GetSegmentCollectionList returns a list of all segment
// collections for an account
func (l *Client) GetSegmentCollectionList() ([]SegmentCollection, error) {
	res := ApiResp{}
	data := []SegmentCollection{}

	err := l.Get(parseLyticsURL(segmentCollectionListEndpoint, nil), nil, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// Other Available Endpoints
// * POST    create segment
// * PUT     update segment
// * DELETE  remove segment

// **************************** START OF SEGMENT SCAN METHODS ****************************

// GetSegmentEntities returns a single page of entities (20max) for the given segment
// also returns the next value if there are more than 20 entities in the segment
// https://www.getlytics.com/developers/rest-api#segment-scan
func (l *Client) GetSegmentEntities(segment, next string, limit int) (interface{}, string, []Entity, error) {
	res := ApiResp{}
	data := []Entity{}
	params := url.Values{}

	params.Add("start", next)
	params.Add("limit", strconv.Itoa(limit))

	// make the request
	err := l.Get(parseLyticsURL(segmentScanEndpoint, map[string]string{"id": segment}), params, nil, &res, &data)
	if err != nil {
		return "", "", data, err
	}

	return res.Status, res.Next, data, nil
}

// CreateScanner generates a segment scanner so that we can process entities as they are loaded
// reduces the load as some segments can have hundreds of thousands of users and it is impractical
// to return a single result
func (l *Client) CreateScanner() error {
	scanner := SegmentScanner{}

	// create loader and shutdown
	loader := make(chan []Entity)
	shutdown := make(chan bool)

	// save to scanner
	scanner.Loader = loader
	scanner.Shutdown = shutdown

	// add the scanner to the client
	l.Scan = &scanner

	return nil
}

// LoadEntity does the heavy lifting when it comes to paging. It loops through all available pages
// and emits the batch of entities to be processed along the way. Also maintains a slice of batch
// counts and total count to help with debugging and reporting.
func (l *Client) LoadEntity() {
	var (
		entities []Entity
		err      error
		fails    int
		maxTries int
	)

	maxTries = 10

	// make calls for next batch of segments until we run out of next params
	for {
		_, l.Scan.Next, entities, err = l.GetSegmentEntities(l.Scan.SegmentID, l.Scan.Next, 100)
		if err != nil {
			fails++

			if fails > maxTries {
				panic(fmt.Sprintf("Failed to get entities, exceeded max tries(%d): %v", maxTries, err))
			}

			// if we fail and have not exceeded the limit, try again
			continue
		}

		// for logging add the batch details to the scanner
		l.Scan.Batches = append(l.Scan.Batches, len(entities))

		// for logging add the total entites returned to the scanner
		l.Scan.Total = l.Scan.Total + len(entities)

		// emit the entities to the loader for processing
		l.Scan.Loader <- entities

		// if there are no more pages we will have a blank next, just break and return
		if l.Scan.Next == "" {
			break
		}
	}

	// everything worked, shutdown safely
	l.Scan.Shutdown <- true
}

// PageMembers sets the segment on the master scanner and initiates the main go routine
// for paging
func (l *Client) PageMembers(segment string) error {
	// save the target segment on the scanner
	l.Scan.SegmentID = segment

	// fire up the go routine for paging entities
	go l.LoadEntity()
	return nil
}
<|endoftext|>


<commit_before>package bitfinex

import (
	"bytes"
	"crypto/md5"
	"fmt"
	"strings"
)

// Ticker represents a payload for a ticker for a trading pair.
type Ticker struct {
	Symbol              string  `json:"symbol,omitempty"`
	Bid                 float32 `json:"bid,omitempty"`
	BidSize             float32 `json:"bid_size,omitempty"`
	Ask                 float32 `json:"ask,omitempty"`
	AskSize             float32 `json:"ask_size,omitempty"`
	DailyChange         float32 `json:"daily_change,omitempty"`
	DailyChangeRelative float32 `json:"daily_change_rel,omitempty"`
	LastPrice           float32 `json:"last_price,omitempty"`
	Volume              float32 `json:"vol,omitempty"`
	High                float32 `json:"high,omitempty"`
	Low                 float32 `json:"low,omitempty"`
	Hash                string  `json:"-"`
}

// ParseTickers parses an array of Tickers.
func ParseTickers(body []byte) ([]Ticker, error) {
	// remove first/last square braces
	body = body[1 : len(body)-2]

	// split into records
	parts := bytes.Split(body, []byte("]"))

	records := [][]byte{}

	for i, r := range parts {
		if i == 0 {
			// first record is clean
			records = append(records, r)
			continue
		}

		// empty
		if len(r) == 0 {
			continue
		}

		// skip the first byte
		records = append(records, r[1:len(r)])
	}

	tickers := make([]Ticker, len(records), len(records))

	for i, r := range records {
		t, err := ParseTicker(r)
		if err != nil {
			return nil, err
		}

		tickers[i] = *t
	}

	return tickers, nil
}

// ParseTicker parses a single Ticker.
func ParseTicker(b []byte) (*Ticker, error) {
	parts := strings.Split(string(b), ",")

	bid, err := parseFloat32(parts[1])
	if err != nil {
		return nil, err
	}

	bidSize, err := parseFloat32(parts[2])
	if err != nil {
		return nil, err
	}

	ask, err := parseFloat32(parts[3])
	if err != nil {
		return nil, err
	}

	askSize, err := parseFloat32(parts[4])
	if err != nil {
		return nil, err
	}

	dailyChange, err := parseFloat32(parts[5])
	if err != nil {
		return nil, err
	}

	dailyChangeRelative, err := parseFloat32(parts[6])
	if err != nil {
		return nil, err
	}

	lastPrice, err := parseFloat32(parts[7])
	if err != nil {
		return nil, err
	}

	volume, err := parseFloat32(parts[8])
	if err != nil {
		return nil, err
	}

	high, err := parseFloat32(parts[9])
	if err != nil {
		return nil, err
	}

	low, err := parseFloat32(parts[10])
	if err != nil {
		return nil, err
	}

	// add a hash of the data to make it easy to check if a Ticker hash changed.
	hash := md5.Sum(b)

	t := Ticker{
		Symbol:              stripStringData(parts[0]),
		Bid:                 bid,
		BidSize:             bidSize,
		Ask:                 ask,
		AskSize:             askSize,
		DailyChange:         dailyChange,
		DailyChangeRelative: dailyChangeRelative,
		LastPrice:           lastPrice,
		Volume:              volume,
		High:                high,
		Low:                 low,
		Hash:                fmt.Sprintf("%x", hash),
	}

	return &t, nil
}
<commit_msg>Return error when attempting to parse an empty ticker body.<commit_after>package bitfinex

import (
	"bytes"
	"crypto/md5"
	"errors"
	"fmt"
	"strings"
)

var (
	// ErrEmptyBody is returned when an empty body is encountered during parsing.
	ErrEmptyBody = errors.New("Empty body")
)

// Ticker represents a payload for a ticker for a trading pair.
type Ticker struct {
	Symbol              string  `json:"symbol,omitempty"`
	Bid                 float32 `json:"bid,omitempty"`
	BidSize             float32 `json:"bid_size,omitempty"`
	Ask                 float32 `json:"ask,omitempty"`
	AskSize             float32 `json:"ask_size,omitempty"`
	DailyChange         float32 `json:"daily_change,omitempty"`
	DailyChangeRelative float32 `json:"daily_change_rel,omitempty"`
	LastPrice           float32 `json:"last_price,omitempty"`
	Volume              float32 `json:"vol,omitempty"`
	High                float32 `json:"high,omitempty"`
	Low                 float32 `json:"low,omitempty"`
	Hash                string  `json:"-"`
}

// ParseTickers parses an array of Tickers.
func ParseTickers(body []byte) ([]Ticker, error) {
	if len(body) == 0 {
		return nil, ErrEmptyBody
	}

	// remove first/last square braces
	body = body[1 : len(body)-2]

	// split into records
	parts := bytes.Split(body, []byte("]"))

	records := [][]byte{}

	for i, r := range parts {
		if i == 0 {
			// first record is clean
			records = append(records, r)
			continue
		}

		// empty
		if len(r) == 0 {
			continue
		}

		// skip the first byte
		records = append(records, r[1:len(r)])
	}

	tickers := make([]Ticker, len(records), len(records))

	for i, r := range records {
		t, err := ParseTicker(r)
		if err != nil {
			return nil, err
		}

		tickers[i] = *t
	}

	return tickers, nil
}

// ParseTicker parses a single Ticker.
func ParseTicker(b []byte) (*Ticker, error) {
	parts := strings.Split(string(b), ",")

	bid, err := parseFloat32(parts[1])
	if err != nil {
		return nil, err
	}

	bidSize, err := parseFloat32(parts[2])
	if err != nil {
		return nil, err
	}

	ask, err := parseFloat32(parts[3])
	if err != nil {
		return nil, err
	}

	askSize, err := parseFloat32(parts[4])
	if err != nil {
		return nil, err
	}

	dailyChange, err := parseFloat32(parts[5])
	if err != nil {
		return nil, err
	}

	dailyChangeRelative, err := parseFloat32(parts[6])
	if err != nil {
		return nil, err
	}

	lastPrice, err := parseFloat32(parts[7])
	if err != nil {
		return nil, err
	}

	volume, err := parseFloat32(parts[8])
	if err != nil {
		return nil, err
	}

	high, err := parseFloat32(parts[9])
	if err != nil {
		return nil, err
	}

	low, err := parseFloat32(parts[10])
	if err != nil {
		return nil, err
	}

	// add a hash of the data to make it easy to check if a Ticker hash changed.
	hash := md5.Sum(b)

	t := Ticker{
		Symbol:              stripStringData(parts[0]),
		Bid:                 bid,
		BidSize:             bidSize,
		Ask:                 ask,
		AskSize:             askSize,
		DailyChange:         dailyChange,
		DailyChangeRelative: dailyChangeRelative,
		LastPrice:           lastPrice,
		Volume:              volume,
		High:                high,
		Low:                 low,
		Hash:                fmt.Sprintf("%x", hash),
	}

	return &t, nil
}
<|endoftext|>


<commit_before>// Copyright © 2016 Abcum Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package orbit

import (
	"time"

	"github.com/robertkrimen/otto"
)

// Task represents a task on the timer channel
type Task struct {
	timer    *time.Timer
	interval bool
	duration time.Duration
	function otto.FunctionCall
}

func quit(ctx *Orbit) {
	ctx.Interrupt = make(chan func(), 1)
	if ctx.timeout > 0 {
		go func() {
			time.Sleep(ctx.timeout * time.Millisecond)
			ctx.Interrupt <- func() {}
		}()
	}
}

func wait(ctx *Orbit) {

	for {

		select {
		case <-ctx.Interrupt:
			for timer := range ctx.timers {
				delete(ctx.timers, timer)
			}
		case timer := <-ctx.loop:
			var arguments []interface{}
			if len(timer.function.ArgumentList) > 2 {
				tmp := timer.function.ArgumentList[2:]
				arguments = make([]interface{}, 2+len(tmp))
				for i, value := range tmp {
					arguments[i+2] = value
				}
			} else {
				arguments = make([]interface{}, 1)
			}
			arguments[0] = timer.function.ArgumentList[0]
			_, err := ctx.Call(`Function.call.call`, nil, arguments...)
			if err != nil {
				for _, timer := range ctx.timers {
					timer.timer.Stop()
					delete(ctx.timers, timer)
					panic(err)
				}
			}
			if timer.interval {
				timer.timer.Reset(timer.duration)
			} else {
				delete(ctx.timers, timer)
			}
		default:
			// Escape valve!
		}

		if len(ctx.timers) == 0 {
			break
		}

	}

}

func init() {

	OnInit(func(ctx *Orbit) {

		ctx.Set("setTimeout", func(call otto.FunctionCall) otto.Value {

			delay, _ := call.Argument(1).ToInteger()
			if delay <= 0 {
				delay = 1
			}

			timer := &Task{
				function: call,
				interval: false,
				duration: time.Duration(delay) * time.Millisecond,
			}

			ctx.timers[timer] = timer

			timer.timer = time.AfterFunc(timer.duration, func() {
				ctx.loop <- timer
			})

			val, err := call.Otto.ToValue(timer)
			if err != nil {
				panic(err)
			}

			return val

		})

		ctx.Set("setInterval", func(call otto.FunctionCall) otto.Value {

			delay, _ := call.Argument(1).ToInteger()
			if delay <= 0 {
				delay = 1
			}

			timer := &Task{
				function: call,
				interval: true,
				duration: time.Duration(delay) * time.Millisecond,
			}

			ctx.timers[timer] = timer

			timer.timer = time.AfterFunc(timer.duration, func() {
				ctx.loop <- timer
			})

			val, err := call.Otto.ToValue(timer)
			if err != nil {
				panic(err)
			}

			return val

		})

		ctx.Set("setImmediate", func(call otto.FunctionCall) otto.Value {

			timer := &Task{
				function: call,
				interval: false,
				duration: time.Millisecond,
			}

			ctx.timers[timer] = timer

			timer.timer = time.AfterFunc(timer.duration, func() {
				ctx.loop <- timer
			})

			val, err := call.Otto.ToValue(timer)
			if err != nil {
				panic(err)
			}

			return val

		})

		ctx.Set("clearTimeout", func(call otto.FunctionCall) otto.Value {
			timer, _ := call.Argument(0).Export()
			if timer, ok := timer.(*Task); ok {
				timer.timer.Stop()
				delete(ctx.timers, timer)
			}
			return otto.UndefinedValue()
		})

		ctx.Set("clearImmediate", func(call otto.FunctionCall) otto.Value {
			timer, _ := call.Argument(0).Export()
			if timer, ok := timer.(*Task); ok {
				timer.timer.Stop()
				delete(ctx.timers, timer)
			}
			return otto.UndefinedValue()
		})

	})

}
<commit_msg>Don't panic on timer failure but notify callbacks with error and quit<commit_after>// Copyright © 2016 Abcum Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package orbit

import (
	"time"

	"github.com/robertkrimen/otto"
)

// Task represents a task on the timer channel
type Task struct {
	timer    *time.Timer
	interval bool
	duration time.Duration
	function otto.FunctionCall
}

func quit(ctx *Orbit) {
	ctx.Interrupt = make(chan func(), 1)
	if ctx.timeout > 0 {
		go func() {
			time.Sleep(ctx.timeout * time.Millisecond)
			ctx.Interrupt <- func() {}
		}()
	}
}

func wait(ctx *Orbit) {

	for {

		select {
		case <-ctx.Interrupt:
			for timer := range ctx.timers {
				delete(ctx.timers, timer)
			}
		case timer := <-ctx.loop:
			var arguments []interface{}
			if len(timer.function.ArgumentList) > 2 {
				tmp := timer.function.ArgumentList[2:]
				arguments = make([]interface{}, 2+len(tmp))
				for i, value := range tmp {
					arguments[i+2] = value
				}
			} else {
				arguments = make([]interface{}, 1)
			}
			arguments[0] = timer.function.ArgumentList[0]
			_, err := ctx.Call(`Function.call.call`, nil, arguments...)
			if err != nil {
				for _, timer := range ctx.timers {
					timer.timer.Stop()
					delete(ctx.timers, timer)
				}
				for _, e := range faults {
					if e.when == "fail" {
						e.what(ctx, err)
					}
				}
			}
			if timer.interval {
				timer.timer.Reset(timer.duration)
			} else {
				delete(ctx.timers, timer)
			}
		default:
			// Escape valve!
		}

		if len(ctx.timers) == 0 {
			break
		}

	}

}

func init() {

	OnInit(func(ctx *Orbit) {

		ctx.Set("setTimeout", func(call otto.FunctionCall) otto.Value {

			delay, _ := call.Argument(1).ToInteger()
			if delay <= 0 {
				delay = 1
			}

			timer := &Task{
				function: call,
				interval: false,
				duration: time.Duration(delay) * time.Millisecond,
			}

			ctx.timers[timer] = timer

			timer.timer = time.AfterFunc(timer.duration, func() {
				ctx.loop <- timer
			})

			val, err := call.Otto.ToValue(timer)
			if err != nil {
				panic(err)
			}

			return val

		})

		ctx.Set("setInterval", func(call otto.FunctionCall) otto.Value {

			delay, _ := call.Argument(1).ToInteger()
			if delay <= 0 {
				delay = 1
			}

			timer := &Task{
				function: call,
				interval: true,
				duration: time.Duration(delay) * time.Millisecond,
			}

			ctx.timers[timer] = timer

			timer.timer = time.AfterFunc(timer.duration, func() {
				ctx.loop <- timer
			})

			val, err := call.Otto.ToValue(timer)
			if err != nil {
				panic(err)
			}

			return val

		})

		ctx.Set("setImmediate", func(call otto.FunctionCall) otto.Value {

			timer := &Task{
				function: call,
				interval: false,
				duration: time.Millisecond,
			}

			ctx.timers[timer] = timer

			timer.timer = time.AfterFunc(timer.duration, func() {
				ctx.loop <- timer
			})

			val, err := call.Otto.ToValue(timer)
			if err != nil {
				panic(err)
			}

			return val

		})

		ctx.Set("clearTimeout", func(call otto.FunctionCall) otto.Value {
			timer, _ := call.Argument(0).Export()
			if timer, ok := timer.(*Task); ok {
				timer.timer.Stop()
				delete(ctx.timers, timer)
			}
			return otto.UndefinedValue()
		})

		ctx.Set("clearImmediate", func(call otto.FunctionCall) otto.Value {
			timer, _ := call.Argument(0).Export()
			if timer, ok := timer.(*Task); ok {
				timer.timer.Stop()
				delete(ctx.timers, timer)
			}
			return otto.UndefinedValue()
		})

	})

}
<|endoftext|>