<commit_before>// deepcopy makes deep copies of things. A standard copy will copy the
// pointers: deep copy copies the values pointed to.  Unexported field
// values are not copied.
//
// Copyright (c)2014-2016, Joel Scoble (github.com/mohae), all rights reserved.
// License: MIT, for more details check the included LICENSE file.
package deepcopy

import "reflect"
import "time"

// Iface is an alias to Copy; this exists for backwards compatibility reasons.
func Iface(iface interface{}) interface{} {
	return Copy(iface)
}

// Copy creates a deep copy of whatever is passed to it and returns the copy
// in an interface{}.  The returned value will need to be asserted to the
// correct type.
func Copy(src interface{}) interface{} {
	if src == nil {
		return nil
	}

	// Make the interface a reflect.Value
	original := reflect.ValueOf(src)

	// Make a copy of the same type as the original.
	cpy := reflect.New(original.Type()).Elem()

	// Recursively copy the original.
	copyRecursive(original, cpy)

	// Return theb copy as an interface.
	return cpy.Interface()
}

// copyRecursive does the actual copying of the interface. It currently has
// limited support for what it can handle. Add as needed.
func copyRecursive(original, cpy reflect.Value) {
	// handle according to original's Kind
	switch original.Kind() {
	case reflect.Ptr:
		// Get the actual value being pointed to.
		originalValue := original.Elem()

		// if  it isn't valid, return.
		if !originalValue.IsValid() {
			return
		}
		cpy.Set(reflect.New(originalValue.Type()))
		copyRecursive(originalValue, cpy.Elem())

	case reflect.Interface:
		// If this is a nil, don't do anything
		if original.IsNil() {
			return
		}
		// Get the value for the interface, not the pointer.
		originalValue := original.Elem()

		// Get the value by calling Elem().
		copyValue := reflect.New(originalValue.Type()).Elem()
		copyRecursive(originalValue, copyValue)
		cpy.Set(copyValue)

	case reflect.Struct:
		t, ok := original.Interface().(time.Time)
		if ok {
			cpy.Set(reflect.ValueOf(t))
			return
		}
		// Go through each field of the struct and copy it.
		for i := 0; i < original.NumField(); i++ {
			// The Type's StructField for a given field is checked to see if StructField.PkgPath
			// is set to determine if the field is exported or not because CanSet() returns false
			// for settable fields.  I'm not sure why.  -mohae
			if original.Type().Field(i).PkgPath != "" {
				continue
			}
			copyRecursive(original.Field(i), cpy.Field(i))
		}

	case reflect.Slice:
		if original.IsNil() {
			return
		}
		// Make a new slice and copy each element.
		cpy.Set(reflect.MakeSlice(original.Type(), original.Len(), original.Cap()))
		for i := 0; i < original.Len(); i++ {
			copyRecursive(original.Index(i), cpy.Index(i))
		}

	case reflect.Map:
		if original.IsNil() {
			return
		}
		cpy.Set(reflect.MakeMap(original.Type()))
		for _, key := range original.MapKeys() {
			originalValue := original.MapIndex(key)
			copyValue := reflect.New(originalValue.Type()).Elem()
			copyRecursive(originalValue, copyValue)
			cpy.SetMapIndex(key, copyValue)
		}

	default:
		cpy.Set(original)
	}
}
<commit_msg>fix spelling error<commit_after>// deepcopy makes deep copies of things. A standard copy will copy the
// pointers: deep copy copies the values pointed to.  Unexported field
// values are not copied.
//
// Copyright (c)2014-2016, Joel Scoble (github.com/mohae), all rights reserved.
// License: MIT, for more details check the included LICENSE file.
package deepcopy

import "reflect"
import "time"

// Iface is an alias to Copy; this exists for backwards compatibility reasons.
func Iface(iface interface{}) interface{} {
	return Copy(iface)
}

// Copy creates a deep copy of whatever is passed to it and returns the copy
// in an interface{}.  The returned value will need to be asserted to the
// correct type.
func Copy(src interface{}) interface{} {
	if src == nil {
		return nil
	}

	// Make the interface a reflect.Value
	original := reflect.ValueOf(src)

	// Make a copy of the same type as the original.
	cpy := reflect.New(original.Type()).Elem()

	// Recursively copy the original.
	copyRecursive(original, cpy)

	// Return the copy as an interface.
	return cpy.Interface()
}

// copyRecursive does the actual copying of the interface. It currently has
// limited support for what it can handle. Add as needed.
func copyRecursive(original, cpy reflect.Value) {
	// handle according to original's Kind
	switch original.Kind() {
	case reflect.Ptr:
		// Get the actual value being pointed to.
		originalValue := original.Elem()

		// if  it isn't valid, return.
		if !originalValue.IsValid() {
			return
		}
		cpy.Set(reflect.New(originalValue.Type()))
		copyRecursive(originalValue, cpy.Elem())

	case reflect.Interface:
		// If this is a nil, don't do anything
		if original.IsNil() {
			return
		}
		// Get the value for the interface, not the pointer.
		originalValue := original.Elem()

		// Get the value by calling Elem().
		copyValue := reflect.New(originalValue.Type()).Elem()
		copyRecursive(originalValue, copyValue)
		cpy.Set(copyValue)

	case reflect.Struct:
		t, ok := original.Interface().(time.Time)
		if ok {
			cpy.Set(reflect.ValueOf(t))
			return
		}
		// Go through each field of the struct and copy it.
		for i := 0; i < original.NumField(); i++ {
			// The Type's StructField for a given field is checked to see if StructField.PkgPath
			// is set to determine if the field is exported or not because CanSet() returns false
			// for settable fields.  I'm not sure why.  -mohae
			if original.Type().Field(i).PkgPath != "" {
				continue
			}
			copyRecursive(original.Field(i), cpy.Field(i))
		}

	case reflect.Slice:
		if original.IsNil() {
			return
		}
		// Make a new slice and copy each element.
		cpy.Set(reflect.MakeSlice(original.Type(), original.Len(), original.Cap()))
		for i := 0; i < original.Len(); i++ {
			copyRecursive(original.Index(i), cpy.Index(i))
		}

	case reflect.Map:
		if original.IsNil() {
			return
		}
		cpy.Set(reflect.MakeMap(original.Type()))
		for _, key := range original.MapKeys() {
			originalValue := original.MapIndex(key)
			copyValue := reflect.New(originalValue.Type()).Elem()
			copyRecursive(originalValue, copyValue)
			cpy.SetMapIndex(key, copyValue)
		}

	default:
		cpy.Set(original)
	}
}
<|endoftext|>


<commit_before>package mesosutil

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"regexp"

	log "github.com/golang/glog"
	mesos "github.com/mesos/mesos-go/mesosproto"
)

func FetchLog(logUrl string) ([]byte, error) {
	resp, err := defaultClient.Get(logUrl)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != 200 {
		return nil, ProxyError{resp.Status, resp.StatusCode}
	}

	defer resp.Body.Close()
	return ioutil.ReadAll(resp.Body)
}

type ProxyError struct {
	Status     string
	StatusCode int
}

func (p ProxyError) Error() string {
	return p.Status
}

type MesosState struct {
	CompletedFrameworks []Framework `json:"completed_frameworks"`
}

func (m MesosState) Directory(frameworkId string) string {
	for _, f := range m.CompletedFrameworks {
		// should we check for the framework?
		for _, e := range f.CompletedExecutors {
			for _, t := range e.CompletedTasks {
				if t.FrameworkId == frameworkId {
					log.V(2).Infoln("Matching task id " + frameworkId)
					return e.Directory
				}
			}
		}
	}
	return ""
}

type Tasks struct {
	FrameworkId string `json:"framework_id"`
}

type Framework struct {
	Name               string
	CompletedExecutors []Executor `json:"completed_executors"`
}

type Executor struct {
	CompletedTasks []Tasks `json:"completed_tasks"`
	Source         string
	Directory      string
}

type MesosTaskStatusMounts []struct {
	Source string `json:"Source"`
}

type MesosTaskStatusConfig struct {
	Hostname   string `json:"Hostname"`
	Domainname string `json:"Domainname"`
}

type MesosTaskStatusData []struct {
	Mounts MesosTaskStatusMounts `json:"Mounts"`
	Config MesosTaskStatusConfig `json:"Config"`
}

type LogData struct {
	Data   string `json:"data"`
	Offset int    `json:"offset"`
}

func FetchLogs(status *mesos.TaskStatus, offset int, file string, frameworkId string) ([]byte, error) {
	var (
		dir      string
		hostname string
		err      error
	)
	switch status.GetState() {
	case mesos.TaskState_TASK_FAILED,
		mesos.TaskState_TASK_KILLED:
		hostname = *status.ContainerStatus.NetworkInfos[0].IpAddress
		url := "http://" + hostname + ":5051/state.json"
		resp, err := defaultClient.Get(url)
		if err != nil {
			return nil, err
		}
		if resp.StatusCode != 200 {
			return nil, ProxyError{resp.Status, resp.StatusCode}
		}
		defer resp.Body.Close()
		var ms MesosState
		err = json.NewDecoder(resp.Body).Decode(&ms)
		if err != nil {
			return nil, err
		}
		dir = ms.Directory(frameworkId)
	default:
		var mtsd MesosTaskStatusData
		err = json.Unmarshal(status.Data, &mtsd)
		if err != nil {
			return nil, err
		}
		// status.Data is an array of one value :( Maybe there is a better way to marshal it?
		firstMtsd := mtsd[0]
		log.V(2).Infof("firstMtsd: %#v", firstMtsd)
		for _, mount := range firstMtsd.Mounts {
			source := mount.Source
			log.V(2).Infoln("mount: ", source)
			matched, _ := regexp.MatchString("slaves.*frameworks.*executors", source)
			if matched {
				dir = source
			}
		}
		domainName := ""
		if firstMtsd.Config.Domainname != "" {
			domainName = "." + firstMtsd.Config.Domainname
		}
		hostname = firstMtsd.Config.Hostname + domainName
	}
	logUrl := fmt.Sprintf("http://%s:5051/files/read.json?path=%s/%s&offset=%d",
		hostname, dir, file, offset)
	bodyData, _ := FetchLog(logUrl)

	var logData LogData
	err = json.Unmarshal(bodyData, &logData)
	if err != nil {
		return nil, err
	}
	return []byte(logData.Data), nil
}
<commit_msg>Depending on the timing, you have to look into these slices as well.<commit_after>package mesosutil

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"regexp"

	log "github.com/golang/glog"
	mesos "github.com/mesos/mesos-go/mesosproto"
)

func FetchLog(logUrl string) ([]byte, error) {
	resp, err := defaultClient.Get(logUrl)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != 200 {
		return nil, ProxyError{resp.Status, resp.StatusCode}
	}

	defer resp.Body.Close()
	return ioutil.ReadAll(resp.Body)
}

type ProxyError struct {
	Status     string
	StatusCode int
}

func (p ProxyError) Error() string {
	return p.Status
}

type MesosState struct {
	CompletedFrameworks []Framework `json:"completed_frameworks"`
	Frameworks          []Framework
}

func (m MesosState) Directory(frameworkId string) string {
	for _, f := range append(m.CompletedFrameworks, m.Frameworks...) {
		// should we check for the framework?
		for _, e := range append(f.CompletedExecutors, f.Executors...) {
			for _, t := range append(e.CompletedTasks, e.Tasks...) {
				if t.FrameworkId == frameworkId {
					log.V(2).Infoln("Matching task id " + frameworkId)
					return e.Directory
				}
			}
		}
	}
	return ""
}

type Tasks struct {
	FrameworkId string `json:"framework_id"`
}

type Framework struct {
	Name               string
	CompletedExecutors []Executor `json:"completed_executors"`
	Executors          []Executor
}

type Executor struct {
	CompletedTasks []Tasks `json:"completed_tasks"`
	Tasks          []Tasks
	Source         string
	Directory      string
}

type MesosTaskStatusMounts []struct {
	Source string `json:"Source"`
}

type MesosTaskStatusConfig struct {
	Hostname   string `json:"Hostname"`
	Domainname string `json:"Domainname"`
}

type MesosTaskStatusData []struct {
	Mounts MesosTaskStatusMounts `json:"Mounts"`
	Config MesosTaskStatusConfig `json:"Config"`
}

type LogData struct {
	Data   string `json:"data"`
	Offset int    `json:"offset"`
}

func FetchLogs(status *mesos.TaskStatus, offset int, file string, frameworkId string) ([]byte, error) {
	var (
		dir      string
		hostname string
		err      error
	)
	switch status.GetState() {
	case mesos.TaskState_TASK_FAILED,
		mesos.TaskState_TASK_KILLED:
		hostname = *status.ContainerStatus.NetworkInfos[0].IpAddress
		url := "http://" + hostname + ":5051/state.json"
		resp, err := defaultClient.Get(url)
		if err != nil {
			return nil, err
		}
		if resp.StatusCode != 200 {
			return nil, ProxyError{resp.Status, resp.StatusCode}
		}
		defer resp.Body.Close()
		var ms MesosState
		err = json.NewDecoder(resp.Body).Decode(&ms)
		if err != nil {
			return nil, err
		}
		dir = ms.Directory(frameworkId)
	default:
		var mtsd MesosTaskStatusData
		err = json.Unmarshal(status.Data, &mtsd)
		if err != nil {
			return nil, err
		}
		// status.Data is an array of one value :( Maybe there is a better way to marshal it?
		firstMtsd := mtsd[0]
		log.V(2).Infof("firstMtsd: %#v", firstMtsd)
		for _, mount := range firstMtsd.Mounts {
			source := mount.Source
			log.V(2).Infoln("mount: ", source)
			matched, _ := regexp.MatchString("slaves.*frameworks.*executors", source)
			if matched {
				dir = source
			}
		}
		domainName := ""
		if firstMtsd.Config.Domainname != "" {
			domainName = "." + firstMtsd.Config.Domainname
		}
		hostname = firstMtsd.Config.Hostname + domainName
	}
	logUrl := fmt.Sprintf("http://%s:5051/files/read.json?path=%s/%s&offset=%d",
		hostname, dir, file, offset)
	bodyData, _ := FetchLog(logUrl)

	var logData LogData
	err = json.Unmarshal(bodyData, &logData)
	if err != nil {
		return nil, err
	}
	return []byte(logData.Data), nil
}
<|endoftext|>


<commit_before>package actor

import (
	"fmt"

	"github.com/emirpasic/gods/sets/hashset"
	"github.com/emirpasic/gods/stacks/linkedliststack"
)

type Context interface {
	Watch(*PID)
	Unwatch(*PID)
	Message() interface{}
	Become(Receive)
	BecomeStacked(Receive)
	UnbecomeStacked()
	Self() *PID
	Parent() *PID
	Spawn(Props) *PID
	SpawnNamed(Props, string) *PID
	Children() []*PID
	Stash()
}

type contextValue struct {
	*actorCell
	message interface{}
}

func (context *contextValue) Message() interface{} {
	return context.message
}

func (context *contextValue) Stash() {
	context.actorCell.stashMessage(context.message)
}

func (cell *actorCell) newContext(message interface{}) Context {
	res := &contextValue{
		actorCell: cell,
		message:   message,
	}
	return res
}

type actorCell struct {
	parent     *PID
	self       *PID
	actor      Actor
	props      Props
	supervisor SupervisionStrategy
	behavior   *linkedliststack.Stack
	children   *hashset.Set
	watchers   *hashset.Set
	watching   *hashset.Set
	stash      *linkedliststack.Stack
	stopping   bool
}

func (cell *actorCell) Children() []*PID {
	values := cell.children.Values()
	children := make([]*PID, len(values))
	for i, child := range values {
		children[i] = child.(*PID)
	}
	return children
}

func (cell *actorCell) Self() *PID {
	return cell.self
}

func (cell *actorCell) Parent() *PID {
	return cell.parent
}

func (cell *actorCell) stashMessage(message interface{}) {
	if cell.stash == nil {
		cell.stash = linkedliststack.New()
	}

	cell.stash.Push(message)
}

func NewActorCell(props Props, parent *PID) *actorCell {

	cell := actorCell{
		parent:     parent,
		props:      props,
		supervisor: props.Supervisor(),
		behavior:   linkedliststack.New(),
		children:   hashset.New(),
		watchers:   hashset.New(),
		watching:   hashset.New(),
	}
	cell.incarnateActor()
	return &cell
}

func (cell *actorCell) incarnateActor() {
	actor := cell.props.ProduceActor()
	cell.actor = actor
	cell.Become(actor.Receive)
}

func (cell *actorCell) invokeSystemMessage(message SystemMessage) {
	switch msg := message.(interface{}).(type) {
	default:
		fmt.Printf("Unknown system message %T", msg)
	case *stop:
		cell.handleStop(msg)
	case *otherStopped:
		cell.handleOtherStopped(msg)
	case *watch:
		cell.watchers.Add(msg.Watcher)
	case *unwatch:
		cell.watchers.Remove(msg.Watcher)
	case *failure:
		cell.handleFailure(msg)
	case *restart:
		cell.handleRestart(msg)
	case *resume:
		cell.self.resume()
	}
}

func (cell *actorCell) handleStop(msg *stop) {
	cell.stopping = true
	cell.invokeUserMessage(Stopping{})
	for _, child := range cell.children.Values() {
		child.(*PID).Stop()
	}
	cell.tryRestartOrTerminate()
}

func (cell *actorCell) handleOtherStopped(msg *otherStopped) {
	cell.children.Remove(msg.Who)
	cell.watching.Remove(msg.Who)
	cell.tryRestartOrTerminate()
}

func (cell *actorCell) handleFailure(msg *failure) {
	directive := cell.supervisor.Handle(msg.Who, msg.Reason)
	switch directive {
	case ResumeDirective:
		//resume the failing child
		msg.Who.sendSystemMessage(&resume{})
	case RestartDirective:
		//restart the failing child
		msg.Who.sendSystemMessage(&restart{})
	case StopDirective:
		//stop the failing child
		msg.Who.Stop()
	case EscalateDirective:
		//send failure to parent
		cell.parent.sendSystemMessage(msg)
	}
}

func (cell *actorCell) handleRestart(msg *restart) {
	cell.stopping = false
	cell.invokeUserMessage(Restarting{})
	for _, child := range cell.children.Values() {
		child.(*PID).Stop()
	}
	cell.tryRestartOrTerminate()
}

func (cell *actorCell) tryRestartOrTerminate() {
	if !cell.children.Empty() {
		return
	}

	if !cell.stopping {
		cell.restart()
		return
	}

	cell.stopped()
}

func (cell *actorCell) restart() {
	cell.incarnateActor()
	cell.invokeUserMessage(Started{})
	if cell.stash != nil {
		for !cell.stash.Empty() {
			msg, _ := cell.stash.Pop()
			cell.invokeUserMessage(msg)
		}
	}
}

func (cell *actorCell) stopped() {
	ProcessRegistry.unregisterPID(cell.self)
	cell.invokeUserMessage(Stopped{})
	otherStopped := &otherStopped{Who: cell.self}
	for _, watcher := range cell.watchers.Values() {
		watcher.(*PID).sendSystemMessage(otherStopped)
	}
}

func (cell *actorCell) invokeUserMessage(message interface{}) {
	defer func() {
		if r := recover(); r != nil {
			failure := &failure{Reason: r, Who: cell.self}
			if cell.parent == nil {
				handleRootFailure(failure, defaultSupervisionStrategy)
			} else {
				cell.self.suspend()
				cell.parent.sendSystemMessage(failure)
			}
		}
	}()
	behavior, _ := cell.behavior.Peek()
	behavior.(Receive)(cell.newContext(message))
}

func (cell *actorCell) Become(behavior Receive) {
	cell.behavior.Clear()
	cell.behavior.Push(behavior)
}

func (cell *actorCell) BecomeStacked(behavior Receive) {
	cell.behavior.Push(behavior)
}

func (cell *actorCell) UnbecomeStacked() {
	if cell.behavior.Size() <= 1 {
		panic("Can not unbecome actor base behavior")
	}
	cell.behavior.Pop()
}

func (cell *actorCell) Watch(who *PID) {
	who.sendSystemMessage(&watch{
		Watcher: cell.self,
	})
	cell.watching.Add(who)
}

func (cell *actorCell) Unwatch(who *PID) {
	who.sendSystemMessage(&unwatch{
		Watcher: cell.self,
	})
	cell.watching.Remove(who)
}

func (cell *actorCell) Spawn(props Props) *PID {
	id := ProcessRegistry.getAutoId()
	return cell.SpawnNamed(props, id)
}

func (cell *actorCell) SpawnNamed(props Props, name string) *PID {
	pid := spawn(name, props, cell.self)
	cell.children.Add(pid)
	cell.Watch(pid)
	return pid
}

func handleRootFailure(msg *failure, supervisor SupervisionStrategy) {
	directive := supervisor.Handle(msg.Who, msg.Reason)
	switch directive {
	case ResumeDirective:
		//resume the fialing child
		msg.Who.sendSystemMessage(&resume{})
	case RestartDirective:
		//restart the failing child
		msg.Who.sendSystemMessage(&restart{})
	case StopDirective:
		//stop the failing child
		msg.Who.Stop()
	case EscalateDirective:
		//send failure to parent
		panic("Can not escalate root level failures")
	}
}
<commit_msg>900k messages per sec<commit_after>package actor

import (
	"fmt"

	"github.com/emirpasic/gods/sets/hashset"
	"github.com/emirpasic/gods/stacks/linkedliststack"
)

type Context interface {
	Watch(*PID)
	Unwatch(*PID)
	Message() interface{}
	Become(Receive)
	BecomeStacked(Receive)
	UnbecomeStacked()
	Self() *PID
	Parent() *PID
	Spawn(Props) *PID
	SpawnNamed(Props, string) *PID
	Children() []*PID
	Stash()
}

func (cell *actorCell) Message() interface{} {
	return cell.message
}

func (cell *actorCell) Stash() {
	cell.stashMessage(cell.message)
}

type actorCell struct {
	message    interface{}
	parent     *PID
	self       *PID
	actor      Actor
	props      Props
	supervisor SupervisionStrategy
	behavior   *linkedliststack.Stack
	children   *hashset.Set
	watchers   *hashset.Set
	watching   *hashset.Set
	stash      *linkedliststack.Stack
	stopping   bool
}

func (cell *actorCell) Children() []*PID {
	values := cell.children.Values()
	children := make([]*PID, len(values))
	for i, child := range values {
		children[i] = child.(*PID)
	}
	return children
}

func (cell *actorCell) Self() *PID {
	return cell.self
}

func (cell *actorCell) Parent() *PID {
	return cell.parent
}

func (cell *actorCell) stashMessage(message interface{}) {
	if cell.stash == nil {
		cell.stash = linkedliststack.New()
	}

	cell.stash.Push(message)
}

func NewActorCell(props Props, parent *PID) *actorCell {

	cell := actorCell{
		parent:     parent,
		props:      props,
		supervisor: props.Supervisor(),
		behavior:   linkedliststack.New(),
		children:   hashset.New(),
		watchers:   hashset.New(),
		watching:   hashset.New(),
	}
	cell.incarnateActor()
	return &cell
}

func (cell *actorCell) incarnateActor() {
	actor := cell.props.ProduceActor()
	cell.actor = actor
	cell.Become(actor.Receive)
}

func (cell *actorCell) invokeSystemMessage(message SystemMessage) {
	switch msg := message.(interface{}).(type) {
	default:
		fmt.Printf("Unknown system message %T", msg)
	case *stop:
		cell.handleStop(msg)
	case *otherStopped:
		cell.handleOtherStopped(msg)
	case *watch:
		cell.watchers.Add(msg.Watcher)
	case *unwatch:
		cell.watchers.Remove(msg.Watcher)
	case *failure:
		cell.handleFailure(msg)
	case *restart:
		cell.handleRestart(msg)
	case *resume:
		cell.self.resume()
	}
}

func (cell *actorCell) handleStop(msg *stop) {
	cell.stopping = true
	cell.invokeUserMessage(Stopping{})
	for _, child := range cell.children.Values() {
		child.(*PID).Stop()
	}
	cell.tryRestartOrTerminate()
}

func (cell *actorCell) handleOtherStopped(msg *otherStopped) {
	cell.children.Remove(msg.Who)
	cell.watching.Remove(msg.Who)
	cell.tryRestartOrTerminate()
}

func (cell *actorCell) handleFailure(msg *failure) {
	directive := cell.supervisor.Handle(msg.Who, msg.Reason)
	switch directive {
	case ResumeDirective:
		//resume the failing child
		msg.Who.sendSystemMessage(&resume{})
	case RestartDirective:
		//restart the failing child
		msg.Who.sendSystemMessage(&restart{})
	case StopDirective:
		//stop the failing child
		msg.Who.Stop()
	case EscalateDirective:
		//send failure to parent
		cell.parent.sendSystemMessage(msg)
	}
}

func (cell *actorCell) handleRestart(msg *restart) {
	cell.stopping = false
	cell.invokeUserMessage(Restarting{})
	for _, child := range cell.children.Values() {
		child.(*PID).Stop()
	}
	cell.tryRestartOrTerminate()
}

func (cell *actorCell) tryRestartOrTerminate() {
	if !cell.children.Empty() {
		return
	}

	if !cell.stopping {
		cell.restart()
		return
	}

	cell.stopped()
}

func (cell *actorCell) restart() {
	cell.incarnateActor()
	cell.invokeUserMessage(Started{})
	if cell.stash != nil {
		for !cell.stash.Empty() {
			msg, _ := cell.stash.Pop()
			cell.invokeUserMessage(msg)
		}
	}
}

func (cell *actorCell) stopped() {
	ProcessRegistry.unregisterPID(cell.self)
	cell.invokeUserMessage(Stopped{})
	otherStopped := &otherStopped{Who: cell.self}
	for _, watcher := range cell.watchers.Values() {
		watcher.(*PID).sendSystemMessage(otherStopped)
	}
}

func (cell *actorCell) invokeUserMessage(message interface{}) {
	defer func() {
		if r := recover(); r != nil {
			failure := &failure{Reason: r, Who: cell.self}
			if cell.parent == nil {
				handleRootFailure(failure, defaultSupervisionStrategy)
			} else {
				cell.self.suspend()
				cell.parent.sendSystemMessage(failure)
			}
		}
	}()
	behavior, _ := cell.behavior.Peek()
	cell.message = message
	behavior.(Receive)(cell)
}

func (cell *actorCell) Become(behavior Receive) {
	cell.behavior.Clear()
	cell.behavior.Push(behavior)
}

func (cell *actorCell) BecomeStacked(behavior Receive) {
	cell.behavior.Push(behavior)
}

func (cell *actorCell) UnbecomeStacked() {
	if cell.behavior.Size() <= 1 {
		panic("Can not unbecome actor base behavior")
	}
	cell.behavior.Pop()
}

func (cell *actorCell) Watch(who *PID) {
	who.sendSystemMessage(&watch{
		Watcher: cell.self,
	})
	cell.watching.Add(who)
}

func (cell *actorCell) Unwatch(who *PID) {
	who.sendSystemMessage(&unwatch{
		Watcher: cell.self,
	})
	cell.watching.Remove(who)
}

func (cell *actorCell) Spawn(props Props) *PID {
	id := ProcessRegistry.getAutoId()
	return cell.SpawnNamed(props, id)
}

func (cell *actorCell) SpawnNamed(props Props, name string) *PID {
	pid := spawn(name, props, cell.self)
	cell.children.Add(pid)
	cell.Watch(pid)
	return pid
}

func handleRootFailure(msg *failure, supervisor SupervisionStrategy) {
	directive := supervisor.Handle(msg.Who, msg.Reason)
	switch directive {
	case ResumeDirective:
		//resume the fialing child
		msg.Who.sendSystemMessage(&resume{})
	case RestartDirective:
		//restart the failing child
		msg.Who.sendSystemMessage(&restart{})
	case StopDirective:
		//stop the failing child
		msg.Who.Stop()
	case EscalateDirective:
		//send failure to parent
		panic("Can not escalate root level failures")
	}
}
<|endoftext|>


<commit_before>package rmq

import (
	"context"
	"fmt"
	"time"
)

type Delivery interface {
	Payload() string

	Ack() error
	Reject() error
	Push() error

	AckWithRetry(context.Context, chan<- error) error
	RejectWithRetry(context.Context, chan<- error) error
	PushWithRetry(context.Context, chan<- error) error
}

// TODO: rename to redisDelivery
type wrapDelivery struct {
	payload     string
	unackedKey  string
	rejectedKey string
	pushKey     string
	redisClient RedisClient
}

func newDelivery(payload, unackedKey, rejectedKey, pushKey string, redisClient RedisClient) *wrapDelivery {
	return &wrapDelivery{
		payload:     payload,
		unackedKey:  unackedKey,
		rejectedKey: rejectedKey,
		pushKey:     pushKey,
		redisClient: redisClient,
	}
}

func (delivery *wrapDelivery) String() string {
	return fmt.Sprintf("[%s %s]", delivery.payload, delivery.unackedKey)
}

func (delivery *wrapDelivery) Payload() string {
	return delivery.payload
}

func (delivery *wrapDelivery) Ack() error {
	count, err := delivery.redisClient.LRem(delivery.unackedKey, 1, delivery.payload)
	if err != nil {
		return err
	}
	if count == 0 {
		return ErrorNotFound
	}
	return nil
}

func (delivery *wrapDelivery) Reject() error {
	return delivery.move(delivery.rejectedKey)
}

func (delivery *wrapDelivery) Push() error {
	if delivery.pushKey == "" {
		return delivery.Reject() // fall back to rejecting
	}

	return delivery.move(delivery.pushKey)
}

func (delivery *wrapDelivery) move(key string) error {
	if _, err := delivery.redisClient.LPush(key, delivery.payload); err != nil {
		return err
	}

	return delivery.Ack()
}

// blocking versions of the functions above with the following behavior:
// 1. return immediately if the operation succeeded or failed with ErrorNotFound
// 2. in case of other redis errors, send them to the errors chan and retry after a sleep
// 3. if the context is cancalled or its timeout exceeded, context.Cancelled or
//    context.DeadlineExceeded will be returned

func (delivery *wrapDelivery) AckWithRetry(ctx context.Context, errors chan<- error) error {
	return delivery.ackWithRetry(ctx, errors, 0)
}

func (delivery *wrapDelivery) ackWithRetry(ctx context.Context, errors chan<- error, errorCount int) error {
	for {
		switch err := delivery.Ack(); err {
		case nil, ErrorNotFound:
			return err
		default: // redis error
			errorCount++

			select { // try to add error to channel, but don't block
			case errors <- &DeliveryError{Delivery: delivery, RedisErr: err, Count: errorCount}:
			default:
			}

			if err := ctx.Err(); err != nil {
				return err
			}

			time.Sleep(time.Second)
		}
	}
}

func (delivery *wrapDelivery) RejectWithRetry(ctx context.Context, errors chan<- error) error {
	return delivery.moveWithRetry(ctx, errors, delivery.rejectedKey)
}

func (delivery *wrapDelivery) PushWithRetry(ctx context.Context, errors chan<- error) error {
	if delivery.pushKey == "" {
		return delivery.RejectWithRetry(ctx, errors) // fall back to rejecting
	}

	return delivery.moveWithRetry(ctx, errors, delivery.pushKey)
}

func (delivery *wrapDelivery) moveWithRetry(ctx context.Context, errors chan<- error, key string) error {
	errorCount := 0
	for {
		_, err := delivery.redisClient.LPush(key, delivery.payload)
		if err == nil { // success
			break
		}
		// error

		errorCount++

		select { // try to add error to channel, but don't block
		case errors <- &DeliveryError{Delivery: delivery, RedisErr: err, Count: errorCount}:
		default:
		}

		time.Sleep(time.Second)
	}

	return delivery.ackWithRetry(ctx, errors, errorCount)
}
<commit_msg>Rename wrapDelivery -> redisDelivery<commit_after>package rmq

import (
	"context"
	"fmt"
	"time"
)

type Delivery interface {
	Payload() string

	Ack() error
	Reject() error
	Push() error

	AckWithRetry(context.Context, chan<- error) error
	RejectWithRetry(context.Context, chan<- error) error
	PushWithRetry(context.Context, chan<- error) error
}

type redisDelivery struct {
	payload     string
	unackedKey  string
	rejectedKey string
	pushKey     string
	redisClient RedisClient
}

func newDelivery(payload, unackedKey, rejectedKey, pushKey string, redisClient RedisClient) *redisDelivery {
	return &redisDelivery{
		payload:     payload,
		unackedKey:  unackedKey,
		rejectedKey: rejectedKey,
		pushKey:     pushKey,
		redisClient: redisClient,
	}
}

func (delivery *redisDelivery) String() string {
	return fmt.Sprintf("[%s %s]", delivery.payload, delivery.unackedKey)
}

func (delivery *redisDelivery) Payload() string {
	return delivery.payload
}

func (delivery *redisDelivery) Ack() error {
	count, err := delivery.redisClient.LRem(delivery.unackedKey, 1, delivery.payload)
	if err != nil {
		return err
	}
	if count == 0 {
		return ErrorNotFound
	}
	return nil
}

func (delivery *redisDelivery) Reject() error {
	return delivery.move(delivery.rejectedKey)
}

func (delivery *redisDelivery) Push() error {
	if delivery.pushKey == "" {
		return delivery.Reject() // fall back to rejecting
	}

	return delivery.move(delivery.pushKey)
}

func (delivery *redisDelivery) move(key string) error {
	if _, err := delivery.redisClient.LPush(key, delivery.payload); err != nil {
		return err
	}

	return delivery.Ack()
}

// blocking versions of the functions above with the following behavior:
// 1. return immediately if the operation succeeded or failed with ErrorNotFound
// 2. in case of other redis errors, send them to the errors chan and retry after a sleep
// 3. if the context is cancalled or its timeout exceeded, context.Cancelled or
//    context.DeadlineExceeded will be returned

func (delivery *redisDelivery) AckWithRetry(ctx context.Context, errors chan<- error) error {
	return delivery.ackWithRetry(ctx, errors, 0)
}

func (delivery *redisDelivery) ackWithRetry(ctx context.Context, errors chan<- error, errorCount int) error {
	for {
		switch err := delivery.Ack(); err {
		case nil, ErrorNotFound:
			return err
		default: // redis error
			errorCount++

			select { // try to add error to channel, but don't block
			case errors <- &DeliveryError{Delivery: delivery, RedisErr: err, Count: errorCount}:
			default:
			}

			if err := ctx.Err(); err != nil {
				return err
			}

			time.Sleep(time.Second)
		}
	}
}

func (delivery *redisDelivery) RejectWithRetry(ctx context.Context, errors chan<- error) error {
	return delivery.moveWithRetry(ctx, errors, delivery.rejectedKey)
}

func (delivery *redisDelivery) PushWithRetry(ctx context.Context, errors chan<- error) error {
	if delivery.pushKey == "" {
		return delivery.RejectWithRetry(ctx, errors) // fall back to rejecting
	}

	return delivery.moveWithRetry(ctx, errors, delivery.pushKey)
}

func (delivery *redisDelivery) moveWithRetry(ctx context.Context, errors chan<- error, key string) error {
	errorCount := 0
	for {
		_, err := delivery.redisClient.LPush(key, delivery.payload)
		if err == nil { // success
			break
		}
		// error

		errorCount++

		select { // try to add error to channel, but don't block
		case errors <- &DeliveryError{Delivery: delivery, RedisErr: err, Count: errorCount}:
		default:
		}

		time.Sleep(time.Second)
	}

	return delivery.ackWithRetry(ctx, errors, errorCount)
}
<|endoftext|>


<commit_before>package cmd

import (
	"log"
	"strings"

	"github.com/codegangsta/cli"
	"github.com/crackcomm/ovh"
	"golang.org/x/net/context"
)

var cmdDomainsSet = cli.Command{
	Name:      "set",
	Usage:     "sets domain name server option",
	ArgsUsage: "[--all] [<domain> [<domain> ...]]",
	Flags: []cli.Flag{
		cli.BoolFlag{
			Name:  "all",
			Usage: "applies settings to all domains on account",
		},
		cli.StringFlag{
			Name:  "name_server_type",
			Usage: "changes domain name server type (external or hosted)",
		},
		cli.StringFlag{
			Name:  "transfer_lock_status",
			Usage: "changes domain transfer lock status (locked or unlocked)",
		},
	},
	Action: func(c *cli.Context) {
		if len(c.Args()) == 0 && !c.Bool("all") {
			log.Fatal("You have to use --all or provide list of domains in arguments.")
		} else if len(c.Args()) > 0 && c.Bool("all") {
			log.Fatal("Cannot use --all with list of domains")
		}

		client := client(c)
		ctx := context.Background()
		patch := &ovh.DomainPatch{
			NameServerType:     c.String("name_server_type"),
			TransferLockStatus: c.String("transfer_lock_status"),
		}

		if v := patch.NameServerType; v != "" {
			log.Printf("Changing name server type to %q", v)
		}
		if v := patch.TransferLockStatus; v != "" {
			log.Printf("Changing transfer lock status to %q", v)
		}

		for _, domain := range domainsFromCtx(c) {
			log.Printf("Updating %s", domain)
			err := client.Domains.Patch(ctx, domain, patch)
			if err != nil {
				log.Fatal(err)
			}
		}

		log.Println("Done")
	},
}

func domainsFromCtx(c *cli.Context) (domains []string) {
	if c.Bool("all") {
		domains, err := client(c).Domains.List(context.Background())
		if err != nil {
			log.Fatalf("Error retrieving list of domains: %v", err)
		}
		return domains
	}
	if l := c.String("domain"); l != "" {
		return strings.Split(l, ",")
	}
	return []string(c.Args())
}
<commit_msg>cmd: fix domains set command flags style<commit_after>package cmd

import (
	"log"
	"strings"

	"github.com/codegangsta/cli"
	"github.com/crackcomm/ovh"
	"golang.org/x/net/context"
)

var cmdDomainsSet = cli.Command{
	Name:      "set",
	Usage:     "sets domain name server option",
	ArgsUsage: "[--all] [<domain> [<domain> ...]]",
	Flags: []cli.Flag{
		cli.BoolFlag{
			Name:  "all",
			Usage: "applies settings to all domains on account",
		},
		cli.StringFlag{
			Name:  "name-server-type",
			Usage: "changes domain name server type (external or hosted)",
		},
		cli.StringFlag{
			Name:  "transfer-lock-status",
			Usage: "changes domain transfer lock status (locked or unlocked)",
		},
	},
	Action: func(c *cli.Context) {
		if len(c.Args()) == 0 && !c.Bool("all") {
			log.Fatal("You have to use --all or provide list of domains in arguments.")
		} else if len(c.Args()) > 0 && c.Bool("all") {
			log.Fatal("Cannot use --all with list of domains")
		}

		client := client(c)
		ctx := context.Background()
		patch := &ovh.DomainPatch{
			NameServerType:     c.String("name-server-type"),
			TransferLockStatus: c.String("transfer-lock-status"),
		}

		if v := patch.NameServerType; v != "" {
			log.Printf("Changing name server type to %q", v)
		}
		if v := patch.TransferLockStatus; v != "" {
			log.Printf("Changing transfer lock status to %q", v)
		}

		for _, domain := range domainsFromCtx(c) {
			log.Printf("Updating %s", domain)
			err := client.Domains.Patch(ctx, domain, patch)
			if err != nil {
				log.Fatal(err)
			}
		}

		log.Println("Done")
	},
}

func domainsFromCtx(c *cli.Context) (domains []string) {
	if c.Bool("all") {
		domains, err := client(c).Domains.List(context.Background())
		if err != nil {
			log.Fatalf("Error retrieving list of domains: %v", err)
		}
		return domains
	}
	if l := c.String("domain"); l != "" {
		return strings.Split(l, ",")
	}
	return []string(c.Args())
}
<|endoftext|>


<commit_before>package packages

import (
	"go/build"
	"os"
	"path/filepath"
	"strings"
)

var (
	cwd       string
	gopathEnv = os.Getenv("GOPATH")
	gopath    []string
)

func init() {
	var err error
	cwd, err = os.Getwd()
	if err != nil {
		panic(err)
	}
}

type Package struct {
	Name string
	Path string
}

func Load(pkgPatterns ...string) (packages []Package) {
	pkgPatterns = parsePatterns(pkgPatterns...)
	for _, pkgPattern := range pkgPatterns {
		pkg, err := build.Import(pkgPattern, cwd, build.AllowBinary)
		if err != nil {
			panic(err)
		}
		packages = append(packages, Package{
			Name: pkg.Name,
			Path: pkg.Dir,
		})
	}
	return
}

func parsePatterns(pkgPatterns ...string) (packages []string) {
	for _, pkgPattern := range pkgPatterns {
		if !strings.HasSuffix(pkgPattern, "...") {
			packages = append(packages, pkgPattern)
			continue
		}
		parent := strings.TrimSuffix(pkgPattern, "...")
		parentPkg, err := build.Import(parent, cwd, build.AllowBinary)
		if err != nil {
			panic(err)
		}
		filepath.Walk(parentPkg.Dir, func(path string, info os.FileInfo, err error) error {
			if !info.IsDir() {
				return nil
			}
			path = strings.Replace(path, parentPkg.Dir, parent, 1)
			if _, err := build.Import(path, cwd, build.AllowBinary); err != nil {
				// This directory doesn't appear to be a go package
				return nil
			}
			packages = append(packages, path)
			return nil
		})
	}
	return
}
<commit_msg>Continue renaming 'packages' to 'dirs'<commit_after>package packages

import (
	"go/ast"
	"go/build"
	"go/parser"
	"go/token"
	"os"
	"path/filepath"
	"strings"
)

var (
	cwd       string
	gopathEnv = os.Getenv("GOPATH")
	gopath    []string
)

func init() {
	var err error
	cwd, err = os.Getwd()
	if err != nil {
		panic(err)
	}
}

type Dir struct {
	path string
}

func (d Dir) Path() string {
	return d.path
}

func (d Dir) Packages() map[string]*ast.Package {
	packages, err := parser.ParseDir(token.NewFileSet(), d.Path(), nil, 0)
	if err != nil {
		panic(err)
	}
	return packages
}

func Load(pkgPatterns ...string) (dirs []Dir) {
	pkgPatterns = parsePatterns(pkgPatterns...)
	for _, pkgPattern := range pkgPatterns {
		pkg, err := build.Import(pkgPattern, cwd, build.AllowBinary)
		if err != nil {
			panic(err)
		}
		dirs = append(dirs, Dir{path: pkg.Dir})
	}
	return
}

func parsePatterns(pkgPatterns ...string) (packages []string) {
	for _, pkgPattern := range pkgPatterns {
		if !strings.HasSuffix(pkgPattern, "...") {
			packages = append(packages, pkgPattern)
			continue
		}
		parent := strings.TrimSuffix(pkgPattern, "...")
		parentPkg, err := build.Import(parent, cwd, build.AllowBinary)
		if err != nil {
			panic(err)
		}
		filepath.Walk(parentPkg.Dir, func(path string, info os.FileInfo, err error) error {
			if !info.IsDir() {
				return nil
			}
			path = strings.Replace(path, parentPkg.Dir, parent, 1)
			if _, err := build.Import(path, cwd, build.AllowBinary); err != nil {
				// This directory doesn't appear to be a go package
				return nil
			}
			packages = append(packages, path)
			return nil
		})
	}
	return
}
<|endoftext|>


<commit_before>package instances

import (
	"github.com/gophercloud/gophercloud"
	db "github.com/gophercloud/gophercloud/openstack/db/v1/databases"
	"github.com/gophercloud/gophercloud/openstack/db/v1/users"
	"github.com/gophercloud/gophercloud/pagination"
)

// CreateOptsBuilder is the top-level interface for create options.
type CreateOptsBuilder interface {
	ToInstanceCreateMap() (map[string]interface{}, error)
}

// DatastoreOpts represents the configuration for how an instance stores data.
type DatastoreOpts struct {
	Version string `json:"version"`
	Type    string `json:"type"`
}

// ToMap converts a DatastoreOpts to a map[string]string (for a request body)
func (opts DatastoreOpts) ToMap() (map[string]interface{}, error) {
	return gophercloud.BuildRequestBody(opts, "")
}

// CreateOpts is the struct responsible for configuring a new database instance.
type CreateOpts struct {
	// Either the integer UUID (in string form) of the flavor, or its URI
	// reference as specified in the response from the List() call. Required.
	FlavorRef string
	// Specifies the volume size in gigabytes (GB). The value must be between 1
	// and 300. Required.
	Size int
	// Name of the instance to create. The length of the name is limited to
	// 255 characters and any characters are permitted. Optional.
	Name string
	// A slice of database information options.
	Databases db.CreateOptsBuilder
	// A slice of user information options.
	Users users.CreateOptsBuilder
	// Options to configure the type of datastore the instance will use. This is
	// optional, and if excluded will default to MySQL.
	Datastore *DatastoreOpts
}

// ToInstanceCreateMap will render a JSON map.
func (opts CreateOpts) ToInstanceCreateMap() (map[string]interface{}, error) {
	if opts.Size > 300 || opts.Size < 1 {
		err := gophercloud.ErrInvalidInput{}
		err.Argument = "instances.CreateOpts.Size"
		err.Value = opts.Size
		err.Info = "Size (GB) must be between 1-300"
		return nil, err
	}

	if opts.FlavorRef == "" {
		return nil, gophercloud.ErrMissingInput{Argument: "instances.CreateOpts.FlavorRef"}
	}

	instance := map[string]interface{}{
		"volume":    map[string]int{"size": opts.Size},
		"flavorRef": opts.FlavorRef,
	}

	if opts.Name != "" {
		instance["name"] = opts.Name
	}
	if opts.Databases != nil {
		dbs, err := opts.Databases.ToDBCreateMap()
		if err != nil {
			return nil, err
		}
		instance["databases"] = dbs["databases"]
	}
	if opts.Users != nil {
		users, err := opts.Users.ToUserCreateMap()
		if err != nil {
			return nil, err
		}
		instance["users"] = users["users"]
	}
	if opts.Datastore != nil {
		datastore, err := opts.Datastore.ToMap()
		if err != nil {
			return nil, err
		}
		instance["datastore"] = datastore
	}

	return map[string]interface{}{"instance": instance}, nil
}

// Create asynchronously provisions a new database instance. It requires the
// user to specify a flavor and a volume size. The API service then provisions
// the instance with the requested flavor and sets up a volume of the specified
// size, which is the storage for the database instance.
//
// Although this call only allows the creation of 1 instance per request, you
// can create an instance with multiple databases and users. The default
// binding for a MySQL instance is port 3306.
func Create(client *gophercloud.ServiceClient, opts CreateOptsBuilder) (r CreateResult) {
	b, err := opts.ToInstanceCreateMap()
	if err != nil {
		r.Err = err
		return
	}
	_, r.Err = client.Post(baseURL(client), &b, &r.Body, &gophercloud.RequestOpts{OkCodes: []int{200}})
	return
}

// List retrieves the status and information for all database instances.
func List(client *gophercloud.ServiceClient) pagination.Pager {
	return pagination.NewPager(client, baseURL(client), func(r pagination.PageResult) pagination.Page {
		return InstancePage{pagination.LinkedPageBase{PageResult: r}}
	})
}

// Get retrieves the status and information for a specified database instance.
func Get(client *gophercloud.ServiceClient, id string) (r GetResult) {
	_, r.Err = client.Get(resourceURL(client, id), &r.Body, nil)
	return
}

// Delete permanently destroys the database instance.
func Delete(client *gophercloud.ServiceClient, id string) (r DeleteResult) {
	_, r.Err = client.Delete(resourceURL(client, id), nil)
	return
}

// EnableRootUser enables the login from any host for the root user and
// provides the user with a generated root password.
func EnableRootUser(client *gophercloud.ServiceClient, id string) (r EnableRootUserResult) {
	_, r.Err = client.Post(userRootURL(client, id), nil, &r.Body, &gophercloud.RequestOpts{OkCodes: []int{200}})
	return
}

// IsRootEnabled checks an instance to see if root access is enabled. It returns
// True if root user is enabled for the specified database instance or False
// otherwise.
func IsRootEnabled(client *gophercloud.ServiceClient, id string) (r IsRootEnabledResult) {
	_, r.Err = client.Get(userRootURL(client, id), &r.Body, nil)
	return
}

// Restart will restart only the MySQL Instance. Restarting MySQL will
// erase any dynamic configuration settings that you have made within MySQL.
// The MySQL service will be unavailable until the instance restarts.
func Restart(client *gophercloud.ServiceClient, id string) (r ActionResult) {
	b := map[string]interface{}{"restart": struct{}{}}
	_, r.Err = client.Post(actionURL(client, id), &b, nil, nil)
	return
}

// Resize changes the memory size of the instance, assuming a valid
// flavorRef is provided. It will also restart the MySQL service.
func Resize(client *gophercloud.ServiceClient, id, flavorRef string) (r ActionResult) {
	b := map[string]interface{}{"resize": map[string]string{"flavorRef": flavorRef}}
	_, r.Err = client.Post(actionURL(client, id), &b, nil, nil)
	return
}

// ResizeVolume will resize the attached volume for an instance. It supports
// only increasing the volume size and does not support decreasing the size.
// The volume size is in gigabytes (GB) and must be an integer.
func ResizeVolume(client *gophercloud.ServiceClient, id string, size int) (r ActionResult) {
	b := map[string]interface{}{"resize": map[string]interface{}{"volume": map[string]int{"size": size}}}
	_, r.Err = client.Post(actionURL(client, id), &b, nil, nil)
	return
}
<commit_msg>db/instance: support giving nics in create opts (#201)<commit_after>package instances

import (
	"github.com/gophercloud/gophercloud"
	db "github.com/gophercloud/gophercloud/openstack/db/v1/databases"
	"github.com/gophercloud/gophercloud/openstack/db/v1/users"
	"github.com/gophercloud/gophercloud/pagination"
)

// CreateOptsBuilder is the top-level interface for create options.
type CreateOptsBuilder interface {
	ToInstanceCreateMap() (map[string]interface{}, error)
}

// DatastoreOpts represents the configuration for how an instance stores data.
type DatastoreOpts struct {
	Version string `json:"version"`
	Type    string `json:"type"`
}

// ToMap converts a DatastoreOpts to a map[string]string (for a request body)
func (opts DatastoreOpts) ToMap() (map[string]interface{}, error) {
	return gophercloud.BuildRequestBody(opts, "")
}

// NetworkOpts is used within CreateOpts to control a new server's network attachments.
type NetworkOpts struct {
	// UUID of a nova-network to attach to the newly provisioned server.
	// Required unless Port is provided.
	UUID string `json:"net-id,omitempty"`

	// Port of a neutron network to attach to the newly provisioned server.
	// Required unless UUID is provided.
	Port string `json:"port-id,omitempty"`

	// V4FixedIP [optional] specifies a fixed IPv4 address to be used on this network.
	V4FixedIP string `json:"v4-fixed-ip,omitempty"`

	// V6FixedIP [optional] specifies a fixed IPv6 address to be used on this network.
	V6FixedIP string `json:"v6-fixed-ip,omitempty"`
}

// ToMap converts a NetworkOpts to a map[string]string (for a request body)
func (opts NetworkOpts) ToMap() (map[string]interface{}, error) {
	return gophercloud.BuildRequestBody(opts, "")
}

// CreateOpts is the struct responsible for configuring a new database instance.
type CreateOpts struct {
	// Either the integer UUID (in string form) of the flavor, or its URI
	// reference as specified in the response from the List() call. Required.
	FlavorRef string
	// Specifies the volume size in gigabytes (GB). The value must be between 1
	// and 300. Required.
	Size int
	// Name of the instance to create. The length of the name is limited to
	// 255 characters and any characters are permitted. Optional.
	Name string
	// A slice of database information options.
	Databases db.CreateOptsBuilder
	// A slice of user information options.
	Users users.CreateOptsBuilder
	// Options to configure the type of datastore the instance will use. This is
	// optional, and if excluded will default to MySQL.
	Datastore *DatastoreOpts
	// Networks dictates how this server will be attached to available networks.
	Networks []NetworkOpts
}

// ToInstanceCreateMap will render a JSON map.
func (opts CreateOpts) ToInstanceCreateMap() (map[string]interface{}, error) {
	if opts.Size > 300 || opts.Size < 1 {
		err := gophercloud.ErrInvalidInput{}
		err.Argument = "instances.CreateOpts.Size"
		err.Value = opts.Size
		err.Info = "Size (GB) must be between 1-300"
		return nil, err
	}

	if opts.FlavorRef == "" {
		return nil, gophercloud.ErrMissingInput{Argument: "instances.CreateOpts.FlavorRef"}
	}

	instance := map[string]interface{}{
		"volume":    map[string]int{"size": opts.Size},
		"flavorRef": opts.FlavorRef,
	}

	if opts.Name != "" {
		instance["name"] = opts.Name
	}
	if opts.Databases != nil {
		dbs, err := opts.Databases.ToDBCreateMap()
		if err != nil {
			return nil, err
		}
		instance["databases"] = dbs["databases"]
	}
	if opts.Users != nil {
		users, err := opts.Users.ToUserCreateMap()
		if err != nil {
			return nil, err
		}
		instance["users"] = users["users"]
	}
	if opts.Datastore != nil {
		datastore, err := opts.Datastore.ToMap()
		if err != nil {
			return nil, err
		}
		instance["datastore"] = datastore
	}

	if len(opts.Networks) > 0 {
		networks := make([]map[string]interface{}, len(opts.Networks))
		for i, net := range opts.Networks {
			var err error
			networks[i], err = net.ToMap()
			if err != nil {
				return nil, err
			}
		}
		instance["nics"] = networks
	}

	return map[string]interface{}{"instance": instance}, nil
}

// Create asynchronously provisions a new database instance. It requires the
// user to specify a flavor and a volume size. The API service then provisions
// the instance with the requested flavor and sets up a volume of the specified
// size, which is the storage for the database instance.
//
// Although this call only allows the creation of 1 instance per request, you
// can create an instance with multiple databases and users. The default
// binding for a MySQL instance is port 3306.
func Create(client *gophercloud.ServiceClient, opts CreateOptsBuilder) (r CreateResult) {
	b, err := opts.ToInstanceCreateMap()
	if err != nil {
		r.Err = err
		return
	}
	_, r.Err = client.Post(baseURL(client), &b, &r.Body, &gophercloud.RequestOpts{OkCodes: []int{200}})
	return
}

// List retrieves the status and information for all database instances.
func List(client *gophercloud.ServiceClient) pagination.Pager {
	return pagination.NewPager(client, baseURL(client), func(r pagination.PageResult) pagination.Page {
		return InstancePage{pagination.LinkedPageBase{PageResult: r}}
	})
}

// Get retrieves the status and information for a specified database instance.
func Get(client *gophercloud.ServiceClient, id string) (r GetResult) {
	_, r.Err = client.Get(resourceURL(client, id), &r.Body, nil)
	return
}

// Delete permanently destroys the database instance.
func Delete(client *gophercloud.ServiceClient, id string) (r DeleteResult) {
	_, r.Err = client.Delete(resourceURL(client, id), nil)
	return
}

// EnableRootUser enables the login from any host for the root user and
// provides the user with a generated root password.
func EnableRootUser(client *gophercloud.ServiceClient, id string) (r EnableRootUserResult) {
	_, r.Err = client.Post(userRootURL(client, id), nil, &r.Body, &gophercloud.RequestOpts{OkCodes: []int{200}})
	return
}

// IsRootEnabled checks an instance to see if root access is enabled. It returns
// True if root user is enabled for the specified database instance or False
// otherwise.
func IsRootEnabled(client *gophercloud.ServiceClient, id string) (r IsRootEnabledResult) {
	_, r.Err = client.Get(userRootURL(client, id), &r.Body, nil)
	return
}

// Restart will restart only the MySQL Instance. Restarting MySQL will
// erase any dynamic configuration settings that you have made within MySQL.
// The MySQL service will be unavailable until the instance restarts.
func Restart(client *gophercloud.ServiceClient, id string) (r ActionResult) {
	b := map[string]interface{}{"restart": struct{}{}}
	_, r.Err = client.Post(actionURL(client, id), &b, nil, nil)
	return
}

// Resize changes the memory size of the instance, assuming a valid
// flavorRef is provided. It will also restart the MySQL service.
func Resize(client *gophercloud.ServiceClient, id, flavorRef string) (r ActionResult) {
	b := map[string]interface{}{"resize": map[string]string{"flavorRef": flavorRef}}
	_, r.Err = client.Post(actionURL(client, id), &b, nil, nil)
	return
}

// ResizeVolume will resize the attached volume for an instance. It supports
// only increasing the volume size and does not support decreasing the size.
// The volume size is in gigabytes (GB) and must be an integer.
func ResizeVolume(client *gophercloud.ServiceClient, id string, size int) (r ActionResult) {
	b := map[string]interface{}{"resize": map[string]interface{}{"volume": map[string]int{"size": size}}}
	_, r.Err = client.Post(actionURL(client, id), &b, nil, nil)
	return
}
<|endoftext|>


<commit_before>package main

import (
	"delay"
	"sync/atomic"

	"stm32/hal/raw/tim"
)

const rhMax = 10000

func setupPWM(t *tim.TIM_Periph, pclk uint, periodms, arr int) {
	t.PSC.U16.Store(uint16(int(pclk/1000)*periodms/(arr+1) - 1))
	t.ARR.U32.Store(uint32(arr))
	t.CCMR1.Store(6<<tim.OC1Mn | 6<<tim.OC2Mn | tim.OC1PE | tim.OC2PE)
	t.CCMR2.Store(7<<tim.OC3Mn | 6<<tim.OC4Mn | tim.OC3PE | tim.OC4PE)
	t.CCR2.Store(0)
	t.CCR3.U32.Store(uint32(arr + 1))
	t.CCR4.Store(0)
	t.CCER.Store(tim.CC2E | tim.CC3E | tim.CC4E)
	t.CR1.Store(tim.URS | tim.CEN)
}

type roomHeaterControl struct {
	desiredTemp16 int // °C/16
	tempSensor    Sensor
}

func (r *roomHeaterControl) TempSensor() *Sensor {
	return &r.tempSensor
}

func (r *roomHeaterControl) DesiredTemp16() int {
	return atomic.LoadInt(&r.desiredTemp16)
}

func (r *roomHeaterControl) SetDesiredTemp16(temp16 int) {
	atomic.StoreInt(&r.desiredTemp16, temp16)
}

func (r *roomHeaterControl) loop(t *tim.TIM_Periph) {
	og := &t.CCR2.U32
	la := &t.CCR3.U32
	st := &t.CCR4.U32
	tempResp := make(chan int, 1)
	for {
		p := 0
		dt := readRTC()
		hm := dt.Hour()*60 + dt.Minute()
		dev := r.tempSensor.Load()
		// Heat only if tempSensor set and cheap energy time: 22-6 and 13-15.
		const offset = -12 // My electric meter is 12 minutes late.
		const margin = 5
		if dt.IsValid() && dev.Type() != 0 &&
			(hm < 6*60-margin+offset || hm > 22*60+margin+offset ||
				hm > 13*60+margin+offset && hm < 15*60-margin+offset) {

			owd.Cmd <- TempCmd{Dev: dev, Resp: tempResp}
			t := <-tempResp
			if t != InvalidTemp {
				desiredTemp16 := atomic.LoadInt(&r.desiredTemp16)
				p = (desiredTemp16 - t) * rhMax / (2 * 16) // 1°C : maxP/2
				// Disable room heater if water heater exceeds 9 kW.
				maxP := (9 - water.LastPower()) * rhMax
				switch {
				case maxP < 0:
					maxP = 0
				case maxP > rhMax:
					maxP = rhMax
				}
				switch {
				case p < 0:
					p = 0
				case p > maxP:
					p = maxP
				}
			}
		}
		og.Store(uint32(p) / 3)     // Small room, PWM mode 6.
		la.Store(rhMax - uint32(p)) // PWM mode 7 (use tail of PWM period first).
		st.Store(uint32(p) * 2 / 3) // Medium room, PWM mode 6.
		delay.Millisec(5e3)
	}
}

func (r *roomHeaterControl) Start(t *tim.TIM_Periph, pclk uint) {
	setupPWM(t, pclk, 500, rhMax-1)
	r.SetDesiredTemp16(20 * 16) // °C/16
	go r.loop(t)
}

var room roomHeaterControl
<commit_msg>l1-discovery/heating: More heat to small room.<commit_after>package main

import (
	"delay"
	"sync/atomic"

	"stm32/hal/raw/tim"
)

const rhMax = 10000

func setupPWM(t *tim.TIM_Periph, pclk uint, periodms, arr int) {
	t.PSC.U16.Store(uint16(int(pclk/1000)*periodms/(arr+1) - 1))
	t.ARR.U32.Store(uint32(arr))
	t.CCMR1.Store(6<<tim.OC1Mn | 6<<tim.OC2Mn | tim.OC1PE | tim.OC2PE)
	t.CCMR2.Store(7<<tim.OC3Mn | 6<<tim.OC4Mn | tim.OC3PE | tim.OC4PE)
	t.CCR2.Store(0)
	t.CCR3.U32.Store(uint32(arr + 1))
	t.CCR4.Store(0)
	t.CCER.Store(tim.CC2E | tim.CC3E | tim.CC4E)
	t.CR1.Store(tim.URS | tim.CEN)
}

type roomHeaterControl struct {
	desiredTemp16 int // °C/16
	tempSensor    Sensor
}

func (r *roomHeaterControl) TempSensor() *Sensor {
	return &r.tempSensor
}

func (r *roomHeaterControl) DesiredTemp16() int {
	return atomic.LoadInt(&r.desiredTemp16)
}

func (r *roomHeaterControl) SetDesiredTemp16(temp16 int) {
	atomic.StoreInt(&r.desiredTemp16, temp16)
}

func (r *roomHeaterControl) loop(t *tim.TIM_Periph) {
	og := &t.CCR2.U32
	la := &t.CCR3.U32
	st := &t.CCR4.U32
	tempResp := make(chan int, 1)
	for {
		p := 0
		dt := readRTC()
		hm := dt.Hour()*60 + dt.Minute()
		dev := r.tempSensor.Load()
		// Heat only if tempSensor set and cheap energy time: 22-6 and 13-15.
		const offset = -12 // My electric meter is 12 minutes late.
		const margin = 5
		if dt.IsValid() && dev.Type() != 0 &&
			(hm < 6*60-margin+offset || hm > 22*60+margin+offset ||
				hm > 13*60+margin+offset && hm < 15*60-margin+offset) {

			owd.Cmd <- TempCmd{Dev: dev, Resp: tempResp}
			t := <-tempResp
			if t != InvalidTemp {
				desiredTemp16 := atomic.LoadInt(&r.desiredTemp16)
				p = (desiredTemp16 - t) * rhMax / (2 * 16) // 1°C : maxP/2
				// Disable room heater if water heater exceeds 9 kW.
				maxP := (9 - water.LastPower()) * rhMax
				switch {
				case maxP < 0:
					maxP = 0
				case maxP > rhMax:
					maxP = rhMax
				}
				switch {
				case p < 0:
					p = 0
				case p > maxP:
					p = maxP
				}
			}
		}
		og.Store(uint32(p) * 2 / 3) // Small room, PWM mode 6.
		la.Store(rhMax - uint32(p)) // PWM mode 7 (use tail of PWM period first).
		st.Store(uint32(p) * 2 / 3) // Medium room, PWM mode 6.
		delay.Millisec(5e3)
	}
}

func (r *roomHeaterControl) Start(t *tim.TIM_Periph, pclk uint) {
	setupPWM(t, pclk, 500, rhMax-1)
	r.SetDesiredTemp16(20 * 16) // °C/16
	go r.loop(t)
}

var room roomHeaterControl
<|endoftext|>


<commit_before>package pivotaltracker

import (
	// Stdlib
	"fmt"
	"strings"

	// Internal
	"github.com/salsaflow/salsaflow/modules/common"
	"github.com/salsaflow/salsaflow/version"

	// Other
	"github.com/salsita/go-pivotaltracker/v5/pivotal"
	"github.com/toqueteos/webbrowser"
)

const ServiceName = "Pivotal Tracker"

type issueTracker struct {
	config Config
}

func Factory() (common.IssueTracker, error) {
	config, err := LoadConfig()
	if err != nil {
		return nil, err
	}
	return &issueTracker{config}, nil
}

func (tracker *issueTracker) ServiceName() string {
	return ServiceName
}

func (tracker *issueTracker) CurrentUser() (common.User, error) {
	me, err := fetchMe()
	if err != nil {
		return nil, err
	}
	return &user{me}, nil
}

func (tracker *issueTracker) StartableStories() (stories []common.Story, err error) {
	var (
		client    = pivotal.NewClient(tracker.config.UserToken())
		projectId = tracker.config.ProjectId()
	)
	ptStories, err := searchStories(client, projectId, "state:%v", pivotal.StoryStateUnstarted)
	if err != nil {
		return nil, err
	}

	// Drop the features that are not estimated.
	estimatedStories := make([]*pivotal.Story, 0, len(ptStories))
	for _, story := range ptStories {
		if !(story.Type == pivotal.StoryTypeFeature && story.Estimate == nil) {
			estimatedStories = append(estimatedStories, story)
		}
	}

	ptStories = storiesMatchingByLabel(estimatedStories, tracker.config.IncludeStoryLabelFilter())

	return toCommonStories(ptStories, tracker), nil
}

func (tracker *issueTracker) StoriesInDevelopment() (stories []common.Story, err error) {
	var (
		client    = pivotal.NewClient(tracker.config.UserToken())
		projectId = tracker.config.ProjectId()
	)
	ptStories, err := searchStories(client, projectId,
		"state:%v AND -label:\"%v\" AND -label:\"%v\"",
		pivotal.StoryStateStarted, tracker.config.ReviewedLabel(), tracker.config.NoReviewLabel())
	if err != nil {
		return nil, err
	}
	return toCommonStories(ptStories, tracker), nil
}

func (tracker *issueTracker) ListStoriesByTag(tags []string) ([]common.Story, error) {
	// Convert tags to ids.
	ids := make([]string, 0, len(tags))
	for _, tag := range tags {
		id, err := tracker.StoryTagToReadableStoryId(tag)
		if err != nil {
			return nil, err
		}
		ids = append(ids, id)
	}

	// Fetch the relevant stories.
	var (
		client    = pivotal.NewClient(tracker.config.UserToken())
		projectId = tracker.config.ProjectId()
	)
	stories, err := listStoriesByIdOrdered(client, projectId, ids)
	if err != nil {
		return nil, err
	}

	// Convert to []common.Story and return.
	return toCommonStories(stories, tracker), nil
}

func (tracker *issueTracker) NextRelease(
	trunkVersion *version.Version,
	nextTrunkVersion *version.Version,
) (common.NextRelease, error) {

	return newNextRelease(tracker, trunkVersion, nextTrunkVersion)
}

func (tracker *issueTracker) RunningRelease(
	releaseVersion *version.Version,
) (common.RunningRelease, error) {

	return newRunningRelease(releaseVersion, tracker)
}

func (tracker *issueTracker) OpenStory(storyId string) error {
	return webbrowser.Open(fmt.Sprintf("https://pivotaltracker.com/story/show/%v", storyId))
}

func (tracker *issueTracker) StoryTagToReadableStoryId(tag string) (storyId string, err error) {
	parts := strings.Split(tag, "/")
	if len(parts) != 3 {
		return "", fmt.Errorf("invalid Pivotal Tracker Story-Id tag: %v", tag)
	}
	return parts[2], nil
}
<commit_msg>pivotaltracker: Planned staries are also startable<commit_after>package pivotaltracker

import (
	// Stdlib
	"fmt"
	"strings"

	// Internal
	"github.com/salsaflow/salsaflow/modules/common"
	"github.com/salsaflow/salsaflow/version"

	// Other
	"github.com/salsita/go-pivotaltracker/v5/pivotal"
	"github.com/toqueteos/webbrowser"
)

const ServiceName = "Pivotal Tracker"

type issueTracker struct {
	config Config
}

func Factory() (common.IssueTracker, error) {
	config, err := LoadConfig()
	if err != nil {
		return nil, err
	}
	return &issueTracker{config}, nil
}

func (tracker *issueTracker) ServiceName() string {
	return ServiceName
}

func (tracker *issueTracker) CurrentUser() (common.User, error) {
	me, err := fetchMe()
	if err != nil {
		return nil, err
	}
	return &user{me}, nil
}

func (tracker *issueTracker) StartableStories() (stories []common.Story, err error) {
	var (
		client    = pivotal.NewClient(tracker.config.UserToken())
		projectId = tracker.config.ProjectId()
	)
	ptStories, err := searchStories(client, projectId,
		"state:%v,%v", pivotal.StoryStateUnstarted, pivotal.StoryStatePlanned)
	if err != nil {
		return nil, err
	}

	// Drop the features that are not estimated.
	estimatedStories := make([]*pivotal.Story, 0, len(ptStories))
	for _, story := range ptStories {
		if !(story.Type == pivotal.StoryTypeFeature && story.Estimate == nil) {
			estimatedStories = append(estimatedStories, story)
		}
	}

	ptStories = storiesMatchingByLabel(estimatedStories, tracker.config.IncludeStoryLabelFilter())

	return toCommonStories(ptStories, tracker), nil
}

func (tracker *issueTracker) StoriesInDevelopment() (stories []common.Story, err error) {
	var (
		client    = pivotal.NewClient(tracker.config.UserToken())
		projectId = tracker.config.ProjectId()
	)
	ptStories, err := searchStories(client, projectId,
		"state:%v AND -label:\"%v\" AND -label:\"%v\"",
		pivotal.StoryStateStarted, tracker.config.ReviewedLabel(), tracker.config.NoReviewLabel())
	if err != nil {
		return nil, err
	}
	return toCommonStories(ptStories, tracker), nil
}

func (tracker *issueTracker) ListStoriesByTag(tags []string) ([]common.Story, error) {
	// Convert tags to ids.
	ids := make([]string, 0, len(tags))
	for _, tag := range tags {
		id, err := tracker.StoryTagToReadableStoryId(tag)
		if err != nil {
			return nil, err
		}
		ids = append(ids, id)
	}

	// Fetch the relevant stories.
	var (
		client    = pivotal.NewClient(tracker.config.UserToken())
		projectId = tracker.config.ProjectId()
	)
	stories, err := listStoriesByIdOrdered(client, projectId, ids)
	if err != nil {
		return nil, err
	}

	// Convert to []common.Story and return.
	return toCommonStories(stories, tracker), nil
}

func (tracker *issueTracker) NextRelease(
	trunkVersion *version.Version,
	nextTrunkVersion *version.Version,
) (common.NextRelease, error) {

	return newNextRelease(tracker, trunkVersion, nextTrunkVersion)
}

func (tracker *issueTracker) RunningRelease(
	releaseVersion *version.Version,
) (common.RunningRelease, error) {

	return newRunningRelease(releaseVersion, tracker)
}

func (tracker *issueTracker) OpenStory(storyId string) error {
	return webbrowser.Open(fmt.Sprintf("https://pivotaltracker.com/story/show/%v", storyId))
}

func (tracker *issueTracker) StoryTagToReadableStoryId(tag string) (storyId string, err error) {
	parts := strings.Split(tag, "/")
	if len(parts) != 3 {
		return "", fmt.Errorf("invalid Pivotal Tracker Story-Id tag: %v", tag)
	}
	return parts[2], nil
}
<|endoftext|>


<commit_before>package main

import (
	//"fmt"
	"log"
	"net/http"
	"os"
	"strconv"
	"sync"

	"github.com/PuerkitoBio/goquery"
)

const (
	BASE_URL = "https://projecteuler.net/problem="
)

var wg sync.WaitGroup

func main() {
	for i := 1; i < 523; i++ {
		wg.Add(1)
		go fetch(i)
	}
	wg.Wait()
}

func fetch(i int) {
	num := strconv.Itoa(i)
	url := BASE_URL + num
	defer wg.Done()
	resp, err := http.Get(url)

	if err != nil {
		log.Fatal(err)
	} else {
		defer resp.Body.Close()
		doc, err := goquery.NewDocumentFromReader(resp.Body)
		if err != nil {
			log.Fatal(err)
		} else {
			var title string = ""
			var problemContent string = ""
			doc.Find("#content h2").Each(func(i int, s *goquery.Selection) {
				title = s.Text()
			})

			doc.Find("#content .problem_content").Each(func(i int, s *goquery.Selection) {
				problemContent = s.Text()
			})

			//write to file
			path, err := os.Getwd()
			if err != nil {
				log.Fatal(err)
			}

			pathSeparator := string(byte(os.PathSeparator))
			filepath := path + pathSeparator + pathSeparator + "prob" + num + ".go"

			f, err := os.Create(filepath)
			if err != nil {
				log.Fatal(err)
			}

			defer f.Close()

			f.WriteString("package main \n\n\n/**\n" + url + "\n\n" + title + "\n" + problemContent + "**/\n")
			f.Sync()
		}
	}
}
<commit_msg>separation<commit_after>package main

import (
	//"fmt"
	"log"
	"net/http"
	"os"
	"strconv"
	"sync"

	"github.com/PuerkitoBio/goquery"
)

const (
	BASE_URL = "https://projecteuler.net/problem="
)

var wg sync.WaitGroup
var pathSeparator string

func main() {
	pathSeparator = string(byte(os.PathSeparator))
	for i := 1; i < 523; i++ {
		wg.Add(1)
		go fetch(i)
	}
	wg.Wait()
}

func fetch(i int) {
	num := strconv.Itoa(i)
	url := BASE_URL + num
	defer wg.Done()
	resp, err := http.Get(url)

	if err != nil {
		log.Fatal(err)
	} else {
		defer resp.Body.Close()
		doc, err := goquery.NewDocumentFromReader(resp.Body)
		if err != nil {
			log.Fatal(err)
		} else {
			var title string = ""
			var problemContent string = ""
			doc.Find("#content h2").Each(func(i int, s *goquery.Selection) {
				title = s.Text()
			})

			doc.Find("#content .problem_content").Each(func(i int, s *goquery.Selection) {
				problemContent = s.Text()
			})

			//write to file
			path, err := os.Getwd()
			if err != nil {
				log.Fatal(err)
			}

			filepath := path + pathSeparator + "prob" + num + ".go"

			f, err := os.Create(filepath)
			if err != nil {
				log.Fatal(err)
			}

			defer f.Close()

			f.WriteString("package main \n\n\n/**\n" + url + "\n\n" + title + "\n" + problemContent + "**/\n")
			f.Sync()
		}
	}
}
<|endoftext|>


<commit_before>// Copyright © 2016 Thomas Rabaix <thomas.rabaix@gmail.com>.
//
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

package npm

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"
	"regexp"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/boltdb/bolt"
	"github.com/rande/goapp"
	"github.com/rande/gonode/core/vault"
	"github.com/rande/pkgmirror"
	"github.com/rande/pkgmirror/mirror/git"
)

var (
	NPM_ARCHIVE = regexp.MustCompile(`http(s|):\/\/([\w\.]+)\/(.*)`)
)

type NpmConfig struct {
	SourceServer    string
	PublicServer    string
	FallbackServers []string
	Path            string
	Code            []byte
}

func NewNpmService() *NpmService {
	return &NpmService{
		Config: &NpmConfig{
			SourceServer: "https://registry.npmjs.org",
			Code:         []byte("npm"),
			Path:         "./data/npm",
		},
		dbLock: &sync.Mutex{},
	}
}

type NpmService struct {
	DB        *bolt.DB
	Config    *NpmConfig
	Logger    *log.Entry
	GitConfig *git.GitConfig
	Vault     *vault.Vault
	lock      bool
	dbLock    *sync.Mutex
}

func (ns *NpmService) Init(app *goapp.App) error {
	var err error

	ns.Logger.Info("Init")

	ns.DB, err = bolt.Open(fmt.Sprintf("%s/%s.db", ns.Config.Path, ns.Config.Code), 0600, &bolt.Options{
		Timeout:  1 * time.Second,
		ReadOnly: false,
	})

	if err != nil {
		return err
	}

	return ns.DB.Update(func(tx *bolt.Tx) error {
		_, err := tx.CreateBucketIfNotExists(ns.Config.Code)

		return err
	})
}

func (ns *NpmService) Serve(state *goapp.GoroutineState) error {
	ns.Logger.Info("Starting Npm Service")

	for {
		ns.Logger.Info("Starting a new sync...")

		ns.SyncPackages()

		ns.Logger.Info("Wait before starting a new sync...")
		time.Sleep(60 * 15 * time.Second)
	}
}

func (ns *NpmService) End() error {
	return nil
}

func (ns *NpmService) SyncPackages() error {
	logger := ns.Logger.WithFields(log.Fields{
		"action": "SyncPackages",
	})

	logger.Info("Starting SyncPackages")

	p := make(map[string]*json.RawMessage)

	logger.WithFields(log.Fields{
		"url": fmt.Sprintf("%s/-/all", ns.Config.SourceServer),
	}).Info("Load all packages")

	f, _ := os.Create(fmt.Sprintf("%s/all_%s.json", ns.Config.Path, string(ns.Config.Code)))
	defer f.Close()

	if resp, err := http.Get(fmt.Sprintf("%s/-/all", ns.Config.SourceServer)); err != nil {
		logger.WithError(err).Error("Unable to download npm packages")

		return err
	} else {
		io.Copy(f, resp.Body)

		resp.Body.Close()
	}

	if err := pkgmirror.LoadStruct(fmt.Sprintf("%s/all_%s.json", ns.Config.Path, string(ns.Config.Code)), &p); err != nil {
		logger.WithError(err).Error("Unable to load all npm packages")
	}

	logger.WithFields(log.Fields{
		"url": fmt.Sprintf("%s/-/all", ns.Config.SourceServer),
	}).Info("End loading packages's metadata")

	dm := pkgmirror.NewWorkerManager(60, func(id int, data <-chan interface{}, result chan interface{}) {
		for raw := range data {
			sp := raw.(ShortPackageDefinition)

			p := &FullPackageDefinition{}
			url := fmt.Sprintf("%s/%s", ns.Config.SourceServer, sp.Name)

			logger.WithFields(log.Fields{
				"package": sp.Name,
				"worker":  id,
				"url":     url,
			}).Debug("Loading package information")

			if err := pkgmirror.LoadRemoteStruct(url, p); err != nil {
				logger.WithFields(log.Fields{
					"package": sp.Name,
					"error":   err.Error(),
					"worker":  id,
				}).Error("Error loading package information")

				continue
			}

			sp.FullPackageDefinition = *p
			result <- sp
		}
	})

	dm.ResultCallback(func(data interface{}) {
		pkg := data.(ShortPackageDefinition)

		ns.savePackage(&pkg)
	})

	dm.Start()

	for name, raw := range p {
		if name == "_updated" {
			continue
		}

		sp := &ShortPackageDefinition{}
		tp := &ShortPackageDefinition{}

		if err := json.Unmarshal(*raw, sp); err != nil {
			logger.WithFields(log.Fields{
				"error":   err,
				"package": name,
			}).Error("Unable to unmarshal remote data")

			continue
		}

		store := true
		ns.DB.View(func(tx *bolt.Tx) error {
			b := tx.Bucket(ns.Config.Code)

			if err := json.Unmarshal(b.Get([]byte(fmt.Sprintf("%s.meta", name))), tp); err == nil {
				if tp.Time.Modified == sp.Time.Modified {
					store = false
				}
			}

			return nil
		})

		if store {
			logger.WithFields(log.Fields{
				"package": name,
			}).Debug("Add/Update package to process")

			dm.Add(*sp)
		} else {
			logger.WithFields(log.Fields{
				"package": name,
			}).Debug("Skip package")
		}
	}

	logger.Info("Wait worker to complete")

	dm.Wait()

	return nil
}

func (ns *NpmService) savePackage(pkg *ShortPackageDefinition) error {
	return ns.DB.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket(ns.Config.Code)

		logger := ns.Logger.WithFields(log.Fields{
			"package": pkg.Name,
		})

		logger.Info("Save package information")

		if data, err := json.Marshal(pkg); err != nil {
			return err
		} else {
			if err := b.Put([]byte(fmt.Sprintf("%s.meta", pkg.Name)), data); err != nil {
				logger.WithError(err).Error("Unable to save package meta")
			} else {
				logger.Debug("Save package meta")
			}
		}

		for _, version := range pkg.FullPackageDefinition.Versions {
			if results := NPM_ARCHIVE.FindStringSubmatch(version.Dist.Tarball); len(results) > 0 {
				version.Dist.Tarball = fmt.Sprintf("%s/npm/%s/%s", ns.Config.PublicServer, string(ns.Config.Code), results[3])
			} else {
				logger.WithFields(log.Fields{
					"error":   "regexp does not match",
					"tarball": version.Dist.Tarball,
				}).Error("Unable to find host")
			}
		}

		data, _ := json.Marshal(&pkg.FullPackageDefinition)

		data, err := pkgmirror.Compress(data)

		if err != nil {
			logger.WithError(err).Error("Unable to compress data")

			return err
		}

		// store the path
		if err := b.Put([]byte(pkg.Name), data); err != nil {
			logger.WithError(err).Error("Error updating/creating definition")

			return err
		} else {
			logger.Debug("Save package")
		}

		return nil
	})
}

func (ns *NpmService) Get(key string) ([]byte, error) {
	var data []byte

	ns.Logger.WithFields(log.Fields{
		"action": "Get",
		"key":    key,
	}).Info("Get raw data")

	err := ns.DB.View(func(tx *bolt.Tx) error {
		b := tx.Bucket(ns.Config.Code)

		raw := b.Get([]byte(key))

		if len(raw) == 0 {
			return pkgmirror.EmptyKeyError
		}

		data = make([]byte, len(raw))

		copy(data, raw)

		return nil
	})

	return data, err
}

func (ns *NpmService) WriteArchive(w io.Writer, pkg, version string) error {

	logger := ns.Logger.WithFields(log.Fields{
		"package": pkg,
		"version": version,
		"action":  "WriteArchive",
	})

	vaultKey := fmt.Sprintf("%s/%s", pkg, version)

	if !ns.Vault.Has(vaultKey) {
		url := fmt.Sprintf("%s/%s/-/%s-%s.tgz", ns.Config.SourceServer, pkg, pkg, version)

		logger.WithField("url", url).Info("Create vault entry")

		resp, err := http.Get(url)

		if err != nil {
			return err
		}

		defer resp.Body.Close()

		if resp.StatusCode != http.StatusOK {
			return pkgmirror.ResourceNotFoundError
		}

		meta := vault.NewVaultMetadata()
		meta["path"] = pkg
		meta["version"] = version

		if _, err := ns.Vault.Put(vaultKey, meta, resp.Body); err != nil {
			logger.WithError(err).Info("Error while writing into vault")

			ns.Vault.Remove(vaultKey)

			return err
		}
	}

	logger.Info("Read vault entry")
	if _, err := ns.Vault.Get(vaultKey, w); err != nil {
		return err
	}

	return nil
}
<commit_msg>feat(npm): tweak path<commit_after>// Copyright © 2016 Thomas Rabaix <thomas.rabaix@gmail.com>.
//
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

package npm

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"
	"regexp"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/boltdb/bolt"
	"github.com/rande/goapp"
	"github.com/rande/gonode/core/vault"
	"github.com/rande/pkgmirror"
	"github.com/rande/pkgmirror/mirror/git"
)

var (
	NPM_ARCHIVE = regexp.MustCompile(`http(s|):\/\/([\w\.]+)\/(.*)`)
)

type NpmConfig struct {
	SourceServer    string
	PublicServer    string
	FallbackServers []string
	Path            string
	Code            []byte
}

func NewNpmService() *NpmService {
	return &NpmService{
		Config: &NpmConfig{
			SourceServer: "https://registry.npmjs.org",
			Code:         []byte("npm"),
			Path:         "./data/npm",
		},
		dbLock: &sync.Mutex{},
	}
}

type NpmService struct {
	DB        *bolt.DB
	Config    *NpmConfig
	Logger    *log.Entry
	GitConfig *git.GitConfig
	Vault     *vault.Vault
	lock      bool
	dbLock    *sync.Mutex
}

func (ns *NpmService) Init(app *goapp.App) error {
	var err error

	ns.Logger.Info("Init")

	ns.DB, err = bolt.Open(fmt.Sprintf("%s/%s.db", ns.Config.Path, ns.Config.Code), 0600, &bolt.Options{
		Timeout:  1 * time.Second,
		ReadOnly: false,
	})

	if err != nil {
		return err
	}

	return ns.DB.Update(func(tx *bolt.Tx) error {
		_, err := tx.CreateBucketIfNotExists(ns.Config.Code)

		return err
	})
}

func (ns *NpmService) Serve(state *goapp.GoroutineState) error {
	ns.Logger.Info("Starting Npm Service")

	for {
		ns.Logger.Info("Starting a new sync...")

		ns.SyncPackages()

		ns.Logger.Info("Wait before starting a new sync...")
		time.Sleep(60 * 15 * time.Second)
	}
}

func (ns *NpmService) End() error {
	return nil
}

func (ns *NpmService) SyncPackages() error {
	logger := ns.Logger.WithFields(log.Fields{
		"action": "SyncPackages",
	})

	logger.Info("Starting SyncPackages")

	p := make(map[string]*json.RawMessage)

	logger.WithFields(log.Fields{
		"url": fmt.Sprintf("%s/-/all", ns.Config.SourceServer),
	}).Info("Load all packages")

	f, _ := os.Create(fmt.Sprintf("%s/%s_all.json", ns.Config.Path, string(ns.Config.Code)))
	defer f.Close()

	if resp, err := http.Get(fmt.Sprintf("%s/-/all", ns.Config.SourceServer)); err != nil {
		logger.WithError(err).Error("Unable to download npm packages")

		return err
	} else {
		io.Copy(f, resp.Body)

		resp.Body.Close()
	}

	if err := pkgmirror.LoadStruct(fmt.Sprintf("%s/%s_all.json", ns.Config.Path, string(ns.Config.Code)), &p); err != nil {
		logger.WithError(err).Error("Unable to load all npm packages")
	}

	logger.WithFields(log.Fields{
		"url": fmt.Sprintf("%s/-/all", ns.Config.SourceServer),
	}).Info("End loading packages's metadata")

	dm := pkgmirror.NewWorkerManager(60, func(id int, data <-chan interface{}, result chan interface{}) {
		for raw := range data {
			sp := raw.(ShortPackageDefinition)

			p := &FullPackageDefinition{}
			url := fmt.Sprintf("%s/%s", ns.Config.SourceServer, sp.Name)

			logger.WithFields(log.Fields{
				"package": sp.Name,
				"worker":  id,
				"url":     url,
			}).Debug("Loading package information")

			if err := pkgmirror.LoadRemoteStruct(url, p); err != nil {
				logger.WithFields(log.Fields{
					"package": sp.Name,
					"error":   err.Error(),
					"worker":  id,
				}).Error("Error loading package information")

				continue
			}

			sp.FullPackageDefinition = *p
			result <- sp
		}
	})

	dm.ResultCallback(func(data interface{}) {
		pkg := data.(ShortPackageDefinition)

		ns.savePackage(&pkg)
	})

	dm.Start()

	for name, raw := range p {
		if name == "_updated" {
			continue
		}

		sp := &ShortPackageDefinition{}
		tp := &ShortPackageDefinition{}

		if err := json.Unmarshal(*raw, sp); err != nil {
			logger.WithFields(log.Fields{
				"error":   err,
				"package": name,
			}).Error("Unable to unmarshal remote data")

			continue
		}

		store := true
		ns.DB.View(func(tx *bolt.Tx) error {
			b := tx.Bucket(ns.Config.Code)

			if err := json.Unmarshal(b.Get([]byte(fmt.Sprintf("%s.meta", name))), tp); err == nil {
				if tp.Time.Modified == sp.Time.Modified {
					store = false
				}
			}

			return nil
		})

		if store {
			logger.WithFields(log.Fields{
				"package": name,
			}).Debug("Add/Update package to process")

			dm.Add(*sp)
		} else {
			logger.WithFields(log.Fields{
				"package": name,
			}).Debug("Skip package")
		}
	}

	logger.Info("Wait worker to complete")

	dm.Wait()

	return nil
}

func (ns *NpmService) savePackage(pkg *ShortPackageDefinition) error {
	return ns.DB.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket(ns.Config.Code)

		logger := ns.Logger.WithFields(log.Fields{
			"package": pkg.Name,
		})

		logger.Info("Save package information")

		if data, err := json.Marshal(pkg); err != nil {
			return err
		} else {
			if err := b.Put([]byte(fmt.Sprintf("%s.meta", pkg.Name)), data); err != nil {
				logger.WithError(err).Error("Unable to save package meta")
			} else {
				logger.Debug("Save package meta")
			}
		}

		for _, version := range pkg.FullPackageDefinition.Versions {
			if results := NPM_ARCHIVE.FindStringSubmatch(version.Dist.Tarball); len(results) > 0 {
				version.Dist.Tarball = fmt.Sprintf("%s/npm/%s/%s", ns.Config.PublicServer, string(ns.Config.Code), results[3])
			} else {
				logger.WithFields(log.Fields{
					"error":   "regexp does not match",
					"tarball": version.Dist.Tarball,
				}).Error("Unable to find host")
			}
		}

		data, _ := json.Marshal(&pkg.FullPackageDefinition)

		data, err := pkgmirror.Compress(data)

		if err != nil {
			logger.WithError(err).Error("Unable to compress data")

			return err
		}

		// store the path
		if err := b.Put([]byte(pkg.Name), data); err != nil {
			logger.WithError(err).Error("Error updating/creating definition")

			return err
		} else {
			logger.Debug("Save package")
		}

		return nil
	})
}

func (ns *NpmService) Get(key string) ([]byte, error) {
	var data []byte

	ns.Logger.WithFields(log.Fields{
		"action": "Get",
		"key":    key,
	}).Info("Get raw data")

	err := ns.DB.View(func(tx *bolt.Tx) error {
		b := tx.Bucket(ns.Config.Code)

		raw := b.Get([]byte(key))

		if len(raw) == 0 {
			return pkgmirror.EmptyKeyError
		}

		data = make([]byte, len(raw))

		copy(data, raw)

		return nil
	})

	return data, err
}

func (ns *NpmService) WriteArchive(w io.Writer, pkg, version string) error {

	logger := ns.Logger.WithFields(log.Fields{
		"package": pkg,
		"version": version,
		"action":  "WriteArchive",
	})

	vaultKey := fmt.Sprintf("%s/%s", pkg, version)

	if !ns.Vault.Has(vaultKey) {
		url := fmt.Sprintf("%s/%s/-/%s-%s.tgz", ns.Config.SourceServer, pkg, pkg, version)

		logger.WithField("url", url).Info("Create vault entry")

		resp, err := http.Get(url)

		if err != nil {
			return err
		}

		defer resp.Body.Close()

		if resp.StatusCode != http.StatusOK {
			return pkgmirror.ResourceNotFoundError
		}

		meta := vault.NewVaultMetadata()
		meta["path"] = pkg
		meta["version"] = version

		if _, err := ns.Vault.Put(vaultKey, meta, resp.Body); err != nil {
			logger.WithError(err).Info("Error while writing into vault")

			ns.Vault.Remove(vaultKey)

			return err
		}
	}

	logger.Info("Read vault entry")
	if _, err := ns.Vault.Get(vaultKey, w); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/x509"
	"io/ioutil"
	"math/big"
	"testing"
)

func TestGetDistributionPoint(t *testing.T) {
	cert, _ := readCertificate("./testdata/certificate.pem")
	server, _ := getCRLDistributionPoint(cert)

	expected := "http://crl3.digicert.com/ssca-sha2-g3.crl"

	if server != expected {
		t.Errorf("expected %q, got %q", expected, server)
	}
}

func TestGetDestributionPointFromCertWithoutCRL(t *testing.T) {
	cert, _ := readCertificate("./testdata/cloudflare_origin_ca_rsa_root.crt")
	server, _ := getCRLDistributionPoint(cert)

	expected := ""

	if server != expected {
		t.Errorf("expected %q, got %q", expected, server)
	}
}

func TestFindCert(t *testing.T) {
	// NOTE: DigiCert SHA2 Extended Validation Server CA CRL
	crl, _ := ioutil.ReadFile("./testdata/sha2-ev-server-g2.crl")
	resp, err := x509.ParseCRL(crl)
	if err != nil {
		t.Fatal(err)
	}

	// Serial belongs to https://censys.io/certificates/39e31c9f5913e4ed68c9582de80c8be4689608f622075d0c81b6fe52dfe2db82
	s := new(big.Int)
	s.SetString("17015245701990644280577643802745589798", 10)

	test := FindCert(s, resp)

	if test == nil {
		t.Errorf("expected to find revoked certificate with serial number %q", s.String())
	}
}

func TestFindNonExistingRevokedCert(t *testing.T) {
	// NOTE: DigiCert SHA2 Extended Validation Server CA CRL
	crl, _ := ioutil.ReadFile("./testdata/sha2-ev-server-g2.crl")
	resp, err := x509.ParseCRL(crl)
	if err != nil {
		t.Fatal(err)
	}

	test := FindCert(big.NewInt(0), resp)

	if test != nil {
		t.Error("did not expect to find a revoked certificate")
	}
}

func TestGetCRLResponse(t *testing.T) {
	client = &MockHttpClient{}
	cert, err := readCertificate("./testdata/cisco_revoked.pem")

	if err != nil {
		t.Fatal(err)
	}

	st, err := GetCRLResponse(client, cert)

	if err != nil {
		t.Fatal(err)
	}

	expected := "Revoked"
	if st.Status != expected {
		t.Errorf("expected %q, got %q", expected, st.Status)
	}
}

func TestGetCRLResponseNotRevoked(t *testing.T) {
	client = &MockHttpClient{}
	cert, err := readCertificate("./testdata/twitter.pem")

	if err != nil {
		t.Fatal(err)
	}

	st, err := GetCRLResponse(client, cert)

	if err != nil {
		t.Fatal(err)
	}

	expected := "Good"
	if st.Status != expected {
		t.Errorf("expected %q, got %q", expected, st.Status)
	}
}
<commit_msg>Fixed method call.<commit_after>package main

import (
	"crypto/x509"
	"io/ioutil"
	"math/big"
	"testing"
)

func TestGetDistributionPoint(t *testing.T) {
	cert, _ := readCertificate("./testdata/certificate.pem")
	server, _ := getCRLDistributionPoint(cert)

	expected := "http://crl3.digicert.com/ssca-sha2-g3.crl"

	if server != expected {
		t.Errorf("expected %q, got %q", expected, server)
	}
}

func TestGetDestributionPointFromCertWithoutCRL(t *testing.T) {
	cert, _ := readCertificate("./testdata/cloudflare_origin_ca_rsa_root.crt")
	server, _ := getCRLDistributionPoint(cert)

	expected := ""

	if server != expected {
		t.Errorf("expected %q, got %q", expected, server)
	}
}

func TestFindCert(t *testing.T) {
	// NOTE: DigiCert SHA2 Extended Validation Server CA CRL
	crl, _ := ioutil.ReadFile("./testdata/sha2-ev-server-g2.crl")
	resp, err := x509.ParseCRL(crl)
	if err != nil {
		t.Fatal(err)
	}

	// Serial belongs to https://censys.io/certificates/39e31c9f5913e4ed68c9582de80c8be4689608f622075d0c81b6fe52dfe2db82
	s := new(big.Int)
	s.SetString("17015245701990644280577643802745589798", 10)

	test := findCert(s, resp)

	if test == nil {
		t.Errorf("expected to find revoked certificate with serial number %q", s.String())
	}
}

func TestFindNonExistingRevokedCert(t *testing.T) {
	// NOTE: DigiCert SHA2 Extended Validation Server CA CRL
	crl, _ := ioutil.ReadFile("./testdata/sha2-ev-server-g2.crl")
	resp, err := x509.ParseCRL(crl)
	if err != nil {
		t.Fatal(err)
	}

	test := findCert(big.NewInt(0), resp)

	if test != nil {
		t.Error("did not expect to find a revoked certificate")
	}
}

func TestGetCRLResponse(t *testing.T) {
	client = &MockHttpClient{}
	cert, err := readCertificate("./testdata/cisco_revoked.pem")

	if err != nil {
		t.Fatal(err)
	}

	st, err := GetCRLResponse(client, cert)

	if err != nil {
		t.Fatal(err)
	}

	expected := "Revoked"
	if st.Status != expected {
		t.Errorf("expected %q, got %q", expected, st.Status)
	}
}

func TestGetCRLResponseNotRevoked(t *testing.T) {
	client = &MockHttpClient{}
	cert, err := readCertificate("./testdata/twitter.pem")

	if err != nil {
		t.Fatal(err)
	}

	st, err := GetCRLResponse(client, cert)

	if err != nil {
		t.Fatal(err)
	}

	expected := "Good"
	if st.Status != expected {
		t.Errorf("expected %q, got %q", expected, st.Status)
	}
}
<|endoftext|>


<commit_before>package osmpbf_test

import (
	"fmt"
	"github.com/qedus/osmpbf"
	"io"
	"os"
	"testing"
	"time"
)

func TestDecoder(t *testing.T) {
	//f, err := os.Open("planet-latest.osm.pbf")
	f, err := os.Open("greater_london.osm.pbf")
	if err != nil {
		t.Fatal(err.Error())
	}
	defer f.Close()

	d := osmpbf.NewDecoder(f)
	n, w, r := 0, 0, 0
	count, start := 0, time.Now()
	now := start
	for {
		if v, err := d.Decode(); err == io.EOF {
			break
		} else if err != nil {
			t.Fatal(err.Error())
		} else {
			switch v := v.(type) {
			case *osmpbf.Node:
				n++
			case *osmpbf.Way:
				w++
			case *osmpbf.Relation:
				r++
			default:
				t.Fatalf("unknwon type %T", v)
			}
		}
		count++
		if count%1000000 == 0 {
			newNow := time.Now()
			dur := newNow.Sub(now)
			fmt.Printf("%s\t%s\t%d\t%d\t%d\t%d\n",
				newNow.Sub(start), dur, count, n, w, r)
			now = newNow
		}
	}
	fmt.Printf("Nodes: %d, Ways: %d, Relations: %d\n", n, w, r)
}
<commit_msg>Add test and benchmark.<commit_after>package osmpbf

import (
	"fmt"
	"io"
	"os"
	"reflect"
	"testing"
	"time"
)

const (
	// originally downloaded from http://download.geofabrik.de/europe/great-britain/england/greater-london.html
	London    = "greater-london-140324.osm.pbf"
	LondonURL = "https://googledrive.com/host/0B8pisLiGtmqDR3dOR3hrWUpRTVE"
)

func init() {
	_, err := os.Stat(London)
	if os.IsNotExist(err) {
		panic(fmt.Sprintf("\nDownload %s from %s.\nFor example: 'wget -O %s %s'", London, LondonURL, London, LondonURL))
	}
}

func TestDecoder(t *testing.T) {
	f, err := os.Open(London)
	if err != nil {
		t.Fatal(err)
	}
	defer f.Close()

	var n *Node
	en := &Node{
		ID:  18088578,
		Lat: 51.5442632,
		Lon: -0.2010027,
		Tags: map[string]string{
			"alt_name":   "The King's Head",
			"amenity":    "pub",
			"created_by": "JOSM",
			"name":       "The Luminaire",
			"note":       "Live music venue too",
		},
	}

	var w *Way
	ew := &Way{
		ID:      4257116,
		NodeIDs: []int64{21544864, 333731851, 333731852, 333731850, 333731855, 333731858, 333731854, 108047, 769984352, 21544864},
		Tags: map[string]string{
			"area":    "yes",
			"highway": "pedestrian",
			"name":    "Fitzroy Square",
		},
	}

	var r *Relation
	er := &Relation{
		ID: 7677,
		Members: []Member{
			Member{ID: 4875932, Type: WayType, Role: "outer"},
			Member{ID: 4894305, Type: WayType, Role: "inner"},
		},
		Tags: map[string]string{
			"created_by": "Potlatch 0.9c",
			"type":       "multipolygon",
		},
	}

	d := NewDecoder(f)
	for {
		if v, err := d.Decode(); err == io.EOF {
			break
		} else if err != nil {
			t.Fatal(err)
		} else {
			switch v := v.(type) {
			case *Node:
				if v.ID == en.ID {
					n = v
				}
			case *Way:
				if v.ID == ew.ID {
					w = v
				}
			case *Relation:
				if v.ID == er.ID {
					r = v
				}
			default:
				t.Fatalf("unknown type %T", v)
			}
		}
	}

	if !reflect.DeepEqual(en, n) {
		t.Errorf("\nExpected: %#v\nActual:   %#v", en, n)
	}
	if !reflect.DeepEqual(ew, w) {
		t.Errorf("\nExpected: %#v\nActual:   %#v", ew, w)
	}
	if !reflect.DeepEqual(er, r) {
		t.Errorf("\nExpected: %#v\nActual:   %#v", er, r)
	}
}

func BenchmarkDecoder(b *testing.B) {
	f, err := os.Open(London)
	if err != nil {
		b.Fatal(err)
	}
	defer f.Close()

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		f.Seek(0, 0)
		d := NewDecoder(f)
		n, w, r, count, start := 0, 0, 0, 0, time.Now()
		for {
			if v, err := d.Decode(); err == io.EOF {
				break
			} else if err != nil {
				b.Fatal(err)
			} else {
				switch v := v.(type) {
				case *Node:
					n++
				case *Way:
					w++
				case *Relation:
					r++
				default:
					b.Fatalf("unknown type %T", v)
				}
			}
			count++
		}
		b.Logf("Done in %.3f seconds. Total: %d, Nodes: %d, Ways: %d, Relations: %d\n",
			time.Now().Sub(start).Seconds(), count, n, w, r)
	}
}
<|endoftext|>