<commit_before>package overview

import (
	"fmt"
	"github.com/pkg/errors"
	"log"
	"sync"

	"github.com/davecgh/go-spew/spew"
	"github.com/heptio/developer-dash/internal/cluster"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/watch"
)

// StopFunc tells a watch to stop watching a namespace.
type StopFunc func()

type Watch2 interface {
	Start() (StopFunc, error)
}

// Watch watches a namespace's objects.
type Watch struct {
	clusterClient cluster.ClientInterface
	cache         Cache
	namespace     string
}

// NewWatch creates an instance of Watch.
func NewWatch(namespace string, clusterClient cluster.ClientInterface, c Cache) *Watch {
	return &Watch{
		namespace:     namespace,
		clusterClient: clusterClient,
		cache:         c,
	}
}

// Start starts the watch. It returns a stop function and an error.
func (w *Watch) Start() (StopFunc, error) {
	resources, err := w.resources()
	if err != nil {
		return nil, err
	}

	var watchers []watch.Interface

	for _, resource := range resources {
		dc, err := w.clusterClient.DynamicClient()
		if err != nil {
			return nil, err
		}

		nri := dc.Resource(resource).Namespace(w.namespace)

		watcher, err := nri.Watch(metav1.ListOptions{})
		if err != nil {
			return nil, errors.Wrap(err, fmt.Sprintf("did not create watcher for %s/%s/%s on %s namespace", resource.Group, resource.Version, resource.Resource, w.namespace))
		}

		watchers = append(watchers, watcher)
	}

	done := make(chan struct{})

	events, shutdownCh := consumeEvents(done, watchers)

	go func() {
		for event := range events {
			w.eventHandler(event)
		}
	}()

	stopFn := func() {
		done <- struct{}{}
		<-shutdownCh
	}

	return stopFn, nil
}

func (w *Watch) eventHandler(event watch.Event) {
	u, ok := event.Object.(*unstructured.Unstructured)
	if !ok {
		return
	}

	switch t := event.Type; t {
	case watch.Added:
		if err := w.cache.Store(u); err != nil {
			log.Printf("store object: %v", err)
		}
	case watch.Modified:
		if err := w.cache.Store(u); err != nil {
			log.Printf("store object: %v", err)
		}
	case watch.Deleted:
		if err := w.cache.Delete(u); err != nil {
			log.Printf("store object: %v", err)
		}
	case watch.Error:
		log.Printf("unknown log err: %s", spew.Sdump(event))
	default:
		log.Printf("unknown event %q", t)
	}
}

func (w *Watch) resources() ([]schema.GroupVersionResource, error) {
	discoveryClient, err := w.clusterClient.DiscoveryClient()
	if err != nil {
		return nil, err
	}

	lists, err := discoveryClient.ServerResources()
	if err != nil {
		return nil, err
	}

	var gvrs []schema.GroupVersionResource

	for _, list := range lists {
		gv, err := schema.ParseGroupVersion(list.GroupVersion)
		if err != nil {
			return nil, err
		}

		for _, res := range list.APIResources {
			if !res.Namespaced {
				continue
			}
			if hasList(res) {

				gvr := schema.GroupVersionResource{
					Group:    gv.Group,
					Version:  gv.Version,
					Resource: res.Name,
				}

				gvrs = append(gvrs, gvr)

			}
		}
	}

	return gvrs, nil
}

func hasList(res metav1.APIResource) bool {
	hasList := false
	hasWatch := false

	for _, verb := range res.Verbs {
		hasList = verb == "list"
		hasWatch = verb == "watch"
	}

	return hasList && hasWatch
}

func consumeEvents(done <-chan struct{}, watchers []watch.Interface) (chan watch.Event, chan struct{}) {
	var wg sync.WaitGroup

	wg.Add(len(watchers))

	events := make(chan watch.Event)

	shutdownComplete := make(chan struct{})

	for _, watcher := range watchers {
		go func(watcher watch.Interface) {
			for event := range watcher.ResultChan() {
				events <- event
			}
		}(watcher)
	}

	go func() {
		<-done
		for _, watch := range watchers {
			watch.Stop()
			wg.Done()
		}

		shutdownComplete <- struct{}{}
	}()

	go func() {
		// wait for all watchers to exit.
		wg.Wait()
		close(events)
	}()

	return events, shutdownComplete
}
<commit_msg>fixes hasList fn<commit_after>package overview

import (
	"fmt"
	"github.com/pkg/errors"
	"log"
	"sync"

	"github.com/davecgh/go-spew/spew"
	"github.com/heptio/developer-dash/internal/cluster"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/watch"
)

// StopFunc tells a watch to stop watching a namespace.
type StopFunc func()

type Watch2 interface {
	Start() (StopFunc, error)
}

// Watch watches a namespace's objects.
type Watch struct {
	clusterClient cluster.ClientInterface
	cache         Cache
	namespace     string
}

// NewWatch creates an instance of Watch.
func NewWatch(namespace string, clusterClient cluster.ClientInterface, c Cache) *Watch {
	return &Watch{
		namespace:     namespace,
		clusterClient: clusterClient,
		cache:         c,
	}
}

// Start starts the watch. It returns a stop function and an error.
func (w *Watch) Start() (StopFunc, error) {
	resources, err := w.resources()
	if err != nil {
		return nil, err
	}

	var watchers []watch.Interface

	for _, resource := range resources {
		dc, err := w.clusterClient.DynamicClient()
		if err != nil {
			return nil, err
		}

		nri := dc.Resource(resource).Namespace(w.namespace)

		watcher, err := nri.Watch(metav1.ListOptions{})
		if err != nil {
			return nil, errors.Wrap(err, fmt.Sprintf("did not create watcher for %s/%s/%s on %s namespace", resource.Group, resource.Version, resource.Resource, w.namespace))
		}

		watchers = append(watchers, watcher)
	}

	done := make(chan struct{})

	events, shutdownCh := consumeEvents(done, watchers)

	go func() {
		for event := range events {
			w.eventHandler(event)
		}
	}()

	stopFn := func() {
		done <- struct{}{}
		<-shutdownCh
	}

	return stopFn, nil
}

func (w *Watch) eventHandler(event watch.Event) {
	u, ok := event.Object.(*unstructured.Unstructured)
	if !ok {
		return
	}

	switch t := event.Type; t {
	case watch.Added:
		if err := w.cache.Store(u); err != nil {
			log.Printf("store object: %v", err)
		}
	case watch.Modified:
		if err := w.cache.Store(u); err != nil {
			log.Printf("store object: %v", err)
		}
	case watch.Deleted:
		if err := w.cache.Delete(u); err != nil {
			log.Printf("store object: %v", err)
		}
	case watch.Error:
		log.Printf("unknown log err: %s", spew.Sdump(event))
	default:
		log.Printf("unknown event %q", t)
	}
}

func (w *Watch) resources() ([]schema.GroupVersionResource, error) {
	discoveryClient, err := w.clusterClient.DiscoveryClient()
	if err != nil {
		return nil, err
	}

	lists, err := discoveryClient.ServerResources()
	if err != nil {
		return nil, err
	}

	var gvrs []schema.GroupVersionResource

	for _, list := range lists {
		gv, err := schema.ParseGroupVersion(list.GroupVersion)
		if err != nil {
			return nil, err
		}

		for _, res := range list.APIResources {
			if !res.Namespaced {
				continue
			}
			if hasList(res) {

				gvr := schema.GroupVersionResource{
					Group:    gv.Group,
					Version:  gv.Version,
					Resource: res.Name,
				}

				gvrs = append(gvrs, gvr)

			}
		}
	}

	return gvrs, nil
}

func hasList(res metav1.APIResource) bool {
	m := make(map[string]bool)

	for _, v := range res.Verbs {
		m[v] = true
	}

	return m["list"] && m["watch"]
}

func consumeEvents(done <-chan struct{}, watchers []watch.Interface) (chan watch.Event, chan struct{}) {
	var wg sync.WaitGroup

	wg.Add(len(watchers))

	events := make(chan watch.Event)

	shutdownComplete := make(chan struct{})

	for _, watcher := range watchers {
		go func(watcher watch.Interface) {
			for event := range watcher.ResultChan() {
				events <- event
			}
		}(watcher)
	}

	go func() {
		<-done
		for _, watch := range watchers {
			watch.Stop()
			wg.Done()
		}

		shutdownComplete <- struct{}{}
	}()

	go func() {
		// wait for all watchers to exit.
		wg.Wait()
		close(events)
	}()

	return events, shutdownComplete
}
<|endoftext|>


<commit_before>package core_test

import (
	"testing"

	"github.com/golang/protobuf/proto"

	"github.com/OpenBazaar/openbazaar-go/core"
	"github.com/OpenBazaar/openbazaar-go/pb"
	"github.com/OpenBazaar/openbazaar-go/test/factory"
)

func TestFactoryCryptoListingCoinDivisibilityMatchesConst(t *testing.T) {
	if factory.NewCryptoListing("blu").Metadata.CoinDivisibility != core.DefaultCoinDivisibility {
		t.Fatal("DefaultCoinDivisibility constant has changed. Please update factory value.")
	}
}

func TestValidShippingRegion(t *testing.T) {
	check := map[int32]error{
		0: core.ErrShippingRegionMustBeSet,

		1:   nil,
		247: nil,
		248: core.ErrShippingRegionUndefined,
		500: nil,

		501: core.ErrShippingRegionMustNotBeContinent,
		502: core.ErrShippingRegionMustNotBeContinent,
		503: core.ErrShippingRegionMustNotBeContinent,
		504: core.ErrShippingRegionMustNotBeContinent,
		505: core.ErrShippingRegionMustNotBeContinent,
		506: core.ErrShippingRegionMustNotBeContinent,
		507: core.ErrShippingRegionMustNotBeContinent,
		508: core.ErrShippingRegionMustNotBeContinent,

		509: core.ErrShippingRegionUndefined,
		510: core.ErrShippingRegionUndefined,
		511: core.ErrShippingRegionUndefined,

		5678:   core.ErrShippingRegionUndefined,
		123456: core.ErrShippingRegionUndefined,
	}
	// check error map
	m1 := make(map[int32]error)
	for v := range check {
		cc := pb.CountryCode(v)
		listing := factory.NewShippingRegionListing("asdfasdf", cc)
		for _, shippingOption := range listing.ShippingOptions {
			if err := core.ValidShippingRegion(shippingOption); err != nil {
				m1[v] = err
			} else {
				m1[v] = nil
			}
		}
	}

	// check the countrycodes.proto
	m2 := make(map[int32]error)
	for v := range pb.CountryCode_name {
		cc := pb.CountryCode(v)
		listing := factory.NewShippingRegionListing("asdfasdf", cc)
		for _, shippingOption := range listing.ShippingOptions {
			if err := core.ValidShippingRegion(shippingOption); err != nil {
				m2[v] = err
			} else {
				m2[v] = nil
			}
		}
	}

	for v, errtype := range m2 {
		if check[v] != errtype {
			t.Fatalf("( cc: %d, '%v' != '%v' ) : CountryCode does not match tests error checking map.\n", v, errtype, check[v])
		}
	}

	check[247] = core.ErrShippingRegionUndefined
	for v, errtype := range m1 {
		if check[v] != errtype {
			t.Logf("Should fail: ( cc: %d, '%v' != '%v' ) : CountryCode does not match tests error checking map.\n", v, errtype, check[v])
		}
	}
	for v, errtype := range m2 {
		if check[v] != errtype {
			t.Logf("Should fail: ( cc: %d, '%v' != '%v' ) : CountryCode does not match tests error checking map.\n", v, errtype, check[v])
		}
	}
}

func TestListingProtobufAlias(t *testing.T) {
	countrycodes := []pb.CountryCode{
		pb.CountryCode(212),
		pb.CountryCode(pb.CountryCode_SWAZILAND),
		pb.CountryCode(pb.CountryCode_ESWATINI),
	}
	for _, cc := range countrycodes {
		listing := factory.NewShippingRegionListing("swaziland_eswatini", cc)
		marshalled, _ := proto.Marshal(listing)
		unmarshalledListing := &pb.Listing{}
		err := proto.Unmarshal(marshalled, unmarshalledListing)
		if err != nil {
			t.Fatal(err)
		}
		for _, region := range unmarshalledListing.ShippingOptions[0].Regions {
			if region != pb.CountryCode_ESWATINI {
				t.Fatal("Error returning pb.CountryCode_ESWATINI")
			}
		}
	}
	countrycodes2 := [][]pb.CountryCode{
		{pb.CountryCode(pb.CountryCode_UNITED_KINGDOM), pb.CountryCode(pb.CountryCode_ESWATINI)},
		{pb.CountryCode(pb.CountryCode_UNITED_KINGDOM), pb.CountryCode(pb.CountryCode_SWAZILAND)},

		{pb.CountryCode(pb.CountryCode_SWAZILAND), pb.CountryCode(pb.CountryCode_ESWATINI)},
		{pb.CountryCode(pb.CountryCode_ESWATINI), pb.CountryCode(pb.CountryCode_SWAZILAND)},
		{pb.CountryCode(pb.CountryCode_SWAZILAND)},
		{pb.CountryCode(pb.CountryCode_ESWATINI)},
	}
	for id, cc := range countrycodes2 {
		listing2 := factory.NewShippingRegionsProtoBufAlias("swaziland_eswatini", cc)
		marshalled2, _ := proto.Marshal(listing2)
		unmarshalled2Listing := &pb.Listing{}
		err := proto.Unmarshal(marshalled2, unmarshalled2Listing)
		if err != nil {
			t.Fatal(err)
		}
		for _, region := range unmarshalled2Listing.ShippingOptions[0].Regions {
			if region != pb.CountryCode(pb.CountryCode_SWAZILAND) && region != pb.CountryCode(pb.CountryCode_ESWATINI) {
				if id == 0 || id == 1 {
					t.Logf("( %v ) : should fail : SWAZILAND/ESWATINI proto allow_alias ", unmarshalled2Listing.ShippingOptions[0].Regions)
				}
				if id != 0 && id != 1 {
					t.Fatalf("( %v ) : failed : SWAZILAND/ESWATINI proto allow_alias", unmarshalled2Listing.ShippingOptions[0].Regions)
				}
			}
		}
	}
}
<commit_msg>[#1207] Simplify tests around Listing proto use of allow_alias<commit_after>package core_test

import (
	"testing"

	"github.com/golang/protobuf/proto"

	"github.com/OpenBazaar/openbazaar-go/core"
	"github.com/OpenBazaar/openbazaar-go/pb"
	"github.com/OpenBazaar/openbazaar-go/test/factory"
)

func TestFactoryCryptoListingCoinDivisibilityMatchesConst(t *testing.T) {
	if factory.NewCryptoListing("blu").Metadata.CoinDivisibility != core.DefaultCoinDivisibility {
		t.Fatal("DefaultCoinDivisibility constant has changed. Please update factory value.")
	}
}

var expectedErrorStatesForValidShippingRegion = map[int32]error{
	0: core.ErrShippingRegionMustBeSet,

	1:   nil,
	247: nil,
	248: core.ErrShippingRegionUndefined,
	500: nil,

	501: core.ErrShippingRegionMustNotBeContinent,
	502: core.ErrShippingRegionMustNotBeContinent,
	503: core.ErrShippingRegionMustNotBeContinent,
	504: core.ErrShippingRegionMustNotBeContinent,
	505: core.ErrShippingRegionMustNotBeContinent,
	506: core.ErrShippingRegionMustNotBeContinent,
	507: core.ErrShippingRegionMustNotBeContinent,
	508: core.ErrShippingRegionMustNotBeContinent,

	509: core.ErrShippingRegionUndefined,
	510: core.ErrShippingRegionUndefined,
	511: core.ErrShippingRegionUndefined,

	5678:   core.ErrShippingRegionUndefined,
	123456: core.ErrShippingRegionUndefined,
}

func TestValidShippingRegionErrorCases(t *testing.T) {
	for example, expectedResult := range expectedErrorStatesForValidShippingRegion {
		listing := factory.NewShippingRegionListing("asdfasdf", pb.CountryCode(example))
		for _, shippingOption := range listing.ShippingOptions {
			if result := core.ValidShippingRegion(shippingOption); result != expectedResult {
				t.Errorf("unexpected result using CountryCode (%d): %s", example, result)
			}
		}
	}
}

func TestValidShippingRegionUsingDefinedCountryCodes(t *testing.T) {
	for countryCode := range pb.CountryCode_name {
		listing := factory.NewShippingRegionListing("asdfasdf", pb.CountryCode(countryCode))
		for _, shippingOption := range listing.ShippingOptions {
			result := core.ValidShippingRegion(shippingOption)
			if result != expectedErrorStatesForValidShippingRegion[countryCode] {
				t.Errorf("unexpected result using CountryCode (%d): %s", countryCode, result)
			}
		}
	}
}

func TestListingProtobufAlias(t *testing.T) {
	countrycodes := []pb.CountryCode{
		pb.CountryCode(212),
		pb.CountryCode(pb.CountryCode_SWAZILAND),
		pb.CountryCode(pb.CountryCode_ESWATINI),
	}
	for _, cc := range countrycodes {
		listing := factory.NewShippingRegionListing("swaziland_eswatini", cc)
		marshalled, _ := proto.Marshal(listing)
		unmarshalledListing := &pb.Listing{}
		err := proto.Unmarshal(marshalled, unmarshalledListing)
		if err != nil {
			t.Fatal(err)
		}
		for _, region := range unmarshalledListing.ShippingOptions[0].Regions {
			if region != pb.CountryCode_ESWATINI {
				t.Fatal("expected aliased CountryCode to always unmarshal as pb.CountryCode_ESWATINI but didn't")
			}
		}
	}
}
<|endoftext|>


<commit_before>package interpreter

import "io"
import "os"
import "os/exec"
import "regexp"

type NextT int

const (
	THROUGH  NextT = 0
	CONTINUE NextT = 1
	SHUTDOWN NextT = 2
)

type Interpreter struct {
	exec.Cmd
	HookCount    int
	IsBackGround bool
	Closer       io.Closer
}

func New() *Interpreter {
	return new(Interpreter)
}

func (this *Interpreter) Clone() *Interpreter {
	rv := new(Interpreter)
	rv.Stdout = this.Stdout
	rv.Stderr = this.Stderr
	rv.Stdin = this.Stdin
	rv.HookCount = this.HookCount
	// Dont Copy 'Closer' and 'IsBackGround'
	return rv
}

type ArgsHookT func(args []string) []string

var argsHook = func(args []string) []string {
	return args
}

func SetArgsHook(argsHook_ ArgsHookT) (rv ArgsHookT) {
	rv, argsHook = argsHook, argsHook_
	return
}

type HookT func(*Interpreter) (NextT, error)

var hook = func(*Interpreter) (NextT, error) {
	return THROUGH, nil
}

func SetHook(hook_ HookT) (rv HookT) {
	rv, hook = hook, hook_
	return
}

var errorStatusPattern = regexp.MustCompile("^exit status ([0-9]+)")
var ErrorLevel string

func (this *Interpreter) Interpret(text string) (NextT, error) {
	statements := Parse(text)
	for _, pipeline := range statements {
		var pipeIn *os.File = nil
		for _, state := range pipeline {
			var cmd Interpreter
			cmd.HookCount = this.HookCount
			if this.Stderr != nil {
				cmd.Stdin = this.Stdin
			} else {
				cmd.Stdin = os.Stdin
			}
			if this.Stdout != nil {
				cmd.Stdout = this.Stdout
			} else {
				cmd.Stdout = os.Stdout
			}
			if this.Stderr != nil {
				cmd.Stderr = this.Stderr
			} else {
				cmd.Stderr = os.Stderr
			}
			if pipeIn != nil {
				cmd.Stdin = pipeIn
				pipeIn = nil
			}
			if state.Redirect[0].Path != "" {
				fd, err := os.Open(state.Redirect[0].Path)
				if err != nil {
					return CONTINUE, err
				}
				defer fd.Close()
				cmd.Stdin = fd
			}
			if state.Redirect[1].Path != "" {
				var fd *os.File
				var err error
				if state.Redirect[1].IsAppend {
					fd, err = os.OpenFile(state.Redirect[1].Path, os.O_APPEND, 0666)
				} else {
					fd, err = os.OpenFile(state.Redirect[1].Path, os.O_CREATE, 0666)
				}
				if err != nil {
					return CONTINUE, err
				}
				defer fd.Close()
				cmd.Stdout = fd
			}
			if state.Redirect[2].Path != "" {
				var fd *os.File
				var err error
				if state.Redirect[2].IsAppend {
					fd, err = os.OpenFile(state.Redirect[2].Path, os.O_APPEND, 0666)
				} else {
					fd, err = os.OpenFile(state.Redirect[2].Path, os.O_CREATE, 0666)
				}
				if err != nil {
					return CONTINUE, err
				}
				defer fd.Close()
				cmd.Stderr = fd
			}
			var err error = nil
			var pipeOut *os.File = nil
			if state.Term == "|" {
				pipeIn, pipeOut, err = os.Pipe()
				if err != nil {
					return CONTINUE, err
				}
				// defer pipeIn.Close()
				cmd.Stdout = pipeOut
			}
			var whatToDo NextT

			isBackGround := (state.Term == "|" || state.Term == "&")

			if len(state.Argv) > 0 {
				if argsHook != nil {
					state.Argv = argsHook(state.Argv)
				}
				cmd.Args = state.Argv
				cmd.IsBackGround = isBackGround
				cmd.Closer = pipeOut
				whatToDo, err = hook(&cmd)
				if whatToDo == THROUGH {
					cmd.Path, err = exec.LookPath(state.Argv[0])
					if err == nil {
						if isBackGround {
							err = cmd.Start()
						} else {
							err = cmd.Run()
						}
					}
				} else {
					pipeOut = nil
				}
			}
			if pipeOut != nil {
				// pipeOut.Close()
			}
			if err != nil {
				m := errorStatusPattern.FindStringSubmatch(err.Error())
				if m != nil {
					ErrorLevel = m[1]
					err = nil
				} else {
					ErrorLevel = "-1"
				}
			} else {
				ErrorLevel = "0"
			}
			if whatToDo == SHUTDOWN {
				return SHUTDOWN, err
			}
			if err != nil {
				return CONTINUE, err
			}
		}
	}
	return CONTINUE, nil
}
<commit_msg>Fix: Redirect output did not truncate existing file.(#27)<commit_after>package interpreter

import "io"
import "os"
import "os/exec"
import "regexp"

type NextT int

const (
	THROUGH  NextT = 0
	CONTINUE NextT = 1
	SHUTDOWN NextT = 2
)

type Interpreter struct {
	exec.Cmd
	HookCount    int
	IsBackGround bool
	Closer       io.Closer
}

func New() *Interpreter {
	return new(Interpreter)
}

func (this *Interpreter) Clone() *Interpreter {
	rv := new(Interpreter)
	rv.Stdout = this.Stdout
	rv.Stderr = this.Stderr
	rv.Stdin = this.Stdin
	rv.HookCount = this.HookCount
	// Dont Copy 'Closer' and 'IsBackGround'
	return rv
}

type ArgsHookT func(args []string) []string

var argsHook = func(args []string) []string {
	return args
}

func SetArgsHook(argsHook_ ArgsHookT) (rv ArgsHookT) {
	rv, argsHook = argsHook, argsHook_
	return
}

type HookT func(*Interpreter) (NextT, error)

var hook = func(*Interpreter) (NextT, error) {
	return THROUGH, nil
}

func SetHook(hook_ HookT) (rv HookT) {
	rv, hook = hook, hook_
	return
}

var errorStatusPattern = regexp.MustCompile("^exit status ([0-9]+)")
var ErrorLevel string

func (this *Interpreter) Interpret(text string) (NextT, error) {
	statements := Parse(text)
	for _, pipeline := range statements {
		var pipeIn *os.File = nil
		for _, state := range pipeline {
			var cmd Interpreter
			cmd.HookCount = this.HookCount
			if this.Stderr != nil {
				cmd.Stdin = this.Stdin
			} else {
				cmd.Stdin = os.Stdin
			}
			if this.Stdout != nil {
				cmd.Stdout = this.Stdout
			} else {
				cmd.Stdout = os.Stdout
			}
			if this.Stderr != nil {
				cmd.Stderr = this.Stderr
			} else {
				cmd.Stderr = os.Stderr
			}
			if pipeIn != nil {
				cmd.Stdin = pipeIn
				pipeIn = nil
			}
			if state.Redirect[0].Path != "" {
				fd, err := os.Open(state.Redirect[0].Path)
				if err != nil {
					return CONTINUE, err
				}
				defer fd.Close()
				cmd.Stdin = fd
			}
			if state.Redirect[1].Path != "" {
				var fd *os.File
				var err error
				if state.Redirect[1].IsAppend {
					fd, err = os.OpenFile(state.Redirect[1].Path,
						os.O_APPEND, 0666)
				} else {
					fd, err = os.OpenFile(state.Redirect[1].Path,
						os.O_CREATE|os.O_TRUNC, 0666)
				}
				if err != nil {
					return CONTINUE, err
				}
				defer fd.Close()
				cmd.Stdout = fd
			}
			if state.Redirect[2].Path != "" {
				var fd *os.File
				var err error
				if state.Redirect[2].IsAppend {
					fd, err = os.OpenFile(state.Redirect[2].Path, os.O_APPEND, 0666)
				} else {
					fd, err = os.OpenFile(state.Redirect[2].Path, os.O_CREATE, 0666)
				}
				if err != nil {
					return CONTINUE, err
				}
				defer fd.Close()
				cmd.Stderr = fd
			}
			var err error = nil
			var pipeOut *os.File = nil
			if state.Term == "|" {
				pipeIn, pipeOut, err = os.Pipe()
				if err != nil {
					return CONTINUE, err
				}
				// defer pipeIn.Close()
				cmd.Stdout = pipeOut
			}
			var whatToDo NextT

			isBackGround := (state.Term == "|" || state.Term == "&")

			if len(state.Argv) > 0 {
				if argsHook != nil {
					state.Argv = argsHook(state.Argv)
				}
				cmd.Args = state.Argv
				cmd.IsBackGround = isBackGround
				cmd.Closer = pipeOut
				whatToDo, err = hook(&cmd)
				if whatToDo == THROUGH {
					cmd.Path, err = exec.LookPath(state.Argv[0])
					if err == nil {
						if isBackGround {
							err = cmd.Start()
						} else {
							err = cmd.Run()
						}
					}
				} else {
					pipeOut = nil
				}
			}
			if pipeOut != nil {
				// pipeOut.Close()
			}
			if err != nil {
				m := errorStatusPattern.FindStringSubmatch(err.Error())
				if m != nil {
					ErrorLevel = m[1]
					err = nil
				} else {
					ErrorLevel = "-1"
				}
			} else {
				ErrorLevel = "0"
			}
			if whatToDo == SHUTDOWN {
				return SHUTDOWN, err
			}
			if err != nil {
				return CONTINUE, err
			}
		}
	}
	return CONTINUE, nil
}
<|endoftext|>


<commit_before>// Copyright 2014 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.

// Package state provides a caching layer atop the Ethereum state trie.
package state

import (
	"fmt"
	"math/big"

	"github.com/ethereum/go-ethereum/common"
	"github.com/ethereum/go-ethereum/core/vm"
	"github.com/ethereum/go-ethereum/ethdb"
	"github.com/ethereum/go-ethereum/logger"
	"github.com/ethereum/go-ethereum/logger/glog"
	"github.com/ethereum/go-ethereum/rlp"
	"github.com/ethereum/go-ethereum/trie"
)

// The starting nonce determines the default nonce when new accounts are being
// created.
var StartingNonce uint64

// StateDBs within the ethereum protocol are used to store anything
// within the merkle trie. StateDBs take care of caching and storing
// nested states. It's the general query interface to retrieve:
// * Contracts
// * Accounts
type StateDB struct {
	db   ethdb.Database
	trie *trie.SecureTrie

	stateObjects map[string]*StateObject

	refund *big.Int

	thash, bhash common.Hash
	txIndex      int
	logs         map[common.Hash]vm.Logs
	logSize      uint
}

// Create a new state from a given trie
func New(root common.Hash, db ethdb.Database) (*StateDB, error) {
	tr, err := trie.NewSecure(root, db)
	if err != nil {
		return nil, err
	}
	return &StateDB{
		db:           db,
		trie:         tr,
		stateObjects: make(map[string]*StateObject),
		refund:       new(big.Int),
		logs:         make(map[common.Hash]vm.Logs),
	}, nil
}

func (self *StateDB) StartRecord(thash, bhash common.Hash, ti int) {
	self.thash = thash
	self.bhash = bhash
	self.txIndex = ti
}

func (self *StateDB) AddLog(log *vm.Log) {
	log.TxHash = self.thash
	log.BlockHash = self.bhash
	log.TxIndex = uint(self.txIndex)
	log.Index = self.logSize
	self.logs[self.thash] = append(self.logs[self.thash], log)
	self.logSize++
}

func (self *StateDB) GetLogs(hash common.Hash) vm.Logs {
	return self.logs[hash]
}

func (self *StateDB) Logs() vm.Logs {
	var logs vm.Logs
	for _, lgs := range self.logs {
		logs = append(logs, lgs...)
	}
	return logs
}

func (self *StateDB) AddRefund(gas *big.Int) {
	self.refund.Add(self.refund, gas)
}

func (self *StateDB) HasAccount(addr common.Address) bool {
	return self.GetStateObject(addr) != nil
}

func (self *StateDB) Exist(addr common.Address) bool {
	return self.GetStateObject(addr) != nil
}

func (self *StateDB) GetAccount(addr common.Address) vm.Account {
	return self.GetStateObject(addr)
}

// Retrieve the balance from the given address or 0 if object not found
func (self *StateDB) GetBalance(addr common.Address) *big.Int {
	stateObject := self.GetStateObject(addr)
	if stateObject != nil {
		return stateObject.balance
	}

	return common.Big0
}

func (self *StateDB) GetNonce(addr common.Address) uint64 {
	stateObject := self.GetStateObject(addr)
	if stateObject != nil {
		return stateObject.nonce
	}

	return 0
}

func (self *StateDB) GetCode(addr common.Address) []byte {
	stateObject := self.GetStateObject(addr)
	if stateObject != nil {
		return stateObject.code
	}

	return nil
}

func (self *StateDB) GetState(a common.Address, b common.Hash) common.Hash {
	stateObject := self.GetStateObject(a)
	if stateObject != nil {
		return stateObject.GetState(b)
	}

	return common.Hash{}
}

func (self *StateDB) IsDeleted(addr common.Address) bool {
	stateObject := self.GetStateObject(addr)
	if stateObject != nil {
		return stateObject.remove
	}
	return false
}

/*
 * SETTERS
 */

func (self *StateDB) AddBalance(addr common.Address, amount *big.Int) {
	stateObject := self.GetOrNewStateObject(addr)
	if stateObject != nil {
		stateObject.AddBalance(amount)
	}
}

func (self *StateDB) SetNonce(addr common.Address, nonce uint64) {
	stateObject := self.GetOrNewStateObject(addr)
	if stateObject != nil {
		stateObject.SetNonce(nonce)
	}
}

func (self *StateDB) SetCode(addr common.Address, code []byte) {
	stateObject := self.GetOrNewStateObject(addr)
	if stateObject != nil {
		stateObject.SetCode(code)
	}
}

func (self *StateDB) SetState(addr common.Address, key common.Hash, value common.Hash) {
	stateObject := self.GetOrNewStateObject(addr)
	if stateObject != nil {
		stateObject.SetState(key, value)
	}
}

func (self *StateDB) Delete(addr common.Address) bool {
	stateObject := self.GetStateObject(addr)
	if stateObject != nil {
		stateObject.MarkForDeletion()
		stateObject.balance = new(big.Int)

		return true
	}

	return false
}

//
// Setting, updating & deleting state object methods
//

// Update the given state object and apply it to state trie
func (self *StateDB) UpdateStateObject(stateObject *StateObject) {
	addr := stateObject.Address()
	data, err := rlp.EncodeToBytes(stateObject)
	if err != nil {
		panic(fmt.Errorf("can't encode object at %x: %v", addr[:], err))
	}
	self.trie.Update(addr[:], data)
}

// Delete the given state object and delete it from the state trie
func (self *StateDB) DeleteStateObject(stateObject *StateObject) {
	stateObject.deleted = true

	addr := stateObject.Address()
	self.trie.Delete(addr[:])
	//delete(self.stateObjects, addr.Str())
}

// Retrieve a state object given my the address. Nil if not found
func (self *StateDB) GetStateObject(addr common.Address) (stateObject *StateObject) {
	stateObject = self.stateObjects[addr.Str()]
	if stateObject != nil {
		if stateObject.deleted {
			stateObject = nil
		}

		return stateObject
	}

	data := self.trie.Get(addr[:])
	if len(data) == 0 {
		return nil
	}
	stateObject, err := DecodeObject(addr, self.db, data)
	if err != nil {
		glog.Errorf("can't decode object at %x: %v", addr[:], err)
		return nil
	}
	self.SetStateObject(stateObject)
	return stateObject
}

func (self *StateDB) SetStateObject(object *StateObject) {
	self.stateObjects[object.Address().Str()] = object
}

// Retrieve a state object or create a new state object if nil
func (self *StateDB) GetOrNewStateObject(addr common.Address) *StateObject {
	stateObject := self.GetStateObject(addr)
	if stateObject == nil || stateObject.deleted {
		stateObject = self.CreateStateObject(addr)
	}

	return stateObject
}

// NewStateObject create a state object whether it exist in the trie or not
func (self *StateDB) newStateObject(addr common.Address) *StateObject {
	if glog.V(logger.Core) {
		glog.Infof("(+) %x\n", addr)
	}

	stateObject := NewStateObject(addr, self.db)
	stateObject.SetNonce(StartingNonce)
	self.stateObjects[addr.Str()] = stateObject

	return stateObject
}

// Creates creates a new state object and takes ownership. This is different from "NewStateObject"
func (self *StateDB) CreateStateObject(addr common.Address) *StateObject {
	// Get previous (if any)
	so := self.GetStateObject(addr)
	// Create a new one
	newSo := self.newStateObject(addr)

	// If it existed set the balance to the new account
	if so != nil {
		newSo.balance = so.balance
	}

	return newSo
}

func (self *StateDB) CreateAccount(addr common.Address) vm.Account {
	return self.CreateStateObject(addr)
}

//
// Setting, copying of the state methods
//

func (self *StateDB) Copy() *StateDB {
	// ignore error - we assume state-to-be-copied always exists
	state, _ := New(common.Hash{}, self.db)
	state.trie = self.trie
	for k, stateObject := range self.stateObjects {
		state.stateObjects[k] = stateObject.Copy()
	}

	state.refund.Set(self.refund)

	for hash, logs := range self.logs {
		state.logs[hash] = make(vm.Logs, len(logs))
		copy(state.logs[hash], logs)
	}
	state.logSize = self.logSize

	return state
}

func (self *StateDB) Set(state *StateDB) {
	self.trie = state.trie
	self.stateObjects = state.stateObjects

	self.refund = state.refund
	self.logs = state.logs
	self.logSize = state.logSize
}

func (self *StateDB) GetRefund() *big.Int {
	return self.refund
}

// IntermediateRoot computes the current root hash of the state trie.
// It is called in between transactions to get the root hash that
// goes into transaction receipts.
func (s *StateDB) IntermediateRoot() common.Hash {
	s.refund = new(big.Int)
	for _, stateObject := range s.stateObjects {
		if stateObject.dirty {
			if stateObject.remove {
				s.DeleteStateObject(stateObject)
			} else {
				stateObject.Update()
				s.UpdateStateObject(stateObject)
			}
			stateObject.dirty = false
		}
	}
	return s.trie.Hash()
}

// Commit commits all state changes to the database.
func (s *StateDB) Commit() (root common.Hash, err error) {
	root, batch := s.CommitBatch()
	return root, batch.Write()
}

// CommitBatch commits all state changes to a write batch but does not
// execute the batch. It is used to validate state changes against
// the root hash stored in a block.
func (s *StateDB) CommitBatch() (root common.Hash, batch ethdb.Batch) {
	batch = s.db.NewBatch()
	root, _ = s.commit(batch)
	return root, batch
}

func (s *StateDB) commit(db trie.DatabaseWriter) (common.Hash, error) {
	s.refund = new(big.Int)

	for _, stateObject := range s.stateObjects {
		if stateObject.remove {
			// If the object has been removed, don't bother syncing it
			// and just mark it for deletion in the trie.
			s.DeleteStateObject(stateObject)
		} else {
			// Write any contract code associated with the state object
			if len(stateObject.code) > 0 {
				if err := db.Put(stateObject.codeHash, stateObject.code); err != nil {
					return common.Hash{}, err
				}
			}
			// Write any storage changes in the state object to its trie.
			stateObject.Update()

			// Commit the trie of the object to the batch.
			// This updates the trie root internally, so
			// getting the root hash of the storage trie
			// through UpdateStateObject is fast.
			if _, err := stateObject.trie.CommitTo(db); err != nil {
				return common.Hash{}, err
			}
			// Update the object in the account trie.
			s.UpdateStateObject(stateObject)
		}
		stateObject.dirty = false
	}
	return s.trie.CommitTo(db)
}

func (self *StateDB) Refunds() *big.Int {
	return self.refund
}

// Debug stuff
func (self *StateDB) CreateOutputForDiff() {
	for _, stateObject := range self.stateObjects {
		stateObject.CreateOutputForDiff()
	}
}
<commit_msg>core/state: return the starting nonce for non-existent accs (testnet)<commit_after>// Copyright 2014 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.

// Package state provides a caching layer atop the Ethereum state trie.
package state

import (
	"fmt"
	"math/big"

	"github.com/ethereum/go-ethereum/common"
	"github.com/ethereum/go-ethereum/core/vm"
	"github.com/ethereum/go-ethereum/ethdb"
	"github.com/ethereum/go-ethereum/logger"
	"github.com/ethereum/go-ethereum/logger/glog"
	"github.com/ethereum/go-ethereum/rlp"
	"github.com/ethereum/go-ethereum/trie"
)

// The starting nonce determines the default nonce when new accounts are being
// created.
var StartingNonce uint64

// StateDBs within the ethereum protocol are used to store anything
// within the merkle trie. StateDBs take care of caching and storing
// nested states. It's the general query interface to retrieve:
// * Contracts
// * Accounts
type StateDB struct {
	db   ethdb.Database
	trie *trie.SecureTrie

	stateObjects map[string]*StateObject

	refund *big.Int

	thash, bhash common.Hash
	txIndex      int
	logs         map[common.Hash]vm.Logs
	logSize      uint
}

// Create a new state from a given trie
func New(root common.Hash, db ethdb.Database) (*StateDB, error) {
	tr, err := trie.NewSecure(root, db)
	if err != nil {
		return nil, err
	}
	return &StateDB{
		db:           db,
		trie:         tr,
		stateObjects: make(map[string]*StateObject),
		refund:       new(big.Int),
		logs:         make(map[common.Hash]vm.Logs),
	}, nil
}

func (self *StateDB) StartRecord(thash, bhash common.Hash, ti int) {
	self.thash = thash
	self.bhash = bhash
	self.txIndex = ti
}

func (self *StateDB) AddLog(log *vm.Log) {
	log.TxHash = self.thash
	log.BlockHash = self.bhash
	log.TxIndex = uint(self.txIndex)
	log.Index = self.logSize
	self.logs[self.thash] = append(self.logs[self.thash], log)
	self.logSize++
}

func (self *StateDB) GetLogs(hash common.Hash) vm.Logs {
	return self.logs[hash]
}

func (self *StateDB) Logs() vm.Logs {
	var logs vm.Logs
	for _, lgs := range self.logs {
		logs = append(logs, lgs...)
	}
	return logs
}

func (self *StateDB) AddRefund(gas *big.Int) {
	self.refund.Add(self.refund, gas)
}

func (self *StateDB) HasAccount(addr common.Address) bool {
	return self.GetStateObject(addr) != nil
}

func (self *StateDB) Exist(addr common.Address) bool {
	return self.GetStateObject(addr) != nil
}

func (self *StateDB) GetAccount(addr common.Address) vm.Account {
	return self.GetStateObject(addr)
}

// Retrieve the balance from the given address or 0 if object not found
func (self *StateDB) GetBalance(addr common.Address) *big.Int {
	stateObject := self.GetStateObject(addr)
	if stateObject != nil {
		return stateObject.balance
	}

	return common.Big0
}

func (self *StateDB) GetNonce(addr common.Address) uint64 {
	stateObject := self.GetStateObject(addr)
	if stateObject != nil {
		return stateObject.nonce
	}

	return StartingNonce
}

func (self *StateDB) GetCode(addr common.Address) []byte {
	stateObject := self.GetStateObject(addr)
	if stateObject != nil {
		return stateObject.code
	}

	return nil
}

func (self *StateDB) GetState(a common.Address, b common.Hash) common.Hash {
	stateObject := self.GetStateObject(a)
	if stateObject != nil {
		return stateObject.GetState(b)
	}

	return common.Hash{}
}

func (self *StateDB) IsDeleted(addr common.Address) bool {
	stateObject := self.GetStateObject(addr)
	if stateObject != nil {
		return stateObject.remove
	}
	return false
}

/*
 * SETTERS
 */

func (self *StateDB) AddBalance(addr common.Address, amount *big.Int) {
	stateObject := self.GetOrNewStateObject(addr)
	if stateObject != nil {
		stateObject.AddBalance(amount)
	}
}

func (self *StateDB) SetNonce(addr common.Address, nonce uint64) {
	stateObject := self.GetOrNewStateObject(addr)
	if stateObject != nil {
		stateObject.SetNonce(nonce)
	}
}

func (self *StateDB) SetCode(addr common.Address, code []byte) {
	stateObject := self.GetOrNewStateObject(addr)
	if stateObject != nil {
		stateObject.SetCode(code)
	}
}

func (self *StateDB) SetState(addr common.Address, key common.Hash, value common.Hash) {
	stateObject := self.GetOrNewStateObject(addr)
	if stateObject != nil {
		stateObject.SetState(key, value)
	}
}

func (self *StateDB) Delete(addr common.Address) bool {
	stateObject := self.GetStateObject(addr)
	if stateObject != nil {
		stateObject.MarkForDeletion()
		stateObject.balance = new(big.Int)

		return true
	}

	return false
}

//
// Setting, updating & deleting state object methods
//

// Update the given state object and apply it to state trie
func (self *StateDB) UpdateStateObject(stateObject *StateObject) {
	addr := stateObject.Address()
	data, err := rlp.EncodeToBytes(stateObject)
	if err != nil {
		panic(fmt.Errorf("can't encode object at %x: %v", addr[:], err))
	}
	self.trie.Update(addr[:], data)
}

// Delete the given state object and delete it from the state trie
func (self *StateDB) DeleteStateObject(stateObject *StateObject) {
	stateObject.deleted = true

	addr := stateObject.Address()
	self.trie.Delete(addr[:])
	//delete(self.stateObjects, addr.Str())
}

// Retrieve a state object given my the address. Nil if not found
func (self *StateDB) GetStateObject(addr common.Address) (stateObject *StateObject) {
	stateObject = self.stateObjects[addr.Str()]
	if stateObject != nil {
		if stateObject.deleted {
			stateObject = nil
		}

		return stateObject
	}

	data := self.trie.Get(addr[:])
	if len(data) == 0 {
		return nil
	}
	stateObject, err := DecodeObject(addr, self.db, data)
	if err != nil {
		glog.Errorf("can't decode object at %x: %v", addr[:], err)
		return nil
	}
	self.SetStateObject(stateObject)
	return stateObject
}

func (self *StateDB) SetStateObject(object *StateObject) {
	self.stateObjects[object.Address().Str()] = object
}

// Retrieve a state object or create a new state object if nil
func (self *StateDB) GetOrNewStateObject(addr common.Address) *StateObject {
	stateObject := self.GetStateObject(addr)
	if stateObject == nil || stateObject.deleted {
		stateObject = self.CreateStateObject(addr)
	}

	return stateObject
}

// NewStateObject create a state object whether it exist in the trie or not
func (self *StateDB) newStateObject(addr common.Address) *StateObject {
	if glog.V(logger.Core) {
		glog.Infof("(+) %x\n", addr)
	}

	stateObject := NewStateObject(addr, self.db)
	stateObject.SetNonce(StartingNonce)
	self.stateObjects[addr.Str()] = stateObject

	return stateObject
}

// Creates creates a new state object and takes ownership. This is different from "NewStateObject"
func (self *StateDB) CreateStateObject(addr common.Address) *StateObject {
	// Get previous (if any)
	so := self.GetStateObject(addr)
	// Create a new one
	newSo := self.newStateObject(addr)

	// If it existed set the balance to the new account
	if so != nil {
		newSo.balance = so.balance
	}

	return newSo
}

func (self *StateDB) CreateAccount(addr common.Address) vm.Account {
	return self.CreateStateObject(addr)
}

//
// Setting, copying of the state methods
//

func (self *StateDB) Copy() *StateDB {
	// ignore error - we assume state-to-be-copied always exists
	state, _ := New(common.Hash{}, self.db)
	state.trie = self.trie
	for k, stateObject := range self.stateObjects {
		state.stateObjects[k] = stateObject.Copy()
	}

	state.refund.Set(self.refund)

	for hash, logs := range self.logs {
		state.logs[hash] = make(vm.Logs, len(logs))
		copy(state.logs[hash], logs)
	}
	state.logSize = self.logSize

	return state
}

func (self *StateDB) Set(state *StateDB) {
	self.trie = state.trie
	self.stateObjects = state.stateObjects

	self.refund = state.refund
	self.logs = state.logs
	self.logSize = state.logSize
}

func (self *StateDB) GetRefund() *big.Int {
	return self.refund
}

// IntermediateRoot computes the current root hash of the state trie.
// It is called in between transactions to get the root hash that
// goes into transaction receipts.
func (s *StateDB) IntermediateRoot() common.Hash {
	s.refund = new(big.Int)
	for _, stateObject := range s.stateObjects {
		if stateObject.dirty {
			if stateObject.remove {
				s.DeleteStateObject(stateObject)
			} else {
				stateObject.Update()
				s.UpdateStateObject(stateObject)
			}
			stateObject.dirty = false
		}
	}
	return s.trie.Hash()
}

// Commit commits all state changes to the database.
func (s *StateDB) Commit() (root common.Hash, err error) {
	root, batch := s.CommitBatch()
	return root, batch.Write()
}

// CommitBatch commits all state changes to a write batch but does not
// execute the batch. It is used to validate state changes against
// the root hash stored in a block.
func (s *StateDB) CommitBatch() (root common.Hash, batch ethdb.Batch) {
	batch = s.db.NewBatch()
	root, _ = s.commit(batch)
	return root, batch
}

func (s *StateDB) commit(db trie.DatabaseWriter) (common.Hash, error) {
	s.refund = new(big.Int)

	for _, stateObject := range s.stateObjects {
		if stateObject.remove {
			// If the object has been removed, don't bother syncing it
			// and just mark it for deletion in the trie.
			s.DeleteStateObject(stateObject)
		} else {
			// Write any contract code associated with the state object
			if len(stateObject.code) > 0 {
				if err := db.Put(stateObject.codeHash, stateObject.code); err != nil {
					return common.Hash{}, err
				}
			}
			// Write any storage changes in the state object to its trie.
			stateObject.Update()

			// Commit the trie of the object to the batch.
			// This updates the trie root internally, so
			// getting the root hash of the storage trie
			// through UpdateStateObject is fast.
			if _, err := stateObject.trie.CommitTo(db); err != nil {
				return common.Hash{}, err
			}
			// Update the object in the account trie.
			s.UpdateStateObject(stateObject)
		}
		stateObject.dirty = false
	}
	return s.trie.CommitTo(db)
}

func (self *StateDB) Refunds() *big.Int {
	return self.refund
}

// Debug stuff
func (self *StateDB) CreateOutputForDiff() {
	for _, stateObject := range self.stateObjects {
		stateObject.CreateOutputForDiff()
	}
}
<|endoftext|>


<commit_before>package environs_test

import (
	"io/ioutil"
	. "launchpad.net/gocheck"
	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/environs/config"
	_ "launchpad.net/juju-core/environs/dummy"
	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/testing"
	"launchpad.net/juju-core/version"
	"os"
	"path/filepath"
)

type suite struct{}

var _ = Suite(suite{})

var invalidConfigTests = []struct {
	env string
	err string
}{
	{"'", "YAML error:.*"},
	{`
default: unknown
environments:
    only:
        type: unknown
`, `default environment .* does not exist`,
	},
}

func (suite) TestInvalidConfig(c *C) {
	for i, t := range invalidConfigTests {
		c.Logf("running test %v", i)
		_, err := environs.ReadEnvironsBytes([]byte(t.env))
		c.Check(err, ErrorMatches, t.err)
	}
}

var invalidEnvTests = []struct {
	env  string
	name string
	err  string
}{
	{`
environments:
    only:
        foo: bar
`, "", `environment "only" has no type`,
	}, {`
environments:
    only:
        foo: bar
`, "only", `environment "only" has no type`,
	}, {`
environments:
    only:
        foo: bar
        type: crazy
`, "only", `environment "only" has an unknown provider type "crazy"`,
	}, {`
environments:
    only:
        type: dummy
`, "only", `.*state-server: expected bool, got nothing`,
	}, {`
environments:
    only:
        type: dummy
        state-server: false
        unknown-value: causes-an-error
`, "only", `.*unknown-value: expected nothing, got "causes-an-error"`,
	},
}

func (suite) TestInvalidEnv(c *C) {
	defer makeFakeHome(c, "only").restore()
	for i, t := range invalidEnvTests {
		c.Logf("running test %v", i)
		es, err := environs.ReadEnvironsBytes([]byte(t.env))
		c.Check(err, IsNil)
		e, err := es.Open(t.name)
		c.Check(err, ErrorMatches, t.err)
		c.Check(e, IsNil)
	}
}

var configTests = []struct {
	env   string
	check func(c *C, es *environs.Environs)
}{
	{`
environments:
    only:
        type: dummy
        state-server: false
`, func(c *C, es *environs.Environs) {
		e, err := es.Open("")
		c.Assert(err, IsNil)
		c.Assert(e.Name(), Equals, "only")
	}}, {`
default:
    invalid
environments:
    valid:
        type: dummy
        state-server: false
    invalid:
        type: crazy
`, func(c *C, es *environs.Environs) {
		e, err := es.Open("")
		c.Assert(err, ErrorMatches, `environment "invalid" has an unknown provider type "crazy"`)
		c.Assert(e, IsNil)
		e, err = es.Open("valid")
		c.Assert(err, IsNil)
		c.Assert(e.Name(), Equals, "valid")
	}}, {`
environments:
    one:
        type: dummy
        state-server: false
    two:
        type: dummy
        state-server: false
`, func(c *C, es *environs.Environs) {
		e, err := es.Open("")
		c.Assert(err, ErrorMatches, `no default environment found`)
		c.Assert(e, IsNil)
	}},
}

func (suite) TestConfig(c *C) {
	defer makeFakeHome(c, "only", "valid", "one", "two").restore()
	for i, t := range configTests {
		c.Logf("running test %v", i)
		es, err := environs.ReadEnvironsBytes([]byte(t.env))
		c.Assert(err, IsNil)
		t.check(c, es)
	}
}

func (suite) TestDefaultConfigFile(c *C) {
	defer makeFakeHome(c, "only").restore()

	env := `
environments:
    only:
        type: dummy
        state-server: false
        authorized-keys: i-am-a-key
`
	err := ioutil.WriteFile(homePath(".juju", "environments.yaml"), []byte(env), 0666)
	c.Assert(err, IsNil)

	es, err := environs.ReadEnvirons("")
	c.Assert(err, IsNil)
	e, err := es.Open("")
	c.Assert(err, IsNil)
	c.Assert(e.Name(), Equals, "only")
}

func (suite) TestNamedConfigFile(c *C) {
	defer makeFakeHome(c, "only").restore()

	env := `
environments:
    only:
        type: dummy
        state-server: false
        authorized-keys: i-am-a-key
`
	path := filepath.Join(c.MkDir(), "a-file")
	err := ioutil.WriteFile(path, []byte(env), 0666)
	c.Assert(err, IsNil)

	es, err := environs.ReadEnvirons(path)
	c.Assert(err, IsNil)
	e, err := es.Open("")
	c.Assert(err, IsNil)
	c.Assert(e.Name(), Equals, "only")
}

func (suite) TestConfigRoundTrip(c *C) {
	cfg, err := config.New(map[string]interface{}{
		"name":            "bladaam",
		"type":            "dummy",
		"state-server":    false,
		"authorized-keys": "i-am-a-key",
		"ca-cert":         testing.CACertPEM,
		"ca-private-key":  "",
	})
	c.Assert(err, IsNil)
	provider, err := environs.Provider(cfg.Type())
	c.Assert(err, IsNil)
	cfg, err = provider.Validate(cfg, nil)
	c.Assert(err, IsNil)
	env, err := environs.New(cfg)
	c.Assert(err, IsNil)
	c.Assert(cfg.AllAttrs(), DeepEquals, env.Config().AllAttrs())
}

func (suite) TestBootstrapConfig(c *C) {
	defer makeFakeHome(c, "bladaam").restore()
	cfg, err := config.New(map[string]interface{}{
		"name":            "bladaam",
		"type":            "dummy",
		"state-server":    false,
		"admin-secret":    "highly",
		"secret":          "um",
		"authorized-keys": "i-am-a-key",
		"ca-cert":         testing.CACertPEM,
		"ca-private-key":  testing.CAKeyPEM,
	})
	c.Assert(err, IsNil)
	provider, err := environs.Provider(cfg.Type())
	c.Assert(err, IsNil)

	tools := &state.Tools{
		URL:    "http://x",
		Binary: version.MustParseBinary("1.2.3-foo-bar"),
	}
	cfg1, err := environs.BootstrapConfig(provider, cfg, tools)
	c.Assert(err, IsNil)

	expect := cfg.AllAttrs()
	delete(expect, "secret")
	expect["admin-secret"] = ""
	expect["ca-private-key"] = nil
	expect["agent-version"] = "1.2.3"
	c.Assert(cfg1.AllAttrs(), DeepEquals, expect)
}

type fakeHome string

func makeFakeHome(c *C, certNames ...string) fakeHome {
	oldHome := os.Getenv("HOME")
	os.Setenv("HOME", c.MkDir())

	err := os.Mkdir(homePath(".juju"), 0777)
	c.Assert(err, IsNil)
	for _, name := range certNames {
		err := ioutil.WriteFile(homePath(".juju", name+"-cert.pem"), []byte(testing.CACertPEM), 0666)
		c.Assert(err, IsNil)
		err = ioutil.WriteFile(homePath(".juju", name+"-private-key.pem"), []byte(testing.CAKeyPEM), 0666)
		c.Assert(err, IsNil)
	}

	err = os.Mkdir(homePath(".ssh"), 0777)
	c.Assert(err, IsNil)
	err = ioutil.WriteFile(homePath(".ssh", "id_rsa.pub"), []byte("auth key\n"), 0666)
	c.Assert(err, IsNil)

	return fakeHome(oldHome)
}

func homePath(names ...string) string {
	all := append([]string{os.Getenv("HOME")}, names...)
	return filepath.Join(all...)
}

func (h fakeHome) restore() {
	os.Setenv("HOME", string(h))
}
<commit_msg>environs: fix ca-private-key nil; \<commit_after>package environs_test

import (
	"io/ioutil"
	. "launchpad.net/gocheck"
	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/environs/config"
	_ "launchpad.net/juju-core/environs/dummy"
	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/testing"
	"launchpad.net/juju-core/version"
	"os"
	"path/filepath"
)

type suite struct{}

var _ = Suite(suite{})

var invalidConfigTests = []struct {
	env string
	err string
}{
	{"'", "YAML error:.*"},
	{`
default: unknown
environments:
    only:
        type: unknown
`, `default environment .* does not exist`,
	},
}

func (suite) TestInvalidConfig(c *C) {
	for i, t := range invalidConfigTests {
		c.Logf("running test %v", i)
		_, err := environs.ReadEnvironsBytes([]byte(t.env))
		c.Check(err, ErrorMatches, t.err)
	}
}

var invalidEnvTests = []struct {
	env  string
	name string
	err  string
}{
	{`
environments:
    only:
        foo: bar
`, "", `environment "only" has no type`,
	}, {`
environments:
    only:
        foo: bar
`, "only", `environment "only" has no type`,
	}, {`
environments:
    only:
        foo: bar
        type: crazy
`, "only", `environment "only" has an unknown provider type "crazy"`,
	}, {`
environments:
    only:
        type: dummy
`, "only", `.*state-server: expected bool, got nothing`,
	}, {`
environments:
    only:
        type: dummy
        state-server: false
        unknown-value: causes-an-error
`, "only", `.*unknown-value: expected nothing, got "causes-an-error"`,
	},
}

func (suite) TestInvalidEnv(c *C) {
	defer makeFakeHome(c, "only").restore()
	for i, t := range invalidEnvTests {
		c.Logf("running test %v", i)
		es, err := environs.ReadEnvironsBytes([]byte(t.env))
		c.Check(err, IsNil)
		e, err := es.Open(t.name)
		c.Check(err, ErrorMatches, t.err)
		c.Check(e, IsNil)
	}
}

var configTests = []struct {
	env   string
	check func(c *C, es *environs.Environs)
}{
	{`
environments:
    only:
        type: dummy
        state-server: false
`, func(c *C, es *environs.Environs) {
		e, err := es.Open("")
		c.Assert(err, IsNil)
		c.Assert(e.Name(), Equals, "only")
	}}, {`
default:
    invalid
environments:
    valid:
        type: dummy
        state-server: false
    invalid:
        type: crazy
`, func(c *C, es *environs.Environs) {
		e, err := es.Open("")
		c.Assert(err, ErrorMatches, `environment "invalid" has an unknown provider type "crazy"`)
		c.Assert(e, IsNil)
		e, err = es.Open("valid")
		c.Assert(err, IsNil)
		c.Assert(e.Name(), Equals, "valid")
	}}, {`
environments:
    one:
        type: dummy
        state-server: false
    two:
        type: dummy
        state-server: false
`, func(c *C, es *environs.Environs) {
		e, err := es.Open("")
		c.Assert(err, ErrorMatches, `no default environment found`)
		c.Assert(e, IsNil)
	}},
}

func (suite) TestConfig(c *C) {
	defer makeFakeHome(c, "only", "valid", "one", "two").restore()
	for i, t := range configTests {
		c.Logf("running test %v", i)
		es, err := environs.ReadEnvironsBytes([]byte(t.env))
		c.Assert(err, IsNil)
		t.check(c, es)
	}
}

func (suite) TestDefaultConfigFile(c *C) {
	defer makeFakeHome(c, "only").restore()

	env := `
environments:
    only:
        type: dummy
        state-server: false
        authorized-keys: i-am-a-key
`
	err := ioutil.WriteFile(homePath(".juju", "environments.yaml"), []byte(env), 0666)
	c.Assert(err, IsNil)

	es, err := environs.ReadEnvirons("")
	c.Assert(err, IsNil)
	e, err := es.Open("")
	c.Assert(err, IsNil)
	c.Assert(e.Name(), Equals, "only")
}

func (suite) TestNamedConfigFile(c *C) {
	defer makeFakeHome(c, "only").restore()

	env := `
environments:
    only:
        type: dummy
        state-server: false
        authorized-keys: i-am-a-key
`
	path := filepath.Join(c.MkDir(), "a-file")
	err := ioutil.WriteFile(path, []byte(env), 0666)
	c.Assert(err, IsNil)

	es, err := environs.ReadEnvirons(path)
	c.Assert(err, IsNil)
	e, err := es.Open("")
	c.Assert(err, IsNil)
	c.Assert(e.Name(), Equals, "only")
}

func (suite) TestConfigRoundTrip(c *C) {
	cfg, err := config.New(map[string]interface{}{
		"name":            "bladaam",
		"type":            "dummy",
		"state-server":    false,
		"authorized-keys": "i-am-a-key",
		"ca-cert":         testing.CACertPEM,
		"ca-private-key":  "",
	})
	c.Assert(err, IsNil)
	provider, err := environs.Provider(cfg.Type())
	c.Assert(err, IsNil)
	cfg, err = provider.Validate(cfg, nil)
	c.Assert(err, IsNil)
	env, err := environs.New(cfg)
	c.Assert(err, IsNil)
	c.Assert(cfg.AllAttrs(), DeepEquals, env.Config().AllAttrs())
}

func (suite) TestBootstrapConfig(c *C) {
	defer makeFakeHome(c, "bladaam").restore()
	cfg, err := config.New(map[string]interface{}{
		"name":            "bladaam",
		"type":            "dummy",
		"state-server":    false,
		"admin-secret":    "highly",
		"secret":          "um",
		"authorized-keys": "i-am-a-key",
		"ca-cert":         testing.CACertPEM,
		"ca-private-key":  testing.CAKeyPEM,
	})
	c.Assert(err, IsNil)
	provider, err := environs.Provider(cfg.Type())
	c.Assert(err, IsNil)

	tools := &state.Tools{
		URL:    "http://x",
		Binary: version.MustParseBinary("1.2.3-foo-bar"),
	}
	cfg1, err := environs.BootstrapConfig(provider, cfg, tools)
	c.Assert(err, IsNil)

	expect := cfg.AllAttrs()
	delete(expect, "secret")
	expect["admin-secret"] = ""
	expect["ca-private-key"] = ""
	expect["agent-version"] = "1.2.3"
	c.Assert(cfg1.AllAttrs(), DeepEquals, expect)
}

type fakeHome string

func makeFakeHome(c *C, certNames ...string) fakeHome {
	oldHome := os.Getenv("HOME")
	os.Setenv("HOME", c.MkDir())

	err := os.Mkdir(homePath(".juju"), 0777)
	c.Assert(err, IsNil)
	for _, name := range certNames {
		err := ioutil.WriteFile(homePath(".juju", name+"-cert.pem"), []byte(testing.CACertPEM), 0666)
		c.Assert(err, IsNil)
		err = ioutil.WriteFile(homePath(".juju", name+"-private-key.pem"), []byte(testing.CAKeyPEM), 0666)
		c.Assert(err, IsNil)
	}

	err = os.Mkdir(homePath(".ssh"), 0777)
	c.Assert(err, IsNil)
	err = ioutil.WriteFile(homePath(".ssh", "id_rsa.pub"), []byte("auth key\n"), 0666)
	c.Assert(err, IsNil)

	return fakeHome(oldHome)
}

func homePath(names ...string) string {
	all := append([]string{os.Getenv("HOME")}, names...)
	return filepath.Join(all...)
}

func (h fakeHome) restore() {
	os.Setenv("HOME", string(h))
}
<|endoftext|>


<commit_before>package ec2

import (
	"fmt"
	"io"
	"launchpad.net/goamz/s3"
	"launchpad.net/juju/go/environs"
	"sync"
	"time"
)

// storage implements environs.Storage on
// an ec2.bucket.
type storage struct {
	bucketMutex sync.Mutex
	madeBucket  bool
	bucket      *s3.Bucket
}

// makeBucket makes the environent's control bucket, the
// place where bootstrap information and deployed charms
// are stored. To avoid two round trips on every PUT operation,
// we do this only once for each environ.
func (s *storage) makeBucket() error {
	s.bucketMutex.Lock()
	defer s.bucketMutex.Unlock()
	// try to make the bucket - PutBucket will succeed if the
	// bucket already exists.
	err := s.bucket.PutBucket(s3.Private)
	if err == nil {
		s.madeBucket = true
	}
	return err
}

func (s *storage) Put(file string, r io.Reader, length int64) error {
	if err := s.makeBucket(); err != nil {
		return fmt.Errorf("cannot make S3 control bucket: %v", err)
	}
	err := s.bucket.PutReader(file, r, length, "binary/octet-stream", s3.Private)
	if err != nil {
		return fmt.Errorf("cannot write file %q to control bucket: %v", file, err)
	}
	return nil
}

func (s *storage) Get(file string) (r io.ReadCloser, err error) {
	for a := shortAttempt.start(); a.next(); {
		r, err = s.bucket.GetReader(file)
		if s3ErrorStatusCode(err) == 404 {
			continue
		}
		return
	}
	return r, maybeNotFound(err)
}

func (s *storage) URL(name string) (string, error) {
	// a year should be good enough.
	return s.bucket.SignedURL(name, time.Now().Add(365*24*time.Hour)), nil
}

// s3ErrorStatusCode returns the HTTP status of the S3 request error,
// if it is an error from an S3 operation, or 0 if it was not.
func s3ErrorStatusCode(err error) int {
	if err, _ := err.(*s3.Error); err != nil {
		return err.StatusCode
	}
	return 0
}

func (s *storage) Remove(file string) error {
	err := s.bucket.Del(file)
	// If we can't delete the object because the bucket doesn't
	// exist, then we don't care.
	if s3ErrorStatusCode(err) == 404 {
		return nil
	}
	return err
}

func (s *storage) List(prefix string) ([]string, error) {
	// TODO cope with more than 1000 objects in the bucket.
	resp, err := s.bucket.List(prefix, "", "", 0)
	if err != nil {
		return nil, maybeNotFound(err)
	}
	var names []string
	for _, key := range resp.Contents {
		names = append(names, key.Key)
	}
	return names, nil
}

func (e *environ) Storage() environs.Storage {
	return &e.storage
}

func (e *environ) PublicStorage() environs.StorageReader {
	// TODO use public storage bucket
	return environs.EmptyStorage
}
<commit_msg>environs/ec2: adjust signature expiry date from 1 to 10 years<commit_after>package ec2

import (
	"fmt"
	"io"
	"launchpad.net/goamz/s3"
	"launchpad.net/juju/go/environs"
	"sync"
	"time"
)

// storage implements environs.Storage on
// an ec2.bucket.
type storage struct {
	bucketMutex sync.Mutex
	madeBucket  bool
	bucket      *s3.Bucket
}

// makeBucket makes the environent's control bucket, the
// place where bootstrap information and deployed charms
// are stored. To avoid two round trips on every PUT operation,
// we do this only once for each environ.
func (s *storage) makeBucket() error {
	s.bucketMutex.Lock()
	defer s.bucketMutex.Unlock()
	// try to make the bucket - PutBucket will succeed if the
	// bucket already exists.
	err := s.bucket.PutBucket(s3.Private)
	if err == nil {
		s.madeBucket = true
	}
	return err
}

func (s *storage) Put(file string, r io.Reader, length int64) error {
	if err := s.makeBucket(); err != nil {
		return fmt.Errorf("cannot make S3 control bucket: %v", err)
	}
	err := s.bucket.PutReader(file, r, length, "binary/octet-stream", s3.Private)
	if err != nil {
		return fmt.Errorf("cannot write file %q to control bucket: %v", file, err)
	}
	return nil
}

func (s *storage) Get(file string) (r io.ReadCloser, err error) {
	for a := shortAttempt.start(); a.next(); {
		r, err = s.bucket.GetReader(file)
		if s3ErrorStatusCode(err) == 404 {
			continue
		}
		return
	}
	return r, maybeNotFound(err)
}

func (s *storage) URL(name string) (string, error) {
	// 10 years should be good enough.
	return s.bucket.SignedURL(name, time.Now().AddDate(1, 0, 0)), nil
}

// s3ErrorStatusCode returns the HTTP status of the S3 request error,
// if it is an error from an S3 operation, or 0 if it was not.
func s3ErrorStatusCode(err error) int {
	if err, _ := err.(*s3.Error); err != nil {
		return err.StatusCode
	}
	return 0
}

func (s *storage) Remove(file string) error {
	err := s.bucket.Del(file)
	// If we can't delete the object because the bucket doesn't
	// exist, then we don't care.
	if s3ErrorStatusCode(err) == 404 {
		return nil
	}
	return err
}

func (s *storage) List(prefix string) ([]string, error) {
	// TODO cope with more than 1000 objects in the bucket.
	resp, err := s.bucket.List(prefix, "", "", 0)
	if err != nil {
		return nil, maybeNotFound(err)
	}
	var names []string
	for _, key := range resp.Contents {
		names = append(names, key.Key)
	}
	return names, nil
}

func (e *environ) Storage() environs.Storage {
	return &e.storage
}

func (e *environ) PublicStorage() environs.StorageReader {
	// TODO use public storage bucket
	return environs.EmptyStorage
}
<|endoftext|>


<commit_before>package test

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"testing"
	"time"
)

type dockerfile struct {
	// Local fs path to the Dockerfile
	base string

	// Commands represents a series of commands to be ran in the image
	commands []*Cmd

	// -t flag with build/run
	tag string

	// any errors specific to the dockerfile from `docker` commands
	err error

	// only run build, tag and run steps once
	wg sync.WaitGroup
}

func Dockerfile(where string) *dockerfile {
	if !strings.HasSuffix(where, "Dockerfile") {
		where = filepath.Join(where, "/Dockerfile")
	}

	// Grab the env name (e.g. envs/$env_name/Dockerfile)
	dir, _ := filepath.Split(where)
	now := time.Now().Unix()
	tag := fmt.Sprintf("cert-manage:%s-%d", filepath.Base(dir), now)

	return &dockerfile{
		base: where,
		tag:  tag,
	}
}

func (d *dockerfile) Run(cmd string, args ...string) {
	d.commands = append(d.commands, Command(cmd, args...))
}

func (d *dockerfile) CertManage(args ...string) {
	d.Run("/cert-manage", args...)
}

func (d *dockerfile) SuccessT(t *testing.T) {
	d.prep()
	t.Helper()

	if d.err != nil {
		t.Fatal(d.err)
	}

	for i := range d.commands {
		d.commands[i].SuccessT(t)
	}
}

func (d *dockerfile) build() {
	d.wg.Add(1)
	defer d.wg.Done()

	// Copy our original image's contents into the dst file
	dir, err := ioutil.TempDir("", d.tag)
	if err != nil {
		d.err = fmt.Errorf("tempfile create err=%v", err)
		return
	}
	dst, err := os.Create(filepath.Join(dir, "Dockerfile"))
	if err != nil {
		d.err = fmt.Errorf("tmp Dockerfile create err=%v", err)
		return
	}
	defer os.Remove(dst.Name())

	src, err := os.Open(d.base)
	if err != nil {
		d.err = fmt.Errorf("tmpfile open err=%v", err)
		return
	}
	if _, err := io.Copy(dst, src); err != nil {
		d.err = fmt.Errorf("src->dst copy err=%v", err)
		return
	}
	if err := src.Close(); err != nil {
		d.err = fmt.Errorf("src close err=%v", err)
		return
	}

	// Add all commands as RUN statements in the image
	command := "CMD exit 0"
	for i := range d.commands {
		command += fmt.Sprintf(" && %s %s", d.commands[i].command, strings.Join(d.commands[i].args, " "))
	}
	if _, err := dst.WriteString(command); err != nil {
		d.err = fmt.Errorf("command=%s err=%v", command, err)
		return
	}

	// Force all writes into our dockerfile
	if err := dst.Sync(); err != nil {
		d.err = fmt.Errorf("dst fsync err=%v", err)
		return
	}

	cmd := exec.Command("docker", "build", "-t", d.tag, dir)
	var stderr bytes.Buffer
	cmd.Stderr = &stderr

	if err := cmd.Run(); err != nil {
		d.err = fmt.Errorf("ERROR: err=%v\nOutput: %s", err, stderr.String())
	}
}

func (d *dockerfile) run() {
	d.wg.Add(1)
	defer d.wg.Done()

	// don't attempt anything if we've already failed
	if d.err != nil {
		return
	}

	cmd := exec.Command("docker", "run", "-t", d.tag)
	var stderr bytes.Buffer
	cmd.Stderr = &stderr

	if err := cmd.Run(); err != nil {
		d.err = fmt.Errorf("ERROR: err=%v\nOutput: %s", err, stderr.String())
	}
}

func (d *dockerfile) prep() {
	d.build()
	d.run()
	d.wg.Wait()
}
<commit_msg>test/docker: cleanup comments<commit_after>package test

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"testing"
	"time"
)

type dockerfile struct {
	// Local fs path to the Dockerfile
	base string

	// Commands represents a series of commands to be ran in the image
	commands []*Cmd

	// -t flag with build/run
	tag string

	// any errors specific to the dockerfile from `docker` commands
	err error

	// only run build, tag and run steps once
	wg sync.WaitGroup
}

func Dockerfile(where string) *dockerfile {
	if !strings.HasSuffix(where, "Dockerfile") {
		where = filepath.Join(where, "/Dockerfile")
	}

	// Grab the env name (e.g. envs/$env_name/Dockerfile)
	dir, _ := filepath.Split(where)
	now := time.Now().Unix()
	tag := fmt.Sprintf("cert-manage:%s-%d", filepath.Base(dir), now)

	return &dockerfile{
		base: where,
		tag:  tag,
	}
}

func (d *dockerfile) Run(cmd string, args ...string) {
	d.commands = append(d.commands, Command(cmd, args...))
}

func (d *dockerfile) CertManage(args ...string) {
	d.Run("/cert-manage", args...)
}

func (d *dockerfile) SuccessT(t *testing.T) {
	d.prep()
	t.Helper()

	if d.err != nil {
		t.Fatal(d.err)
	}

	for i := range d.commands {
		d.commands[i].SuccessT(t)
	}
}

func (d *dockerfile) build() {
	d.wg.Add(1)
	defer d.wg.Done()

	// Copy our original image's contents into the dst file
	dir, err := ioutil.TempDir("", d.tag)
	if err != nil {
		d.err = fmt.Errorf("tempfile create err=%v", err)
		return
	}
	dst, err := os.Create(filepath.Join(dir, "Dockerfile"))
	if err != nil {
		d.err = fmt.Errorf("tmp Dockerfile create err=%v", err)
		return
	}
	defer os.Remove(dst.Name())

	src, err := os.Open(d.base)
	if err != nil {
		d.err = fmt.Errorf("tmpfile open err=%v", err)
		return
	}
	if _, err := io.Copy(dst, src); err != nil {
		d.err = fmt.Errorf("src->dst copy err=%v", err)
		return
	}
	if err := src.Close(); err != nil {
		d.err = fmt.Errorf("src close err=%v", err)
		return
	}

	// Add all commands to the Dockerfile
	command := "CMD exit 0"
	for i := range d.commands {
		command += fmt.Sprintf(" && %s %s", d.commands[i].command, strings.Join(d.commands[i].args, " "))
	}
	if _, err := dst.WriteString(command); err != nil {
		d.err = fmt.Errorf("command=%s err=%v", command, err)
		return
	}

	// Force all writes into our Dockerfile
	if err := dst.Sync(); err != nil {
		d.err = fmt.Errorf("dst fsync err=%v", err)
		return
	}

	cmd := exec.Command("docker", "build", "-t", d.tag, dir)
	var stderr bytes.Buffer
	cmd.Stderr = &stderr

	if err := cmd.Run(); err != nil {
		d.err = fmt.Errorf("ERROR: err=%v\nOutput: %s", err, stderr.String())
	}
}

func (d *dockerfile) run() {
	d.wg.Add(1)
	defer d.wg.Done()

	// don't attempt anything if we've already failed
	if d.err != nil {
		return
	}

	cmd := exec.Command("docker", "run", "-t", d.tag)
	var stderr bytes.Buffer
	cmd.Stderr = &stderr

	if err := cmd.Run(); err != nil {
		d.err = fmt.Errorf("ERROR: err=%v\nOutput: %s", err, stderr.String())
	}
}

func (d *dockerfile) prep() {
	d.build()
	d.run()
	d.wg.Wait()
}
<|endoftext|>


<commit_before>package queue

import (
	"github.com/nathan-osman/go-cannon/util"

	"errors"
	"fmt"
	"net"
	"net/smtp"
	"syscall"
	"time"
)

// Individual message for sending to a host.
type Message struct {
	From    string
	To      []string
	Message []byte
}

// Persistent connection to an SMTP host.
type Host struct {
	newMessage *util.NonBlockingChan
	stop       chan bool
}

// Attempt to find the mail servers for the specified host.
func findMailServers(host string) []string {

	// First check for MX records - if one or more were found, convert the
	// records into a list of strings (already sorted by priority) - if none
	// were found, then simply return the host that was originally provided
	if mx, err := net.LookupMX(host); err == nil {
		servers := make([]string, len(mx))
		for i, record := range mx {
			servers[i] = record.Host
		}
		return servers
	} else {
		return []string{host}
	}
}

// Attempt to connect to the specified host.
func connectToMailServer(host string, stop chan bool) (*smtp.Client, error) {

	// Obtain the list of mail servers to try
	servers := findMailServers(host)

	// RFC 5321 (4.5.4) describes the process for retrying connections to a
	// mail server after failure. The recommended strategy is to retry twice
	// with 30 minute intervals and continue at 120 minute intervals until four
	// days have elapsed.
	for i := 0; i < 50; i++ {

		// Attempt to connect to each of the mail servers from the list in the
		// order that was provided - return immediately if a connection is made
		for _, server := range servers {
			if client, err := smtp.Dial(fmt.Sprintf("%s:25", server)); err == nil {
				return client, nil
			}
		}

		// None of the connections succeeded, so it is time to wait either for
		// the specified timeout duration or a receive on the stop channel
		var dur time.Duration
		if i < 2 {
			dur = 30 * time.Minute
		} else {
			dur = 2 * time.Hour
		}

		select {
		case <-time.After(dur):
		case <-stop:
			return nil, nil
		}
	}

	// All attempts have failed, let the caller know we tried :)
	return nil, errors.New("unable to connect to a mail server")
}

// Attempt to deliver the specified message to the server.
func deliverToMailServer(client *smtp.Client, msg *Message) error {

	// Specify the sender of the message
	if err := client.Mail(msg.From); err != nil {
		return err
	}

	// Add each of the recipients
	for _, to := range msg.To {
		if err := client.Rcpt(to); err != nil {
			return err
		}
	}

	// Obtain a writer for writing the actual message
	writer, err := client.Data()
	if err != nil {
		return err
	}
	defer writer.Close()

	// Write the message
	if _, err = writer.Write(msg.Message); err != nil {
		return err
	}

	return nil
}

// Create a new host connection.
func NewHost(host string) *Host {

	// Create the host, including the channel used for delivering new messages
	h := &Host{
		newMessage: util.NewNonBlockingChan(),
		stop:       make(chan bool),
	}

	// Start a goroutine to manage the lifecycle of the host connection
	go func() {

		// Close the stop channel when the goroutine exits
		defer close(h.stop)

		// Enter a loop that will continue to deliver messages
	queue:
		for {

			// Obtain the next message for delivery
			var msg *Message

			select {
			case i := <-h.newMessage.Recv:
				msg = i.(*Message)
			case <-h.stop:
				break queue
			}

		connect:
			for {

				// Attempt to connect to a server
				client, err := connectToMailServer(host, h.stop)
				if client == nil {

					// Stop if there was no client and no error - otherwise,
					// discard the current message and enter the loop again
					if err == nil {
						break queue
					} else {
						// TODO: log something somewhere?
						// TODO: discard remaining messages?
						continue queue
					}
				}

			deliver:
				for {

					// Attempt to deliver the message
					if err = deliverToMailServer(client, msg); err != nil {

						// If the type of error has anything to do with a syscall,
						// assume that the connection was broken and try
						// reconnecting - otherwise, discard the message
						if _, ok := err.(syscall.Errno); ok {
							continue connect
						} else {
							continue queue
						}
					}

					// If a new message comes in then send it, if 5 minutes
					// elapses, close the connection and wait for a new message
					select {
					case i := <-h.newMessage.Recv:
						msg = i.(*Message)
						continue deliver
					case <-time.After(5 * time.Minute):
						client.Quit()
						continue queue
					case <-h.stop:
						client.Quit()
						break queue
					}
				}
			}
		}
	}()

	return h
}

// Attempt to deliver a message to the host.
func (h *Host) Deliver(msg *Message) {
	h.newMessage.Send <- msg
}

// Abort the connection to the host..
func (h *Host) Stop() {

	// Send on the channel to stop it and wait for it to be closed
	h.stop <- true
	<-h.stop
}
<commit_msg>Simplified the control flow of the host queue and added a method for fetching connection idle time.<commit_after>package queue

import (
	"github.com/nathan-osman/go-cannon/util"

	"errors"
	"fmt"
	"net"
	"net/smtp"
	"sync"
	"syscall"
	"time"
)

// Individual message for sending to a host.
type Message struct {
	From    string
	To      []string
	Message []byte
}

// Persistent connection to an SMTP host.
type Host struct {
	sync.Mutex
	lastActivity time.Time
	newMessage   *util.NonBlockingChan
	stop         chan bool
}

// Attempt to find the mail servers for the specified host.
func findMailServers(host string) []string {

	// First check for MX records - if one or more were found, convert the
	// records into a list of strings (already sorted by priority) - if none
	// were found, then simply return the host that was originally provided
	if mx, err := net.LookupMX(host); err == nil {
		servers := make([]string, len(mx))
		for i, record := range mx {
			servers[i] = record.Host
		}
		return servers
	} else {
		return []string{host}
	}
}

// Attempt to connect to the specified host.
func connectToMailServer(host string, stop chan bool) (*smtp.Client, error) {

	// Obtain the list of mail servers to try
	servers := findMailServers(host)

	// RFC 5321 (4.5.4) describes the process for retrying connections to a
	// mail server after failure. The recommended strategy is to retry twice
	// with 30 minute intervals and continue at 120 minute intervals until four
	// days have elapsed.
	for i := 0; i < 50; i++ {

		// Attempt to connect to each of the mail servers from the list in the
		// order that was provided - return immediately if a connection is made
		for _, server := range servers {
			if client, err := smtp.Dial(fmt.Sprintf("%s:25", server)); err == nil {
				return client, nil
			}
		}

		// None of the connections succeeded, so it is time to wait either for
		// the specified timeout duration or a receive on the stop channel
		var dur time.Duration
		if i < 2 {
			dur = 30 * time.Minute
		} else {
			dur = 2 * time.Hour
		}

		select {
		case <-time.After(dur):
		case <-stop:
			return nil, nil
		}
	}

	// All attempts have failed, let the caller know we tried :)
	return nil, errors.New("unable to connect to a mail server")
}

// Attempt to deliver the specified message to the server.
func deliverToMailServer(client *smtp.Client, msg *Message) error {

	// Specify the sender of the message
	if err := client.Mail(msg.From); err != nil {
		return err
	}

	// Add each of the recipients
	for _, to := range msg.To {
		if err := client.Rcpt(to); err != nil {
			return err
		}
	}

	// Obtain a writer for writing the actual message
	writer, err := client.Data()
	if err != nil {
		return err
	}
	defer writer.Close()

	// Write the message
	if _, err = writer.Write(msg.Message); err != nil {
		return err
	}

	return nil
}

// Create a new host connection.
func NewHost(host string) *Host {

	// Create the host, including the channel used for delivering new messages
	h := &Host{
		newMessage: util.NewNonBlockingChan(),
		stop:       make(chan bool),
	}

	// Start a goroutine to manage the lifecycle of the host connection
	go func() {

		// Close the stop channel when the goroutine exits
		defer close(h.stop)

		// If the sight of a "goto" statement makes you cringe, you should
		// probably close your eyes and skip over the next section. Although
		// what follows could in theory be written without "goto", it wouldn't
		// be nearly as concise or easy to follow. Therefore it is used here.

		var (
			client *smtp.Client
			msg    *Message
		)

		// Receive a new message from the channel
	receive:

		h.Lock()
		h.lastActivity = time.Now()
		h.Unlock()

		select {
		case i := <-h.newMessage.Recv:
			msg = i.(*Message)
		case <-h.stop:
			goto quit
		}

		h.Lock()
		h.lastActivity = time.Time{}
		h.Unlock()

		// Connect to the mail server (if not connected) and deliver a message
	connect:

		if client == nil {
			client, err := connectToMailServer(host, h.stop)
			if client == nil {

				// Stop if there was no client and no error - otherwise,
				// discard the current message and wait for the next one
				if err == nil {
					goto quit
				} else {
					// TODO: log something somewhere?
					// TODO: discard remaining messages?
					goto receive
				}
			}
		}

		// Attempt to deliver the message and then wait for the next one
		if err := deliverToMailServer(client, msg); err != nil {

			// If the type of error has anything to do with a syscall, assume
			// that the connection was broken and try reconnecting - otherwise,
			// discard the message
			if _, ok := err.(syscall.Errno); ok {
				client = nil
				goto connect
			}
		}

		// Receive the next message
		goto receive

		// Close the connection (if open) and quit
	quit:

		if client != nil {
			client.Quit()
		}

	}()

	return h
}

// Attempt to deliver a message to the host.
func (h *Host) Deliver(msg *Message) {
	h.newMessage.Send <- msg
}

// Retrieve the connection idle time.
func (h *Host) Idle() time.Duration {
	h.Lock()
	defer h.Unlock()
	if h.lastActivity.IsZero() {
		return 0
	} else {
		return time.Since(h.lastActivity)
	}
}

// Close the connection to the host.
func (h *Host) Stop() {

	// Send on the channel to stop it and wait for it to be closed
	h.stop <- true
	<-h.stop
}
<|endoftext|>


<commit_before>package game

import (
	"runtime"
	"sync"
	"time"

	"buildblast/server/lib/coords"
	"buildblast/server/lib/mapgen"
)

type ChunkGenerator struct {
	// Chunks are sent to this channel as they are generated
	Generated chan ChunkGenerationResult

	chunks       map[coords.Chunk]ChunkStatus
	mutex        sync.Mutex
	queuedChunks chan coords.Chunk
	generator    mapgen.Generator
}

type ChunkGenerationResult struct {
	cc    coords.Chunk
	chunk *mapgen.Chunk
}

type State int

const (
	Queued     State = iota
	Generating State = iota
	Generated  State = iota
)

type ChunkStatus struct {
	state    State
	priority int
}

func NewChunkGenerator(generator mapgen.Generator) *ChunkGenerator {
	cm := new(ChunkGenerator)
	cm.chunks = make(map[coords.Chunk]ChunkStatus, 10)
	cm.generator = generator

	// The number of workers we will run.
	maxActiveThreads := runtime.NumCPU() - 1
	if maxActiveThreads < 1 {
		maxActiveThreads = 1
	}

	// Room for 2*workers worth of work and results.
	cm.Generated = make(chan ChunkGenerationResult, maxActiveThreads*2)
	cm.queuedChunks = make(chan coords.Chunk, maxActiveThreads*2)

	// Start the workers.
	for i := 0; i < maxActiveThreads; i++ {
		go cm.runGenerationWorker()
	}

	return cm
}

// cm.Lock() MUST BE HELD by the caller, or else calling
// this function is unsafe.
func (cm *ChunkGenerator) queue(cc coords.Chunk, priority int) {
	status := cm.chunks[cc]
	if status.state != Queued {
		// Only adjust the priority if it is still queued.
		return
	}
	status.priority += priority
	cm.chunks[cc] = status
}

func (cm *ChunkGenerator) QueueChunksNearby(wc coords.World) {
	cm.mutex.Lock()
	defer cm.mutex.Unlock()

	queue := func(cc coords.Chunk, priority int) {
		cm.queue(cc, priority)
	}

	EachChunkNearby(wc, queue)
}

func (cm *ChunkGenerator) Top() (cc coords.Chunk, valid bool) {
	cm.mutex.Lock()
	defer cm.mutex.Unlock()

	highest := -1
	for key, val := range cm.chunks {
		if val.priority > highest && val.state == Queued {
			highest = val.priority
			cc = key
		}
	}
	if highest != -1 {
		return cc, true
	}
	return cc, false
}

func (cm *ChunkGenerator) Run() {
	for {
		cc, valid := cm.Top()
		if !valid {
			<-time.After(time.Second / 60)
			continue
		}

		cm.mutex.Lock()
		status := cm.chunks[cc]
		status.state = Generating
		cm.chunks[cc] = status
		cm.mutex.Unlock()

		cm.queuedChunks <- cc
	}
}

func (cm *ChunkGenerator) runGenerationWorker() {
	for {
		cc := <-cm.queuedChunks
		chunk := cm.generator.Chunk(cc)

		cm.Generated <- ChunkGenerationResult{
			cc:    cc,
			chunk: chunk,
		}
		cm.mutex.Lock()
		status := cm.chunks[cc]
		status.state = Generated
		cm.chunks[cc] = status
		cm.mutex.Unlock()
	}
}

func EachChunkNearby(wc coords.World, cb func(cc coords.Chunk, priority int)) {
	occ := func(cc coords.Chunk, x, y, z int) coords.Chunk {
		return coords.Chunk{
			X: cc.X + x,
			Y: cc.Y + y,
			Z: cc.Z + z,
		}
	}

	eachWithin := func(cc coords.Chunk, xdist, ydist, zdist int, cb func(newCC coords.Chunk, dist int)) {
		abs := func(n int) int {
			if n < 0 {
				return -n
			}
			return n
		}
		dist := func(x, y, z int) int {
			return abs(x) + abs(y) + abs(z)
		}

		cb(cc, 0)
		for x := -xdist; x <= xdist; x++ {
			for y := -ydist; y <= ydist; y++ {
				for z := -zdist; z <= zdist; z++ {
					cb(occ(cc, x, y, z), dist(x, y, z))
				}
			}
		}
	}

	cc := wc.Chunk()
	eachWithin(cc, 10, 3, 10, func(newCC coords.Chunk, dist int) {
		// We want to prioritize further away chunks lower, but the
		// priority must be a positive integer.
		cb(newCC, 10-dist)
	})

	oc := wc.Offset()
	if oc.Y <= 4 {
		cb(occ(cc, 0, -1, 0), 1)
	} else if oc.Y >= 28 {
		cb(occ(cc, 0, 1, 0), 1)
	}
}
<commit_msg>Revert long distance thing.<commit_after>package game

import (
	"runtime"
	"sync"
	"time"

	"buildblast/server/lib/coords"
	"buildblast/server/lib/mapgen"
)

type ChunkGenerator struct {
	// Chunks are sent to this channel as they are generated
	Generated chan ChunkGenerationResult

	chunks       map[coords.Chunk]ChunkStatus
	mutex        sync.Mutex
	queuedChunks chan coords.Chunk
	generator    mapgen.Generator
}

type ChunkGenerationResult struct {
	cc    coords.Chunk
	chunk *mapgen.Chunk
}

type State int

const (
	Queued     State = iota
	Generating State = iota
	Generated  State = iota
)

type ChunkStatus struct {
	state    State
	priority int
}

func NewChunkGenerator(generator mapgen.Generator) *ChunkGenerator {
	cm := new(ChunkGenerator)
	cm.chunks = make(map[coords.Chunk]ChunkStatus, 10)
	cm.generator = generator

	// The number of workers we will run.
	maxActiveThreads := runtime.NumCPU() - 1
	if maxActiveThreads < 1 {
		maxActiveThreads = 1
	}

	// Room for 2*workers worth of work and results.
	cm.Generated = make(chan ChunkGenerationResult, maxActiveThreads*2)
	cm.queuedChunks = make(chan coords.Chunk, maxActiveThreads*2)

	// Start the workers.
	for i := 0; i < maxActiveThreads; i++ {
		go cm.runGenerationWorker()
	}

	return cm
}

// cm.Lock() MUST BE HELD by the caller, or else calling
// this function is unsafe.
func (cm *ChunkGenerator) queue(cc coords.Chunk, priority int) {
	status := cm.chunks[cc]
	if status.state != Queued {
		// Only adjust the priority if it is still queued.
		return
	}
	status.priority += priority
	cm.chunks[cc] = status
}

func (cm *ChunkGenerator) QueueChunksNearby(wc coords.World) {
	cm.mutex.Lock()
	defer cm.mutex.Unlock()

	queue := func(cc coords.Chunk, priority int) {
		cm.queue(cc, priority)
	}

	EachChunkNearby(wc, queue)
}

func (cm *ChunkGenerator) Top() (cc coords.Chunk, valid bool) {
	cm.mutex.Lock()
	defer cm.mutex.Unlock()

	highest := -1
	for key, val := range cm.chunks {
		if val.priority > highest && val.state == Queued {
			highest = val.priority
			cc = key
		}
	}
	if highest != -1 {
		return cc, true
	}
	return cc, false
}

func (cm *ChunkGenerator) Run() {
	for {
		cc, valid := cm.Top()
		if !valid {
			<-time.After(time.Second / 60)
			continue
		}

		cm.mutex.Lock()
		status := cm.chunks[cc]
		status.state = Generating
		cm.chunks[cc] = status
		cm.mutex.Unlock()

		cm.queuedChunks <- cc
	}
}

func (cm *ChunkGenerator) runGenerationWorker() {
	for {
		cc := <-cm.queuedChunks
		chunk := cm.generator.Chunk(cc)

		cm.Generated <- ChunkGenerationResult{
			cc:    cc,
			chunk: chunk,
		}
		cm.mutex.Lock()
		status := cm.chunks[cc]
		status.state = Generated
		cm.chunks[cc] = status
		cm.mutex.Unlock()
	}
}

func EachChunkNearby(wc coords.World, cb func(cc coords.Chunk, priority int)) {
	occ := func(cc coords.Chunk, x, y, z int) coords.Chunk {
		return coords.Chunk{
			X: cc.X + x,
			Y: cc.Y + y,
			Z: cc.Z + z,
		}
	}

	eachWithin := func(cc coords.Chunk, xdist, ydist, zdist int, cb func(newCC coords.Chunk, dist int)) {
		abs := func(n int) int {
			if n < 0 {
				return -n
			}
			return n
		}
		dist := func(x, y, z int) int {
			return abs(x) + abs(y) + abs(z)
		}

		cb(cc, 0)
		for x := -xdist; x <= xdist; x++ {
			for y := -ydist; y <= ydist; y++ {
				for z := -zdist; z <= zdist; z++ {
					cb(occ(cc, x, y, z), dist(x, y, z))
				}
			}
		}
	}

	cc := wc.Chunk()
	eachWithin(cc, 2, 1, 2, func(newCC coords.Chunk, dist int) {
		// We want to prioritize further away chunks lower, but the
		// priority must be a positive integer.
		cb(newCC, 10-dist)
	})

	oc := wc.Offset()
	if oc.Y <= 4 {
		cb(occ(cc, 0, -1, 0), 1)
	} else if oc.Y >= 28 {
		cb(occ(cc, 0, 1, 0), 1)
	}
}
<|endoftext|>


<commit_before>package middleware

import (
	"github.com/gin-gonic/gin"

	oauth_service "github.com/torinos-io/api/service/oauth_service"
	user_store "github.com/torinos-io/api/store/user_store"
	"github.com/torinos-io/api/type/system"
)

const (
	currentuser = "CurrentUser"
)

// SetCurrentUser sets current authenticated user from authorization header
func SetCurrentUser(appContext *system.AppContext) gin.HandlerFunc {
	return func(c *gin.Context) {

		h := c.GetHeader("Authorization")

		if h == "" {
			c.Next()
			return
		}

		userStore := user_store.New(appContext.MainDB)
		service := oauth_service.New(oauth_service.Context{
			Config:    appContext.Config,
			UserStore: userStore,
		})

		user, err := service.FindByAuthToken(h)

		if err != nil {
			c.Next()
			return
		}

		c.Set(currentuser, user)
		c.Next()
	}
}
<commit_msg>Rename current user const value name<commit_after>package middleware

import (
	"github.com/gin-gonic/gin"

	oauth_service "github.com/torinos-io/api/service/oauth_service"
	user_store "github.com/torinos-io/api/store/user_store"
	"github.com/torinos-io/api/type/system"
)

const (
	currentUserContextName = "CurrentUser"
)

// SetCurrentUser sets current authenticated user from authorization header
func SetCurrentUser(appContext *system.AppContext) gin.HandlerFunc {
	return func(c *gin.Context) {
		h := c.GetHeader("Authorization")

		if h == "" {
			c.Next()
			return
		}

		userStore := user_store.New(appContext.MainDB)
		service := oauth_service.New(oauth_service.Context{
			Config:    appContext.Config,
			UserStore: userStore,
		})

		user, err := service.FindByAuthToken(h)

		if err != nil {
			c.Next()
			return
		}

		c.Set(currentUserContextName, user)
		c.Next()
	}
}
<|endoftext|>