<commit_before>package main

import (
	"bytes"
	"context"
	"flag"
	"fmt"
	traceparser "internal/trace"
	"io/ioutil"
	"reflect"
	"runtime/debug"
	"runtime/trace"
	"sync"
	"testing"
	"time"
)

var saveTraces = flag.Bool("savetraces", false, "save traces collected by tests")

func TestOverlappingDuration(t *testing.T) {
	cases := []struct {
		start0, end0, start1, end1 int64
		want                       time.Duration
	}{
		{
			1, 10, 11, 20, 0,
		},
		{
			1, 10, 5, 20, 5 * time.Nanosecond,
		},
		{
			1, 10, 2, 8, 6 * time.Nanosecond,
		},
	}

	for _, tc := range cases {
		s0, e0, s1, e1 := tc.start0, tc.end0, tc.start1, tc.end1
		if got := overlappingDuration(s0, e0, s1, e1); got != tc.want {
			t.Errorf("overlappingDuration(%d, %d, %d, %d)=%v; want %v", s0, e0, s1, e1, got, tc.want)
		}
		if got := overlappingDuration(s1, e1, s0, e0); got != tc.want {
			t.Errorf("overlappingDuration(%d, %d, %d, %d)=%v; want %v", s1, e1, s0, e0, got, tc.want)
		}
	}
}

// prog0 starts three goroutines.
//
//   goroutine 1: taskless span
//   goroutine 2: starts task0, do work in task0.span0, starts task1 which ends immediately.
//   goroutine 3: do work in task0.span1 and task0.span2, ends task0
func prog0() {
	ctx := context.Background()

	var wg sync.WaitGroup

	wg.Add(1)
	go func() { // goroutine 1
		defer wg.Done()
		trace.WithSpan(ctx, "taskless.span", func(ctx context.Context) {
			trace.Log(ctx, "key0", "val0")
		})
	}()

	wg.Add(1)
	go func() { // goroutine 2
		defer wg.Done()
		ctx, taskDone := trace.NewContext(ctx, "task0")
		trace.WithSpan(ctx, "task0.span0", func(ctx context.Context) {
			wg.Add(1)
			go func() { // goroutine 3
				defer wg.Done()
				defer taskDone()
				trace.WithSpan(ctx, "task0.span1", func(ctx context.Context) {
					trace.WithSpan(ctx, "task0.span2", func(ctx context.Context) {
						trace.Log(ctx, "key2", "val2")
					})
					trace.Log(ctx, "key1", "val1")
				})
			}()
		})
		ctx2, taskDone2 := trace.NewContext(ctx, "task1")
		trace.Log(ctx2, "key3", "val3")
		taskDone2()
	}()
	wg.Wait()
}

func TestAnalyzeAnnotations(t *testing.T) {
	// TODO: classify taskless spans

	// Run prog0 and capture the execution trace.
	if err := traceProgram(t, prog0, "TestAnalyzeAnnotations"); err != nil {
		t.Fatalf("failed to trace the program: %v", err)
	}

	res, err := analyzeAnnotations()
	if err != nil {
		t.Fatalf("failed to analyzeAnnotations: %v", err)
	}
	tasks := res.tasks

	// For prog0, we expect
	//   - task with name = "task0", with three spans.
	//   - task with name = "task1", with no span.
	wantTasks := map[string]struct {
		complete   bool
		goroutines int
		spans      []string
	}{
		"task0": {
			complete:   true,
			goroutines: 2,
			spans:      []string{"task0.span0", "task0.span1", "task0.span2"},
		},
		"task1": {
			complete:   true,
			goroutines: 1,
		},
	}

	for _, task := range tasks {
		want, ok := wantTasks[task.name]
		if !ok {
			t.Errorf("unexpected task: %s", task)
			continue
		}
		if task.complete() != want.complete || len(task.goroutines) != want.goroutines || !reflect.DeepEqual(spanNames(task), want.spans) {
			t.Errorf("got %v; want %+v", task, want)
		}

		delete(wantTasks, task.name)
	}
	if len(wantTasks) > 0 {
		t.Errorf("no more tasks; want %+v", wantTasks)
	}
}

// prog1 creates a task hierarchy consisting of three tasks.
func prog1() {
	ctx := context.Background()
	ctx1, done1 := trace.NewContext(ctx, "task1")
	defer done1()
	trace.WithSpan(ctx1, "task1.span", func(ctx context.Context) {
		ctx2, done2 := trace.NewContext(ctx, "task2")
		defer done2()
		trace.WithSpan(ctx2, "task2.span", func(ctx context.Context) {
			ctx3, done3 := trace.NewContext(ctx, "task3")
			defer done3()
			trace.WithSpan(ctx3, "task3.span", func(ctx context.Context) {
			})
		})
	})
}

func TestAnalyzeAnnotationTaskTree(t *testing.T) {
	// Run prog1 and capture the execution trace.
	if err := traceProgram(t, prog1, "TestAnalyzeAnnotationTaskTree"); err != nil {
		t.Fatalf("failed to trace the program: %v", err)
	}

	res, err := analyzeAnnotations()
	if err != nil {
		t.Fatalf("failed to analyzeAnnotations: %v", err)
	}
	tasks := res.tasks

	// For prog0, we expect
	//   - task with name = "", with taskless.span in spans.
	//   - task with name = "task0", with three spans.
	wantTasks := map[string]struct {
		parent   string
		children []string
		spans    []string
	}{
		"task1": {
			parent:   "",
			children: []string{"task2"},
			spans:    []string{"task1.span"},
		},
		"task2": {
			parent:   "task1",
			children: []string{"task3"},
			spans:    []string{"task2.span"},
		},
		"task3": {
			parent:   "task2",
			children: nil,
			spans:    []string{"task3.span"},
		},
	}

	for _, task := range tasks {
		want, ok := wantTasks[task.name]
		if !ok {
			t.Errorf("unexpected task: %s", task)
			continue
		}
		delete(wantTasks, task.name)

		if parentName(task) != want.parent ||
			!reflect.DeepEqual(childrenNames(task), want.children) ||
			!reflect.DeepEqual(spanNames(task), want.spans) {
			t.Errorf("got %v; want %+v", task, want)
		}
	}

	if len(wantTasks) > 0 {
		t.Errorf("no more tasks; want %+v", wantTasks)
	}
}

// prog2 starts two tasks; "taskWithGC" that overlaps with GC
// and "taskWithoutGC" that doesn't. In order to run this reliably,
// the caller needs to set up to prevent GC from running automatically.
// prog2 returns the upper-bound gc time that overlaps with the first task.
func prog2() (gcTime time.Duration) {
	ch := make(chan bool)
	ctx1, done := trace.NewContext(context.Background(), "taskWithGC")
	trace.WithSpan(ctx1, "taskWithGC.span1", func(ctx context.Context) {
		go func() {
			defer trace.StartSpan(ctx, "taskWithGC.span2")()
			<-ch
		}()
		s := time.Now()
		debug.FreeOSMemory() // task1 affected by gc
		gcTime = time.Since(s)
		close(ch)
	})
	done()

	ctx2, done2 := trace.NewContext(context.Background(), "taskWithoutGC")
	trace.WithSpan(ctx2, "taskWithoutGC.span1", func(ctx context.Context) {
		// do nothing.
	})
	done2()
	return gcTime
}

func TestAnalyzeAnnotationGC(t *testing.T) {
	var gcTime time.Duration
	err := traceProgram(t, func() {
		oldGC := debug.SetGCPercent(10000) // gc, and effectively disable GC
		defer debug.SetGCPercent(oldGC)

		gcTime = prog2()
	}, "TestAnalyzeAnnotationGC")
	if err != nil {
		t.Fatalf("failed to trace the program: %v", err)
	}

	res, err := analyzeAnnotations()
	if err != nil {
		t.Fatalf("failed to analyzeAnnotations: %v", err)
	}

	// Check collected GC Start events are all sorted and non-overlapping.
	lastTS := int64(0)
	for i, ev := range res.gcEvents {
		if ev.Type != traceparser.EvGCStart {
			t.Errorf("unwanted event in gcEvents: %v", ev)
		}
		if i > 0 && lastTS > ev.Ts {
			t.Errorf("overlapping GC events:\n%d: %v\n%d: %v", i-1, res.gcEvents[i-1], i, res.gcEvents[i])
		}
		if ev.Link != nil {
			lastTS = ev.Link.Ts
		}
	}

	// Check whether only taskWithGC reports overlapping duration.
	for _, task := range res.tasks {
		got := task.overlappingGCDuration(res.gcEvents)
		switch task.name {
		case "taskWithoutGC":
			if got != 0 {
				t.Errorf("%s reported %v as overlapping GC time; want 0: %v", task.name, got, task)
			}
		case "taskWithGC":
			upperBound := task.duration()
			// TODO(hyangah): a tighter upper bound is gcTime, but
			// use of it will make the test flaky due to the issue
			// described in golang.org/issue/16755. Tighten the upper
			// bound when the issue with the timestamp computed
			// based on clockticks is resolved.
			if got <= 0 || got > upperBound {
				t.Errorf("%s reported %v as overlapping GC time; want (0, %v):\n%v", task.name, got, upperBound, task)
				buf := new(bytes.Buffer)
				fmt.Fprintln(buf, "GC Events")
				for _, ev := range res.gcEvents {
					fmt.Fprintf(buf, " %s -> %s\n", ev, ev.Link)
				}
				fmt.Fprintln(buf, "Events in Task")
				for i, ev := range task.events {
					fmt.Fprintf(buf, " %d: %s\n", i, ev)
				}

				t.Logf("\n%s", buf)
			}
		}
	}
}

// traceProgram runs the provided function while tracing is enabled,
// parses the captured trace, and sets the global trace loader to
// point to the parsed trace.
//
// If savetraces flag is set, the captured trace will be saved in the named file.
func traceProgram(t *testing.T, f func(), name string) error {
	t.Helper()
	buf := new(bytes.Buffer)
	if err := trace.Start(buf); err != nil {
		return err
	}
	f()
	trace.Stop()

	saveTrace(buf, name)
	res, err := traceparser.Parse(buf, name+".faketrace")
	if err == traceparser.ErrTimeOrder {
		t.Skipf("skipping due to golang.org/issue/16755: %v", err)
	} else if err != nil {
		return err
	}

	swapLoaderData(res, err)
	return nil
}

func spanNames(task *taskDesc) (ret []string) {
	for _, s := range task.spans {
		ret = append(ret, s.name)
	}
	return ret
}

func parentName(task *taskDesc) string {
	if task.parent != nil {
		return task.parent.name
	}
	return ""
}

func childrenNames(task *taskDesc) (ret []string) {
	for _, s := range task.children {
		ret = append(ret, s.name)
	}
	return ret
}

func swapLoaderData(res traceparser.ParseResult, err error) {
	// swap loader's data.
	parseTrace() // fool loader.once.
	loader.res = res
	loader.err = err
}

func saveTrace(buf *bytes.Buffer, name string) {
	if !*saveTraces {
		return
	}
	if err := ioutil.WriteFile(name+".trace", buf.Bytes(), 0600); err != nil {
		panic(fmt.Errorf("failed to write trace file: %v", err))
	}
}
<commit_msg>cmd/trace: remove unused variable in tests<commit_after>package main

import (
	"bytes"
	"context"
	"flag"
	"fmt"
	traceparser "internal/trace"
	"io/ioutil"
	"reflect"
	"runtime/debug"
	"runtime/trace"
	"sync"
	"testing"
	"time"
)

var saveTraces = flag.Bool("savetraces", false, "save traces collected by tests")

func TestOverlappingDuration(t *testing.T) {
	cases := []struct {
		start0, end0, start1, end1 int64
		want                       time.Duration
	}{
		{
			1, 10, 11, 20, 0,
		},
		{
			1, 10, 5, 20, 5 * time.Nanosecond,
		},
		{
			1, 10, 2, 8, 6 * time.Nanosecond,
		},
	}

	for _, tc := range cases {
		s0, e0, s1, e1 := tc.start0, tc.end0, tc.start1, tc.end1
		if got := overlappingDuration(s0, e0, s1, e1); got != tc.want {
			t.Errorf("overlappingDuration(%d, %d, %d, %d)=%v; want %v", s0, e0, s1, e1, got, tc.want)
		}
		if got := overlappingDuration(s1, e1, s0, e0); got != tc.want {
			t.Errorf("overlappingDuration(%d, %d, %d, %d)=%v; want %v", s1, e1, s0, e0, got, tc.want)
		}
	}
}

// prog0 starts three goroutines.
//
//   goroutine 1: taskless span
//   goroutine 2: starts task0, do work in task0.span0, starts task1 which ends immediately.
//   goroutine 3: do work in task0.span1 and task0.span2, ends task0
func prog0() {
	ctx := context.Background()

	var wg sync.WaitGroup

	wg.Add(1)
	go func() { // goroutine 1
		defer wg.Done()
		trace.WithSpan(ctx, "taskless.span", func(ctx context.Context) {
			trace.Log(ctx, "key0", "val0")
		})
	}()

	wg.Add(1)
	go func() { // goroutine 2
		defer wg.Done()
		ctx, taskDone := trace.NewContext(ctx, "task0")
		trace.WithSpan(ctx, "task0.span0", func(ctx context.Context) {
			wg.Add(1)
			go func() { // goroutine 3
				defer wg.Done()
				defer taskDone()
				trace.WithSpan(ctx, "task0.span1", func(ctx context.Context) {
					trace.WithSpan(ctx, "task0.span2", func(ctx context.Context) {
						trace.Log(ctx, "key2", "val2")
					})
					trace.Log(ctx, "key1", "val1")
				})
			}()
		})
		ctx2, taskDone2 := trace.NewContext(ctx, "task1")
		trace.Log(ctx2, "key3", "val3")
		taskDone2()
	}()
	wg.Wait()
}

func TestAnalyzeAnnotations(t *testing.T) {
	// TODO: classify taskless spans

	// Run prog0 and capture the execution trace.
	if err := traceProgram(t, prog0, "TestAnalyzeAnnotations"); err != nil {
		t.Fatalf("failed to trace the program: %v", err)
	}

	res, err := analyzeAnnotations()
	if err != nil {
		t.Fatalf("failed to analyzeAnnotations: %v", err)
	}
	tasks := res.tasks

	// For prog0, we expect
	//   - task with name = "task0", with three spans.
	//   - task with name = "task1", with no span.
	wantTasks := map[string]struct {
		complete   bool
		goroutines int
		spans      []string
	}{
		"task0": {
			complete:   true,
			goroutines: 2,
			spans:      []string{"task0.span0", "task0.span1", "task0.span2"},
		},
		"task1": {
			complete:   true,
			goroutines: 1,
		},
	}

	for _, task := range tasks {
		want, ok := wantTasks[task.name]
		if !ok {
			t.Errorf("unexpected task: %s", task)
			continue
		}
		if task.complete() != want.complete || len(task.goroutines) != want.goroutines || !reflect.DeepEqual(spanNames(task), want.spans) {
			t.Errorf("got %v; want %+v", task, want)
		}

		delete(wantTasks, task.name)
	}
	if len(wantTasks) > 0 {
		t.Errorf("no more tasks; want %+v", wantTasks)
	}
}

// prog1 creates a task hierarchy consisting of three tasks.
func prog1() {
	ctx := context.Background()
	ctx1, done1 := trace.NewContext(ctx, "task1")
	defer done1()
	trace.WithSpan(ctx1, "task1.span", func(ctx context.Context) {
		ctx2, done2 := trace.NewContext(ctx, "task2")
		defer done2()
		trace.WithSpan(ctx2, "task2.span", func(ctx context.Context) {
			ctx3, done3 := trace.NewContext(ctx, "task3")
			defer done3()
			trace.WithSpan(ctx3, "task3.span", func(ctx context.Context) {
			})
		})
	})
}

func TestAnalyzeAnnotationTaskTree(t *testing.T) {
	// Run prog1 and capture the execution trace.
	if err := traceProgram(t, prog1, "TestAnalyzeAnnotationTaskTree"); err != nil {
		t.Fatalf("failed to trace the program: %v", err)
	}

	res, err := analyzeAnnotations()
	if err != nil {
		t.Fatalf("failed to analyzeAnnotations: %v", err)
	}
	tasks := res.tasks

	// For prog0, we expect
	//   - task with name = "", with taskless.span in spans.
	//   - task with name = "task0", with three spans.
	wantTasks := map[string]struct {
		parent   string
		children []string
		spans    []string
	}{
		"task1": {
			parent:   "",
			children: []string{"task2"},
			spans:    []string{"task1.span"},
		},
		"task2": {
			parent:   "task1",
			children: []string{"task3"},
			spans:    []string{"task2.span"},
		},
		"task3": {
			parent:   "task2",
			children: nil,
			spans:    []string{"task3.span"},
		},
	}

	for _, task := range tasks {
		want, ok := wantTasks[task.name]
		if !ok {
			t.Errorf("unexpected task: %s", task)
			continue
		}
		delete(wantTasks, task.name)

		if parentName(task) != want.parent ||
			!reflect.DeepEqual(childrenNames(task), want.children) ||
			!reflect.DeepEqual(spanNames(task), want.spans) {
			t.Errorf("got %v; want %+v", task, want)
		}
	}

	if len(wantTasks) > 0 {
		t.Errorf("no more tasks; want %+v", wantTasks)
	}
}

// prog2 starts two tasks; "taskWithGC" that overlaps with GC
// and "taskWithoutGC" that doesn't. In order to run this reliably,
// the caller needs to set up to prevent GC from running automatically.
// prog2 returns the upper-bound gc time that overlaps with the first task.
func prog2() (gcTime time.Duration) {
	ch := make(chan bool)
	ctx1, done := trace.NewContext(context.Background(), "taskWithGC")
	trace.WithSpan(ctx1, "taskWithGC.span1", func(ctx context.Context) {
		go func() {
			defer trace.StartSpan(ctx, "taskWithGC.span2")()
			<-ch
		}()
		s := time.Now()
		debug.FreeOSMemory() // task1 affected by gc
		gcTime = time.Since(s)
		close(ch)
	})
	done()

	ctx2, done2 := trace.NewContext(context.Background(), "taskWithoutGC")
	trace.WithSpan(ctx2, "taskWithoutGC.span1", func(ctx context.Context) {
		// do nothing.
	})
	done2()
	return gcTime
}

func TestAnalyzeAnnotationGC(t *testing.T) {
	err := traceProgram(t, func() {
		oldGC := debug.SetGCPercent(10000) // gc, and effectively disable GC
		defer debug.SetGCPercent(oldGC)
		prog2()
	}, "TestAnalyzeAnnotationGC")
	if err != nil {
		t.Fatalf("failed to trace the program: %v", err)
	}

	res, err := analyzeAnnotations()
	if err != nil {
		t.Fatalf("failed to analyzeAnnotations: %v", err)
	}

	// Check collected GC Start events are all sorted and non-overlapping.
	lastTS := int64(0)
	for i, ev := range res.gcEvents {
		if ev.Type != traceparser.EvGCStart {
			t.Errorf("unwanted event in gcEvents: %v", ev)
		}
		if i > 0 && lastTS > ev.Ts {
			t.Errorf("overlapping GC events:\n%d: %v\n%d: %v", i-1, res.gcEvents[i-1], i, res.gcEvents[i])
		}
		if ev.Link != nil {
			lastTS = ev.Link.Ts
		}
	}

	// Check whether only taskWithGC reports overlapping duration.
	for _, task := range res.tasks {
		got := task.overlappingGCDuration(res.gcEvents)
		switch task.name {
		case "taskWithoutGC":
			if got != 0 {
				t.Errorf("%s reported %v as overlapping GC time; want 0: %v", task.name, got, task)
			}
		case "taskWithGC":
			upperBound := task.duration()
			// TODO(hyangah): a tighter upper bound is gcTime, but
			// use of it will make the test flaky due to the issue
			// described in golang.org/issue/16755. Tighten the upper
			// bound when the issue with the timestamp computed
			// based on clockticks is resolved.
			if got <= 0 || got > upperBound {
				t.Errorf("%s reported %v as overlapping GC time; want (0, %v):\n%v", task.name, got, upperBound, task)
				buf := new(bytes.Buffer)
				fmt.Fprintln(buf, "GC Events")
				for _, ev := range res.gcEvents {
					fmt.Fprintf(buf, " %s -> %s\n", ev, ev.Link)
				}
				fmt.Fprintln(buf, "Events in Task")
				for i, ev := range task.events {
					fmt.Fprintf(buf, " %d: %s\n", i, ev)
				}

				t.Logf("\n%s", buf)
			}
		}
	}
}

// traceProgram runs the provided function while tracing is enabled,
// parses the captured trace, and sets the global trace loader to
// point to the parsed trace.
//
// If savetraces flag is set, the captured trace will be saved in the named file.
func traceProgram(t *testing.T, f func(), name string) error {
	t.Helper()
	buf := new(bytes.Buffer)
	if err := trace.Start(buf); err != nil {
		return err
	}
	f()
	trace.Stop()

	saveTrace(buf, name)
	res, err := traceparser.Parse(buf, name+".faketrace")
	if err == traceparser.ErrTimeOrder {
		t.Skipf("skipping due to golang.org/issue/16755: %v", err)
	} else if err != nil {
		return err
	}

	swapLoaderData(res, err)
	return nil
}

func spanNames(task *taskDesc) (ret []string) {
	for _, s := range task.spans {
		ret = append(ret, s.name)
	}
	return ret
}

func parentName(task *taskDesc) string {
	if task.parent != nil {
		return task.parent.name
	}
	return ""
}

func childrenNames(task *taskDesc) (ret []string) {
	for _, s := range task.children {
		ret = append(ret, s.name)
	}
	return ret
}

func swapLoaderData(res traceparser.ParseResult, err error) {
	// swap loader's data.
	parseTrace() // fool loader.once.
	loader.res = res
	loader.err = err
}

func saveTrace(buf *bytes.Buffer, name string) {
	if !*saveTraces {
		return
	}
	if err := ioutil.WriteFile(name+".trace", buf.Bytes(), 0600); err != nil {
		panic(fmt.Errorf("failed to write trace file: %v", err))
	}
}
<|endoftext|>


<commit_before>package LLVMCodegen

import (
	"io/ioutil"
	"os"
	"os/exec"
	"strings"

	"github.com/ark-lang/ark/src/parser"
	"llvm.org/llvm/bindings/go/llvm"
)

type OutputType int

const (
	OUTPUT_ASSEMBLY OutputType = iota
	OUTPUT_OBJECT
	OUTPUT_LLVM_IR
	OUTPUT_LLVM_BC
	OUTPUT_EXECUTABLE
)

func (v *Codegen) createBitcode(file *parser.Module) string {
	filename := file.Name + ".bc"

	fileHandle, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666)
	if err != nil {
		v.err("Couldn't create bitcode file "+filename+": `%s`", err.Error())
	}
	defer fileHandle.Close()

	if err := llvm.WriteBitcodeToFile(file.Module, fileHandle); err != nil {
		v.err("failed to write bitcode to file for "+file.Name+": `%s`", err.Error())
	}

	return filename
}

func (v *Codegen) bitcodeToASM(filename string) string {
	asmName := filename

	if strings.HasSuffix(filename, ".bc") {
		asmName = asmName[:len(asmName)-3]
	}

	asmName += ".s"

	cmd := exec.Command("llc", filename, "-o", asmName)
	if out, err := cmd.CombinedOutput(); err != nil {
		v.err("Failed to convert bitcode to assembly: `%s`\n%s", err.Error(), string(out))
	}

	return asmName
}

func (v *Codegen) asmToObject(filename string) string {
	objName := filename

	if strings.HasSuffix(filename, ".s") {
		objName = objName[:len(objName)-2]
	}

	objName += ".o"

	if v.Compiler == "" {
		v.Compiler = "cc"
	}

	args := append(v.CompilerArgs, "-fno-PIE", "-c", filename, "-o", objName)

	cmd := exec.Command(v.Compiler, args...)
	if out, err := cmd.CombinedOutput(); err != nil {
		v.err("Failed to convert assembly to object file: `%s`\n%s", err.Error(), string(out))
	}

	return objName
}

func (v *Codegen) createIR(mod *parser.Module) string {
	filename := mod.Name + ".ll"

	err := ioutil.WriteFile(filename, []byte(mod.Module.String()), 0666)
	if err != nil {
		v.err("Couldn't write IR file "+filename+": `%s`", err.Error())
	}

	return filename
}

func (v *Codegen) createBinary() {
	// god this is a long and ugly function

	if v.OutputType == OUTPUT_LLVM_IR {
		for _, file := range v.input {
			v.createIR(file)
		}
		return
	}

	linkArgs := append(v.LinkerArgs, "-fno-PIE", "-nodefaultlibs", "-lc", "-lm")

	bitcodeFiles := []string{}

	for _, file := range v.input {
		bitcodeFiles = append(bitcodeFiles, v.createBitcode(file))
	}

	if v.OutputType == OUTPUT_LLVM_BC {
		return
	}

	asmFiles := []string{}

	for _, name := range bitcodeFiles {
		asmName := v.bitcodeToASM(name)
		asmFiles = append(asmFiles, asmName)
	}

	for _, bc := range bitcodeFiles {
		if err := os.Remove(bc); err != nil {
			v.err("Failed to remove "+bc+": `%s`", err.Error())
		}
	}

	if v.OutputType == OUTPUT_ASSEMBLY {
		return
	}

	objFiles := []string{}

	for _, asmFile := range asmFiles {
		objName := v.asmToObject(asmFile)

		objFiles = append(objFiles, objName)
		linkArgs = append(linkArgs, objName)
	}

	for _, asmFile := range asmFiles {
		os.Remove(asmFile)
	}

	if v.OutputType == OUTPUT_OBJECT {
		return
	}

	if v.OutputName == "" {
		panic("OutputName is empty")
	}

	linkArgs = append(linkArgs, "-o", v.OutputName)

	if v.Linker == "" {
		v.Linker = "cc"
	}

	cmd := exec.Command(v.Linker, linkArgs...)
	if out, err := cmd.CombinedOutput(); err != nil {
		v.err("failed to link object files: `%s`\n%s", err.Error(), string(out))
	}

	for _, objFile := range objFiles {
		os.Remove(objFile)
	}
}
<commit_msg>Use enviroment CC if available<commit_after>package LLVMCodegen

import (
	"io/ioutil"
	"os"
	"os/exec"
	"strings"

	"github.com/ark-lang/ark/src/parser"
	"llvm.org/llvm/bindings/go/llvm"
)

type OutputType int

const (
	OUTPUT_ASSEMBLY OutputType = iota
	OUTPUT_OBJECT
	OUTPUT_LLVM_IR
	OUTPUT_LLVM_BC
	OUTPUT_EXECUTABLE
)

func (v *Codegen) createBitcode(file *parser.Module) string {
	filename := file.Name + ".bc"

	fileHandle, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666)
	if err != nil {
		v.err("Couldn't create bitcode file "+filename+": `%s`", err.Error())
	}
	defer fileHandle.Close()

	if err := llvm.WriteBitcodeToFile(file.Module, fileHandle); err != nil {
		v.err("failed to write bitcode to file for "+file.Name+": `%s`", err.Error())
	}

	return filename
}

func (v *Codegen) bitcodeToASM(filename string) string {
	asmName := filename

	if strings.HasSuffix(filename, ".bc") {
		asmName = asmName[:len(asmName)-3]
	}

	asmName += ".s"

	cmd := exec.Command("llc", filename, "-o", asmName)
	if out, err := cmd.CombinedOutput(); err != nil {
		v.err("Failed to convert bitcode to assembly: `%s`\n%s", err.Error(), string(out))
	}

	return asmName
}

func (v *Codegen) asmToObject(filename string) string {
	objName := filename

	if strings.HasSuffix(filename, ".s") {
		objName = objName[:len(objName)-2]
	}

	objName += ".o"

	if v.Compiler == "" {
		envcc := os.Getenv("CC")
		if envcc != "" {
			v.Compiler = envcc
		} else {
			v.Compiler = "cc"
		}
	}

	args := append(v.CompilerArgs, "-fno-PIE", "-c", filename, "-o", objName)

	cmd := exec.Command(v.Compiler, args...)
	if out, err := cmd.CombinedOutput(); err != nil {
		v.err("Failed to convert assembly to object file: `%s`\n%s", err.Error(), string(out))
	}

	return objName
}

func (v *Codegen) createIR(mod *parser.Module) string {
	filename := mod.Name + ".ll"

	err := ioutil.WriteFile(filename, []byte(mod.Module.String()), 0666)
	if err != nil {
		v.err("Couldn't write IR file "+filename+": `%s`", err.Error())
	}

	return filename
}

func (v *Codegen) createBinary() {
	// god this is a long and ugly function

	if v.OutputType == OUTPUT_LLVM_IR {
		for _, file := range v.input {
			v.createIR(file)
		}
		return
	}

	linkArgs := append(v.LinkerArgs, "-fno-PIE", "-nodefaultlibs", "-lc", "-lm")

	bitcodeFiles := []string{}

	for _, file := range v.input {
		bitcodeFiles = append(bitcodeFiles, v.createBitcode(file))
	}

	if v.OutputType == OUTPUT_LLVM_BC {
		return
	}

	asmFiles := []string{}

	for _, name := range bitcodeFiles {
		asmName := v.bitcodeToASM(name)
		asmFiles = append(asmFiles, asmName)
	}

	for _, bc := range bitcodeFiles {
		if err := os.Remove(bc); err != nil {
			v.err("Failed to remove "+bc+": `%s`", err.Error())
		}
	}

	if v.OutputType == OUTPUT_ASSEMBLY {
		return
	}

	objFiles := []string{}

	for _, asmFile := range asmFiles {
		objName := v.asmToObject(asmFile)

		objFiles = append(objFiles, objName)
		linkArgs = append(linkArgs, objName)
	}

	for _, asmFile := range asmFiles {
		os.Remove(asmFile)
	}

	if v.OutputType == OUTPUT_OBJECT {
		return
	}

	if v.OutputName == "" {
		panic("OutputName is empty")
	}

	linkArgs = append(linkArgs, "-o", v.OutputName)

	if v.Linker == "" {
		v.Linker = "cc"
	}

	cmd := exec.Command(v.Linker, linkArgs...)
	if out, err := cmd.CombinedOutput(); err != nil {
		v.err("failed to link object files: `%s`\n%s", err.Error(), string(out))
	}

	for _, objFile := range objFiles {
		os.Remove(objFile)
	}
}
<|endoftext|>


<commit_before>package goriak

import (
	"errors"
	riak "github.com/basho/riak-go-client"
)

// Command is the main query builder object
type Command struct {
	// Key information
	bucket     string
	bucketType string
}

// Result contains your query result data from Run()
type Result struct {
	NotFound bool   // Wether or not the item was not found when using Get, GetJSON, or GetRaw.
	Key      string // Returns your automatically generated key when using Set, SetJSON, or SetRaw.
	Context  []byte // Returns the Riak Context used in map operations. Is set when using Get.
	VClock   []byte
}

// Bucket specifies the bucket and bucket type that your following command will be performed on.
func Bucket(bucket, bucketType string) *Command {
	return &Command{
		bucket:     bucket,
		bucketType: bucketType,
	}
}

func (c *Command) resultListKeysCommand(cmd *riak.ListKeysCommand) (*Result, error) {
	if !cmd.Success() {
		return nil, errors.New("Not successful")
	}

	return &Result{}, nil
}

func (c *Command) resultDeleteValueCommand(cmd *riak.DeleteValueCommand) (*Result, error) {
	if !cmd.Success() {
		return nil, errors.New("Not successful")
	}

	return &Result{}, nil
}

func (c *Command) resultSecondaryIndexQueryCommand(cmd *riak.SecondaryIndexQueryCommand) (*Result, error) {
	if !cmd.Success() {
		return nil, errors.New("Not successful")
	}

	return &Result{}, nil
}
<commit_msg>#43 - Remove unused code<commit_after>package goriak

import (
	"errors"
	riak "github.com/basho/riak-go-client"
)

// Command is the main query builder object
type Command struct {
	// Key information
	bucket     string
	bucketType string
}

// Result contains your query result data from Run()
type Result struct {
	NotFound bool   // Wether or not the item was not found when using Get, GetJSON, or GetRaw.
	Key      string // Returns your automatically generated key when using Set, SetJSON, or SetRaw.
	Context  []byte // Returns the Riak Context used in map operations. Is set when using Get.
	VClock   []byte
}

// Bucket specifies the bucket and bucket type that your following command will be performed on.
func Bucket(bucket, bucketType string) *Command {
	return &Command{
		bucket:     bucket,
		bucketType: bucketType,
	}
}
<|endoftext|>


<commit_before>package cellnet

import (
	"fmt"
	"log"
	"runtime/debug"
	"sync"
)

// 事件队列
type EventQueue interface {
	// 事件队列开始工作
	StartLoop() EventQueue

	// 停止事件队列
	StopLoop() EventQueue

	// 等待退出
	Wait()

	// 投递事件, 通过队列到达消费者端
	Post(callback func())

	// 是否捕获异常
	EnableCapturePanic(v bool)
}

type CapturePanicNotifyFunc func(interface{}, EventQueue)

type eventQueue struct {
	*Pipe

	endSignal sync.WaitGroup

	capturePanic bool

	onPanic CapturePanicNotifyFunc
}

// 启动崩溃捕获
func (self *eventQueue) EnableCapturePanic(v bool) {
	self.capturePanic = v
}

// 设置捕获崩溃通知
func (self *eventQueue) SetCapturePanicNotify(callback CapturePanicNotifyFunc) {
	self.onPanic = callback
}

// 派发事件处理回调到队列中
func (self *eventQueue) Post(callback func()) {

	if callback == nil {
		return
	}

	self.Add(callback)
}

// 保护调用用户函数
func (self *eventQueue) protectedCall(callback func()) {

	if self.capturePanic {
		defer func() {

			if err := recover(); err != nil {
				self.onPanic(err, self)
			}

		}()
	}

	callback()
}

// 开启事件循环
func (self *eventQueue) StartLoop() EventQueue {

	self.endSignal.Add(1)

	go func() {

		var writeList []interface{}

		for {
			writeList = writeList[0:0]
			exit := self.Pick(&writeList)

			// 遍历要发送的数据
			for _, msg := range writeList {
				switch t := msg.(type) {
				case func():
					self.protectedCall(t)
				case nil:
					break
				default:
					log.Printf("unexpected type %T", t)
				}
			}

			if exit {
				break
			}
		}

		self.endSignal.Done()
	}()

	return self
}

// 停止事件循环
func (self *eventQueue) StopLoop() EventQueue {
	self.Add(nil)
	return self
}

// 等待退出消息
func (self *eventQueue) Wait() {
	self.endSignal.Wait()
}

// 创建默认长度的队列
func NewEventQueue() EventQueue {

	return &eventQueue{
		Pipe: NewPipe(),

		// 默认的崩溃捕获打印
		onPanic: func(raw interface{}, queue EventQueue) {

			fmt.Printf("%v \n%s\n", raw, string(debug.Stack()))
			debug.PrintStack()
		},
	}
}

// 在会话对应的Peer上的事件队列中执行callback，如果没有队列，则马上执行
func SessionQueuedCall(ses Session, callback func()) {
	if ses == nil {
		return
	}
	q := ses.Peer().(interface {
		Queue() EventQueue
	}).Queue()

	QueuedCall(q, callback)
}

// 有队列时队列调用，无队列时直接调用
func QueuedCall(queue EventQueue, callback func()) {
	if queue == nil {
		callback()
	} else {
		queue.Post(callback)
	}
}
<commit_msg>崩溃时打印时间<commit_after>package cellnet

import (
	"fmt"
	"log"
	"runtime/debug"
	"sync"
	"time"
)

// 事件队列
type EventQueue interface {
	// 事件队列开始工作
	StartLoop() EventQueue

	// 停止事件队列
	StopLoop() EventQueue

	// 等待退出
	Wait()

	// 投递事件, 通过队列到达消费者端
	Post(callback func())

	// 是否捕获异常
	EnableCapturePanic(v bool)
}

type CapturePanicNotifyFunc func(interface{}, EventQueue)

type eventQueue struct {
	*Pipe

	endSignal sync.WaitGroup

	capturePanic bool

	onPanic CapturePanicNotifyFunc
}

// 启动崩溃捕获
func (self *eventQueue) EnableCapturePanic(v bool) {
	self.capturePanic = v
}

// 设置捕获崩溃通知
func (self *eventQueue) SetCapturePanicNotify(callback CapturePanicNotifyFunc) {
	self.onPanic = callback
}

// 派发事件处理回调到队列中
func (self *eventQueue) Post(callback func()) {

	if callback == nil {
		return
	}

	self.Add(callback)
}

// 保护调用用户函数
func (self *eventQueue) protectedCall(callback func()) {

	if self.capturePanic {
		defer func() {

			if err := recover(); err != nil {
				self.onPanic(err, self)
			}

		}()
	}

	callback()
}

// 开启事件循环
func (self *eventQueue) StartLoop() EventQueue {

	self.endSignal.Add(1)

	go func() {

		var writeList []interface{}

		for {
			writeList = writeList[0:0]
			exit := self.Pick(&writeList)

			// 遍历要发送的数据
			for _, msg := range writeList {
				switch t := msg.(type) {
				case func():
					self.protectedCall(t)
				case nil:
					break
				default:
					log.Printf("unexpected type %T", t)
				}
			}

			if exit {
				break
			}
		}

		self.endSignal.Done()
	}()

	return self
}

// 停止事件循环
func (self *eventQueue) StopLoop() EventQueue {
	self.Add(nil)
	return self
}

// 等待退出消息
func (self *eventQueue) Wait() {
	self.endSignal.Wait()
}

// 创建默认长度的队列
func NewEventQueue() EventQueue {

	return &eventQueue{
		Pipe: NewPipe(),

		// 默认的崩溃捕获打印
		onPanic: func(raw interface{}, queue EventQueue) {

			fmt.Printf("%s: %v \n%s\n", time.Now().Format("2006-01-02 15:04:05"), raw, string(debug.Stack()))
			debug.PrintStack()
		},
	}
}

// 在会话对应的Peer上的事件队列中执行callback，如果没有队列，则马上执行
func SessionQueuedCall(ses Session, callback func()) {
	if ses == nil {
		return
	}
	q := ses.Peer().(interface {
		Queue() EventQueue
	}).Queue()

	QueuedCall(q, callback)
}

// 有队列时队列调用，无队列时直接调用
func QueuedCall(queue EventQueue, callback func()) {
	if queue == nil {
		callback()
	} else {
		queue.Post(callback)
	}
}
<|endoftext|>


<commit_before>package varys

import "github.com/garyburd/redigo/redis"

const (
	queueReady   = "queue-ready"
	queuePending = "queue-pending"
	queueDone    = "queue-done"
	queueFailed  = "queue-failed"
)

type Queue interface {
	Enqueue(urls ...string) error
	Dequeue() (string, error)

	Repaire() error

	DoneURL(url string) error
	RetryURL(url string) error

	FailedURLs() []string

	Cleanup()
}

type RedisQueue struct {
	pool *redis.Pool
}

func NewRedisQueue() *RedisQueue {
	return &RedisQueue{
		pool: &redis.Pool{
			Dial: func() (redis.Conn, error) {
				return redis.Dial("tcp", "127.0.0.1:6379")
			},
			Wait: true,
		},
	}
}

func (q *RedisQueue) Enqueue(urls ...string) error {
	conn := q.pool.Get()
	defer conn.Close()

	for _, url := range urls {
		if dup, err := redis.Bool(conn.Do("SISMEMBER", queueFailed, url)); err == nil && dup {
			continue
		}
		if dup, err := redis.Bool(conn.Do("SISMEMBER", queueDone, url)); err == nil && dup {
			continue
		}
		if dup, err := redis.Bool(conn.Do("SISMEMBER", queuePending, url)); err == nil && dup {
			continue
		}
		_, err := conn.Do("SADD", queueReady, url)
		if err != nil {
			return err
		}
	}
	return nil
}

func (q *RedisQueue) Dequeue() (url string, err error) {
	conn := q.pool.Get()
	defer conn.Close()

	url, err = redis.String(conn.Do("SPOP", queueReady))
	if err != nil {
		return "", nil
	}
	_, err = conn.Do("SADD", queuePending, url)

	return
}

func (q *RedisQueue) Repaire() error {
	conn := q.pool.Get()
	defer conn.Close()
	_, err := conn.Do("SUNIONSTORE", queueReady, queueReady, queuePending)
	return err
}

func (q *RedisQueue) DoneURL(url string) error {
	conn := q.pool.Get()
	defer conn.Close()
	_, err := conn.Do("SMOVE", queuePending, queueDone, url)
	return err
}

func (q *RedisQueue) RetryURL(url string) error {
	conn := q.pool.Get()
	defer conn.Close()
	_, err := conn.Do("SMOVE", queuePending, queueFailed, url)
	return err
}

func (q *RedisQueue) FailedURLs() []string {
	conn := q.pool.Get()
	defer conn.Close()
	urls, err := redis.Strings(conn.Do("SMEMBERS", queueFailed))
	if err != nil {
		return nil
	}
	return urls
}

func (q *RedisQueue) Cleanup() {

}
<commit_msg>mv queue name into struct<commit_after>package varys

import "github.com/garyburd/redigo/redis"

type Queue interface {
	Enqueue(urls ...string) error
	Dequeue() (string, error)

	Repaire() error

	DoneURL(url string) error
	RetryURL(url string) error

	FailedURLs() []string

	Cleanup()
}

type RedisQueue struct {
	pool *redis.Pool

	QueueReady   string
	QueuePending string
	QueueDone    string
	QueueFailed  string
}

func NewRedisQueue() *RedisQueue {
	return &RedisQueue{
		pool: &redis.Pool{
			Dial: func() (redis.Conn, error) {
				return redis.Dial("tcp", "127.0.0.1:6379")
			},
			Wait: true,
		},
		QueueReady:   "queue-ready",
		QueuePending: "queue-pending",
		QueueDone:    "queue-done",
		QueueFailed:  "queue-failed",
	}
}

func (q *RedisQueue) Enqueue(urls ...string) error {
	conn := q.pool.Get()
	defer conn.Close()

	for _, url := range urls {
		if dup, err := redis.Bool(conn.Do("SISMEMBER", q.QueueFailed, url)); err == nil && dup {
			continue
		}
		if dup, err := redis.Bool(conn.Do("SISMEMBER", q.QueueDone, url)); err == nil && dup {
			continue
		}
		if dup, err := redis.Bool(conn.Do("SISMEMBER", q.QueuePending, url)); err == nil && dup {
			continue
		}
		_, err := conn.Do("SADD", q.QueueReady, url)
		if err != nil {
			return err
		}
	}
	return nil
}

func (q *RedisQueue) Dequeue() (url string, err error) {
	conn := q.pool.Get()
	defer conn.Close()

	url, err = redis.String(conn.Do("SPOP", q.QueueReady))
	if err != nil {
		return "", nil
	}
	_, err = conn.Do("SADD", q.QueuePending, url)

	return
}

func (q *RedisQueue) Repaire() error {
	conn := q.pool.Get()
	defer conn.Close()
	_, err := conn.Do("SUNIONSTORE", q.QueueReady, q.QueueReady, q.QueuePending)
	return err
}

func (q *RedisQueue) DoneURL(url string) error {
	conn := q.pool.Get()
	defer conn.Close()
	_, err := conn.Do("SMOVE", q.QueuePending, q.QueueDone, url)
	return err
}

func (q *RedisQueue) RetryURL(url string) error {
	conn := q.pool.Get()
	defer conn.Close()
	_, err := conn.Do("SMOVE", q.QueuePending, q.QueueFailed, url)
	return err
}

func (q *RedisQueue) FailedURLs() []string {
	conn := q.pool.Get()
	defer conn.Close()
	urls, err := redis.Strings(conn.Do("SMEMBERS", q.QueueFailed))
	if err != nil {
		return nil
	}
	return urls
}

func (q *RedisQueue) Cleanup() {

}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"log"
	"os"
	"strings"
)

func main() {
	lgr := log.New(os.Stderr, "", 0)

	err := run(os.Stdin, os.Stdout)
	if err != nil {
		lgr.Printf("run: %v", err)
		os.Exit(1)
	}
}

func run(stdin io.Reader, stdout io.Writer) error {
	var raw string
	for {
		_, err := fmt.Fprintf(stdout, "> ")
		if err != nil {
			return err
		}

		_, err = fmt.Fscanf(stdin, "%s", &raw)
		if err != nil {
			return err
		}

		cmd := strings.TrimSpace(raw)
		if cmd == "exit" {
			break
		}

		_, err = fmt.Fprintln(stdout, cmd)
		if err != nil {
			return fmt.Errorf("write: %v", err)
		}
	}

	return nil
}
<commit_msg>shell: clean run<commit_after>package main

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"os"
	"strings"
)

func main() {
	lgr := log.New(os.Stderr, "", 0)

	err := run(os.Stdin, os.Stdout)
	if err != nil {
		lgr.Printf("run: %v", err)
		os.Exit(1)
	}
}

func run(stdin io.Reader, stdout io.Writer) error {
	r := bufio.NewReader(stdin)
	for {
		_, err := fmt.Fprintf(stdout, "> ")
		if err != nil {
			return err
		}

		raw, err := r.ReadString('\n')
		if err != nil {
			return err
		}

		cmd := strings.TrimSpace(raw)
		if cmd == "exit" {
			break
		}

		_, err = fmt.Fprintln(stdout, cmd)
		if err != nil {
			return fmt.Errorf("write: %v", err)
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import "github.com/gotgo/storm"

func main() {
	runSpout()
}

func runSpout() {
	s := storm.NewStormSession()
	spout := storm.NewSpout(s, &MySpout{})
	go spout.Run()

	select {
	case <-s.Done:
	}

	close(s.Done)
}

func runBolt() {
	s := storm.NewStormSession()
	b := storm.NewBolt(s, &MyBolt{})
	go b.Process()

	select {
	case <-s.Done:
	}

	close(s.Done)
}

type MySpout struct {
}

func (s *MySpout) Emit() *storm.TupleMessage {
	return nil
}
func (s *MySpout) Ack(id string) {

}
func (s *MySpout) Fail(id string) {

}
func (s *MySpout) AssociateTasks(id string, taskIds []int) {

}

type MyBolt struct {
}

func (mb *MyBolt) Process(tuple *storm.TupleMessage) (error, *storm.TupleMessage) {
	return nil, nil
}

func createTopology() {
	t := storm.NewTopology("word counter")
	parallelism := int32(2)

	randomScentence := t.AddSpout("randomScentence", &storm.ComponentDef{
		ShellCommand: "/opt/myapp/app",
		OutputFields: []string{},
		Direct:       false,
		Parallelism:  parallelism,
	})

	splitter := t.AddBolt("wordSplitter", &storm.ComponentDef{
		ShellCommand: "/opt/mybolt/app",
		OutputFields: []string{},
		Direct:       false,
		Parallelism:  parallelism,
	})
	splitter.Input(randomScentence, storm.DistributeByShuffle, nil)

	wordCounter := t.AddBolt("wordCounter", &storm.ComponentDef{
		ShellCommand: "/opt/mybolt/app",
		OutputFields: []string{},
		Direct:       false,
		Parallelism:  parallelism,
	})
	wordCounter.Input(splitter, storm.DistributeByShuffle, nil)
}
<commit_msg>updated example<commit_after>package main

import "github.com/gotgo/storm"

func main() {
	runSpout()
}

func runSpout() {
	s := storm.NewStorm()
	s.Run()
	spout := storm.NewSpout(s, &MySpout{})
	go spout.Run()

	//TODO: block on os signal
	s.End()
}

func runBolt() {
	s := storm.NewStorm()
	s.Run()
	b := storm.NewBolt(s, &MyBolt{})
	go b.Run()
	//TODO: block on os signal
	s.End()
}

type MySpout struct {
}

func (s *MySpout) Emit() *storm.TupleMessage {
	return nil
}
func (s *MySpout) Ack(id string) {

}
func (s *MySpout) Fail(id string) {

}
func (s *MySpout) AssociateTasks(id string, taskIds []int) {

}

type MyBolt struct {
}

func (b *MyBolt) Process(tuple *storm.TupleMessage) (*storm.TupleMessage, error) {
	return nil, nil
}

func (b *MyBolt) TrackIndirectEmit(taskIds []int) {}

func createTopology() {
	t := storm.NewTopology("word counter")
	parallelism := int32(2)

	randomScentence := t.AddSpout("randomScentence", &storm.ComponentDef{
		ShellCommand: "/opt/myapp/app",
		OutputFields: []string{},
		Direct:       false,
		Parallelism:  parallelism,
	})

	splitter := t.AddBolt("wordSplitter", &storm.ComponentDef{
		ShellCommand: "/opt/mybolt/app",
		OutputFields: []string{},
		Direct:       false,
		Parallelism:  parallelism,
	})
	splitter.Input(randomScentence, storm.DistributeByShuffle, nil)

	wordCounter := t.AddBolt("wordCounter", &storm.ComponentDef{
		ShellCommand: "/opt/mybolt/app",
		OutputFields: []string{},
		Direct:       false,
		Parallelism:  parallelism,
	})
	wordCounter.Input(splitter, storm.DistributeByShuffle, nil)
}
<|endoftext|>


<commit_before>package oci8_test

import (
	"context"
	"database/sql"
	"fmt"
	"log"
	"os"
	"time"

	"github.com/mattn/go-oci8"
)

func Example_sqlSelect() {
	// Example shows how to do a basic select

	oci8.OCI8Driver.Logger = log.New(os.Stderr, "oci8 ", log.Ldate|log.Ltime|log.LUTC|log.Llongfile)

	var openString string
	// [username/[password]@]host[:port][/instance_name][?param1=value1&...&paramN=valueN]
	if len(oci8.TestUsername) > 0 {
		if len(oci8.TestPassword) > 0 {
			openString = oci8.TestUsername + "/" + oci8.TestPassword + "@"
		} else {
			openString = oci8.TestUsername + "@"
		}
	}
	openString += oci8.TestHostValid

	// A normal simple Open to localhost would look like:
	// db, err := sql.Open("oci8", "127.0.0.1")
	// For testing, need to use additional variables
	db, err := sql.Open("oci8", openString)
	if err != nil {
		fmt.Printf("Open error is not nil: %v", err)
		return
	}
	if db == nil {
		fmt.Println("db is nil")
		return
	}

	ctx, cancel := context.WithTimeout(context.Background(), 55*time.Second)
	defer cancel()
	rows, err := db.QueryContext(ctx, "select 1 from dual")
	if err != nil {
		fmt.Println("QueryContext error is not nil:", err)
		return
	}
	if !rows.Next() {
		fmt.Println("no Next rows")
		return
	}

	dest := make([]interface{}, 1)
	destPointer := make([]interface{}, 1)
	destPointer[0] = &dest[0]
	err = rows.Scan(destPointer...)
	if err != nil {
		fmt.Println("Scan error is not nil:", err)
		return
	}

	if len(dest) != 1 {
		fmt.Println("len dest != 1")
		return
	}
	data, ok := dest[0].(float64)
	if !ok {
		fmt.Println("dest type not float64")
		return
	}
	if data != 1 {
		fmt.Println("data not equal to 1")
		return
	}

	if rows.Next() {
		fmt.Println("has Next rows")
		return
	}

	err = rows.Err()
	if err != nil {
		fmt.Println("Err error is not nil:", err)
		return
	}
	err = rows.Close()
	if err != nil {
		fmt.Println("Close error is not nil:", err)
		return
	}
	cancel()

	err = db.Close()
	if err != nil {
		fmt.Println("Close error is not nil:", err)
		return
	}

	fmt.Println(data)

	// output: 1
}
<commit_msg>Added Example_sqlFunction<commit_after>package oci8_test

import (
	"context"
	"database/sql"
	"fmt"
	"log"
	"os"
	"time"

	"github.com/mattn/go-oci8"
)

func Example_sqlSelect() {
	// Example shows how to do a basic select

	oci8.OCI8Driver.Logger = log.New(os.Stderr, "oci8 ", log.Ldate|log.Ltime|log.LUTC|log.Llongfile)

	var openString string
	// [username/[password]@]host[:port][/instance_name][?param1=value1&...&paramN=valueN]
	if len(oci8.TestUsername) > 0 {
		if len(oci8.TestPassword) > 0 {
			openString = oci8.TestUsername + "/" + oci8.TestPassword + "@"
		} else {
			openString = oci8.TestUsername + "@"
		}
	}
	openString += oci8.TestHostValid

	// A normal simple Open to localhost would look like:
	// db, err := sql.Open("oci8", "127.0.0.1")
	// For testing, need to use additional variables
	db, err := sql.Open("oci8", openString)
	if err != nil {
		fmt.Printf("Open error is not nil: %v", err)
		return
	}
	if db == nil {
		fmt.Println("db is nil")
		return
	}

	ctx, cancel := context.WithTimeout(context.Background(), 55*time.Second)
	defer cancel()
	rows, err := db.QueryContext(ctx, "select 1 from dual")
	if err != nil {
		fmt.Println("QueryContext error is not nil:", err)
		return
	}
	if !rows.Next() {
		fmt.Println("no Next rows")
		return
	}

	dest := make([]interface{}, 1)
	destPointer := make([]interface{}, 1)
	destPointer[0] = &dest[0]
	err = rows.Scan(destPointer...)
	if err != nil {
		fmt.Println("Scan error is not nil:", err)
		return
	}

	if len(dest) != 1 {
		fmt.Println("len dest != 1")
		return
	}
	data, ok := dest[0].(float64)
	if !ok {
		fmt.Println("dest type not float64")
		return
	}
	if data != 1 {
		fmt.Println("data not equal to 1")
		return
	}

	if rows.Next() {
		fmt.Println("has Next rows")
		return
	}

	err = rows.Err()
	if err != nil {
		fmt.Println("Err error is not nil:", err)
		return
	}
	err = rows.Close()
	if err != nil {
		fmt.Println("Close error is not nil:", err)
		return
	}
	cancel()

	err = db.Close()
	if err != nil {
		fmt.Println("Close error is not nil:", err)
		return
	}

	fmt.Println(data)

	// output: 1
}

func Example_sqlFunction() {
	// Example shows how to do a function call with binds

	oci8.OCI8Driver.Logger = log.New(os.Stderr, "oci8 ", log.Ldate|log.Ltime|log.LUTC|log.Llongfile)

	var openString string
	// [username/[password]@]host[:port][/instance_name][?param1=value1&...&paramN=valueN]
	if len(oci8.TestUsername) > 0 {
		if len(oci8.TestPassword) > 0 {
			openString = oci8.TestUsername + "/" + oci8.TestPassword + "@"
		} else {
			openString = oci8.TestUsername + "@"
		}
	}
	openString += oci8.TestHostValid

	// A normal simple Open to localhost would look like:
	// db, err := sql.Open("oci8", "127.0.0.1")
	// For testing, need to use additional variables
	db, err := sql.Open("oci8", openString)
	if err != nil {
		fmt.Printf("Open error is not nil: %v", err)
		return
	}
	if db == nil {
		fmt.Println("db is nil")
		return
	}

	number := int64(2)
	query := `
declare
	function ADD_ONE(p_number INTEGER) return INTEGER as
	begin
		return p_number + 1;
	end ADD_ONE;
begin
	:num1 := ADD_ONE(:num1);
end;`

	ctx, cancel := context.WithTimeout(context.Background(), 55*time.Second)
	defer cancel()
	_, err = db.ExecContext(ctx, query, sql.Out{Dest: &number, In: true})
	if err != nil {
		fmt.Println("ExecContext error is not nil:", err)
		return
	}

	if number != 3 {
		fmt.Println("number != 3")
		return
	}

	fmt.Println(number)

	// output: 3
}
<|endoftext|>


<commit_before>package sia

import (
	"bytes"
	"errors"
	"math/big"
	"sort"
	"time"
)

// Used to keep track of how many signatures an input has been signed by.
type InputSignatures struct {
	RemainingSignatures uint8
	PossibleKeys        []PublicKey
	UsedKeys            map[uint8]struct{}
}

// checkMaps looks through the maps known to the state and sees if the block id
// has been cached anywhere.
func (s *State) checkMaps(b *Block) (parentBlockNode *BlockNode, err error) {
	// See if the block is a known invalid block.
	_, exists := s.BadBlocks[b.ID()]
	if exists {
		err = errors.New("block is known to be invalid")
		return
	}

	// See if the block is a known valid block.
	_, exists = s.BlockMap[b.ID()]
	if exists {
		err = errors.New("Block exists in block map.")
		return
	}

	/*
		// See if the block is a known orphan.
		_, exists = s.OrphanBlocks[b.ID()]
		if exists {
			err = errors.New("Block exists in orphan list")
			return
		}
	*/

	// See if the block's parent is known.
	parentBlockNode, exists = s.BlockMap[b.ParentBlock]
	if !exists {
		// OrphanBlocks[b.ID()] = b
		err = errors.New("Block is an orphan")
		return
	}

	return
}

// Returns true if timestamp is valid, and if target value is reached.
func (s *State) validateHeader(parent *BlockNode, b *Block) (err error) {
	// Check that the block is not too far in the future.
	skew := b.Timestamp - Timestamp(time.Now().Unix())
	if skew > FutureThreshold {
		// Do something so that you will return to considering this
		// block once it's no longer too far in the future.
		err = errors.New("timestamp too far in future")
		return
	}

	// If timestamp is too far in the past, reject and put in bad blocks.
	var intTimestamps []int
	for _, timestamp := range parent.RecentTimestamps {
		intTimestamps = append(intTimestamps, int(timestamp))
	}
	sort.Ints(intTimestamps)
	if Timestamp(intTimestamps[5]) > b.Timestamp {
		s.BadBlocks[b.ID()] = struct{}{}
		err = errors.New("timestamp invalid for being in the past")
		return
	}

	// Check the id meets the target.
	blockHash := b.ID()
	if bytes.Compare(parent.Target[:], blockHash[:]) < 0 {
		err = errors.New("block does not meet target")
		return
	}

	return
}

// Takes a block and a parent node, and adds the block as a child to the parent
// node.
func (s *State) addBlockToTree(parentNode *BlockNode, b *Block) (newNode *BlockNode) {
	// Create the child node.
	newNode = new(BlockNode)
	newNode.Block = b
	newNode.Height = parentNode.Height + 1

	// Copy over the timestamps.
	copy(newNode.RecentTimestamps[:], parentNode.RecentTimestamps[1:])
	newNode.RecentTimestamps[10] = b.Timestamp

	// Calculate target and depth.

	// Add the node to the block map and the list of its parents children.
	s.BlockMap[b.ID()] = newNode
	parentNode.Children = append(parentNode.Children, newNode)

	return
}

// Add a block to the state struct.
func (s *State) AcceptBlock(b *Block) (err error) {
	// Check the maps in the state to see if the block is already known.
	parentBlockNode, err := s.checkMaps(b)
	if err != nil {
		return
	}

	// Check that the header of the block is valid.
	err = s.validateHeader(parentBlockNode, b)
	if err != nil {
		return
	}

	newBlockNode := s.addBlockToTree(parentBlockNode, b)

	///////////// Can be made into a function for calculating adjusted difficulty.
	var timePassed Timestamp
	var expectedTimePassed Timestamp
	var blockWindow BlockHeight
	if newBlockNode.Height < 5000 {
		// Calculate new target, using block 0 timestamp.
		timePassed = b.Timestamp - s.BlockRoot.Block.Timestamp
		expectedTimePassed = TargetSecondsPerBlock * Timestamp(newBlockNode.Height)
		blockWindow = newBlockNode.Height
	} else {
		// Calculate new target, using block Height-5000 timestamp.
		timePassed = b.Timestamp - s.BlockMap[s.ConsensusState.CurrentPath[newBlockNode.Height-5000]].Block.Timestamp
		expectedTimePassed = TargetSecondsPerBlock * 5000
		blockWindow = 5000
	}

	// Adjustment as a float = timePassed / expectedTimePassed / blockWindow.
	targetAdjustment := big.NewRat(int64(timePassed), int64(expectedTimePassed)*int64(blockWindow))

	// Enforce a maximum targetAdjustment
	if targetAdjustment.Cmp(MaxAdjustmentUp) == 1 {
		targetAdjustment = MaxAdjustmentUp
	} else if targetAdjustment.Cmp(MaxAdjustmentDown) == -1 {
		targetAdjustment = MaxAdjustmentDown
	}

	// Take the target adjustment and apply it to the target slice,
	// using rational numbers. Truncate the result.
	oldTarget := new(big.Int).SetBytes(parentBlockNode.Target[:])
	ratOldTarget := new(big.Rat).SetInt(oldTarget)
	ratNewTarget := ratOldTarget.Mul(targetAdjustment, ratOldTarget)
	intNewTarget := new(big.Int).Div(ratNewTarget.Num(), ratNewTarget.Denom())
	newTargetBytes := intNewTarget.Bytes()
	offset := len(newBlockNode.Target[:]) - len(newTargetBytes)
	copy(newBlockNode.Target[offset:], newTargetBytes)

	// Add the parent target to the depth of the block in the tree.
	blockWeight := new(big.Rat).SetFrac(big.NewInt(1), new(big.Int).SetBytes(parentBlockNode.Target[:]))
	newBlockNode.Depth = BlockWeight(new(big.Rat).Add(parentBlockNode.Depth, blockWeight))

	///////////////// Can be made into a function for following a fork.
	// If the new node is .5% heavier than the other node, switch to the new fork.
	currentWeight := new(big.Rat).SetFrac(big.NewInt(1), new(big.Int).SetBytes(s.BlockMap[s.ConsensusState.CurrentBlock].Target[:]))
	threshold := new(big.Rat).Mul(currentWeight, SurpassThreshold)
	requiredDepth := new(big.Rat).Add(s.BlockMap[s.ConsensusState.CurrentBlock].Depth, threshold)
	if (*big.Rat)(newBlockNode.Depth).Cmp(requiredDepth) == 1 {
		// Find the common parent between the new fork and the current
		// fork, keeping track of which path is taken through the
		// children of the parents so that we can re-trace as we
		// validate the blocks.
		currentNode := parentBlockNode
		value := s.ConsensusState.CurrentPath[currentNode.Height]
		var parentHistory []BlockID
		for value != currentNode.Block.ID() {
			parentHistory = append(parentHistory, currentNode.Block.ID())
			currentNode = s.BlockMap[currentNode.Block.ParentBlock]
			value = s.ConsensusState.CurrentPath[currentNode.Height]
		}

		// Remove blocks from the ConsensusState until we get to the
		// same parent that we are forking from.
		var rewoundBlocks []BlockID
		for s.ConsensusState.CurrentBlock != currentNode.Block.ID() {
			rewoundBlocks = append(rewoundBlocks, s.ConsensusState.CurrentBlock)
			s.RewindABlock()
		}

		// Validate each block in the parent history in order, updating
		// the state as we go.  If at some point a block doesn't
		// verify, you get to walk all the way backwards and forwards
		// again.
		validatedBlocks := 0
		for i := len(parentHistory) - 1; i >= 0; i-- {
			err = s.ValidateBlock(b)
			if err != nil {
				// Add the whole tree of blocks to BadBlocks,
				// deleting them from BlockMap

				// Rewind the validated blocks
				for i := 0; i < validatedBlocks; i++ {
					s.RewindABlock()
				}

				// Integrate the rewound blocks
				for i := len(rewoundBlocks) - 1; i >= 0; i-- {
					err = s.ValidateBlock(s.BlockMap[rewoundBlocks[i]].Block)
					if err != nil {
						panic(err)
					}
				}

				break
			}
			validatedBlocks += 1
		}

		// Do something to the transaction pool.
	} else {
		// Do something to the transaction pool.
	}

	return
}

// ValidateBlock will both verify the block AND update the consensus state.
// Calling integrate block is not needed.
func (s *State) ValidateBlock(b *Block) (err error) {
	// Check the hash on the merkle tree of transactions.

	var appliedTransactions []Transaction
	minerSubsidy := Currency(0)
	for _, txn := range b.Transactions {
		err = s.ValidateTxn(txn, s.BlockMap[b.ID()].Height)
		if err != nil {
			s.BadBlocks[b.ID()] = struct{}{}
			break
		}

		// Apply the transaction to the ConsensusState, adding it to the list of applied transactions.
		s.ApplyTransaction(txn)
		appliedTransactions = append(appliedTransactions, txn)

		minerSubsidy += txn.MinerFee
	}

	if err != nil {
		// Rewind transactions added to ConsensusState.
		for i := len(appliedTransactions) - 1; i >= 0; i-- {
			s.ReverseTransaction(appliedTransactions[i])
		}
		return
	}

	// Add outputs for all of the missed proofs in the open transactions.

	// Add coin inflation to the miner subsidy.

	// Add output contianing miner fees + block subsidy.
	bid := b.ID()
	minerSubsidyID := OutputID(HashBytes(append(bid[:], []byte("blockReward")...)))
	minerSubsidyOutput := Output{
		Value:     minerSubsidy,
		SpendHash: b.MinerAddress,
	}
	s.ConsensusState.UnspentOutputs[minerSubsidyID] = minerSubsidyOutput

	// s.BlockMap[b.ID()].Verified = true

	s.ConsensusState.CurrentBlock = b.ID()
	s.ConsensusState.CurrentPath[s.BlockMap[b.ID()].Height] = b.ID()

	return
}

// Add a function that integrates a block without verifying it.

/// Can probably split the validation of each piece into a different function,
//but perhaps not.
func (s *State) ValidateTxn(t Transaction, currentHeight BlockHeight) (err error) {
	inputSum := Currency(0)
	outputSum := t.MinerFee
	var inputSignaturesMap map[OutputID]InputSignatures
	for _, input := range t.Inputs {
		utxo, exists := s.ConsensusState.UnspentOutputs[input.OutputID]
		if !exists {
			err = errors.New("Transaction spends a nonexisting output")
			return
		}

		inputSum += utxo.Value

		// Check that the spend conditions match the hash listed in the output.

		// Check the timelock on the spend conditions is expired.
		if input.SpendConditions.TimeLock < currentHeight {
			err = errors.New("Output spent before timelock expiry.")
			return
		}

		// Create the condition for the input signatures and add it to the input signatures map.
		_, exists = inputSignaturesMap[input.OutputID]
		if exists {
			err = errors.New("Output spent twice in same transaction")
			return
		}
		var newInputSignatures InputSignatures
		newInputSignatures.RemainingSignatures = input.SpendConditions.NumSignatures
		newInputSignatures.PossibleKeys = input.SpendConditions.PublicKeys
		inputSignaturesMap[input.OutputID] = newInputSignatures
	}

	for _, output := range t.Outputs {
		outputSum += output.Value
	}

	for _, contract := range t.FileContracts {
		if contract.Start < currentHeight {
			err = errors.New("Contract starts in the future.")
			return
		}
		if contract.End <= contract.Start {
			err = errors.New("Contract duration must be at least one block.")
			return
		}
	}

	/*
		for _, proof := range t.StorageProofs {
			// Check that the proof passes.
			// Check that the proof has not already been submitted.
		}
	*/

	if inputSum != outputSum {
		err = errors.New("Inputs do not equal outputs for transaction.")
		return
	}

	for _, sig := range t.Signatures {
		// Check that each signature signs a unique pubkey where
		// RemainingSignatures > 0.
		if inputSignaturesMap[sig.InputID].RemainingSignatures == 0 {
			err = errors.New("Friviolous Signature detected.")
			return
		}
		_, exists := inputSignaturesMap[sig.InputID].UsedKeys[sig.PublicKeyIndex]
		if exists {
			err = errors.New("public key used twice while signing")
			return
		}

		// Check the timelock on the signature.
		if sig.TimeLock < currentHeight {
			err = errors.New("signature timelock has not expired")
			return
		}

		// Check that the actual signature is valid, following the covered fields struct.
	}

	return
}

func (s *State) ApplyTransaction(t Transaction) {
	// Remove all inputs from the unspent outputs list
	for _, input := range t.Inputs {
		s.ConsensusState.SpentOutputs[input.OutputID] = s.ConsensusState.UnspentOutputs[input.OutputID]
		delete(s.ConsensusState.UnspentOutputs, input.OutputID)
	}

	// Add all outputs to the unspent outputs list
	for i, output := range t.Outputs {
		newOutputID := OutputID(HashBytes(append((t.Inputs[0].OutputID)[:], EncUint64(uint64(i))...)))
		s.ConsensusState.UnspentOutputs[newOutputID] = output
	}

	// Add all outputs created by storage proofs.
	/*
		for _, sp := range t.StorageProofs {
			// Need to check that the contract fund has sufficient funds remaining.

			newOutputID := HashBytes(append(ContractID), []byte(n))
			output := Output {
				Value: s.ConsensusState.OpenContracts[sp.ContractID].ValidProofPayout,
				SpendHash: s.ConsensusState.OpenContracts[sp.ContractID].ValidProofAddress,
			}
			s.ConsensusState.UnspentOutputs[newOutputID] = output

			// need a counter or some way to determine what the index of
			// the window is.
		}
	*/
}

// Pulls just this transaction out of the ConsensusState.
func (s *State) ReverseTransaction(t Transaction) {
	// Remove all outputs created by storage proofs.

	// Remove all outputs created by outputs.
	for i := range t.Outputs {
		outputID := OutputID(HashBytes(append((t.Inputs[0].OutputID)[:], EncUint64(uint64(i))...)))
		delete(s.ConsensusState.UnspentOutputs, outputID)
	}

	// Add all outputs spent by inputs.
	for _, input := range t.Inputs {
		s.ConsensusState.UnspentOutputs[input.OutputID] = s.ConsensusState.SpentOutputs[input.OutputID]
		delete(s.ConsensusState.SpentOutputs, input.OutputID)
	}
}

// Pulls the most recent block out of the ConsensusState.
func (s *State) RewindABlock() {
	block := s.BlockMap[s.ConsensusState.CurrentBlock].Block
	for i := len(block.Transactions) - 1; i >= 0; i-- {
		s.ReverseTransaction(block.Transactions[i])
	}

	s.ConsensusState.CurrentBlock = block.ParentBlock
	delete(s.ConsensusState.CurrentPath, s.BlockMap[block.ID()].Height)
}
<commit_msg>move depth setting to its own function<commit_after>package sia

import (
	"bytes"
	"errors"
	"math/big"
	"sort"
	"time"
)

// Used to keep track of how many signatures an input has been signed by.
type InputSignatures struct {
	RemainingSignatures uint8
	PossibleKeys        []PublicKey
	UsedKeys            map[uint8]struct{}
}

// checkMaps looks through the maps known to the state and sees if the block id
// has been cached anywhere.
func (s *State) checkMaps(b *Block) (parentBlockNode *BlockNode, err error) {
	// See if the block is a known invalid block.
	_, exists := s.BadBlocks[b.ID()]
	if exists {
		err = errors.New("block is known to be invalid")
		return
	}

	// See if the block is a known valid block.
	_, exists = s.BlockMap[b.ID()]
	if exists {
		err = errors.New("Block exists in block map.")
		return
	}

	/*
		// See if the block is a known orphan.
		_, exists = s.OrphanBlocks[b.ID()]
		if exists {
			err = errors.New("Block exists in orphan list")
			return
		}
	*/

	// See if the block's parent is known.
	parentBlockNode, exists = s.BlockMap[b.ParentBlock]
	if !exists {
		// OrphanBlocks[b.ID()] = b
		err = errors.New("Block is an orphan")
		return
	}

	return
}

// Returns true if timestamp is valid, and if target value is reached.
func (s *State) validateHeader(parent *BlockNode, b *Block) (err error) {
	// Check that the block is not too far in the future.
	skew := b.Timestamp - Timestamp(time.Now().Unix())
	if skew > FutureThreshold {
		// Do something so that you will return to considering this
		// block once it's no longer too far in the future.
		err = errors.New("timestamp too far in future")
		return
	}

	// If timestamp is too far in the past, reject and put in bad blocks.
	var intTimestamps []int
	for _, timestamp := range parent.RecentTimestamps {
		intTimestamps = append(intTimestamps, int(timestamp))
	}
	sort.Ints(intTimestamps)
	if Timestamp(intTimestamps[5]) > b.Timestamp {
		s.BadBlocks[b.ID()] = struct{}{}
		err = errors.New("timestamp invalid for being in the past")
		return
	}

	// Check the id meets the target.
	blockHash := b.ID()
	if bytes.Compare(parent.Target[:], blockHash[:]) < 0 {
		err = errors.New("block does not meet target")
		return
	}

	return
}

func (s *State) childDepth(parentNode *BlockNode) BlockWeight {
	blockWeight := new(big.Rat).SetFrac(big.NewInt(1), new(big.Int).SetBytes(parentNode.Target[:]))
	return BlockWeight(new(big.Rat).Add(parentNode.Depth, blockWeight))
}

// Takes a block and a parent node, and adds the block as a child to the parent
// node.
func (s *State) addBlockToTree(parentNode *BlockNode, b *Block) (newNode *BlockNode) {
	// Create the child node.
	newNode = new(BlockNode)
	newNode.Block = b
	newNode.Height = parentNode.Height + 1

	// Copy over the timestamps.
	copy(newNode.RecentTimestamps[:], parentNode.RecentTimestamps[1:])
	newNode.RecentTimestamps[10] = b.Timestamp

	// Calculate target and depth.
	newNode.Depth = s.childDepth(parentNode)

	// Add the node to the block map and the list of its parents children.
	s.BlockMap[b.ID()] = newNode
	parentNode.Children = append(parentNode.Children, newNode)

	return
}

// Add a block to the state struct.
func (s *State) AcceptBlock(b *Block) (err error) {
	// Check the maps in the state to see if the block is already known.
	parentBlockNode, err := s.checkMaps(b)
	if err != nil {
		return
	}

	// Check that the header of the block is valid.
	err = s.validateHeader(parentBlockNode, b)
	if err != nil {
		return
	}

	newBlockNode := s.addBlockToTree(parentBlockNode, b)

	///////////// Can be made into a function for calculating adjusted difficulty.
	var timePassed Timestamp
	var expectedTimePassed Timestamp
	var blockWindow BlockHeight
	if newBlockNode.Height < 5000 {
		// Calculate new target, using block 0 timestamp.
		timePassed = b.Timestamp - s.BlockRoot.Block.Timestamp
		expectedTimePassed = TargetSecondsPerBlock * Timestamp(newBlockNode.Height)
		blockWindow = newBlockNode.Height
	} else {
		// Calculate new target, using block Height-5000 timestamp.
		timePassed = b.Timestamp - s.BlockMap[s.ConsensusState.CurrentPath[newBlockNode.Height-5000]].Block.Timestamp
		expectedTimePassed = TargetSecondsPerBlock * 5000
		blockWindow = 5000
	}

	// Adjustment as a float = timePassed / expectedTimePassed / blockWindow.
	targetAdjustment := big.NewRat(int64(timePassed), int64(expectedTimePassed)*int64(blockWindow))

	// Enforce a maximum targetAdjustment
	if targetAdjustment.Cmp(MaxAdjustmentUp) == 1 {
		targetAdjustment = MaxAdjustmentUp
	} else if targetAdjustment.Cmp(MaxAdjustmentDown) == -1 {
		targetAdjustment = MaxAdjustmentDown
	}

	// Take the target adjustment and apply it to the target slice,
	// using rational numbers. Truncate the result.
	oldTarget := new(big.Int).SetBytes(parentBlockNode.Target[:])
	ratOldTarget := new(big.Rat).SetInt(oldTarget)
	ratNewTarget := ratOldTarget.Mul(targetAdjustment, ratOldTarget)
	intNewTarget := new(big.Int).Div(ratNewTarget.Num(), ratNewTarget.Denom())
	newTargetBytes := intNewTarget.Bytes()
	offset := len(newBlockNode.Target[:]) - len(newTargetBytes)
	copy(newBlockNode.Target[offset:], newTargetBytes)

	///////////////// Can be made into a function for following a fork.
	// If the new node is .5% heavier than the other node, switch to the new fork.
	currentWeight := new(big.Rat).SetFrac(big.NewInt(1), new(big.Int).SetBytes(s.BlockMap[s.ConsensusState.CurrentBlock].Target[:]))
	threshold := new(big.Rat).Mul(currentWeight, SurpassThreshold)
	requiredDepth := new(big.Rat).Add(s.BlockMap[s.ConsensusState.CurrentBlock].Depth, threshold)
	if (*big.Rat)(newBlockNode.Depth).Cmp(requiredDepth) == 1 {
		// Find the common parent between the new fork and the current
		// fork, keeping track of which path is taken through the
		// children of the parents so that we can re-trace as we
		// validate the blocks.
		currentNode := parentBlockNode
		value := s.ConsensusState.CurrentPath[currentNode.Height]
		var parentHistory []BlockID
		for value != currentNode.Block.ID() {
			parentHistory = append(parentHistory, currentNode.Block.ID())
			currentNode = s.BlockMap[currentNode.Block.ParentBlock]
			value = s.ConsensusState.CurrentPath[currentNode.Height]
		}

		// Remove blocks from the ConsensusState until we get to the
		// same parent that we are forking from.
		var rewoundBlocks []BlockID
		for s.ConsensusState.CurrentBlock != currentNode.Block.ID() {
			rewoundBlocks = append(rewoundBlocks, s.ConsensusState.CurrentBlock)
			s.RewindABlock()
		}

		// Validate each block in the parent history in order, updating
		// the state as we go.  If at some point a block doesn't
		// verify, you get to walk all the way backwards and forwards
		// again.
		validatedBlocks := 0
		for i := len(parentHistory) - 1; i >= 0; i-- {
			err = s.ValidateBlock(b)
			if err != nil {
				// Add the whole tree of blocks to BadBlocks,
				// deleting them from BlockMap

				// Rewind the validated blocks
				for i := 0; i < validatedBlocks; i++ {
					s.RewindABlock()
				}

				// Integrate the rewound blocks
				for i := len(rewoundBlocks) - 1; i >= 0; i-- {
					err = s.ValidateBlock(s.BlockMap[rewoundBlocks[i]].Block)
					if err != nil {
						panic(err)
					}
				}

				break
			}
			validatedBlocks += 1
		}

		// Do something to the transaction pool.
	} else {
		// Do something to the transaction pool.
	}

	return
}

// ValidateBlock will both verify the block AND update the consensus state.
// Calling integrate block is not needed.
func (s *State) ValidateBlock(b *Block) (err error) {
	// Check the hash on the merkle tree of transactions.

	var appliedTransactions []Transaction
	minerSubsidy := Currency(0)
	for _, txn := range b.Transactions {
		err = s.ValidateTxn(txn, s.BlockMap[b.ID()].Height)
		if err != nil {
			s.BadBlocks[b.ID()] = struct{}{}
			break
		}

		// Apply the transaction to the ConsensusState, adding it to the list of applied transactions.
		s.ApplyTransaction(txn)
		appliedTransactions = append(appliedTransactions, txn)

		minerSubsidy += txn.MinerFee
	}

	if err != nil {
		// Rewind transactions added to ConsensusState.
		for i := len(appliedTransactions) - 1; i >= 0; i-- {
			s.ReverseTransaction(appliedTransactions[i])
		}
		return
	}

	// Add outputs for all of the missed proofs in the open transactions.

	// Add coin inflation to the miner subsidy.

	// Add output contianing miner fees + block subsidy.
	bid := b.ID()
	minerSubsidyID := OutputID(HashBytes(append(bid[:], []byte("blockReward")...)))
	minerSubsidyOutput := Output{
		Value:     minerSubsidy,
		SpendHash: b.MinerAddress,
	}
	s.ConsensusState.UnspentOutputs[minerSubsidyID] = minerSubsidyOutput

	// s.BlockMap[b.ID()].Verified = true

	s.ConsensusState.CurrentBlock = b.ID()
	s.ConsensusState.CurrentPath[s.BlockMap[b.ID()].Height] = b.ID()

	return
}

// Add a function that integrates a block without verifying it.

/// Can probably split the validation of each piece into a different function,
//but perhaps not.
func (s *State) ValidateTxn(t Transaction, currentHeight BlockHeight) (err error) {
	inputSum := Currency(0)
	outputSum := t.MinerFee
	var inputSignaturesMap map[OutputID]InputSignatures
	for _, input := range t.Inputs {
		utxo, exists := s.ConsensusState.UnspentOutputs[input.OutputID]
		if !exists {
			err = errors.New("Transaction spends a nonexisting output")
			return
		}

		inputSum += utxo.Value

		// Check that the spend conditions match the hash listed in the output.

		// Check the timelock on the spend conditions is expired.
		if input.SpendConditions.TimeLock < currentHeight {
			err = errors.New("Output spent before timelock expiry.")
			return
		}

		// Create the condition for the input signatures and add it to the input signatures map.
		_, exists = inputSignaturesMap[input.OutputID]
		if exists {
			err = errors.New("Output spent twice in same transaction")
			return
		}
		var newInputSignatures InputSignatures
		newInputSignatures.RemainingSignatures = input.SpendConditions.NumSignatures
		newInputSignatures.PossibleKeys = input.SpendConditions.PublicKeys
		inputSignaturesMap[input.OutputID] = newInputSignatures
	}

	for _, output := range t.Outputs {
		outputSum += output.Value
	}

	for _, contract := range t.FileContracts {
		if contract.Start < currentHeight {
			err = errors.New("Contract starts in the future.")
			return
		}
		if contract.End <= contract.Start {
			err = errors.New("Contract duration must be at least one block.")
			return
		}
	}

	/*
		for _, proof := range t.StorageProofs {
			// Check that the proof passes.
			// Check that the proof has not already been submitted.
		}
	*/

	if inputSum != outputSum {
		err = errors.New("Inputs do not equal outputs for transaction.")
		return
	}

	for _, sig := range t.Signatures {
		// Check that each signature signs a unique pubkey where
		// RemainingSignatures > 0.
		if inputSignaturesMap[sig.InputID].RemainingSignatures == 0 {
			err = errors.New("Friviolous Signature detected.")
			return
		}
		_, exists := inputSignaturesMap[sig.InputID].UsedKeys[sig.PublicKeyIndex]
		if exists {
			err = errors.New("public key used twice while signing")
			return
		}

		// Check the timelock on the signature.
		if sig.TimeLock < currentHeight {
			err = errors.New("signature timelock has not expired")
			return
		}

		// Check that the actual signature is valid, following the covered fields struct.
	}

	return
}

func (s *State) ApplyTransaction(t Transaction) {
	// Remove all inputs from the unspent outputs list
	for _, input := range t.Inputs {
		s.ConsensusState.SpentOutputs[input.OutputID] = s.ConsensusState.UnspentOutputs[input.OutputID]
		delete(s.ConsensusState.UnspentOutputs, input.OutputID)
	}

	// Add all outputs to the unspent outputs list
	for i, output := range t.Outputs {
		newOutputID := OutputID(HashBytes(append((t.Inputs[0].OutputID)[:], EncUint64(uint64(i))...)))
		s.ConsensusState.UnspentOutputs[newOutputID] = output
	}

	// Add all outputs created by storage proofs.
	/*
		for _, sp := range t.StorageProofs {
			// Need to check that the contract fund has sufficient funds remaining.

			newOutputID := HashBytes(append(ContractID), []byte(n))
			output := Output {
				Value: s.ConsensusState.OpenContracts[sp.ContractID].ValidProofPayout,
				SpendHash: s.ConsensusState.OpenContracts[sp.ContractID].ValidProofAddress,
			}
			s.ConsensusState.UnspentOutputs[newOutputID] = output

			// need a counter or some way to determine what the index of
			// the window is.
		}
	*/
}

// Pulls just this transaction out of the ConsensusState.
func (s *State) ReverseTransaction(t Transaction) {
	// Remove all outputs created by storage proofs.

	// Remove all outputs created by outputs.
	for i := range t.Outputs {
		outputID := OutputID(HashBytes(append((t.Inputs[0].OutputID)[:], EncUint64(uint64(i))...)))
		delete(s.ConsensusState.UnspentOutputs, outputID)
	}

	// Add all outputs spent by inputs.
	for _, input := range t.Inputs {
		s.ConsensusState.UnspentOutputs[input.OutputID] = s.ConsensusState.SpentOutputs[input.OutputID]
		delete(s.ConsensusState.SpentOutputs, input.OutputID)
	}
}

// Pulls the most recent block out of the ConsensusState.
func (s *State) RewindABlock() {
	block := s.BlockMap[s.ConsensusState.CurrentBlock].Block
	for i := len(block.Transactions) - 1; i >= 0; i-- {
		s.ReverseTransaction(block.Transactions[i])
	}

	s.ConsensusState.CurrentBlock = block.ParentBlock
	delete(s.ConsensusState.CurrentPath, s.BlockMap[block.ID()].Height)
}
<|endoftext|>


<commit_before>package turnpike

import (
	"fmt"
	"sync"
	"time"
)

const (
	defaultAuthTimeout = 2 * time.Minute
)

// A Realm is a WAMP routing and administrative domain.
//
// Clients that have connected to a WAMP router are joined to a realm and all
// message delivery is handled by the realm.
type Realm struct {
	Broker
	Dealer
	Authorizer
	Interceptor

	URI              URI
	CRAuthenticators map[string]CRAuthenticator
	Authenticators   map[string]Authenticator
	AuthTimeout      time.Duration

	sessions     map[ID]*Session
	sessionsLock sync.RWMutex

	closed     bool
	closedLock sync.RWMutex

	localClient
	acts chan func()
}

type localClient struct {
	*Client
}

func (r *Realm) init() {
	r.sessions = make(map[ID]*Session)
	r.acts = make(chan func())
	p, _ := r.getPeer(nil)

	r.localClient.Client = NewClient(p)

	if r.Broker == nil {
		r.Broker = NewDefaultBroker()
	}
	if r.Dealer == nil {
		r.Dealer = NewDefaultDealer()
	}
	if r.Authorizer == nil {
		r.Authorizer = NewDefaultAuthorizer()
	}
	if r.Interceptor == nil {
		r.Interceptor = NewDefaultInterceptor()
	}
	if r.AuthTimeout == 0 {
		r.AuthTimeout = defaultAuthTimeout
	}

	go r.localClient.Receive()
	go r.run()
}

func (r *Realm) Closed() bool {
	r.closedLock.RLock()
	defer r.closedLock.RUnlock()
	return r.closed
}

// Close disconnects all clients after sending a goodbye message
func (r *Realm) Close() {
	r.acts <- func() {
		for _, client := range r.sessions {
			client.kill <- ErrSystemShutdown
		}
	}

	var (
		s        = make(chan struct{})
		nclients int
	)
	for {
		r.acts <- func() {
			nclients = len(r.sessions)
			s <- struct{}{}
		}
		<-s
		if nclients == 0 {
			break
		}
	}

	close(r.acts)
}

func (r *Realm) run() {
	for {
		if act, ok := <-r.acts; ok {
			act()
		} else {
			return
		}
	}
}

func (l *localClient) onJoin(details map[string]interface{}) {
	l.Publish("wamp.session.on_join", nil, []interface{}{details}, nil)
}

func (l *localClient) onLeave(session ID) {
	l.Publish("wamp.session.on_leave", nil, []interface{}{session}, nil)
}

func (r *Realm) handleSession(sess *Session) {
	s := make(chan struct{})
	r.acts <- func() {
		r.sessions[sess.Id] = sess
		r.onJoin(sess.Details)
		s <- struct{}{}
	}
	<-s
	defer func() {
		r.acts <- func() {
			delete(r.sessions, sess.Id)
			r.Dealer.RemoveSession(sess)
			r.Broker.RemoveSession(sess)
			r.onLeave(sess.Id)
		}
	}()
	c := sess.Receive()
	// TODO: what happens if the realm is closed?

	for {
		var msg Message
		var open bool
		select {
		case msg, open = <-c:
			if !open {
				log.Println("lost session:", sess)
				return
			}
		case reason := <-sess.kill:
			logErr(sess.Send(&Goodbye{Reason: reason, Details: make(map[string]interface{})}))
			log.Printf("kill session %s: %v", sess, reason)
			// TODO: wait for client Goodbye?
			return
		}

		log.Printf("[%s] %s: %+v", sess, msg.MessageType(), msg)
		if isAuthz, err := r.Authorizer.Authorize(sess, msg); !isAuthz {
			errMsg := &Error{Type: msg.MessageType()}
			switch msg := msg.(type) {
			case *Publish:
				errMsg.Request = msg.Request
			case *Subscribe:
				errMsg.Request = msg.Request
			case *Unsubscribe:
				errMsg.Request = msg.Request
			case *Register:
				errMsg.Request = msg.Request
			case *Unregister:
				errMsg.Request = msg.Request
			case *Call:
				errMsg.Request = msg.Request
			case *Yield:
				errMsg.Request = msg.Request
			}
			if err != nil {
				errMsg.Error = ErrAuthorizationFailed
				log.Printf("[%s] authorization failed: %v", sess, err)
			} else {
				errMsg.Error = ErrNotAuthorized
				log.Printf("[%s] %s UNAUTHORIZED", sess, msg.MessageType())
			}
			logErr(sess.Send(errMsg))
			continue
		}

		r.Interceptor.Intercept(sess, &msg)

		switch msg := msg.(type) {
		case *Goodbye:
			logErr(sess.Send(&Goodbye{Reason: ErrGoodbyeAndOut, Details: make(map[string]interface{})}))
			log.Printf("[%s] leaving: %v", sess, msg.Reason)
			return

		// Broker messages
		case *Publish:
			r.Broker.Publish(sess, msg)
		case *Subscribe:
			r.Broker.Subscribe(sess, msg)
		case *Unsubscribe:
			r.Broker.Unsubscribe(sess, msg)

		// Dealer messages
		case *Register:
			r.Dealer.Register(sess, msg)
		case *Unregister:
			r.Dealer.Unregister(sess, msg)
		case *Call:
			r.Dealer.Call(sess, msg)
		case *Yield:
			r.Dealer.Yield(sess, msg)

		// Error messages
		case *Error:
			if msg.Type == MessageTypeInvocation {
				// the only type of ERROR message the router should receive
				r.Dealer.Error(sess, msg)
			} else {
				log.Printf("invalid ERROR message received: %v", msg)
			}

		default:
			log.Println("Unhandled message:", msg.MessageType())
		}
	}
}

func (r *Realm) handleAuth(client Peer, details map[string]interface{}) (*Welcome, error) {
	msg, err := r.authenticate(details)
	if err != nil {
		return nil, err
	}
	// we should never get anything besides WELCOME and CHALLENGE
	if msg.MessageType() == MessageTypeWelcome {
		return msg.(*Welcome), nil
	}
	// Challenge response
	challenge := msg.(*Challenge)
	if err := client.Send(challenge); err != nil {
		return nil, err
	}

	msg, err = GetMessageTimeout(client, r.AuthTimeout)
	if err != nil {
		return nil, err
	}
	log.Printf("%s: %+v", msg.MessageType(), msg)
	if authenticate, ok := msg.(*Authenticate); !ok {
		return nil, fmt.Errorf("unexpected %s message received", msg.MessageType())
	} else {
		return r.checkResponse(challenge, authenticate)
	}
}

// Authenticate either authenticates a client or returns a challenge message if
// challenge/response authentication is to be used.
func (r *Realm) authenticate(details map[string]interface{}) (Message, error) {
	log.Println("details:", details)
	if len(r.Authenticators) == 0 && len(r.CRAuthenticators) == 0 {
		return &Welcome{}, nil
	}
	// TODO: this might not always be a []interface{}. Using the JSON unmarshaller it will be,
	// but we may have serializations that preserve more of the original type.
	// For now, the tests just explicitly send a []interface{}
	_authmethods, ok := details["authmethods"].([]interface{})
	if !ok {
		return nil, fmt.Errorf("No authentication supplied")
	}
	authmethods := []string{}
	for _, method := range _authmethods {
		if m, ok := method.(string); ok {
			authmethods = append(authmethods, m)
		} else {
			log.Printf("invalid authmethod value: %v", method)
		}
	}
	for _, method := range authmethods {
		if auth, ok := r.CRAuthenticators[method]; ok {
			if challenge, err := auth.Challenge(details); err != nil {
				return nil, err
			} else {
				return &Challenge{AuthMethod: method, Extra: challenge}, nil
			}
		}
		if auth, ok := r.Authenticators[method]; ok {
			if authDetails, err := auth.Authenticate(details); err != nil {
				return nil, err
			} else {
				return &Welcome{Details: addAuthMethod(authDetails, method)}, nil
			}
		}
	}
	// TODO: check default auth (special '*' auth?)
	return nil, fmt.Errorf("could not authenticate with any method")
}

// checkResponse determines whether the response to the challenge is sufficient to gain access to the Realm.
func (r *Realm) checkResponse(chal *Challenge, auth *Authenticate) (*Welcome, error) {
	authenticator, ok := r.CRAuthenticators[chal.AuthMethod]
	if !ok {
		return nil, fmt.Errorf("authentication method has been removed")
	}
	if details, err := authenticator.Authenticate(chal.Extra, auth.Signature); err != nil {
		return nil, err
	} else {
		return &Welcome{Details: addAuthMethod(details, chal.AuthMethod)}, nil
	}
}

func (r *Realm) getPeer(details map[string]interface{}) (Peer, error) {
	peerA, peerB := localPipe()
	if details == nil {
		details = make(map[string]interface{})
	}
	sess := Session{Peer: peerA, Id: NewID(), Details: details, kill: make(chan URI, 1)}
	go r.handleSession(&sess)
	log.Println("Established internal session:", sess)
	return peerB, nil
}

func addAuthMethod(details map[string]interface{}, method string) map[string]interface{} {
	if details == nil {
		details = make(map[string]interface{})
	}
	details["authmethod"] = method
	return details
}
<commit_msg>Removing "acts" channel in Realm.  Could still use further cleanup.<commit_after>package turnpike

import (
	"context"
	"fmt"
	"sync"
	"time"
)

const (
	defaultAuthTimeout = 2 * time.Minute
)

// A Realm is a WAMP routing and administrative domain.
//
// Clients that have connected to a WAMP router are joined to a realm and all
// message delivery is handled by the realm.
type Realm struct {
	Broker
	Dealer
	Authorizer
	Interceptor

	ctx context.Context

	URI              URI
	CRAuthenticators map[string]CRAuthenticator
	Authenticators   map[string]Authenticator
	AuthTimeout      time.Duration

	sessions     map[ID]*Session
	sessionsLock sync.RWMutex

	closed     bool
	closedLock sync.RWMutex

	localClient
}

type localClient struct {
	*Client
}

func (r *Realm) init() {
	r.sessions = make(map[ID]*Session)

	r.ctx = context.Background()

	p, _ := r.getPeer(nil)

	r.localClient.Client = NewClient(p)

	if r.Broker == nil {
		r.Broker = NewDefaultBroker()
	}
	if r.Dealer == nil {
		r.Dealer = NewDefaultDealer()
	}
	if r.Authorizer == nil {
		r.Authorizer = NewDefaultAuthorizer()
	}
	if r.Interceptor == nil {
		r.Interceptor = NewDefaultInterceptor()
	}
	if r.AuthTimeout == 0 {
		r.AuthTimeout = defaultAuthTimeout
	}

	go r.localClient.Receive()
}

func (r *Realm) Closed() bool {
	r.closedLock.RLock()
	defer r.closedLock.RUnlock()
	return r.closed
}

// Close disconnects all clients after sending a goodbye message
func (r *Realm) Close() {
	r.closedLock.Lock()

	if r.closed {
		log.Printf("Realm \"%s\" is already closing", string(r.URI))
		r.closedLock.Unlock()
		return
	}

	r.closedLock.Unlock()

	sLen := len(r.sessions)

	// if there are no active sessions, move on.
	if 0 == sLen {
		return
	}

	wg := &sync.WaitGroup{}
	wg.Add(sLen)

	for _, session := range r.sessions {
		go func(s *Session) {

			// will contain the message from session close
			closeChan := make(chan error)

			// we'll give them 2 seconds to ack the disconnect...
			ctx, cancel := context.WithTimeout(r.ctx, 2*time.Second)

			// attempt to close client connection gracefully...
			go func(s *Session, c chan error) { c <- s.Close() }(s, closeChan)

			// wait around for something to happen...
			select {
			case <-ctx.Done():
				logErr(fmt.Errorf("Unable to close session \"%d\": %s", s.Id, ctx.Err()))
			case err := <-closeChan:
				logErr(err)
			}

			// decrement wait group
			wg.Done()

			// do you even cancel, bro?
			cancel()
		}(session)
	}

	wg.Wait()

	log.Printf("Realm \"%s\" is now closed.", string(r.URI))
}

func (l *localClient) onJoin(details map[string]interface{}) {
	l.Publish("wamp.session.on_join", nil, []interface{}{details}, nil)
}

func (l *localClient) onLeave(session ID) {
	l.Publish("wamp.session.on_leave", nil, []interface{}{session}, nil)
}

func (r *Realm) handleSession(sess *Session) {
	r.closedLock.RLock()
	if r.closed {
		log.Printf("Will not handle session \"%d\" as realm \"%s\" is already closed", sess.Id, string(r.URI))
		r.closedLock.RUnlock()
		return
	}

	r.closedLock.RUnlock()

	r.sessionsLock.Lock()
	r.sessions[sess.Id] = sess
	r.sessionsLock.Unlock()

	r.onJoin(sess.Details)

	defer func(sid ID) {
		r.sessionsLock.Lock()
		delete(r.sessions, sid)
		r.sessionsLock.Unlock()

		r.Dealer.RemoveSession(sess)
		r.Broker.RemoveSession(sess)
		r.onLeave(sid)
	}(sess.Id)

	for {
		var msg Message
		var open bool
		select {
		case msg, open = <-sess.Receive():
			if !open {
				log.Println("lost session:", sess)
				return
			}
		case reason := <-sess.kill:
			logErr(sess.Send(&Goodbye{Reason: reason, Details: make(map[string]interface{})}))
			log.Printf("kill session %s: %v", sess, reason)
			// TODO: wait for client Goodbye?
			return
		}

		log.Printf("[%s] %s: %+v", sess, msg.MessageType(), msg)
		if isAuthz, err := r.Authorizer.Authorize(sess, msg); !isAuthz {
			errMsg := &Error{Type: msg.MessageType()}
			switch msg := msg.(type) {
			case *Publish:
				errMsg.Request = msg.Request
			case *Subscribe:
				errMsg.Request = msg.Request
			case *Unsubscribe:
				errMsg.Request = msg.Request
			case *Register:
				errMsg.Request = msg.Request
			case *Unregister:
				errMsg.Request = msg.Request
			case *Call:
				errMsg.Request = msg.Request
			case *Yield:
				errMsg.Request = msg.Request
			}
			if err != nil {
				errMsg.Error = ErrAuthorizationFailed
				log.Printf("[%s] authorization failed: %v", sess, err)
			} else {
				errMsg.Error = ErrNotAuthorized
				log.Printf("[%s] %s UNAUTHORIZED", sess, msg.MessageType())
			}
			logErr(sess.Send(errMsg))
			continue
		}

		r.Interceptor.Intercept(sess, &msg)

		switch msg := msg.(type) {
		case *Goodbye:
			logErr(sess.Send(&Goodbye{Reason: ErrGoodbyeAndOut, Details: make(map[string]interface{})}))
			log.Printf("[%s] leaving: %v", sess, msg.Reason)
			return

		// Broker messages
		case *Publish:
			r.Broker.Publish(sess, msg)
		case *Subscribe:
			r.Broker.Subscribe(sess, msg)
		case *Unsubscribe:
			r.Broker.Unsubscribe(sess, msg)

		// Dealer messages
		case *Register:
			r.Dealer.Register(sess, msg)
		case *Unregister:
			r.Dealer.Unregister(sess, msg)
		case *Call:
			r.Dealer.Call(sess, msg)
		case *Yield:
			r.Dealer.Yield(sess, msg)

		// Error messages
		case *Error:
			if msg.Type == MessageTypeInvocation {
				// the only type of ERROR message the router should receive
				r.Dealer.Error(sess, msg)
			} else {
				log.Printf("invalid ERROR message received: %v", msg)
			}

		default:
			log.Println("Unhandled message:", msg.MessageType())
		}
	}
}

func (r *Realm) handleAuth(client Peer, details map[string]interface{}) (*Welcome, error) {
	msg, err := r.authenticate(details)
	if err != nil {
		return nil, err
	}
	// we should never get anything besides WELCOME and CHALLENGE
	if msg.MessageType() == MessageTypeWelcome {
		return msg.(*Welcome), nil
	}
	// Challenge response
	challenge := msg.(*Challenge)
	if err := client.Send(challenge); err != nil {
		return nil, err
	}

	msg, err = GetMessageTimeout(client, r.AuthTimeout)
	if err != nil {
		return nil, err
	}
	log.Printf("%s: %+v", msg.MessageType(), msg)
	if authenticate, ok := msg.(*Authenticate); !ok {
		return nil, fmt.Errorf("unexpected %s message received", msg.MessageType())
	} else {
		return r.checkResponse(challenge, authenticate)
	}
}

// Authenticate either authenticates a client or returns a challenge message if
// challenge/response authentication is to be used.
func (r *Realm) authenticate(details map[string]interface{}) (Message, error) {
	log.Println("details:", details)
	if len(r.Authenticators) == 0 && len(r.CRAuthenticators) == 0 {
		return &Welcome{}, nil
	}
	// TODO: this might not always be a []interface{}. Using the JSON unmarshaller it will be,
	// but we may have serializations that preserve more of the original type.
	// For now, the tests just explicitly send a []interface{}
	_authmethods, ok := details["authmethods"].([]interface{})
	if !ok {
		return nil, fmt.Errorf("No authentication supplied")
	}
	authmethods := []string{}
	for _, method := range _authmethods {
		if m, ok := method.(string); ok {
			authmethods = append(authmethods, m)
		} else {
			log.Printf("invalid authmethod value: %v", method)
		}
	}
	for _, method := range authmethods {
		if auth, ok := r.CRAuthenticators[method]; ok {
			if challenge, err := auth.Challenge(details); err != nil {
				return nil, err
			} else {
				return &Challenge{AuthMethod: method, Extra: challenge}, nil
			}
		}
		if auth, ok := r.Authenticators[method]; ok {
			if authDetails, err := auth.Authenticate(details); err != nil {
				return nil, err
			} else {
				return &Welcome{Details: addAuthMethod(authDetails, method)}, nil
			}
		}
	}
	// TODO: check default auth (special '*' auth?)
	return nil, fmt.Errorf("could not authenticate with any method")
}

// checkResponse determines whether the response to the challenge is sufficient to gain access to the Realm.
func (r *Realm) checkResponse(chal *Challenge, auth *Authenticate) (*Welcome, error) {
	authenticator, ok := r.CRAuthenticators[chal.AuthMethod]
	if !ok {
		return nil, fmt.Errorf("authentication method has been removed")
	}
	if details, err := authenticator.Authenticate(chal.Extra, auth.Signature); err != nil {
		return nil, err
	} else {
		return &Welcome{Details: addAuthMethod(details, chal.AuthMethod)}, nil
	}
}

func (r *Realm) getPeer(details map[string]interface{}) (Peer, error) {
	peerA, peerB := localPipe()
	if details == nil {
		details = make(map[string]interface{})
	}
	sess := Session{Peer: peerA, Id: NewID(), Details: details, kill: make(chan URI, 1)}
	go r.handleSession(&sess)
	log.Println("Established internal session:", sess)
	return peerB, nil
}

func addAuthMethod(details map[string]interface{}, method string) map[string]interface{} {
	if details == nil {
		details = make(map[string]interface{})
	}
	details["authmethod"] = method
	return details
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"errors"
	"fmt"
	"github.com/howeyc/fsnotify"
	"go/build"
	"log"
	"os"
	"os/exec"
	"path"
	"path/filepath"
)

func install(buildpath, lastError string) (installed bool, errorOutput string, err error) {
	cmdline := []string{"go", "install", "-v", buildpath}

	cmd := exec.Command("go", cmdline[1:]...)
	bufOut := bytes.NewBuffer([]byte{})
	bufErr := bytes.NewBuffer([]byte{})
	cmd.Stdout = bufOut
	cmd.Stderr = bufErr

	err = cmd.Run()

	if bufOut.Len() != 0 {
		errorOutput = bufOut.String()
		if errorOutput != lastError {
			fmt.Print(bufOut)
		}
		err = errors.New("compile error")
		return
	}

	installed = bufErr.Len() != 0

	return
}

func run(binName, binPath string, args []string) (runch chan bool) {
	runch = make(chan bool)
	go func() {
		cmdline := append([]string{binName}, args...)
		var proc *os.Process
		for _ = range runch {
			if proc != nil {
				proc.Kill()
			}
			cmd := exec.Command(binPath, args...)
			cmd.Stdout = os.Stdout
			cmd.Stderr = os.Stderr
			log.Print(cmdline)
			cmd.Start()
			proc = cmd.Process
		}
	}()
	return
}

func getWatcher(buildpath string) (watcher *fsnotify.Watcher, err error) {
	watcher, err = fsnotify.NewWatcher()
	addToWatcher(watcher, buildpath, map[string]bool{})
	return
}

func addToWatcher(watcher *fsnotify.Watcher, importpath string, watching map[string]bool) {
	pkg, err := build.Import(importpath, "", 0)
	if err != nil {
		return
	}
	if pkg.Goroot {
		return
	}
	watcher.Watch(pkg.Dir)
	watching[importpath] = true
	for _, imp := range pkg.Imports {
		if !watching[imp] {
			addToWatcher(watcher, imp, watching)
		}
	}
}

func rerun(buildpath string, args []string) (err error) {
	log.Printf("setting up %s %v", buildpath, args)

	pkg, err := build.Import(buildpath, "", 0)
	if err != nil {
		return
	}

	if pkg.Name != "main" {
		err = errors.New(fmt.Sprintf("expected package %q, got %q", "main", pkg.Name))
		return
	}

	_, binName := path.Split(buildpath)
	binPath := filepath.Join(pkg.BinDir, binName)

	runch := run(binName, binPath, args)

	var errorOutput string
	_, errorOutput, ierr := install(buildpath, errorOutput)
	if ierr == nil {
		runch <- true
	}

	var watcher *fsnotify.Watcher
	watcher, err = getWatcher(buildpath)
	if err != nil {
		return
	}

	for {
		we := <-watcher.Event
		var installed bool
		installed, errorOutput, _ = install(buildpath, errorOutput)
		if installed {
			log.Print(we.Name)
			runch <- true
			watcher.Close()
			/* empty the buffer */
			go func(events chan *fsnotify.FileEvent, errors chan error) {
				for _ = range events {

				}
				for _ = range errors {

				}
			}(watcher.Event, watcher.Error)
			/* rescan */
			log.Println("rescanning")
			watcher, err = getWatcher(buildpath)
			if err != nil {
				return
			}
		}
	}
	return
}

func main() {
	if len(os.Args) < 2 {
		log.Fatal("Usage: rerun <import path> [arg]*")
	}
	err := rerun(os.Args[1], os.Args[2:])
	if err != nil {
		log.Print(err)
	}
}
<commit_msg>read from events and errors independently<commit_after>package main

import (
	"bytes"
	"errors"
	"fmt"
	"github.com/howeyc/fsnotify"
	"go/build"
	"log"
	"os"
	"os/exec"
	"path"
	"path/filepath"
)

func install(buildpath, lastError string) (installed bool, errorOutput string, err error) {
	cmdline := []string{"go", "install", "-v", buildpath}

	cmd := exec.Command("go", cmdline[1:]...)
	bufOut := bytes.NewBuffer([]byte{})
	bufErr := bytes.NewBuffer([]byte{})
	cmd.Stdout = bufOut
	cmd.Stderr = bufErr

	err = cmd.Run()

	if bufOut.Len() != 0 {
		errorOutput = bufOut.String()
		if errorOutput != lastError {
			fmt.Print(bufOut)
		}
		err = errors.New("compile error")
		return
	}

	installed = bufErr.Len() != 0

	return
}

func run(binName, binPath string, args []string) (runch chan bool) {
	runch = make(chan bool)
	go func() {
		cmdline := append([]string{binName}, args...)
		var proc *os.Process
		for _ = range runch {
			if proc != nil {
				proc.Kill()
			}
			cmd := exec.Command(binPath, args...)
			cmd.Stdout = os.Stdout
			cmd.Stderr = os.Stderr
			log.Print(cmdline)
			cmd.Start()
			proc = cmd.Process
		}
	}()
	return
}

func getWatcher(buildpath string) (watcher *fsnotify.Watcher, err error) {
	watcher, err = fsnotify.NewWatcher()
	addToWatcher(watcher, buildpath, map[string]bool{})
	return
}

func addToWatcher(watcher *fsnotify.Watcher, importpath string, watching map[string]bool) {
	pkg, err := build.Import(importpath, "", 0)
	if err != nil {
		return
	}
	if pkg.Goroot {
		return
	}
	watcher.Watch(pkg.Dir)
	watching[importpath] = true
	for _, imp := range pkg.Imports {
		if !watching[imp] {
			addToWatcher(watcher, imp, watching)
		}
	}
}

func rerun(buildpath string, args []string) (err error) {
	log.Printf("setting up %s %v", buildpath, args)

	pkg, err := build.Import(buildpath, "", 0)
	if err != nil {
		return
	}

	if pkg.Name != "main" {
		err = errors.New(fmt.Sprintf("expected package %q, got %q", "main", pkg.Name))
		return
	}

	_, binName := path.Split(buildpath)
	binPath := filepath.Join(pkg.BinDir, binName)

	runch := run(binName, binPath, args)

	var errorOutput string
	_, errorOutput, ierr := install(buildpath, errorOutput)
	if ierr == nil {
		runch <- true
	}

	var watcher *fsnotify.Watcher
	watcher, err = getWatcher(buildpath)
	if err != nil {
		return
	}

	for {
		we, _ := <-watcher.Event
		var installed bool
		installed, errorOutput, _ = install(buildpath, errorOutput)
		if installed {
			log.Print(we.Name)
			runch <- true
			watcher.Close()
			/* empty the buffer */
			go func(events chan *fsnotify.FileEvent) {
				for _ = range events {

				}
			}(watcher.Event)
			go func(errors chan error) {
				for _ = range errors {

				}
			}(watcher.Error)
			/* rescan */
			log.Println("rescanning")
			watcher, err = getWatcher(buildpath)
			if err != nil {
				return
			}
		}
	}
	return
}

func main() {
	if len(os.Args) < 2 {
		log.Fatal("Usage: rerun <import path> [arg]*")
	}
	err := rerun(os.Args[1], os.Args[2:])
	if err != nil {
		log.Print(err)
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2015-2016 Jeevanandam M (jeeva@myjeeva.com), All rights reserved.
// resty source code and usage is governed by a MIT style
// license that can be found in the LICENSE file.

// Package resty provides simple HTTP and REST client for Go inspired by Ruby rest-client.
package resty

// Version # of go-resty
var Version = "0.10"
<commit_msg>version bump to v0.11<commit_after>// Copyright (c) 2015-2016 Jeevanandam M (jeeva@myjeeva.com), All rights reserved.
// resty source code and usage is governed by a MIT style
// license that can be found in the LICENSE file.

// Package resty provides simple HTTP and REST client for Go inspired by Ruby rest-client.
package resty

// Version # of resty
const Version = "0.11"
<|endoftext|>


<commit_before>package backoff

import "time"

// Retry the function f until it does not return error or BackOff stops.
//
// Example:
// 	operation := func() error {
// 		// An operation that may fail
// 	}
//
// 	err := backoff.Retry(operation, backoff.NewExponentialBackoff())
// 	if err != nil {
// 		// handle error
// 	}
//
// 	// operation is successfull
func Retry(f func() error, b BackOff) error {
	var err error
	var next time.Duration

	b.Reset()
	for {
		if err = f(); err == nil {
			return nil
		}

		if next = b.NextBackOff(); next == Stop {
			return err
		}

		time.Sleep(next)
	}
}
<commit_msg>update retry function comment<commit_after>package backoff

import "time"

// Retry the function f until it does not return error or BackOff stops.
// f is guaranteed to be run at least once.
//
// Example:
// 	operation := func() error {
// 		// An operation that may fail
// 	}
//
// 	err := backoff.Retry(operation, backoff.NewExponentialBackoff())
// 	if err != nil {
// 		// handle error
// 	}
//
// 	// operation is successfull
func Retry(f func() error, b BackOff) error {
	var err error
	var next time.Duration

	b.Reset()
	for {
		if err = f(); err == nil {
			return nil
		}

		if next = b.NextBackOff(); next == Stop {
			return err
		}

		time.Sleep(next)
	}
}
<|endoftext|>