<commit_before>package buffer

import (
	"crypto/rand"
	"fmt"
	"io"
	"reflect"
	"testing"
	"unsafe"
)

func toByteSlice(p unsafe.Pointer, n int) []byte {
	sh := reflect.SliceHeader{
		Data: uintptr(p),
		Len:  n,
		Cap:  n,
	}

	return *(*[]byte)(unsafe.Pointer(&sh))
}

// fillWithGarbage writes random data to [p, p+n).
func fillWithGarbage(p unsafe.Pointer, n int) (err error) {
	b := toByteSlice(p, n)
	_, err = io.ReadFull(rand.Reader, b)
	return
}

func randBytes(n int) (b []byte, err error) {
	b = make([]byte, n)
	_, err = io.ReadFull(rand.Reader, b)
	return
}

// findNonZero finds the offset of the first non-zero byte in [p, p+n). If
// none, it returns n.
func findNonZero(p unsafe.Pointer, n int) int {
	b := toByteSlice(p, n)
	for i, x := range b {
		if x != 0 {
			return i
		}
	}

	return n
}

func TestMemclr(t *testing.T) {
	// All sizes up to 32 bytes.
	var sizes []int
	for i := 0; i <= 32; i++ {
		sizes = append(sizes, i)
	}

	// And a few hand-chosen sizes.
	sizes = append(sizes, []int{
		39, 41, 64, 127, 128, 129,
		1<<20 - 1,
		1 << 20,
		1<<20 + 1,
	}...)

	// For each size, fill a buffer with random bytes and then zero it.
	for _, size := range sizes {
		size := size
		t.Run(fmt.Sprintf("size=%d", size), func(t *testing.T) {
			// Generate
			b, err := randBytes(size)
			if err != nil {
				t.Fatalf("randBytes: %v", err)
			}

			// Clear
			var p unsafe.Pointer
			if len(b) != 0 {
				p = unsafe.Pointer(&b[0])
			}

			memclr(p, uintptr(len(b)))

			// Check
			if i := findNonZero(p, len(b)); i != len(b) {
				t.Fatalf("non-zero byte at offset %d", i)
			}
		})
	}
}

func TestOutMessageAppend(t *testing.T) {
	t.Fatal("TODO")
}

func TestOutMessageAppendString(t *testing.T) {
	t.Fatal("TODO")
}

func TestOutMessageShrinkTo(t *testing.T) {
	t.Fatal("TODO")
}

func TestOutMessageHeader(t *testing.T) {
	t.Fatal("TODO")
}

func TestOutMessageReset(t *testing.T) {
	var om OutMessage
	h := om.OutHeader()

	const trials = 10
	for i := 0; i < trials; i++ {
		// Fill the header with garbage.
		err := fillWithGarbage(unsafe.Pointer(h), int(unsafe.Sizeof(*h)))
		if err != nil {
			t.Fatalf("fillWithGarbage: %v", err)
		}

		// Ensure a non-zero payload length.
		if p := om.GrowNoZero(128); p == nil {
			t.Fatal("GrowNoZero failed")
		}

		// Reset.
		om.Reset()

		// Check that the length was updated.
		if got, want := int(om.Len()), int(OutMessageInitialSize); got != want {
			t.Fatalf("om.Len() = %d, want %d", got, want)
		}

		// Check that the header was zeroed.
		if h.Len != 0 {
			t.Fatalf("non-zero Len %v", h.Len)
		}

		if h.Error != 0 {
			t.Fatalf("non-zero Error %v", h.Error)
		}

		if h.Unique != 0 {
			t.Fatalf("non-zero Unique %v", h.Unique)
		}
	}
}

func TestOutMessageGrow(t *testing.T) {
	var om OutMessage

	// Overwrite with garbage.
	err := fillWithGarbage(unsafe.Pointer(&om), int(unsafe.Sizeof(om)))
	if err != nil {
		t.Fatalf("fillWithGarbage: %v", err)
	}

	// Zero the header.
	om.Reset()

	// Grow to the max size. This should zero the message.
	if p := om.Grow(MaxReadSize); p == nil {
		t.Fatal("Grow returned nil")
	}

	// Check that everything has been zeroed.
	b := om.Bytes()
	for i, x := range b {
		if x != 0 {
			t.Fatalf("non-zero byte 0x%02x at offset %d", x, i)
		}
	}
}

func BenchmarkOutMessageReset(b *testing.B) {
	// A single buffer, which should fit in some level of CPU cache.
	b.Run("Single buffer", func(b *testing.B) {
		var om OutMessage
		for i := 0; i < b.N; i++ {
			om.Reset()
		}

		b.SetBytes(int64(om.offset))
	})

	// Many megabytes worth of buffers, which should defeat the CPU cache.
	b.Run("Many buffers", func(b *testing.B) {
		// The number of messages; intentionally a power of two.
		const numMessages = 128

		var oms [numMessages]OutMessage
		if s := unsafe.Sizeof(oms); s < 128<<20 {
			panic(fmt.Sprintf("Array is too small; total size: %d", s))
		}

		for i := 0; i < b.N; i++ {
			oms[i%numMessages].Reset()
		}

		b.SetBytes(int64(oms[0].offset))
	})
}

func BenchmarkOutMessageGrowShrink(b *testing.B) {
	// A single buffer, which should fit in some level of CPU cache.
	b.Run("Single buffer", func(b *testing.B) {
		var om OutMessage
		for i := 0; i < b.N; i++ {
			om.Grow(MaxReadSize)
			om.ShrinkTo(OutMessageInitialSize)
		}

		b.SetBytes(int64(MaxReadSize))
	})

	// Many megabytes worth of buffers, which should defeat the CPU cache.
	b.Run("Many buffers", func(b *testing.B) {
		// The number of messages; intentionally a power of two.
		const numMessages = 128

		var oms [numMessages]OutMessage
		if s := unsafe.Sizeof(oms); s < 128<<20 {
			panic(fmt.Sprintf("Array is too small; total size: %d", s))
		}

		for i := 0; i < b.N; i++ {
			oms[i%numMessages].Grow(MaxReadSize)
			oms[i%numMessages].ShrinkTo(OutMessageInitialSize)
		}

		b.SetBytes(int64(MaxReadSize))
	})
}
<commit_msg>buffer_test: expand the coverage of TestOutMessageGrow.<commit_after>package buffer

import (
	"crypto/rand"
	"fmt"
	"io"
	"reflect"
	"testing"
	"unsafe"
)

func toByteSlice(p unsafe.Pointer, n int) []byte {
	sh := reflect.SliceHeader{
		Data: uintptr(p),
		Len:  n,
		Cap:  n,
	}

	return *(*[]byte)(unsafe.Pointer(&sh))
}

// fillWithGarbage writes random data to [p, p+n).
func fillWithGarbage(p unsafe.Pointer, n int) (err error) {
	b := toByteSlice(p, n)
	_, err = io.ReadFull(rand.Reader, b)
	return
}

func randBytes(n int) (b []byte, err error) {
	b = make([]byte, n)
	_, err = io.ReadFull(rand.Reader, b)
	return
}

// findNonZero finds the offset of the first non-zero byte in [p, p+n). If
// none, it returns n.
func findNonZero(p unsafe.Pointer, n int) int {
	b := toByteSlice(p, n)
	for i, x := range b {
		if x != 0 {
			return i
		}
	}

	return n
}

func TestMemclr(t *testing.T) {
	// All sizes up to 32 bytes.
	var sizes []int
	for i := 0; i <= 32; i++ {
		sizes = append(sizes, i)
	}

	// And a few hand-chosen sizes.
	sizes = append(sizes, []int{
		39, 41, 64, 127, 128, 129,
		1<<20 - 1,
		1 << 20,
		1<<20 + 1,
	}...)

	// For each size, fill a buffer with random bytes and then zero it.
	for _, size := range sizes {
		size := size
		t.Run(fmt.Sprintf("size=%d", size), func(t *testing.T) {
			// Generate
			b, err := randBytes(size)
			if err != nil {
				t.Fatalf("randBytes: %v", err)
			}

			// Clear
			var p unsafe.Pointer
			if len(b) != 0 {
				p = unsafe.Pointer(&b[0])
			}

			memclr(p, uintptr(len(b)))

			// Check
			if i := findNonZero(p, len(b)); i != len(b) {
				t.Fatalf("non-zero byte at offset %d", i)
			}
		})
	}
}

func TestOutMessageAppend(t *testing.T) {
	t.Fatal("TODO")
}

func TestOutMessageAppendString(t *testing.T) {
	t.Fatal("TODO")
}

func TestOutMessageShrinkTo(t *testing.T) {
	t.Fatal("TODO")
}

func TestOutMessageHeader(t *testing.T) {
	t.Fatal("TODO")
}

func TestOutMessageReset(t *testing.T) {
	var om OutMessage
	h := om.OutHeader()

	const trials = 10
	for i := 0; i < trials; i++ {
		// Fill the header with garbage.
		err := fillWithGarbage(unsafe.Pointer(h), int(unsafe.Sizeof(*h)))
		if err != nil {
			t.Fatalf("fillWithGarbage: %v", err)
		}

		// Ensure a non-zero payload length.
		if p := om.GrowNoZero(128); p == nil {
			t.Fatal("GrowNoZero failed")
		}

		// Reset.
		om.Reset()

		// Check that the length was updated.
		if got, want := int(om.Len()), int(OutMessageInitialSize); got != want {
			t.Fatalf("om.Len() = %d, want %d", got, want)
		}

		// Check that the header was zeroed.
		if h.Len != 0 {
			t.Fatalf("non-zero Len %v", h.Len)
		}

		if h.Error != 0 {
			t.Fatalf("non-zero Error %v", h.Error)
		}

		if h.Unique != 0 {
			t.Fatalf("non-zero Unique %v", h.Unique)
		}
	}
}

func TestOutMessageGrow(t *testing.T) {
	var om OutMessage
	om.Reset()

	// Set up garbage where the payload will soon be.
	const payloadSize = 1234
	{
		p := om.GrowNoZero(payloadSize)
		if p == nil {
			t.Fatal("GrowNoZero failed")
		}

		err := fillWithGarbage(p, payloadSize)
		if err != nil {
			t.Fatalf("fillWithGarbage: %v", err)
		}

		om.ShrinkTo(OutMessageInitialSize)
	}

	// Call Grow.
	if p := om.Grow(payloadSize); p == nil {
		t.Fatal("Grow failed")
	}

	// Check the resulting length in two ways.
	const wantLen = int(payloadSize + OutMessageInitialSize)
	if got, want := om.Len(), wantLen; got != want {
		t.Errorf("om.Len() = %d, want %d", got)
	}

	b := om.Bytes()
	if got, want := len(b), wantLen; got != want {
		t.Fatalf("len(om.Len()) = %d, want %d", got)
	}

	// Check that the payload was zeroed.
	for i, x := range b[OutMessageInitialSize:] {
		if x != 0 {
			t.Fatalf("non-zero byte 0x%02x at payload offset %d", x, i)
		}
	}
}

func BenchmarkOutMessageReset(b *testing.B) {
	// A single buffer, which should fit in some level of CPU cache.
	b.Run("Single buffer", func(b *testing.B) {
		var om OutMessage
		for i := 0; i < b.N; i++ {
			om.Reset()
		}

		b.SetBytes(int64(om.offset))
	})

	// Many megabytes worth of buffers, which should defeat the CPU cache.
	b.Run("Many buffers", func(b *testing.B) {
		// The number of messages; intentionally a power of two.
		const numMessages = 128

		var oms [numMessages]OutMessage
		if s := unsafe.Sizeof(oms); s < 128<<20 {
			panic(fmt.Sprintf("Array is too small; total size: %d", s))
		}

		for i := 0; i < b.N; i++ {
			oms[i%numMessages].Reset()
		}

		b.SetBytes(int64(oms[0].offset))
	})
}

func BenchmarkOutMessageGrowShrink(b *testing.B) {
	// A single buffer, which should fit in some level of CPU cache.
	b.Run("Single buffer", func(b *testing.B) {
		var om OutMessage
		for i := 0; i < b.N; i++ {
			om.Grow(MaxReadSize)
			om.ShrinkTo(OutMessageInitialSize)
		}

		b.SetBytes(int64(MaxReadSize))
	})

	// Many megabytes worth of buffers, which should defeat the CPU cache.
	b.Run("Many buffers", func(b *testing.B) {
		// The number of messages; intentionally a power of two.
		const numMessages = 128

		var oms [numMessages]OutMessage
		if s := unsafe.Sizeof(oms); s < 128<<20 {
			panic(fmt.Sprintf("Array is too small; total size: %d", s))
		}

		for i := 0; i < b.N; i++ {
			oms[i%numMessages].Grow(MaxReadSize)
			oms[i%numMessages].ShrinkTo(OutMessageInitialSize)
		}

		b.SetBytes(int64(MaxReadSize))
	})
}
<|endoftext|>


<commit_before>package main

import (
	"log"

	"github.com/bitfinexcom/bitfinex-api-go/v2/rest"
	"github.com/davecgh/go-spew/spew"
)

func main() {
	c := rest.NewClient()

	pulseHist, err := c.Pulse.PublicPulseHistory("", "")
	if err != nil {
		log.Fatalf("PublicPulseHistory: %s", err)
	}

	spew.Dump(pulseHist)
}
<commit_msg>public hist example fix<commit_after>package main

import (
	"log"

	"github.com/bitfinexcom/bitfinex-api-go/v2/rest"
	"github.com/davecgh/go-spew/spew"
)

func main() {
	c := rest.NewClient()

	pulseHist, err := c.Pulse.PublicPulseHistory(0, 0)
	if err != nil {
		log.Fatalf("PublicPulseHistory: %s", err)
	}

	spew.Dump(pulseHist)
}
<|endoftext|>


<commit_before>package shell_test

import (
	"testing"

	"github.com/stretchr/testify/assert"
	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/common"
	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/helpers"
	"io/ioutil"
	"os"
	"time"
)

func TestBashShellSuccessRun(t *testing.T) {
	if helpers.SkipIntegrationTests(t, "bash") {
		return
	}

	build := &common.Build{
		GetBuildResponse: common.SuccessfulBuild,
		Runner: &common.RunnerConfig{
			RunnerSettings: common.RunnerSettings{
				Executor: "shell",
				Shell:    "bash",
			},
		},
	}

	err := build.Run(&common.Config{}, &common.Trace{Writer: os.Stdout})
	assert.NoError(t, err)
}

func TestWindowsBatchSuccessRun(t *testing.T) {
	if helpers.SkipIntegrationTests(t, "cmd.exe") {
		return
	}

	build := &common.Build{
		GetBuildResponse: common.SuccessfulBuild,
		Runner: &common.RunnerConfig{
			RunnerSettings: common.RunnerSettings{
				Executor: "shell",
				Shell:    "cmd",
			},
		},
	}

	err := build.Run(&common.Config{}, &common.Trace{Writer: os.Stdout})
	assert.NoError(t, err)
}

func TestPowerShellSuccessRun(t *testing.T) {
	if helpers.SkipIntegrationTests(t, "powershell.exe") {
		return
	}

	build := &common.Build{
		GetBuildResponse: common.SuccessfulBuild,
		Runner: &common.RunnerConfig{
			RunnerSettings: common.RunnerSettings{
				Executor: "shell",
				Shell:    "powershell",
			},
		},
	}

	err := build.Run(&common.Config{}, &common.Trace{Writer: os.Stdout})
	assert.NoError(t, err)
}

func TestShellBuildAbort(t *testing.T) {
	if helpers.SkipIntegrationTests(t) {
		return
	}

	build := &common.Build{
		GetBuildResponse: common.LongRunningBuild,
		Runner: &common.RunnerConfig{
			RunnerSettings: common.RunnerSettings{
				Executor: "shell",
			},
		},
		SystemInterrupt: make(chan os.Signal, 1),
	}

	abortTimer := time.AfterFunc(time.Second, func() {
		t.Log("Interrupt")
		build.SystemInterrupt <- os.Interrupt
	})
	defer abortTimer.Stop()

	timeoutTimer := time.AfterFunc(time.Second*3, func() {
		t.Log("Timedout")
		t.FailNow()
	})
	defer timeoutTimer.Stop()

	err := build.Run(&common.Config{}, &common.Trace{Writer: os.Stdout})
	assert.EqualError(t, err, "aborted: interrupt")
}

func TestShellBuildCancel(t *testing.T) {
	if helpers.SkipIntegrationTests(t) {
		return
	}

	build := &common.Build{
		GetBuildResponse: common.LongRunningBuild,
		Runner: &common.RunnerConfig{
			RunnerSettings: common.RunnerSettings{
				Executor: "shell",
			},
		},
	}

	trace := &common.Trace{Writer: os.Stdout, Abort: make(chan interface{}, 1)}

	abortTimer := time.AfterFunc(time.Second, func() {
		t.Log("Interrupt")
		trace.Abort <- true
	})
	defer abortTimer.Stop()

	timeoutTimer := time.AfterFunc(time.Second*3, func() {
		t.Log("Timedout")
		t.FailNow()
	})
	defer timeoutTimer.Stop()

	err := build.Run(&common.Config{}, trace)
	assert.EqualError(t, err, "canceled")
	assert.IsType(t, err, &common.BuildError{})
}

func runBuildWithIndexLockForShell(t *testing.T, shell string) {
	if helpers.SkipIntegrationTests(t, shell) {
		return
	}

	build := &common.Build{
		GetBuildResponse: common.SuccessfulBuild,
		Runner: &common.RunnerConfig{
			RunnerSettings: common.RunnerSettings{
				Executor: "shell",
				Shell:    shell,
			},
		},
	}
	err := build.Run(&common.Config{}, &common.Trace{Writer: os.Stdout})
	assert.NoError(t, err)

	build.GetBuildResponse.AllowGitFetch = true
	ioutil.WriteFile(build.BuildDir+"/.git/index.lock", []byte{}, os.ModeSticky)

	err = build.Run(&common.Config{}, &common.Trace{Writer: os.Stdout})
	assert.NoError(t, err)
}

func TestShellBuildWithIndexLockBash(t *testing.T) {
	runBuildWithIndexLockForShell(t, "bash")
}

func TestShellBuildWithIndexLockCmd(t *testing.T) {
	runBuildWithIndexLockForShell(t, "cmd")
}

func TestShellBuildWithIndexLockPowershell(t *testing.T) {
	runBuildWithIndexLockForShell(t, "powershell")
}
<commit_msg>Refactor shell executor tests<commit_after>package shell_test

import (
	"bytes"
	"io/ioutil"
	"os"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"

	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/common"
	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/helpers"
)

func onEachShell(t *testing.T, f func(t *testing.T, shell string)) {
	t.Run("bash", func(t *testing.T) {
		if helpers.SkipIntegrationTests(t, "bash") {
			t.Skip()
		}

		f(t, "bash")
	})

	t.Run("cmd.exe", func(t *testing.T) {
		if helpers.SkipIntegrationTests(t, "cmd.exe") {
			t.Skip()
		}

		f(t, "cmd")
	})

	t.Run("powershell.exe", func(t *testing.T) {
		if helpers.SkipIntegrationTests(t, "powershell.exe") {
			t.Skip()
		}

		f(t, "powershell")
	})
}

func runBuildWithTrace(t *testing.T, build *common.Build, trace *common.Trace) error {
	timeoutTimer := time.AfterFunc(10*time.Second, func() {
		t.Log("Timed out")
		t.FailNow()
	})
	defer timeoutTimer.Stop()

	return build.Run(&common.Config{}, trace)
}

func runBuild(t *testing.T, build *common.Build) error {
	return runBuildWithTrace(t, build, &common.Trace{Writer: os.Stdout})
}

func runBuildReturningOutput(t *testing.T, build *common.Build) (string, error) {
	buf := bytes.NewBuffer(nil)
	err := runBuildWithTrace(t, build, &common.Trace{Writer: buf})
	output := buf.String()
	t.Log(output)

	return output, err
}

func newBuild(t *testing.T, getBuildResponse common.GetBuildResponse, shell string) (*common.Build, func()) {
	dir, err := ioutil.TempDir("", "gitlab-runner-shell-executor-test")
	if err != nil {
		t.Fatal(err)
	}

	t.Log("Build directory:", dir)

	build := &common.Build{
		GetBuildResponse: getBuildResponse,
		Runner: &common.RunnerConfig{
			RunnerSettings: common.RunnerSettings{
				BuildsDir: dir,
				Executor:  "shell",
				Shell:     shell,
			},
		},
		SystemInterrupt: make(chan os.Signal, 1),
	}

	cleanup := func() {
		os.RemoveAll(dir)
	}

	return build, cleanup
}

func TestBuildSuccess(t *testing.T) {
	onEachShell(t, func(t *testing.T, shell string) {
		build, cleanup := newBuild(t, common.SuccessfulBuild, shell)
		defer cleanup()

		err := runBuild(t, build)
		assert.NoError(t, err)
	})
}

func TestBuildAbort(t *testing.T) {
	onEachShell(t, func(t *testing.T, shell string) {
		build, cleanup := newBuild(t, common.LongRunningBuild, shell)
		defer cleanup()

		abortTimer := time.AfterFunc(time.Second, func() {
			t.Log("Interrupt")
			build.SystemInterrupt <- os.Interrupt
		})
		defer abortTimer.Stop()

		err := runBuild(t, build)
		assert.EqualError(t, err, "aborted: interrupt")
	})
}

func TestBuildCancel(t *testing.T) {
	onEachShell(t, func(t *testing.T, shell string) {
		build, cleanup := newBuild(t, common.LongRunningBuild, shell)
		defer cleanup()

		cancelChan := make(chan interface{}, 1)
		cancelTimer := time.AfterFunc(time.Second, func() {
			t.Log("Cancel")
			cancelChan <- true
		})
		defer cancelTimer.Stop()

		err := runBuildWithTrace(t, build, &common.Trace{Writer: os.Stdout, Abort: cancelChan})
		assert.EqualError(t, err, "canceled")
		assert.IsType(t, err, &common.BuildError{})
	})
}

func TestBuildWithIndexLock(t *testing.T) {
	onEachShell(t, func(t *testing.T, shell string) {
		build, cleanup := newBuild(t, common.SuccessfulBuild, shell)
		defer cleanup()

		err := runBuild(t, build)
		assert.NoError(t, err)

		build.GetBuildResponse.AllowGitFetch = true
		ioutil.WriteFile(build.BuildDir+"/.git/index.lock", []byte{}, os.ModeSticky)

		err = runBuild(t, build)
		assert.NoError(t, err)
	})
}
<|endoftext|>


<commit_before>package nyb

import (
	"encoding/json"
	"errors"
	"fmt"
	"strings"
	"time"

	kitty "github.com/ugjka/kittybot"
	"gopkg.in/ugjka/go-tz.v2/tz"
)

const helpMsg = "COMMANDS: '%shny <location>', '%stime <location>', '%snext', '%sprevious', '%sremaining', '%shelp', '%ssource'"

func (bot *Settings) addTriggers() {
	irc := bot.irc

	//Log Notices
	irc.AddTrigger(kitty.Trigger{
		Condition: func(b *kitty.Bot, m *kitty.Message) bool {
			return m.Command == "NOTICE"
		},
		Action: func(b *kitty.Bot, m *kitty.Message) {
			b.Info("[NOTICE] " + m.Content)
		},
	})

	//Trigger for !source
	irc.AddTrigger(kitty.Trigger{
		Condition: func(b *kitty.Bot, m *kitty.Message) bool {
			return m.Command == "PRIVMSG" &&
				strings.HasPrefix(normalize(m.Content), bot.Prefix+"source")
		},
		Action: func(b *kitty.Bot, m *kitty.Message) {
			b.Reply(m, "https://github.com/ugjka/newyearsbot")
		},
	})

	//Trigger for !help
	irc.AddTrigger(kitty.Trigger{
		Condition: func(b *kitty.Bot, m *kitty.Message) bool {
			return m.Command == "PRIVMSG" &&
				strings.HasPrefix(normalize(m.Content), bot.Prefix+"help") ||
				normalize(m.Content) == bot.Prefix+"hny"
		},
		Action: func(b *kitty.Bot, m *kitty.Message) {
			b.Info("Querying help...")
			b.Reply(m, fmt.Sprintf(helpMsg, bot.Prefix, bot.Prefix, bot.Prefix, bot.Prefix, bot.Prefix, bot.Prefix, bot.Prefix))
		},
	})

	//Trigger for !next
	irc.AddTrigger(kitty.Trigger{
		Condition: func(b *kitty.Bot, m *kitty.Message) bool {
			return m.Command == "PRIVMSG" &&
				strings.HasPrefix(normalize(m.Content), bot.Prefix+"next")
		},
		Action: func(b *kitty.Bot, m *kitty.Message) {
			b.Info("Querying next...")
			dur := time.Minute * time.Duration(bot.next.Offset*60)
			if timeNow().UTC().Add(dur).After(target) {
				b.Reply(m, fmt.Sprintf("No more next, %d is here AoE", target.Year()))
				return
			}
			hdur := humanDur(target.Sub(timeNow().UTC().Add(dur)))
			b.Reply(m, fmt.Sprintf("Next New Year in %s in %s",
				hdur, bot.next))
		},
	})

	//Trigger for !previous
	irc.AddTrigger(kitty.Trigger{
		Condition: func(b *kitty.Bot, m *kitty.Message) bool {
			return m.Command == "PRIVMSG" &&
				strings.HasPrefix(normalize(m.Content), bot.Prefix+"previous")
		},
		Action: func(b *kitty.Bot, m *kitty.Message) {
			b.Info("Querying previous...")
			dur := time.Minute * time.Duration(bot.previous.Offset*60)
			hdur := humanDur(timeNow().UTC().Add(dur).Sub(target))
			if bot.previous.Offset == -12 {
				hdur = humanDur(timeNow().UTC().Add(dur).Sub(target.AddDate(-1, 0, 0)))
			}
			b.Reply(m, fmt.Sprintf("Previous New Year %s ago in %s",
				hdur, bot.previous))
		},
	})

	//Trigger for !remaining
	irc.AddTrigger(kitty.Trigger{
		Condition: func(b *kitty.Bot, m *kitty.Message) bool {
			return m.Command == "PRIVMSG" &&
				strings.HasPrefix(normalize(m.Content), bot.Prefix+"remaining")
		},
		Action: func(b *kitty.Bot, m *kitty.Message) {
			b.Info("Querying remaining...")
			plural := "s"
			if bot.remaining == 1 {
				plural = ""
			}
			b.Reply(m, fmt.Sprintf("%d timezone%s remaining. %d%% are in the new year", bot.remaining, (bot.zones - bot.remaining) / bot.zones, plural))
		},
	})

	//Trigger for time in location
	irc.AddTrigger(kitty.Trigger{
		Condition: func(b *kitty.Bot, m *kitty.Message) bool {
			return m.Command == "PRIVMSG" &&
				strings.HasPrefix(normalize(m.Content), bot.Prefix+"time ")
		},
		Action: func(b *kitty.Bot, m *kitty.Message) {
			b.Info("Querying time...")
			result, err := bot.time(normalize(m.Content)[len(bot.Prefix)+len("time")+1:])
			if err == errNoZone || err == errNoPlace {
				b.Warn("Query error: " + err.Error())
				b.Reply(m, err.Error())
				return
			}
			if err != nil {
				b.Warn("Query error: " + err.Error())
				b.Reply(m, "Some error occurred!")
				return
			}
			b.Reply(m, result)
		},
	})

	//Trigger for UTC time
	irc.AddTrigger(kitty.Trigger{
		Condition: func(b *kitty.Bot, m *kitty.Message) bool {
			return m.Command == "PRIVMSG" &&
				normalize(m.Content) == bot.Prefix+"time"
		},
		Action: func(b *kitty.Bot, m *kitty.Message) {
			b.Info("Querying time...")
			result := "Time is " + time.Now().UTC().Format("Mon Jan 2 15:04:05 -0700 MST 2006")
			b.Reply(m, result)
		},
	})

	//Trigger for new year in location
	irc.AddTrigger(kitty.Trigger{
		Condition: func(b *kitty.Bot, m *kitty.Message) bool {
			return m.Command == "PRIVMSG" &&
				strings.HasPrefix(normalize(m.Content), bot.Prefix+"hny ")
		},
		Action: func(b *kitty.Bot, m *kitty.Message) {
			result, err := bot.newYear(normalize(m.Content)[len(bot.Prefix)+len("hny")+1:])
			if err == errNoZone || err == errNoPlace {
				b.Warn("Query error: " + err.Error())
				b.Reply(m, err.Error())
				return
			}
			if err != nil {
				b.Warn("Query error: " + err.Error())
				b.Reply(m, "Some error occurred!")
				return
			}
			b.Reply(m, result)
		},
	})
}

var (
	errNoZone  = errors.New("couldn't get timezone for that location")
	errNoPlace = errors.New("Couldn't find that place")
)

func (bot *Settings) time(location string) (string, error) {
	bot.irc.Info("Querying location: " + location)
	data, err := NominatimFetcher(&bot.Email, &bot.Nominatim, &location)
	if err != nil {
		bot.irc.Warn("Nominatim error: " + err.Error())
		return "", err
	}
	var res NominatimResults
	if err = json.Unmarshal(data, &res); err != nil {
		bot.irc.Warn("Nominatim JSON error: " + err.Error())
		return "", err
	}
	if len(res) == 0 {
		return "", errNoPlace
	}
	p := tz.Point{
		Lat: res[0].Lat,
		Lon: res[0].Lon,
	}
	tzid, err := tz.GetZone(p)
	if err != nil {
		return "", errNoZone
	}
	zone, err := time.LoadLocation(tzid[0])
	if err != nil {
		return "", errNoZone
	}
	address := res[0].DisplayName
	msg := fmt.Sprintf("Time in %s is %s", address, time.Now().In(zone).Format("Mon Jan 2 15:04:05 -0700 MST 2006"))
	return msg, nil
}

func (bot *Settings) newYear(location string) (string, error) {
	bot.irc.Info("Querying location: " + location)
	data, err := NominatimFetcher(&bot.Email, &bot.Nominatim, &location)
	if err != nil {
		bot.irc.Warn("Nominatim error: " + err.Error())
		return "", err
	}
	var res NominatimResults
	if err = json.Unmarshal(data, &res); err != nil {
		bot.irc.Warn("Nominatim JSON error: " + err.Error())
		return "", err
	}
	if len(res) == 0 {
		return "", errNoPlace
	}
	p := tz.Point{
		Lat: res[0].Lat,
		Lon: res[0].Lon,
	}
	tzid, err := tz.GetZone(p)
	if err != nil {
		return "", errNoZone
	}
	zone, err := time.LoadLocation(tzid[0])
	if err != nil {
		return "", errNoZone
	}
	offset := zoneOffset(target, zone)
	address := res[0].DisplayName
	if timeNow().UTC().Add(offset).Before(target) {
		hdur := humanDur(target.Sub(timeNow().UTC().Add(offset)))
		const newYearFutureMsg = "New Year in %s will happen in %s"
		return fmt.Sprintf(newYearFutureMsg, address, hdur), nil
	}
	hdur := humanDur(timeNow().UTC().Add(offset).Sub(target))
	const newYearPastMsg = "New Year in %s happened %s ago"
	return fmt.Sprintf(newYearPastMsg, address, hdur), nil
}
<commit_msg>multiply for percentage int<commit_after>package nyb

import (
	"encoding/json"
	"errors"
	"fmt"
	"strings"
	"time"

	kitty "github.com/ugjka/kittybot"
	"gopkg.in/ugjka/go-tz.v2/tz"
)

const helpMsg = "COMMANDS: '%shny <location>', '%stime <location>', '%snext', '%sprevious', '%sremaining', '%shelp', '%ssource'"

func (bot *Settings) addTriggers() {
	irc := bot.irc

	//Log Notices
	irc.AddTrigger(kitty.Trigger{
		Condition: func(b *kitty.Bot, m *kitty.Message) bool {
			return m.Command == "NOTICE"
		},
		Action: func(b *kitty.Bot, m *kitty.Message) {
			b.Info("[NOTICE] " + m.Content)
		},
	})

	//Trigger for !source
	irc.AddTrigger(kitty.Trigger{
		Condition: func(b *kitty.Bot, m *kitty.Message) bool {
			return m.Command == "PRIVMSG" &&
				strings.HasPrefix(normalize(m.Content), bot.Prefix+"source")
		},
		Action: func(b *kitty.Bot, m *kitty.Message) {
			b.Reply(m, "https://github.com/ugjka/newyearsbot")
		},
	})

	//Trigger for !help
	irc.AddTrigger(kitty.Trigger{
		Condition: func(b *kitty.Bot, m *kitty.Message) bool {
			return m.Command == "PRIVMSG" &&
				strings.HasPrefix(normalize(m.Content), bot.Prefix+"help") ||
				normalize(m.Content) == bot.Prefix+"hny"
		},
		Action: func(b *kitty.Bot, m *kitty.Message) {
			b.Info("Querying help...")
			b.Reply(m, fmt.Sprintf(helpMsg, bot.Prefix, bot.Prefix, bot.Prefix, bot.Prefix, bot.Prefix, bot.Prefix, bot.Prefix))
		},
	})

	//Trigger for !next
	irc.AddTrigger(kitty.Trigger{
		Condition: func(b *kitty.Bot, m *kitty.Message) bool {
			return m.Command == "PRIVMSG" &&
				strings.HasPrefix(normalize(m.Content), bot.Prefix+"next")
		},
		Action: func(b *kitty.Bot, m *kitty.Message) {
			b.Info("Querying next...")
			dur := time.Minute * time.Duration(bot.next.Offset*60)
			if timeNow().UTC().Add(dur).After(target) {
				b.Reply(m, fmt.Sprintf("No more next, %d is here AoE", target.Year()))
				return
			}
			hdur := humanDur(target.Sub(timeNow().UTC().Add(dur)))
			b.Reply(m, fmt.Sprintf("Next New Year in %s in %s",
				hdur, bot.next))
		},
	})

	//Trigger for !previous
	irc.AddTrigger(kitty.Trigger{
		Condition: func(b *kitty.Bot, m *kitty.Message) bool {
			return m.Command == "PRIVMSG" &&
				strings.HasPrefix(normalize(m.Content), bot.Prefix+"previous")
		},
		Action: func(b *kitty.Bot, m *kitty.Message) {
			b.Info("Querying previous...")
			dur := time.Minute * time.Duration(bot.previous.Offset*60)
			hdur := humanDur(timeNow().UTC().Add(dur).Sub(target))
			if bot.previous.Offset == -12 {
				hdur = humanDur(timeNow().UTC().Add(dur).Sub(target.AddDate(-1, 0, 0)))
			}
			b.Reply(m, fmt.Sprintf("Previous New Year %s ago in %s",
				hdur, bot.previous))
		},
	})

	//Trigger for !remaining
	irc.AddTrigger(kitty.Trigger{
		Condition: func(b *kitty.Bot, m *kitty.Message) bool {
			return m.Command == "PRIVMSG" &&
				strings.HasPrefix(normalize(m.Content), bot.Prefix+"remaining")
		},
		Action: func(b *kitty.Bot, m *kitty.Message) {
			b.Info("Querying remaining...")
			plural := "s"
			if bot.remaining == 1 {
				plural = ""
			}
			b.Reply(m, fmt.Sprintf("%d timezone%s remaining. %d%% are in the new year", bot.remaining, (bot.zones - bot.remaining) * 100 / (bot.zones * 100), plural))
		},
	})

	//Trigger for time in location
	irc.AddTrigger(kitty.Trigger{
		Condition: func(b *kitty.Bot, m *kitty.Message) bool {
			return m.Command == "PRIVMSG" &&
				strings.HasPrefix(normalize(m.Content), bot.Prefix+"time ")
		},
		Action: func(b *kitty.Bot, m *kitty.Message) {
			b.Info("Querying time...")
			result, err := bot.time(normalize(m.Content)[len(bot.Prefix)+len("time")+1:])
			if err == errNoZone || err == errNoPlace {
				b.Warn("Query error: " + err.Error())
				b.Reply(m, err.Error())
				return
			}
			if err != nil {
				b.Warn("Query error: " + err.Error())
				b.Reply(m, "Some error occurred!")
				return
			}
			b.Reply(m, result)
		},
	})

	//Trigger for UTC time
	irc.AddTrigger(kitty.Trigger{
		Condition: func(b *kitty.Bot, m *kitty.Message) bool {
			return m.Command == "PRIVMSG" &&
				normalize(m.Content) == bot.Prefix+"time"
		},
		Action: func(b *kitty.Bot, m *kitty.Message) {
			b.Info("Querying time...")
			result := "Time is " + time.Now().UTC().Format("Mon Jan 2 15:04:05 -0700 MST 2006")
			b.Reply(m, result)
		},
	})

	//Trigger for new year in location
	irc.AddTrigger(kitty.Trigger{
		Condition: func(b *kitty.Bot, m *kitty.Message) bool {
			return m.Command == "PRIVMSG" &&
				strings.HasPrefix(normalize(m.Content), bot.Prefix+"hny ")
		},
		Action: func(b *kitty.Bot, m *kitty.Message) {
			result, err := bot.newYear(normalize(m.Content)[len(bot.Prefix)+len("hny")+1:])
			if err == errNoZone || err == errNoPlace {
				b.Warn("Query error: " + err.Error())
				b.Reply(m, err.Error())
				return
			}
			if err != nil {
				b.Warn("Query error: " + err.Error())
				b.Reply(m, "Some error occurred!")
				return
			}
			b.Reply(m, result)
		},
	})
}

var (
	errNoZone  = errors.New("couldn't get timezone for that location")
	errNoPlace = errors.New("Couldn't find that place")
)

func (bot *Settings) time(location string) (string, error) {
	bot.irc.Info("Querying location: " + location)
	data, err := NominatimFetcher(&bot.Email, &bot.Nominatim, &location)
	if err != nil {
		bot.irc.Warn("Nominatim error: " + err.Error())
		return "", err
	}
	var res NominatimResults
	if err = json.Unmarshal(data, &res); err != nil {
		bot.irc.Warn("Nominatim JSON error: " + err.Error())
		return "", err
	}
	if len(res) == 0 {
		return "", errNoPlace
	}
	p := tz.Point{
		Lat: res[0].Lat,
		Lon: res[0].Lon,
	}
	tzid, err := tz.GetZone(p)
	if err != nil {
		return "", errNoZone
	}
	zone, err := time.LoadLocation(tzid[0])
	if err != nil {
		return "", errNoZone
	}
	address := res[0].DisplayName
	msg := fmt.Sprintf("Time in %s is %s", address, time.Now().In(zone).Format("Mon Jan 2 15:04:05 -0700 MST 2006"))
	return msg, nil
}

func (bot *Settings) newYear(location string) (string, error) {
	bot.irc.Info("Querying location: " + location)
	data, err := NominatimFetcher(&bot.Email, &bot.Nominatim, &location)
	if err != nil {
		bot.irc.Warn("Nominatim error: " + err.Error())
		return "", err
	}
	var res NominatimResults
	if err = json.Unmarshal(data, &res); err != nil {
		bot.irc.Warn("Nominatim JSON error: " + err.Error())
		return "", err
	}
	if len(res) == 0 {
		return "", errNoPlace
	}
	p := tz.Point{
		Lat: res[0].Lat,
		Lon: res[0].Lon,
	}
	tzid, err := tz.GetZone(p)
	if err != nil {
		return "", errNoZone
	}
	zone, err := time.LoadLocation(tzid[0])
	if err != nil {
		return "", errNoZone
	}
	offset := zoneOffset(target, zone)
	address := res[0].DisplayName
	if timeNow().UTC().Add(offset).Before(target) {
		hdur := humanDur(target.Sub(timeNow().UTC().Add(offset)))
		const newYearFutureMsg = "New Year in %s will happen in %s"
		return fmt.Sprintf(newYearFutureMsg, address, hdur), nil
	}
	hdur := humanDur(timeNow().UTC().Add(offset).Sub(target))
	const newYearPastMsg = "New Year in %s happened %s ago"
	return fmt.Sprintf(newYearPastMsg, address, hdur), nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved.
// Licensed under the Mozilla Public License v2.0

package oci

import (
	"log"
)

const Version = "3.88.0"

func PrintVersion() {
	log.Printf("[INFO] terraform-provider-oci %s\n", Version)
}
<commit_msg>Finalize changelog and release for version v3.89.0<commit_after>// Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved.
// Licensed under the Mozilla Public License v2.0

package oci

import (
	"log"
)

const Version = "3.89.0"

func PrintVersion() {
	log.Printf("[INFO] terraform-provider-oci %s\n", Version)
}
<|endoftext|>


<commit_before>package stl

// This file defines the top level reading and writing operations
// for the stl package

import (
	"bufio"
	"errors"
	"io"
	"os"
)

// ErrIncompleteBinaryHeader is used when reading binary STL files with incomplete header.
var ErrIncompleteBinaryHeader = errors.New("incomplete STL binary header, 84 bytes expected")

// ErrUnexpectedEOF is used by ReadFile and ReadAll to signify an incomplete file.
var ErrUnexpectedEOF = errors.New("unexpected end of file")

// ReadFile reads the contents of a file into a new Solid object. The file
// can be either in STL ASCII format, beginning with "solid ", or in
// STL binary format, beginning with a 84 byte header. Shorthand for os.Open and ReadAll
func ReadFile(filename string) (solid *Solid, err error) {
	var s Solid
	err = CopyFile(filename, &s)
	if err == nil {
		solid = &s
	}
	return
}

// ReadAll reads the contents of a file into a new Solid object. The file
// can be either in STL ASCII format, beginning with "solid ", or in
// STL binary format, beginning with a 84 byte header. Because of this,
// the file pointer has to be at the beginning of the file.
func ReadAll(r io.ReadSeeker) (solid *Solid, err error) {
	var s Solid
	err = CopyAll(r, &s)
	if err == nil {
		solid = &s
	}
	return
}

func CopyFile(filename string, sw Writer) (err error) {
	file, openErr := os.Open(filename)
	if openErr != nil {
		err = openErr
		return
	}
	err = CopyAll(file, sw)
	closeErr := file.Close()
	if err == nil {
		err = closeErr
	}
	return
}

func CopyAll(r io.ReadSeeker, sw Writer) (err error) {
	isBinary, err := isBinaryFile(r)
	if err != nil {
		return
	}
	if _, err = r.Seek(0, io.SeekStart); err != nil {
		return
	}

	if isBinary {
		sw.SetASCII(false)
		err = readAllBinary(r, sw)
	} else {
		sw.SetASCII(true)
		err = readAllASCII(r, sw)
	}

	return
}

// isBinaryFile returns true if the seekable stream tests as a binary file by
// matching triangle count (in header) and file size
func isBinaryFile(r io.ReadSeeker) (isBinary bool, err error) {
	var header [binaryHeaderSize]byte
	_, err = r.Read(header[:])
	if err != nil {
		if err == io.EOF { // too short to meet spec
			err = nil
		}
		return
	}
	triangleCount := triangleCountFromBinaryHeader(header[:])
	expectedFileLength := int64(triangleCount)*binaryTriangleSize + binaryHeaderSize
	actualFileLength, err := r.Seek(0, io.SeekEnd)
	if err != nil {
		return
	}
	isBinary = expectedFileLength == actualFileLength
	return
}

// WriteFile creates file with name filename and write contents of this Solid.
// Shorthand for os.Create and Solid.WriteAll
func (s *Solid) WriteFile(filename string) (err error) {
	file, err := os.Create(filename)
	if err != nil {
		return err
	}

	bufWriter := bufio.NewWriter(file)
	err = s.WriteAll(bufWriter)
	flushErr := bufWriter.Flush()
	closeErr := file.Close()
	if err == nil {
		err = flushErr
	}
	if err == nil {
		err = closeErr
	}
	return
}

// WriteAll writes the contents of this solid to an io.Writer. Depending on solid.IsAscii
// the STL ASCII format, or the STL binary format is used. If IsAscii
// is false, and the binary format is used, solid.Name will be used for
// the header, if solid.BinaryHeader is empty.
func (s *Solid) WriteAll(w io.Writer) error {
	if s.IsAscii {
		return writeSolidASCII(w, s)
	}
	return writeSolidBinary(w, s)
}

// Extracts an ASCII string from a byte slice. Reads all characters
// from the beginning until a \0 or a non-ASCII character is found.
func extractASCIIString(byteData []byte) string {
	i := 0
	for i < len(byteData) && byteData[i] < byte(128) && byteData[i] != byte(0) {
		i++
	}
	return string(byteData[0:i])
}
<commit_msg>use bufio.Reader for reading STL files for speed-up (works best on binary)<commit_after>package stl

// This file defines the top level reading and writing operations
// for the stl package

import (
	"bufio"
	"errors"
	"io"
	"os"
)

// ErrIncompleteBinaryHeader is used when reading binary STL files with incomplete header.
var ErrIncompleteBinaryHeader = errors.New("incomplete STL binary header, 84 bytes expected")

// ErrUnexpectedEOF is used by ReadFile and ReadAll to signify an incomplete file.
var ErrUnexpectedEOF = errors.New("unexpected end of file")

// ReadFile reads the contents of a file into a new Solid object. The file
// can be either in STL ASCII format, beginning with "solid ", or in
// STL binary format, beginning with a 84 byte header. Shorthand for os.Open and ReadAll
func ReadFile(filename string) (solid *Solid, err error) {
	var s Solid
	err = CopyFile(filename, &s)
	if err == nil {
		solid = &s
	}
	return
}

// ReadAll reads the contents of a file into a new Solid object. The file
// can be either in STL ASCII format, beginning with "solid ", or in
// STL binary format, beginning with a 84 byte header. Because of this,
// the file pointer has to be at the beginning of the file.
func ReadAll(r io.ReadSeeker) (solid *Solid, err error) {
	var s Solid
	err = CopyAll(r, &s)
	if err == nil {
		solid = &s
	}
	return
}

func CopyFile(filename string, sw Writer) (err error) {
	file, openErr := os.Open(filename)
	if openErr != nil {
		err = openErr
		return
	}
	err = CopyAll(file, sw)
	closeErr := file.Close()
	if err == nil {
		err = closeErr
	}
	return
}

func CopyAll(r io.ReadSeeker, sw Writer) (err error) {
	isBinary, err := isBinaryFile(r)
	if err != nil {
		return
	}
	if _, err = r.Seek(0, io.SeekStart); err != nil {
		return
	}
	br := bufio.NewReader(r)

	if isBinary {
		sw.SetASCII(false)
		err = readAllBinary(br, sw)
	} else {
		sw.SetASCII(true)
		err = readAllASCII(br, sw)
	}

	return
}

// isBinaryFile returns true if the seekable stream tests as a binary file by
// matching triangle count (in header) and file size
func isBinaryFile(r io.ReadSeeker) (isBinary bool, err error) {
	var header [binaryHeaderSize]byte
	_, err = r.Read(header[:])
	if err != nil {
		if err == io.EOF { // too short to meet spec
			err = nil
		}
		return
	}
	triangleCount := triangleCountFromBinaryHeader(header[:])
	expectedFileLength := int64(triangleCount)*binaryTriangleSize + binaryHeaderSize
	actualFileLength, err := r.Seek(0, io.SeekEnd)
	if err != nil {
		return
	}
	isBinary = expectedFileLength == actualFileLength
	return
}

// WriteFile creates file with name filename and write contents of this Solid.
// Shorthand for os.Create and Solid.WriteAll
func (s *Solid) WriteFile(filename string) (err error) {
	file, err := os.Create(filename)
	if err != nil {
		return err
	}

	bufWriter := bufio.NewWriter(file)
	err = s.WriteAll(bufWriter)
	flushErr := bufWriter.Flush()
	closeErr := file.Close()
	if err == nil {
		err = flushErr
	}
	if err == nil {
		err = closeErr
	}
	return
}

// WriteAll writes the contents of this solid to an io.Writer. Depending on solid.IsAscii
// the STL ASCII format, or the STL binary format is used. If IsAscii
// is false, and the binary format is used, solid.Name will be used for
// the header, if solid.BinaryHeader is empty.
func (s *Solid) WriteAll(w io.Writer) error {
	if s.IsAscii {
		return writeSolidASCII(w, s)
	}
	return writeSolidBinary(w, s)
}

// Extracts an ASCII string from a byte slice. Reads all characters
// from the beginning until a \0 or a non-ASCII character is found.
func extractASCIIString(byteData []byte) string {
	i := 0
	for i < len(byteData) && byteData[i] < byte(128) && byteData[i] != byte(0) {
		i++
	}
	return string(byteData[0:i])
}
<|endoftext|>


<commit_before>package main

import (
	"archive/tar"
	"bytes"
	"fmt"
	"io"
	"log"
	"os"
	"path"
	"regexp"
	"strings"
	"time"

	"github.com/flynn/flynn/controller/client"
	ct "github.com/flynn/flynn/controller/types"
	"github.com/flynn/flynn/discoverd/client"
	"github.com/flynn/flynn/pkg/cluster"
	"github.com/flynn/flynn/pkg/exec"
	"github.com/flynn/flynn/pkg/random"
)

var clusterc *cluster.Client

func init() {
	log.SetFlags(0)

	var err error
	clusterc, err = cluster.NewClient()
	if err != nil {
		log.Fatalln("Error connecting to cluster leader:", err)
	}
}

var typesPattern = regexp.MustCompile("types.* -> (.+)\n")

func main() {
	client, err := controller.NewClient("", os.Getenv("CONTROLLER_AUTH_KEY"))
	if err != nil {
		log.Fatalln("Unable to connect to controller:", err)
	}
	// TODO: use discoverd http dialer here?
	services, err := discoverd.Services("blobstore", discoverd.DefaultTimeout)
	if err != nil || len(services) < 1 {
		log.Fatalf("Unable to discover blobstore %q", err)
	}
	blobstoreHost := services[0].Addr

	appName := os.Args[1]

	app, err := client.GetApp(appName)
	if err == controller.ErrNotFound {
		log.Fatalf("Unknown app %q", appName)
	} else if err != nil {
		log.Fatalln("Error retrieving app:", err)
	}
	prevRelease, err := client.GetAppRelease(app.Name)
	if err == controller.ErrNotFound {
		prevRelease = &ct.Release{}
	} else if err != nil {
		log.Fatalln("Error getting current app release:", err)
	}

	fmt.Printf("-----> Building %s...\n", app.Name)

	var output bytes.Buffer
	slugURL := fmt.Sprintf("http://%s/%s.tgz", blobstoreHost, random.UUID())
	cmd := exec.Command(exec.DockerImage("flynn/slugbuilder", os.Getenv("SLUGBUILDER_IMAGE_ID")), slugURL)
	cmd.Stdout = io.MultiWriter(os.Stdout, &output)
	cmd.Stderr = os.Stderr
	stdin, err := cmd.StdinPipe()
	if err != nil {
		log.Fatalln(err)
	}
	go appendEnvDir(os.Stdin, stdin, prevRelease.Env)
	if buildpackURL, ok := prevRelease.Env["BUILDPACK_URL"]; ok {
		cmd.Env = map[string]string{"BUILDPACK_URL": buildpackURL}
	}

	if err := cmd.Run(); err != nil {
		log.Fatalln("Build failed:", err)
	}

	var types []string
	if match := typesPattern.FindSubmatch(output.Bytes()); match != nil {
		types = strings.Split(string(match[1]), ", ")
	}

	fmt.Printf("-----> Creating release...\n")

	artifact := &ct.Artifact{Type: "docker", URI: "https://registry.hub.docker.com/flynn/slugrunner?id=" + os.Getenv("SLUGRUNNER_IMAGE_ID")}
	if err := client.CreateArtifact(artifact); err != nil {
		log.Fatalln("Error creating artifact:", err)
	}

	release := &ct.Release{
		ArtifactID: artifact.ID,
		Env:        prevRelease.Env,
	}
	procs := make(map[string]ct.ProcessType)
	for _, t := range types {
		proc := prevRelease.Processes[t]
		proc.Cmd = []string{"start", t}
		if t == "web" {
			proc.Ports = []ct.Port{{Proto: "tcp"}}
			if proc.Env == nil {
				proc.Env = make(map[string]string)
			}
			proc.Env["SD_NAME"] = app.Name + "-web"
		}
		procs[t] = proc
	}
	release.Processes = procs
	if release.Env == nil {
		release.Env = make(map[string]string)
	}
	release.Env["SLUG_URL"] = slugURL

	if err := client.CreateRelease(release); err != nil {
		log.Fatalln("Error creating release:", err)
	}
	if err := client.SetAppRelease(app.Name, release.ID); err != nil {
		log.Fatalln("Error setting app release:", err)
	}

	fmt.Println("=====> Application deployed")

	// If the app is new and the web process type exists,
	// it should scale to one process after the release is created.
	if _, ok := procs["web"]; ok && prevRelease.ID == "" {
		formation := &ct.Formation{
			AppID:     app.ID,
			ReleaseID: release.ID,
			Processes: map[string]int{"web": 1},
		}
		if err := client.PutFormation(formation); err != nil {
			log.Fatalln("Error putting formation:", err)
		}

		fmt.Println("=====> Added default web=1 formation")
	}
}

func appendEnvDir(stdin io.Reader, pipe io.WriteCloser, env map[string]string) {
	defer pipe.Close()
	tr := tar.NewReader(stdin)
	tw := tar.NewWriter(pipe)
	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			// end of tar archive
			break
		}
		if err != nil {
			log.Fatalln(err)
		}
		hdr.Name = path.Join("app", hdr.Name)
		if err := tw.WriteHeader(hdr); err != nil {
			log.Fatalln(err)
		}
		if _, err := io.Copy(tw, tr); err != nil {
			log.Fatalln(err)
		}
	}
	// append env dir
	for key, value := range env {
		hdr := &tar.Header{
			Name:    path.Join("env", key),
			Mode:    0400,
			ModTime: time.Now(),
			Size:    int64(len(value)),
		}

		if err := tw.WriteHeader(hdr); err != nil {
			log.Fatalln(err)
		}
		if _, err := tw.Write([]byte(value)); err != nil {
			log.Fatalln(err)
		}
	}
	hdr := &tar.Header{
		Name:    ".ENV_DIR_bdca46b87df0537eaefe79bb632d37709ff1df18",
		Mode:    0400,
		ModTime: time.Now(),
		Size:    0,
	}
	if err := tw.WriteHeader(hdr); err != nil {
		log.Fatalln(err)
	}
}
<commit_msg>receiver: Only append ENV dir if non-empty<commit_after>package main

import (
	"archive/tar"
	"bytes"
	"fmt"
	"io"
	"log"
	"os"
	"path"
	"regexp"
	"strings"
	"time"

	"github.com/flynn/flynn/controller/client"
	ct "github.com/flynn/flynn/controller/types"
	"github.com/flynn/flynn/discoverd/client"
	"github.com/flynn/flynn/pkg/cluster"
	"github.com/flynn/flynn/pkg/exec"
	"github.com/flynn/flynn/pkg/random"
)

var clusterc *cluster.Client

func init() {
	log.SetFlags(0)

	var err error
	clusterc, err = cluster.NewClient()
	if err != nil {
		log.Fatalln("Error connecting to cluster leader:", err)
	}
}

var typesPattern = regexp.MustCompile("types.* -> (.+)\n")

func main() {
	client, err := controller.NewClient("", os.Getenv("CONTROLLER_AUTH_KEY"))
	if err != nil {
		log.Fatalln("Unable to connect to controller:", err)
	}
	// TODO: use discoverd http dialer here?
	services, err := discoverd.Services("blobstore", discoverd.DefaultTimeout)
	if err != nil || len(services) < 1 {
		log.Fatalf("Unable to discover blobstore %q", err)
	}
	blobstoreHost := services[0].Addr

	appName := os.Args[1]

	app, err := client.GetApp(appName)
	if err == controller.ErrNotFound {
		log.Fatalf("Unknown app %q", appName)
	} else if err != nil {
		log.Fatalln("Error retrieving app:", err)
	}
	prevRelease, err := client.GetAppRelease(app.Name)
	if err == controller.ErrNotFound {
		prevRelease = &ct.Release{}
	} else if err != nil {
		log.Fatalln("Error getting current app release:", err)
	}

	fmt.Printf("-----> Building %s...\n", app.Name)

	var output bytes.Buffer
	slugURL := fmt.Sprintf("http://%s/%s.tgz", blobstoreHost, random.UUID())
	cmd := exec.Command(exec.DockerImage("flynn/slugbuilder", os.Getenv("SLUGBUILDER_IMAGE_ID")), slugURL)
	cmd.Stdout = io.MultiWriter(os.Stdout, &output)
	cmd.Stderr = os.Stderr
	if len(prevRelease.Env) > 0 {
		stdin, err := cmd.StdinPipe()
		if err != nil {
			log.Fatalln(err)
		}
		go appendEnvDir(os.Stdin, stdin, prevRelease.Env)
	} else {
		cmd.Stdin = os.Stdin
	}
	if buildpackURL, ok := prevRelease.Env["BUILDPACK_URL"]; ok {
		cmd.Env = map[string]string{"BUILDPACK_URL": buildpackURL}
	}

	if err := cmd.Run(); err != nil {
		log.Fatalln("Build failed:", err)
	}

	var types []string
	if match := typesPattern.FindSubmatch(output.Bytes()); match != nil {
		types = strings.Split(string(match[1]), ", ")
	}

	fmt.Printf("-----> Creating release...\n")

	artifact := &ct.Artifact{Type: "docker", URI: "https://registry.hub.docker.com/flynn/slugrunner?id=" + os.Getenv("SLUGRUNNER_IMAGE_ID")}
	if err := client.CreateArtifact(artifact); err != nil {
		log.Fatalln("Error creating artifact:", err)
	}

	release := &ct.Release{
		ArtifactID: artifact.ID,
		Env:        prevRelease.Env,
	}
	procs := make(map[string]ct.ProcessType)
	for _, t := range types {
		proc := prevRelease.Processes[t]
		proc.Cmd = []string{"start", t}
		if t == "web" {
			proc.Ports = []ct.Port{{Proto: "tcp"}}
			if proc.Env == nil {
				proc.Env = make(map[string]string)
			}
			proc.Env["SD_NAME"] = app.Name + "-web"
		}
		procs[t] = proc
	}
	release.Processes = procs
	if release.Env == nil {
		release.Env = make(map[string]string)
	}
	release.Env["SLUG_URL"] = slugURL

	if err := client.CreateRelease(release); err != nil {
		log.Fatalln("Error creating release:", err)
	}
	if err := client.SetAppRelease(app.Name, release.ID); err != nil {
		log.Fatalln("Error setting app release:", err)
	}

	fmt.Println("=====> Application deployed")

	// If the app is new and the web process type exists,
	// it should scale to one process after the release is created.
	if _, ok := procs["web"]; ok && prevRelease.ID == "" {
		formation := &ct.Formation{
			AppID:     app.ID,
			ReleaseID: release.ID,
			Processes: map[string]int{"web": 1},
		}
		if err := client.PutFormation(formation); err != nil {
			log.Fatalln("Error putting formation:", err)
		}

		fmt.Println("=====> Added default web=1 formation")
	}
}

func appendEnvDir(stdin io.Reader, pipe io.WriteCloser, env map[string]string) {
	defer pipe.Close()
	tr := tar.NewReader(stdin)
	tw := tar.NewWriter(pipe)
	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			// end of tar archive
			break
		}
		if err != nil {
			log.Fatalln(err)
		}
		hdr.Name = path.Join("app", hdr.Name)
		if err := tw.WriteHeader(hdr); err != nil {
			log.Fatalln(err)
		}
		if _, err := io.Copy(tw, tr); err != nil {
			log.Fatalln(err)
		}
	}
	// append env dir
	for key, value := range env {
		hdr := &tar.Header{
			Name:    path.Join("env", key),
			Mode:    0400,
			ModTime: time.Now(),
			Size:    int64(len(value)),
		}

		if err := tw.WriteHeader(hdr); err != nil {
			log.Fatalln(err)
		}
		if _, err := tw.Write([]byte(value)); err != nil {
			log.Fatalln(err)
		}
	}
	hdr := &tar.Header{
		Name:    ".ENV_DIR_bdca46b87df0537eaefe79bb632d37709ff1df18",
		Mode:    0400,
		ModTime: time.Now(),
		Size:    0,
	}
	if err := tw.WriteHeader(hdr); err != nil {
		log.Fatalln(err)
	}
}
<|endoftext|>


<commit_before>package anaconda_test

import (
	"net/url"
	"reflect"
	"testing"

	"github.com/ChimeraCoder/anaconda"
)

func TestOEmbed(t *testing.T) {
	// It is the only one that can be tested without auth
	api := anaconda.NewTwitterApi("", "")
	o, err := api.GetOEmbed(url.Values{"id": []string{"99530515043983360"}})
	if err != nil {
		t.Error(err)
	}

	if !reflect.DeepEqual(o, expectedOEmbed) {
		t.Error("Actual OEmbed differs from expected", o)
	}
}

var expectedOEmbed anaconda.OEmbed = anaconda.OEmbed{
	Cache_age:     "3153600000",
	Url:           "https://twitter.com/twitter/statuses/99530515043983360",
	Height:        0,
	Provider_url:  "https://twitter.com",
	Provider_name: "Twitter",
	Author_name:   "Twitter",
	Version:       "1.0",
	Author_url:    "https://twitter.com/twitter",
	Type:          "rich",
	Html:          "\u003Cblockquote class=\"twitter-tweet\"\u003E\u003Cp\u003ECool! \u201C\u003Ca href=\"https://twitter.com/tw1tt3rart\"\u003E@tw1tt3rart\u003C/a\u003E: \u003Ca href=\"https://twitter.com/hashtag/TWITTERART?src=hash\"\u003E#TWITTERART\u003C/a\u003E \u2571\u2571\u2571\u2571\u2571\u2571\u2571\u2571 \u2571\u2571\u256D\u2501\u2501\u2501\u2501\u256E\u2571\u2571\u256D\u2501\u2501\u2501\u2501\u256E \u2571\u2571\u2503\u2587\u2506\u2506\u2587\u2503\u2571\u256D\u252B\u24E6\u24D4\u24D4\u24DA\u2503 \u2571\u2571\u2503\u25BD\u25BD\u25BD\u25BD\u2503\u2501\u256F\u2503\u2661\u24D4\u24DD\u24D3\u2503 \u2571\u256D\u252B\u25B3\u25B3\u25B3\u25B3\u2523\u256E\u2571\u2570\u2501\u2501\u2501\u2501\u256F \u2571\u2503\u2503\u2506\u2506\u2506\u2506\u2503\u2503\u2571\u2571\u2571\u2571\u2571\u2571 \u2571\u2517\u252B\u2506\u250F\u2513\u2506\u2523\u251B\u2571\u2571\u2571\u2571\u2571\u201D\u003C/p\u003E— Twitter (@twitter) \u003Ca href=\"https://twitter.com/twitter/status/99530515043983360\"\u003EAugust 5, 2011\u003C/a\u003E\u003C/blockquote\u003E\n\u003Cscript async src=\"//platform.twitter.com/widgets.js\" charset=\"utf-8\"\u003E\u003C/script\u003E",
	Width:         550,
}
<commit_msg>test(TestOEmbed): fix test to make it pass<commit_after>package anaconda_test

import (
	"net/url"
	"reflect"
	"testing"

	"github.com/ChimeraCoder/anaconda"
)

func TestOEmbed(t *testing.T) {
	// It is the only one that can be tested without auth
	api := anaconda.NewTwitterApi("", "")
	o, err := api.GetOEmbed(url.Values{"id": []string{"99530515043983360"}})
	if err != nil {
		t.Error(err)
	}

	if !reflect.DeepEqual(o, expectedOEmbed) {
		t.Errorf("Actual OEmbed differs expected:\n%+v\n Got: \n%+v\n", expectedOEmbed, o)
	}
}

var expectedOEmbed anaconda.OEmbed = anaconda.OEmbed{
	Cache_age:     "3153600000",
	Url:           "https://twitter.com/twitter/statuses/99530515043983360",
	Height:        0,
	Provider_url:  "https://twitter.com",
	Provider_name: "Twitter",
	Author_name:   "Twitter",
	Version:       "1.0",
	Author_url:    "https://twitter.com/twitter",
	Type:          "rich",
	Html: `

Cool! “@tw1tt3rart: #TWITTERART ╱╱╱╱╱╱╱╱ ╱╱╭━━━━╮╱╱╭━━━━╮ ╱╱┃▇┆┆▇┃╱╭┫ⓦⓔⓔⓚ┃ ╱╱┃▽▽▽▽┃━╯┃♡ⓔⓝⓓ┃ ╱╭┫△△△△┣╮╱╰━━━━╯ ╱┃┃┆┆┆┆┃┃╱╱╱╱╱╱ ╱┗┫┆┏┓┆┣┛╱╱╱╱╱”
— Twitter (@twitter) August 5, 2011

`,
	Width: 550,
}
<|endoftext|>


<commit_before>package groupByNode

import (
	"fmt"
	"strings"

	"github.com/go-graphite/carbonapi/expr/consolidations"
	"github.com/go-graphite/carbonapi/expr/helper"
	"github.com/go-graphite/carbonapi/expr/interfaces"
	"github.com/go-graphite/carbonapi/expr/types"
	"github.com/go-graphite/carbonapi/pkg/parser"
)

type groupByNode struct {
	interfaces.FunctionBase
}

func GetOrder() interfaces.Order {
	return interfaces.Any
}

func New(configFile string) []interfaces.FunctionMetadata {
	res := make([]interfaces.FunctionMetadata, 0)
	f := &groupByNode{}
	functions := []string{"groupByNode", "groupByNodes"}
	for _, n := range functions {
		res = append(res, interfaces.FunctionMetadata{Name: n, F: f})
	}
	return res
}

// groupByNode(seriesList, nodeNum, callback)
// groupByNodes(seriesList, callback, *nodes)
func (f *groupByNode) Do(e parser.Expr, from, until int64, values map[parser.MetricRequest][]*types.MetricData) ([]*types.MetricData, error) {
	args, err := helper.GetSeriesArg(e.Args()[0], from, until, values)
	if err != nil {
		return nil, err
	}
	var callback string
	var fields []int

	if e.Target() == "groupByNode" {
		field, err := e.GetIntArg(1)
		if err != nil {
			return nil, err
		}

		callback, err = e.GetStringArg(2)
		if err != nil {
			return nil, err
		}
		fields = []int{field}
	} else {
		callback, err = e.GetStringArg(1)
		if err != nil {
			return nil, err
		}

		fields, err = e.GetIntArgs(2)
		if err != nil {
			return nil, err
		}
	}

	var results []*types.MetricData

	groups := make(map[string][]*types.MetricData)
	nodeList := []string{}

	for _, a := range args {

		metric := helper.ExtractMetric(a.Name)
		nodes := strings.Split(metric, ".")
		nodeKey := make([]string, 0, len(fields))
		for _, f := range fields {
			nodeKey = append(nodeKey, nodes[f])
		}
		node := strings.Join(nodeKey, ".")
		if len(groups[node]) == 0 {
			nodeList = append(nodeList, node)
		}

		groups[node] = append(groups[node], a)
	}

	for _, k := range nodeList {
		k := k // k's reference is used later, so it's important to make it unique per loop
		v := groups[k]

		// Ensure that names won't be parsed as consts, appending stub to them
		expr := fmt.Sprintf("%s(stub_%s)", callback, k)

		// create a stub context to evaluate the callback in
		nexpr, _, err := parser.ParseExpr(expr)
		// remove all stub_ prefixes we've prepended before
		nexpr.SetRawArgs(strings.Replace(nexpr.RawArgs(), "stub_", "", 1))
		for argIdx := range nexpr.Args() {
			nexpr.Args()[argIdx].SetTarget(strings.Replace(nexpr.Args()[0].Target(), "stub_", "", 1))
		}
		if err != nil {
			return nil, err
		}

		nvalues := values
		if e.Target() == "groupByNode" || e.Target() == "groupByNodes" {
			nvalues = map[parser.MetricRequest][]*types.MetricData{
				parser.MetricRequest{k, from, until}: v,
			}
		}

		r, _ := f.Evaluator.Eval(nexpr, from, until, nvalues)
		if r != nil {
			r[0].Name = k
			results = append(results, r...)
		}
	}

	return results, nil
}

// Description is auto-generated description, based on output of https://github.com/graphite-project/graphite-web
func (f *groupByNode) Description() map[string]types.FunctionDescription {
	return map[string]types.FunctionDescription{
		"groupByNode": {
			Description: "Takes a serieslist and maps a callback to subgroups within as defined by a common node\n\n.. code-block:: none\n\n  &target=groupByNode(ganglia.by-function.*.*.cpu.load5,2,\"sumSeries\")\n\nWould return multiple series which are each the result of applying the \"sumSeries\" function\nto groups joined on the second node (0 indexed) resulting in a list of targets like\n\n.. code-block :: none\n\n  sumSeries(ganglia.by-function.server1.*.cpu.load5),sumSeries(ganglia.by-function.server2.*.cpu.load5),...\n\nNode may be an integer referencing a node in the series name or a string identifying a tag.\n\nThis is an alias for using :py:func:`groupByNodes <groupByNodes>` with a single node.",
			Function:    "groupByNode(seriesList, nodeNum, callback='average')",
			Group:       "Combine",
			Module:      "graphite.render.functions",
			Name:        "groupByNode",
			Params: []types.FunctionParam{
				{
					Name:     "seriesList",
					Required: true,
					Type:     types.SeriesList,
				},
				{
					Name:     "nodeNum",
					Required: true,
					Type:     types.NodeOrTag,
				},
				{
					Default:  types.NewSuggestion("average"),
					Name:     "callback",
					Options:  consolidations.AvailableSummarizers,
					Required: true,
					Type:     types.AggFunc,
				},
			},
		},
		"groupByNodes": {
			Description: "Takes a serieslist and maps a callback to subgroups within as defined by multiple nodes\n\n.. code-block:: none\n\n  &target=groupByNodes(ganglia.server*.*.cpu.load*,\"sum\",1,4)\n\nWould return multiple series which are each the result of applying the \"sum\" aggregation\nto groups joined on the nodes' list (0 indexed) resulting in a list of targets like\n\n.. code-block :: none\n\n  sumSeries(ganglia.server1.*.cpu.load5),sumSeries(ganglia.server1.*.cpu.load10),sumSeries(ganglia.server1.*.cpu.load15),sumSeries(ganglia.server2.*.cpu.load5),sumSeries(ganglia.server2.*.cpu.load10),sumSeries(ganglia.server2.*.cpu.load15),...\n\nThis function can be used with all aggregation functions supported by\n:py:func:`aggregate <aggregate>`: ``average``, ``median``, ``sum``, ``min``, ``max``, ``diff``,\n``stddev``, ``range`` & ``multiply``.\n\nEach node may be an integer referencing a node in the series name or a string identifying a tag.\n\n.. code-block :: none\n\n  &target=seriesByTag(\"name=~cpu.load.*\", \"server=~server[1-9}+\", \"datacenter=~dc[1-9}+\")|groupByNodes(\"average\", \"datacenter\", 1)\n\n  # will produce output series like\n  # dc1.load5, dc2.load5, dc1.load10, dc2.load10\n\nThis complements :py:func:`aggregateWithWildcards <aggregateWithWildcards>` which takes a list of wildcard nodes.",
			Function:    "groupByNodes(seriesList, callback, *nodes)",
			Group:       "Combine",
			Module:      "graphite.render.functions",
			Name:        "groupByNodes",
			Params: []types.FunctionParam{
				{
					Name:     "seriesList",
					Required: true,
					Type:     types.SeriesList,
				},
				{
					Name:     "callback",
					Options:  consolidations.AvailableSummarizers,
					Required: true,
					Type:     types.AggFunc,
				},
				{
					Multiple: true,
					Name:     "nodes",
					Required: true,
					Type:     types.NodeOrTag,
				},
			},
		},
	}
}
<commit_msg>groupByNode: check for errors as soon as it's possible<commit_after>package groupByNode

import (
	"fmt"
	"strings"

	"github.com/go-graphite/carbonapi/expr/consolidations"
	"github.com/go-graphite/carbonapi/expr/helper"
	"github.com/go-graphite/carbonapi/expr/interfaces"
	"github.com/go-graphite/carbonapi/expr/types"
	"github.com/go-graphite/carbonapi/pkg/parser"
)

type groupByNode struct {
	interfaces.FunctionBase
}

func GetOrder() interfaces.Order {
	return interfaces.Any
}

func New(configFile string) []interfaces.FunctionMetadata {
	res := make([]interfaces.FunctionMetadata, 0)
	f := &groupByNode{}
	functions := []string{"groupByNode", "groupByNodes"}
	for _, n := range functions {
		res = append(res, interfaces.FunctionMetadata{Name: n, F: f})
	}
	return res
}

// groupByNode(seriesList, nodeNum, callback)
// groupByNodes(seriesList, callback, *nodes)
func (f *groupByNode) Do(e parser.Expr, from, until int64, values map[parser.MetricRequest][]*types.MetricData) ([]*types.MetricData, error) {
	args, err := helper.GetSeriesArg(e.Args()[0], from, until, values)
	if err != nil {
		return nil, err
	}
	var callback string
	var fields []int

	if e.Target() == "groupByNode" {
		field, err := e.GetIntArg(1)
		if err != nil {
			return nil, err
		}

		callback, err = e.GetStringArg(2)
		if err != nil {
			return nil, err
		}
		fields = []int{field}
	} else {
		callback, err = e.GetStringArg(1)
		if err != nil {
			return nil, err
		}

		fields, err = e.GetIntArgs(2)
		if err != nil {
			return nil, err
		}
	}

	var results []*types.MetricData

	groups := make(map[string][]*types.MetricData)
	nodeList := []string{}

	for _, a := range args {

		metric := helper.ExtractMetric(a.Name)
		nodes := strings.Split(metric, ".")
		nodeKey := make([]string, 0, len(fields))
		for _, f := range fields {
			nodeKey = append(nodeKey, nodes[f])
		}
		node := strings.Join(nodeKey, ".")
		if len(groups[node]) == 0 {
			nodeList = append(nodeList, node)
		}

		groups[node] = append(groups[node], a)
	}

	for _, k := range nodeList {
		k := k // k's reference is used later, so it's important to make it unique per loop
		v := groups[k]

		// Ensure that names won't be parsed as consts, appending stub to them
		expr := fmt.Sprintf("%s(stub_%s)", callback, k)

		// create a stub context to evaluate the callback in
		nexpr, _, err := parser.ParseExpr(expr)
		if err != nil {
			return nil, err
		}
		// remove all stub_ prefixes we've prepended before
		nexpr.SetRawArgs(strings.Replace(nexpr.RawArgs(), "stub_", "", 1))
		for argIdx := range nexpr.Args() {
			nexpr.Args()[argIdx].SetTarget(strings.Replace(nexpr.Args()[0].Target(), "stub_", "", 1))
		}

		nvalues := values
		if e.Target() == "groupByNode" || e.Target() == "groupByNodes" {
			nvalues = map[parser.MetricRequest][]*types.MetricData{
				parser.MetricRequest{k, from, until}: v,
			}
		}

		r, _ := f.Evaluator.Eval(nexpr, from, until, nvalues)
		if r != nil {
			r[0].Name = k
			results = append(results, r...)
		}
	}

	return results, nil
}

// Description is auto-generated description, based on output of https://github.com/graphite-project/graphite-web
func (f *groupByNode) Description() map[string]types.FunctionDescription {
	return map[string]types.FunctionDescription{
		"groupByNode": {
			Description: "Takes a serieslist and maps a callback to subgroups within as defined by a common node\n\n.. code-block:: none\n\n  &target=groupByNode(ganglia.by-function.*.*.cpu.load5,2,\"sumSeries\")\n\nWould return multiple series which are each the result of applying the \"sumSeries\" function\nto groups joined on the second node (0 indexed) resulting in a list of targets like\n\n.. code-block :: none\n\n  sumSeries(ganglia.by-function.server1.*.cpu.load5),sumSeries(ganglia.by-function.server2.*.cpu.load5),...\n\nNode may be an integer referencing a node in the series name or a string identifying a tag.\n\nThis is an alias for using :py:func:`groupByNodes <groupByNodes>` with a single node.",
			Function:    "groupByNode(seriesList, nodeNum, callback='average')",
			Group:       "Combine",
			Module:      "graphite.render.functions",
			Name:        "groupByNode",
			Params: []types.FunctionParam{
				{
					Name:     "seriesList",
					Required: true,
					Type:     types.SeriesList,
				},
				{
					Name:     "nodeNum",
					Required: true,
					Type:     types.NodeOrTag,
				},
				{
					Default:  types.NewSuggestion("average"),
					Name:     "callback",
					Options:  consolidations.AvailableSummarizers,
					Required: true,
					Type:     types.AggFunc,
				},
			},
		},
		"groupByNodes": {
			Description: "Takes a serieslist and maps a callback to subgroups within as defined by multiple nodes\n\n.. code-block:: none\n\n  &target=groupByNodes(ganglia.server*.*.cpu.load*,\"sum\",1,4)\n\nWould return multiple series which are each the result of applying the \"sum\" aggregation\nto groups joined on the nodes' list (0 indexed) resulting in a list of targets like\n\n.. code-block :: none\n\n  sumSeries(ganglia.server1.*.cpu.load5),sumSeries(ganglia.server1.*.cpu.load10),sumSeries(ganglia.server1.*.cpu.load15),sumSeries(ganglia.server2.*.cpu.load5),sumSeries(ganglia.server2.*.cpu.load10),sumSeries(ganglia.server2.*.cpu.load15),...\n\nThis function can be used with all aggregation functions supported by\n:py:func:`aggregate <aggregate>`: ``average``, ``median``, ``sum``, ``min``, ``max``, ``diff``,\n``stddev``, ``range`` & ``multiply``.\n\nEach node may be an integer referencing a node in the series name or a string identifying a tag.\n\n.. code-block :: none\n\n  &target=seriesByTag(\"name=~cpu.load.*\", \"server=~server[1-9}+\", \"datacenter=~dc[1-9}+\")|groupByNodes(\"average\", \"datacenter\", 1)\n\n  # will produce output series like\n  # dc1.load5, dc2.load5, dc1.load10, dc2.load10\n\nThis complements :py:func:`aggregateWithWildcards <aggregateWithWildcards>` which takes a list of wildcard nodes.",
			Function:    "groupByNodes(seriesList, callback, *nodes)",
			Group:       "Combine",
			Module:      "graphite.render.functions",
			Name:        "groupByNodes",
			Params: []types.FunctionParam{
				{
					Name:     "seriesList",
					Required: true,
					Type:     types.SeriesList,
				},
				{
					Name:     "callback",
					Options:  consolidations.AvailableSummarizers,
					Required: true,
					Type:     types.AggFunc,
				},
				{
					Multiple: true,
					Name:     "nodes",
					Required: true,
					Type:     types.NodeOrTag,
				},
			},
		},
	}
}
<|endoftext|>


<commit_before>package ooyalaV2SDK

import (
	"bytes"
	"crypto/sha256"
	"encoding/base64"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"sort"
	"strconv"
	"time"
)

// NewAPI Factory function returns a new api instance.
func NewAPI(apiKey, apiSecret string, expires int64) *OoyalaApi {
	api := OoyalaApi{}
	params := make(map[string]string)
	params["api_key"] = apiKey
	params["expires"] = fmt.Sprint(int64(time.Now().Unix() + expires))
	api.Params = params

	api.BaseURL = "https://api.ooyala.com"
	api.CacheBaseURL = "https://cdn-api.ooyala.com"
	api.Secret = apiSecret

	return &api
}

// OoyalaAPI implements the Ooyala Api
type OoyalaAPI struct {
	Params          map[string]string
	BaseURL         string
	CacheBaseURL    string
	UsedBaseURL     string
	Secret          string
	Body            string
	HTTPMethod      string
	RequestPath     string
	Response        string
	ResponseHeaders string
	Filter          string
	Signature       string
	FinalURL        string
}

// Request retry wrapper ensures success
// eliminating transient errors
func (a *OoyalaAPI) send() error {
	currentRetry := 0
	retryCount := 3

	for {
		err := a.sendRequest()
		if err != nil {
			currentRetry++
			if currentRetry > retryCount {
				return err
			}
			continue
		}
		return err
	}
}

func (a *OoyalaAPI) generateFinalURL() {
	if val, ok := a.Params["where"]; ok {
		a.FinalURL = fmt.Sprintf(
			"%s%s?api_key=%s&where=%s&signature=%s&expires=%s",
			a.UsedBaseURL,
			a.RequestPath,
			a.Params["api_key"],
			url.QueryEscape(val),
			a.Signature,
			a.Params["expires"],
		)
	} else {
		a.FinalURL = fmt.Sprintf(
			"%s%s?api_key=%s&signature=%s&expires=%s",
			a.UsedBaseURL,
			a.RequestPath,
			a.Params["api_key"],
			a.Signature,
			a.Params["expires"],
		)
	}

	for key, value := range a.Params {
		switch key {
		case "user_permission":
			a.FinalURL += fmt.Sprintf("&%s=%s", key, value)
		case "limit":
			a.FinalURL += fmt.Sprintf("&%s=%s", key, value)
		case "page_token":
			a.FinalURL += fmt.Sprintf("&%s=%s", key, value)
		case "include":
			a.FinalURL += fmt.Sprintf("&%s=%s", key, value)
		}
	}
}

// Send Http Request
func (a *OoyalaAPI) sendRequest() error {
	a.GenerateSignature()

	a.UsedBaseURL = a.CacheBaseURL

	if a.HTTPMethod != "GET" {
		a.UsedBaseURL = a.BaseURL
	}

	a.generateFinalURL()

	client := &http.Client{}
	request, err := http.NewRequest(a.HTTPMethod, a.FinalURL, bytes.NewReader([]byte(a.Body)))
	if err != nil {
		return err
	}
	request.Header.Add("Content-Length", strconv.Itoa(len(a.Body)))
	request.Header.Add("Content-Type", "application/json; charset=utf-8")

	response, err := client.Do(request)
	if err != nil {
		return err
	}

	defer response.Body.Close()

	switch response.StatusCode {
	case 200:
		contents, err := ioutil.ReadAll(response.Body)
		if err != nil {
			return err
		}
		a.Response = string(contents)

	case 204:
		return errors.New("NO CONTENT")
	case 400:
		return errors.New("BAD REQUEST")
	case 401:
		return errors.New("NOT AUTHORISED")
	case 403:
		return errors.New("FORBIDDEN")
	case 404:
		return errors.New("NOT FOUND")
	case 429:
		return errors.New("INSUFFICIENT API CREDITS")
	}
	return nil
}

// Get or view a resource
func (a *OoyalaAPI) Get() error {
	a.HTTPMethod = "GET"
	a.GenerateSignature()
	return a.send()
}

// Patch or update an existing resource
func (a *OoyalaAPI) Patch() error {
	a.HTTPMethod = "PATCH"
	if a.Body == "" {
		return errors.New("NO DATA TO UPDATE")
	}
	a.GenerateSignature()
	return a.send()
}

// Post or create a new resource
func (a *OoyalaAPI) Post() error {
	a.HTTPMethod = "POST"
	if a.Body == "" {
		return errors.New("NO NEW ASSET DATA")
	}
	a.GenerateSignature()
	// return a.send_request()
	return a.send()
}

// Put or replace an existing reource
func (a *OoyalaAPI) Put() error {
	a.HTTPMethod = "PUT"
	a.GenerateSignature()
	return a.send()
}

// Delete a resource
func (a *OoyalaAPI) Delete() error {
	a.HTTPMethod = "DELETE"
	a.GenerateSignature()
	return a.send()
}

// GenerateSignature Generates the signature for a request
func (a *OoyalaAPI) GenerateSignature() {
	signature := a.Secret + a.HTTPMethod + a.RequestPath
	hash := sha256.New()

	var keys []string

	for k := range a.Params {
		keys = append(keys, k)
	}

	sort.Strings(keys)

	for _, k := range keys {
		signature += k + "=" + a.Params[k]
	}

	signature += a.Body

	_, err := io.WriteString(hash, signature)
	if err != nil {
		log.Fatalln(err)
	}

	signature = base64.StdEncoding.EncodeToString(hash.Sum(nil))[0:43]
	signature = url.QueryEscape(signature)

	a.Signature = signature
}
<commit_msg>Update ooyalaV2SDK.go<commit_after>package ooyalaV2SDK

import (
	"bytes"
	"crypto/sha256"
	"encoding/base64"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"sort"
	"strconv"
	"time"
)

// NewAPI Factory function returns a new api instance.
func NewAPI(apiKey, apiSecret string, expires int64) *OoyalaAPI {
	api := OoyalaAPI{}
	params := make(map[string]string)
	params["api_key"] = apiKey
	params["expires"] = fmt.Sprint(int64(time.Now().Unix() + expires))
	api.Params = params

	api.BaseURL = "https://api.ooyala.com"
	api.CacheBaseURL = "https://cdn-api.ooyala.com"
	api.Secret = apiSecret

	return &api
}

// OoyalaAPI implements the Ooyala Api
type OoyalaAPI struct {
	Params          map[string]string
	BaseURL         string
	CacheBaseURL    string
	UsedBaseURL     string
	Secret          string
	Body            string
	HTTPMethod      string
	RequestPath     string
	Response        string
	ResponseHeaders string
	Filter          string
	Signature       string
	FinalURL        string
}

// Request retry wrapper ensures success
// eliminating transient errors
func (a *OoyalaAPI) send() error {
	currentRetry := 0
	retryCount := 3

	for {
		err := a.sendRequest()
		if err != nil {
			currentRetry++
			if currentRetry > retryCount {
				return err
			}
			continue
		}
		return err
	}
}

func (a *OoyalaAPI) generateFinalURL() {
	if val, ok := a.Params["where"]; ok {
		a.FinalURL = fmt.Sprintf(
			"%s%s?api_key=%s&where=%s&signature=%s&expires=%s",
			a.UsedBaseURL,
			a.RequestPath,
			a.Params["api_key"],
			url.QueryEscape(val),
			a.Signature,
			a.Params["expires"],
		)
	} else {
		a.FinalURL = fmt.Sprintf(
			"%s%s?api_key=%s&signature=%s&expires=%s",
			a.UsedBaseURL,
			a.RequestPath,
			a.Params["api_key"],
			a.Signature,
			a.Params["expires"],
		)
	}

	for key, value := range a.Params {
		switch key {
		case "user_permission":
			a.FinalURL += fmt.Sprintf("&%s=%s", key, value)
		case "limit":
			a.FinalURL += fmt.Sprintf("&%s=%s", key, value)
		case "page_token":
			a.FinalURL += fmt.Sprintf("&%s=%s", key, value)
		case "include":
			a.FinalURL += fmt.Sprintf("&%s=%s", key, value)
		}
	}
}

// Send Http Request
func (a *OoyalaAPI) sendRequest() error {
	a.GenerateSignature()

	a.UsedBaseURL = a.CacheBaseURL

	if a.HTTPMethod != "GET" {
		a.UsedBaseURL = a.BaseURL
	}

	a.generateFinalURL()

	client := &http.Client{}
	request, err := http.NewRequest(a.HTTPMethod, a.FinalURL, bytes.NewReader([]byte(a.Body)))
	if err != nil {
		return err
	}
	request.Header.Add("Content-Length", strconv.Itoa(len(a.Body)))
	request.Header.Add("Content-Type", "application/json; charset=utf-8")

	response, err := client.Do(request)
	if err != nil {
		return err
	}

	defer response.Body.Close()

	switch response.StatusCode {
	case 200:
		contents, err := ioutil.ReadAll(response.Body)
		if err != nil {
			return err
		}
		a.Response = string(contents)

	case 204:
		return errors.New("NO CONTENT")
	case 400:
		return errors.New("BAD REQUEST")
	case 401:
		return errors.New("NOT AUTHORISED")
	case 403:
		return errors.New("FORBIDDEN")
	case 404:
		return errors.New("NOT FOUND")
	case 429:
		return errors.New("INSUFFICIENT API CREDITS")
	}
	return nil
}

// Get or view a resource
func (a *OoyalaAPI) Get() error {
	a.HTTPMethod = "GET"
	a.GenerateSignature()
	return a.send()
}

// Patch or update an existing resource
func (a *OoyalaAPI) Patch() error {
	a.HTTPMethod = "PATCH"
	if a.Body == "" {
		return errors.New("NO DATA TO UPDATE")
	}
	a.GenerateSignature()
	return a.send()
}

// Post or create a new resource
func (a *OoyalaAPI) Post() error {
	a.HTTPMethod = "POST"
	if a.Body == "" {
		return errors.New("NO NEW ASSET DATA")
	}
	a.GenerateSignature()
	// return a.send_request()
	return a.send()
}

// Put or replace an existing reource
func (a *OoyalaAPI) Put() error {
	a.HTTPMethod = "PUT"
	a.GenerateSignature()
	return a.send()
}

// Delete a resource
func (a *OoyalaAPI) Delete() error {
	a.HTTPMethod = "DELETE"
	a.GenerateSignature()
	return a.send()
}

// GenerateSignature Generates the signature for a request
func (a *OoyalaAPI) GenerateSignature() {
	signature := a.Secret + a.HTTPMethod + a.RequestPath
	hash := sha256.New()

	var keys []string

	for k := range a.Params {
		keys = append(keys, k)
	}

	sort.Strings(keys)

	for _, k := range keys {
		signature += k + "=" + a.Params[k]
	}

	signature += a.Body

	_, err := io.WriteString(hash, signature)
	if err != nil {
		log.Fatalln(err)
	}

	signature = base64.StdEncoding.EncodeToString(hash.Sum(nil))[0:43]
	signature = url.QueryEscape(signature)

	a.Signature = signature
}
<|endoftext|>


<commit_before>package eccore

import (
	"database/sql"
	"fmt"
	"log"
)

type dbOrderStore struct {
	db     *sql.DB
	static StaticItems
}

type DBOrderStore interface {
	UpdateOrders(region Region, mt MarketType, orders []MarketOrder) error
}

func NewOrderStore(db *sql.DB, static StaticItems) (DBOrderStore, error) {
	log.Println("Building new order store")
	return &dbOrderStore{db: db, static: static}, nil
}

func (d *dbOrderStore) UpdateOrders(region Region, mt MarketType, orders []MarketOrder) error {
	tx, err := d.db.Begin()
	if err != nil {
		return err
	}

	_, err = tx.Exec("DELETE FROM current_market WHERE regionid = $1 AND typeid = $2", region.Id, mt.Id)
	if err != nil {
		tx.Rollback()
		log.Printf("Orders can't delete due to %s", err)
		return err
	}

	insCurrent, err := tx.Prepare(`
INSERT INTO current_market
(regionid, systemid, stationid, typeid, bid, price, orderid,
minvolume, volremain, volenter, issued, duration, range, reportedby, reportedtime)
VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, CAST ($12 AS INTERVAL), $13, 0, $14)`)

	if err != nil {
		tx.Rollback()
		log.Println(err)
		return err
	}

	insArchive, err := tx.Prepare(`
INSERT INTO archive_market
(regionid, systemid, stationid, typeid, bid, price, orderid,
minvolume, volremain, volenter, issued, duration, range, reportedby, source)
VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, CAST ($12 AS INTERVAL), $13, 0, 'evec_upload_cqache')`)

	if err != nil {
		tx.Rollback()
		log.Println(err)
		return err
	}

	for _, order := range orders {
		duration := fmt.Sprintf("%d", order.Expires.Hours()/24)
		_, err = insCurrent.Exec(region.Id, order.Station.SolarSystem.Id, order.Station.Id,
			mt.Id, order.Bid, order.Price, order.MinVolume, order.VolRemain, order.VolEnter,
			order.Issued, duration, order.Range, order.ReportedAt)
		if err != nil {
			tx.Rollback()
			log.Println(err)
			return err
		}
		_, err = insArchive.Exec(region.Id, order.Station.SolarSystem.Id, order.Station.Id,
			mt.Id, order.Bid, order.Price, order.MinVolume, order.VolRemain, order.VolEnter,
			order.Issued, duration, order.Range)

		if err != nil {
			tx.Rollback()
			log.Println(err)
			return err
		}
	}

	insArchive.Close()
	insCurrent.Close()
	err = tx.Commit()
	if err != nil {
		log.Println(err)
	}
	return err
}
<commit_msg>DB bug fixes<commit_after>package eccore

import (
	"database/sql"
	"fmt"
	"log"
)

type dbOrderStore struct {
	db     *sql.DB
	static StaticItems
}

type DBOrderStore interface {
	UpdateOrders(region Region, mt MarketType, orders []MarketOrder) error
}

func NewOrderStore(db *sql.DB, static StaticItems) (DBOrderStore, error) {
	log.Println("Building new order store")
	return &dbOrderStore{db: db, static: static}, nil
}

func (d *dbOrderStore) UpdateOrders(region Region, mt MarketType, orders []MarketOrder) error {
	tx, err := d.db.Begin()
	if err != nil {
		return err
	}

	_, err = tx.Exec("DELETE FROM current_market WHERE regionid = $1 AND typeid = $2", region.Id, mt.Id)
	if err != nil {
		tx.Rollback()
		log.Printf("Orders can't delete due to %s", err)
		return err
	}

	insCurrent, err := tx.Prepare(`
INSERT INTO current_market
(regionid, systemid, stationid, typeid, bid, price, orderid,
minvolume, volremain, volenter, issued, duration, range, reportedby, reportedtime)
VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, CAST ($12 AS INTERVAL), $13, 0, $14)`)

	if err != nil {
		tx.Rollback()
		log.Println(err)
		return err
	}

	insArchive, err := tx.Prepare(`
INSERT INTO archive_market
(regionid, systemid, stationid, typeid, bid, price, orderid,
minvolume, volremain, volenter, issued, duration, range, reportedby, source)
VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, CAST ($12 AS INTERVAL), $13, 0, 'evec_upload_cqache')`)

	if err != nil {
		tx.Rollback()
		log.Println(err)
		return err
	}

	for _, order := range orders {
		duration := fmt.Sprintf("%d", int(order.Expires.Hours()/24))
		// Historically, this is an integer column. welp.
		bid := 0
		if order.Bid {
			bid = 1
		}
		_, err = insCurrent.Exec(region.Id, order.Station.SolarSystem.Id, order.Station.Id,
			mt.Id, bid, order.Price, order.OrderId, order.MinVolume, order.VolRemain, order.VolEnter,
			order.Issued, duration, order.Range, order.ReportedAt)
		if err != nil {
			tx.Rollback()
			log.Println(err)
			return err
		}
		_, err = insArchive.Exec(region.Id, order.Station.SolarSystem.Id, order.Station.Id,
			mt.Id, bid, order.Price, order.OrderId, order.MinVolume, order.VolRemain, order.VolEnter,
			order.Issued, duration, order.Range)

		if err != nil {
			tx.Rollback()
			log.Println(err)
			return err
		}
	}

	insArchive.Close()
	insCurrent.Close()
	err = tx.Commit()
	if err != nil {
		log.Println(err)
	}
	return err
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"os"
	"math/big"
	"crypto/rand"
	"fmt"
	"log"
	"math"
)

var help = flag.Bool("h", false, "Help")
var rand32 = flag.Bool("rand32", false, "Print a 32 bit integer random number to stdout")

func main() {
	flag.Parse()
	if (*help) {
		flag.Usage()
		os.Exit(1)
	}

	if *rand32 {
		processRand32()
	}

	os.Exit(0)
}

func processRand32() {
	val, err := rand.Int(rand.Reader, big.NewInt(math.MaxInt32))
	if err != nil {
		log.Fatalf("Error creating a random 32 bit number. %v", err)
	}
	fmt.Fprintf(os.Stdout, "%v", int32(val.Int64()))
}
<commit_msg>now32<commit_after>package main

import (
	"flag"
	"os"
	"math/big"
	"crypto/rand"
	"fmt"
	"log"
	"math"
	"time"
)

var help = flag.Bool("h", false, "Help")
var rand32 = flag.Bool("rand32", false, "Print a 32 bit integer random number to stdout")
var now32 = flag.Bool("now32", false, "Print the 32 bits of time.Now().Unixnano to stdout")

func main() {
	flag.Parse()
	if (*help) {
		flag.Usage()
		os.Exit(1)
	}

	if *rand32 {
		processRand32()
	}
	if *now32 {
		processNow32()
	}

	os.Exit(0)
}

func processRand32() {
	val, err := rand.Int(rand.Reader, big.NewInt(math.MaxInt32))
	if err != nil {
		log.Fatalf("Error creating a random 32 bit number. %v", err)
	}
	fmt.Fprintf(os.Stdout, "%v", int32(val.Int64()))
}

func processNow32() {
	now := time.Now().UnixNano()
	fmt.Fprintf(os.Stdout, "%v", int32(now >> 32))
}
<|endoftext|>


<commit_before>// +build testtools

package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"strconv"
	"time"

	"github.com/github/git-lfs/api"
	"github.com/github/git-lfs/httputil"
	"github.com/github/git-lfs/progress"
	"github.com/github/git-lfs/tools"
)

// This test custom adapter just acts as a bridge for uploads/downloads
// in order to demonstrate & test the custom transfer adapter protocols
// All we actually do is relay the requests back to the normal storage URLs
// of our test server for simplicity, but this proves the principle
func main() {

	scanner := bufio.NewScanner(os.Stdin)
	writer := bufio.NewWriter(os.Stdout)
	errWriter := bufio.NewWriter(os.Stderr)

	for scanner.Scan() {
		line := scanner.Text()
		var req request
		if err := json.Unmarshal([]byte(line), &req); err != nil {
			errWriter.WriteString(fmt.Sprintf("Unable to parse request: %v\n", line))
			continue
		}

		switch req.Id {
		case "init":
			errWriter.WriteString(fmt.Sprintf("Initialised test custom adapter for %s\n", req.Operation))
			resp := &initResponse{}
			sendResponse(resp, writer)
		case "download":
			errWriter.WriteString(fmt.Sprintf("Received download request for %s\n", req.Oid))
			performDownload(req.Oid, req.Size, req.Action, writer, errWriter)
		case "upload":
			errWriter.WriteString(fmt.Sprintf("Received upload request for %s\n", req.Oid))
			performUpload(req.Oid, req.Size, req.Action, req.Path, writer, errWriter)
		case "terminate":
			errWriter.WriteString("Terminating test custom adapter gracefully.\n")
			break
		}
	}

}

func sendResponse(r interface{}, writer *bufio.Writer) error {
	b, err := json.Marshal(r)
	if err != nil {
		return err
	}
	// Line oriented JSON
	b = append(b, '\n')
	_, err = writer.Write(b)
	if err != nil {
		return err
	}
	writer.Flush()
	return nil
}

func sendTransferError(oid string, code int, message string, writer *bufio.Writer, errWriter *bufio.Writer) {
	resp := &transferResponse{"complete", oid, "", &transferError{code, message}}
	err := sendResponse(resp, writer)
	if err != nil {
		errWriter.WriteString(fmt.Sprintf("Unable to send transfer error: %v", err))
	}
}

func sendProgress(oid string, bytesSoFar int64, bytesSinceLast int, writer *bufio.Writer, errWriter *bufio.Writer) {
	resp := &progressResponse{"progress", oid, bytesSoFar, bytesSinceLast}
	err := sendResponse(resp, writer)
	if err != nil {
		errWriter.WriteString(fmt.Sprintf("Unable to send progress update: %v", err))
	}
}

func performDownload(oid string, size int64, a *action, writer *bufio.Writer, errWriter *bufio.Writer) {
	// We just use the URLs we're given, so we're just a proxy for the direct method
	// but this is enough to test intermediate custom adapters
	req, err := httputil.NewHttpRequest("GET", a.Href, a.Header)
	if err != nil {
		sendTransferError(oid, 2, err.Error(), writer, errWriter)
		return
	}
	res, err := httputil.DoHttpRequest(req, true)
	if err != nil {
		sendTransferError(oid, res.StatusCode, err.Error(), writer, errWriter)
		return
	}
	defer res.Body.Close()

	dlFile, err := ioutil.TempFile("", "lfscustomdl")
	if err != nil {
		sendTransferError(oid, 3, err.Error(), writer, errWriter)
		return
	}
	defer dlFile.Close()
	dlfilename := dlFile.Name()
	// Wrap callback to give name context
	cb := func(totalSize int64, readSoFar int64, readSinceLast int) error {
		sendProgress(oid, readSoFar, readSinceLast, writer, errWriter)
		return nil
	}
	_, err = tools.CopyWithCallback(dlFile, res.Body, res.ContentLength, cb)
	if err != nil {
		sendTransferError(oid, 4, fmt.Sprintf("cannot write data to tempfile %q: %v", dlfilename, err), writer, errWriter)
		os.Remove(dlfilename)
		return
	}
	if err := dlFile.Close(); err != nil {
		sendTransferError(oid, 5, fmt.Sprintf("can't close tempfile %q: %v", dlfilename, err), writer, errWriter)
		os.Remove(dlfilename)
		return
	}

	// completed
	complete := &transferResponse{"complete", oid, dlfilename, nil}
	err = sendResponse(complete, writer)
	if err != nil {
		errWriter.WriteString(fmt.Sprintf("Unable to send transfer error: %v", err))
	}
}

func performUpload(oid string, size int64, a *action, fromPath string, writer *bufio.Writer, errWriter *bufio.Writer) {
	// We just use the URLs we're given, so we're just a proxy for the direct method
	// but this is enough to test intermediate custom adapters
	req, err := httputil.NewHttpRequest("PUT", a.Href, a.Header)
	if err != nil {
		sendTransferError(oid, 2, err.Error(), writer, errWriter)
		return
	}

	if len(req.Header.Get("Content-Type")) == 0 {
		req.Header.Set("Content-Type", "application/octet-stream")
	}

	if req.Header.Get("Transfer-Encoding") == "chunked" {
		req.TransferEncoding = []string{"chunked"}
	} else {
		req.Header.Set("Content-Length", strconv.FormatInt(size, 10))
	}

	req.ContentLength = size

	f, err := os.OpenFile(fromPath, os.O_RDONLY, 0644)
	if err != nil {
		sendTransferError(oid, 3, fmt.Sprintf("Cannot read data from %q: %v", fromPath, err), writer, errWriter)
		return
	}
	defer f.Close()

	// Ensure progress callbacks made while uploading
	// Wrap callback to give name context
	cb := func(totalSize int64, readSoFar int64, readSinceLast int) error {
		sendProgress(oid, readSoFar, readSinceLast, writer, errWriter)
		return nil
	}
	var reader io.Reader
	reader = &progress.CallbackReader{
		C:         cb,
		TotalSize: size,
		Reader:    f,
	}

	req.Body = ioutil.NopCloser(reader)

	res, err := httputil.DoHttpRequest(req, true)
	if err != nil {
		sendTransferError(oid, res.StatusCode, fmt.Sprintf("Error uploading data for %s: %v", oid, err), writer, errWriter)
		return
	}

	if res.StatusCode > 299 {
		sendTransferError(oid, res.StatusCode, fmt.Sprintf("Invalid status for %s: %d", httputil.TraceHttpReq(req), res.StatusCode), writer, errWriter)
		return
	}

	io.Copy(ioutil.Discard, res.Body)
	res.Body.Close()

}

// Structs reimplemented so closer to a real external implementation
type header struct {
	Key   string `json:"key"`
	Value string `json:"value"`
}
type action struct {
	Href      string            `json:"href"`
	Header    map[string]string `json:"header,omitempty"`
	ExpiresAt time.Time         `json:"expires_at,omitempty"`
}
type transferError struct {
	Code    int    `json:"code"`
	Message string `json:"message"`
}

// Combined request struct which can accept anything
type request struct {
	Id                  string  `json:"id"`
	Operation           string  `json:"operation"`
	Concurrent          bool    `json:"concurrent"`
	ConcurrentTransfers int     `json:"concurrenttransfers"`
	Oid                 string  `json:"oid"`
	Size                int64   `json:"size"`
	Path                string  `json:"path"`
	Action              *action `json:"action"`
}

type initResponse struct {
	Error *api.ObjectError `json:"error,omitempty"`
}
type transferResponse struct {
	Id    string         `json:"id"`
	Oid   string         `json:"oid"`
	Path  string         `json:"path,omitempty"` // always blank for upload
	Error *transferError `json:"error,omitempty"`
}
type progressResponse struct {
	Id             string `json:"id"`
	Oid            string `json:"oid"`
	BytesSoFar     int64  `json:"bytesSoFar"`
	BytesSinceLast int    `json:"bytesSinceLast"`
}
<commit_msg>Tidy up signatures<commit_after>// +build testtools

package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"strconv"
	"time"

	"github.com/github/git-lfs/api"
	"github.com/github/git-lfs/httputil"
	"github.com/github/git-lfs/progress"
	"github.com/github/git-lfs/tools"
)

// This test custom adapter just acts as a bridge for uploads/downloads
// in order to demonstrate & test the custom transfer adapter protocols
// All we actually do is relay the requests back to the normal storage URLs
// of our test server for simplicity, but this proves the principle
func main() {

	scanner := bufio.NewScanner(os.Stdin)
	writer := bufio.NewWriter(os.Stdout)
	errWriter := bufio.NewWriter(os.Stderr)

	for scanner.Scan() {
		line := scanner.Text()
		var req request
		if err := json.Unmarshal([]byte(line), &req); err != nil {
			errWriter.WriteString(fmt.Sprintf("Unable to parse request: %v\n", line))
			continue
		}

		switch req.Id {
		case "init":
			errWriter.WriteString(fmt.Sprintf("Initialised test custom adapter for %s\n", req.Operation))
			resp := &initResponse{}
			sendResponse(resp, writer)
		case "download":
			errWriter.WriteString(fmt.Sprintf("Received download request for %s\n", req.Oid))
			performDownload(req.Oid, req.Size, req.Action, writer, errWriter)
		case "upload":
			errWriter.WriteString(fmt.Sprintf("Received upload request for %s\n", req.Oid))
			performUpload(req.Oid, req.Size, req.Action, req.Path, writer, errWriter)
		case "terminate":
			errWriter.WriteString("Terminating test custom adapter gracefully.\n")
			break
		}
	}

}

func sendResponse(r interface{}, writer *bufio.Writer) error {
	b, err := json.Marshal(r)
	if err != nil {
		return err
	}
	// Line oriented JSON
	b = append(b, '\n')
	_, err = writer.Write(b)
	if err != nil {
		return err
	}
	writer.Flush()
	return nil
}

func sendTransferError(oid string, code int, message string, writer, errWriter *bufio.Writer) {
	resp := &transferResponse{"complete", oid, "", &transferError{code, message}}
	err := sendResponse(resp, writer)
	if err != nil {
		errWriter.WriteString(fmt.Sprintf("Unable to send transfer error: %v", err))
	}
}

func sendProgress(oid string, bytesSoFar int64, bytesSinceLast int, writer, errWriter *bufio.Writer) {
	resp := &progressResponse{"progress", oid, bytesSoFar, bytesSinceLast}
	err := sendResponse(resp, writer)
	if err != nil {
		errWriter.WriteString(fmt.Sprintf("Unable to send progress update: %v", err))
	}
}

func performDownload(oid string, size int64, a *action, writer, errWriter *bufio.Writer) {
	// We just use the URLs we're given, so we're just a proxy for the direct method
	// but this is enough to test intermediate custom adapters
	req, err := httputil.NewHttpRequest("GET", a.Href, a.Header)
	if err != nil {
		sendTransferError(oid, 2, err.Error(), writer, errWriter)
		return
	}
	res, err := httputil.DoHttpRequest(req, true)
	if err != nil {
		sendTransferError(oid, res.StatusCode, err.Error(), writer, errWriter)
		return
	}
	defer res.Body.Close()

	dlFile, err := ioutil.TempFile("", "lfscustomdl")
	if err != nil {
		sendTransferError(oid, 3, err.Error(), writer, errWriter)
		return
	}
	defer dlFile.Close()
	dlfilename := dlFile.Name()
	// Wrap callback to give name context
	cb := func(totalSize int64, readSoFar int64, readSinceLast int) error {
		sendProgress(oid, readSoFar, readSinceLast, writer, errWriter)
		return nil
	}
	_, err = tools.CopyWithCallback(dlFile, res.Body, res.ContentLength, cb)
	if err != nil {
		sendTransferError(oid, 4, fmt.Sprintf("cannot write data to tempfile %q: %v", dlfilename, err), writer, errWriter)
		os.Remove(dlfilename)
		return
	}
	if err := dlFile.Close(); err != nil {
		sendTransferError(oid, 5, fmt.Sprintf("can't close tempfile %q: %v", dlfilename, err), writer, errWriter)
		os.Remove(dlfilename)
		return
	}

	// completed
	complete := &transferResponse{"complete", oid, dlfilename, nil}
	err = sendResponse(complete, writer)
	if err != nil {
		errWriter.WriteString(fmt.Sprintf("Unable to send transfer error: %v", err))
	}
}

func performUpload(oid string, size int64, a *action, fromPath string, writer, errWriter *bufio.Writer) {
	// We just use the URLs we're given, so we're just a proxy for the direct method
	// but this is enough to test intermediate custom adapters
	req, err := httputil.NewHttpRequest("PUT", a.Href, a.Header)
	if err != nil {
		sendTransferError(oid, 2, err.Error(), writer, errWriter)
		return
	}

	if len(req.Header.Get("Content-Type")) == 0 {
		req.Header.Set("Content-Type", "application/octet-stream")
	}

	if req.Header.Get("Transfer-Encoding") == "chunked" {
		req.TransferEncoding = []string{"chunked"}
	} else {
		req.Header.Set("Content-Length", strconv.FormatInt(size, 10))
	}

	req.ContentLength = size

	f, err := os.OpenFile(fromPath, os.O_RDONLY, 0644)
	if err != nil {
		sendTransferError(oid, 3, fmt.Sprintf("Cannot read data from %q: %v", fromPath, err), writer, errWriter)
		return
	}
	defer f.Close()

	// Ensure progress callbacks made while uploading
	// Wrap callback to give name context
	cb := func(totalSize int64, readSoFar int64, readSinceLast int) error {
		sendProgress(oid, readSoFar, readSinceLast, writer, errWriter)
		return nil
	}
	var reader io.Reader
	reader = &progress.CallbackReader{
		C:         cb,
		TotalSize: size,
		Reader:    f,
	}

	req.Body = ioutil.NopCloser(reader)

	res, err := httputil.DoHttpRequest(req, true)
	if err != nil {
		sendTransferError(oid, res.StatusCode, fmt.Sprintf("Error uploading data for %s: %v", oid, err), writer, errWriter)
		return
	}

	if res.StatusCode > 299 {
		sendTransferError(oid, res.StatusCode, fmt.Sprintf("Invalid status for %s: %d", httputil.TraceHttpReq(req), res.StatusCode), writer, errWriter)
		return
	}

	io.Copy(ioutil.Discard, res.Body)
	res.Body.Close()

}

// Structs reimplemented so closer to a real external implementation
type header struct {
	Key   string `json:"key"`
	Value string `json:"value"`
}
type action struct {
	Href      string            `json:"href"`
	Header    map[string]string `json:"header,omitempty"`
	ExpiresAt time.Time         `json:"expires_at,omitempty"`
}
type transferError struct {
	Code    int    `json:"code"`
	Message string `json:"message"`
}

// Combined request struct which can accept anything
type request struct {
	Id                  string  `json:"id"`
	Operation           string  `json:"operation"`
	Concurrent          bool    `json:"concurrent"`
	ConcurrentTransfers int     `json:"concurrenttransfers"`
	Oid                 string  `json:"oid"`
	Size                int64   `json:"size"`
	Path                string  `json:"path"`
	Action              *action `json:"action"`
}

type initResponse struct {
	Error *api.ObjectError `json:"error,omitempty"`
}
type transferResponse struct {
	Id    string         `json:"id"`
	Oid   string         `json:"oid"`
	Path  string         `json:"path,omitempty"` // always blank for upload
	Error *transferError `json:"error,omitempty"`
}
type progressResponse struct {
	Id             string `json:"id"`
	Oid            string `json:"oid"`
	BytesSoFar     int64  `json:"bytesSoFar"`
	BytesSinceLast int    `json:"bytesSinceLast"`
}
<|endoftext|>