<commit_before>package lfs

import (
	"testing"

	"github.com/github/git-lfs/vendor/_nuts/github.com/technoweenie/assert"
)

func TestEndpointDefaultsToOrigin(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{"remote.origin.lfsurl": "abc"},
		remotes:   []string{},
	}

	endpoint := config.Endpoint()
	assert.Equal(t, "abc", endpoint.Url)
	assert.Equal(t, "", endpoint.SshUserAndHost)
	assert.Equal(t, "", endpoint.SshPath)
}

func TestEndpointOverridesOrigin(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{
			"lfs.url":              "abc",
			"remote.origin.lfsurl": "def",
		},
		remotes: []string{},
	}

	endpoint := config.Endpoint()
	assert.Equal(t, "abc", endpoint.Url)
	assert.Equal(t, "", endpoint.SshUserAndHost)
	assert.Equal(t, "", endpoint.SshPath)
}

func TestEndpointNoOverrideDefaultRemote(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{
			"remote.origin.lfsurl": "abc",
			"remote.other.lfsurl":  "def",
		},
		remotes: []string{},
	}

	endpoint := config.Endpoint()
	assert.Equal(t, "abc", endpoint.Url)
	assert.Equal(t, "", endpoint.SshUserAndHost)
	assert.Equal(t, "", endpoint.SshPath)
}

func TestEndpointUseAlternateRemote(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{
			"remote.origin.lfsurl": "abc",
			"remote.other.lfsurl":  "def",
		},
		remotes: []string{},
	}

	config.CurrentRemote = "other"

	endpoint := config.Endpoint()
	assert.Equal(t, "def", endpoint.Url)
	assert.Equal(t, "", endpoint.SshUserAndHost)
	assert.Equal(t, "", endpoint.SshPath)
}

func TestEndpointAddsLfsSuffix(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{"remote.origin.url": "https://example.com/foo/bar"},
		remotes:   []string{},
	}

	endpoint := config.Endpoint()
	assert.Equal(t, "https://example.com/foo/bar.git/info/lfs", endpoint.Url)
	assert.Equal(t, "", endpoint.SshUserAndHost)
	assert.Equal(t, "", endpoint.SshPath)
}

func TestBareEndpointAddsLfsSuffix(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{"remote.origin.url": "https://example.com/foo/bar.git"},
		remotes:   []string{},
	}

	endpoint := config.Endpoint()
	assert.Equal(t, "https://example.com/foo/bar.git/info/lfs", endpoint.Url)
	assert.Equal(t, "", endpoint.SshUserAndHost)
	assert.Equal(t, "", endpoint.SshPath)
}

func TestSSHEndpointOverridden(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{
			"remote.origin.url":    "git@example.com:foo/bar",
			"remote.origin.lfsurl": "lfs",
		},
		remotes: []string{},
	}

	endpoint := config.Endpoint()
	assert.Equal(t, "lfs", endpoint.Url)
	assert.Equal(t, "", endpoint.SshUserAndHost)
	assert.Equal(t, "", endpoint.SshPath)
	assert.Equal(t, "", endpoint.SshPort)
}

func TestSSHEndpointAddsLfsSuffix(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{"remote.origin.url": "ssh://git@example.com/foo/bar"},
		remotes:   []string{},
	}

	endpoint := config.Endpoint()
	assert.Equal(t, "https://example.com/foo/bar.git/info/lfs", endpoint.Url)
	assert.Equal(t, "git@example.com", endpoint.SshUserAndHost)
	assert.Equal(t, "foo/bar", endpoint.SshPath)
	assert.Equal(t, "", endpoint.SshPort)
}

func TestSSHCustomPortEndpointAddsLfsSuffix(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{"remote.origin.url": "ssh://git@example.com:9000/foo/bar"},
		remotes:   []string{},
	}

	endpoint := config.Endpoint()
	assert.Equal(t, "https://example.com/foo/bar.git/info/lfs", endpoint.Url)
	assert.Equal(t, "git@example.com", endpoint.SshUserAndHost)
	assert.Equal(t, "foo/bar", endpoint.SshPath)
	assert.Equal(t, "9000", endpoint.SshPort)
}

func TestBareSSHEndpointAddsLfsSuffix(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{"remote.origin.url": "git@example.com:foo/bar.git"},
		remotes:   []string{},
	}

	endpoint := config.Endpoint()
	assert.Equal(t, "https://example.com/foo/bar.git/info/lfs", endpoint.Url)
	assert.Equal(t, "git@example.com", endpoint.SshUserAndHost)
	assert.Equal(t, "foo/bar.git", endpoint.SshPath)
	assert.Equal(t, "", endpoint.SshPort)
}

func TestSSHEndpointFromGlobalLfsUrl(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{"lfs.url": "git@example.com:foo/bar.git"},
		remotes:   []string{},
	}

	endpoint := config.Endpoint()
	assert.Equal(t, "https://example.com/foo/bar.git", endpoint.Url)
	assert.Equal(t, "git@example.com", endpoint.SshUserAndHost)
	assert.Equal(t, "foo/bar.git", endpoint.SshPath)
	assert.Equal(t, "", endpoint.SshPort)
}

func TestHTTPEndpointAddsLfsSuffix(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{"remote.origin.url": "http://example.com/foo/bar"},
		remotes:   []string{},
	}

	endpoint := config.Endpoint()
	assert.Equal(t, "http://example.com/foo/bar.git/info/lfs", endpoint.Url)
	assert.Equal(t, "", endpoint.SshUserAndHost)
	assert.Equal(t, "", endpoint.SshPath)
	assert.Equal(t, "", endpoint.SshPort)
}

func TestBareHTTPEndpointAddsLfsSuffix(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{"remote.origin.url": "http://example.com/foo/bar.git"},
		remotes:   []string{},
	}

	endpoint := config.Endpoint()
	assert.Equal(t, "http://example.com/foo/bar.git/info/lfs", endpoint.Url)
	assert.Equal(t, "", endpoint.SshUserAndHost)
	assert.Equal(t, "", endpoint.SshPath)
	assert.Equal(t, "", endpoint.SshPort)
}

func TestObjectUrl(t *testing.T) {
	defer Config.ResetConfig()
	tests := map[string]string{
		"http://example.com":      "http://example.com/objects/oid",
		"http://example.com/":     "http://example.com/objects/oid",
		"http://example.com/foo":  "http://example.com/foo/objects/oid",
		"http://example.com/foo/": "http://example.com/foo/objects/oid",
	}

	for endpoint, expected := range tests {
		Config.SetConfig("lfs.url", endpoint)
		u, err := Config.ObjectUrl("oid")
		if err != nil {
			t.Errorf("Error building URL for %s: %s", endpoint, err)
		} else {
			if actual := u.String(); expected != actual {
				t.Errorf("Expected %s, got %s", expected, u.String())
			}
		}
	}
}

func TestObjectsUrl(t *testing.T) {
	defer Config.ResetConfig()

	tests := map[string]string{
		"http://example.com":      "http://example.com/objects",
		"http://example.com/":     "http://example.com/objects",
		"http://example.com/foo":  "http://example.com/foo/objects",
		"http://example.com/foo/": "http://example.com/foo/objects",
	}

	for endpoint, expected := range tests {
		Config.SetConfig("lfs.url", endpoint)
		u, err := Config.ObjectUrl("")
		if err != nil {
			t.Errorf("Error building URL for %s: %s", endpoint, err)
		} else {
			if actual := u.String(); expected != actual {
				t.Errorf("Expected %s, got %s", expected, u.String())
			}
		}
	}
}

func TestConcurrentTransfersSetValue(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{
			"lfs.concurrenttransfers": "5",
		},
	}

	n := config.ConcurrentTransfers()
	assert.Equal(t, 5, n)
}

func TestConcurrentTransfersDefault(t *testing.T) {
	config := &Configuration{}

	n := config.ConcurrentTransfers()
	assert.Equal(t, 3, n)
}

func TestConcurrentTransfersZeroValue(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{
			"lfs.concurrenttransfers": "0",
		},
	}

	n := config.ConcurrentTransfers()
	assert.Equal(t, 3, n)
}

func TestConcurrentTransfersNonNumeric(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{
			"lfs.concurrenttransfers": "elephant",
		},
	}

	n := config.ConcurrentTransfers()
	assert.Equal(t, 3, n)
}

func TestConcurrentTransfersNegativeValue(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{
			"lfs.concurrenttransfers": "-5",
		},
	}

	n := config.ConcurrentTransfers()
	assert.Equal(t, 3, n)
}

func TestBatchTrue(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{
			"lfs.batch": "true",
		},
	}

	v := config.BatchTransfer()
	assert.Equal(t, true, v)
}

func TestBatchNumeric1IsTrue(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{
			"lfs.batch": "1",
		},
	}

	v := config.BatchTransfer()
	assert.Equal(t, true, v)
}

func TestBatchNumeric0IsFalse(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{
			"lfs.batch": "0",
		},
	}

	v := config.BatchTransfer()
	assert.Equal(t, false, v)
}

func TestBatchOtherNumericsAreTrue(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{
			"lfs.batch": "42",
		},
	}

	v := config.BatchTransfer()
	assert.Equal(t, true, v)
}

func TestBatchNegativeNumericsAreTrue(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{
			"lfs.batch": "-1",
		},
	}

	v := config.BatchTransfer()
	assert.Equal(t, true, v)
}

func TestBatchNonBooleanIsFalse(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{
			"lfs.batch": "elephant",
		},
	}

	v := config.BatchTransfer()
	assert.Equal(t, false, v)
}

func TestBatchPresentButBlankIsTrue(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{
			"lfs.batch": "",
		},
	}

	v := config.BatchTransfer()
	assert.Equal(t, true, v)
}

func TestBatchAbsentIsTrue(t *testing.T) {
	config := &Configuration{}

	v := config.BatchTransfer()
	assert.Equal(t, true, v)
}

func TestLoadValidExtension(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{},
		extensions: map[string]Extension{
			"foo": Extension{
				"foo",
				"foo-clean %f",
				"foo-smudge %f",
				2,
			},
		},
	}

	ext := config.Extensions()["foo"]

	assert.Equal(t, "foo", ext.Name)
	assert.Equal(t, "foo-clean %f", ext.Clean)
	assert.Equal(t, "foo-smudge %f", ext.Smudge)
	assert.Equal(t, 2, ext.Priority)
}

func TestLoadInvalidExtension(t *testing.T) {
	config := &Configuration{}

	ext := config.Extensions()["foo"]

	assert.Equal(t, "", ext.Name)
	assert.Equal(t, "", ext.Clean)
	assert.Equal(t, "", ext.Smudge)
	assert.Equal(t, 0, ext.Priority)
}

func TestFetchPruneConfigDefault(t *testing.T) {
	config := &Configuration{}
	fp := config.FetchPruneConfig()

	assert.Equal(t, 7, fp.FetchRecentRefsDays)
	assert.Equal(t, 0, fp.FetchRecentCommitsDays)
	assert.Equal(t, 3, fp.PruneOffsetDays)
	assert.Equal(t, true, fp.FetchRecentRefsIncludeRemotes)

}
func TestFetchPruneConfigCustom(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{
			"lfs.fetchrecentrefsdays":    "12",
			"lfs.fetchrecentremoterefs":  "false",
			"lfs.fetchrecentcommitsdays": "9",
			"lfs.pruneoffsetdays":        "30",
		},
	}
	fp := config.FetchPruneConfig()

	assert.Equal(t, 12, fp.FetchRecentRefsDays)
	assert.Equal(t, 9, fp.FetchRecentCommitsDays)
	assert.Equal(t, 30, fp.PruneOffsetDays)
	assert.Equal(t, false, fp.FetchRecentRefsIncludeRemotes)
}

// only used for tests
func (c *Configuration) SetConfig(key, value string) {
	if c.loadGitConfig() {
		c.loading.Lock()
		c.origConfig = make(map[string]string)
		for k, v := range c.gitConfig {
			c.origConfig[k] = v
		}
		c.loading.Unlock()
	}

	c.gitConfig[key] = value
}

func (c *Configuration) ResetConfig() {
	c.loading.Lock()
	c.gitConfig = make(map[string]string)
	for k, v := range c.origConfig {
		c.gitConfig[k] = v
	}
	c.loading.Unlock()
}
<commit_msg>アー アアアア アーアー<commit_after>package lfs

import (
	"testing"

	"github.com/github/git-lfs/vendor/_nuts/github.com/technoweenie/assert"
)

func TestEndpointDefaultsToOrigin(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{"remote.origin.lfsurl": "abc"},
		remotes:   []string{},
	}

	endpoint := config.Endpoint()
	assert.Equal(t, "abc", endpoint.Url)
	assert.Equal(t, "", endpoint.SshUserAndHost)
	assert.Equal(t, "", endpoint.SshPath)
}

func TestEndpointOverridesOrigin(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{
			"lfs.url":              "abc",
			"remote.origin.lfsurl": "def",
		},
		remotes: []string{},
	}

	endpoint := config.Endpoint()
	assert.Equal(t, "abc", endpoint.Url)
	assert.Equal(t, "", endpoint.SshUserAndHost)
	assert.Equal(t, "", endpoint.SshPath)
}

func TestEndpointNoOverrideDefaultRemote(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{
			"remote.origin.lfsurl": "abc",
			"remote.other.lfsurl":  "def",
		},
		remotes: []string{},
	}

	endpoint := config.Endpoint()
	assert.Equal(t, "abc", endpoint.Url)
	assert.Equal(t, "", endpoint.SshUserAndHost)
	assert.Equal(t, "", endpoint.SshPath)
}

func TestEndpointUseAlternateRemote(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{
			"remote.origin.lfsurl": "abc",
			"remote.other.lfsurl":  "def",
		},
		remotes: []string{},
	}

	config.CurrentRemote = "other"

	endpoint := config.Endpoint()
	assert.Equal(t, "def", endpoint.Url)
	assert.Equal(t, "", endpoint.SshUserAndHost)
	assert.Equal(t, "", endpoint.SshPath)
}

func TestEndpointAddsLfsSuffix(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{"remote.origin.url": "https://example.com/foo/bar"},
		remotes:   []string{},
	}

	endpoint := config.Endpoint()
	assert.Equal(t, "https://example.com/foo/bar.git/info/lfs", endpoint.Url)
	assert.Equal(t, "", endpoint.SshUserAndHost)
	assert.Equal(t, "", endpoint.SshPath)
}

func TestBareEndpointAddsLfsSuffix(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{"remote.origin.url": "https://example.com/foo/bar.git"},
		remotes:   []string{},
	}

	endpoint := config.Endpoint()
	assert.Equal(t, "https://example.com/foo/bar.git/info/lfs", endpoint.Url)
	assert.Equal(t, "", endpoint.SshUserAndHost)
	assert.Equal(t, "", endpoint.SshPath)
}

func TestSSHEndpointOverridden(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{
			"remote.origin.url":    "git@example.com:foo/bar",
			"remote.origin.lfsurl": "lfs",
		},
		remotes: []string{},
	}

	endpoint := config.Endpoint()
	assert.Equal(t, "lfs", endpoint.Url)
	assert.Equal(t, "", endpoint.SshUserAndHost)
	assert.Equal(t, "", endpoint.SshPath)
	assert.Equal(t, "", endpoint.SshPort)
}

func TestSSHEndpointAddsLfsSuffix(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{"remote.origin.url": "ssh://git@example.com/foo/bar"},
		remotes:   []string{},
	}

	endpoint := config.Endpoint()
	assert.Equal(t, "https://example.com/foo/bar.git/info/lfs", endpoint.Url)
	assert.Equal(t, "git@example.com", endpoint.SshUserAndHost)
	assert.Equal(t, "foo/bar", endpoint.SshPath)
	assert.Equal(t, "", endpoint.SshPort)
}

func TestSSHCustomPortEndpointAddsLfsSuffix(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{"remote.origin.url": "ssh://git@example.com:9000/foo/bar"},
		remotes:   []string{},
	}

	endpoint := config.Endpoint()
	assert.Equal(t, "https://example.com/foo/bar.git/info/lfs", endpoint.Url)
	assert.Equal(t, "git@example.com", endpoint.SshUserAndHost)
	assert.Equal(t, "foo/bar", endpoint.SshPath)
	assert.Equal(t, "9000", endpoint.SshPort)
}

func TestBareSSHEndpointAddsLfsSuffix(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{"remote.origin.url": "git@example.com:foo/bar.git"},
		remotes:   []string{},
	}

	endpoint := config.Endpoint()
	assert.Equal(t, "https://example.com/foo/bar.git/info/lfs", endpoint.Url)
	assert.Equal(t, "git@example.com", endpoint.SshUserAndHost)
	assert.Equal(t, "foo/bar.git", endpoint.SshPath)
	assert.Equal(t, "", endpoint.SshPort)
}

func TestSSHEndpointFromGlobalLfsUrl(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{"lfs.url": "git@example.com:foo/bar.git"},
		remotes:   []string{},
	}

	endpoint := config.Endpoint()
	assert.Equal(t, "https://example.com/foo/bar.git", endpoint.Url)
	assert.Equal(t, "git@example.com", endpoint.SshUserAndHost)
	assert.Equal(t, "foo/bar.git", endpoint.SshPath)
	assert.Equal(t, "", endpoint.SshPort)
}

func TestHTTPEndpointAddsLfsSuffix(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{"remote.origin.url": "http://example.com/foo/bar"},
		remotes:   []string{},
	}

	endpoint := config.Endpoint()
	assert.Equal(t, "http://example.com/foo/bar.git/info/lfs", endpoint.Url)
	assert.Equal(t, "", endpoint.SshUserAndHost)
	assert.Equal(t, "", endpoint.SshPath)
	assert.Equal(t, "", endpoint.SshPort)
}

func TestBareHTTPEndpointAddsLfsSuffix(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{"remote.origin.url": "http://example.com/foo/bar.git"},
		remotes:   []string{},
	}

	endpoint := config.Endpoint()
	assert.Equal(t, "http://example.com/foo/bar.git/info/lfs", endpoint.Url)
	assert.Equal(t, "", endpoint.SshUserAndHost)
	assert.Equal(t, "", endpoint.SshPath)
	assert.Equal(t, "", endpoint.SshPort)
}

func TestObjectUrl(t *testing.T) {
	defer Config.ResetConfig()
	tests := map[string]string{
		"http://example.com":      "http://example.com/objects/oid",
		"http://example.com/":     "http://example.com/objects/oid",
		"http://example.com/foo":  "http://example.com/foo/objects/oid",
		"http://example.com/foo/": "http://example.com/foo/objects/oid",
	}

	for endpoint, expected := range tests {
		Config.SetConfig("lfs.url", endpoint)
		u, err := Config.ObjectUrl("oid")
		if err != nil {
			t.Errorf("Error building URL for %s: %s", endpoint, err)
		} else {
			if actual := u.String(); expected != actual {
				t.Errorf("Expected %s, got %s", expected, u.String())
			}
		}
	}
}

func TestObjectsUrl(t *testing.T) {
	defer Config.ResetConfig()

	tests := map[string]string{
		"http://example.com":      "http://example.com/objects",
		"http://example.com/":     "http://example.com/objects",
		"http://example.com/foo":  "http://example.com/foo/objects",
		"http://example.com/foo/": "http://example.com/foo/objects",
	}

	for endpoint, expected := range tests {
		Config.SetConfig("lfs.url", endpoint)
		u, err := Config.ObjectUrl("")
		if err != nil {
			t.Errorf("Error building URL for %s: %s", endpoint, err)
		} else {
			if actual := u.String(); expected != actual {
				t.Errorf("Expected %s, got %s", expected, u.String())
			}
		}
	}
}

func TestConcurrentTransfersSetValue(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{
			"lfs.concurrenttransfers": "5",
		},
	}

	n := config.ConcurrentTransfers()
	assert.Equal(t, 5, n)
}

func TestConcurrentTransfersDefault(t *testing.T) {
	config := &Configuration{}

	n := config.ConcurrentTransfers()
	assert.Equal(t, 3, n)
}

func TestConcurrentTransfersZeroValue(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{
			"lfs.concurrenttransfers": "0",
		},
	}

	n := config.ConcurrentTransfers()
	assert.Equal(t, 3, n)
}

func TestConcurrentTransfersNonNumeric(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{
			"lfs.concurrenttransfers": "elephant",
		},
	}

	n := config.ConcurrentTransfers()
	assert.Equal(t, 3, n)
}

func TestConcurrentTransfersNegativeValue(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{
			"lfs.concurrenttransfers": "-5",
		},
	}

	n := config.ConcurrentTransfers()
	assert.Equal(t, 3, n)
}

func TestBatch(t *testing.T) {
	tests := map[string]bool{
		"true":     true,
		"1":        true,
		"42":       true,
		"-1":       true,
		"":         true,
		"0":        false,
		"false":    false,
		"elephant": false,
	}

	for value, expected := range tests {
		config := &Configuration{
			gitConfig: map[string]string{"lfs.batch": value},
		}

		if actual := config.BatchTransfer(); actual != expected {
			t.Errorf("lfs.batch %q == %v, not %v", value, actual, expected)
		}
	}
}

func TestBatchAbsentIsTrue(t *testing.T) {
	config := &Configuration{}

	v := config.BatchTransfer()
	assert.Equal(t, true, v)
}

func TestLoadValidExtension(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{},
		extensions: map[string]Extension{
			"foo": Extension{
				"foo",
				"foo-clean %f",
				"foo-smudge %f",
				2,
			},
		},
	}

	ext := config.Extensions()["foo"]

	assert.Equal(t, "foo", ext.Name)
	assert.Equal(t, "foo-clean %f", ext.Clean)
	assert.Equal(t, "foo-smudge %f", ext.Smudge)
	assert.Equal(t, 2, ext.Priority)
}

func TestLoadInvalidExtension(t *testing.T) {
	config := &Configuration{}

	ext := config.Extensions()["foo"]

	assert.Equal(t, "", ext.Name)
	assert.Equal(t, "", ext.Clean)
	assert.Equal(t, "", ext.Smudge)
	assert.Equal(t, 0, ext.Priority)
}

func TestFetchPruneConfigDefault(t *testing.T) {
	config := &Configuration{}
	fp := config.FetchPruneConfig()

	assert.Equal(t, 7, fp.FetchRecentRefsDays)
	assert.Equal(t, 0, fp.FetchRecentCommitsDays)
	assert.Equal(t, 3, fp.PruneOffsetDays)
	assert.Equal(t, true, fp.FetchRecentRefsIncludeRemotes)

}
func TestFetchPruneConfigCustom(t *testing.T) {
	config := &Configuration{
		gitConfig: map[string]string{
			"lfs.fetchrecentrefsdays":    "12",
			"lfs.fetchrecentremoterefs":  "false",
			"lfs.fetchrecentcommitsdays": "9",
			"lfs.pruneoffsetdays":        "30",
		},
	}
	fp := config.FetchPruneConfig()

	assert.Equal(t, 12, fp.FetchRecentRefsDays)
	assert.Equal(t, 9, fp.FetchRecentCommitsDays)
	assert.Equal(t, 30, fp.PruneOffsetDays)
	assert.Equal(t, false, fp.FetchRecentRefsIncludeRemotes)
}

// only used for tests
func (c *Configuration) SetConfig(key, value string) {
	if c.loadGitConfig() {
		c.loading.Lock()
		c.origConfig = make(map[string]string)
		for k, v := range c.gitConfig {
			c.origConfig[k] = v
		}
		c.loading.Unlock()
	}

	c.gitConfig[key] = value
}

func (c *Configuration) ResetConfig() {
	c.loading.Lock()
	c.gitConfig = make(map[string]string)
	for k, v := range c.origConfig {
		c.gitConfig[k] = v
	}
	c.loading.Unlock()
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package apiserver

import (
	"context"
	"fmt"
	"net/http"
	"net/url"
	"sync/atomic"

	"github.com/golang/glog"

	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/httpstream"
	"k8s.io/apimachinery/pkg/util/httpstream/spdy"
	utilnet "k8s.io/apimachinery/pkg/util/net"
	"k8s.io/apimachinery/pkg/util/proxy"
	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
	genericfeatures "k8s.io/apiserver/pkg/features"
	utilfeature "k8s.io/apiserver/pkg/util/feature"
	restclient "k8s.io/client-go/rest"
	"k8s.io/client-go/transport"
	apiregistrationapi "k8s.io/kube-aggregator/pkg/apis/apiregistration"
)

// proxyHandler provides a http.Handler which will proxy traffic to locations
// specified by items implementing Redirector.
type proxyHandler struct {
	// localDelegate is used to satisfy local APIServices
	localDelegate http.Handler

	// proxyClientCert/Key are the client cert used to identify this proxy. Backing APIServices use
	// this to confirm the proxy's identity
	proxyClientCert []byte
	proxyClientKey  []byte
	proxyTransport  *http.Transport

	// Endpoints based routing to map from cluster IP to routable IP
	serviceResolver ServiceResolver

	handlingInfo atomic.Value
}

type proxyHandlingInfo struct {
	// local indicates that this APIService is locally satisfied
	local bool

	// restConfig holds the information for building a roundtripper
	restConfig *restclient.Config
	// transportBuildingError is an error produced while building the transport.  If this
	// is non-nil, it will be reported to clients.
	transportBuildingError error
	// proxyRoundTripper is the re-useable portion of the transport.  It does not vary with any request.
	proxyRoundTripper http.RoundTripper
	// serviceName is the name of the service this handler proxies to
	serviceName string
	// namespace is the namespace the service lives in
	serviceNamespace string
	// serviceAvailable indicates this APIService is available or not
	serviceAvailable bool
}

func (r *proxyHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	value := r.handlingInfo.Load()
	if value == nil {
		r.localDelegate.ServeHTTP(w, req)
		return
	}
	handlingInfo := value.(proxyHandlingInfo)
	if handlingInfo.local {
		if r.localDelegate == nil {
			http.Error(w, "", http.StatusNotFound)
			return
		}
		r.localDelegate.ServeHTTP(w, req)
		return
	}

	if !handlingInfo.serviceAvailable {
		http.Error(w, "service unavailable", http.StatusServiceUnavailable)
		return
	}

	if handlingInfo.transportBuildingError != nil {
		http.Error(w, handlingInfo.transportBuildingError.Error(), http.StatusInternalServerError)
		return
	}

	user, ok := genericapirequest.UserFrom(req.Context())
	if !ok {
		http.Error(w, "missing user", http.StatusInternalServerError)
		return
	}

	// write a new location based on the existing request pointed at the target service
	location := &url.URL{}
	location.Scheme = "https"
	rloc, err := r.serviceResolver.ResolveEndpoint(handlingInfo.serviceNamespace, handlingInfo.serviceName)
	if err != nil {
		glog.Errorf("error resolving %s/%s: %v", handlingInfo.serviceNamespace, handlingInfo.serviceName, err)
		http.Error(w, "service unavailable", http.StatusServiceUnavailable)
		return
	}
	location.Host = rloc.Host
	location.Path = req.URL.Path
	location.RawQuery = req.URL.Query().Encode()

	// WithContext creates a shallow clone of the request with the new context.
	newReq := req.WithContext(context.Background())
	newReq.Header = utilnet.CloneHeader(req.Header)
	newReq.URL = location

	if handlingInfo.proxyRoundTripper == nil {
		http.Error(w, "", http.StatusNotFound)
		return
	}

	// we need to wrap the roundtripper in another roundtripper which will apply the front proxy headers
	proxyRoundTripper, upgrade, err := maybeWrapForConnectionUpgrades(handlingInfo.restConfig, handlingInfo.proxyRoundTripper, req)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	proxyRoundTripper = transport.NewAuthProxyRoundTripper(user.GetName(), user.GetGroups(), user.GetExtra(), proxyRoundTripper)

	// if we are upgrading, then the upgrade path tries to use this request with the TLS config we provide, but it does
	// NOT use the roundtripper.  Its a direct call that bypasses the round tripper.  This means that we have to
	// attach the "correct" user headers to the request ahead of time.  After the initial upgrade, we'll be back
	// at the roundtripper flow, so we only have to muck with this request, but we do have to do it.
	if upgrade {
		transport.SetAuthProxyHeaders(newReq, user.GetName(), user.GetGroups(), user.GetExtra())
	}

	handler := proxy.NewUpgradeAwareHandler(location, proxyRoundTripper, true, upgrade, &responder{w: w})
	handler.ServeHTTP(w, newReq)
}

// maybeWrapForConnectionUpgrades wraps the roundtripper for upgrades.  The bool indicates if it was wrapped
func maybeWrapForConnectionUpgrades(restConfig *restclient.Config, rt http.RoundTripper, req *http.Request) (http.RoundTripper, bool, error) {
	if !httpstream.IsUpgradeRequest(req) {
		return rt, false, nil
	}

	tlsConfig, err := restclient.TLSConfigFor(restConfig)
	if err != nil {
		return nil, true, err
	}
	followRedirects := utilfeature.DefaultFeatureGate.Enabled(genericfeatures.StreamingProxyRedirects)
	upgradeRoundTripper := spdy.NewRoundTripper(tlsConfig, followRedirects)
	wrappedRT, err := restclient.HTTPWrappersForConfig(restConfig, upgradeRoundTripper)
	if err != nil {
		return nil, true, err
	}

	return wrappedRT, true, nil
}

// responder implements rest.Responder for assisting a connector in writing objects or errors.
type responder struct {
	w http.ResponseWriter
}

// TODO this should properly handle content type negotiation
// if the caller asked for protobuf and you write JSON bad things happen.
func (r *responder) Object(statusCode int, obj runtime.Object) {
	responsewriters.WriteRawJSON(statusCode, obj, r.w)
}

func (r *responder) Error(_ http.ResponseWriter, _ *http.Request, err error) {
	http.Error(r.w, err.Error(), http.StatusInternalServerError)
}

// these methods provide locked access to fields

func (r *proxyHandler) updateAPIService(apiService *apiregistrationapi.APIService) {
	if apiService.Spec.Service == nil {
		r.handlingInfo.Store(proxyHandlingInfo{local: true})
		return
	}

	newInfo := proxyHandlingInfo{
		restConfig: &restclient.Config{
			TLSClientConfig: restclient.TLSClientConfig{
				Insecure:   apiService.Spec.InsecureSkipTLSVerify,
				ServerName: apiService.Spec.Service.Name + "." + apiService.Spec.Service.Namespace + ".svc",
				CertData:   r.proxyClientCert,
				KeyData:    r.proxyClientKey,
				CAData:     apiService.Spec.CABundle,
			},
		},
		serviceName:      apiService.Spec.Service.Name,
		serviceNamespace: apiService.Spec.Service.Namespace,
		serviceAvailable: apiregistrationapi.IsAPIServiceConditionTrue(apiService, apiregistrationapi.Available),
	}
	newInfo.proxyRoundTripper, newInfo.transportBuildingError = restclient.TransportFor(newInfo.restConfig)
	if newInfo.transportBuildingError == nil && r.proxyTransport != nil && r.proxyTransport.DialContext != nil {
		switch transport := newInfo.proxyRoundTripper.(type) {
		case *http.Transport:
			transport.DialContext = r.proxyTransport.DialContext
		default:
			newInfo.transportBuildingError = fmt.Errorf("unable to set dialer for %s/%s as rest transport is of type %T", apiService.Spec.Service.Namespace, apiService.Spec.Service.Name, newInfo.proxyRoundTripper)
			glog.Warning(newInfo.transportBuildingError.Error())
		}
	}
	r.handlingInfo.Store(newInfo)
}
<commit_msg>fix kube-aggregator dailer<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package apiserver

import (
	"context"
	"net/http"
	"net/url"
	"sync/atomic"

	"github.com/golang/glog"

	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/httpstream"
	"k8s.io/apimachinery/pkg/util/httpstream/spdy"
	utilnet "k8s.io/apimachinery/pkg/util/net"
	"k8s.io/apimachinery/pkg/util/proxy"
	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
	genericfeatures "k8s.io/apiserver/pkg/features"
	utilfeature "k8s.io/apiserver/pkg/util/feature"
	restclient "k8s.io/client-go/rest"
	"k8s.io/client-go/transport"
	apiregistrationapi "k8s.io/kube-aggregator/pkg/apis/apiregistration"
)

// proxyHandler provides a http.Handler which will proxy traffic to locations
// specified by items implementing Redirector.
type proxyHandler struct {
	// localDelegate is used to satisfy local APIServices
	localDelegate http.Handler

	// proxyClientCert/Key are the client cert used to identify this proxy. Backing APIServices use
	// this to confirm the proxy's identity
	proxyClientCert []byte
	proxyClientKey  []byte
	proxyTransport  *http.Transport

	// Endpoints based routing to map from cluster IP to routable IP
	serviceResolver ServiceResolver

	handlingInfo atomic.Value
}

type proxyHandlingInfo struct {
	// local indicates that this APIService is locally satisfied
	local bool

	// restConfig holds the information for building a roundtripper
	restConfig *restclient.Config
	// transportBuildingError is an error produced while building the transport.  If this
	// is non-nil, it will be reported to clients.
	transportBuildingError error
	// proxyRoundTripper is the re-useable portion of the transport.  It does not vary with any request.
	proxyRoundTripper http.RoundTripper
	// serviceName is the name of the service this handler proxies to
	serviceName string
	// namespace is the namespace the service lives in
	serviceNamespace string
	// serviceAvailable indicates this APIService is available or not
	serviceAvailable bool
}

func (r *proxyHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	value := r.handlingInfo.Load()
	if value == nil {
		r.localDelegate.ServeHTTP(w, req)
		return
	}
	handlingInfo := value.(proxyHandlingInfo)
	if handlingInfo.local {
		if r.localDelegate == nil {
			http.Error(w, "", http.StatusNotFound)
			return
		}
		r.localDelegate.ServeHTTP(w, req)
		return
	}

	if !handlingInfo.serviceAvailable {
		http.Error(w, "service unavailable", http.StatusServiceUnavailable)
		return
	}

	if handlingInfo.transportBuildingError != nil {
		http.Error(w, handlingInfo.transportBuildingError.Error(), http.StatusInternalServerError)
		return
	}

	user, ok := genericapirequest.UserFrom(req.Context())
	if !ok {
		http.Error(w, "missing user", http.StatusInternalServerError)
		return
	}

	// write a new location based on the existing request pointed at the target service
	location := &url.URL{}
	location.Scheme = "https"
	rloc, err := r.serviceResolver.ResolveEndpoint(handlingInfo.serviceNamespace, handlingInfo.serviceName)
	if err != nil {
		glog.Errorf("error resolving %s/%s: %v", handlingInfo.serviceNamespace, handlingInfo.serviceName, err)
		http.Error(w, "service unavailable", http.StatusServiceUnavailable)
		return
	}
	location.Host = rloc.Host
	location.Path = req.URL.Path
	location.RawQuery = req.URL.Query().Encode()

	// WithContext creates a shallow clone of the request with the new context.
	newReq := req.WithContext(context.Background())
	newReq.Header = utilnet.CloneHeader(req.Header)
	newReq.URL = location

	if handlingInfo.proxyRoundTripper == nil {
		http.Error(w, "", http.StatusNotFound)
		return
	}

	// we need to wrap the roundtripper in another roundtripper which will apply the front proxy headers
	proxyRoundTripper, upgrade, err := maybeWrapForConnectionUpgrades(handlingInfo.restConfig, handlingInfo.proxyRoundTripper, req)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	proxyRoundTripper = transport.NewAuthProxyRoundTripper(user.GetName(), user.GetGroups(), user.GetExtra(), proxyRoundTripper)

	// if we are upgrading, then the upgrade path tries to use this request with the TLS config we provide, but it does
	// NOT use the roundtripper.  Its a direct call that bypasses the round tripper.  This means that we have to
	// attach the "correct" user headers to the request ahead of time.  After the initial upgrade, we'll be back
	// at the roundtripper flow, so we only have to muck with this request, but we do have to do it.
	if upgrade {
		transport.SetAuthProxyHeaders(newReq, user.GetName(), user.GetGroups(), user.GetExtra())
	}

	handler := proxy.NewUpgradeAwareHandler(location, proxyRoundTripper, true, upgrade, &responder{w: w})
	handler.ServeHTTP(w, newReq)
}

// maybeWrapForConnectionUpgrades wraps the roundtripper for upgrades.  The bool indicates if it was wrapped
func maybeWrapForConnectionUpgrades(restConfig *restclient.Config, rt http.RoundTripper, req *http.Request) (http.RoundTripper, bool, error) {
	if !httpstream.IsUpgradeRequest(req) {
		return rt, false, nil
	}

	tlsConfig, err := restclient.TLSConfigFor(restConfig)
	if err != nil {
		return nil, true, err
	}
	followRedirects := utilfeature.DefaultFeatureGate.Enabled(genericfeatures.StreamingProxyRedirects)
	upgradeRoundTripper := spdy.NewRoundTripper(tlsConfig, followRedirects)
	wrappedRT, err := restclient.HTTPWrappersForConfig(restConfig, upgradeRoundTripper)
	if err != nil {
		return nil, true, err
	}

	return wrappedRT, true, nil
}

// responder implements rest.Responder for assisting a connector in writing objects or errors.
type responder struct {
	w http.ResponseWriter
}

// TODO this should properly handle content type negotiation
// if the caller asked for protobuf and you write JSON bad things happen.
func (r *responder) Object(statusCode int, obj runtime.Object) {
	responsewriters.WriteRawJSON(statusCode, obj, r.w)
}

func (r *responder) Error(_ http.ResponseWriter, _ *http.Request, err error) {
	http.Error(r.w, err.Error(), http.StatusInternalServerError)
}

// these methods provide locked access to fields

func (r *proxyHandler) updateAPIService(apiService *apiregistrationapi.APIService) {
	if apiService.Spec.Service == nil {
		r.handlingInfo.Store(proxyHandlingInfo{local: true})
		return
	}

	newInfo := proxyHandlingInfo{
		restConfig: &restclient.Config{
			TLSClientConfig: restclient.TLSClientConfig{
				Insecure:   apiService.Spec.InsecureSkipTLSVerify,
				ServerName: apiService.Spec.Service.Name + "." + apiService.Spec.Service.Namespace + ".svc",
				CertData:   r.proxyClientCert,
				KeyData:    r.proxyClientKey,
				CAData:     apiService.Spec.CABundle,
			},
		},
		serviceName:      apiService.Spec.Service.Name,
		serviceNamespace: apiService.Spec.Service.Namespace,
		serviceAvailable: apiregistrationapi.IsAPIServiceConditionTrue(apiService, apiregistrationapi.Available),
	}
	if r.proxyTransport != nil && r.proxyTransport.DialContext != nil {
		newInfo.restConfig.Dial = r.proxyTransport.DialContext
	}
	newInfo.proxyRoundTripper, newInfo.transportBuildingError = restclient.TransportFor(newInfo.restConfig)
	if newInfo.transportBuildingError != nil {
		glog.Warning(newInfo.transportBuildingError.Error())
	}
	r.handlingInfo.Store(newInfo)
}
<|endoftext|>


<commit_before>// Copyright (c) 2014 Pagoda Box Inc.
//
// This Source Code Form is subject to the terms of the Mozilla Public License,
// v. 2.0. If a copy of the MPL was not distributed with this file, You can
// obtain one at http://mozilla.org/MPL/2.0/.

//
package jobs

//
import (
	"strings"

	"github.com/nanobox-io/nanobox-boxfile"
	"github.com/nanobox-io/nanobox-golang-stylish"
	// "github.com/nanobox-io/nanobox-logtap"
	"github.com/nanobox-io/nanobox-router"
	"github.com/nanobox-io/nanobox-server/config"
	"github.com/nanobox-io/nanobox-server/util"
	"github.com/nanobox-io/nanobox-server/util/docker"
	"github.com/nanobox-io/nanobox-server/util/fs"
	"github.com/nanobox-io/nanobox-server/util/script"
	"github.com/nanobox-io/nanobox-server/util/worker"
)

//
type Deploy struct {
	ID    string
	Reset bool
	Run   bool

	payload map[string]interface{}
}

// Proccess syncronies your docker containers with the boxfile specification
func (j *Deploy) Process() {
	// add a lock so the service wont go down whil im running
	util.Lock()
	defer util.Unlock()

	// set routing to watch logs
	router.ErrorHandler = router.DeployInProgress{}

	// remove all code containers
	util.LogInfo(stylish.Bullet("Cleaning containers"))

	// might as well remove bootstraps and execs too
	containers, _ := docker.ListContainers("code", "build", "bootstrap", "exec", "tcp", "udp")
	for _, container := range containers {
		util.RemoveForward(container.NetworkSettings.IPAddress)
		if err := docker.RemoveContainer(container.ID); err != nil {
			util.HandleError(stylish.Error("Failed to remove old containers", err.Error()))
			util.UpdateStatus(j, "errored")
			return
		}
	}

	// Make sure we have the directories
	if err := fs.CreateDirs(); err != nil {
		util.HandleError(stylish.Error("Failed to create dirs", err.Error()))
		util.UpdateStatus(j, "errored")
		return
	}

	// wipe the previous deploy data if reset == true
	if j.Reset {
		util.LogInfo(stylish.Bullet("Emptying cache"))
		if err := fs.Clean(); err != nil {
			util.HandleError(stylish.Warning("Failed to reset cache and code directories:\n%v", err.Error()))
		}
	}

	// parse the boxfile
	util.LogDebug(stylish.Bullet("Parsing Boxfile"))
	box := boxfile.NewFromPath("/vagrant/code/" + config.App + "/Boxfile")

	image := "nanobox/build"

	if stab := box.Node("build").StringValue("stability"); stab != "" {
		image = image + ":" + stab
	}

	// if the build image doesn't exist it needs to be downloaded
	if !docker.ImageExists(image) {
		util.LogInfo(stylish.Bullet("Pulling the latest build image (this may take awhile)... "))
		docker.InstallImage(image)
	}

	util.LogDebug(stylish.Bullet("image name: %v", image))

	// create a build container
	util.LogInfo(stylish.Bullet("Creating build container"))

	_, err := docker.CreateContainer(docker.CreateConfig{Image: image, Category: "build", UID: "build1"})
	if err != nil {
		util.HandleError(stylish.Error("Failed to create build container", err.Error()))
		util.UpdateStatus(j, "errored")
		return
	}

	// define the deploy payload
	j.payload = map[string]interface{}{
		"platform":    "local",
		"app":         config.App,
		"dns":         []string{config.App + ".dev"},
		"port":        "8080",
		"boxfile":     box.Node("build").Parsed,
		"logtap_host": config.LogtapHost,
	}

	evar := map[string]string{}
	if box.Node("env").Valid {
		for key, val := range box.Node("env").Parsed {
			if str, ok := val.(string); ok {
				evar[key] = str
			}
		}
	}

	evar["APP_NAME"] = config.App
	j.payload["env"] = evar

	// run the default-user hook to get ssh keys setup
	if out, err := script.Exec("default-user", "build1", fs.UserPayload()); err != nil {
		util.LogDebug("Failed script output: \n %s", out)
		util.HandleError(stylish.Error("Failed to run user script", err.Error()))
		util.UpdateStatus(j, "errored")
		return
	}

	// run configure hook (blocking)
	if out, err := script.Exec("default-configure", "build1", j.payload); err != nil {
		util.LogDebug("Failed script output: \n %s", out)
		util.HandleError(stylish.Error("Failed to run configure script", err.Error()))
		util.UpdateStatus(j, "errored")
		return
	}

	// run detect script (blocking)
	if out, err := script.Exec("default-detect", "build1", j.payload); err != nil {
		util.LogDebug("Failed script output: \n %s", out)
		util.HandleError(stylish.Error("Failed to run detect script", err.Error()))
		util.UpdateStatus(j, "errored")
		return
	}

	// run sync script (blocking)
	if out, err := script.Exec("default-sync", "build1", j.payload); err != nil {
		util.LogDebug("Failed script output: \n %s", out)
		util.HandleError(stylish.Error("Failed to run sync script", err.Error()))
		util.UpdateStatus(j, "errored")
		return
	}

	// run setup script (blocking)
	if out, err := script.Exec("default-setup", "build1", j.payload); err != nil {
		util.LogDebug("Failed script output: \n %s", out)
		util.HandleError(stylish.Error("Failed to run setup script", err.Error()))
		util.UpdateStatus(j, "errored")
		return
	}

	// run boxfile script (blocking)
	if !box.Node("build").BoolValue("disable_engine_boxfile") {
		if out, err := script.Exec("default-boxfile", "build1", j.payload); err != nil {
			util.LogDebug("Failed script output: \n %s", out)
			util.HandleError(stylish.Error("Failed to run boxfile script", err.Error()))
			util.UpdateStatus(j, "errored")
			return

			// if the script runs succesfully merge the boxfiles
		} else {
			util.LogDebug(stylish.Bullet("Merging Boxfiles..."))
			box.Merge(boxfile.New([]byte(out)))
		}
	}

	// add the missing storage nodes to the boxfile
	box.AddStorageNode()

	// remove any containers no longer in the boxfile
	util.LogDebug(stylish.Bullet("Removing old containers..."))
	serviceContainers, _ := docker.ListContainers("service")
	for _, container := range serviceContainers {
		if !box.Node(container.Config.Labels["uid"]).Valid {
			util.RemoveForward(container.NetworkSettings.IPAddress)
			docker.RemoveContainer(container.ID)
		}
	}

	worker := worker.New()
	worker.Blocking = true
	worker.Concurrent = true

	//
	serviceStarts := []*ServiceStart{}

	// build service containers according to boxfile
	for _, node := range box.Nodes("service") {
		if _, err := docker.GetContainer(node); err != nil {
			// container doesn't exist so we need to create it
			s := ServiceStart{
				Boxfile: box.Node(node),
				UID:     node,
				EVars:   map[string]string{},
			}

			serviceStarts = append(serviceStarts, &s)

			worker.Queue(&s)
		}
	}

	if worker.Count() > 0 {
		util.LogInfo(stylish.Bullet("Launching data services"))
	}

	worker.Process()

	// ensure all services started correctly before continuing
	for _, starts := range serviceStarts {
		if !starts.Success {
			util.HandleError(stylish.ErrorHead("Failed to start %v", starts.UID))
			util.HandleError(stylish.ErrorBody(""))
			util.UpdateStatus(j, "errored")
			return
		}
	}

	// grab the environment data from all service containers
	evars := j.payload["env"].(map[string]string)

	// clear out the old ports from the previous deploy
	clearPorts()

	//
	serviceEnvs := []*ServiceEnv{}

	serviceContainers, _ = docker.ListContainers("service")
	for _, container := range serviceContainers {

		s := ServiceEnv{UID: container.Config.Labels["uid"]}
		serviceEnvs = append(serviceEnvs, &s)

		worker.Queue(&s)
	}

	worker.Process()

	for _, env := range serviceEnvs {
		if !env.Success {
			util.HandleError(stylish.ErrorHead("Failed to configure %v's environment variables", env.UID))
			util.HandleError(stylish.ErrorBody(""))
			util.UpdateStatus(j, "errored")
			return
		}

		for key, val := range env.EVars {
			evars[strings.ToUpper(env.UID+"_"+key)] = val
		}
	}

	j.payload["env"] = evars

	// run prepare script (blocking)
	if out, err := script.Exec("default-prepare", "build1", j.payload); err != nil {
		util.LogDebug("Failed script output: \n %s", out)
		util.HandleError(stylish.Error("Failed to run prepare script", err.Error()))
		util.UpdateStatus(j, "errored")
		return
	}

	if j.Run {
		// run build script (blocking)
		if out, err := script.Exec("default-build", "build1", j.payload); err != nil {
			util.LogDebug("Failed script output: \n %s", out)
			util.HandleError(stylish.Error("Failed to run build script", err.Error()))
			util.UpdateStatus(j, "errored")
			return
		}

		// run publish script (blocking)
		if out, err := script.Exec("default-publish", "build1", j.payload); err != nil {
			util.LogDebug("Failed script output: \n %s", out)
			util.HandleError(stylish.Error("Failed to run publish script", err.Error()))
			util.UpdateStatus(j, "errored")
			return
		}
	}

	// run cleanup script (blocking)
	if out, err := script.Exec("default-cleanup", "build1", j.payload); err != nil {
		util.LogDebug("Failed script output: \n %s", out)
		util.HandleError(stylish.Error("Failed to run cleanup script", err.Error()))
		util.UpdateStatus(j, "errored")
		return
	}

	// we will only create new code nodes if we are
	// supposed to be running
	if j.Run {

		// build new code containers
		codeServices := []*ServiceStart{}
		for _, node := range box.Nodes("code") {
			if _, err := docker.GetContainer(node); err != nil {
				// container doesn't exist so we need to create it
				s := ServiceStart{
					Boxfile: box.Node(node),
					UID:     node,
					EVars:   evars,
				}

				codeServices = append(codeServices, &s)

				worker.Queue(&s)
			}
			if worker.Count() > 0 {
				util.LogInfo(stylish.Bullet("Launching Code services"))
			}
		}

		worker.Process()

		for _, serv := range codeServices {
			if !serv.Success {
				util.HandleError("A Service was not started correctly (" + serv.UID + ")")
				util.UpdateStatus(j, "errored")
				return
			}
		}
	}

	util.LogDebug(stylish.Bullet("Running before deploy scripts..."))

	// run before deploy scripts
	for _, node := range box.Nodes() {
		bd := box.Node(node).Value("before_deploy")
		bda := box.Node(node).Value("before_deploy_all")
		if bd != nil || bda != nil {

			// run before deploy script (blocking)
			if out, err := script.Exec("default-before_deploy", node, map[string]interface{}{"before_deploy": bd, "before_deploy_all": bda}); err != nil {
				util.LogDebug("Failed script output: \n %s", out)
				util.HandleError(stylish.Error("Failed to run before_deploy script", err.Error()))
				util.UpdateStatus(j, "errored")
				return
			}
		}
	}

	// configure the port forwards per service
	err = configurePorts(box)
	if err != nil {
		util.HandleError(stylish.Error("Failed to configure Ports", err.Error()))
		util.UpdateStatus(j, "errored")
		return
	}

	// configure the routing mesh for any web services
	err = configureRoutes(box)
	if err != nil {
		util.HandleError(stylish.Error("Failed to configure Routes", err.Error()))
		util.UpdateStatus(j, "errored")
		return
	}

	//
	util.LogDebug(stylish.Bullet("Running after deploy hooks..."))

	// after deploy hooks
	for _, node := range box.Nodes() {
		ad := box.Node(node).Value("after_deploy")
		ada := box.Node(node).Value("after_deploy_all")
		if ad != nil || ada != nil {

			// run after deploy hook (blocking)
			if out, err := script.Exec("default-after_deploy", node, map[string]interface{}{"after_deploy": ad, "after_deploy_all": ada}); err != nil {
				util.LogDebug("Failed script output: \n %s", out)
				util.HandleError(stylish.Error("Failed to run after_deploy script", err.Error()))
				util.UpdateStatus(j, "errored")
				return
			}
		}
	}

	util.UpdateStatus(j, "complete")
}
<commit_msg>update the boxfile node in the payload after merging the two boxfiles<commit_after>// Copyright (c) 2014 Pagoda Box Inc.
//
// This Source Code Form is subject to the terms of the Mozilla Public License,
// v. 2.0. If a copy of the MPL was not distributed with this file, You can
// obtain one at http://mozilla.org/MPL/2.0/.

//
package jobs

//
import (
	"strings"

	"github.com/nanobox-io/nanobox-boxfile"
	"github.com/nanobox-io/nanobox-golang-stylish"
	// "github.com/nanobox-io/nanobox-logtap"
	"github.com/nanobox-io/nanobox-router"
	"github.com/nanobox-io/nanobox-server/config"
	"github.com/nanobox-io/nanobox-server/util"
	"github.com/nanobox-io/nanobox-server/util/docker"
	"github.com/nanobox-io/nanobox-server/util/fs"
	"github.com/nanobox-io/nanobox-server/util/script"
	"github.com/nanobox-io/nanobox-server/util/worker"
)

//
type Deploy struct {
	ID    string
	Reset bool
	Run   bool

	payload map[string]interface{}
}

// Proccess syncronies your docker containers with the boxfile specification
func (j *Deploy) Process() {
	// add a lock so the service wont go down whil im running
	util.Lock()
	defer util.Unlock()

	// set routing to watch logs
	router.ErrorHandler = router.DeployInProgress{}

	// remove all code containers
	util.LogInfo(stylish.Bullet("Cleaning containers"))

	// might as well remove bootstraps and execs too
	containers, _ := docker.ListContainers("code", "build", "bootstrap", "exec", "tcp", "udp")
	for _, container := range containers {
		util.RemoveForward(container.NetworkSettings.IPAddress)
		if err := docker.RemoveContainer(container.ID); err != nil {
			util.HandleError(stylish.Error("Failed to remove old containers", err.Error()))
			util.UpdateStatus(j, "errored")
			return
		}
	}

	// Make sure we have the directories
	if err := fs.CreateDirs(); err != nil {
		util.HandleError(stylish.Error("Failed to create dirs", err.Error()))
		util.UpdateStatus(j, "errored")
		return
	}

	// wipe the previous deploy data if reset == true
	if j.Reset {
		util.LogInfo(stylish.Bullet("Emptying cache"))
		if err := fs.Clean(); err != nil {
			util.HandleError(stylish.Warning("Failed to reset cache and code directories:\n%v", err.Error()))
		}
	}

	// parse the boxfile
	util.LogDebug(stylish.Bullet("Parsing Boxfile"))
	box := boxfile.NewFromPath("/vagrant/code/" + config.App + "/Boxfile")

	image := "nanobox/build"

	if stab := box.Node("build").StringValue("stability"); stab != "" {
		image = image + ":" + stab
	}

	// if the build image doesn't exist it needs to be downloaded
	if !docker.ImageExists(image) {
		util.LogInfo(stylish.Bullet("Pulling the latest build image (this may take awhile)... "))
		docker.InstallImage(image)
	}

	util.LogDebug(stylish.Bullet("image name: %v", image))

	// create a build container
	util.LogInfo(stylish.Bullet("Creating build container"))

	_, err := docker.CreateContainer(docker.CreateConfig{Image: image, Category: "build", UID: "build1"})
	if err != nil {
		util.HandleError(stylish.Error("Failed to create build container", err.Error()))
		util.UpdateStatus(j, "errored")
		return
	}

	// define the deploy payload
	j.payload = map[string]interface{}{
		"platform":    "local",
		"app":         config.App,
		"dns":         []string{config.App + ".dev"},
		"port":        "8080",
		"boxfile":     box.Node("build").Parsed,
		"logtap_host": config.LogtapHost,
	}

	evar := map[string]string{}
	if box.Node("env").Valid {
		for key, val := range box.Node("env").Parsed {
			if str, ok := val.(string); ok {
				evar[key] = str
			}
		}
	}

	evar["APP_NAME"] = config.App
	j.payload["env"] = evar

	// run the default-user hook to get ssh keys setup
	if out, err := script.Exec("default-user", "build1", fs.UserPayload()); err != nil {
		util.LogDebug("Failed script output: \n %s", out)
		util.HandleError(stylish.Error("Failed to run user script", err.Error()))
		util.UpdateStatus(j, "errored")
		return
	}

	// run configure hook (blocking)
	if out, err := script.Exec("default-configure", "build1", j.payload); err != nil {
		util.LogDebug("Failed script output: \n %s", out)
		util.HandleError(stylish.Error("Failed to run configure script", err.Error()))
		util.UpdateStatus(j, "errored")
		return
	}

	// run detect script (blocking)
	if out, err := script.Exec("default-detect", "build1", j.payload); err != nil {
		util.LogDebug("Failed script output: \n %s", out)
		util.HandleError(stylish.Error("Failed to run detect script", err.Error()))
		util.UpdateStatus(j, "errored")
		return
	}

	// run sync script (blocking)
	if out, err := script.Exec("default-sync", "build1", j.payload); err != nil {
		util.LogDebug("Failed script output: \n %s", out)
		util.HandleError(stylish.Error("Failed to run sync script", err.Error()))
		util.UpdateStatus(j, "errored")
		return
	}

	// run setup script (blocking)
	if out, err := script.Exec("default-setup", "build1", j.payload); err != nil {
		util.LogDebug("Failed script output: \n %s", out)
		util.HandleError(stylish.Error("Failed to run setup script", err.Error()))
		util.UpdateStatus(j, "errored")
		return
	}

	// run boxfile script (blocking)
	if !box.Node("build").BoolValue("disable_engine_boxfile") {
		if out, err := script.Exec("default-boxfile", "build1", j.payload); err != nil {
			util.LogDebug("Failed script output: \n %s", out)
			util.HandleError(stylish.Error("Failed to run boxfile script", err.Error()))
			util.UpdateStatus(j, "errored")
			return

			// if the script runs succesfully merge the boxfiles
		} else {
			util.LogDebug(stylish.Bullet("Merging Boxfiles..."))
			box.Merge(boxfile.New([]byte(out)))
		}
	}

	// add the missing storage nodes to the boxfile
	box.AddStorageNode()
	j.Payload["boxfile"] = box.Node("build").Parsed

	// remove any containers no longer in the boxfile
	util.LogDebug(stylish.Bullet("Removing old containers..."))
	serviceContainers, _ := docker.ListContainers("service")
	for _, container := range serviceContainers {
		if !box.Node(container.Config.Labels["uid"]).Valid {
			util.RemoveForward(container.NetworkSettings.IPAddress)
			docker.RemoveContainer(container.ID)
		}
	}

	worker := worker.New()
	worker.Blocking = true
	worker.Concurrent = true

	//
	serviceStarts := []*ServiceStart{}

	// build service containers according to boxfile
	for _, node := range box.Nodes("service") {
		if _, err := docker.GetContainer(node); err != nil {
			// container doesn't exist so we need to create it
			s := ServiceStart{
				Boxfile: box.Node(node),
				UID:     node,
				EVars:   map[string]string{},
			}

			serviceStarts = append(serviceStarts, &s)

			worker.Queue(&s)
		}
	}

	if worker.Count() > 0 {
		util.LogInfo(stylish.Bullet("Launching data services"))
	}

	worker.Process()

	// ensure all services started correctly before continuing
	for _, starts := range serviceStarts {
		if !starts.Success {
			util.HandleError(stylish.ErrorHead("Failed to start %v", starts.UID))
			util.HandleError(stylish.ErrorBody(""))
			util.UpdateStatus(j, "errored")
			return
		}
	}

	// grab the environment data from all service containers
	evars := j.payload["env"].(map[string]string)

	// clear out the old ports from the previous deploy
	clearPorts()

	//
	serviceEnvs := []*ServiceEnv{}

	serviceContainers, _ = docker.ListContainers("service")
	for _, container := range serviceContainers {

		s := ServiceEnv{UID: container.Config.Labels["uid"]}
		serviceEnvs = append(serviceEnvs, &s)

		worker.Queue(&s)
	}

	worker.Process()

	for _, env := range serviceEnvs {
		if !env.Success {
			util.HandleError(stylish.ErrorHead("Failed to configure %v's environment variables", env.UID))
			util.HandleError(stylish.ErrorBody(""))
			util.UpdateStatus(j, "errored")
			return
		}

		for key, val := range env.EVars {
			evars[strings.ToUpper(env.UID+"_"+key)] = val
		}
	}

	j.payload["env"] = evars

	// run prepare script (blocking)
	if out, err := script.Exec("default-prepare", "build1", j.payload); err != nil {
		util.LogDebug("Failed script output: \n %s", out)
		util.HandleError(stylish.Error("Failed to run prepare script", err.Error()))
		util.UpdateStatus(j, "errored")
		return
	}

	if j.Run {
		// run build script (blocking)
		if out, err := script.Exec("default-build", "build1", j.payload); err != nil {
			util.LogDebug("Failed script output: \n %s", out)
			util.HandleError(stylish.Error("Failed to run build script", err.Error()))
			util.UpdateStatus(j, "errored")
			return
		}

		// run publish script (blocking)
		if out, err := script.Exec("default-publish", "build1", j.payload); err != nil {
			util.LogDebug("Failed script output: \n %s", out)
			util.HandleError(stylish.Error("Failed to run publish script", err.Error()))
			util.UpdateStatus(j, "errored")
			return
		}
	}

	// run cleanup script (blocking)
	if out, err := script.Exec("default-cleanup", "build1", j.payload); err != nil {
		util.LogDebug("Failed script output: \n %s", out)
		util.HandleError(stylish.Error("Failed to run cleanup script", err.Error()))
		util.UpdateStatus(j, "errored")
		return
	}

	// we will only create new code nodes if we are
	// supposed to be running
	if j.Run {

		// build new code containers
		codeServices := []*ServiceStart{}
		for _, node := range box.Nodes("code") {
			if _, err := docker.GetContainer(node); err != nil {
				// container doesn't exist so we need to create it
				s := ServiceStart{
					Boxfile: box.Node(node),
					UID:     node,
					EVars:   evars,
				}

				codeServices = append(codeServices, &s)

				worker.Queue(&s)
			}
			if worker.Count() > 0 {
				util.LogInfo(stylish.Bullet("Launching Code services"))
			}
		}

		worker.Process()

		for _, serv := range codeServices {
			if !serv.Success {
				util.HandleError("A Service was not started correctly (" + serv.UID + ")")
				util.UpdateStatus(j, "errored")
				return
			}
		}
	}

	util.LogDebug(stylish.Bullet("Running before deploy scripts..."))

	// run before deploy scripts
	for _, node := range box.Nodes() {
		bd := box.Node(node).Value("before_deploy")
		bda := box.Node(node).Value("before_deploy_all")
		if bd != nil || bda != nil {

			// run before deploy script (blocking)
			if out, err := script.Exec("default-before_deploy", node, map[string]interface{}{"before_deploy": bd, "before_deploy_all": bda}); err != nil {
				util.LogDebug("Failed script output: \n %s", out)
				util.HandleError(stylish.Error("Failed to run before_deploy script", err.Error()))
				util.UpdateStatus(j, "errored")
				return
			}
		}
	}

	// configure the port forwards per service
	err = configurePorts(box)
	if err != nil {
		util.HandleError(stylish.Error("Failed to configure Ports", err.Error()))
		util.UpdateStatus(j, "errored")
		return
	}

	// configure the routing mesh for any web services
	err = configureRoutes(box)
	if err != nil {
		util.HandleError(stylish.Error("Failed to configure Routes", err.Error()))
		util.UpdateStatus(j, "errored")
		return
	}

	//
	util.LogDebug(stylish.Bullet("Running after deploy hooks..."))

	// after deploy hooks
	for _, node := range box.Nodes() {
		ad := box.Node(node).Value("after_deploy")
		ada := box.Node(node).Value("after_deploy_all")
		if ad != nil || ada != nil {

			// run after deploy hook (blocking)
			if out, err := script.Exec("default-after_deploy", node, map[string]interface{}{"after_deploy": ad, "after_deploy_all": ada}); err != nil {
				util.LogDebug("Failed script output: \n %s", out)
				util.HandleError(stylish.Error("Failed to run after_deploy script", err.Error()))
				util.UpdateStatus(j, "errored")
				return
			}
		}
	}

	util.UpdateStatus(j, "complete")
}
<|endoftext|>


<commit_before>package main

import (
    "encoding/binary"
    "xcl"
    "fmt"
)

func main() {
    // Allocate a 'world' for interacting with kernels
    world := xcl.NewWorld()
    defer world.Release()

    // Import the kernel.
    // Right now these two identifiers are hard coded as an output from the build process
    krnl := world.Import("kernel_test").GetKernel("reconfigure_io_sdaccel_builder_stub_0_1")
    defer krnl.Release()

    // Create/get data and pass arguments to the kernel as required. These could be small pieces of data,
    // pointers to memory, data lengths so the Kernel knows what to expect. This all depends on your project.
    // We have passed three arguments here, you can pass more as neccessary

    // make an array to send to the kernel for processing
    input := make([]uint32, 10)

	  // seed it with incrementing values
  	for i, _ := range input {
  		input[i] = uint32(i)
  	}

    fmt.Println("Here is our example array:")

    for _, val := range input {
      print(val)
    }

    // Create space in shared memory for our array input
  	buff := world.Malloc(xcl.ReadOnly, uint(binary.Size(input)))
  	defer buff.Free()

    // Create a variable to hold the output from the FPGA
  	var output [10]uint32

  	// Create space in the shared memory for the output from the FPGA
  	outputBuff := world.Malloc(xcl.ReadWrite, uint(binary.Size(output)))
  	defer outputBuff.Free()

  	// write our input to the shared memory at the location we specified previously
  	binary.Write(buff.Writer(), binary.LittleEndian, &input)

  	// zero out output space
  	binary.Write(outputBuff.Writer(), binary.LittleEndian, &output)

    // Send the location of the input array as the first argument
    krnl.SetMemoryArg(0, buff)
    // Send the location the FPGA should put the result as the second argument
    krnl.SetMemoryArg(1, outputBuff)
    // Send the length of the input array, so the kernel knows what to expect, as the third argument
    krnl.SetArg(2, uint32(len(input)))

    // Run the kernel with the supplied arguments. This is the same for all projects.
    // The arguments ``(1, 1, 1)`` relate to x, y, z co-ordinates and correspond to our current
    // underlying technology.
    krnl.Run(1, 1, 1)

    // Display/use the results returned from the FPGA as required!

    binary.Read(outputBuff.Reader(), binary.LittleEndian, &output);

    fmt.Println("...and here is our array with each integer multiplied by 2.")

    for _, val := range output {
      print(val)
    }

}
<commit_msg>take text out<commit_after>package main

import (
    "encoding/binary"
    "xcl"
    "fmt"
)

func main() {
    // Allocate a 'world' for interacting with kernels
    world := xcl.NewWorld()
    defer world.Release()

    // Import the kernel.
    // Right now these two identifiers are hard coded as an output from the build process
    krnl := world.Import("kernel_test").GetKernel("reconfigure_io_sdaccel_builder_stub_0_1")
    defer krnl.Release()

    // Create/get data and pass arguments to the kernel as required. These could be small pieces of data,
    // pointers to memory, data lengths so the Kernel knows what to expect. This all depends on your project.
    // We have passed three arguments here, you can pass more as neccessary

    // make an array to send to the kernel for processing
    input := make([]uint32, 10)

	  // seed it with incrementing values
  	for i, _ := range input {
  		input[i] = uint32(i)
  	}

    // Create space in shared memory for our array input
  	buff := world.Malloc(xcl.ReadOnly, uint(binary.Size(input)))
  	defer buff.Free()

    // Create a variable to hold the output from the FPGA
  	var output [10]uint32

  	// Create space in the shared memory for the output from the FPGA
  	outputBuff := world.Malloc(xcl.ReadWrite, uint(binary.Size(output)))
  	defer outputBuff.Free()

  	// write our input to the shared memory at the location we specified previously
  	binary.Write(buff.Writer(), binary.LittleEndian, &input)

  	// zero out output space
  	binary.Write(outputBuff.Writer(), binary.LittleEndian, &output)

    // Send the location of the input array as the first argument
    krnl.SetMemoryArg(0, buff)
    // Send the location the FPGA should put the result as the second argument
    krnl.SetMemoryArg(1, outputBuff)
    // Send the length of the input array, so the kernel knows what to expect, as the third argument
    krnl.SetArg(2, uint32(len(input)))

    // Run the kernel with the supplied arguments. This is the same for all projects.
    // The arguments ``(1, 1, 1)`` relate to x, y, z co-ordinates and correspond to our current
    // underlying technology.
    krnl.Run(1, 1, 1)

    // Display/use the results returned from the FPGA as required!

    binary.Read(outputBuff.Reader(), binary.LittleEndian, &output);

    for _, val := range output {
      print(val)
    }

}
<|endoftext|>


<commit_before>package cockroach

import (
	//_ "github.com/denisenkom/go-postgresdb"
	postgre "github.com/rbastic/dyndao/adapters/postgres"
	sg "github.com/rbastic/dyndao/sqlgen"
)

// New shows off a sort of inheritance/composition-using-vtables approach.
// It receives the SQLBuilder composed by Core and then overrides any
// methods that it needs to. In some instances, this could be all methods,
// or hardly any.
func New(g *sg.SQLBuilder) *sg.SQLBuilder {
	g.IsPOSTGRES = true
	g.FixLastInsertIDbug = true
	g.IsStringType = sg.FnIsStringType(postgre.IsStringType)
	g.MapJSONPath = sg.FnMapJSONPath(postgre.MapJSONPath)
	g.IsNumberType = sg.FnIsNumberType(postgre.IsNumberType)
	g.IsFloatingType = sg.FnIsFloatingType(postgre.IsFloatingType)
	g.IsTimestampType = sg.FnIsTimestampType(postgre.IsTimestampType)
	g.IsLOBType = sg.FnIsLOBType(postgre.IsLOBType)
	g.BindingInsertSQL = sg.FnBindingInsertSQL(postgre.BindingInsertSQL)
	g.RenderCreateColumn = sg.FnRenderCreateColumn(postgre.RenderCreateColumn)
	g.GenPlaceholder = sg.FnGenPlaceholder(postgre.GenPlaceholder)
	g.BindingUpdate = sg.FnBindingUpdate(postgre.BindingUpdate)
	g.BindObject = sg.FnBindObject(postgre.BindObject)
	g.MakeColumnPointers = sg.FnMakeColumnPointers(postgre.MakeColumnPointers)
	return g
}
<commit_msg>Minor compatibility fix for CockroachDB<commit_after>package cockroach

import (
	postgre "github.com/rbastic/dyndao/adapters/postgres"
	sg "github.com/rbastic/dyndao/sqlgen"
)

// New shows off a sort of inheritance/composition-using-vtables approach.
// It receives the SQLBuilder composed by Core and then overrides any
// methods that it needs to. In some instances, this could be all methods,
// or hardly any.
func New(g *sg.SQLBuilder) *sg.SQLBuilder {
	g.IsPOSTGRES = true
	g.FixLastInsertIDbug = true
	g.CreateTable = sg.FnCreateTable(postgre.CreateTable)
	g.DropTable = sg.FnDropTable(postgre.DropTable)
	g.IsStringType = sg.FnIsStringType(postgre.IsStringType)
	g.MapJSONPath = sg.FnMapJSONPath(postgre.MapJSONPath)
	g.IsNumberType = sg.FnIsNumberType(postgre.IsNumberType)
	g.IsFloatingType = sg.FnIsFloatingType(postgre.IsFloatingType)
	g.IsTimestampType = sg.FnIsTimestampType(postgre.IsTimestampType)
	g.IsLOBType = sg.FnIsLOBType(postgre.IsLOBType)
	g.BindingInsertSQL = sg.FnBindingInsertSQL(postgre.BindingInsertSQL)
	g.RenderCreateColumn = sg.FnRenderCreateColumn(postgre.RenderCreateColumn)
	g.GenPlaceholder = sg.FnGenPlaceholder(postgre.GenPlaceholder)
	g.BindingUpdate = sg.FnBindingUpdate(postgre.BindingUpdate)
	//g.BindObject = sg.FnBindObject(postgre.BindObject)
	//g.MakeColumnPointers = sg.FnMakeColumnPointers(postgre.MakeColumnPointers)
	return g
}
<|endoftext|>


<commit_before>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package lease

import (
	"encoding/binary"
	"fmt"
	"math"
	"sync"
	"time"

	"github.com/coreos/etcd/lease/leasepb"
	"github.com/coreos/etcd/pkg/idutil"
	"github.com/coreos/etcd/storage/backend"
)

var (
	minLeaseTerm = 5 * time.Second

	leaseBucketName = []byte("lease")
)

type LeaseID int64

// DeleteableRange defines an interface with DeleteRange method.
// We define this interface only for lessor to limit the number
// of methods of storage.KV to what lessor actually needs.
//
// Having a minimum interface makes testing easy.
type DeleteableRange interface {
	DeleteRange(key, end []byte) (int64, int64)
}

// a lessor is the owner of leases. It can grant, revoke,
// renew and modify leases for lessee.
// TODO: use clockwork for testability.
type lessor struct {
	mu sync.Mutex
	// TODO: probably this should be a heap with a secondary
	// id index.
	// Now it is O(N) to loop over the leases to find expired ones.
	// We want to make Grant, Revoke, and FindExpired all O(logN) and
	// Renew O(1).
	// FindExpired and Renew should be the most frequent operations.
	leaseMap map[LeaseID]*lease

	// A DeleteableRange the lessor operates on.
	// When a lease expires, the lessor will delete the
	// leased range (or key) from the DeleteableRange.
	dr DeleteableRange

	// backend to persist leases. We only persist lease ID and expiry for now.
	// The leased items can be recovered by iterating all the keys in kv.
	b backend.Backend

	idgen *idutil.Generator
}

func NewLessor(lessorID uint8, b backend.Backend, dr DeleteableRange) *lessor {
	l := &lessor{
		leaseMap: make(map[LeaseID]*lease),
		b:        b,
		dr:       dr,
		idgen:    idutil.NewGenerator(lessorID, time.Now()),
	}
	l.initAndRecover()

	return l
}

// Grant grants a lease that expires at least after TTL seconds.
// TODO: when lessor is under high load, it should give out lease
// with longer TTL to reduce renew load.
func (le *lessor) Grant(ttl int64) *lease {
	// TODO: define max TTL
	expiry := time.Now().Add(time.Duration(ttl) * time.Second)
	expiry = minExpiry(time.Now(), expiry)

	id := LeaseID(le.idgen.Next())

	le.mu.Lock()
	defer le.mu.Unlock()

	l := &lease{id: id, ttl: ttl, expiry: expiry, itemSet: make(map[leaseItem]struct{})}
	if _, ok := le.leaseMap[id]; ok {
		panic("lease: unexpected duplicate ID!")
	}

	le.leaseMap[id] = l
	l.persistTo(le.b)

	return l
}

// Revoke revokes a lease with given ID. The item attached to the
// given lease will be removed. If the ID does not exist, an error
// will be returned.
func (le *lessor) Revoke(id LeaseID) error {
	le.mu.Lock()
	defer le.mu.Unlock()

	l := le.leaseMap[id]
	if l == nil {
		return fmt.Errorf("lease: cannot find lease %x", id)
	}

	for item := range l.itemSet {
		le.dr.DeleteRange([]byte(item.key), nil)
	}

	delete(le.leaseMap, l.id)
	l.removeFrom(le.b)

	return nil
}

// Renew renews an existing lease. If the given lease does not exist or
// has expired, an error will be returned.
// TODO: return new TTL?
func (le *lessor) Renew(id LeaseID) error {
	le.mu.Lock()
	defer le.mu.Unlock()

	l := le.leaseMap[id]
	if l == nil {
		return fmt.Errorf("lease: cannot find lease %x", id)
	}

	expiry := time.Now().Add(time.Duration(l.ttl) * time.Second)
	l.expiry = minExpiry(time.Now(), expiry)
	return nil
}

// Attach attaches items to the lease with given ID. When the lease
// expires, the attached items will be automatically removed.
// If the given lease does not exist, an error will be returned.
func (le *lessor) Attach(id LeaseID, items []leaseItem) error {
	le.mu.Lock()
	defer le.mu.Unlock()

	l := le.leaseMap[id]
	if l == nil {
		return fmt.Errorf("lease: cannot find lease %x", id)
	}

	for _, it := range items {
		l.itemSet[it] = struct{}{}
	}
	return nil
}

// findExpiredLeases loops all the leases in the leaseMap and returns the expired
// leases that needed to be revoked.
func (le *lessor) findExpiredLeases() []*lease {
	le.mu.Lock()
	defer le.mu.Unlock()

	leases := make([]*lease, 0, 16)
	now := time.Now()

	for _, l := range le.leaseMap {
		if l.expiry.Sub(now) <= 0 {
			leases = append(leases, l)
		}
	}

	return leases
}

// get gets the lease with given id.
// get is a helper fucntion for testing, at least for now.
func (le *lessor) get(id LeaseID) *lease {
	le.mu.Lock()
	defer le.mu.Unlock()

	return le.leaseMap[id]
}

func (le *lessor) initAndRecover() {
	tx := le.b.BatchTx()
	tx.Lock()
	defer tx.Unlock()

	tx.UnsafeCreateBucket(leaseBucketName)
	_, vs := tx.UnsafeRange(leaseBucketName, int64ToBytes(0), int64ToBytes(math.MaxInt64), 0)
	// TODO: copy vs and do decoding outside tx lock if lock contention becomes an issue.
	for i := range vs {
		var lpb leasepb.Lease
		err := lpb.Unmarshal(vs[i])
		if err != nil {
			panic("failed to unmarshal lease proto item")
		}
		id := LeaseID(lpb.ID)
		le.leaseMap[id] = &lease{
			id:  id,
			ttl: lpb.TTL,

			// itemSet will be filled in when recover key-value pairs
			expiry: minExpiry(time.Now(), time.Now().Add(time.Second*time.Duration(lpb.TTL))),
		}
	}
	le.b.ForceCommit()
}

type lease struct {
	id  LeaseID
	ttl int64 // time to live in seconds

	itemSet map[leaseItem]struct{}
	// expiry time in unixnano
	expiry time.Time
}

func (l lease) persistTo(b backend.Backend) {
	key := int64ToBytes(int64(l.id))

	lpb := leasepb.Lease{ID: int64(l.id), TTL: int64(l.ttl)}
	val, err := lpb.Marshal()
	if err != nil {
		panic("failed to marshal lease proto item")
	}

	b.BatchTx().Lock()
	b.BatchTx().UnsafePut(leaseBucketName, key, val)
	b.BatchTx().Unlock()
}

func (l lease) removeFrom(b backend.Backend) {
	key := int64ToBytes(int64(l.id))

	b.BatchTx().Lock()
	b.BatchTx().UnsafeDelete(leaseBucketName, key)
	b.BatchTx().Unlock()
}

type leaseItem struct {
	key string
}

// minExpiry returns a minimal expiry. A minimal expiry is the larger on
// between now + minLeaseTerm and the given expectedExpiry.
func minExpiry(now time.Time, expectedExpiry time.Time) time.Time {
	minExpiry := time.Now().Add(minLeaseTerm)
	if expectedExpiry.Sub(minExpiry) < 0 {
		expectedExpiry = minExpiry
	}
	return expectedExpiry
}

func int64ToBytes(n int64) []byte {
	bytes := make([]byte, 8)
	binary.BigEndian.PutUint64(bytes, uint64(n))
	return bytes
}
<commit_msg>lease: unlock before another batch operation<commit_after>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package lease

import (
	"encoding/binary"
	"fmt"
	"math"
	"sync"
	"time"

	"github.com/coreos/etcd/lease/leasepb"
	"github.com/coreos/etcd/pkg/idutil"
	"github.com/coreos/etcd/storage/backend"
)

var (
	minLeaseTerm = 5 * time.Second

	leaseBucketName = []byte("lease")
)

type LeaseID int64

// DeleteableRange defines an interface with DeleteRange method.
// We define this interface only for lessor to limit the number
// of methods of storage.KV to what lessor actually needs.
//
// Having a minimum interface makes testing easy.
type DeleteableRange interface {
	DeleteRange(key, end []byte) (int64, int64)
}

// a lessor is the owner of leases. It can grant, revoke,
// renew and modify leases for lessee.
// TODO: use clockwork for testability.
type lessor struct {
	mu sync.Mutex
	// TODO: probably this should be a heap with a secondary
	// id index.
	// Now it is O(N) to loop over the leases to find expired ones.
	// We want to make Grant, Revoke, and FindExpired all O(logN) and
	// Renew O(1).
	// FindExpired and Renew should be the most frequent operations.
	leaseMap map[LeaseID]*lease

	// A DeleteableRange the lessor operates on.
	// When a lease expires, the lessor will delete the
	// leased range (or key) from the DeleteableRange.
	dr DeleteableRange

	// backend to persist leases. We only persist lease ID and expiry for now.
	// The leased items can be recovered by iterating all the keys in kv.
	b backend.Backend

	idgen *idutil.Generator
}

func NewLessor(lessorID uint8, b backend.Backend, dr DeleteableRange) *lessor {
	l := &lessor{
		leaseMap: make(map[LeaseID]*lease),
		b:        b,
		dr:       dr,
		idgen:    idutil.NewGenerator(lessorID, time.Now()),
	}
	l.initAndRecover()

	return l
}

// Grant grants a lease that expires at least after TTL seconds.
// TODO: when lessor is under high load, it should give out lease
// with longer TTL to reduce renew load.
func (le *lessor) Grant(ttl int64) *lease {
	// TODO: define max TTL
	expiry := time.Now().Add(time.Duration(ttl) * time.Second)
	expiry = minExpiry(time.Now(), expiry)

	id := LeaseID(le.idgen.Next())

	le.mu.Lock()
	defer le.mu.Unlock()

	l := &lease{id: id, ttl: ttl, expiry: expiry, itemSet: make(map[leaseItem]struct{})}
	if _, ok := le.leaseMap[id]; ok {
		panic("lease: unexpected duplicate ID!")
	}

	le.leaseMap[id] = l
	l.persistTo(le.b)

	return l
}

// Revoke revokes a lease with given ID. The item attached to the
// given lease will be removed. If the ID does not exist, an error
// will be returned.
func (le *lessor) Revoke(id LeaseID) error {
	le.mu.Lock()
	defer le.mu.Unlock()

	l := le.leaseMap[id]
	if l == nil {
		return fmt.Errorf("lease: cannot find lease %x", id)
	}

	for item := range l.itemSet {
		le.dr.DeleteRange([]byte(item.key), nil)
	}

	delete(le.leaseMap, l.id)
	l.removeFrom(le.b)

	return nil
}

// Renew renews an existing lease. If the given lease does not exist or
// has expired, an error will be returned.
// TODO: return new TTL?
func (le *lessor) Renew(id LeaseID) error {
	le.mu.Lock()
	defer le.mu.Unlock()

	l := le.leaseMap[id]
	if l == nil {
		return fmt.Errorf("lease: cannot find lease %x", id)
	}

	expiry := time.Now().Add(time.Duration(l.ttl) * time.Second)
	l.expiry = minExpiry(time.Now(), expiry)
	return nil
}

// Attach attaches items to the lease with given ID. When the lease
// expires, the attached items will be automatically removed.
// If the given lease does not exist, an error will be returned.
func (le *lessor) Attach(id LeaseID, items []leaseItem) error {
	le.mu.Lock()
	defer le.mu.Unlock()

	l := le.leaseMap[id]
	if l == nil {
		return fmt.Errorf("lease: cannot find lease %x", id)
	}

	for _, it := range items {
		l.itemSet[it] = struct{}{}
	}
	return nil
}

// findExpiredLeases loops all the leases in the leaseMap and returns the expired
// leases that needed to be revoked.
func (le *lessor) findExpiredLeases() []*lease {
	le.mu.Lock()
	defer le.mu.Unlock()

	leases := make([]*lease, 0, 16)
	now := time.Now()

	for _, l := range le.leaseMap {
		if l.expiry.Sub(now) <= 0 {
			leases = append(leases, l)
		}
	}

	return leases
}

// get gets the lease with given id.
// get is a helper fucntion for testing, at least for now.
func (le *lessor) get(id LeaseID) *lease {
	le.mu.Lock()
	defer le.mu.Unlock()

	return le.leaseMap[id]
}

func (le *lessor) initAndRecover() {
	tx := le.b.BatchTx()
	tx.Lock()

	tx.UnsafeCreateBucket(leaseBucketName)
	_, vs := tx.UnsafeRange(leaseBucketName, int64ToBytes(0), int64ToBytes(math.MaxInt64), 0)
	// TODO: copy vs and do decoding outside tx lock if lock contention becomes an issue.
	for i := range vs {
		var lpb leasepb.Lease
		err := lpb.Unmarshal(vs[i])
		if err != nil {
			tx.Unlock()
			panic("failed to unmarshal lease proto item")
		}
		id := LeaseID(lpb.ID)
		le.leaseMap[id] = &lease{
			id:  id,
			ttl: lpb.TTL,

			// itemSet will be filled in when recover key-value pairs
			expiry: minExpiry(time.Now(), time.Now().Add(time.Second*time.Duration(lpb.TTL))),
		}
	}
	tx.Unlock()

	le.b.ForceCommit()
}

type lease struct {
	id  LeaseID
	ttl int64 // time to live in seconds

	itemSet map[leaseItem]struct{}
	// expiry time in unixnano
	expiry time.Time
}

func (l lease) persistTo(b backend.Backend) {
	key := int64ToBytes(int64(l.id))

	lpb := leasepb.Lease{ID: int64(l.id), TTL: int64(l.ttl)}
	val, err := lpb.Marshal()
	if err != nil {
		panic("failed to marshal lease proto item")
	}

	b.BatchTx().Lock()
	b.BatchTx().UnsafePut(leaseBucketName, key, val)
	b.BatchTx().Unlock()
}

func (l lease) removeFrom(b backend.Backend) {
	key := int64ToBytes(int64(l.id))

	b.BatchTx().Lock()
	b.BatchTx().UnsafeDelete(leaseBucketName, key)
	b.BatchTx().Unlock()
}

type leaseItem struct {
	key string
}

// minExpiry returns a minimal expiry. A minimal expiry is the larger on
// between now + minLeaseTerm and the given expectedExpiry.
func minExpiry(now time.Time, expectedExpiry time.Time) time.Time {
	minExpiry := time.Now().Add(minLeaseTerm)
	if expectedExpiry.Sub(minExpiry) < 0 {
		expectedExpiry = minExpiry
	}
	return expectedExpiry
}

func int64ToBytes(n int64) []byte {
	bytes := make([]byte, 8)
	binary.BigEndian.PutUint64(bytes, uint64(n))
	return bytes
}
<|endoftext|>


<commit_before>package stream_aggregator

import (
        "fmt"
        "time"
        "github.com/mozilla-services/heka/message"
        . "github.com/mozilla-services/heka/pipeline"
        "code.google.com/p/go-uuid/uuid"
        "sync"
)

type StreamAggregatorFilter struct {
        *StreamAggregatorFilterConfig
        batchChan chan []byte
        backChan  chan []byte
        msgLoopCount uint
}

type StreamAggregatorFilterConfig struct {
        FlushInterval uint32 `toml:"flush_interval"`
        FlushBytes    int    `toml:"flush_bytes"`
        StreamAggregatorTag       string `toml:"stream_aggregator_tag"`
        EncoderName   string `toml:"encoder"`
}

func (f *StreamAggregatorFilter) ConfigStruct() interface{} {
        return &StreamAggregatorFilterConfig{
        FlushInterval: 1000,
        FlushBytes:    10,
        StreamAggregatorTag:       "aggregated",
        }
}

func (f *StreamAggregatorFilter) Init(config interface{}) (err error) {
        f.StreamAggregatorFilterConfig = config.(*StreamAggregatorFilterConfig)
        f.batchChan = make(chan []byte)
        f.backChan = make(chan []byte, 2)

        if f.StreamAggregatorTag == "" {
            return fmt.Errorf(`A stream_aggregator_tag value must be specified for the StreamAggregatorTag Field`)
        }

        if f.EncoderName == "" {
            return fmt.Errorf(`An encoder must be specified`)
        }

        return
}

func (f *StreamAggregatorFilter) committer(fr FilterRunner, h PluginHelper, wg *sync.WaitGroup) {
        initBatch := make([]byte, 0, 10000)
        f.backChan <- initBatch
        var (
            tag string
            //ok bool
            outBatch []byte
        )
        tag = f.StreamAggregatorTag

        for outBatch = range f.batchChan {
                pack := h.PipelinePack(f.msgLoopCount)
                if pack == nil {
                        fr.LogError(fmt.Errorf("exceeded MaxMsgLoops = %d",
                                h.PipelineConfig().Globals.MaxMsgLoops))
            break
                }

                tagField, _ := message.NewField("StreamAggregatorTag", tag, "")
                pack.Message.AddField(tagField)
                pack.Message.SetUuid(uuid.NewRandom())
                pack.Message.SetPayload(string(outBatch))
                fr.Inject(pack)

                outBatch = outBatch[:0]
                f.backChan <- outBatch
        }
        wg.Done()
}

func (f *StreamAggregatorFilter) receiver(fr FilterRunner, h PluginHelper, encoder Encoder, wg *sync.WaitGroup) {
        var (
                pack *PipelinePack
                ok bool
                e        error
        )
        ok = true
        outBatch := make([]byte, 0, 10000)
        outBytes := make([]byte, 0, 10000)
        ticker := time.Tick(time.Duration(f.FlushInterval) * time.Millisecond)
        inChan := fr.InChan()

        for ok {
                select {
                case pack, ok = <-inChan:
                        if !ok {
                                // Closed inChan => we're shutting down, flush data
                                if len(outBatch) > 0 {
                                        f.batchChan <- outBatch
                                }
                                close(f.batchChan)
                                break
                        }
                        f.msgLoopCount = pack.MsgLoopCount

                        if outBytes, e = encoder.Encode(pack); e != nil {
                                fr.LogError(fmt.Errorf("Error encoding message: %s", e))
                        } else {
                            if len(outBytes) > 0 {
                                outBytes = append(outBytes, '\n')
                                outBatch = append(outBatch, outBytes...)

                                if len(outBatch) > f.FlushBytes {
                                        f.batchChan <- outBatch
                                        outBatch = <-f.backChan
                                }
                            }
                            outBytes = outBytes[:0]
                        }
                        pack.Recycle()
                case <-ticker:
                        if len(outBatch) > 0 {
                        f.batchChan <- outBatch
                        outBatch = <-f.backChan
                        }
                }
        }

        wg.Done()
}

func (f *StreamAggregatorFilter) Run(fr FilterRunner, h PluginHelper) (err error) {
        base_name := f.EncoderName
        full_name := fr.Name() + "-" + f.EncoderName
        encoder, ok := h.Encoder(base_name, full_name)
        if !ok {
            return fmt.Errorf("Encoder not found: %s", full_name)
        }

        var wg sync.WaitGroup
        wg.Add(2)
        go f.receiver(fr, h, encoder, &wg)
        go f.committer(fr, h, &wg)
        wg.Wait()

    return
}

func init() {
    RegisterPlugin("StreamAggregatorFilter", func() interface{} {
        return new(StreamAggregatorFilter)
    })
}
<commit_msg>Adding delimitter config option<commit_after>package stream_aggregator

import (
        "fmt"
        "time"
        "github.com/mozilla-services/heka/message"
        . "github.com/mozilla-services/heka/pipeline"
        "code.google.com/p/go-uuid/uuid"
        "sync"
)

type StreamAggregatorFilter struct {
        *StreamAggregatorFilterConfig
        batchChan chan []byte
        backChan  chan []byte
        msgLoopCount uint
}

type StreamAggregatorFilterConfig struct {
        Delimitter          string `toml:"delimitter"` // Delimitter used to append to end of each protobuf for splitting on when decoding later.
                                                       // Defaults to '\n'
        FlushInterval uint32 `toml:"flush_interval"`
        FlushBytes    int    `toml:"flush_bytes"`
        StreamAggregatorTag       string `toml:"stream_aggregator_tag"`
        EncoderName   string `toml:"encoder"`
}

func (f *StreamAggregatorFilter) ConfigStruct() interface{} {
        return &StreamAggregatorFilterConfig{
        Delimitter:     "\n",
        FlushInterval: 1000,
        FlushBytes:    10,
        StreamAggregatorTag:       "aggregated",
        }
}

func (f *StreamAggregatorFilter) Init(config interface{}) (err error) {
        f.StreamAggregatorFilterConfig = config.(*StreamAggregatorFilterConfig)
        f.batchChan = make(chan []byte)
        f.backChan = make(chan []byte, 2)

        if f.StreamAggregatorTag == "" {
            return fmt.Errorf(`A stream_aggregator_tag value must be specified for the StreamAggregatorTag Field`)
        }

        if f.EncoderName == "" {
            return fmt.Errorf(`An encoder must be specified`)
        }

        return
}

func (f *StreamAggregatorFilter) committer(fr FilterRunner, h PluginHelper, wg *sync.WaitGroup) {
        initBatch := make([]byte, 0, 10000)
        f.backChan <- initBatch
        var (
            tag string
            outBatch []byte
        )
        tag = f.StreamAggregatorTag

        for outBatch = range f.batchChan {
                pack := h.PipelinePack(f.msgLoopCount)
                if pack == nil {
                        fr.LogError(fmt.Errorf("exceeded MaxMsgLoops = %d",
                                h.PipelineConfig().Globals.MaxMsgLoops))
            break
                }

                tagField, _ := message.NewField("StreamAggregatorTag", tag, "")
                pack.Message.AddField(tagField)
                pack.Message.SetUuid(uuid.NewRandom())
                pack.Message.SetPayload(string(outBatch))
                fr.Inject(pack)

                outBatch = outBatch[:0]
                f.backChan <- outBatch
        }
        wg.Done()
}

func (f *StreamAggregatorFilter) receiver(fr FilterRunner, h PluginHelper, encoder Encoder, wg *sync.WaitGroup) {
        var (
                pack *PipelinePack
                ok bool
                e        error
        )
        ok = true
        delimitter := f.Delimitter
        outBatch := make([]byte, 0, 10000)
        outBytes := make([]byte, 0, 10000)
        ticker := time.Tick(time.Duration(f.FlushInterval) * time.Millisecond)
        inChan := fr.InChan()

        for ok {
                select {
                case pack, ok = <-inChan:
                        if !ok {
                                // Closed inChan => we're shutting down, flush data
                                if len(outBatch) > 0 {
                                        f.batchChan <- outBatch
                                }
                                close(f.batchChan)
                                break
                        }
                        f.msgLoopCount = pack.MsgLoopCount

                        if outBytes, e = encoder.Encode(pack); e != nil {
                                fr.LogError(fmt.Errorf("Error encoding message: %s", e))
                        } else {
                            if len(outBytes) > 0 {
                                outBatch = append(outBatch, outBytes...)
                                outBatch = append(outBatch, delimitter...)

                                if len(outBatch) > f.FlushBytes {
                                        f.batchChan <- outBatch
                                        outBatch = <-f.backChan
                                }
                            }
                            outBytes = outBytes[:0]
                        }
                        pack.Recycle()
                case <-ticker:
                        if len(outBatch) > 0 {
                        f.batchChan <- outBatch
                        outBatch = <-f.backChan
                        }
                }
        }

        wg.Done()
}

func (f *StreamAggregatorFilter) Run(fr FilterRunner, h PluginHelper) (err error) {
        base_name := f.EncoderName
        full_name := fr.Name() + "-" + f.EncoderName
        encoder, ok := h.Encoder(base_name, full_name)
        if !ok {
            return fmt.Errorf("Encoder not found: %s", full_name)
        }

        var wg sync.WaitGroup
        wg.Add(2)
        go f.receiver(fr, h, encoder, &wg)
        go f.committer(fr, h, &wg)
        wg.Wait()

    return
}

func init() {
    RegisterPlugin("StreamAggregatorFilter", func() interface{} {
        return new(StreamAggregatorFilter)
    })
}
<|endoftext|>


<commit_before>package spec

import (
	"fmt"
	"os"
	"path/filepath"
)

import (
	mixers "butler/mixer"
	xmlhelp "gokogiri/help"
	"golog"
	"runtime/debug"
	"steno/dummy"
	"time"
	"tritium"
	"tritium/packager/legacy"
	tp "tritium/proto"
	"tritium/whale"
)

func All(command string, directory string, options ...string) {

	var mixerPath string
	if len(options) == 1 {
		//TODO: Instead of the mixer path, we should pass in just the name
		//and the version.
		mixerPath = filepath.Base(options[0])
	}

	logger := golog.NewLogger("tritium")
	debugger := &dummy.DummyDebugger{}

	logger.AddProcessor("info", golog.NewConsoleProcessor(golog.LOG_INFO, true))
	var eng tritium.Engine
	if command == "test" {
		eng = whale.NewEngine(debugger)
	}

	var pkg *tp.Package

	if len(mixerPath) > 0 {
		// Used when testing in ambrosia
		mixer, err := mixers.GetMixerFromFile(mixerPath)
		if err != nil {
			panic("Error, could not load mixer: " + mixerPath)
		}

		pkg = mixer.Package
	} else {
		bigPackage := legacy.BuildDefaultPackage()
		pkg = bigPackage.Package
	}

	globalResult := NewResult()
	globalResult.all(directory, pkg, eng, logger)

	// TODO : Walk over the results here and print errors.

	var foundError = false

	for _, err := range globalResult.Errors {
		foundError = true
		println("\n=========================================", err.Location, "\n")
		if err.Panic {
			fmt.Printf(err.Message)
		} else {
			fmt.Printf("\n==========\n%v :: %v \n\n Got \n----------\n%v\n\n Expected \n----------\n%v\n", err.Name, err.Message, err.Got, err.Expected)
		}
	}
	println("\n\n")
	println("+++TEST COMPLETE+++\n\n")

	if foundError {
		os.Exit(1)
	}
	eng.Free()
	xmlhelp.LibxmlCleanUpParser()
	if xmlhelp.LibxmlGetMemoryAllocation() != 0 {
		fmt.Printf("Memeory leaks %d!!!", xmlhelp.LibxmlGetMemoryAllocation())
		xmlhelp.LibxmlReportMemoryLeak()
	}
}

func (result *Result) all(directory string, pkg *tp.Package, eng tritium.Engine, logger *golog.Logger) {
	paths, err := filepath.Glob(filepath.Join(directory, "main.ts"))
	if err == nil && len(paths) == 1 {
		newResult := RunSpec(directory, pkg, eng, logger)
		result.Merge(newResult)
	}

	subdirs, _ := filepath.Glob(filepath.Join(directory, "*"))
	for _, subdir := range subdirs {
		fi, err := os.Stat(subdir)
		if err != nil {
			continue
		}
		if !fi.IsDir() {
			continue
		}
		result.all(subdir, pkg, eng, logger)
	}

}

func RunSpec(dir string, pkg *tp.Package, eng tritium.Engine, logger *golog.Logger) (result *Result) {
	result = NewResult()

	defer func() {
		//log.Println("done")  // Println executes normally even in there is a panic
		if x := recover(); x != nil {
			err, ok := x.(error)
			if ok {
				logger.Errorf(dir + " === " + err.Error() + "\n\n" + string(debug.Stack()))
			} else {
				logger.Errorf(dir + " === " + x.(string) + "\n\n" + string(debug.Stack()))
			}
		}
		print(result.CharStatus())
	}()

	spec, err := LoadSpec(dir, pkg)
	if err != nil {
		result.Error(dir, err.Error())
	} else {
		d, _ := time.ParseDuration("1m")
		result.Merge(spec.Compare(eng.Run(spec.Script, nil, spec.Input, spec.Vars, time.Now().Add(d), "test", "test", "test", make([]string, 0), false)))
	}
	return
}
<commit_msg>missed this on my first pass...<commit_after>package spec

import (
	"fmt"
	"os"
	"path/filepath"
)

import (
	mixers "butler/mixer"
	xmlhelp "gokogiri/help"
	"golog"
	"runtime/debug"
	"steno/dummy"
	"time"
	"tritium"
	"tritium/packager/legacy"
	tp "tritium/proto"
	"tritium/whale"
)

func All(command string, directory string, options ...string) {

	var mixerPath string
	if len(options) == 1 {
		//TODO: Instead of the mixer path, we should pass in just the name
		//and the version.
		mixerPath = filepath.Base(options[0])
	}

	logger := golog.NewLogger("tritium")
	debugger := &dummy.DummyDebugger{}

	logger.AddProcessor("info", golog.NewConsoleProcessor(golog.LOG_INFO, true))
	var eng tritium.Engine
	if command == "test" {
		eng = whale.NewEngine(debugger)
	}

	var pkg *tp.Package

	if len(mixerPath) > 0 {
		// Used when testing in ambrosia
		mixer, err := mixers.GetMixerFromFile(mixerPath)
		if err != nil {
			panic("Error, could not load mixer: " + mixerPath)
		}

		pkg = mixer.Package
	} else {
		bigPackage := legacy.BuildDefaultPackage()
		pkg = bigPackage.Package
	}

	globalResult := NewResult()
	globalResult.all(directory, pkg, eng, logger)

	// TODO : Walk over the results here and print errors.

	var foundError = false

	for _, err := range globalResult.Errors {
		foundError = true
		println("\n=========================================", err.Location, "\n")
		if err.Panic {
			fmt.Printf(err.Message)
		} else {
			fmt.Printf("\n==========\n%v :: %v \n\n Got \n----------\n%v\n\n Expected \n----------\n%v\n", err.Name, err.Message, err.Got, err.Expected)
		}
	}
	println("\n\n")
	println("+++TEST COMPLETE+++\n\n")

	if foundError {
		os.Exit(1)
	}
	eng.Free()
	xmlhelp.LibxmlCleanUpParser()
	if xmlhelp.LibxmlGetMemoryAllocation() != 0 {
		fmt.Printf("Memeory leaks %d!!!", xmlhelp.LibxmlGetMemoryAllocation())
		xmlhelp.LibxmlReportMemoryLeak()
	}
}

func (result *Result) all(directory string, pkg *tp.Package, eng tritium.Engine, logger *golog.Logger) {
	paths, err := filepath.Glob(filepath.Join(directory, "main.ts"))
	if err == nil && len(paths) == 1 {
		newResult := RunSpec(directory, pkg, eng, logger)
		result.Merge(newResult)
	}

	subdirs, _ := filepath.Glob(filepath.Join(directory, "*"))
	for _, subdir := range subdirs {
		fi, err := os.Stat(subdir)
		if err != nil {
			continue
		}
		if !fi.IsDir() {
			continue
		}
		result.all(subdir, pkg, eng, logger)
	}

}

func RunSpec(dir string, pkg *tp.Package, eng tritium.Engine, logger *golog.Logger) (result *Result) {
	result = NewResult()

	defer func() {
		//log.Println("done")  // Println executes normally even in there is a panic
		if x := recover(); x != nil {
			err, ok := x.(error)
			if ok {
				logger.Errorf(dir + " === " + err.Error() + "\n\n" + string(debug.Stack()))
			} else {
				logger.Errorf(dir + " === " + x.(string) + "\n\n" + string(debug.Stack()))
			}
		}
		print(result.CharStatus())
	}()

	spec, err := LoadSpec(dir, pkg)
	if err != nil {
		result.Error(dir, err.Error())
	} else {
		d, _ := time.ParseDuration("1m")
		result.Merge(spec.Compare(eng.Run(spec.Script, nil, spec.Input, spec.Vars, map[string]string{}, time.Now().Add(d), "test", "test", "test", make([]string, 0), false)))
	}
	return
}
<|endoftext|>