<commit_before>package blobsfile

import (
	"bytes"
	"crypto/rand"
	"fmt"
	"io/ioutil"
	mrand "math/rand"
	"os"
	"reflect"
	"sort"
	"testing"

	"a4.io/blobstash/pkg/hashutil"
)

func check(e error) {
	if e != nil {
		panic(e)
	}
}

func BenchmarkBlobsFilePut512B(b *testing.B) {
	back, err := New(&Opts{Directory: "./tmp_blobsfile_test", DisableCompression: true})
	check(err)
	defer back.Close()
	defer os.RemoveAll("./tmp_blobsfile_test")
	benchmarkBlobsFilePut(back, 512, b)
}

func BenchmarkBlobsFilePut512KB(b *testing.B) {
	back, err := New(&Opts{Directory: "./tmp_blobsfile_test", DisableCompression: true})
	check(err)
	defer back.Close()
	defer os.RemoveAll("./tmp_blobsfile_test")
	benchmarkBlobsFilePut(back, 512000, b)
}

func BenchmarkBlobsFilePut2MB(b *testing.B) {
	back, err := New(&Opts{Directory: "./tmp_blobsfile_test", DisableCompression: true})
	check(err)
	defer back.Close()
	defer os.RemoveAll("./tmp_blobsfile_test")
	benchmarkBlobsFilePut(back, 2000000, b)
}

func BenchmarkBlobsFilePut512BCompressed(b *testing.B) {
	back, err := New(&Opts{Directory: "./tmp_blobsfile_test"})
	check(err)
	defer back.Close()
	defer os.RemoveAll("./tmp_blobsfile_test")
	benchmarkBlobsFilePut(back, 512, b)
}

func BenchmarkBlobsFilePut512KBCompressed(b *testing.B) {
	back, err := New(&Opts{Directory: "./tmp_blobsfile_test"})
	check(err)
	defer back.Close()
	defer os.RemoveAll("./tmp_blobsfile_test")
	benchmarkBlobsFilePut(back, 512000, b)
}

func BenchmarkBlobsFilePut2MBCompressed(b *testing.B) {
	back, err := New(&Opts{Directory: "./tmp_blobsfile_test"})
	check(err)
	defer back.Close()
	defer os.RemoveAll("./tmp_blobsfile_test")
	benchmarkBlobsFilePut(back, 2000000, b)
}

func benchmarkBlobsFilePut(back *BlobsFiles, blobSize int, b *testing.B) {
	// b.ResetTimer()
	// b.StopTimer()
	for i := 0; i < b.N; i++ {
		b.StopTimer()
		h, blob := randBlob(blobSize)
		b.StartTimer()
		if err := back.Put(h, blob); err != nil {
			panic(err)
		}
		b.StopTimer()
	}
	b.SetBytes(int64(blobSize))
}

func TestBlobsFileReedSolomon(t *testing.T) {
	b, err := New(&Opts{Directory: "./tmp_blobsfile_test", DisableCompression: true, BlobsFileSize: 16000000})
	check(err)
	defer os.RemoveAll("./tmp_blobsfile_test")
	testParity(t, b, true, nil)
	fname := b.filename(0)
	b.Close()
	// // Corrupt the file

	// f, err := os.OpenFile(fname, os.O_RDWR, 0755)
	// if err != nil {
	// 	panic(err)
	// }
	// FIXME(tsileo): test this
	// if _, err := f.Seek(defaultMaxBlobsFileSize/10*3, os.SEEK_SET); err != nil {
	// if _, err := f.Seek(defaultMaxBlobsFileSize/10, os.SEEK_SET); err != nil {
	// if _, err := f.Seek(16000000/10*2, os.SEEK_SET); err != nil {
	data, err := ioutil.ReadFile(fname)
	if err != nil {
		panic(err)
	}
	punchOffset := int64(16000000/10*5) - 10
	t.Logf("punch at %d\n", punchOffset)
	fmt.Printf("punch at %d/%d\n", punchOffset, 16000000)
	ndata := []byte("blobsfilelol")
	copy(data[punchOffset:punchOffset+int64(len(ndata))], ndata)
	if err := ioutil.WriteFile(fname, []byte(data), 0644); err != nil {
		panic(err)
	}
	// Reopen the db
	b, err = New(&Opts{Directory: "./tmp_blobsfile_test", DisableCompression: true, BlobsFileSize: 16000000})
	check(err)
	defer b.Close()
	// Ensure we can recover from this corruption
	cb := func(err error) error {
		return b.checkBlobsFile(0)
	}
	testParity(t, b, false, cb)

}

func TestBlobsFileReedSolomonWithCompression(t *testing.T) {
	b, err := New(&Opts{Directory: "./tmp_blobsfile_test", DisableCompression: true, BlobsFileSize: 16000000})
	check(err)
	defer b.Close()
	defer os.RemoveAll("./tmp_blobsfile_test")
	testParity(t, b, true, nil)
}

func testParity(t *testing.T, b *BlobsFiles, insert bool, cb func(error) error) {
	if insert {
		for i := 0; i < 31+10; i++ {
			h, blob := randBlob(512000)
			if err := b.Put(h, blob); err != nil {
				panic(err)
			}
		}
	}
	if err := b.checkParityBlobs(0); err != nil {
		if cb == nil {
			panic(err)
		}
		if err := cb(err); err != nil {
			panic(err)
		}
	}
}

func randBlob(size int) (string, []byte) {
	blob := make([]byte, size)
	if _, err := rand.Read(blob); err != nil {
		panic(err)
	}
	return hashutil.Compute(blob), blob
}

func TestBlobsFileBlobPutGetEnumerate(t *testing.T) {
	b, err := New(&Opts{Directory: "./tmp_blobsfile_test"})
	check(err)
	defer os.RemoveAll("./tmp_blobsfile_test")
	hashes, blobs := testBackendPutGetEnumerate(t, b, 50)
	b.Close()
	// Test we can still read everything when closing/reopening the blobsfile
	b, err = New(&Opts{Directory: "./tmp_blobsfile_test"})
	check(err)
	testBackendEnumerate(t, b, hashes)
	testBackendGet(t, b, hashes, blobs)
	b.Close()
	b.RemoveIndex()
	// Try with the index and removed and test re-indexing
	b, err = New(&Opts{Directory: "./tmp_blobsfile_test"})
	check(err)
	testBackendEnumerate(t, b, hashes)
	testBackendGet(t, b, hashes, blobs)
}

func backendPut(t *testing.T, b *BlobsFiles, blobsCount int) ([]string, [][]byte) {
	blobs := [][]byte{}
	hashes := []string{}
	// TODO(tsileo): 50 blobs if in short mode
	for i := 0; i < blobsCount; i++ {
		h, blob := randBlob(mrand.Intn(4000000-32) + 32)
		hashes = append(hashes, h)
		blobs = append(blobs, blob)
		if err := b.Put(h, blob); err != nil {
			panic(err)
		}
	}

	stats, err := b.Stats()
	if err != nil {
		panic(err)
	}
	fmt.Printf("stats=%+v\n", stats)

	return hashes, blobs
}

func testBackendPutGetEnumerate(t *testing.T, b *BlobsFiles, blobsCount int) ([]string, [][]byte) {
	hashes, blobs := backendPut(t, b, blobsCount)
	testBackendGet(t, b, hashes, blobs)
	testBackendEnumerate(t, b, hashes)
	return hashes, blobs
}

func testBackendGet(t *testing.T, b *BlobsFiles, hashes []string, blobs [][]byte) {
	blobsIndex := map[string]bool{}
	for _, blob := range blobs {
		blobsIndex[hashutil.Compute(blob)] = true
	}
	for _, h := range hashes {
		if _, err := b.Get(h); err != nil {
			panic(err)
		}
		_, ok := blobsIndex[h]
		if !ok {
			t.Errorf("blob %s should be index", h)
		}
		delete(blobsIndex, h)
	}
	if len(blobsIndex) > 0 {
		t.Errorf("index should have been emptied, got len %d", len(blobsIndex))
	}
}

func testBackendEnumerate(t *testing.T, b *BlobsFiles, hashes []string) []string {
	sort.Strings(hashes)
	bchan := make(chan *Blob)
	errc := make(chan error, 1)
	go func() {
		errc <- b.Enumerate(bchan, "", "\xff", 0)
	}()
	enumHashes := []string{}
	for ref := range bchan {
		enumHashes = append(enumHashes, ref.Hash)
	}
	if err := <-errc; err != nil {
		panic(err)
	}
	if !sort.StringsAreSorted(enumHashes) {
		t.Errorf("enum hashes should already be sorted")
	}
	if !reflect.DeepEqual(hashes, enumHashes) {
		t.Errorf("bad enumerate results")
	}
	return enumHashes
}

func TestBlobsFileBlobEncodingNoCompression(t *testing.T) {
	b, err := New(&Opts{Directory: "./tmp_blobsfile_test", DisableCompression: true})
	check(err)
	defer b.Close()
	defer os.RemoveAll("./tmp_blobsfile_test")
	_, blob := randBlob(512)
	_, data := b.encodeBlob(blob, flagBlob)
	size, blob2, f := b.decodeBlob(data)
	if f != flagBlob {
		t.Errorf("bad flag, got %v, expected %v", f, flagBlob)
	}
	if size != 512 || !bytes.Equal(blob, blob2) {
		t.Errorf("Error blob encoding, got size:%v, expected:512, got blob:%v, expected:%v", size, blob2[:10], blob[:10])
	}
}

func TestBlobsFileBlobEncoding(t *testing.T) {
	b, err := New(&Opts{Directory: "./tmp_blobsfile_test"})
	check(err)
	defer b.Close()
	defer os.RemoveAll("./tmp_blobsfile_test")
	_, blob := randBlob(512)
	_, data := b.encodeBlob(blob, flagBlob)
	size, blob2, f := b.decodeBlob(data)
	if f != flagBlob {
		t.Errorf("bad flag, got %v, expected %v", f, flagBlob)
	}
	// Don't check the size are as the returned size is the size of the compressed blob
	if !bytes.Equal(blob, blob2) {
		t.Errorf("Error blob encoding, got size:%v, expected:512, got blob:%v, expected:%v", size, blob2[:10], blob[:10])
	}
}
<commit_msg>add more tests<commit_after>package blobsfile

import (
	"bytes"
	"crypto/rand"
	"fmt"
	"io/ioutil"
	mrand "math/rand"
	"os"
	"reflect"
	"sort"
	"testing"

	"a4.io/blobstash/pkg/hashutil"
)

func check(e error) {
	if e != nil {
		panic(e)
	}
}

func BenchmarkBlobsFilePut512B(b *testing.B) {
	back, err := New(&Opts{Directory: "./tmp_blobsfile_test", DisableCompression: true})
	check(err)
	defer back.Close()
	defer os.RemoveAll("./tmp_blobsfile_test")
	benchmarkBlobsFilePut(back, 512, b)
}

func BenchmarkBlobsFilePut512KB(b *testing.B) {
	back, err := New(&Opts{Directory: "./tmp_blobsfile_test", DisableCompression: true})
	check(err)
	defer back.Close()
	defer os.RemoveAll("./tmp_blobsfile_test")
	benchmarkBlobsFilePut(back, 512000, b)
}

func BenchmarkBlobsFilePut2MB(b *testing.B) {
	back, err := New(&Opts{Directory: "./tmp_blobsfile_test", DisableCompression: true})
	check(err)
	defer back.Close()
	defer os.RemoveAll("./tmp_blobsfile_test")
	benchmarkBlobsFilePut(back, 2000000, b)
}

func BenchmarkBlobsFilePut512BCompressed(b *testing.B) {
	back, err := New(&Opts{Directory: "./tmp_blobsfile_test"})
	check(err)
	defer back.Close()
	defer os.RemoveAll("./tmp_blobsfile_test")
	benchmarkBlobsFilePut(back, 512, b)
}

func BenchmarkBlobsFilePut512KBCompressed(b *testing.B) {
	back, err := New(&Opts{Directory: "./tmp_blobsfile_test"})
	check(err)
	defer back.Close()
	defer os.RemoveAll("./tmp_blobsfile_test")
	benchmarkBlobsFilePut(back, 512000, b)
}

func BenchmarkBlobsFilePut2MBCompressed(b *testing.B) {
	back, err := New(&Opts{Directory: "./tmp_blobsfile_test"})
	check(err)
	defer back.Close()
	defer os.RemoveAll("./tmp_blobsfile_test")
	benchmarkBlobsFilePut(back, 2000000, b)
}

func benchmarkBlobsFilePut(back *BlobsFiles, blobSize int, b *testing.B) {
	// b.ResetTimer()
	// b.StopTimer()
	for i := 0; i < b.N; i++ {
		b.StopTimer()
		h, blob := randBlob(blobSize)
		b.StartTimer()
		if err := back.Put(h, blob); err != nil {
			panic(err)
		}
		b.StopTimer()
	}
	b.SetBytes(int64(blobSize))
}

func TestBlobsFileReedSolomon(t *testing.T) {
	b, err := New(&Opts{Directory: "./tmp_blobsfile_test", DisableCompression: true, BlobsFileSize: 16000000})
	check(err)
	defer os.RemoveAll("./tmp_blobsfile_test")
	testParity(t, b, true, nil)
	fname := b.filename(0)
	b.Close()
	// // Corrupt the file

	// f, err := os.OpenFile(fname, os.O_RDWR, 0755)
	// if err != nil {
	// 	panic(err)
	// }
	// FIXME(tsileo): test this
	// if _, err := f.Seek(defaultMaxBlobsFileSize/10*3, os.SEEK_SET); err != nil {
	// if _, err := f.Seek(defaultMaxBlobsFileSize/10, os.SEEK_SET); err != nil {
	// if _, err := f.Seek(16000000/10*2, os.SEEK_SET); err != nil {
	data, err := ioutil.ReadFile(fname)
	if err != nil {
		panic(err)
	}
	punchOffset := int64(16000000/10*5) - 10
	t.Logf("punch at %d\n", punchOffset)
	fmt.Printf("punch at %d/%d\n", punchOffset, 16000000)
	ndata := []byte("blobsfilelol")
	copy(data[punchOffset:punchOffset+int64(len(ndata))], ndata)
	if err := ioutil.WriteFile(fname, []byte(data), 0644); err != nil {
		panic(err)
	}
	// Reopen the db
	b, err = New(&Opts{Directory: "./tmp_blobsfile_test", DisableCompression: true, BlobsFileSize: 16000000})
	check(err)
	defer b.Close()
	// Ensure we can recover from this corruption
	cb := func(err error) error {
		return b.checkBlobsFile(0)
	}
	testParity(t, b, false, cb)

}

func TestBlobsFileReedSolomonWithCompression(t *testing.T) {
	b, err := New(&Opts{Directory: "./tmp_blobsfile_test", DisableCompression: true, BlobsFileSize: 16000000})
	check(err)
	defer b.Close()
	defer os.RemoveAll("./tmp_blobsfile_test")
	testParity(t, b, true, nil)
}

func testParity(t *testing.T, b *BlobsFiles, insert bool, cb func(error) error) {
	if insert {
		for i := 0; i < 31+10; i++ {
			h, blob := randBlob(512000)
			if err := b.Put(h, blob); err != nil {
				panic(err)
			}
		}
	}
	if err := b.checkParityBlobs(0); err != nil {
		if cb == nil {
			panic(err)
		}
		if err := cb(err); err != nil {
			panic(err)
		}
	}
}

func randBlob(size int) (string, []byte) {
	blob := make([]byte, size)
	if _, err := rand.Read(blob); err != nil {
		panic(err)
	}
	return hashutil.Compute(blob), blob
}

func TestBlobsFilePutIdempotent(t *testing.T) {
	back, err := New(&Opts{Directory: "./tmp_blobsfile_test"})
	check(err)
	defer back.Close()
	defer os.RemoveAll("./tmp_blobsfile_test")
	h, blob := randBlob(512)
	for i := 0; i < 10; i++ {
		if err := back.Put(h, blob); err != nil {
			panic(err)
		}
	}
	stats, err := back.Stats()
	if err != nil {
		panic(err)
	}
	if stats.BlobsCount != 1 || stats.BlobsSize != 512 {
		t.Errorf("bad stats: %+v", stats)
	}
}

func TestBlobsFileBlobPutGetEnumerate(t *testing.T) {
	b, err := New(&Opts{Directory: "./tmp_blobsfile_test"})
	check(err)
	defer os.RemoveAll("./tmp_blobsfile_test")
	hashes, blobs := testBackendPutGetEnumerate(t, b, 50)
	b.Close()
	// Test we can still read everything when closing/reopening the blobsfile
	b, err = New(&Opts{Directory: "./tmp_blobsfile_test"})
	check(err)
	testBackendEnumerate(t, b, hashes)
	testBackendGet(t, b, hashes, blobs)
	b.Close()
	b.RemoveIndex()
	// Try with the index and removed and test re-indexing
	b, err = New(&Opts{Directory: "./tmp_blobsfile_test"})
	check(err)
	testBackendEnumerate(t, b, hashes)
	testBackendGet(t, b, hashes, blobs)
}

func backendPut(t *testing.T, b *BlobsFiles, blobsCount int) ([]string, [][]byte) {
	blobs := [][]byte{}
	hashes := []string{}
	// TODO(tsileo): 50 blobs if in short mode
	for i := 0; i < blobsCount; i++ {
		h, blob := randBlob(mrand.Intn(4000000-32) + 32)
		hashes = append(hashes, h)
		blobs = append(blobs, blob)
		if err := b.Put(h, blob); err != nil {
			panic(err)
		}
	}

	stats, err := b.Stats()
	if err != nil {
		panic(err)
	}
	fmt.Printf("stats=%+v\n", stats)

	return hashes, blobs
}

func testBackendPutGetEnumerate(t *testing.T, b *BlobsFiles, blobsCount int) ([]string, [][]byte) {
	hashes, blobs := backendPut(t, b, blobsCount)
	testBackendGet(t, b, hashes, blobs)
	testBackendEnumerate(t, b, hashes)
	return hashes, blobs
}

func testBackendGet(t *testing.T, b *BlobsFiles, hashes []string, blobs [][]byte) {
	blobsIndex := map[string]bool{}
	for _, blob := range blobs {
		blobsIndex[hashutil.Compute(blob)] = true
	}
	for _, h := range hashes {
		if _, err := b.Get(h); err != nil {
			panic(err)
		}
		_, ok := blobsIndex[h]
		if !ok {
			t.Errorf("blob %s should be index", h)
		}
		delete(blobsIndex, h)
	}
	if len(blobsIndex) > 0 {
		t.Errorf("index should have been emptied, got len %d", len(blobsIndex))
	}
}

func testBackendEnumerate(t *testing.T, b *BlobsFiles, hashes []string) []string {
	sort.Strings(hashes)
	bchan := make(chan *Blob)
	errc := make(chan error, 1)
	go func() {
		errc <- b.Enumerate(bchan, "", "\xff", 0)
	}()
	enumHashes := []string{}
	for ref := range bchan {
		enumHashes = append(enumHashes, ref.Hash)
	}
	if err := <-errc; err != nil {
		panic(err)
	}
	if !sort.StringsAreSorted(enumHashes) {
		t.Errorf("enum hashes should already be sorted")
	}
	if !reflect.DeepEqual(hashes, enumHashes) {
		t.Errorf("bad enumerate results")
	}
	return enumHashes
}

func TestBlobsFileBlobEncodingNoCompression(t *testing.T) {
	b, err := New(&Opts{Directory: "./tmp_blobsfile_test", DisableCompression: true})
	check(err)
	defer b.Close()
	defer os.RemoveAll("./tmp_blobsfile_test")
	_, blob := randBlob(512)
	_, data := b.encodeBlob(blob, flagBlob)
	size, blob2, f := b.decodeBlob(data)
	if f != flagBlob {
		t.Errorf("bad flag, got %v, expected %v", f, flagBlob)
	}
	if size != 512 || !bytes.Equal(blob, blob2) {
		t.Errorf("Error blob encoding, got size:%v, expected:512, got blob:%v, expected:%v", size, blob2[:10], blob[:10])
	}
}

func TestBlobsFileBlobEncoding(t *testing.T) {
	b, err := New(&Opts{Directory: "./tmp_blobsfile_test"})
	check(err)
	defer b.Close()
	defer os.RemoveAll("./tmp_blobsfile_test")
	_, blob := randBlob(512)
	_, data := b.encodeBlob(blob, flagBlob)
	size, blob2, f := b.decodeBlob(data)
	if f != flagBlob {
		t.Errorf("bad flag, got %v, expected %v", f, flagBlob)
	}
	// Don't check the size are as the returned size is the size of the compressed blob
	if !bytes.Equal(blob, blob2) {
		t.Errorf("Error blob encoding, got size:%v, expected:512, got blob:%v, expected:%v", size, blob2[:10], blob[:10])
	}
}
<|endoftext|>


<commit_before>package scm_test

import (
	"capsulecd/pkg/config/mock"
	"capsulecd/pkg/pipeline"
	"capsulecd/pkg/scm"
	"github.com/golang/mock/gomock"
	"github.com/stretchr/testify/require"
	"github.com/stretchr/testify/suite"
	"testing"
)

// Define the suite, and absorb the built-in basic suite
// functionality from testify - including a T() method which
// returns the current testing context
type ScmTestSuite struct {
	suite.Suite
	MockCtrl     *gomock.Controller
	Config       *mock_config.MockInterface
	PipelineData *pipeline.Data
}

// Make sure that VariableThatShouldStartAtFive is set to five
// before each test
func (suite *ScmTestSuite) SetupTest() {
	suite.MockCtrl = gomock.NewController(suite.T())

	suite.PipelineData = new(pipeline.Data)

	suite.Config = mock_config.NewMockInterface(suite.MockCtrl)

}

func (suite *ScmTestSuite) TearDownTest() {
	suite.MockCtrl.Finish()
}

func (suite *ScmTestSuite) TestCreate_Invalid() {
	//test
	testEngine, cerr := scm.Create("invalidtype", suite.PipelineData, suite.Config, nil)

	//assert
	require.Error(suite.T(), cerr, "should return an erro")
	require.Nil(suite.T(), testEngine, "engine should be nil")
}

func (suite *ScmTestSuite) TestCreate_Github() {
	//setup
	suite.Config.EXPECT().GetString("scm_github_access_token").Return("placeholder")
	suite.Config.EXPECT().IsSet("scm_github_api_endpoint").Return(false)
	suite.Config.EXPECT().IsSet("scm_github_access_token").Return(true)
	suite.Config.EXPECT().IsSet("scm_git_parent_path").Return(false)

	//test
	testScm, cerr := scm.Create("github", suite.PipelineData, suite.Config, nil)

	//assert
	require.NoError(suite.T(), cerr)
	require.NotNil(suite.T(), testScm)
}

func (suite *ScmTestSuite) TestCreate_Bitbucket() {
	//setup
	suite.Config.EXPECT().IsSet("scm_bitbucket_username").Return(true)
	suite.Config.EXPECT().IsSet("scm_bitbucket_password").Return(true)
	suite.Config.EXPECT().GetString("scm_bitbucket_username").Return("placeholder")
	suite.Config.EXPECT().GetString("scm_bitbucket_password").Return("placeholder")
	suite.Config.EXPECT().IsSet("scm_git_parent_path").Return(false)

	//test
	testScm, cerr := scm.Create("bitbucket", suite.PipelineData, suite.Config, nil)

	//assert
	require.NoError(suite.T(), cerr)
	require.NotNil(suite.T(), testScm)
}

// In order for 'go test' to run this suite, we need to create
// a normal test function and pass our suite to suite.Run
func TestFactoryTestSuite(t *testing.T) {
	suite.Run(t, new(ScmTestSuite))
}
<commit_msg>fix SCM factory.<commit_after>package scm_test

import (
	"capsulecd/pkg/config/mock"
	"capsulecd/pkg/pipeline"
	"capsulecd/pkg/scm"
	"github.com/golang/mock/gomock"
	"github.com/stretchr/testify/require"
	"github.com/stretchr/testify/suite"
	"testing"
)

// Define the suite, and absorb the built-in basic suite
// functionality from testify - including a T() method which
// returns the current testing context
type ScmTestSuite struct {
	suite.Suite
	MockCtrl     *gomock.Controller
	Config       *mock_config.MockInterface
	PipelineData *pipeline.Data
}

// Make sure that VariableThatShouldStartAtFive is set to five
// before each test
func (suite *ScmTestSuite) SetupTest() {
	suite.MockCtrl = gomock.NewController(suite.T())

	suite.PipelineData = new(pipeline.Data)

	suite.Config = mock_config.NewMockInterface(suite.MockCtrl)

}

func (suite *ScmTestSuite) TearDownTest() {
	suite.MockCtrl.Finish()
}

func (suite *ScmTestSuite) TestCreate_Invalid() {
	//test
	testEngine, cerr := scm.Create("invalidtype", suite.PipelineData, suite.Config, nil)

	//assert
	require.Error(suite.T(), cerr, "should return an erro")
	require.Nil(suite.T(), testEngine, "engine should be nil")
}

func (suite *ScmTestSuite) TestCreate_Github() {
	//setup
	suite.Config.EXPECT().GetString("scm_github_access_token").Return("placeholder")
	suite.Config.EXPECT().IsSet("scm_github_api_endpoint").Return(false)
	suite.Config.EXPECT().IsSet("scm_github_access_token").Return(true)
	suite.Config.EXPECT().IsSet("scm_git_parent_path").Return(false)

	//test
	testScm, cerr := scm.Create("github", suite.PipelineData, suite.Config, nil)

	//assert
	require.NoError(suite.T(), cerr)
	require.NotNil(suite.T(), testScm)
}

func (suite *ScmTestSuite) TestCreate_Bitbucket() {
	//setup
	suite.Config.EXPECT().IsSet("scm_bitbucket_username").Return(true)
	suite.Config.EXPECT().IsSet("scm_bitbucket_password").MinTimes(1).Return(true)
	suite.Config.EXPECT().GetString("scm_bitbucket_username").Return("placeholder")
	suite.Config.EXPECT().GetString("scm_bitbucket_password").MinTimes(1).Return("placeholder")
	suite.Config.EXPECT().IsSet("scm_git_parent_path").Return(false)

	//test
	testScm, cerr := scm.Create("bitbucket", suite.PipelineData, suite.Config, nil)

	//assert
	require.NoError(suite.T(), cerr)
	require.NotNil(suite.T(), testScm)
}

// In order for 'go test' to run this suite, we need to create
// a normal test function and pass our suite to suite.Run
func TestFactoryTestSuite(t *testing.T) {
	suite.Run(t, new(ScmTestSuite))
}
<|endoftext|>


<commit_before>package tanka

import (
	"fmt"
	"strings"

	"github.com/pkg/errors"

	"github.com/grafana/tanka/pkg/jsonnet"
	"github.com/grafana/tanka/pkg/jsonnet/jpath"
)

// EvalJsonnet evaluates the jsonnet environment at the given file system path
func evalJsonnet(path string, opts jsonnet.Opts) (raw string, err error) {
	entrypoint, err := jpath.Entrypoint(path)
	if err != nil {
		return "", err
	}

	// evaluate Jsonnet
	if opts.EvalScript != "" {
		var tla []string
		for k := range opts.TLACode {
			tla = append(tla, k+"="+k)
		}
		evalScript := fmt.Sprintf(`
  local main = (import '%s');
  %s
`, entrypoint, opts.EvalScript)

		if len(tla) != 0 {
			tlaJoin := strings.Join(tla, ", ")
			evalScript = fmt.Sprintf(`
function(%s)
  local main = (import '%s')(%s);
  %s
`, tlaJoin, entrypoint, tlaJoin, opts.EvalScript)
		}

		raw, err = jsonnet.Evaluate(path, evalScript, opts)
		if err != nil {
			return "", errors.Wrap(err, "evaluating jsonnet")
		}
		return raw, nil
	}

	raw, err = jsonnet.EvaluateFile(entrypoint, opts)
	if err != nil {
		return "", errors.Wrap(err, "evaluating jsonnet")
	}
	return raw, nil
}

const PatternEvalScript = "main.%s"

// MetadataEvalScript finds the Environment object (without its .data object)
const MetadataEvalScript = `
local noDataEnv(object) =
  std.prune(
    if std.isObject(object)
    then
      if std.objectHas(object, 'apiVersion')
         && std.objectHas(object, 'kind')
      then
        if object.kind == 'Environment'
        then object { data:: {} }
        else {}
      else
        std.mapWithKey(
          function(key, obj)
            noDataEnv(obj),
          object
        )
    else if std.isArray(object)
    then
      std.map(
        function(obj)
          noDataEnv(obj),
        object
      )
    else {}
  );

noDataEnv(main)
`

// MetadataSingleEnvEvalScript returns a Single Environment object
const MetadataSingleEnvEvalScript = `
local singleEnv(object) =
  std.prune(
    if std.isObject(object)
    then
      if std.objectHas(object, 'apiVersion')
         && std.objectHas(object, 'kind')
      then
        if object.kind == 'Environment'
        && object.metadata.name == '%s'
        then object { data:: {} }
        else {}
      else
        std.mapWithKey(
          function(key, obj)
            singleEnv(obj),
          object
        )
    else if std.isArray(object)
    then
      std.map(
        function(obj)
          singleEnv(obj),
        object
      )
    else {}
  );

singleEnv(main)
`

// SingleEnvEvalScript returns a Single Environment object
const SingleEnvEvalScript = `
local singleEnv(object) =
  if std.isObject(object)
  then
    if std.objectHas(object, 'apiVersion')
       && std.objectHas(object, 'kind')
    then
      if object.kind == 'Environment'
      && object.metadata.name == '%s'
      then object
      else {}
    else
      std.mapWithKey(
        function(key, obj)
          singleEnv(obj),
        object
      )
  else if std.isArray(object)
  then
    std.map(
      function(obj)
        singleEnv(obj),
      object
    )
  else {};

singleEnv(main)
`
<commit_msg>fix: Do not remove data (#585)<commit_after>package tanka

import (
	"fmt"
	"strings"

	"github.com/pkg/errors"

	"github.com/grafana/tanka/pkg/jsonnet"
	"github.com/grafana/tanka/pkg/jsonnet/jpath"
)

// EvalJsonnet evaluates the jsonnet environment at the given file system path
func evalJsonnet(path string, opts jsonnet.Opts) (raw string, err error) {
	entrypoint, err := jpath.Entrypoint(path)
	if err != nil {
		return "", err
	}

	// evaluate Jsonnet
	if opts.EvalScript != "" {
		var tla []string
		for k := range opts.TLACode {
			tla = append(tla, k+"="+k)
		}
		evalScript := fmt.Sprintf(`
  local main = (import '%s');
  %s
`, entrypoint, opts.EvalScript)

		if len(tla) != 0 {
			tlaJoin := strings.Join(tla, ", ")
			evalScript = fmt.Sprintf(`
function(%s)
  local main = (import '%s')(%s);
  %s
`, tlaJoin, entrypoint, tlaJoin, opts.EvalScript)
		}

		raw, err = jsonnet.Evaluate(path, evalScript, opts)
		if err != nil {
			return "", errors.Wrap(err, "evaluating jsonnet")
		}
		return raw, nil
	}

	raw, err = jsonnet.EvaluateFile(entrypoint, opts)
	if err != nil {
		return "", errors.Wrap(err, "evaluating jsonnet")
	}
	return raw, nil
}

const PatternEvalScript = "main.%s"

// MetadataEvalScript finds the Environment object (without its .data object)
const MetadataEvalScript = `
local noDataEnv(object) =
  std.prune(
    if std.isObject(object)
    then
      if std.objectHas(object, 'apiVersion')
         && std.objectHas(object, 'kind')
      then
        if object.kind == 'Environment'
        then object { data+:: {} }
        else {}
      else
        std.mapWithKey(
          function(key, obj)
            noDataEnv(obj),
          object
        )
    else if std.isArray(object)
    then
      std.map(
        function(obj)
          noDataEnv(obj),
        object
      )
    else {}
  );

noDataEnv(main)
`

// MetadataSingleEnvEvalScript returns a Single Environment object
const MetadataSingleEnvEvalScript = `
local singleEnv(object) =
  std.prune(
    if std.isObject(object)
    then
      if std.objectHas(object, 'apiVersion')
         && std.objectHas(object, 'kind')
      then
        if object.kind == 'Environment'
        && object.metadata.name == '%s'
        then object { data:: super.data }
        else {}
      else
        std.mapWithKey(
          function(key, obj)
            singleEnv(obj),
          object
        )
    else if std.isArray(object)
    then
      std.map(
        function(obj)
          singleEnv(obj),
        object
      )
    else {}
  );

singleEnv(main)
`

// SingleEnvEvalScript returns a Single Environment object
const SingleEnvEvalScript = `
local singleEnv(object) =
  if std.isObject(object)
  then
    if std.objectHas(object, 'apiVersion')
       && std.objectHas(object, 'kind')
    then
      if object.kind == 'Environment'
      && object.metadata.name == '%s'
      then object
      else {}
    else
      std.mapWithKey(
        function(key, obj)
          singleEnv(obj),
        object
      )
  else if std.isArray(object)
  then
    std.map(
      function(obj)
        singleEnv(obj),
      object
    )
  else {};

singleEnv(main)
`
<|endoftext|>


<commit_before>package mysql

import (
	"database/sql"
	"fmt"
	"reflect"
	"strconv"
	"strings"

	"github.com/go-sql-driver/mysql"
	"github.com/go-xorm/core"
	"github.com/grafana/grafana/pkg/log"
	"github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/tsdb"
)

func init() {
	tsdb.RegisterTsdbQueryEndpoint("mysql", newMysqlQueryEndpoint)
}

func newMysqlQueryEndpoint(datasource *models.DataSource) (tsdb.TsdbQueryEndpoint, error) {
	logger := log.New("tsdb.mysql")

	protocol := "tcp"
	if strings.HasPrefix(datasource.Url, "/") {
		protocol = "unix"
	}
	cnnstr := fmt.Sprintf("%s:%s@%s(%s)/%s?collation=utf8mb4_unicode_ci&parseTime=true&loc=UTC&allowNativePasswords=true",
		datasource.User,
		datasource.Password,
		protocol,
		datasource.Url,
		datasource.Database,
	)

	tlsConfig, err := datasource.GetTLSConfig()
	if err != nil {
		return nil, err
	}

	if tlsConfig.RootCAs != nil || len(tlsConfig.Certificates) > 0 {
		mysql.RegisterTLSConfig(datasource.Name, tlsConfig)
		cnnstr += "&tls=" + datasource.Name
	}

	logger.Debug("getEngine", "connection", cnnstr)

	config := tsdb.SqlQueryEndpointConfiguration{
		DriverName:        "mysql",
		ConnectionString:  cnnstr,
		Datasource:        datasource,
		TimeColumnNames:   []string{"time", "time_sec"},
		MetricColumnTypes: []string{"CHAR", "VARCHAR", "TINYTEXT", "TEXT", "MEDIUMTEXT", "LONGTEXT"},
	}

	rowTransformer := mysqlRowTransformer{
		log: logger,
	}

	return tsdb.NewSqlQueryEndpoint(&config, &rowTransformer, newMysqlMacroEngine(), logger)
}

type mysqlRowTransformer struct {
	log log.Logger
}

func (t *mysqlRowTransformer) Transform(columnTypes []*sql.ColumnType, rows *core.Rows) (tsdb.RowValues, error) {
	values := make([]interface{}, len(columnTypes))

	for i := range values {
		scanType := columnTypes[i].ScanType()
		values[i] = reflect.New(scanType).Interface()

		if columnTypes[i].DatabaseTypeName() == "BIT" {
			values[i] = new([]byte)
		}
	}

	if err := rows.Scan(values...); err != nil {
		return nil, err
	}

	for i := 0; i < len(columnTypes); i++ {
		typeName := reflect.ValueOf(values[i]).Type().String()

		switch typeName {
		case "*sql.RawBytes":
			values[i] = string(*values[i].(*sql.RawBytes))
		case "*mysql.NullTime":
			sqlTime := (*values[i].(*mysql.NullTime))
			if sqlTime.Valid {
				values[i] = sqlTime.Time
			} else {
				values[i] = nil
			}
		case "*sql.NullInt64":
			nullInt64 := (*values[i].(*sql.NullInt64))
			if nullInt64.Valid {
				values[i] = nullInt64.Int64
			} else {
				values[i] = nil
			}
		case "*sql.NullFloat64":
			nullFloat64 := (*values[i].(*sql.NullFloat64))
			if nullFloat64.Valid {
				values[i] = nullFloat64.Float64
			} else {
				values[i] = nil
			}
		}

		if columnTypes[i].DatabaseTypeName() == "DECIMAL" {
			f, err := strconv.ParseFloat(values[i].(string), 64)

			if err == nil {
				values[i] = f
			} else {
				values[i] = nil
			}
		}
	}

	return values, nil
}
<commit_msg>use unique datasource id when registering mysql tls config<commit_after>package mysql

import (
	"database/sql"
	"fmt"
	"reflect"
	"strconv"
	"strings"

	"github.com/go-sql-driver/mysql"
	"github.com/go-xorm/core"
	"github.com/grafana/grafana/pkg/log"
	"github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/tsdb"
)

func init() {
	tsdb.RegisterTsdbQueryEndpoint("mysql", newMysqlQueryEndpoint)
}

func newMysqlQueryEndpoint(datasource *models.DataSource) (tsdb.TsdbQueryEndpoint, error) {
	logger := log.New("tsdb.mysql")

	protocol := "tcp"
	if strings.HasPrefix(datasource.Url, "/") {
		protocol = "unix"
	}
	cnnstr := fmt.Sprintf("%s:%s@%s(%s)/%s?collation=utf8mb4_unicode_ci&parseTime=true&loc=UTC&allowNativePasswords=true",
		datasource.User,
		datasource.Password,
		protocol,
		datasource.Url,
		datasource.Database,
	)

	tlsConfig, err := datasource.GetTLSConfig()
	if err != nil {
		return nil, err
	}

	if tlsConfig.RootCAs != nil || len(tlsConfig.Certificates) > 0 {
		tlsConfigString := fmt.Sprintf("ds%d", datasource.Id)
		mysql.RegisterTLSConfig(tlsConfigString, tlsConfig)
		cnnstr += "&tls=" + tlsConfigString
	}

	logger.Debug("getEngine", "connection", cnnstr)

	config := tsdb.SqlQueryEndpointConfiguration{
		DriverName:        "mysql",
		ConnectionString:  cnnstr,
		Datasource:        datasource,
		TimeColumnNames:   []string{"time", "time_sec"},
		MetricColumnTypes: []string{"CHAR", "VARCHAR", "TINYTEXT", "TEXT", "MEDIUMTEXT", "LONGTEXT"},
	}

	rowTransformer := mysqlRowTransformer{
		log: logger,
	}

	return tsdb.NewSqlQueryEndpoint(&config, &rowTransformer, newMysqlMacroEngine(), logger)
}

type mysqlRowTransformer struct {
	log log.Logger
}

func (t *mysqlRowTransformer) Transform(columnTypes []*sql.ColumnType, rows *core.Rows) (tsdb.RowValues, error) {
	values := make([]interface{}, len(columnTypes))

	for i := range values {
		scanType := columnTypes[i].ScanType()
		values[i] = reflect.New(scanType).Interface()

		if columnTypes[i].DatabaseTypeName() == "BIT" {
			values[i] = new([]byte)
		}
	}

	if err := rows.Scan(values...); err != nil {
		return nil, err
	}

	for i := 0; i < len(columnTypes); i++ {
		typeName := reflect.ValueOf(values[i]).Type().String()

		switch typeName {
		case "*sql.RawBytes":
			values[i] = string(*values[i].(*sql.RawBytes))
		case "*mysql.NullTime":
			sqlTime := (*values[i].(*mysql.NullTime))
			if sqlTime.Valid {
				values[i] = sqlTime.Time
			} else {
				values[i] = nil
			}
		case "*sql.NullInt64":
			nullInt64 := (*values[i].(*sql.NullInt64))
			if nullInt64.Valid {
				values[i] = nullInt64.Int64
			} else {
				values[i] = nil
			}
		case "*sql.NullFloat64":
			nullFloat64 := (*values[i].(*sql.NullFloat64))
			if nullFloat64.Valid {
				values[i] = nullFloat64.Float64
			} else {
				values[i] = nil
			}
		}

		if columnTypes[i].DatabaseTypeName() == "DECIMAL" {
			f, err := strconv.ParseFloat(values[i].(string), 64)

			if err == nil {
				values[i] = f
			} else {
				values[i] = nil
			}
		}
	}

	return values, nil
}
<|endoftext|>


<commit_before>/*
 * Collisiondetector testing.
 */

package collisiondetector

import (
	"github.com/DiscoViking/goBrains/entity"
	"testing"
)

// Dummy entity structure for testing.
type testEntity struct {
	radius float64
}

func (te testEntity) GetRadius() float64 {
	return te.radius
}

// Test co-ordinate handling; coords and DeltaCoords.
func TestCoord(t *testing.T) {
	loc := coord{0, 0}

	// Update location and verify it.
	deltaLoc := CoordDelta{1, 2}
	loc.update(deltaLoc)

	if loc.locX != 1 {
		t.Errorf("Expected x-location update to %v, got %v.", 1, loc.locX)
	}
	if loc.locY != 2 {
		t.Errorf("Expected y-location update to %v, got %v.", 2, loc.locY)
	}
}

// Test the circle hitboxes.
func TestCircleHitbox(t *testing.T) {

	// Update the location of the hitbox.
	hb := circleHitbox{coord{0, 0}, 10, testEntity{}}

	move := CoordDelta{1, 2}
	hb.update(move)

	if hb.centre.locX != 1 {
		t.Errorf("Expected x-location update to %v, got %v.", 1, hb.centre.locX)
	}
	if hb.centre.locY != 2 {
		t.Errorf("Expected y-location update to %v, got %v.", 2, hb.centre.locY)
	}

	// Run checks on points inside and outside the hitbox.
	hb = circleHitbox{coord{0, 0}, 10, testEntity{}}

	loc := coord{1, 2}
	if !hb.isInside(loc) {
		t.Errorf("Expected location (1, 2) to be inside hitbox.  It wasn't.")
	}

	loc = coord{12, 8}
	if hb.isInside(loc) {
		t.Errorf("Expected location (12, 8) to be outside hitbox.  It wasn't.")
	}
}

// Test basic collision detection.
func TestCollisionDetection(t *testing.T) {
	errorStr := "[%v] Expected %v hitboxes, actual: %v"

	// A test location find collisions here.
	var loc CoordDelta

	// Entities at a location.
	var col []entity.Entity

	// Set up a new collision detector.
	cm := newCollisionManager()

	// Add two entities to be managed.
	ent1 := testEntity{5}
	ent2 := testEntity{5}

	cm.addEntity(ent1)
	cm.addEntity(ent2)

	if len(cm.hitboxes) != 2 {
		t.Errorf(errorStr, 1, 2, len(cm.hitboxes))
		cm.printDebug()
	}

	// Check there are two hitboxes found at the origin.
	loc = CoordDelta{0, 0}
	col = cm.getCollisions(loc, ent1)
	if len(col) != 2 {
		t.Errorf(errorStr, 2, 2, len(col))
		cm.printDebug()
	}

	// Move a hitbox and verify it's moved.
	move := CoordDelta{10, 10}
	cm.changeLocation(move, ent2)

	col = cm.getCollisions(loc, ent1)
	if len(col) != 1 {
		t.Errorf(errorStr, 3, 1, len(col))
		cm.printDebug()
	}

	// Verify that we can detect the moved entity.
	loc = CoordDelta{10, 10}
	col = cm.getCollisions(loc, ent1)
	if len(col) != 1 {
		t.Errorf(errorStr, 4, 1, len(col))
		cm.printDebug()
	}

	// Reduce radius of the entity and verify we stop detecting it.
	loc = CoordDelta{2, 0}
	col = cm.getCollisions(loc, ent1)
	if len(col) != 1 {
		t.Errorf(errorStr, 5, 1, len(col))
		cm.printDebug()
	}

	cm.changeRadius(1, ent1)
	col = cm.getCollisions(loc, ent1)
	if len(col) != 0 {
		t.Error(errorStr, 6, 0, len(col))
		cm.printDebug()
	}
}
<commit_msg>Finally fix the UTs.<commit_after>/*
 * Collisiondetector testing.
 */

package collisiondetector

import (
	"github.com/DiscoViking/goBrains/entity"
	"github.com/DiscoViking/goBrains/food"
	"testing"
)

// Test co-ordinate handling; coords and DeltaCoords.
func TestCoord(t *testing.T) {
	loc := coord{0, 0}

	// Update location and verify it.
	deltaLoc := CoordDelta{1, 2}
	loc.update(deltaLoc)

	if loc.locX != 1 {
		t.Errorf("Expected x-location update to %v, got %v.", 1, loc.locX)
	}
	if loc.locY != 2 {
		t.Errorf("Expected y-location update to %v, got %v.", 2, loc.locY)
	}
}

// Test the circle hitboxes.
func TestCircleHitbox(t *testing.T) {

	// Update the location of the hitbox.
	hb := circleHitbox{coord{0, 0}, 10, food.NewFood(0)}

	move := CoordDelta{1, 2}
	hb.update(move)

	if hb.centre.locX != 1 {
		t.Errorf("Expected x-location update to %v, got %v.", 1, hb.centre.locX)
	}
	if hb.centre.locY != 2 {
		t.Errorf("Expected y-location update to %v, got %v.", 2, hb.centre.locY)
	}

	// Run checks on points inside and outside the hitbox.
	hb = circleHitbox{coord{0, 0}, 10, food.NewFood(0)}

	loc := coord{1, 2}
	if !hb.isInside(loc) {
		t.Errorf("Expected location (1, 2) to be inside hitbox.  It wasn't.")
	}

	loc = coord{12, 8}
	if hb.isInside(loc) {
		t.Errorf("Expected location (12, 8) to be outside hitbox.  It wasn't.")
	}
}

// Test basic collision detection.
func TestCollisionDetection(t *testing.T) {
	errorStr := "[%v] Expected %v hitboxes, actual: %v"

	// A test location find collisions here.
	var loc CoordDelta

	// Entities at a location.
	var col []entity.Entity

	// Set up a new collision detector.
	cm := newCollisionManager()

	// Add two entities to be managed.
	// These MUST be different, else we cannot determine the difference between the two.
	// This results in hilarious test failures that are hard to debug.
	ent1 := food.NewFood(25)
	ent2 := food.NewFood(16)

	cm.addEntity(ent1)
	cm.addEntity(ent2)

	if len(cm.hitboxes) != 2 {
		t.Errorf(errorStr, 1, 2, len(cm.hitboxes))
		cm.printDebug()
	}

	// Check there are two hitboxes found at the origin.
	loc = CoordDelta{0, 0}
	col = cm.getCollisions(loc, ent1)
	if len(col) != 2 {
		t.Errorf(errorStr, 2, 2, len(col))
		cm.printDebug()
	}

	// Move a hitbox and verify it's moved.
	move := CoordDelta{10, 10}
	cm.changeLocation(move, ent2)

	col = cm.getCollisions(loc, ent1)
	if len(col) != 1 {
		t.Errorf(errorStr, 3, 1, len(col))
		cm.printDebug()
	}

	// Verify that we can detect the moved entity.
	loc = CoordDelta{10, 10}
	col = cm.getCollisions(loc, ent1)
	if len(col) != 1 {
		t.Errorf(errorStr, 4, 1, len(col))
		cm.printDebug()
	}

	// Reduce radius of the entity at the origin and verify we stop detecting it.
	loc = CoordDelta{2, 0}
	col = cm.getCollisions(loc, ent1)
	if len(col) != 1 {
		t.Errorf(errorStr, 5, 1, len(col))
		cm.printDebug()
	}

	cm.changeRadius(1, ent1)
	col = cm.getCollisions(loc, ent1)
	if len(col) != 0 {
		t.Error(errorStr, 6, 0, len(col))
		cm.printDebug()
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

// +build go1.3

package lxd

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"strings"

	"github.com/juju/errors"

	"github.com/juju/utils/packaging/config"
	"github.com/juju/utils/packaging/manager"

	"github.com/juju/juju/container"
)

const lxdBridgeFile = "/etc/default/lxd-bridge"

var requiredPackages = []string{
	"lxd",
}

var xenialPackages = []string{
	"zfsutils-linux",
}

type containerInitialiser struct {
	series string
}

// containerInitialiser implements container.Initialiser.
var _ container.Initialiser = (*containerInitialiser)(nil)

// NewContainerInitialiser returns an instance used to perform the steps
// required to allow a host machine to run a LXC container.
func NewContainerInitialiser(series string) container.Initialiser {
	return &containerInitialiser{series}
}

// Initialise is specified on the container.Initialiser interface.
func (ci *containerInitialiser) Initialise() error {
	err := ensureDependencies(ci.series)
	if err != nil {
		return err
	}

	err = configureLXDBridge()
	if err != nil {
		return err
	}

	if ci.series >= "xenial" {
		configureZFS()
	}

	return nil
}

// getPackageManager is a helper function which returns the
// package manager implementation for the current system.
func getPackageManager(series string) (manager.PackageManager, error) {
	return manager.NewPackageManager(series)
}

// getPackagingConfigurer is a helper function which returns the
// packaging configuration manager for the current system.
func getPackagingConfigurer(series string) (config.PackagingConfigurer, error) {
	return config.NewPackagingConfigurer(series)
}

var configureZFS = func() {
	/* create a 100 GB pool by default (sparse, so it won't actually fill
	 * that immediately)
	 */
	output, err := exec.Command(
		"lxd",
		"init",
		"--auto",
		"--storage-backend", "zfs",
		"--storage-pool", "lxd",
		"--storage-create-loop", "100",
	).CombinedOutput()

	if err != nil {
		logger.Warningf("configuring zfs failed with %s: %s", err, string(output))
	}
}

var configureLXDBridge = func() error {
	f, err := os.OpenFile(lxdBridgeFile, os.O_RDWR, 0777)
	if err != nil {
		/* We're using an old version of LXD which doesn't have
		 * lxd-bridge; let's not fail here.
		 */
		if os.IsNotExist(err) {
			logger.Warningf("couldn't find %s, not configuring it", lxdBridgeFile)
			return nil
		}
		return errors.Trace(err)
	}
	defer f.Close()

	existing, err := ioutil.ReadAll(f)
	if err != nil {
		return errors.Trace(err)
	}

	newBridgeCfg, err := bridgeConfiguration(string(existing))
	if err != nil {
		return errors.Trace(err)
	}

	if newBridgeCfg == string(existing) {
		return nil
	}

	_, err = f.Seek(0, 0)
	if err != nil {
		return errors.Trace(err)
	}
	_, err = f.WriteString(newBridgeCfg)
	if err != nil {
		return errors.Trace(err)
	}

	/* non-systemd systems don't have the lxd-bridge service, so this always fails */
	_ = exec.Command("service", "lxd-bridge", "restart").Run()
	return exec.Command("service", "lxd", "restart").Run()
}

var interfaceAddrs = func() ([]net.Addr, error) {
	return net.InterfaceAddrs()
}

func editLXDBridgeFile(input string, subnet string) string {
	buffer := bytes.Buffer{}

	newValues := map[string]string{
		"USE_LXD_BRIDGE":      "true",
		"EXISTING_BRIDGE":     "",
		"LXD_BRIDGE":          "lxdbr0",
		"LXD_IPV4_ADDR":       fmt.Sprintf("10.0.%s.1", subnet),
		"LXD_IPV4_NETMASK":    "255.255.255.0",
		"LXD_IPV4_NETWORK":    fmt.Sprintf("10.0.%s.1/24", subnet),
		"LXD_IPV4_DHCP_RANGE": fmt.Sprintf("10.0.%s.2,10.0.%s.254", subnet, subnet),
		"LXD_IPV4_DHCP_MAX":   "253",
		"LXD_IPV4_NAT":        "true",
		"LXD_IPV6_PROXY":      "false",
	}
	found := map[string]bool{}

	for _, line := range strings.Split(input, "\n") {
		out := line

		for prefix, value := range newValues {
			if strings.HasPrefix(line, prefix+"=") {
				out = fmt.Sprintf(`%s="%s"`, prefix, value)
				found[prefix] = true
				break
			}
		}

		buffer.WriteString(out)
		buffer.WriteString("\n")
	}

	for prefix, value := range newValues {
		if !found[prefix] {
			buffer.WriteString(prefix)
			buffer.WriteString("=")
			buffer.WriteString(value)
			buffer.WriteString("\n")
			found[prefix] = true // not necessary but keeps "found" logically consistent
		}
	}

	return buffer.String()
}

// ensureDependencies creates a set of install packages using
// apt.GetPreparePackages and runs each set of packages through
// apt.GetInstall.
func ensureDependencies(series string) error {
	if series == "precise" {
		return fmt.Errorf("LXD is not supported in precise.")
	}

	pacman, err := getPackageManager(series)
	if err != nil {
		return err
	}
	pacconfer, err := getPackagingConfigurer(series)
	if err != nil {
		return err
	}

	for _, pack := range requiredPackages {
		pkg := pack
		if config.SeriesRequiresCloudArchiveTools(series) &&
			pacconfer.IsCloudArchivePackage(pack) {
			pkg = strings.Join(pacconfer.ApplyCloudArchiveTarget(pack), " ")
		}

		if config.RequiresBackports(series, pack) {
			pkg = fmt.Sprintf("--target-release %s-backports %s", series, pkg)
		}

		if err := pacman.Install(pkg); err != nil {
			return err
		}
	}

	if series >= "xenial" {
		for _, pack := range xenialPackages {
			pacman.Install(fmt.Sprintf("--no-install-recommends %s", pack))
		}
	}

	return err
}

func findAvailableSubnet() (string, error) {
	addrs, err := interfaceAddrs()
	if err != nil {
		return "", errors.Annotatef(err, "cannot get network interface addresses")
	}

	max := 0
	usedSubnets := make(map[int]bool)

	for _, address := range addrs {
		_, network, err := net.ParseCIDR(address.String())
		if err != nil {
			logger.Warningf("cannot parse address %q: %v (ignoring)", address.String(), err)
			continue
		}
		if network.IP[0] != 10 || network.IP[1] != 0 {
			continue
		}
		subnet := int(network.IP[2])
		usedSubnets[subnet] = true
		if subnet > max {
			max = subnet
		}
	}

	for i := 0; i < 256; i++ {
		max = (max + 1) % 256
		if _, inUse := usedSubnets[max]; !inUse {
			return fmt.Sprintf("%d", max), nil
		}
	}

	return "", errors.New("could not find unused subnet")
}

func parseLXDBridgeConfigValues(input string) map[string]string {
	values := make(map[string]string)

	for _, line := range strings.Split(input, "\n") {
		line = strings.TrimSpace(line)

		if line == "" || strings.HasPrefix(line, "#") || !strings.Contains(line, "=") {
			continue
		}

		tokens := strings.Split(line, "=")

		if tokens[0] == "" {
			continue // no key
		}

		value := ""

		if len(tokens) > 1 {
			value = tokens[1]
			if strings.HasPrefix(value, `"`) && strings.HasSuffix(value, `"`) {
				value = strings.Trim(value, `"`)
			}
		}

		values[tokens[0]] = value
	}
	return values
}

// bridgeConfiguration ensures that input has a valid setting for
// LXD_IPV4_ADDR, returning the existing input if is already set, and
// allocating the next available subnet if it is not.
func bridgeConfiguration(input string) (string, error) {
	values := parseLXDBridgeConfigValues(input)
	ipAddr := net.ParseIP(values["LXD_IPV4_ADDR"])

	if ipAddr == nil || ipAddr.To4() == nil {
		subnet, err := findAvailableSubnet()
		if err != nil {
			return "", errors.Trace(err)
		}
		return editLXDBridgeFile(input, subnet), nil
	}
	return input, nil
}
<commit_msg>Add commentary to findAvailableSubnet()<commit_after>// Copyright 2016 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

// +build go1.3

package lxd

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"strings"

	"github.com/juju/errors"

	"github.com/juju/utils/packaging/config"
	"github.com/juju/utils/packaging/manager"

	"github.com/juju/juju/container"
)

const lxdBridgeFile = "/etc/default/lxd-bridge"

var requiredPackages = []string{
	"lxd",
}

var xenialPackages = []string{
	"zfsutils-linux",
}

type containerInitialiser struct {
	series string
}

// containerInitialiser implements container.Initialiser.
var _ container.Initialiser = (*containerInitialiser)(nil)

// NewContainerInitialiser returns an instance used to perform the steps
// required to allow a host machine to run a LXC container.
func NewContainerInitialiser(series string) container.Initialiser {
	return &containerInitialiser{series}
}

// Initialise is specified on the container.Initialiser interface.
func (ci *containerInitialiser) Initialise() error {
	err := ensureDependencies(ci.series)
	if err != nil {
		return err
	}

	err = configureLXDBridge()
	if err != nil {
		return err
	}

	if ci.series >= "xenial" {
		configureZFS()
	}

	return nil
}

// getPackageManager is a helper function which returns the
// package manager implementation for the current system.
func getPackageManager(series string) (manager.PackageManager, error) {
	return manager.NewPackageManager(series)
}

// getPackagingConfigurer is a helper function which returns the
// packaging configuration manager for the current system.
func getPackagingConfigurer(series string) (config.PackagingConfigurer, error) {
	return config.NewPackagingConfigurer(series)
}

var configureZFS = func() {
	/* create a 100 GB pool by default (sparse, so it won't actually fill
	 * that immediately)
	 */
	output, err := exec.Command(
		"lxd",
		"init",
		"--auto",
		"--storage-backend", "zfs",
		"--storage-pool", "lxd",
		"--storage-create-loop", "100",
	).CombinedOutput()

	if err != nil {
		logger.Warningf("configuring zfs failed with %s: %s", err, string(output))
	}
}

var configureLXDBridge = func() error {
	f, err := os.OpenFile(lxdBridgeFile, os.O_RDWR, 0777)
	if err != nil {
		/* We're using an old version of LXD which doesn't have
		 * lxd-bridge; let's not fail here.
		 */
		if os.IsNotExist(err) {
			logger.Warningf("couldn't find %s, not configuring it", lxdBridgeFile)
			return nil
		}
		return errors.Trace(err)
	}
	defer f.Close()

	existing, err := ioutil.ReadAll(f)
	if err != nil {
		return errors.Trace(err)
	}

	newBridgeCfg, err := bridgeConfiguration(string(existing))
	if err != nil {
		return errors.Trace(err)
	}

	if newBridgeCfg == string(existing) {
		return nil
	}

	_, err = f.Seek(0, 0)
	if err != nil {
		return errors.Trace(err)
	}
	_, err = f.WriteString(newBridgeCfg)
	if err != nil {
		return errors.Trace(err)
	}

	/* non-systemd systems don't have the lxd-bridge service, so this always fails */
	_ = exec.Command("service", "lxd-bridge", "restart").Run()
	return exec.Command("service", "lxd", "restart").Run()
}

var interfaceAddrs = func() ([]net.Addr, error) {
	return net.InterfaceAddrs()
}

func editLXDBridgeFile(input string, subnet string) string {
	buffer := bytes.Buffer{}

	newValues := map[string]string{
		"USE_LXD_BRIDGE":      "true",
		"EXISTING_BRIDGE":     "",
		"LXD_BRIDGE":          "lxdbr0",
		"LXD_IPV4_ADDR":       fmt.Sprintf("10.0.%s.1", subnet),
		"LXD_IPV4_NETMASK":    "255.255.255.0",
		"LXD_IPV4_NETWORK":    fmt.Sprintf("10.0.%s.1/24", subnet),
		"LXD_IPV4_DHCP_RANGE": fmt.Sprintf("10.0.%s.2,10.0.%s.254", subnet, subnet),
		"LXD_IPV4_DHCP_MAX":   "253",
		"LXD_IPV4_NAT":        "true",
		"LXD_IPV6_PROXY":      "false",
	}
	found := map[string]bool{}

	for _, line := range strings.Split(input, "\n") {
		out := line

		for prefix, value := range newValues {
			if strings.HasPrefix(line, prefix+"=") {
				out = fmt.Sprintf(`%s="%s"`, prefix, value)
				found[prefix] = true
				break
			}
		}

		buffer.WriteString(out)
		buffer.WriteString("\n")
	}

	for prefix, value := range newValues {
		if !found[prefix] {
			buffer.WriteString(prefix)
			buffer.WriteString("=")
			buffer.WriteString(value)
			buffer.WriteString("\n")
			found[prefix] = true // not necessary but keeps "found" logically consistent
		}
	}

	return buffer.String()
}

// ensureDependencies creates a set of install packages using
// apt.GetPreparePackages and runs each set of packages through
// apt.GetInstall.
func ensureDependencies(series string) error {
	if series == "precise" {
		return fmt.Errorf("LXD is not supported in precise.")
	}

	pacman, err := getPackageManager(series)
	if err != nil {
		return err
	}
	pacconfer, err := getPackagingConfigurer(series)
	if err != nil {
		return err
	}

	for _, pack := range requiredPackages {
		pkg := pack
		if config.SeriesRequiresCloudArchiveTools(series) &&
			pacconfer.IsCloudArchivePackage(pack) {
			pkg = strings.Join(pacconfer.ApplyCloudArchiveTarget(pack), " ")
		}

		if config.RequiresBackports(series, pack) {
			pkg = fmt.Sprintf("--target-release %s-backports %s", series, pkg)
		}

		if err := pacman.Install(pkg); err != nil {
			return err
		}
	}

	if series >= "xenial" {
		for _, pack := range xenialPackages {
			pacman.Install(fmt.Sprintf("--no-install-recommends %s", pack))
		}
	}

	return err
}

// findAvailableSubnet scans the list of interfaces on the machine
// looking for 10.0.x.y networks and returns the first subnet not in
// use having detected the highest subnet in use. The next subnet can
// actually be lower if we overflowed 255 whilst seeking out the next
// unused subnet. If all subnets in 10.0.x.0 are in use an error is
// returned. If no interfaces are found that use a 10.0.X.0 subnet
// then "0" is returned.
//
// TODO(frobware): this is not an ideal solution as it doesn't take
// into account any static routes that may set up on the machine.
func findAvailableSubnet() (string, error) {
	addrs, err := interfaceAddrs()
	if err != nil {
		return "", errors.Annotatef(err, "cannot get network interface addresses")
	}

	max := 0
	usedSubnets := make(map[int]bool)

	for _, address := range addrs {
		_, network, err := net.ParseCIDR(address.String())
		if err != nil {
			logger.Warningf("cannot parse address %q: %v (ignoring)", address.String(), err)
			continue
		}
		if network.IP[0] != 10 || network.IP[1] != 0 {
			continue
		}
		subnet := int(network.IP[2])
		usedSubnets[subnet] = true
		if subnet > max {
			max = subnet
		}
	}

	for i := 0; i < 256; i++ {
		max = (max + 1) % 256
		if _, inUse := usedSubnets[max]; !inUse {
			return fmt.Sprintf("%d", max), nil
		}
	}

	return "", errors.New("could not find unused subnet")
}

func parseLXDBridgeConfigValues(input string) map[string]string {
	values := make(map[string]string)

	for _, line := range strings.Split(input, "\n") {
		line = strings.TrimSpace(line)

		if line == "" || strings.HasPrefix(line, "#") || !strings.Contains(line, "=") {
			continue
		}

		tokens := strings.Split(line, "=")

		if tokens[0] == "" {
			continue // no key
		}

		value := ""

		if len(tokens) > 1 {
			value = tokens[1]
			if strings.HasPrefix(value, `"`) && strings.HasSuffix(value, `"`) {
				value = strings.Trim(value, `"`)
			}
		}

		values[tokens[0]] = value
	}
	return values
}

// bridgeConfiguration ensures that input has a valid setting for
// LXD_IPV4_ADDR, returning the existing input if is already set, and
// allocating the next available subnet if it is not.
func bridgeConfiguration(input string) (string, error) {
	values := parseLXDBridgeConfigValues(input)
	ipAddr := net.ParseIP(values["LXD_IPV4_ADDR"])

	if ipAddr == nil || ipAddr.To4() == nil {
		subnet, err := findAvailableSubnet()
		if err != nil {
			return "", errors.Trace(err)
		}
		return editLXDBridgeFile(input, subnet), nil
	}
	return input, nil
}
<|endoftext|>


<commit_before>package tracker

import (
	"errors"
	"fmt"
	"os"
	"strings"

	"github.com/jinzhu/gorm"

	// Needed for sqlite gorm support.
	_ "github.com/mattn/go-sqlite3"
)

// NIBLookup struct is used to represent entries in the database.
type NIBLookup struct {
	ID    int64
	NIBID string `sql:"size:256;unique" gorm:"column:nib_id"`
	Path  string `sql:"size:4096;unique"`
}

// TableName returns the name of the SQLite NIB table.
func (n NIBLookup) TableName() string {
	return "nib_lookups"
}

// NewDatabaseNIBTracker initializes a new object which uses a database
// to track NIB changes and implements the NIBTracker repository.
func NewDatabaseNIBTracker(dbLocation string) (NIBTracker, error) {
	nibTracker := &DatabaseNIBTracker{
		dbLocation: dbLocation,
	}
	_, statErr := os.Stat(dbLocation)

	db, err := gorm.Open("sqlite3", nibTracker.dbLocation)
	nibTracker.db = &db
	if err == nil && os.IsNotExist(statErr) {
		err = nibTracker.createDb()
	}

	return nibTracker, err
}

// DatabaseNIBTracker implements the NIBTracker interface and utilizes
// a sqlite database backend for persistence.
type DatabaseNIBTracker struct {
	dbLocation string
	db         *gorm.DB
}

// createDb initializes the tables in the database structure.
func (d *DatabaseNIBTracker) createDb() error {
	db := d.db.CreateTable(&NIBLookup{})
	return db.Error
}

// Add registers the given nibID for the given path.
func (d *DatabaseNIBTracker) Add(path string, nibID string) error {
	if len(path) > MaxPathSize {
		return errors.New("Path longer than maximal allowed path.")
	}
	tx := d.db.Begin()
	res, err := d.get(path, tx)

	var db *gorm.DB
	if err == nil && res != nil {
		res.NIBID = nibID
		fmt.Println("SAVE")
		db = tx.Save(res)
	} else {
		res = &NIBLookup{
			NIBID: nibID,
			Path:  path,
		}

		db = tx.Create(res)
	}

	tx.Commit()
	return db.Error
}

// whereFor returns a where statement for the
func (d *DatabaseNIBTracker) whereFor(path string, db *gorm.DB) *gorm.DB {
	return db.Where(map[string]interface{}{"path": path})
}

// lookupToNIB converts the lookup nib to a search response.
func (d *DatabaseNIBTracker) lookupToNIB(nibLookup *NIBLookup) *NIBSearchResponse {
	return &NIBSearchResponse{
		NIBID:          nibLookup.NIBID,
		Path:           nibLookup.Path,
		repositoryPath: "",
	}
}

// get returns the database object for the given path.
func (d *DatabaseNIBTracker) get(path string, db *gorm.DB) (*NIBLookup, error) {
	stmt := d.whereFor(path, db)
	data := &NIBLookup{}
	res := stmt.First(data)
	if res.Error != nil {
		return nil, res.Error
	}
	return data, nil
}

// Get returns the nibID for the given path.
func (d *DatabaseNIBTracker) Get(path string) (*NIBSearchResponse, error) {
	data, err := d.get(path, d.db)
	if err != nil {
		return nil, err
	}

	return d.lookupToNIB(data), err
}

// SearchPrefix returns all nibIDs with the given path.
// The map being returned has the paths
func (d *DatabaseNIBTracker) SearchPrefix(prefix string) ([]*NIBSearchResponse, error) {
	var resp []NIBLookup

	prefix = strings.TrimSuffix(prefix, "/")
	directoryPrefix := prefix + "/"
	db := d.db.Where("path LIKE ? or path = ?", directoryPrefix+"%", prefix).Find(&resp)

	searchResponse := []*NIBSearchResponse{}
	for _, item := range resp {
		searchResponse = append(searchResponse, d.lookupToNIB(&item))
	}

	return searchResponse, db.Error
}

// Remove removes the given path from being tracked.
func (d *DatabaseNIBTracker) Remove(path string) error {
	tx := d.db.Begin()
	db := d.whereFor(path, tx).Delete(NIBLookup{})
	if db.Error != nil {
		tx.Rollback()
	} else if db.Error == nil && db.RowsAffected < 1 {
		tx.Rollback()
		return errors.New("Entry not found")
	} else {
		tx.Commit()
	}
	return db.Error
}
<commit_msg>repository: removed missed debug statement.<commit_after>package tracker

import (
	"errors"
	"fmt"
	"os"
	"strings"

	"github.com/jinzhu/gorm"

	// Needed for sqlite gorm support.
	_ "github.com/mattn/go-sqlite3"
)

// NIBLookup struct is used to represent entries in the database.
type NIBLookup struct {
	ID    int64
	NIBID string `sql:"size:256;unique" gorm:"column:nib_id"`
	Path  string `sql:"size:4096;unique"`
}

// TableName returns the name of the SQLite NIB table.
func (n NIBLookup) TableName() string {
	return "nib_lookups"
}

// NewDatabaseNIBTracker initializes a new object which uses a database
// to track NIB changes and implements the NIBTracker repository.
func NewDatabaseNIBTracker(dbLocation string) (NIBTracker, error) {
	nibTracker := &DatabaseNIBTracker{
		dbLocation: dbLocation,
	}
	_, statErr := os.Stat(dbLocation)

	db, err := gorm.Open("sqlite3", nibTracker.dbLocation)
	nibTracker.db = &db
	if err == nil && os.IsNotExist(statErr) {
		err = nibTracker.createDb()
	}

	return nibTracker, err
}

// DatabaseNIBTracker implements the NIBTracker interface and utilizes
// a sqlite database backend for persistence.
type DatabaseNIBTracker struct {
	dbLocation string
	db         *gorm.DB
}

// createDb initializes the tables in the database structure.
func (d *DatabaseNIBTracker) createDb() error {
	db := d.db.CreateTable(&NIBLookup{})
	return db.Error
}

// Add registers the given nibID for the given path.
func (d *DatabaseNIBTracker) Add(path string, nibID string) error {
	if len(path) > MaxPathSize {
		return errors.New("Path longer than maximal allowed path.")
	}
	tx := d.db.Begin()
	res, err := d.get(path, tx)

	var db *gorm.DB
	if err == nil && res != nil {
		res.NIBID = nibID
		db = tx.Save(res)
	} else {
		res = &NIBLookup{
			NIBID: nibID,
			Path:  path,
		}

		db = tx.Create(res)
	}

	tx.Commit()
	return db.Error
}

// whereFor returns a where statement for the
func (d *DatabaseNIBTracker) whereFor(path string, db *gorm.DB) *gorm.DB {
	return db.Where(map[string]interface{}{"path": path})
}

// lookupToNIB converts the lookup nib to a search response.
func (d *DatabaseNIBTracker) lookupToNIB(nibLookup *NIBLookup) *NIBSearchResponse {
	return &NIBSearchResponse{
		NIBID:          nibLookup.NIBID,
		Path:           nibLookup.Path,
		repositoryPath: "",
	}
}

// get returns the database object for the given path.
func (d *DatabaseNIBTracker) get(path string, db *gorm.DB) (*NIBLookup, error) {
	stmt := d.whereFor(path, db)
	data := &NIBLookup{}
	res := stmt.First(data)
	if res.Error != nil {
		return nil, res.Error
	}
	return data, nil
}

// Get returns the nibID for the given path.
func (d *DatabaseNIBTracker) Get(path string) (*NIBSearchResponse, error) {
	data, err := d.get(path, d.db)
	if err != nil {
		return nil, err
	}

	return d.lookupToNIB(data), err
}

// SearchPrefix returns all nibIDs with the given path.
// The map being returned has the paths
func (d *DatabaseNIBTracker) SearchPrefix(prefix string) ([]*NIBSearchResponse, error) {
	var resp []NIBLookup

	prefix = strings.TrimSuffix(prefix, "/")
	directoryPrefix := prefix + "/"
	db := d.db.Where("path LIKE ? or path = ?", directoryPrefix+"%", prefix).Find(&resp)

	searchResponse := []*NIBSearchResponse{}
	for _, item := range resp {
		searchResponse = append(searchResponse, d.lookupToNIB(&item))
	}

	return searchResponse, db.Error
}

// Remove removes the given path from being tracked.
func (d *DatabaseNIBTracker) Remove(path string) error {
	tx := d.db.Begin()
	db := d.whereFor(path, tx).Delete(NIBLookup{})
	if db.Error != nil {
		tx.Rollback()
	} else if db.Error == nil && db.RowsAffected < 1 {
		tx.Rollback()
		return errors.New("Entry not found")
	} else {
		tx.Commit()
	}
	return db.Error
}
<|endoftext|>


<commit_before>// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package aws

import (
	"encoding/base64"
	"fmt"
	"net"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/ec2"
)

func (a *API) AddKey(name, key string) error {
	_, err := a.ec2.ImportKeyPair(&ec2.ImportKeyPairInput{
		KeyName:           &name,
		PublicKeyMaterial: []byte(key),
	})

	return err
}

func (a *API) DeleteKey(name string) error {
	_, err := a.ec2.DeleteKeyPair(&ec2.DeleteKeyPairInput{
		KeyName: &name,
	})

	return err
}

// CheckInstances waits until a set of EC2 instances are accessible by SSH, waiting a maximum of 'd' time.
func (a *API) CheckInstances(ids []string, d time.Duration) error {
	after := time.After(d)
	online := make(map[string]bool)

	// loop until all machines are online
	for len(ids) != len(online) {
		select {
		case <-after:
			return fmt.Errorf("timed out waiting for instances to run")
		default:
		}

		// don't make api calls too quickly, or we will hit the rate limit
		time.Sleep(10 * time.Second)

		getinst := &ec2.DescribeInstancesInput{
			InstanceIds: aws.StringSlice(ids),
		}

		insts, err := a.ec2.DescribeInstances(getinst)
		if err != nil {
			return err
		}

		for _, r := range insts.Reservations {
			for _, i := range r.Instances {
				switch *i.State.Name {
				case ec2.InstanceStateNamePending:
					// continue
				case ec2.InstanceStateNameRunning:
					// skip instances known to be up
					if online[*i.InstanceId] {
						continue
					}

					if i.PublicIpAddress == nil {
						continue
					}

					// XXX: ssh is a terrible way to check this, but it is all we have.
					c, err := net.DialTimeout("tcp", *i.PublicIpAddress+":22", 3*time.Second)
					if err != nil {
						continue
					}
					c.Close()

					online[*i.InstanceId] = true
				default:
					// instances should not be stopping, shutting-down, terminated, etc.
					return fmt.Errorf("instance %v in unexpected state %q", *i.InstanceId, *i.State.Name)
				}
			}
		}
	}

	return nil
}

// CreateInstances creates EC2 instances with a given name tag, ssh key name, user data. The image ID, instance type, and security group set in the API will be used. If wait is true, CreateInstances will block until all instances are reachable by SSH.
func (a *API) CreateInstances(name, keyname, userdata string, count uint64, wait bool) ([]*ec2.Instance, error) {
	cnt := int64(count)

	var ud *string
	if len(userdata) > 0 {
		tud := base64.StdEncoding.EncodeToString([]byte(userdata))
		ud = &tud
	}

	inst := ec2.RunInstancesInput{
		ImageId:        &a.opts.AMI,
		MinCount:       &cnt,
		MaxCount:       &cnt,
		KeyName:        &keyname,
		InstanceType:   &a.opts.InstanceType,
		SecurityGroups: []*string{&a.opts.SecurityGroup},
		UserData:       ud,
	}

	reservations, err := a.ec2.RunInstances(&inst)
	if err != nil {
		return nil, err
	}

	ids := make([]string, len(reservations.Instances))
	for i, inst := range reservations.Instances {
		ids[i] = *inst.InstanceId
	}

	for {
		err := a.CreateTags(ids, map[string]string{
			"Name": name,
		})
		if err == nil {
			break
		}
		if awserr, ok := err.(awserr.Error); !ok || awserr.Code() != "InvalidInstanceID.NotFound" {
			a.TerminateInstances(ids)
			return nil, err
		}
		// eventual consistency
		time.Sleep(5 * time.Second)
	}

	if !wait {
		return reservations.Instances, nil
	}

	// 5 minutes is a pretty reasonable timeframe for AWS instances to work.
	if err := a.CheckInstances(ids, 10*time.Minute); err != nil {
		a.TerminateInstances(ids)
		return nil, err
	}

	// call DescribeInstances to get machine IP
	getinst := &ec2.DescribeInstancesInput{
		InstanceIds: aws.StringSlice(ids),
	}

	insts, err := a.ec2.DescribeInstances(getinst)
	if err != nil {
		a.TerminateInstances(ids)
		return nil, err
	}

	return insts.Reservations[0].Instances, nil
}

// TerminateInstances schedules EC2 instances to be terminated.
func (a *API) TerminateInstances(ids []string) error {
	input := &ec2.TerminateInstancesInput{
		InstanceIds: aws.StringSlice(ids),
	}

	if _, err := a.ec2.TerminateInstances(input); err != nil {
		return err
	}

	return nil
}

func (a *API) CreateTags(resources []string, tags map[string]string) error {
	tagObjs := make([]*ec2.Tag, 0, len(tags))
	for key, value := range tags {
		tagObjs = append(tagObjs, &ec2.Tag{
			Key:   aws.String(key),
			Value: aws.String(value),
		})
	}
	_, err := a.ec2.CreateTags(&ec2.CreateTagsInput{
		Resources: aws.StringSlice(resources),
		Tags:      tagObjs,
	})
	if err != nil {
		return fmt.Errorf("error creating tags: %v", err)
	}
	return err
}
<commit_msg>platform/aws: automatically create SGs<commit_after>// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package aws

import (
	"encoding/base64"
	"fmt"
	"net"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/ec2"
)

func (a *API) AddKey(name, key string) error {
	_, err := a.ec2.ImportKeyPair(&ec2.ImportKeyPairInput{
		KeyName:           &name,
		PublicKeyMaterial: []byte(key),
	})

	return err
}

func (a *API) DeleteKey(name string) error {
	_, err := a.ec2.DeleteKeyPair(&ec2.DeleteKeyPairInput{
		KeyName: &name,
	})

	return err
}

// CheckInstances waits until a set of EC2 instances are accessible by SSH, waiting a maximum of 'd' time.
func (a *API) CheckInstances(ids []string, d time.Duration) error {
	after := time.After(d)
	online := make(map[string]bool)

	// loop until all machines are online
	for len(ids) != len(online) {
		select {
		case <-after:
			return fmt.Errorf("timed out waiting for instances to run")
		default:
		}

		// don't make api calls too quickly, or we will hit the rate limit
		time.Sleep(10 * time.Second)

		getinst := &ec2.DescribeInstancesInput{
			InstanceIds: aws.StringSlice(ids),
		}

		insts, err := a.ec2.DescribeInstances(getinst)
		if err != nil {
			return err
		}

		for _, r := range insts.Reservations {
			for _, i := range r.Instances {
				switch *i.State.Name {
				case ec2.InstanceStateNamePending:
					// continue
				case ec2.InstanceStateNameRunning:
					// skip instances known to be up
					if online[*i.InstanceId] {
						continue
					}

					if i.PublicIpAddress == nil {
						continue
					}

					// XXX: ssh is a terrible way to check this, but it is all we have.
					c, err := net.DialTimeout("tcp", *i.PublicIpAddress+":22", 3*time.Second)
					if err != nil {
						continue
					}
					c.Close()

					online[*i.InstanceId] = true
				default:
					// instances should not be stopping, shutting-down, terminated, etc.
					return fmt.Errorf("instance %v in unexpected state %q", *i.InstanceId, *i.State.Name)
				}
			}
		}
	}

	return nil
}

// CreateInstances creates EC2 instances with a given name tag, ssh key name, user data. The image ID, instance type, and security group set in the API will be used. If wait is true, CreateInstances will block until all instances are reachable by SSH.
func (a *API) CreateInstances(name, keyname, userdata string, count uint64, wait bool) ([]*ec2.Instance, error) {
	cnt := int64(count)

	var ud *string
	if len(userdata) > 0 {
		tud := base64.StdEncoding.EncodeToString([]byte(userdata))
		ud = &tud
	}

	sgId, err := a.getSecurityGroupID(a.opts.SecurityGroup)
	if err != nil {
		return nil, fmt.Errorf("error resolving security group: %v", err)
	}
	inst := ec2.RunInstancesInput{
		ImageId:          &a.opts.AMI,
		MinCount:         &cnt,
		MaxCount:         &cnt,
		KeyName:          &keyname,
		InstanceType:     &a.opts.InstanceType,
		SecurityGroupIds: []*string{&sgId},
		UserData:         ud,
	}

	reservations, err := a.ec2.RunInstances(&inst)
	if err != nil {
		return nil, fmt.Errorf("error running instances: %v", err)
	}

	ids := make([]string, len(reservations.Instances))
	for i, inst := range reservations.Instances {
		ids[i] = *inst.InstanceId
	}

	for {
		err := a.CreateTags(ids, map[string]string{
			"Name": name,
		})
		if err == nil {
			break
		}
		if awserr, ok := err.(awserr.Error); !ok || awserr.Code() != "InvalidInstanceID.NotFound" {
			a.TerminateInstances(ids)
			return nil, err
		}
		// eventual consistency
		time.Sleep(5 * time.Second)
	}

	if !wait {
		return reservations.Instances, nil
	}

	// 5 minutes is a pretty reasonable timeframe for AWS instances to work.
	if err := a.CheckInstances(ids, 10*time.Minute); err != nil {
		a.TerminateInstances(ids)
		return nil, err
	}

	// call DescribeInstances to get machine IP
	getinst := &ec2.DescribeInstancesInput{
		InstanceIds: aws.StringSlice(ids),
	}

	insts, err := a.ec2.DescribeInstances(getinst)
	if err != nil {
		a.TerminateInstances(ids)
		return nil, err
	}

	return insts.Reservations[0].Instances, nil
}

// TerminateInstances schedules EC2 instances to be terminated.
func (a *API) TerminateInstances(ids []string) error {
	input := &ec2.TerminateInstancesInput{
		InstanceIds: aws.StringSlice(ids),
	}

	if _, err := a.ec2.TerminateInstances(input); err != nil {
		return err
	}

	return nil
}

func (a *API) CreateTags(resources []string, tags map[string]string) error {
	tagObjs := make([]*ec2.Tag, 0, len(tags))
	for key, value := range tags {
		tagObjs = append(tagObjs, &ec2.Tag{
			Key:   aws.String(key),
			Value: aws.String(value),
		})
	}
	_, err := a.ec2.CreateTags(&ec2.CreateTagsInput{
		Resources: aws.StringSlice(resources),
		Tags:      tagObjs,
	})
	if err != nil {
		return fmt.Errorf("error creating tags: %v", err)
	}
	return err
}

// getSecurityGroupID gets a security group matching the given name.
// If the security group does not exist, it's created.
func (a *API) getSecurityGroupID(name string) (string, error) {
	sgIds, err := a.ec2.DescribeSecurityGroups(&ec2.DescribeSecurityGroupsInput{
		GroupNames: []*string{&name},
	})
	if isSecurityGroupNotExist(err) {
		return a.createSecurityGroup(name)
	}

	if err != nil {
		return "", fmt.Errorf("unable to get security group named %v: %v", name, err)
	}
	if len(sgIds.SecurityGroups) == 0 {
		return "", fmt.Errorf("zero security groups matched name %v", name)
	}
	return *sgIds.SecurityGroups[0].GroupId, nil
}

// createSecurityGroup creates a security group with tcp/22 access allowed from the
// internet.
func (a *API) createSecurityGroup(name string) (string, error) {
	sg, err := a.ec2.CreateSecurityGroup(&ec2.CreateSecurityGroupInput{
		GroupName:   aws.String(name),
		Description: aws.String("mantle security group for testing"),
	})
	if err != nil {
		return "", err
	}
	plog.Debugf("created security group %v", *sg.GroupId)

	allowedIngresses := []ec2.AuthorizeSecurityGroupIngressInput{
		{
			// SSH access from the public internet
			GroupId: sg.GroupId,
			IpPermissions: []*ec2.IpPermission{
				{
					IpProtocol: aws.String("tcp"),
					IpRanges: []*ec2.IpRange{
						{
							CidrIp: aws.String("0.0.0.0/0"),
						},
					},
					FromPort: aws.Int64(22),
					ToPort:   aws.Int64(22),
				},
			},
		},
		{
			// Access from all things in this vpc with the same SG (e.g. other
			// machines in our kola cluster)
			GroupId:                 sg.GroupId,
			SourceSecurityGroupName: aws.String(name),
		},
	}

	for _, input := range allowedIngresses {
		_, err := a.ec2.AuthorizeSecurityGroupIngress(&input)

		if err != nil {
			// We created the SG but can't add all the needed rules, let's try to
			// bail gracefully
			_, delErr := a.ec2.DeleteSecurityGroup(&ec2.DeleteSecurityGroupInput{
				GroupId: sg.GroupId,
			})
			if delErr != nil {
				return "", fmt.Errorf("created sg %v (%v) but couldn't authorize it. Manual deletion may be required: %v", *sg.GroupId, name, err)
			}
			return "", fmt.Errorf("created sg %v (%v), but couldn't authorize it and thus deleted it: %v", *sg.GroupId, name, err)
		}
	}
	return *sg.GroupId, err
}

func isSecurityGroupNotExist(err error) bool {
	if err == nil {
		return false
	}
	if awsErr, ok := (err).(awserr.Error); ok {
		if awsErr.Code() == "InvalidGroup.NotFound" {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before><commit_msg>:green_heart: Define tempEnv and use it to keep environment variables<commit_after><|endoftext|>


<commit_before>package command

import (
	"fmt"
	"log"
	"os"
	"strings"

	"github.com/wantedly/developers-account-mapper/models"
	"github.com/wantedly/developers-account-mapper/store"
)

type RegisterCommand struct {
	Meta
}

func (c *RegisterCommand) Run(args []string) int {
	var loginName, githubUsername, slackUsername string
	if len(args) == 2 {
		loginName = os.Getenv("USER")
		githubUsername = args[0]
		slackUsername = args[1]
	} else if len(args) == 3 {
		loginName = args[0]
		githubUsername = args[1]
		slackUsername = args[2]
	} else {
		log.Println(c.Help())
		return 1
	}

	s := store.NewDynamoDB()

	user := models.NewUser(loginName, githubUsername, slackUsername, "")
	err := user.RetrieveSlackUserId()
	if err != nil {
		log.Println(err)
		return 1
	}
	if err := s.AddUser(user); err != nil {
		log.Println(err)
		return 1
	}

	userSummary, err := user.String()
	if err != nil {
		log.Println(err)
		return 1
	}
	fmt.Printf("user %q added.\n", userSummary)

	return 0
}

func (c *RegisterCommand) Synopsis() string {
	return "Register LoginName and other accounts mapping"
}

func (c *RegisterCommand) Help() string {
	helpText := `

`
	return strings.TrimSpace(helpText)
}
<commit_msg>Never use env USER<commit_after>package command

import (
	"fmt"
	"log"
	"strings"

	"github.com/wantedly/developers-account-mapper/models"
	"github.com/wantedly/developers-account-mapper/store"
)

type RegisterCommand struct {
	Meta
}

func (c *RegisterCommand) Run(args []string) int {
	var loginName, githubUsername, slackUsername string
	if len(args) == 3 {
		loginName = args[0]
		githubUsername = args[1]
		slackUsername = args[2]
	} else {
		log.Println(c.Help())
		return 1
	}

	s := store.NewDynamoDB()

	user := models.NewUser(loginName, githubUsername, slackUsername, "")
	err := user.RetrieveSlackUserId()
	if err != nil {
		log.Println(err)
		return 1
	}
	if err := s.AddUser(user); err != nil {
		log.Println(err)
		return 1
	}

	userSummary, err := user.String()
	if err != nil {
		log.Println(err)
		return 1
	}
	fmt.Printf("user %q added.\n", userSummary)

	return 0
}

func (c *RegisterCommand) Synopsis() string {
	return "Register LoginName and other accounts mapping"
}

func (c *RegisterCommand) Help() string {
	helpText := `

`
	return strings.TrimSpace(helpText)
}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"strings"
)

func initRIO() *RObject {
	obj := &RObject{}
	obj.name = "RIO"
	obj.ivars = make(map[string]Object)
	obj.class = nil
	obj.methods = make(map[string]*RMethod)

	// RString method initialization
	obj.methods["readlines"] = &RMethod{gofunc: RIO_readlines}

	return obj
}

// IO.readlines(filename)
// [RString]
func RIO_readlines(vm *GobiesVM, receiver Object, v []Object) Object {
	filename := v[0].(*RObject).val.str

	obj := RArray_new(vm, receiver, nil)

	content, _ := ioutil.ReadFile(filename)
	str := string(content[:])

	lines := strings.SplitAfter(str, "\n")
	for _, line := range lines {
		dummy_obj := make([]Object, 1, 1)
		dummy_obj[0] = line
		rstr := RString_new(vm, receiver, dummy_obj)
		dummy_obj[0] = rstr
		RArray_append(vm, obj, dummy_obj)
	}

	return obj
}
<commit_msg>[IO] Fixed naming error in comment<commit_after>package main

import (
	"io/ioutil"
	"strings"
)

func initRIO() *RObject {
	obj := &RObject{}
	obj.name = "RIO"
	obj.ivars = make(map[string]Object)
	obj.class = nil
	obj.methods = make(map[string]*RMethod)

	// RIO method initialization
	obj.methods["readlines"] = &RMethod{gofunc: RIO_readlines}

	return obj
}

// IO.readlines(filename)
// [RString]
func RIO_readlines(vm *GobiesVM, receiver Object, v []Object) Object {
	filename := v[0].(*RObject).val.str

	obj := RArray_new(vm, receiver, nil)

	content, _ := ioutil.ReadFile(filename)
	str := string(content[:])

	lines := strings.SplitAfter(str, "\n")
	for _, line := range lines {
		dummy_obj := make([]Object, 1, 1)
		dummy_obj[0] = line
		rstr := RString_new(vm, receiver, dummy_obj)
		dummy_obj[0] = rstr
		RArray_append(vm, obj, dummy_obj)
	}

	return obj
}
<|endoftext|>


<commit_before>// Copyright 2014 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package functional

import (
	"strings"
	"testing"

	"github.com/coreos/fleet/functional/platform"
)

// TestUnitRunnable is the simplest test possible, deplying a single-node
// cluster and ensuring a unit can enter an 'active' state
func TestUnitRunnable(t *testing.T) {
	cluster, err := platform.NewNspawnCluster("smoke")
	if err != nil {
		t.Fatal(err)
	}
	defer cluster.Destroy()

	m0, err := cluster.CreateMember()
	if err != nil {
		t.Fatal(err)
	}
	_, err = cluster.WaitForNMachines(m0, 1)
	if err != nil {
		t.Fatal(err)
	}

	if stdout, stderr, err := cluster.Fleetctl(m0, "start", "fixtures/units/hello.service"); err != nil {
		t.Fatalf("Unable to start fleet unit: \nstdout: %s\nstderr: %s\nerr: %v", stdout, stderr, err)
	}

	units, err := cluster.WaitForNActiveUnits(m0, 1)
	if err != nil {
		t.Fatal(err)
	}
	_, found := units["hello.service"]
	if len(units) != 1 || !found {
		t.Fatalf("Expected hello.service to be sole active unit, got %v", units)
	}
}

func TestUnitSubmit(t *testing.T) {
	cluster, err := platform.NewNspawnCluster("smoke")
	if err != nil {
		t.Fatal(err)
	}
	defer cluster.Destroy()

	m, err := cluster.CreateMember()
	if err != nil {
		t.Fatal(err)
	}
	_, err = cluster.WaitForNMachines(m, 1)
	if err != nil {
		t.Fatal(err)
	}

	// submit a unit and assert it shows up
	if _, _, err := cluster.Fleetctl(m, "submit", "fixtures/units/hello.service"); err != nil {
		t.Fatalf("Unable to submit fleet unit: %v", err)
	}
	stdout, _, err := cluster.Fleetctl(m, "list-units", "--no-legend")
	if err != nil {
		t.Fatalf("Failed to run list-units: %v", err)
	}
	units := strings.Split(strings.TrimSpace(stdout), "\n")
	if len(units) != 1 {
		t.Fatalf("Did not find 1 unit in cluster: \n%s", stdout)
	}

	// submitting the same unit should not fail
	if _, _, err = cluster.Fleetctl(m, "submit", "fixtures/units/hello.service"); err != nil {
		t.Fatalf("Expected no failure when double-submitting unit, got this: %v", err)
	}

	// destroy the unit and ensure it disappears from the unit list
	if _, _, err := cluster.Fleetctl(m, "destroy", "fixtures/units/hello.service"); err != nil {
		t.Fatalf("Failed to destroy unit: %v", err)
	}
	stdout, _, err = cluster.Fleetctl(m, "list-units", "--no-legend")
	if err != nil {
		t.Fatalf("Failed to run list-units: %v", err)
	}
	if strings.TrimSpace(stdout) != "" {
		t.Fatalf("Did not find 0 units in cluster: \n%s", stdout)
	}

	// submitting the unit after destruction should succeed
	if _, _, err := cluster.Fleetctl(m, "submit", "fixtures/units/hello.service"); err != nil {
		t.Fatalf("Unable to submit fleet unit: %v", err)
	}
	stdout, _, err = cluster.Fleetctl(m, "list-units", "--no-legend")
	if err != nil {
		t.Fatalf("Failed to run list-units: %v", err)
	}
	units = strings.Split(strings.TrimSpace(stdout), "\n")
	if len(units) != 1 {
		t.Fatalf("Did not find 1 unit in cluster: \n%s", stdout)
	}
}

func TestUnitRestart(t *testing.T) {
	cluster, err := platform.NewNspawnCluster("smoke")
	if err != nil {
		t.Fatal(err)
	}
	defer cluster.Destroy()

	m, err := cluster.CreateMember()
	if err != nil {
		t.Fatal(err)
	}
	_, err = cluster.WaitForNMachines(m, 1)
	if err != nil {
		t.Fatal(err)
	}

	if stdout, stderr, err := cluster.Fleetctl(m, "start", "fixtures/units/hello.service"); err != nil {
		t.Fatalf("Unable to start fleet unit: \nstdout: %s\nstderr: %s\nerr: %v", stdout, stderr, err)
	}

	units, err := cluster.WaitForNActiveUnits(m, 1)
	if err != nil {
		t.Fatal(err)
	}
	_, found := units["hello.service"]
	if len(units) != 1 || !found {
		t.Fatalf("Expected hello.service to be sole active unit, got %v", units)
	}

	if _, _, err := cluster.Fleetctl(m, "stop", "hello.service"); err != nil {
		t.Fatal(err)
	}
	units, err = cluster.WaitForNActiveUnits(m, 0)
	if err != nil {
		t.Fatal(err)
	}
	if len(units) != 0 {
		t.Fatalf("Zero units should be running, found %v", units)
	}

	if stdout, stderr, err := cluster.Fleetctl(m, "start", "hello.service"); err != nil {
		t.Fatalf("Unable to start fleet unit: \nstdout: %s\nstderr: %s\nerr: %v", stdout, stderr, err)
	}
	units, err = cluster.WaitForNActiveUnits(m, 1)
	if err != nil {
		t.Fatal(err)
	}
	_, found = units["hello.service"]
	if len(units) != 1 || !found {
		t.Fatalf("Expected hello.service to be sole active unit, got %v", units)
	}

}

func TestUnitSSHActions(t *testing.T) {
	cluster, err := platform.NewNspawnCluster("smoke")
	if err != nil {
		t.Fatal(err)
	}
	defer cluster.Destroy()

	m, err := cluster.CreateMember()
	if err != nil {
		t.Fatal(err)
	}
	_, err = cluster.WaitForNMachines(m, 1)
	if err != nil {
		t.Fatal(err)
	}

	if stdout, stderr, err := cluster.Fleetctl(m, "start", "--no-block", "fixtures/units/hello.service"); err != nil {
		t.Fatalf("Unable to start fleet unit: \nstdout: %s\nstderr: %s\nerr: %v", stdout, stderr, err)
	}

	units, err := cluster.WaitForNActiveUnits(m, 1)
	if err != nil {
		t.Fatal(err)
	}

	_, found := units["hello.service"]
	if len(units) != 1 || !found {
		t.Fatalf("Expected hello.service to be sole active unit, got %v", units)
	}

	stdout, stderr, err := cluster.Fleetctl(m, "--strict-host-key-checking=false", "ssh", "hello.service", "echo", "foo")
	if err != nil {
		t.Errorf("Failure occurred while calling fleetctl ssh: %v\nstdout: %v\nstderr: %v", err, stdout, stderr)
	}

	if !strings.Contains(stdout, "foo") {
		t.Errorf("Could not find expected string in command output:\n%s", stdout)
	}

	stdout, stderr, err = cluster.Fleetctl(m, "--strict-host-key-checking=false", "status", "hello.service")
	if err != nil {
		t.Errorf("Failure occurred while calling fleetctl status: %v\nstdout: %v\nstderr: %v", err, stdout, stderr)
	}

	if !strings.Contains(stdout, "Active: active") {
		t.Errorf("Could not find expected string in status output:\n%s", stdout)
	}

	stdout, stderr, err = cluster.Fleetctl(m, "--strict-host-key-checking=false", "journal", "--sudo", "hello.service")
	if err != nil {
		t.Errorf("Failure occurred while calling fleetctl journal: %v\nstdout: %v\nstderr: %v", err, stdout, stderr)
	}

	if !strings.Contains(stdout, "Hello, World!") {
		t.Errorf("Could not find expected string in journal output:\n%s", stdout)
	}
}
<commit_msg>functional: introduce a new test TestUnitCat for fleetctl cat<commit_after>// Copyright 2014 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package functional

import (
	"io/ioutil"
	"path"
	"strings"
	"testing"

	"github.com/coreos/fleet/functional/platform"
)

// TestUnitRunnable is the simplest test possible, deplying a single-node
// cluster and ensuring a unit can enter an 'active' state
func TestUnitRunnable(t *testing.T) {
	cluster, err := platform.NewNspawnCluster("smoke")
	if err != nil {
		t.Fatal(err)
	}
	defer cluster.Destroy()

	m0, err := cluster.CreateMember()
	if err != nil {
		t.Fatal(err)
	}
	_, err = cluster.WaitForNMachines(m0, 1)
	if err != nil {
		t.Fatal(err)
	}

	if stdout, stderr, err := cluster.Fleetctl(m0, "start", "fixtures/units/hello.service"); err != nil {
		t.Fatalf("Unable to start fleet unit: \nstdout: %s\nstderr: %s\nerr: %v", stdout, stderr, err)
	}

	units, err := cluster.WaitForNActiveUnits(m0, 1)
	if err != nil {
		t.Fatal(err)
	}
	_, found := units["hello.service"]
	if len(units) != 1 || !found {
		t.Fatalf("Expected hello.service to be sole active unit, got %v", units)
	}
}

func TestUnitSubmit(t *testing.T) {
	cluster, err := platform.NewNspawnCluster("smoke")
	if err != nil {
		t.Fatal(err)
	}
	defer cluster.Destroy()

	m, err := cluster.CreateMember()
	if err != nil {
		t.Fatal(err)
	}
	_, err = cluster.WaitForNMachines(m, 1)
	if err != nil {
		t.Fatal(err)
	}

	// submit a unit and assert it shows up
	if _, _, err := cluster.Fleetctl(m, "submit", "fixtures/units/hello.service"); err != nil {
		t.Fatalf("Unable to submit fleet unit: %v", err)
	}
	stdout, _, err := cluster.Fleetctl(m, "list-units", "--no-legend")
	if err != nil {
		t.Fatalf("Failed to run list-units: %v", err)
	}
	units := strings.Split(strings.TrimSpace(stdout), "\n")
	if len(units) != 1 {
		t.Fatalf("Did not find 1 unit in cluster: \n%s", stdout)
	}

	// submitting the same unit should not fail
	if _, _, err = cluster.Fleetctl(m, "submit", "fixtures/units/hello.service"); err != nil {
		t.Fatalf("Expected no failure when double-submitting unit, got this: %v", err)
	}

	// destroy the unit and ensure it disappears from the unit list
	if _, _, err := cluster.Fleetctl(m, "destroy", "fixtures/units/hello.service"); err != nil {
		t.Fatalf("Failed to destroy unit: %v", err)
	}
	stdout, _, err = cluster.Fleetctl(m, "list-units", "--no-legend")
	if err != nil {
		t.Fatalf("Failed to run list-units: %v", err)
	}
	if strings.TrimSpace(stdout) != "" {
		t.Fatalf("Did not find 0 units in cluster: \n%s", stdout)
	}

	// submitting the unit after destruction should succeed
	if _, _, err := cluster.Fleetctl(m, "submit", "fixtures/units/hello.service"); err != nil {
		t.Fatalf("Unable to submit fleet unit: %v", err)
	}
	stdout, _, err = cluster.Fleetctl(m, "list-units", "--no-legend")
	if err != nil {
		t.Fatalf("Failed to run list-units: %v", err)
	}
	units = strings.Split(strings.TrimSpace(stdout), "\n")
	if len(units) != 1 {
		t.Fatalf("Did not find 1 unit in cluster: \n%s", stdout)
	}
}

func TestUnitRestart(t *testing.T) {
	cluster, err := platform.NewNspawnCluster("smoke")
	if err != nil {
		t.Fatal(err)
	}
	defer cluster.Destroy()

	m, err := cluster.CreateMember()
	if err != nil {
		t.Fatal(err)
	}
	_, err = cluster.WaitForNMachines(m, 1)
	if err != nil {
		t.Fatal(err)
	}

	if stdout, stderr, err := cluster.Fleetctl(m, "start", "fixtures/units/hello.service"); err != nil {
		t.Fatalf("Unable to start fleet unit: \nstdout: %s\nstderr: %s\nerr: %v", stdout, stderr, err)
	}

	units, err := cluster.WaitForNActiveUnits(m, 1)
	if err != nil {
		t.Fatal(err)
	}
	_, found := units["hello.service"]
	if len(units) != 1 || !found {
		t.Fatalf("Expected hello.service to be sole active unit, got %v", units)
	}

	if _, _, err := cluster.Fleetctl(m, "stop", "hello.service"); err != nil {
		t.Fatal(err)
	}
	units, err = cluster.WaitForNActiveUnits(m, 0)
	if err != nil {
		t.Fatal(err)
	}
	if len(units) != 0 {
		t.Fatalf("Zero units should be running, found %v", units)
	}

	if stdout, stderr, err := cluster.Fleetctl(m, "start", "hello.service"); err != nil {
		t.Fatalf("Unable to start fleet unit: \nstdout: %s\nstderr: %s\nerr: %v", stdout, stderr, err)
	}
	units, err = cluster.WaitForNActiveUnits(m, 1)
	if err != nil {
		t.Fatal(err)
	}
	_, found = units["hello.service"]
	if len(units) != 1 || !found {
		t.Fatalf("Expected hello.service to be sole active unit, got %v", units)
	}

}

func TestUnitSSHActions(t *testing.T) {
	cluster, err := platform.NewNspawnCluster("smoke")
	if err != nil {
		t.Fatal(err)
	}
	defer cluster.Destroy()

	m, err := cluster.CreateMember()
	if err != nil {
		t.Fatal(err)
	}
	_, err = cluster.WaitForNMachines(m, 1)
	if err != nil {
		t.Fatal(err)
	}

	if stdout, stderr, err := cluster.Fleetctl(m, "start", "--no-block", "fixtures/units/hello.service"); err != nil {
		t.Fatalf("Unable to start fleet unit: \nstdout: %s\nstderr: %s\nerr: %v", stdout, stderr, err)
	}

	units, err := cluster.WaitForNActiveUnits(m, 1)
	if err != nil {
		t.Fatal(err)
	}

	_, found := units["hello.service"]
	if len(units) != 1 || !found {
		t.Fatalf("Expected hello.service to be sole active unit, got %v", units)
	}

	stdout, stderr, err := cluster.Fleetctl(m, "--strict-host-key-checking=false", "ssh", "hello.service", "echo", "foo")
	if err != nil {
		t.Errorf("Failure occurred while calling fleetctl ssh: %v\nstdout: %v\nstderr: %v", err, stdout, stderr)
	}

	if !strings.Contains(stdout, "foo") {
		t.Errorf("Could not find expected string in command output:\n%s", stdout)
	}

	stdout, stderr, err = cluster.Fleetctl(m, "--strict-host-key-checking=false", "status", "hello.service")
	if err != nil {
		t.Errorf("Failure occurred while calling fleetctl status: %v\nstdout: %v\nstderr: %v", err, stdout, stderr)
	}

	if !strings.Contains(stdout, "Active: active") {
		t.Errorf("Could not find expected string in status output:\n%s", stdout)
	}

	stdout, stderr, err = cluster.Fleetctl(m, "--strict-host-key-checking=false", "journal", "--sudo", "hello.service")
	if err != nil {
		t.Errorf("Failure occurred while calling fleetctl journal: %v\nstdout: %v\nstderr: %v", err, stdout, stderr)
	}

	if !strings.Contains(stdout, "Hello, World!") {
		t.Errorf("Could not find expected string in journal output:\n%s", stdout)
	}
}

// TestUnitCat simply compares body of a unit file with that of a unit fetched
// from the remote cluster using "fleetctl cat".
func TestUnitCat(t *testing.T) {
	cluster, err := platform.NewNspawnCluster("smoke")
	if err != nil {
		t.Fatal(err)
	}
	defer cluster.Destroy()

	m, err := cluster.CreateMember()
	if err != nil {
		t.Fatal(err)
	}
	_, err = cluster.WaitForNMachines(m, 1)
	if err != nil {
		t.Fatal(err)
	}

	// read a sample unit file to a buffer
	unitFile := "fixtures/units/hello.service"
	fileBuf, err := ioutil.ReadFile(unitFile)
	if err != nil {
		t.Fatal(err)
	}
	fileBody := strings.TrimSpace(string(fileBuf))

	// submit a unit and assert it shows up
	_, _, err = cluster.Fleetctl(m, "submit", unitFile)
	if err != nil {
		t.Fatalf("Unable to submit fleet unit: %v", err)
	}
	// wait until the unit gets submitted up to 15 seconds
	_, err = cluster.WaitForNUnitFiles(m, 1)
	if err != nil {
		t.Fatalf("Failed to run list-units: %v", err)
	}

	// cat the unit file and compare it with the original unit body
	stdout, _, err := cluster.Fleetctl(m, "cat", path.Base(unitFile))
	if err != nil {
		t.Fatalf("Unable to submit fleet unit: %v", err)
	}
	catBody := strings.TrimSpace(stdout)

	if strings.Compare(catBody, fileBody) != 0 {
		t.Fatalf("unit body changed across fleetctl cat: \noriginal:%s\nnew:%s", fileBody, catBody)
	}
}
<|endoftext|>


<commit_before>package libcontainer

import "io"

// ErrorCode is the API error code type.
type ErrorCode int

// API error codes.
const (
	// Factory errors
	IdInUse ErrorCode = iota
	InvalidIdFormat

	// Container errors
	ContainerNotExists
	ContainerPaused
	ContainerNotStopped
	ContainerNotRunning
	ContainerNotPaused

	// Process errors
	NoProcessOps

	// Common errors
	ConfigInvalid
	ConsoleExists
	SystemError
)

func (c ErrorCode) String() string {
	switch c {
	case IdInUse:
		return "Id already in use"
	case InvalidIdFormat:
		return "Invalid format"
	case ContainerPaused:
		return "Container paused"
	case ConfigInvalid:
		return "Invalid configuration"
	case SystemError:
		return "System error"
	case ContainerNotExists:
		return "Container does not exist"
	case ContainerNotStopped:
		return "Container is not stopped"
	case ContainerNotRunning:
		return "Container is not running"
	case ConsoleExists:
		return "Console exists for process"
	case ContainerNotPaused:
		return "Container is not paused"
	case NoProcessOps:
		return "No process operations"
	default:
		return "Unknown error"
	}
}

// Error is the API error type.
type Error interface {
	error

	// Returns an error if it failed to write the detail of the Error to w.
	// The detail of the Error may include the error message and a
	// representation of the stack trace.
	Detail(w io.Writer) error

	// Returns the error code for this error.
	Code() ErrorCode
}
<commit_msg>libct/error: rm ConsoleExists<commit_after>package libcontainer

import "io"

// ErrorCode is the API error code type.
type ErrorCode int

// API error codes.
const (
	// Factory errors
	IdInUse ErrorCode = iota
	InvalidIdFormat

	// Container errors
	ContainerNotExists
	ContainerPaused
	ContainerNotStopped
	ContainerNotRunning
	ContainerNotPaused

	// Process errors
	NoProcessOps

	// Common errors
	ConfigInvalid
	SystemError
)

func (c ErrorCode) String() string {
	switch c {
	case IdInUse:
		return "Id already in use"
	case InvalidIdFormat:
		return "Invalid format"
	case ContainerPaused:
		return "Container paused"
	case ConfigInvalid:
		return "Invalid configuration"
	case SystemError:
		return "System error"
	case ContainerNotExists:
		return "Container does not exist"
	case ContainerNotStopped:
		return "Container is not stopped"
	case ContainerNotRunning:
		return "Container is not running"
	case ContainerNotPaused:
		return "Container is not paused"
	case NoProcessOps:
		return "No process operations"
	default:
		return "Unknown error"
	}
}

// Error is the API error type.
type Error interface {
	error

	// Returns an error if it failed to write the detail of the Error to w.
	// The detail of the Error may include the error message and a
	// representation of the stack trace.
	Detail(w io.Writer) error

	// Returns the error code for this error.
	Code() ErrorCode
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/psmithuk/xlsx"
)

func main() {

	c := []xlsx.Column{
		xlsx.Column{Name: "Col1", Width: 10},
		xlsx.Column{Name: "Col2", Width: 10},
	}

	sh := xlsx.NewSheetWithColumns(c)

	for i := 0; i < 100000; i++ {

		r := sh.NewRow()

		r.Cells[0] = xlsx.Cell{
			Type:  xlsx.CellTypeNumber,
			Value: "1",
		}
		r.Cells[1] = xlsx.Cell{
			Type:  xlsx.CellTypeNumber,
			Value: "2",
		}

		sh.AppendRow(r)
    }

    err := sh.SaveToFile("test.xlsx")
    _ = err
}
<commit_msg>Use fork of xlsx in large test<commit_after>package main

import (
	"github.com/sean-duffy/xlsx"
)

func main() {

	c := []xlsx.Column{
		xlsx.Column{Name: "Col1", Width: 10},
		xlsx.Column{Name: "Col2", Width: 10},
	}

	sh := xlsx.NewSheetWithColumns(c)

	for i := 0; i < 10; i++ {

		r := sh.NewRow()

		r.Cells[0] = xlsx.Cell{
			Type:  xlsx.CellTypeNumber,
			Value: "1",
		}
		r.Cells[1] = xlsx.Cell{
			Type:  xlsx.CellTypeNumber,
			Value: "2",
		}

		sh.AppendRow(r)
	}

	err := sh.SaveToFile("test.xlsx")
	_ = err
}
<|endoftext|>