<commit_before>package bolt_test

import (
	"context"
	"fmt"
	"reflect"
	"testing"

	"github.com/influxdata/chronograf"
)

func TestUsersStore_Get(t *testing.T) {
	type args struct {
		ctx context.Context
		ID  string
	}
	tests := []struct {
		name    string
		args    args
		want    *chronograf.User
		wantErr bool
	}{
		{
			name: "User not found",
			args: args{
				ctx: context.Background(),
				ID:  "1337",
			},
			wantErr: true,
		},
	}
	for _, tt := range tests {
		client, err := NewTestClient()
		if err != nil {
			t.Fatal(err)
		}
		if err := client.Open(context.TODO()); err != nil {
			t.Fatal(err)
		}
		defer client.Close()

		s := client.UsersStore
		got, err := s.Get(tt.args.ctx, tt.args.ID)
		if (err != nil) != tt.wantErr {
			t.Errorf("%q. UsersStore.Get() error = %v, wantErr %v", tt.name, err, tt.wantErr)
			continue
		}
		if !reflect.DeepEqual(got, tt.want) {
			t.Errorf("%q. UsersStore.Get() = %v, want %v", tt.name, got, tt.want)
		}
	}
}

func TestUsersStore_Add(t *testing.T) {
	type args struct {
		ctx context.Context
		u   *chronograf.User
	}
	tests := []struct {
		name    string
		args    args
		want    *chronograf.User
		wantErr bool
	}{
		{
			name: "Add new user",
			args: args{
				ctx: context.Background(),
				u: &chronograf.User{
					Name:     "docbrown",
					Provider: "GitHub",
					Scheme:   "OAuth2",
					Roles: []chronograf.Role{
						chronograf.DefaultUserRoles[chronograf.EditorRole],
					},
				},
			},
			want: &chronograf.User{
				Name:     "docbrown",
				Provider: "GitHub",
				Scheme:   "OAuth2",
				Roles: []chronograf.Role{
					chronograf.DefaultUserRoles[chronograf.EditorRole],
				},
			},
		},
	}
	for _, tt := range tests {
		client, err := NewTestClient()
		if err != nil {
			t.Fatal(err)
		}
		if err := client.Open(context.TODO()); err != nil {
			t.Fatal(err)
		}
		defer client.Close()
		s := client.UsersStore
		got, err := s.Add(tt.args.ctx, tt.args.u)
		if (err != nil) != tt.wantErr {
			t.Errorf("%q. UsersStore.Add() error = %v, wantErr %v", tt.name, err, tt.wantErr)
			continue
		}

		got, err = s.Get(tt.args.ctx, fmt.Sprintf("%d", got.ID))
		if err != nil {
			t.Fatalf("failed to get user: %v", err)
		}
		if got.Name != tt.want.Name {
			t.Errorf("%q. UsersStore.Add() .Name:\ngot %v, want %v", tt.name, got.Name, tt.want.Name)
		}
		if got.Provider != tt.want.Provider {
			t.Errorf("%q. UsersStore.Add() .Provider:\ngot %v, want %v", tt.name, got.Provider, tt.want.Provider)
		}
		if got.Scheme != tt.want.Scheme {
			t.Errorf("%q. UsersStore.Add() .Scheme:\ngot %v, want %v", tt.name, got.Scheme, tt.want.Scheme)
		}
		if len(got.Roles) != len(tt.want.Roles) {
			t.Errorf("%q. UsersStore.Add() .Roles:\ngot %v, want %v", tt.name, got.Roles, tt.want.Roles)
			continue
		}
		if len(got.Roles) > 0 && len(tt.want.Roles) > 0 {
			for i, gotRole := range got.Roles {
				wantRole := tt.want.Roles[i]
				if wantRole.Name != gotRole.Name {
					t.Errorf("%q. UsersStore.Add() .Roles[%d]:\ngot %v, want %v", tt.name, i, gotRole, wantRole)
				}
			}
		}
	}
}

func TestUsersStore_Delete(t *testing.T) {
	type args struct {
		ctx  context.Context
		user *chronograf.User
	}
	tests := []struct {
		name     string
		args     args
		addFirst bool
		wantErr  bool
	}{
		{
			name: "No such user",
			args: args{
				ctx: context.Background(),
				user: &chronograf.User{
					ID: 10,
				},
			},
			wantErr: true,
		},
		{
			name: "Delete new user",
			args: args{
				ctx: context.Background(),
				user: &chronograf.User{
					Name: "noone",
				},
			},
			addFirst: true,
		},
	}
	for _, tt := range tests {
		client, err := NewTestClient()
		if err != nil {
			t.Fatal(err)
		}
		if err := client.Open(context.TODO()); err != nil {
			t.Fatal(err)
		}
		defer client.Close()
		s := client.UsersStore

		if tt.addFirst {
			tt.args.user, _ = s.Add(tt.args.ctx, tt.args.user)
		}
		if err := s.Delete(tt.args.ctx, tt.args.user); (err != nil) != tt.wantErr {
			t.Errorf("%q. UsersStore.Delete() error = %v, wantErr %v", tt.name, err, tt.wantErr)
		}
	}
}

func TestUsersStore_Update(t *testing.T) {
	type args struct {
		ctx      context.Context
		usr      *chronograf.User
		roles    []chronograf.Role
		provider string
		scheme   string
		// TODO: add name
	}
	tests := []struct {
		name     string
		args     args
		addFirst bool
		want     *chronograf.User
		wantErr  bool
	}{
		{
			name: "No such user",
			args: args{
				ctx: context.Background(),
				usr: &chronograf.User{
					ID: 10,
				},
			},
			wantErr: true,
		},
		{
			name: "Update user role",
			args: args{
				ctx: context.Background(),
				usr: &chronograf.User{
					Name:     "bobetta",
					Provider: "GitHub",
					Scheme:   "OAuth2",
					Roles: []chronograf.Role{
						chronograf.DefaultUserRoles[chronograf.ViewerRole],
					},
				},
				roles: []chronograf.Role{
					chronograf.DefaultUserRoles[chronograf.EditorRole],
				},
			},
			want: &chronograf.User{
				Name:     "bobetta",
				Provider: "GitHub",
				Scheme:   "OAuth2",
				Roles: []chronograf.Role{
					chronograf.DefaultUserRoles[chronograf.EditorRole],
				},
			},
			addFirst: true,
		},
		{
			name: "Update user provider and scheme",
			args: args{
				ctx: context.Background(),
				usr: &chronograf.User{
					Name:     "bobetta",
					Provider: "GitHub",
					Scheme:   "OAuth2",
				},
				provider: "Google",
				scheme:   "LDAP",
			},
			want: &chronograf.User{
				Name:     "bobetta",
				Provider: "Google",
				Scheme:   "LDAP",
			},
			addFirst: true,
		},
	}
	for _, tt := range tests {
		client, err := NewTestClient()
		if err != nil {
			t.Fatal(err)
		}
		if err := client.Open(context.TODO()); err != nil {
			t.Fatal(err)
		}
		defer client.Close()
		s := client.UsersStore

		if tt.addFirst {
			tt.args.usr, err = s.Add(tt.args.ctx, tt.args.usr)
			if err != nil {
				t.Fatal(err)
			}
		}

		if tt.args.roles != nil {
			tt.args.usr.Roles = tt.args.roles
		}

		if tt.args.provider != "" {
			tt.args.usr.Provider = tt.args.provider
		}

		if tt.args.scheme != "" {
			tt.args.usr.Scheme = tt.args.scheme
		}

		if err := s.Update(tt.args.ctx, tt.args.usr); (err != nil) != tt.wantErr {
			t.Errorf("%q. UsersStore.Update() error = %v, wantErr %v", tt.name, err, tt.wantErr)
		}

		// for the empty test
		if tt.want == nil {
			continue
		}

		got, err := s.Get(tt.args.ctx, fmt.Sprintf("%d", tt.args.usr.ID))
		if err != nil {
			t.Fatalf("failed to get user: %v", err)
		}
		if got.Name != tt.want.Name {
			t.Errorf("%q. UsersStore.Update() .Name:\ngot %v, want %v", tt.name, got.Name, tt.want.Name)
		}
		if got.Provider != tt.want.Provider {
			t.Errorf("%q. UsersStore.Update() .Provider:\ngot %v, want %v", tt.name, got.Provider, tt.want.Provider)
		}
		if got.Scheme != tt.want.Scheme {
			t.Errorf("%q. UsersStore.Update() .Scheme:\ngot %v, want %v", tt.name, got.Scheme, tt.want.Scheme)
		}
		if len(got.Roles) != len(tt.want.Roles) {
			t.Errorf("%q. UsersStore.Update() .Roles:\ngot %v, want %v", tt.name, got.Roles, tt.want.Roles)
			continue
		}
		if len(got.Roles) != len(tt.want.Roles) {
			t.Errorf("%q. UsersStore.Update() .Roles:\ngot %v, want %v", tt.name, got.Roles, tt.want.Roles)
			continue
		}
		if len(got.Roles) > 0 && len(tt.want.Roles) > 0 {
			for i, gotRole := range got.Roles {
				wantRole := tt.want.Roles[i]
				if wantRole.Name != gotRole.Name {
					t.Errorf("%q. UsersStore.Update() .Roles[%d]:\ngot %v, want %v", tt.name, i, gotRole, wantRole)
				}
			}
		}
	}
}

func TestUsersStore_All(t *testing.T) {
	tests := []struct {
		name     string
		ctx      context.Context
		want     []chronograf.User
		addFirst bool
		wantErr  bool
	}{
		{
			name: "No users",
		},
		{
			name: "Update new user",
			want: []chronograf.User{
				{
					Name:     "howdy",
					Provider: "GitHub",
					Scheme:   "OAuth2",
					Roles: []chronograf.Role{
						chronograf.DefaultUserRoles[chronograf.ViewerRole],
					},
				},
				{
					Name:     "doody",
					Provider: "GitHub",
					Scheme:   "OAuth2",
					Roles: []chronograf.Role{
						chronograf.DefaultUserRoles[chronograf.EditorRole],
					},
				},
			},
			addFirst: true,
		},
	}
	for _, tt := range tests {
		client, err := NewTestClient()
		if err != nil {
			t.Fatal(err)
		}
		if err := client.Open(context.TODO()); err != nil {
			t.Fatal(err)
		}
		defer client.Close()
		s := client.UsersStore

		if tt.addFirst {
			for _, u := range tt.want {
				s.Add(tt.ctx, &u)
			}
		}
		gots, err := s.All(tt.ctx)
		if (err != nil) != tt.wantErr {
			t.Errorf("%q. UsersStore.All() error = %v, wantErr %v", tt.name, err, tt.wantErr)
			continue
		}
		for i, got := range gots {
			want := tt.want[i]
			if got.Name != want.Name {
				t.Errorf("%q. UsersStore.All() .Name:\ngot %v, want %v", tt.name, got.Name, want.Name)
			}
			if got.Provider != want.Provider {
				t.Errorf("%q. UsersStore.All() .Provider:\ngot %v, want %v", tt.name, got.Provider, want.Provider)
			}
			if got.Scheme != want.Scheme {
				t.Errorf("%q. UsersStore.All() .Scheme:\ngot %v, want %v", tt.name, got.Scheme, want.Scheme)
			}
			if len(got.Roles) != len(want.Roles) {
				t.Errorf("%q. UsersStore.All() .Roles:\ngot %v, want %v", tt.name, got.Roles, want.Roles)
				continue
			}
			if len(got.Roles) > 0 && len(want.Roles) > 0 {
				for i, gotRole := range got.Roles {
					wantRole := want.Roles[i]
					if wantRole.Name != gotRole.Name {
						t.Errorf("%q. UsersStore.All() .Roles[%d]:\ngot %v, want %v", tt.name, i, gotRole, wantRole)
					}
				}
			}
		}
	}
}
<commit_msg>Factor out code in common bolt UsersStore tests<commit_after>package bolt_test

import (
	"context"
	"fmt"
	"reflect"
	"testing"

	"github.com/influxdata/chronograf"
)

func TestUsersStore_Get(t *testing.T) {
	type args struct {
		ctx context.Context
		ID  string
	}
	tests := []struct {
		name    string
		args    args
		want    *chronograf.User
		wantErr bool
	}{
		{
			name: "User not found",
			args: args{
				ctx: context.Background(),
				ID:  "1337",
			},
			wantErr: true,
		},
	}
	for _, tt := range tests {
		client, err := NewTestClient()
		if err != nil {
			t.Fatal(err)
		}
		if err := client.Open(context.TODO()); err != nil {
			t.Fatal(err)
		}
		defer client.Close()

		s := client.UsersStore
		got, err := s.Get(tt.args.ctx, tt.args.ID)
		if (err != nil) != tt.wantErr {
			t.Errorf("%q. UsersStore.Get() error = %v, wantErr %v", tt.name, err, tt.wantErr)
			continue
		}
		if !reflect.DeepEqual(got, tt.want) {
			t.Errorf("%q. UsersStore.Get() = %v, want %v", tt.name, got, tt.want)
		}
	}
}

func TestUsersStore_Add(t *testing.T) {
	type args struct {
		ctx context.Context
		u   *chronograf.User
	}
	tests := []struct {
		name    string
		args    args
		want    *chronograf.User
		wantErr bool
	}{
		{
			name: "Add new user",
			args: args{
				ctx: context.Background(),
				u: &chronograf.User{
					Name:     "docbrown",
					Provider: "GitHub",
					Scheme:   "OAuth2",
					Roles: []chronograf.Role{
						chronograf.DefaultUserRoles[chronograf.EditorRole],
					},
				},
			},
			want: &chronograf.User{
				Name:     "docbrown",
				Provider: "GitHub",
				Scheme:   "OAuth2",
				Roles: []chronograf.Role{
					chronograf.DefaultUserRoles[chronograf.EditorRole],
				},
			},
		},
	}
	for _, tt := range tests {
		client, err := NewTestClient()
		if err != nil {
			t.Fatal(err)
		}
		if err := client.Open(context.TODO()); err != nil {
			t.Fatal(err)
		}
		defer client.Close()
		s := client.UsersStore
		got, err := s.Add(tt.args.ctx, tt.args.u)
		if (err != nil) != tt.wantErr {
			t.Errorf("%q. UsersStore.Add() error = %v, wantErr %v", tt.name, err, tt.wantErr)
			continue
		}

		got, err = s.Get(tt.args.ctx, fmt.Sprintf("%d", got.ID))
		if err != nil {
			t.Fatalf("failed to get user: %v", err)
		}
		usersEqual(t, tt.name, "Add", got, tt.want)
	}
}

func TestUsersStore_Delete(t *testing.T) {
	type args struct {
		ctx  context.Context
		user *chronograf.User
	}
	tests := []struct {
		name     string
		args     args
		addFirst bool
		wantErr  bool
	}{
		{
			name: "No such user",
			args: args{
				ctx: context.Background(),
				user: &chronograf.User{
					ID: 10,
				},
			},
			wantErr: true,
		},
		{
			name: "Delete new user",
			args: args{
				ctx: context.Background(),
				user: &chronograf.User{
					Name: "noone",
				},
			},
			addFirst: true,
		},
	}
	for _, tt := range tests {
		client, err := NewTestClient()
		if err != nil {
			t.Fatal(err)
		}
		if err := client.Open(context.TODO()); err != nil {
			t.Fatal(err)
		}
		defer client.Close()
		s := client.UsersStore

		if tt.addFirst {
			tt.args.user, _ = s.Add(tt.args.ctx, tt.args.user)
		}
		if err := s.Delete(tt.args.ctx, tt.args.user); (err != nil) != tt.wantErr {
			t.Errorf("%q. UsersStore.Delete() error = %v, wantErr %v", tt.name, err, tt.wantErr)
		}
	}
}

func TestUsersStore_Update(t *testing.T) {
	type args struct {
		ctx      context.Context
		usr      *chronograf.User
		roles    []chronograf.Role
		provider string
		scheme   string
		// TODO: add name
	}
	tests := []struct {
		name     string
		args     args
		addFirst bool
		want     *chronograf.User
		wantErr  bool
	}{
		{
			name: "No such user",
			args: args{
				ctx: context.Background(),
				usr: &chronograf.User{
					ID: 10,
				},
			},
			wantErr: true,
		},
		{
			name: "Update user role",
			args: args{
				ctx: context.Background(),
				usr: &chronograf.User{
					Name:     "bobetta",
					Provider: "GitHub",
					Scheme:   "OAuth2",
					Roles: []chronograf.Role{
						chronograf.DefaultUserRoles[chronograf.ViewerRole],
					},
				},
				roles: []chronograf.Role{
					chronograf.DefaultUserRoles[chronograf.EditorRole],
				},
			},
			want: &chronograf.User{
				Name:     "bobetta",
				Provider: "GitHub",
				Scheme:   "OAuth2",
				Roles: []chronograf.Role{
					chronograf.DefaultUserRoles[chronograf.EditorRole],
				},
			},
			addFirst: true,
		},
		{
			name: "Update user provider and scheme",
			args: args{
				ctx: context.Background(),
				usr: &chronograf.User{
					Name:     "bobetta",
					Provider: "GitHub",
					Scheme:   "OAuth2",
				},
				provider: "Google",
				scheme:   "LDAP",
			},
			want: &chronograf.User{
				Name:     "bobetta",
				Provider: "Google",
				Scheme:   "LDAP",
			},
			addFirst: true,
		},
	}
	for _, tt := range tests {
		client, err := NewTestClient()
		if err != nil {
			t.Fatal(err)
		}
		if err := client.Open(context.TODO()); err != nil {
			t.Fatal(err)
		}
		defer client.Close()
		s := client.UsersStore

		if tt.addFirst {
			tt.args.usr, err = s.Add(tt.args.ctx, tt.args.usr)
			if err != nil {
				t.Fatal(err)
			}
		}

		if tt.args.roles != nil {
			tt.args.usr.Roles = tt.args.roles
		}

		if tt.args.provider != "" {
			tt.args.usr.Provider = tt.args.provider
		}

		if tt.args.scheme != "" {
			tt.args.usr.Scheme = tt.args.scheme
		}

		if err := s.Update(tt.args.ctx, tt.args.usr); (err != nil) != tt.wantErr {
			t.Errorf("%q. UsersStore.Update() error = %v, wantErr %v", tt.name, err, tt.wantErr)
		}

		// for the empty test
		if tt.want == nil {
			continue
		}

		got, err := s.Get(tt.args.ctx, fmt.Sprintf("%d", tt.args.usr.ID))
		if err != nil {
			t.Fatalf("failed to get user: %v", err)
		}
		usersEqual(t, tt.name, "Update", got, tt.want)
	}
}

func TestUsersStore_All(t *testing.T) {
	tests := []struct {
		name     string
		ctx      context.Context
		want     []chronograf.User
		addFirst bool
		wantErr  bool
	}{
		{
			name: "No users",
		},
		{
			name: "Update new user",
			want: []chronograf.User{
				{
					Name:     "howdy",
					Provider: "GitHub",
					Scheme:   "OAuth2",
					Roles: []chronograf.Role{
						chronograf.DefaultUserRoles[chronograf.ViewerRole],
					},
				},
				{
					Name:     "doody",
					Provider: "GitHub",
					Scheme:   "OAuth2",
					Roles: []chronograf.Role{
						chronograf.DefaultUserRoles[chronograf.EditorRole],
					},
				},
			},
			addFirst: true,
		},
	}
	for _, tt := range tests {
		client, err := NewTestClient()
		if err != nil {
			t.Fatal(err)
		}
		if err := client.Open(context.TODO()); err != nil {
			t.Fatal(err)
		}
		defer client.Close()
		s := client.UsersStore

		if tt.addFirst {
			for _, u := range tt.want {
				s.Add(tt.ctx, &u)
			}
		}
		gots, err := s.All(tt.ctx)
		if (err != nil) != tt.wantErr {
			t.Errorf("%q. UsersStore.All() error = %v, wantErr %v", tt.name, err, tt.wantErr)
			continue
		}
		for i, got := range gots {
			usersEqual(t, tt.name, "All", &got, &tt.want[i])
		}
	}
}

func usersEqual(t *testing.T, name, method string, u1, u2 *chronograf.User) {
	if u1.Name != u2.Name {
		t.Errorf("%q. UsersStore.%s() .Name:\ngot %v, want %v", name, method, u1.Name, u2.Name)
	}
	if u1.Provider != u2.Provider {
		t.Errorf("%q. UsersStore.%s() .Provider:\ngot %v, want %v", name, method, u1.Provider, u2.Provider)
	}
	if u1.Scheme != u2.Scheme {
		t.Errorf("%q. UsersStore.%s() .Scheme:\ngot %v, want %v", name, method, u1.Scheme, u2.Scheme)
	}
	if len(u1.Roles) != len(u2.Roles) {
		t.Errorf("%q. UsersStore.%s() .Roles:\ngot %v, want %v", name, method, u1.Roles, u2.Roles)
		return
	}
	if len(u1.Roles) > 0 && len(u2.Roles) > 0 {
		for i, gotRole := range u1.Roles {
			wantRole := u2.Roles[i]
			if wantRole.Name != gotRole.Name {
				t.Errorf("%q. UsersStore.%s() .Roles[%d]:\ngot %v, want %v", name, method, i, gotRole, wantRole)
			}
		}
	}
}
<|endoftext|>


<commit_before>package bosh

import (
	"github.com/EngineerBetter/concourse-up/bosh/internal/aws"
	"github.com/EngineerBetter/concourse-up/bosh/internal/boshenv"
	"github.com/EngineerBetter/concourse-up/db"
)

func (client *AWSClient) Deploy(state, creds []byte, detach bool) (newState, newCreds []byte, err error) {
	bosh, err := boshenv.New(boshenv.DownloadBOSH())
	if err != nil {
		return state, creds, err
	}

	state, creds, err = client.createEnv(bosh, state, creds)
	if err != nil {
		return state, creds, err
	}

	if err = client.updateCloudConfig(bosh); err != nil {
		return state, creds, err
	}
	if err = client.uploadConcourseStemcell(bosh); err != nil {
		return state, creds, err
	}
	if err = client.createDefaultDatabases(); err != nil {
		return state, creds, err
	}

	creds, err = client.deployConcourse(creds, detach)
	if err != nil {
		return state, creds, err
	}

	return state, creds, err
}

func (client *AWSClient) createEnv(bosh *boshenv.BOSHCLI, state, creds []byte) (newState, newCreds []byte, err error) {
	tags, err := splitTags(client.config.Tags)
	if err != nil {
		return state, creds, err
	}
	tags["concourse-up-project"] = client.config.Project
	tags["concourse-up-component"] = "concourse"
	//TODO(px): pull up this so that we use aws.Store
	store := temporaryStore{
		"vars.yaml":  creds,
		"state.json": state,
	}

	boshUserAccessKeyID, err1 := client.metadata.Get("BoshUserAccessKeyID")
	if err1 != nil {
		return state, creds, err1
	}
	boshSecretAccessKey, err1 := client.metadata.Get("BoshSecretAccessKey")
	if err1 != nil {
		return state, creds, err1
	}
	publicSubnetID, err1 := client.metadata.Get("PublicSubnetID")
	if err1 != nil {
		return state, creds, err1
	}
	privateSubnetID, err1 := client.metadata.Get("PrivateSubnetID")
	if err1 != nil {
		return state, creds, err1
	}
	directorPublicIP, err1 := client.metadata.Get("DirectorPublicIP")
	if err1 != nil {
		return state, creds, err1
	}
	atcSecurityGroupID, err1 := client.metadata.Get("ATCSecurityGroupID")
	if err1 != nil {
		return state, creds, err1
	}
	vmSecurityGroupID, err1 := client.metadata.Get("VMsSecurityGroupID")
	if err1 != nil {
		return state, creds, err1
	}
	blobstoreBucket, err1 := client.metadata.Get("BlobstoreBucket")
	if err1 != nil {
		return state, creds, err1
	}
	boshDBAddress, err1 := client.metadata.Get("BoshDBAddress")
	if err1 != nil {
		return state, creds, err1
	}
	boshDbPort, err1 := client.metadata.Get("BoshDBPort")
	if err1 != nil {
		return state, creds, err1
	}
	blobstoreUserAccessKeyID, err1 := client.metadata.Get("BlobstoreUserAccessKeyID")
	if err1 != nil {
		return state, creds, err1
	}
	blobstoreSecretAccessKey, err1 := client.metadata.Get("BlobstoreSecretAccessKey")
	if err1 != nil {
		return state, creds, err1
	}
	directorKeyPair, err1 := client.metadata.Get("DirectorKeyPair")
	if err1 != nil {
		return state, creds, err1
	}
	directorSecurityGroup, err1 := client.metadata.Get("DirectorSecurityGroupID")
	if err1 != nil {
		return state, creds, err1
	}
	err1 = bosh.CreateEnv(store, aws.Environment{
		InternalCIDR:    "10.0.0.0/24",
		InternalGateway: "10.0.0.1",
		InternalIP:      "10.0.0.6",
		AccessKeyID:     boshUserAccessKeyID,
		SecretAccessKey: boshSecretAccessKey,
		Region:          client.config.Region,
		AZ:              client.config.AvailabilityZone,
		DefaultKeyName:  directorKeyPair,
		DefaultSecurityGroups: []string{
			directorSecurityGroup,
			vmSecurityGroupID,
		},
		PrivateKey:           client.config.PrivateKey,
		PublicSubnetID:       publicSubnetID,
		PrivateSubnetID:      privateSubnetID,
		ExternalIP:           directorPublicIP,
		ATCSecurityGroup:     atcSecurityGroupID,
		VMSecurityGroup:      vmSecurityGroupID,
		BlobstoreBucket:      blobstoreBucket,
		DBCACert:             db.RDSRootCert,
		DBHost:               boshDBAddress,
		DBName:               client.config.RDSDefaultDatabaseName,
		DBPassword:           client.config.RDSPassword,
		DBPort:               boshDbPort,
		DBUsername:           client.config.RDSUsername,
		S3AWSAccessKeyID:     blobstoreUserAccessKeyID,
		S3AWSSecretAccessKey: blobstoreSecretAccessKey,
		Spot:                 client.config.Spot,
		WorkerType:           client.config.WorkerType}, client.config.DirectorPassword, client.config.DirectorCert, client.config.DirectorKey, client.config.DirectorCACert, tags)
	if err1 != nil {
		return store["state.json"], store["vars.yaml"], err1
	}
	return store["state.json"], store["vars.yaml"], err
}

func (client *AWSClient) updateCloudConfig(bosh *boshenv.BOSHCLI) error {
	publicSubnetID, err := client.metadata.Get("PublicSubnetID")
	if err != nil {
		return err
	}
	privateSubnetID, err := client.metadata.Get("PrivateSubnetID")
	if err != nil {
		return err
	}
	aTCSecurityGroupID, err := client.metadata.Get("ATCSecurityGroupID")
	if err != nil {
		return err
	}
	vMsSecurityGroupID, err := client.metadata.Get("VMsSecurityGroupID")
	if err != nil {
		return err
	}
	directorPublicIP, err := client.metadata.Get("DirectorPublicIP")
	if err != nil {
		return err
	}
	return bosh.UpdateCloudConfig(aws.Environment{
		AZ:               client.config.AvailabilityZone,
		PublicSubnetID:   publicSubnetID,
		PrivateSubnetID:  privateSubnetID,
		ATCSecurityGroup: aTCSecurityGroupID,
		VMSecurityGroup:  vMsSecurityGroupID,
		Spot:             client.config.Spot,
		ExternalIP:       directorPublicIP,
		WorkerType:       client.config.WorkerType,
	}, directorPublicIP, client.config.DirectorPassword, client.config.DirectorCACert)

	return nil
}
func (client *AWSClient) uploadConcourseStemcell(bosh *boshenv.BOSHCLI) error {
	directorPublicIP, err := client.metadata.Get("DirectorPublicIP")
	if err != nil {
		return err
	}
	return bosh.UploadConcourseStemcell(aws.Environment{
		ExternalIP: directorPublicIP,
	}, directorPublicIP, client.config.DirectorPassword, client.config.DirectorCACert)
	return nil
}

func (client *AWSClient) saveStateFile(bytes []byte) (string, error) {
	if bytes == nil {
		return client.director.PathInWorkingDir(StateFilename), nil
	}

	return client.director.SaveFileToWorkingDir(StateFilename, bytes)
}

func (client *AWSClient) saveCredsFile(bytes []byte) (string, error) {
	if bytes == nil {
		return client.director.PathInWorkingDir(CredsFilename), nil
	}

	return client.director.SaveFileToWorkingDir(CredsFilename, bytes)
}
<commit_msg>linter is happy<commit_after>package bosh

import (
	"github.com/EngineerBetter/concourse-up/bosh/internal/aws"
	"github.com/EngineerBetter/concourse-up/bosh/internal/boshenv"
	"github.com/EngineerBetter/concourse-up/db"
)

// Deploy implements deploy for AWS client
func (client *AWSClient) Deploy(state, creds []byte, detach bool) (newState, newCreds []byte, err error) {
	bosh, err := boshenv.New(boshenv.DownloadBOSH())
	if err != nil {
		return state, creds, err
	}

	state, creds, err = client.createEnv(bosh, state, creds)
	if err != nil {
		return state, creds, err
	}

	if err = client.updateCloudConfig(bosh); err != nil {
		return state, creds, err
	}
	if err = client.uploadConcourseStemcell(bosh); err != nil {
		return state, creds, err
	}
	if err = client.createDefaultDatabases(); err != nil {
		return state, creds, err
	}

	creds, err = client.deployConcourse(creds, detach)
	if err != nil {
		return state, creds, err
	}

	return state, creds, err
}

func (client *AWSClient) createEnv(bosh *boshenv.BOSHCLI, state, creds []byte) (newState, newCreds []byte, err error) {
	tags, err := splitTags(client.config.Tags)
	if err != nil {
		return state, creds, err
	}
	tags["concourse-up-project"] = client.config.Project
	tags["concourse-up-component"] = "concourse"
	//TODO(px): pull up this so that we use aws.Store
	store := temporaryStore{
		"vars.yaml":  creds,
		"state.json": state,
	}

	boshUserAccessKeyID, err1 := client.metadata.Get("BoshUserAccessKeyID")
	if err1 != nil {
		return state, creds, err1
	}
	boshSecretAccessKey, err1 := client.metadata.Get("BoshSecretAccessKey")
	if err1 != nil {
		return state, creds, err1
	}
	publicSubnetID, err1 := client.metadata.Get("PublicSubnetID")
	if err1 != nil {
		return state, creds, err1
	}
	privateSubnetID, err1 := client.metadata.Get("PrivateSubnetID")
	if err1 != nil {
		return state, creds, err1
	}
	directorPublicIP, err1 := client.metadata.Get("DirectorPublicIP")
	if err1 != nil {
		return state, creds, err1
	}
	atcSecurityGroupID, err1 := client.metadata.Get("ATCSecurityGroupID")
	if err1 != nil {
		return state, creds, err1
	}
	vmSecurityGroupID, err1 := client.metadata.Get("VMsSecurityGroupID")
	if err1 != nil {
		return state, creds, err1
	}
	blobstoreBucket, err1 := client.metadata.Get("BlobstoreBucket")
	if err1 != nil {
		return state, creds, err1
	}
	boshDBAddress, err1 := client.metadata.Get("BoshDBAddress")
	if err1 != nil {
		return state, creds, err1
	}
	boshDbPort, err1 := client.metadata.Get("BoshDBPort")
	if err1 != nil {
		return state, creds, err1
	}
	blobstoreUserAccessKeyID, err1 := client.metadata.Get("BlobstoreUserAccessKeyID")
	if err1 != nil {
		return state, creds, err1
	}
	blobstoreSecretAccessKey, err1 := client.metadata.Get("BlobstoreSecretAccessKey")
	if err1 != nil {
		return state, creds, err1
	}
	directorKeyPair, err1 := client.metadata.Get("DirectorKeyPair")
	if err1 != nil {
		return state, creds, err1
	}
	directorSecurityGroup, err1 := client.metadata.Get("DirectorSecurityGroupID")
	if err1 != nil {
		return state, creds, err1
	}
	err1 = bosh.CreateEnv(store, aws.Environment{
		InternalCIDR:    "10.0.0.0/24",
		InternalGateway: "10.0.0.1",
		InternalIP:      "10.0.0.6",
		AccessKeyID:     boshUserAccessKeyID,
		SecretAccessKey: boshSecretAccessKey,
		Region:          client.config.Region,
		AZ:              client.config.AvailabilityZone,
		DefaultKeyName:  directorKeyPair,
		DefaultSecurityGroups: []string{
			directorSecurityGroup,
			vmSecurityGroupID,
		},
		PrivateKey:           client.config.PrivateKey,
		PublicSubnetID:       publicSubnetID,
		PrivateSubnetID:      privateSubnetID,
		ExternalIP:           directorPublicIP,
		ATCSecurityGroup:     atcSecurityGroupID,
		VMSecurityGroup:      vmSecurityGroupID,
		BlobstoreBucket:      blobstoreBucket,
		DBCACert:             db.RDSRootCert,
		DBHost:               boshDBAddress,
		DBName:               client.config.RDSDefaultDatabaseName,
		DBPassword:           client.config.RDSPassword,
		DBPort:               boshDbPort,
		DBUsername:           client.config.RDSUsername,
		S3AWSAccessKeyID:     blobstoreUserAccessKeyID,
		S3AWSSecretAccessKey: blobstoreSecretAccessKey,
		Spot:                 client.config.Spot,
		WorkerType:           client.config.WorkerType}, client.config.DirectorPassword, client.config.DirectorCert, client.config.DirectorKey, client.config.DirectorCACert, tags)
	if err1 != nil {
		return store["state.json"], store["vars.yaml"], err1
	}
	return store["state.json"], store["vars.yaml"], err
}

func (client *AWSClient) updateCloudConfig(bosh *boshenv.BOSHCLI) error {
	publicSubnetID, err := client.metadata.Get("PublicSubnetID")
	if err != nil {
		return err
	}
	privateSubnetID, err := client.metadata.Get("PrivateSubnetID")
	if err != nil {
		return err
	}
	aTCSecurityGroupID, err := client.metadata.Get("ATCSecurityGroupID")
	if err != nil {
		return err
	}
	vMsSecurityGroupID, err := client.metadata.Get("VMsSecurityGroupID")
	if err != nil {
		return err
	}
	directorPublicIP, err := client.metadata.Get("DirectorPublicIP")
	if err != nil {
		return err
	}
	return bosh.UpdateCloudConfig(aws.Environment{
		AZ:               client.config.AvailabilityZone,
		PublicSubnetID:   publicSubnetID,
		PrivateSubnetID:  privateSubnetID,
		ATCSecurityGroup: aTCSecurityGroupID,
		VMSecurityGroup:  vMsSecurityGroupID,
		Spot:             client.config.Spot,
		ExternalIP:       directorPublicIP,
		WorkerType:       client.config.WorkerType,
	}, directorPublicIP, client.config.DirectorPassword, client.config.DirectorCACert)

	return nil
}
func (client *AWSClient) uploadConcourseStemcell(bosh *boshenv.BOSHCLI) error {
	directorPublicIP, err := client.metadata.Get("DirectorPublicIP")
	if err != nil {
		return err
	}
	return bosh.UploadConcourseStemcell(aws.Environment{
		ExternalIP: directorPublicIP,
	}, directorPublicIP, client.config.DirectorPassword, client.config.DirectorCACert)
	return nil
}

func (client *AWSClient) saveStateFile(bytes []byte) (string, error) {
	if bytes == nil {
		return client.director.PathInWorkingDir(StateFilename), nil
	}

	return client.director.SaveFileToWorkingDir(StateFilename, bytes)
}

func (client *AWSClient) saveCredsFile(bytes []byte) (string, error) {
	if bytes == nil {
		return client.director.PathInWorkingDir(CredsFilename), nil
	}

	return client.director.SaveFileToWorkingDir(CredsFilename, bytes)
}
<|endoftext|>


<commit_before>package flags

import (
    "github.com/codegangsta/cli"
)

const defaultApiUrl = "https://bintray.com/api/v1/"

type Flags interface {

    Get(args interface{}) []cli.Flag
}<commit_msg>Update flags.go<commit_after>package flags

import (
    "github.com/codegangsta/cli"
)

type Flags interface {

    Get(args interface{}) []cli.Flag
}
<|endoftext|>


<commit_before>/*
Copyright 2014 Workiva, LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package palm

import (
	"log"
	"sync"
	"sync/atomic"
	"time"

	"github.com/Workiva/go-datastructures/futures"
	"github.com/Workiva/go-datastructures/queue"
)

type operation int

const (
	get operation = iota
	add
	remove
)

type recursiveBuild struct {
	keys   []Key
	nodes  []*node
	parent *node
}

type pending struct {
	reads    actions
	writes   Keys
	number   uint64
	signal   *futures.Future
	signaler chan interface{}
}

type ptree struct {
	root        *node
	ary, number uint64
	pending     *pending
	lock, write sync.Mutex
	waiter      *queue.Queue
}

func (ptree *ptree) initPending() {
	signaler := make(chan interface{})
	future := futures.New(signaler, 30*time.Minute)
	ptree.pending = &pending{
		signal:   future,
		signaler: signaler,
	}
}

func (ptree *ptree) listen() {
	for {
		_, err := ptree.waiter.Get(10)
		if err != nil {
			return
		}

		ptree.runOperations()
	}
}

func (ptree *ptree) runReads(reads actions, wg *sync.WaitGroup) {
	var key Key
	var i uint64
	for _, action := range reads {
		for {
			key, i = action.getKey()
			if key == nil {
				break
			}

			n := getParent(ptree.root, key)
			switch action.operation() {
			case get:
				if n == nil {
					action.addResult(i, nil)
				}
				index := n.keys.search(key)
				k := n.keys.byPosition(index)
				if index < n.keys.len() && k.Compare(key) == 0 {
					action.addResult(i, k)
				} else {
					action.addResult(i, nil)
				}
			}
		}
	}

	wg.Done()
}

func (ptree *ptree) runOperations() {
	ptree.lock.Lock()
	toPerform := ptree.pending
	ptree.initPending()
	ptree.lock.Unlock()

	if toPerform.number == 0 {
		return
	}

	var wg sync.WaitGroup
	var offset int
	var inserts []*node
	wg.Add(1) // for the gets

	if len(toPerform.writes) > 0 {
		inserts = make([]*node, len(toPerform.writes))
		chunks := chunkKeys(toPerform.writes, 8)
		wg.Add(len(chunks)) // for the inserts
		go ptree.runReads(toPerform.reads, &wg)

		for _, chunk := range chunks {
			go func(offset int, chunk Keys) {
				for i, k := range chunk {
					n := getParent(ptree.root, k)
					inserts[offset+i] = n
				}

				wg.Done()
			}(offset, chunk)
			offset += len(chunk)
		}
	} else {
		go ptree.runReads(toPerform.reads, &wg)
	}

	wg.Wait()
	if len(toPerform.writes) == 0 {
		return
	}
	writeOperations := make(map[*node]Keys)
	for i, n := range inserts {
		writeOperations[n] = append(writeOperations[n], toPerform.writes[i])
	}

	toPerform.signaler <- true
	ptree.runAdds(writeOperations)
}

func (ptree *ptree) recursiveSplit(n, parent, left *node, nodes *[]*node, keys *Keys) {
	if !n.needsSplit(ptree.ary) {
		return
	}

	key, l, r := n.split()
	if left != nil {
		left.right = l
	}
	l.parent = parent
	r.parent = parent
	*keys = append(*keys, key)
	*nodes = append(*nodes, l, r)
	ptree.recursiveSplit(l, parent, left, nodes, keys)
	ptree.recursiveSplit(r, parent, l, nodes, keys)
}

func (ptree *ptree) recursiveAdd(layer map[*node][]*recursiveBuild, setRoot bool) {
	if len(layer) == 0 {
		return
	}

	if setRoot && len(layer) > 1 {
		panic(`SHOULD ONLY HAVE ONE ROOT`)
	}

	q := queue.New(int64(len(layer)))
	for _, rbs := range layer {
		q.Put(rbs)
	}

	layer = make(map[*node][]*recursiveBuild, len(layer))
	dummyRoot := &node{
		keys:  newKeys(),
		nodes: newNodes(),
	}
	queue.ExecuteInParallel(q, func(ifc interface{}) {
		rbs := ifc.([]*recursiveBuild)

		if len(rbs) == 0 {
			return
		}

		n := rbs[0].parent
		if setRoot {
			ptree.root = n
		}

		parent := n.parent
		if parent == nil {
			parent = dummyRoot
			setRoot = true
		}

		for _, rb := range rbs {
			for i, k := range rb.keys {
				if n.keys.len() == 0 {
					n.keys.insert(k)
					n.nodes.push(rb.nodes[i*2])
					n.nodes.push(rb.nodes[i*2+1])
					continue
				}

				n.keys.insert(k)
				index := n.search(k)
				n.nodes.replaceAt(index, rb.nodes[i*2])
				n.nodes.insertAt(index+1, rb.nodes[i*2+1])
			}
		}

		if n.needsSplit(ptree.ary) {
			keys := make(Keys, 0, n.keys.len())
			nodes := make([]*node, 0, n.nodes.len())
			ptree.recursiveSplit(n, parent, nil, &nodes, &keys)
			ptree.write.Lock()
			layer[parent] = append(
				layer[parent], &recursiveBuild{keys: keys, nodes: nodes, parent: parent},
			)
			ptree.write.Unlock()
		}
	})

	ptree.recursiveAdd(layer, setRoot)
}

func (ptree *ptree) runAdds(addOperations map[*node]Keys) {
	if len(addOperations) == 0 {
		return
	}

	q := queue.New(int64(len(addOperations)))

	for n := range addOperations {
		q.Put(n)
	}

	nextLayer := make(map[*node][]*recursiveBuild)
	dummyRoot := &node{
		keys:  newKeys(),
		nodes: newNodes(),
	} // constructed in case we need it
	var needRoot uint64
	queue.ExecuteInParallel(q, func(ifc interface{}) {
		n := ifc.(*node)
		keys := addOperations[n]

		if len(keys) == 0 {
			return
		}

		parent := n.parent
		if parent == nil {
			parent = dummyRoot
			atomic.AddUint64(&needRoot, 1)
		}

		for _, key := range keys {
			oldKey := n.keys.insert(key)
			if oldKey == nil {
				atomic.AddUint64(&ptree.number, 1)
			}
		}

		if n.needsSplit(ptree.ary) {
			keys := make(Keys, 0, n.keys.len())
			nodes := make([]*node, 0, n.nodes.len())
			ptree.recursiveSplit(n, parent, nil, &nodes, &keys)
			ptree.write.Lock()
			nextLayer[parent] = append(
				nextLayer[parent], &recursiveBuild{keys: keys, nodes: nodes, parent: parent},
			)
			ptree.write.Unlock()
		}
	})

	setRoot := needRoot > 0

	ptree.recursiveAdd(nextLayer, setRoot)
}

// Insert will add the provided keys to the tree.
func (ptree *ptree) Insert(keys ...Key) {
	var signaler *futures.Future
	ptree.lock.Lock()
	ptree.pending.writes = append(ptree.pending.writes, keys...)
	ptree.pending.number += uint64(len(keys))
	signaler = ptree.pending.signal
	ptree.lock.Unlock()

	ptree.waiter.Put(true)
	signaler.GetResult()
}

// Get will retrieve a list of keys from the provided keys.
func (ptree *ptree) Get(keys ...Key) Keys {
	ga := newGetAction(keys)
	ptree.lock.Lock()
	ptree.pending.reads = append(ptree.pending.reads, ga)
	ptree.pending.number += uint64(len(keys))
	ptree.lock.Unlock()

	ptree.waiter.Put(true)
	return <-ga.completer
}

// Len returns the number of items in the tree.
func (ptree *ptree) Len() uint64 {
	return atomic.LoadUint64(&ptree.number)
}

// Dispose will clean up any resources used by this tree.  This
// must be called to prevent a memory leak.
func (ptree *ptree) Dispose() {
	ptree.waiter.Dispose()
}

func (ptree *ptree) print(output *log.Logger) {
	println(`PRINTING TREE`)
	if ptree.root == nil {
		return
	}

	ptree.root.print(output)
}

func newTree(ary uint64) *ptree {
	ptree := &ptree{
		root:    newNode(true, newKeys(), newNodes()),
		ary:     ary,
		pending: &pending{},
		waiter:  queue.New(10),
	}
	ptree.initPending()
	go ptree.listen()
	return ptree
}

// New will allocate, initialize, and return a new B-Tree based
// on PALM principles.  This type of tree is suited for in-memory
// indices in a multi-threaded environment.
func New(ary uint64) BTree {
	return newTree(ary)
}
<commit_msg>Use specified type for list of keys.<commit_after>/*
Copyright 2014 Workiva, LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package palm

import (
	"log"
	"sync"
	"sync/atomic"
	"time"

	"github.com/Workiva/go-datastructures/futures"
	"github.com/Workiva/go-datastructures/queue"
)

type operation int

const (
	get operation = iota
	add
	remove
)

type recursiveBuild struct {
	keys   Keys
	nodes  []*node
	parent *node
}

type pending struct {
	reads    actions
	writes   Keys
	number   uint64
	signal   *futures.Future
	signaler chan interface{}
}

type ptree struct {
	root        *node
	ary, number uint64
	pending     *pending
	lock, write sync.Mutex
	waiter      *queue.Queue
}

func (ptree *ptree) initPending() {
	signaler := make(chan interface{})
	future := futures.New(signaler, 30*time.Minute)
	ptree.pending = &pending{
		signal:   future,
		signaler: signaler,
	}
}

func (ptree *ptree) listen() {
	for {
		_, err := ptree.waiter.Get(10)
		if err != nil {
			return
		}

		ptree.runOperations()
	}
}

func (ptree *ptree) runReads(reads actions, wg *sync.WaitGroup) {
	var key Key
	var i uint64
	for _, action := range reads {
		for {
			key, i = action.getKey()
			if key == nil {
				break
			}

			n := getParent(ptree.root, key)
			switch action.operation() {
			case get:
				if n == nil {
					action.addResult(i, nil)
				}
				index := n.keys.search(key)
				k := n.keys.byPosition(index)
				if index < n.keys.len() && k.Compare(key) == 0 {
					action.addResult(i, k)
				} else {
					action.addResult(i, nil)
				}
			}
		}
	}

	wg.Done()
}

func (ptree *ptree) runOperations() {
	ptree.lock.Lock()
	toPerform := ptree.pending
	ptree.initPending()
	ptree.lock.Unlock()

	if toPerform.number == 0 {
		return
	}

	var wg sync.WaitGroup
	var offset int
	var inserts []*node
	wg.Add(1) // for the gets

	if len(toPerform.writes) > 0 {
		inserts = make([]*node, len(toPerform.writes))
		chunks := chunkKeys(toPerform.writes, 8)
		wg.Add(len(chunks)) // for the inserts
		go ptree.runReads(toPerform.reads, &wg)

		for _, chunk := range chunks {
			go func(offset int, chunk Keys) {
				for i, k := range chunk {
					n := getParent(ptree.root, k)
					inserts[offset+i] = n
				}

				wg.Done()
			}(offset, chunk)
			offset += len(chunk)
		}
	} else {
		go ptree.runReads(toPerform.reads, &wg)
	}

	wg.Wait()
	if len(toPerform.writes) == 0 {
		return
	}
	writeOperations := make(map[*node]Keys)
	for i, n := range inserts {
		writeOperations[n] = append(writeOperations[n], toPerform.writes[i])
	}

	toPerform.signaler <- true
	ptree.runAdds(writeOperations)
}

func (ptree *ptree) recursiveSplit(n, parent, left *node, nodes *[]*node, keys *Keys) {
	if !n.needsSplit(ptree.ary) {
		return
	}

	key, l, r := n.split()
	if left != nil {
		left.right = l
	}
	l.parent = parent
	r.parent = parent
	*keys = append(*keys, key)
	*nodes = append(*nodes, l, r)
	ptree.recursiveSplit(l, parent, left, nodes, keys)
	ptree.recursiveSplit(r, parent, l, nodes, keys)
}

func (ptree *ptree) recursiveAdd(layer map[*node][]*recursiveBuild, setRoot bool) {
	if len(layer) == 0 {
		return
	}

	if setRoot && len(layer) > 1 {
		panic(`SHOULD ONLY HAVE ONE ROOT`)
	}

	q := queue.New(int64(len(layer)))
	for _, rbs := range layer {
		q.Put(rbs)
	}

	layer = make(map[*node][]*recursiveBuild, len(layer))
	dummyRoot := &node{
		keys:  newKeys(),
		nodes: newNodes(),
	}
	queue.ExecuteInParallel(q, func(ifc interface{}) {
		rbs := ifc.([]*recursiveBuild)

		if len(rbs) == 0 {
			return
		}

		n := rbs[0].parent
		if setRoot {
			ptree.root = n
		}

		parent := n.parent
		if parent == nil {
			parent = dummyRoot
			setRoot = true
		}

		for _, rb := range rbs {
			for i, k := range rb.keys {
				if n.keys.len() == 0 {
					n.keys.insert(k)
					n.nodes.push(rb.nodes[i*2])
					n.nodes.push(rb.nodes[i*2+1])
					continue
				}

				n.keys.insert(k)
				index := n.search(k)
				n.nodes.replaceAt(index, rb.nodes[i*2])
				n.nodes.insertAt(index+1, rb.nodes[i*2+1])
			}
		}

		if n.needsSplit(ptree.ary) {
			keys := make(Keys, 0, n.keys.len())
			nodes := make([]*node, 0, n.nodes.len())
			ptree.recursiveSplit(n, parent, nil, &nodes, &keys)
			ptree.write.Lock()
			layer[parent] = append(
				layer[parent], &recursiveBuild{keys: keys, nodes: nodes, parent: parent},
			)
			ptree.write.Unlock()
		}
	})

	ptree.recursiveAdd(layer, setRoot)
}

func (ptree *ptree) runAdds(addOperations map[*node]Keys) {
	if len(addOperations) == 0 {
		return
	}

	q := queue.New(int64(len(addOperations)))

	for n := range addOperations {
		q.Put(n)
	}

	nextLayer := make(map[*node][]*recursiveBuild)
	dummyRoot := &node{
		keys:  newKeys(),
		nodes: newNodes(),
	} // constructed in case we need it
	var needRoot uint64
	queue.ExecuteInParallel(q, func(ifc interface{}) {
		n := ifc.(*node)
		keys := addOperations[n]

		if len(keys) == 0 {
			return
		}

		parent := n.parent
		if parent == nil {
			parent = dummyRoot
			atomic.AddUint64(&needRoot, 1)
		}

		for _, key := range keys {
			oldKey := n.keys.insert(key)
			if oldKey == nil {
				atomic.AddUint64(&ptree.number, 1)
			}
		}

		if n.needsSplit(ptree.ary) {
			keys := make(Keys, 0, n.keys.len())
			nodes := make([]*node, 0, n.nodes.len())
			ptree.recursiveSplit(n, parent, nil, &nodes, &keys)
			ptree.write.Lock()
			nextLayer[parent] = append(
				nextLayer[parent], &recursiveBuild{keys: keys, nodes: nodes, parent: parent},
			)
			ptree.write.Unlock()
		}
	})

	setRoot := needRoot > 0

	ptree.recursiveAdd(nextLayer, setRoot)
}

// Insert will add the provided keys to the tree.
func (ptree *ptree) Insert(keys ...Key) {
	var signaler *futures.Future
	ptree.lock.Lock()
	ptree.pending.writes = append(ptree.pending.writes, keys...)
	ptree.pending.number += uint64(len(keys))
	signaler = ptree.pending.signal
	ptree.lock.Unlock()

	ptree.waiter.Put(true)
	signaler.GetResult()
}

// Get will retrieve a list of keys from the provided keys.
func (ptree *ptree) Get(keys ...Key) Keys {
	ga := newGetAction(keys)
	ptree.lock.Lock()
	ptree.pending.reads = append(ptree.pending.reads, ga)
	ptree.pending.number += uint64(len(keys))
	ptree.lock.Unlock()

	ptree.waiter.Put(true)
	return <-ga.completer
}

// Len returns the number of items in the tree.
func (ptree *ptree) Len() uint64 {
	return atomic.LoadUint64(&ptree.number)
}

// Dispose will clean up any resources used by this tree.  This
// must be called to prevent a memory leak.
func (ptree *ptree) Dispose() {
	ptree.waiter.Dispose()
}

func (ptree *ptree) print(output *log.Logger) {
	println(`PRINTING TREE`)
	if ptree.root == nil {
		return
	}

	ptree.root.print(output)
}

func newTree(ary uint64) *ptree {
	ptree := &ptree{
		root:    newNode(true, newKeys(), newNodes()),
		ary:     ary,
		pending: &pending{},
		waiter:  queue.New(10),
	}
	ptree.initPending()
	go ptree.listen()
	return ptree
}

// New will allocate, initialize, and return a new B-Tree based
// on PALM principles.  This type of tree is suited for in-memory
// indices in a multi-threaded environment.
func New(ary uint64) BTree {
	return newTree(ary)
}
<|endoftext|>


<commit_before>package svnwatch

import (
	"encoding/xml"

	"github.com/pkg/errors"
)

type Watches struct {
	XMLName xml.Name `xml:"watches"`
	Watches []Watch  `xml:"watch"`
}

type Watch struct {
	XMLName  xml.Name  `xml:"watch"`
	URL      string    `xml:"url,attr"`
	Commands []Command `xml:"command"`
}

func (w Watch) Update(repositories *Repositories) error {
	repo := repositories.ForURL(w.URL)

	updated, err := repo.Update()

	if err != nil {
		return errors.Wrapf(err, "failed to update watch for %s", w.URL)
	}

	if updated {
		for _, cmd := range w.Commands {
			if err := cmd.Execute(*repo); err != nil {
				return errors.Wrapf(err, "failed to execute command")
			}
		}
	}

	return nil
}
<commit_msg>Make <watch> elements require a URL<commit_after>package svnwatch

import (
	"encoding/xml"

	"github.com/pkg/errors"
)

type Watches struct {
	XMLName xml.Name `xml:"watches"`
	Watches []Watch  `xml:"watch"`
}

type Watch struct {
	XMLName  xml.Name  `xml:"watch"`
	URL      string    `xml:"url,attr"`
	Commands []Command `xml:"command"`
}

func (w Watch) Update(repositories *Repositories) error {
	repo := repositories.ForURL(w.URL)

	updated, err := repo.Update()

	if err != nil {
		return errors.Wrapf(err, "failed to update watch for %s", w.URL)
	}

	if updated {
		for _, cmd := range w.Commands {
			if err := cmd.Execute(*repo); err != nil {
				return errors.Wrapf(err, "failed to execute command")
			}
		}
	}

	return nil
}

func (w *Watch) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
	watch := struct {
		URL *string `xml:"url,attr"`
	}{}

	if err := d.DecodeElement(&watch, &start); err != nil {
		return err
	}

	if watch.URL == nil {
		return errors.New("missing URL from watch")
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package syslog

import (
	"errors"
	"fmt"
	"io"
	"os"

	"github.com/venicegeo/pz-gocommon/elasticsearch"
	piazza "github.com/venicegeo/pz-gocommon/gocommon"
)

const (
	SyslogdNetwork = ""
	SyslogdRaddr   = ""
)

func GetRequiredEnvVars() (string, string, string, error) {
	var loggerIndex, loggerType, auditType string
	if loggerIndex = os.Getenv("LOGGER_INDEX"); loggerIndex == "" {
		return "", "", "", errors.New("Elasticsearch index name not set")
	}
	if loggerType = os.Getenv("LOGGER_TYPE"); loggerType == "" {
		return "", "", "", errors.New("Elasticsearch type name not set")
	}
	if auditType = os.Getenv("AUDIT_TYPE"); auditType == "" {
		return "", "", "", errors.New("Elasticsearch audit type name not set")
	}
	return loggerIndex, loggerType, auditType, nil
}

func GetRequiredESIWriters(esi elasticsearch.IIndex, loggerType string, auditType string) (*ElasticWriter, *ElasticWriter, error) {
	var logWriter, auditWriter ElasticWriter
	var indexExisted, typeExisted, auditTypeExisted bool
	var err error
	logWriter = ElasticWriter{Esi: esi}
	if err = logWriter.SetType(loggerType); err != nil {
		return nil, nil, err
	}
	if indexExisted, err = logWriter.CreateIndex(); err != nil {
		return nil, nil, err
	}
	if !indexExisted {
		fmt.Println("Created index:", esi.IndexName())
	}
	if typeExisted, err = logWriter.CreateType(LogMapping); err != nil {
		return nil, nil, err
	}
	if !typeExisted {
		fmt.Println("Created type:", loggerType)
	}
	auditWriter = ElasticWriter{Esi: esi}
	if err = auditWriter.SetType(auditType); err != nil {
		return &logWriter, nil, err
	}
	if auditTypeExisted, err = auditWriter.CreateType(LogMapping); err != nil {
		return &logWriter, nil, err
	}
	if !auditTypeExisted {
		fmt.Println("Created type:", auditType)
	}
	return &logWriter, &auditWriter, err
}

//---------------------------------------------------------------------

// Writer is an interface for writing a Message to some sort of output.
type Writer interface {
	Write(*Message) error
	Close() error
}

// Reader is an interface for reading Messages from some sort of input.
// count is the number of messages to read: 1 means the latest message,
// 2 means the two latest messages, etc. The newest message is at the end
// of the array.
type Reader interface {
	Read(count int) ([]Message, error)
	GetMessages(*piazza.JsonPagination, *piazza.HttpQueryParams) ([]Message, int, error)
}

type WriterReader interface {
	Reader
	Writer
}

//---------------------------------------------------------------------

// FileWriter implements the Writer interface, writing to a given file
type FileWriter struct {
	FileName string
	file     *os.File
}

// Write writes the message to the supplied file.
func (w *FileWriter) Write(mssg *Message) error {
	var _ Writer = (*FileWriter)(nil)

	var err error

	if w == nil || w.FileName == "" {
		return fmt.Errorf("writer not set not set")
	}

	if w.file == nil {
		w.file, err = os.OpenFile(w.FileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0777)
		if err != nil {
			return err
		}
	}

	s := mssg.String()
	s += "\n"

	_, err = io.WriteString(w.file, s)
	return err
}

// Close closes the file. The creator of the FileWriter must call this.
func (w *FileWriter) Close() error {
	return w.file.Close()
}

//---------------------------------------------------------------------

//STDOUTWriter writes messages to STDOUT
type STDOUTWriter struct {
}

//Writes message to STDOUT
func (w *STDOUTWriter) Write(mssg *Message) error {
	var _ Writer = (*FileWriter)(nil)
	fmt.Println(mssg.String())
	return nil
}

//Nothing to close for this writer
func (w *STDOUTWriter) Close() error {
	return nil
}

//---------------------------------------------------------------------

// MessageWriter implements Reader and Writer, using an array of Messages
// as the backing store
type LocalReaderWriter struct {
	messages []Message
}

// Write writes the message to the backing array
func (w *LocalReaderWriter) Write(mssg *Message) error {
	var _ Writer = (*LocalReaderWriter)(nil)

	if w.messages == nil {
		w.messages = make([]Message, 0)
	}

	w.messages = append(w.messages, *mssg)

	return nil
}

// Read reads messages from the backing array. Will only return as many as are
// available; asking for too many is not an error.
func (w *LocalReaderWriter) Read(count int) ([]Message, error) {
	var _ Reader = (*LocalReaderWriter)(nil)

	if count < 0 {
		return nil, fmt.Errorf("invalid count: %d", count)
	}

	if w.messages == nil || count == 0 {
		return make([]Message, 0), nil
	}

	if count > len(w.messages) {
		count = len(w.messages)
	}

	n := len(w.messages)
	a := w.messages[n-count : n]

	return a, nil
}

func (w *LocalReaderWriter) GetMessages(*piazza.JsonPagination, *piazza.HttpQueryParams) ([]Message, int, error) {
	return nil, 0, fmt.Errorf("LocalReaderWriter.GetMessages() not supported")
}

func (w *LocalReaderWriter) Close() error {
	return nil
}

//---------------------------------------------------------------------

// HttpWriter implements Writer, by talking to the actual pz-logger service
type HttpWriter struct {
	url string
	h   piazza.Http
}

func NewHttpWriter(url string, apiKey string) (*HttpWriter, error) {
	w := &HttpWriter{url: url}
	w.h = piazza.Http{
		BaseUrl: url,
		ApiKey:  apiKey,
		//FlightCheck: &piazza.SimpleFlightCheck{},
	}

	return w, nil
}

func (w *HttpWriter) Write(mssg *Message) error {
	var _ Writer = (*HttpWriter)(nil)

	jresp := w.h.PzPost("/syslog", mssg)
	if jresp.IsError() {
		return jresp.ToError()
	}
	return nil
}

func (w *HttpWriter) Close() error {
	return nil
}

func (w *HttpWriter) Read(count int) ([]Message, error) {
	var _ Reader = (*HttpWriter)(nil)

	format := &piazza.JsonPagination{
		Page:    0,
		PerPage: count,
		SortBy:  "timeStamp",
		Order:   "desc",
	}
	params := &piazza.HttpQueryParams{}

	mssgs, _, err := w.GetMessages(format, params)
	return mssgs, err

}

// GetMessages is only implemented for HttpWriter as it is most likely the only
// Writer to be used for reading back data.
func (w *HttpWriter) GetMessages(
	format *piazza.JsonPagination,
	params *piazza.HttpQueryParams) ([]Message, int, error) {
	var _ Reader = (*HttpWriter)(nil)

	formatString := format.String()
	paramString := params.String()

	var ext string
	if formatString != "" && paramString != "" {
		ext = "?" + formatString + "&" + paramString
	} else if formatString == "" && paramString != "" {
		ext = "?" + paramString
	} else if formatString != "" && paramString == "" {
		ext = "?" + formatString
	} else if formatString == "" && paramString == "" {
		ext = ""
	} else {
		return nil, 0, errors.New("Internal error: failed to parse query params")
	}

	endpoint := "/syslog" + ext
	jresp := w.h.PzGet(endpoint)
	if jresp.IsError() {
		return nil, 0, jresp.ToError()
	}
	var mssgs []Message
	err := jresp.ExtractData(&mssgs)
	if err != nil {
		return nil, 0, err
	}

	return mssgs, jresp.Pagination.Count, nil
}

//---------------------------------------------------------------------

// SyslogdWriter implements a Writer that writes to the syslogd system service.
// This will almost certainly not work on Windows, but that is okay because Piazza
// does not support Windows.
type SyslogdWriter struct {
	writer *DaemonWriter
}

func (w *SyslogdWriter) initWriter() error {
	if w.writer != nil {
		return nil
	}

	tw, err := Dial(SyslogdNetwork, SyslogdRaddr)
	if err != nil {
		return err
	}

	w.writer = tw

	return nil
}

// Write writes the message to the OS's syslogd system.
func (w *SyslogdWriter) Write(mssg *Message) error {
	// compile-time check if interface is implemented
	var _ Writer = (*SyslogdWriter)(nil)

	var err error

	err = w.initWriter()
	if err != nil {
		return err
	}

	s := mssg.String()

	return w.writer.Write(s)
}

// Close closes the underlying network connection.
func (w *SyslogdWriter) Close() error {
	if w.writer == nil {
		return nil
	}
	return w.writer.Close()
}

//---------------------------------------------------------------------

// ElasticWriter implements the Writer, writing to elasticsearch
type ElasticWriter struct {
	Esi elasticsearch.IIndex
	typ string
	id  string
}

func NewElasticWriter(esi elasticsearch.IIndex, typ string) *ElasticWriter {
	ew := &ElasticWriter{
		Esi: esi,
		typ: typ,
	}
	return ew
}

// Write writes the message to the elasticsearch index, type, id
func (w *ElasticWriter) Write(mssg *Message) error {
	var _ Writer = (*ElasticWriter)(nil)

	var err error

	if w == nil || w.Esi == nil || w.typ == "" {
		return fmt.Errorf("writer not set not set")
	}

	_, err = w.Esi.PostData(w.typ, w.id, mssg)
	return err
}

// SetType sets the type to write to
func (w *ElasticWriter) SetType(typ string) error {
	if w == nil {
		return fmt.Errorf("writer not set not set")
	}
	w.typ = typ
	return nil
}

func (w *ElasticWriter) CreateIndex() (bool, error) {
	if w == nil || w.Esi == nil {
		return false, fmt.Errorf("writer not set not set")
	}
	exists, err := w.Esi.IndexExists()
	if err != nil {
		return exists, err
	}
	if !exists {
		return exists, w.Esi.Create("")
	}
	return exists, nil
}

func (w *ElasticWriter) CreateType(mapping string) (bool, error) {
	if w == nil || w.Esi == nil || w.typ == "" {
		return false, fmt.Errorf("writer not set not set")
	}
	exists, err := w.Esi.TypeExists(w.typ)
	if err != nil {
		return exists, err
	}
	if !exists {
		return exists, w.Esi.SetMapping(w.typ, piazza.JsonString(mapping))
	}
	return exists, nil
}

// SetID sets the id to write to
func (w *ElasticWriter) SetID(id string) error {
	if w == nil {
		return fmt.Errorf("writer not set not set")
	}
	w.id = id
	return nil
}

// Close does nothing but satisfy an interface.
func (w *ElasticWriter) Close() error {
	return nil
}

func (w *ElasticWriter) GetMessages(*piazza.JsonPagination, *piazza.HttpQueryParams) ([]Message, int, error) {
	return nil, 0, fmt.Errorf("ElasticWriter.GetMessages not supported")
}
func (w *ElasticWriter) Read(count int) ([]Message, error) {
	return nil, fmt.Errorf("ElasticWriter.Read not supported")
}

//-------------------------------

// NilWriter doesn't do anything
type NilWriter struct {
}

func (*NilWriter) Write(*Message) error {
	return nil
}

func (*NilWriter) Close() error {
	return nil
}

//---------------------------------------------------------------------

// MultiWriter will write to N different Writers at once. When doing a
// Write or a Close, we call the function on all of them. If any have failed,
// we return the error from the first one that failed.
type MultiWriter struct {
	writers []Writer
}

func NewMultiWriter(ws []Writer) *MultiWriter {
	mw := &MultiWriter{}
	mw.writers = make([]Writer, len(ws))
	for i, w := range ws {
		mw.writers[i] = w
	}
	return mw
}

func (mw *MultiWriter) Write(m *Message) error {
	var err error

	for _, w := range mw.writers {
		e := w.Write(m)
		if e != nil && err != nil {
			err = e
		}
	}

	return err
}

func (mw *MultiWriter) Close() error {
	var err error

	for _, w := range mw.writers {
		e := w.Close()
		if e != nil && err != nil {
			err = e
		}
	}

	return err
}
<commit_msg>commented out some noise<commit_after>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package syslog

import (
	"errors"
	"fmt"
	"io"
	"os"

	"github.com/venicegeo/pz-gocommon/elasticsearch"
	piazza "github.com/venicegeo/pz-gocommon/gocommon"
)

const (
	SyslogdNetwork = ""
	SyslogdRaddr   = ""
)

func GetRequiredEnvVars() (string, string, string, error) {
	var loggerIndex, loggerType, auditType string
	if loggerIndex = os.Getenv("LOGGER_INDEX"); loggerIndex == "" {
		return "", "", "", errors.New("Elasticsearch index name not set")
	}
	if loggerType = os.Getenv("LOGGER_TYPE"); loggerType == "" {
		return "", "", "", errors.New("Elasticsearch type name not set")
	}
	if auditType = os.Getenv("AUDIT_TYPE"); auditType == "" {
		return "", "", "", errors.New("Elasticsearch audit type name not set")
	}
	return loggerIndex, loggerType, auditType, nil
}

func GetRequiredESIWriters(esi elasticsearch.IIndex, loggerType string, auditType string) (*ElasticWriter, *ElasticWriter, error) {
	var logWriter, auditWriter ElasticWriter
	var indexExisted, typeExisted, auditTypeExisted bool
	var err error
	logWriter = ElasticWriter{Esi: esi}
	if err = logWriter.SetType(loggerType); err != nil {
		return nil, nil, err
	}
	if indexExisted, err = logWriter.CreateIndex(); err != nil {
		return nil, nil, err
	}
	if !indexExisted {
		//fmt.Println("Created index:", esi.IndexName())
	}
	if typeExisted, err = logWriter.CreateType(LogMapping); err != nil {
		return nil, nil, err
	}
	if !typeExisted {
		//fmt.Println("Created type:", loggerType)
	}
	auditWriter = ElasticWriter{Esi: esi}
	if err = auditWriter.SetType(auditType); err != nil {
		return &logWriter, nil, err
	}
	if auditTypeExisted, err = auditWriter.CreateType(LogMapping); err != nil {
		return &logWriter, nil, err
	}
	if !auditTypeExisted {
		//fmt.Println("Created type:", auditType)
	}
	return &logWriter, &auditWriter, err
}

//---------------------------------------------------------------------

// Writer is an interface for writing a Message to some sort of output.
type Writer interface {
	Write(*Message) error
	Close() error
}

// Reader is an interface for reading Messages from some sort of input.
// count is the number of messages to read: 1 means the latest message,
// 2 means the two latest messages, etc. The newest message is at the end
// of the array.
type Reader interface {
	Read(count int) ([]Message, error)
	GetMessages(*piazza.JsonPagination, *piazza.HttpQueryParams) ([]Message, int, error)
}

type WriterReader interface {
	Reader
	Writer
}

//---------------------------------------------------------------------

// FileWriter implements the Writer interface, writing to a given file
type FileWriter struct {
	FileName string
	file     *os.File
}

// Write writes the message to the supplied file.
func (w *FileWriter) Write(mssg *Message) error {
	var _ Writer = (*FileWriter)(nil)

	var err error

	if w == nil || w.FileName == "" {
		return fmt.Errorf("writer not set not set")
	}

	if w.file == nil {
		w.file, err = os.OpenFile(w.FileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0777)
		if err != nil {
			return err
		}
	}

	s := mssg.String()
	s += "\n"

	_, err = io.WriteString(w.file, s)
	return err
}

// Close closes the file. The creator of the FileWriter must call this.
func (w *FileWriter) Close() error {
	return w.file.Close()
}

//---------------------------------------------------------------------

//STDOUTWriter writes messages to STDOUT
type STDOUTWriter struct {
}

//Writes message to STDOUT
func (w *STDOUTWriter) Write(mssg *Message) error {
	var _ Writer = (*FileWriter)(nil)
	fmt.Println(mssg.String())
	return nil
}

//Nothing to close for this writer
func (w *STDOUTWriter) Close() error {
	return nil
}

//---------------------------------------------------------------------

// MessageWriter implements Reader and Writer, using an array of Messages
// as the backing store
type LocalReaderWriter struct {
	messages []Message
}

// Write writes the message to the backing array
func (w *LocalReaderWriter) Write(mssg *Message) error {
	var _ Writer = (*LocalReaderWriter)(nil)

	if w.messages == nil {
		w.messages = make([]Message, 0)
	}

	w.messages = append(w.messages, *mssg)

	return nil
}

// Read reads messages from the backing array. Will only return as many as are
// available; asking for too many is not an error.
func (w *LocalReaderWriter) Read(count int) ([]Message, error) {
	var _ Reader = (*LocalReaderWriter)(nil)

	if count < 0 {
		return nil, fmt.Errorf("invalid count: %d", count)
	}

	if w.messages == nil || count == 0 {
		return make([]Message, 0), nil
	}

	if count > len(w.messages) {
		count = len(w.messages)
	}

	n := len(w.messages)
	a := w.messages[n-count : n]

	return a, nil
}

func (w *LocalReaderWriter) GetMessages(*piazza.JsonPagination, *piazza.HttpQueryParams) ([]Message, int, error) {
	return nil, 0, fmt.Errorf("LocalReaderWriter.GetMessages() not supported")
}

func (w *LocalReaderWriter) Close() error {
	return nil
}

//---------------------------------------------------------------------

// HttpWriter implements Writer, by talking to the actual pz-logger service
type HttpWriter struct {
	url string
	h   piazza.Http
}

func NewHttpWriter(url string, apiKey string) (*HttpWriter, error) {
	w := &HttpWriter{url: url}
	w.h = piazza.Http{
		BaseUrl: url,
		ApiKey:  apiKey,
		//FlightCheck: &piazza.SimpleFlightCheck{},
	}

	return w, nil
}

func (w *HttpWriter) Write(mssg *Message) error {
	var _ Writer = (*HttpWriter)(nil)

	jresp := w.h.PzPost("/syslog", mssg)
	if jresp.IsError() {
		return jresp.ToError()
	}
	return nil
}

func (w *HttpWriter) Close() error {
	return nil
}

func (w *HttpWriter) Read(count int) ([]Message, error) {
	var _ Reader = (*HttpWriter)(nil)

	format := &piazza.JsonPagination{
		Page:    0,
		PerPage: count,
		SortBy:  "timeStamp",
		Order:   "desc",
	}
	params := &piazza.HttpQueryParams{}

	mssgs, _, err := w.GetMessages(format, params)
	return mssgs, err

}

// GetMessages is only implemented for HttpWriter as it is most likely the only
// Writer to be used for reading back data.
func (w *HttpWriter) GetMessages(
	format *piazza.JsonPagination,
	params *piazza.HttpQueryParams) ([]Message, int, error) {
	var _ Reader = (*HttpWriter)(nil)

	formatString := format.String()
	paramString := params.String()

	var ext string
	if formatString != "" && paramString != "" {
		ext = "?" + formatString + "&" + paramString
	} else if formatString == "" && paramString != "" {
		ext = "?" + paramString
	} else if formatString != "" && paramString == "" {
		ext = "?" + formatString
	} else if formatString == "" && paramString == "" {
		ext = ""
	} else {
		return nil, 0, errors.New("Internal error: failed to parse query params")
	}

	endpoint := "/syslog" + ext
	jresp := w.h.PzGet(endpoint)
	if jresp.IsError() {
		return nil, 0, jresp.ToError()
	}
	var mssgs []Message
	err := jresp.ExtractData(&mssgs)
	if err != nil {
		return nil, 0, err
	}

	return mssgs, jresp.Pagination.Count, nil
}

//---------------------------------------------------------------------

// SyslogdWriter implements a Writer that writes to the syslogd system service.
// This will almost certainly not work on Windows, but that is okay because Piazza
// does not support Windows.
type SyslogdWriter struct {
	writer *DaemonWriter
}

func (w *SyslogdWriter) initWriter() error {
	if w.writer != nil {
		return nil
	}

	tw, err := Dial(SyslogdNetwork, SyslogdRaddr)
	if err != nil {
		return err
	}

	w.writer = tw

	return nil
}

// Write writes the message to the OS's syslogd system.
func (w *SyslogdWriter) Write(mssg *Message) error {
	// compile-time check if interface is implemented
	var _ Writer = (*SyslogdWriter)(nil)

	var err error

	err = w.initWriter()
	if err != nil {
		return err
	}

	s := mssg.String()

	return w.writer.Write(s)
}

// Close closes the underlying network connection.
func (w *SyslogdWriter) Close() error {
	if w.writer == nil {
		return nil
	}
	return w.writer.Close()
}

//---------------------------------------------------------------------

// ElasticWriter implements the Writer, writing to elasticsearch
type ElasticWriter struct {
	Esi elasticsearch.IIndex
	typ string
	id  string
}

func NewElasticWriter(esi elasticsearch.IIndex, typ string) *ElasticWriter {
	ew := &ElasticWriter{
		Esi: esi,
		typ: typ,
	}
	return ew
}

// Write writes the message to the elasticsearch index, type, id
func (w *ElasticWriter) Write(mssg *Message) error {
	var _ Writer = (*ElasticWriter)(nil)

	var err error

	if w == nil || w.Esi == nil || w.typ == "" {
		return fmt.Errorf("writer not set not set")
	}

	_, err = w.Esi.PostData(w.typ, w.id, mssg)
	return err
}

// SetType sets the type to write to
func (w *ElasticWriter) SetType(typ string) error {
	if w == nil {
		return fmt.Errorf("writer not set not set")
	}
	w.typ = typ
	return nil
}

func (w *ElasticWriter) CreateIndex() (bool, error) {
	if w == nil || w.Esi == nil {
		return false, fmt.Errorf("writer not set not set")
	}
	exists, err := w.Esi.IndexExists()
	if err != nil {
		return exists, err
	}
	if !exists {
		return exists, w.Esi.Create("")
	}
	return exists, nil
}

func (w *ElasticWriter) CreateType(mapping string) (bool, error) {
	if w == nil || w.Esi == nil || w.typ == "" {
		return false, fmt.Errorf("writer not set not set")
	}
	exists, err := w.Esi.TypeExists(w.typ)
	if err != nil {
		return exists, err
	}
	if !exists {
		return exists, w.Esi.SetMapping(w.typ, piazza.JsonString(mapping))
	}
	return exists, nil
}

// SetID sets the id to write to
func (w *ElasticWriter) SetID(id string) error {
	if w == nil {
		return fmt.Errorf("writer not set not set")
	}
	w.id = id
	return nil
}

// Close does nothing but satisfy an interface.
func (w *ElasticWriter) Close() error {
	return nil
}

func (w *ElasticWriter) GetMessages(*piazza.JsonPagination, *piazza.HttpQueryParams) ([]Message, int, error) {
	return nil, 0, fmt.Errorf("ElasticWriter.GetMessages not supported")
}
func (w *ElasticWriter) Read(count int) ([]Message, error) {
	return nil, fmt.Errorf("ElasticWriter.Read not supported")
}

//-------------------------------

// NilWriter doesn't do anything
type NilWriter struct {
}

func (*NilWriter) Write(*Message) error {
	return nil
}

func (*NilWriter) Close() error {
	return nil
}

//---------------------------------------------------------------------

// MultiWriter will write to N different Writers at once. When doing a
// Write or a Close, we call the function on all of them. If any have failed,
// we return the error from the first one that failed.
type MultiWriter struct {
	writers []Writer
}

func NewMultiWriter(ws []Writer) *MultiWriter {
	mw := &MultiWriter{}
	mw.writers = make([]Writer, len(ws))
	for i, w := range ws {
		mw.writers[i] = w
	}
	return mw
}

func (mw *MultiWriter) Write(m *Message) error {
	var err error

	for _, w := range mw.writers {
		e := w.Write(m)
		if e != nil && err != nil {
			err = e
		}
	}

	return err
}

func (mw *MultiWriter) Close() error {
	var err error

	for _, w := range mw.writers {
		e := w.Close()
		if e != nil && err != nil {
			err = e
		}
	}

	return err
}
<|endoftext|>


<commit_before>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build 386 amd64

package cpu

const CacheLinePadSize = 64

// cpuid is implemented in cpu_x86.s.
func cpuid(eaxArg, ecxArg uint32) (eax, ebx, ecx, edx uint32)

// xgetbv with ecx = 0 is implemented in cpu_x86.s.
func xgetbv() (eax, edx uint32)

const (
	// edx bits
	cpuid_SSE2 = 1 << 26

	// ecx bits
	cpuid_SSE3      = 1 << 0
	cpuid_PCLMULQDQ = 1 << 1
	cpuid_SSSE3     = 1 << 9
	cpuid_FMA       = 1 << 12
	cpuid_SSE41     = 1 << 19
	cpuid_SSE42     = 1 << 20
	cpuid_POPCNT    = 1 << 23
	cpuid_AES       = 1 << 25
	cpuid_OSXSAVE   = 1 << 27
	cpuid_AVX       = 1 << 28

	// ebx bits
	cpuid_BMI1 = 1 << 3
	cpuid_AVX2 = 1 << 5
	cpuid_BMI2 = 1 << 8
	cpuid_ERMS = 1 << 9
	cpuid_ADX  = 1 << 19
)

var maxExtendedFunctionInformation uint32

func doinit() {
	options = []option{
		{Name: "adx", Feature: &X86.HasADX},
		{Name: "aes", Feature: &X86.HasAES},
		{Name: "avx", Feature: &X86.HasAVX},
		{Name: "avx2", Feature: &X86.HasAVX2},
		{Name: "bmi1", Feature: &X86.HasBMI1},
		{Name: "bmi2", Feature: &X86.HasBMI2},
		{Name: "erms", Feature: &X86.HasERMS},
		{Name: "fma", Feature: &X86.HasFMA},
		{Name: "pclmulqdq", Feature: &X86.HasPCLMULQDQ},
		{Name: "popcnt", Feature: &X86.HasPOPCNT},
		{Name: "sse3", Feature: &X86.HasSSE3},
		{Name: "sse41", Feature: &X86.HasSSE41},
		{Name: "sse42", Feature: &X86.HasSSE42},
		{Name: "ssse3", Feature: &X86.HasSSSE3},

		// These capabilities should always be enabled on amd64:
		{Name: "sse2", Feature: &X86.HasSSE2, Required: GOARCH == "amd64"},
	}

	maxID, _, _, _ := cpuid(0, 0)

	if maxID < 1 {
		return
	}

	maxExtendedFunctionInformation, _, _, _ = cpuid(0x80000000, 0)

	_, _, ecx1, edx1 := cpuid(1, 0)
	X86.HasSSE2 = isSet(edx1, cpuid_SSE2)

	X86.HasSSE3 = isSet(ecx1, cpuid_SSE3)
	X86.HasPCLMULQDQ = isSet(ecx1, cpuid_PCLMULQDQ)
	X86.HasSSSE3 = isSet(ecx1, cpuid_SSSE3)
	X86.HasFMA = isSet(ecx1, cpuid_FMA)
	X86.HasSSE41 = isSet(ecx1, cpuid_SSE41)
	X86.HasSSE42 = isSet(ecx1, cpuid_SSE42)
	X86.HasPOPCNT = isSet(ecx1, cpuid_POPCNT)
	X86.HasAES = isSet(ecx1, cpuid_AES)
	X86.HasOSXSAVE = isSet(ecx1, cpuid_OSXSAVE)

	osSupportsAVX := false
	// For XGETBV, OSXSAVE bit is required and sufficient.
	if X86.HasOSXSAVE {
		eax, _ := xgetbv()
		// Check if XMM and YMM registers have OS support.
		osSupportsAVX = isSet(eax, 1<<1) && isSet(eax, 1<<2)
	}

	X86.HasAVX = isSet(ecx1, cpuid_AVX) && osSupportsAVX

	if maxID < 7 {
		return
	}

	_, ebx7, _, _ := cpuid(7, 0)
	X86.HasBMI1 = isSet(ebx7, cpuid_BMI1)
	X86.HasAVX2 = isSet(ebx7, cpuid_AVX2) && osSupportsAVX
	X86.HasBMI2 = isSet(ebx7, cpuid_BMI2)
	X86.HasERMS = isSet(ebx7, cpuid_ERMS)
	X86.HasADX = isSet(ebx7, cpuid_ADX)
}

func isSet(hwc uint32, value uint32) bool {
	return hwc&value != 0
}

// Name returns the CPU name given by the vendor.
// If the CPU name can not be determined an
// empty string is returned.
func Name() string {
	if maxExtendedFunctionInformation < 0x80000004 {
		return ""
	}

	data := make([]byte, 0, 3*4*4)

	var eax, ebx, ecx, edx uint32
	eax, ebx, ecx, edx = cpuid(0x80000002, 0)
	data = appendBytes(data, eax, ebx, ecx, edx)
	eax, ebx, ecx, edx = cpuid(0x80000003, 0)
	data = appendBytes(data, eax, ebx, ecx, edx)
	eax, ebx, ecx, edx = cpuid(0x80000004, 0)
	data = appendBytes(data, eax, ebx, ecx, edx)

	// Trim leading spaces.
	for len(data) > 0 && data[0] == ' ' {
		data = data[1:]
	}

	// Trim tail after and including the first null byte.
	for i, c := range data {
		if c == '\x00' {
			data = data[:i]
			break
		}
	}

	return string(data)
}

func appendBytes(b []byte, args ...uint32) []byte {
	for _, arg := range args {
		b = append(b,
			byte((arg >> 0)),
			byte((arg >> 8)),
			byte((arg >> 16)),
			byte((arg >> 24)))
	}
	return b
}
<commit_msg>internal/cpu: disable FMA when OSXSAVE is not enabled on x86<commit_after>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build 386 amd64

package cpu

const CacheLinePadSize = 64

// cpuid is implemented in cpu_x86.s.
func cpuid(eaxArg, ecxArg uint32) (eax, ebx, ecx, edx uint32)

// xgetbv with ecx = 0 is implemented in cpu_x86.s.
func xgetbv() (eax, edx uint32)

const (
	// edx bits
	cpuid_SSE2 = 1 << 26

	// ecx bits
	cpuid_SSE3      = 1 << 0
	cpuid_PCLMULQDQ = 1 << 1
	cpuid_SSSE3     = 1 << 9
	cpuid_FMA       = 1 << 12
	cpuid_SSE41     = 1 << 19
	cpuid_SSE42     = 1 << 20
	cpuid_POPCNT    = 1 << 23
	cpuid_AES       = 1 << 25
	cpuid_OSXSAVE   = 1 << 27
	cpuid_AVX       = 1 << 28

	// ebx bits
	cpuid_BMI1 = 1 << 3
	cpuid_AVX2 = 1 << 5
	cpuid_BMI2 = 1 << 8
	cpuid_ERMS = 1 << 9
	cpuid_ADX  = 1 << 19
)

var maxExtendedFunctionInformation uint32

func doinit() {
	options = []option{
		{Name: "adx", Feature: &X86.HasADX},
		{Name: "aes", Feature: &X86.HasAES},
		{Name: "avx", Feature: &X86.HasAVX},
		{Name: "avx2", Feature: &X86.HasAVX2},
		{Name: "bmi1", Feature: &X86.HasBMI1},
		{Name: "bmi2", Feature: &X86.HasBMI2},
		{Name: "erms", Feature: &X86.HasERMS},
		{Name: "fma", Feature: &X86.HasFMA},
		{Name: "pclmulqdq", Feature: &X86.HasPCLMULQDQ},
		{Name: "popcnt", Feature: &X86.HasPOPCNT},
		{Name: "sse3", Feature: &X86.HasSSE3},
		{Name: "sse41", Feature: &X86.HasSSE41},
		{Name: "sse42", Feature: &X86.HasSSE42},
		{Name: "ssse3", Feature: &X86.HasSSSE3},

		// These capabilities should always be enabled on amd64:
		{Name: "sse2", Feature: &X86.HasSSE2, Required: GOARCH == "amd64"},
	}

	maxID, _, _, _ := cpuid(0, 0)

	if maxID < 1 {
		return
	}

	maxExtendedFunctionInformation, _, _, _ = cpuid(0x80000000, 0)

	_, _, ecx1, edx1 := cpuid(1, 0)
	X86.HasSSE2 = isSet(edx1, cpuid_SSE2)

	X86.HasSSE3 = isSet(ecx1, cpuid_SSE3)
	X86.HasPCLMULQDQ = isSet(ecx1, cpuid_PCLMULQDQ)
	X86.HasSSSE3 = isSet(ecx1, cpuid_SSSE3)
	X86.HasSSE41 = isSet(ecx1, cpuid_SSE41)
	X86.HasSSE42 = isSet(ecx1, cpuid_SSE42)
	X86.HasPOPCNT = isSet(ecx1, cpuid_POPCNT)
	X86.HasAES = isSet(ecx1, cpuid_AES)

	// OSXSAVE can be false when using older Operating Systems
	// or when explicitly disabled on newer Operating Systems by
	// e.g. setting the xsavedisable boot option on Windows 10.
	X86.HasOSXSAVE = isSet(ecx1, cpuid_OSXSAVE)

	// The FMA instruction set extension only has VEX prefixed instructions.
	// VEX prefixed instructions require OSXSAVE to be enabled.
	// See Intel 64 and IA-32 Architecture Software Developer’s Manual Volume 2
	// Section 2.4 "AVX and SSE Instruction Exception Specification"
	X86.HasFMA = isSet(ecx1, cpuid_FMA) && X86.HasOSXSAVE

	osSupportsAVX := false
	// For XGETBV, OSXSAVE bit is required and sufficient.
	if X86.HasOSXSAVE {
		eax, _ := xgetbv()
		// Check if XMM and YMM registers have OS support.
		osSupportsAVX = isSet(eax, 1<<1) && isSet(eax, 1<<2)
	}

	X86.HasAVX = isSet(ecx1, cpuid_AVX) && osSupportsAVX

	if maxID < 7 {
		return
	}

	_, ebx7, _, _ := cpuid(7, 0)
	X86.HasBMI1 = isSet(ebx7, cpuid_BMI1)
	X86.HasAVX2 = isSet(ebx7, cpuid_AVX2) && osSupportsAVX
	X86.HasBMI2 = isSet(ebx7, cpuid_BMI2)
	X86.HasERMS = isSet(ebx7, cpuid_ERMS)
	X86.HasADX = isSet(ebx7, cpuid_ADX)
}

func isSet(hwc uint32, value uint32) bool {
	return hwc&value != 0
}

// Name returns the CPU name given by the vendor.
// If the CPU name can not be determined an
// empty string is returned.
func Name() string {
	if maxExtendedFunctionInformation < 0x80000004 {
		return ""
	}

	data := make([]byte, 0, 3*4*4)

	var eax, ebx, ecx, edx uint32
	eax, ebx, ecx, edx = cpuid(0x80000002, 0)
	data = appendBytes(data, eax, ebx, ecx, edx)
	eax, ebx, ecx, edx = cpuid(0x80000003, 0)
	data = appendBytes(data, eax, ebx, ecx, edx)
	eax, ebx, ecx, edx = cpuid(0x80000004, 0)
	data = appendBytes(data, eax, ebx, ecx, edx)

	// Trim leading spaces.
	for len(data) > 0 && data[0] == ' ' {
		data = data[1:]
	}

	// Trim tail after and including the first null byte.
	for i, c := range data {
		if c == '\x00' {
			data = data[:i]
			break
		}
	}

	return string(data)
}

func appendBytes(b []byte, args ...uint32) []byte {
	for _, arg := range args {
		b = append(b,
			byte((arg >> 0)),
			byte((arg >> 8)),
			byte((arg >> 16)),
			byte((arg >> 24)))
	}
	return b
}
<|endoftext|>


<commit_before>package wallet

import (
	"errors"

	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

var (
	errOutOfBounds = errors.New("requesting transactions at unknown confirmation heights")
)

// AddressTransactions returns all of the wallet transactions associated with a
// single unlock hash.
func (w *Wallet) AddressTransactions(uh types.UnlockHash) (pts []modules.ProcessedTransaction) {
	// ensure durability of reported transactions
	w.mu.Lock()
	defer w.mu.Unlock()
	w.syncDB()

	it := dbProcessedTransactionsIterator(w.dbTx)
	for it.next() {
		pt := it.value()
		relevant := false
		for _, input := range pt.Inputs {
			relevant = relevant || input.RelatedAddress == uh
		}
		for _, output := range pt.Outputs {
			relevant = relevant || output.RelatedAddress == uh
		}
		if relevant {
			pts = append(pts, pt)
		}
	}

	return pts
}

// AddressUnconfirmedHistory returns all of the unconfirmed wallet transactions
// related to a specific address.
func (w *Wallet) AddressUnconfirmedTransactions(uh types.UnlockHash) (pts []modules.ProcessedTransaction) {
	// ensure durability of reported transactions
	w.mu.Lock()
	defer w.mu.Unlock()
	w.syncDB()

	// Scan the full list of unconfirmed transactions to see if there are any
	// related transactions.
	for _, pt := range w.unconfirmedProcessedTransactions {
		relevant := false
		for _, input := range pt.Inputs {
			if input.RelatedAddress == uh {
				relevant = true
				break
			}
		}
		for _, output := range pt.Outputs {
			if output.RelatedAddress == uh {
				relevant = true
				break
			}
		}
		if relevant {
			pts = append(pts, pt)
		}
	}
	return pts
}

// Transaction returns the transaction with the given id. 'False' is returned
// if the transaction does not exist.
func (w *Wallet) Transaction(txid types.TransactionID) (pt modules.ProcessedTransaction, found bool) {
	// ensure durability of reported transaction
	w.mu.Lock()
	defer w.mu.Unlock()
	w.syncDB()

	it := dbProcessedTransactionsIterator(w.dbTx)
	for it.next() {
		pt := it.value()
		if pt.TransactionID == txid {
			return pt, true
		}
	}
	return modules.ProcessedTransaction{}, false
}

// Transactions returns all transactions relevant to the wallet that were
// confirmed in the range [startHeight, endHeight].
func (w *Wallet) Transactions(startHeight, endHeight types.BlockHeight) (pts []modules.ProcessedTransaction, err error) {
	// ensure durability of reported transactions
	w.mu.Lock()
	defer w.mu.Unlock()
	w.syncDB()

	height, err := dbGetConsensusHeight(w.dbTx)
	if err != nil {
		return
	} else if startHeight > height || startHeight > endHeight {
		return nil, errOutOfBounds
	}

	it := dbProcessedTransactionsIterator(w.dbTx)
	for it.next() {
		pt := it.value()
		if pt.ConfirmationHeight < startHeight {
			continue
		} else if pt.ConfirmationHeight > endHeight {
			// transactions are stored in chronological order, so we can
			// break as soon as we are above endHeight
			break
		} else {
			pts = append(pts, pt)
		}
	}
	return
}

// UnconfirmedTransactions returns the set of unconfirmed transactions that are
// relevant to the wallet.
func (w *Wallet) UnconfirmedTransactions() []modules.ProcessedTransaction {
	w.mu.RLock()
	defer w.mu.RUnlock()
	return w.unconfirmedProcessedTransactions
}
<commit_msg>modify Transactions to use binary search<commit_after>package wallet

import (
	"encoding/binary"
	"errors"
	"sort"

	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

var (
	errOutOfBounds = errors.New("requesting transactions at unknown confirmation heights")
)

// AddressTransactions returns all of the wallet transactions associated with a
// single unlock hash.
func (w *Wallet) AddressTransactions(uh types.UnlockHash) (pts []modules.ProcessedTransaction) {
	// ensure durability of reported transactions
	w.mu.Lock()
	defer w.mu.Unlock()
	w.syncDB()

	it := dbProcessedTransactionsIterator(w.dbTx)
	for it.next() {
		pt := it.value()
		relevant := false
		for _, input := range pt.Inputs {
			relevant = relevant || input.RelatedAddress == uh
		}
		for _, output := range pt.Outputs {
			relevant = relevant || output.RelatedAddress == uh
		}
		if relevant {
			pts = append(pts, pt)
		}
	}

	return pts
}

// AddressUnconfirmedHistory returns all of the unconfirmed wallet transactions
// related to a specific address.
func (w *Wallet) AddressUnconfirmedTransactions(uh types.UnlockHash) (pts []modules.ProcessedTransaction) {
	// ensure durability of reported transactions
	w.mu.Lock()
	defer w.mu.Unlock()
	w.syncDB()

	// Scan the full list of unconfirmed transactions to see if there are any
	// related transactions.
	for _, pt := range w.unconfirmedProcessedTransactions {
		relevant := false
		for _, input := range pt.Inputs {
			if input.RelatedAddress == uh {
				relevant = true
				break
			}
		}
		for _, output := range pt.Outputs {
			if output.RelatedAddress == uh {
				relevant = true
				break
			}
		}
		if relevant {
			pts = append(pts, pt)
		}
	}
	return pts
}

// Transaction returns the transaction with the given id. 'False' is returned
// if the transaction does not exist.
func (w *Wallet) Transaction(txid types.TransactionID) (pt modules.ProcessedTransaction, found bool) {
	// ensure durability of reported transaction
	w.mu.Lock()
	defer w.mu.Unlock()
	w.syncDB()

	it := dbProcessedTransactionsIterator(w.dbTx)
	for it.next() {
		pt := it.value()
		if pt.TransactionID == txid {
			return pt, true
		}
	}
	return modules.ProcessedTransaction{}, false
}

// decodeProcessedTransaction decodes a marshalled processedTransaction
func decodeProcessedTransaction(ptBytes []byte, pt *modules.ProcessedTransaction) bool {
	err := encoding.Unmarshal(ptBytes, pt)
	if err != nil {
		// COMPATv1.2.1: try decoding into old transaction type
		var oldpt v121ProcessedTransaction
		err = encoding.Unmarshal(ptBytes, &oldpt)
		*pt = convertProcessedTransaction(oldpt)
	}
	return err == nil
}

// Transactions returns all transactions relevant to the wallet that were
// confirmed in the range [startHeight, endHeight].
func (w *Wallet) Transactions(startHeight, endHeight types.BlockHeight) (pts []modules.ProcessedTransaction, err error) {
	// ensure durability of reported transactions
	w.mu.Lock()
	defer w.mu.Unlock()
	w.syncDB()

	height, err := dbGetConsensusHeight(w.dbTx)
	if err != nil {
		return
	} else if startHeight > height || startHeight > endHeight {
		return nil, errOutOfBounds
	}

	// Conduct the binary search
	var pt modules.ProcessedTransaction
	cursor := w.dbTx.Bucket(bucketProcessedTransactions).Cursor()

	// Get the largest key present in the processed transactions bucket
	lastKeyBytes, _ := cursor.Last()
	if lastKeyBytes == nil {
		// bucket is empty
		return
	}

	// Decode it to the the largest key
	lastKey := binary.BigEndian.Uint64(lastKeyBytes)

	// Conduct the binary search
	sort.Search(int(lastKey), func(i int) bool {
		// Create the key for the index
		keyBytes := make([]byte, 8)
		binary.BigEndian.PutUint64(keyBytes, uint64(i))

		// Retrieve the processed transaction
		key, ptBytes := cursor.Seek(keyBytes)
		if key == nil {
			panic("Failed to retrieve processed Transaction by key")
		}

		// Decode the transaction
		if success := decodeProcessedTransaction(ptBytes, &pt); !success {
			panic("Failed to decode the processed transaction")
		}

		return pt.ConfirmationHeight >= startHeight
	})

	// Gather all transactions until endHeight is reached
	for pt.ConfirmationHeight <= endHeight {
		if !(pt.ConfirmationHeight < startHeight) {
			pts = append(pts, pt)
		}

		// Get next processed transaction
		key, ptBytes := cursor.Next()
		if key == nil {
			break
		}

		// Decode the transaction
		if success := decodeProcessedTransaction(ptBytes, &pt); !success {
			panic("Failed to decode the processed transaction")
		}
	}
	return
}

// UnconfirmedTransactions returns the set of unconfirmed transactions that are
// relevant to the wallet.
func (w *Wallet) UnconfirmedTransactions() []modules.ProcessedTransaction {
	w.mu.RLock()
	defer w.mu.RUnlock()
	return w.unconfirmedProcessedTransactions
}
<|endoftext|>


<commit_before>package ops_test

import (
	"testing"

	"github.com/concourse/concourse/integration/internal/flytest"
	"github.com/stretchr/testify/require"
)

func setupUpgradeDowngrade(t *testing.T, fly flytest.Cmd) {
	t.Run("set up test pipeline", func(t *testing.T) {
		err := fly.Run("set-pipeline", "-p", "test", "-c", "pipelines/smoke-pipeline.yml", "-n")
		require.NoError(t, err)

		err = fly.Run("unpause-pipeline", "-p", "test")
		require.NoError(t, err)

		err = fly.Run("trigger-job", "-j", "test/say-hello", "-w")
		require.NoError(t, err)
	})
}

func verifyUpgradeDowngrade(t *testing.T, fly flytest.Cmd) {
	t.Run("pipeline and build still exists", func(t *testing.T) {
		err := fly.Run("get-pipeline", "-p", "test")
		require.NoError(t, err)

		out, err := fly.Output("watch", "-j", "test/say-hello")
		require.NoError(t, err)
		require.Contains(t, out, "Hello, world!")
	})

	t.Run("can still run pipeline builds", func(t *testing.T) {
		err := fly.Run("trigger-job", "-j", "test/say-hello", "-w")
		require.NoError(t, err)
	})

	t.Run("can still run checks", func(t *testing.T) {
		err := fly.Run("check-resource", "-r", "test/mockery")
		require.NoError(t, err)
	})

	t.Run("can still reach the internet", func(t *testing.T) {
		out, err := fly.Output("trigger-job", "-j", "test/use-the-internet", "-w")
		require.NoError(t, err)
		require.Contains(t, out, "Example Domain")
	})

	t.Run("can still run one-off builds", func(t *testing.T) {
		out, err := fly.Output("execute", "-c", "tasks/hello.yml")
		require.NoError(t, err)
		require.Contains(t, out, "hello")
	})
}
<commit_msg>integration: make fly downgrade ignore new events<commit_after>package ops_test

import (
	"testing"

	"github.com/concourse/concourse/integration/internal/flytest"
	"github.com/stretchr/testify/require"
)

func setupUpgradeDowngrade(t *testing.T, fly flytest.Cmd) {
	t.Run("set up test pipeline", func(t *testing.T) {
		err := fly.Run("set-pipeline", "-p", "test", "-c", "pipelines/smoke-pipeline.yml", "-n")
		require.NoError(t, err)

		err = fly.Run("unpause-pipeline", "-p", "test")
		require.NoError(t, err)

		err = fly.Run("trigger-job", "-j", "test/say-hello", "-w")
		require.NoError(t, err)
	})
}

func verifyUpgradeDowngrade(t *testing.T, fly flytest.Cmd) {
	t.Run("pipeline and build still exists", func(t *testing.T) {
		err := fly.Run("get-pipeline", "-p", "test")
		require.NoError(t, err)

		out, err := fly.Output("watch", "-j", "test/say-hello", "--ignore-event-parsing-errors")
		require.NoError(t, err)
		require.Contains(t, out, "Hello, world!")
	})

	t.Run("can still run pipeline builds", func(t *testing.T) {
		err := fly.Run("trigger-job", "-j", "test/say-hello", "-w")
		require.NoError(t, err)
	})

	t.Run("can still run checks", func(t *testing.T) {
		err := fly.Run("check-resource", "-r", "test/mockery")
		require.NoError(t, err)
	})

	t.Run("can still reach the internet", func(t *testing.T) {
		out, err := fly.Output("trigger-job", "-j", "test/use-the-internet", "-w")
		require.NoError(t, err)
		require.Contains(t, out, "Example Domain")
	})

	t.Run("can still run one-off builds", func(t *testing.T) {
		out, err := fly.Output("execute", "-c", "tasks/hello.yml")
		require.NoError(t, err)
		require.Contains(t, out, "hello")
	})
}
<|endoftext|>


<commit_before>package proto_test

import (
	"bytes"
	"strings"
	"testing"

	"github.com/go-redis/redis/internal/proto"
)

func newReader(s string) *proto.Reader {
	return proto.NewReader(strings.NewReader(s))
}

func BenchmarkReader_ParseReply_Status(b *testing.B) {
	benchmarkParseReply(b, "+OK\r\n", nil, false)
}

func BenchmarkReader_ParseReply_Int(b *testing.B) {
	benchmarkParseReply(b, ":1\r\n", nil, false)
}

func BenchmarkReader_ParseReply_Error(b *testing.B) {
	benchmarkParseReply(b, "-Error message\r\n", nil, true)
}

func BenchmarkReader_ParseReply_String(b *testing.B) {
	benchmarkParseReply(b, "$5\r\nhello\r\n", nil, false)
}

func BenchmarkReader_ParseReply_Slice(b *testing.B) {
	benchmarkParseReply(b, "*2\r\n$5\r\nhello\r\n$5\r\nworld\r\n", multiBulkParse, false)
}

func benchmarkParseReply(b *testing.B, reply string, m proto.MultiBulkParse, wanterr bool) {
	buf := new(bytes.Buffer)
	for i := 0; i < b.N; i++ {
		buf.WriteString(reply)
	}
	p := proto.NewReader(buf)
	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		_, err := p.ReadReply(m)
		if !wanterr && err != nil {
			b.Fatal(err)
		}
	}
}

func multiBulkParse(p *proto.Reader, n int64) (interface{}, error) {
	vv := make([]interface{}, 0, n)
	for i := int64(0); i < n; i++ {
		v, err := p.ReadReply(multiBulkParse)
		if err != nil {
			return nil, err
		}
		vv = append(vv, v)
	}
	return vv, nil
}
<commit_msg>remove unused function<commit_after>package proto_test

import (
	"bytes"
	"testing"

	"github.com/go-redis/redis/internal/proto"
)

func BenchmarkReader_ParseReply_Status(b *testing.B) {
	benchmarkParseReply(b, "+OK\r\n", nil, false)
}

func BenchmarkReader_ParseReply_Int(b *testing.B) {
	benchmarkParseReply(b, ":1\r\n", nil, false)
}

func BenchmarkReader_ParseReply_Error(b *testing.B) {
	benchmarkParseReply(b, "-Error message\r\n", nil, true)
}

func BenchmarkReader_ParseReply_String(b *testing.B) {
	benchmarkParseReply(b, "$5\r\nhello\r\n", nil, false)
}

func BenchmarkReader_ParseReply_Slice(b *testing.B) {
	benchmarkParseReply(b, "*2\r\n$5\r\nhello\r\n$5\r\nworld\r\n", multiBulkParse, false)
}

func benchmarkParseReply(b *testing.B, reply string, m proto.MultiBulkParse, wanterr bool) {
	buf := new(bytes.Buffer)
	for i := 0; i < b.N; i++ {
		buf.WriteString(reply)
	}
	p := proto.NewReader(buf)
	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		_, err := p.ReadReply(m)
		if !wanterr && err != nil {
			b.Fatal(err)
		}
	}
}

func multiBulkParse(p *proto.Reader, n int64) (interface{}, error) {
	vv := make([]interface{}, 0, n)
	for i := int64(0); i < n; i++ {
		v, err := p.ReadReply(multiBulkParse)
		if err != nil {
			return nil, err
		}
		vv = append(vv, v)
	}
	return vv, nil
}
<|endoftext|>