<commit_before>package write

import (
	"bufio"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"strings"
	"testing"
	"time"

	"github.com/google/go-cmp/cmp"
	platform "github.com/influxdata/influxdb"
	"github.com/influxdata/influxdb/mock"
)

func TestScanLines(t *testing.T) {
	tests := []struct {
		name    string
		input   string
		want    []string
		wantErr bool
	}{
		{
			name:  "3 lines produced including their newlines",
			input: "m1,t1=v1 f1=1\nm2,t2=v2 f2=2\nm3,t3=v3 f3=3",
			want:  []string{"m1,t1=v1 f1=1\n", "m2,t2=v2 f2=2\n", "m3,t3=v3 f3=3"},
		},
		{
			name:  "single line without newline",
			input: "m1,t1=v1 f1=1",
			want:  []string{"m1,t1=v1 f1=1"},
		},
		{
			name:  "single line with newline",
			input: "m1,t1=v1 f1=1\n",
			want:  []string{"m1,t1=v1 f1=1\n"},
		},
		{
			name:  "no lines",
			input: "",
			want:  []string{},
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			scanner := bufio.NewScanner(strings.NewReader(tt.input))
			scanner.Split(ScanLines)
			got := []string{}
			for scanner.Scan() {
				got = append(got, scanner.Text())
			}
			err := scanner.Err()

			if (err != nil) != tt.wantErr {
				t.Errorf("ScanLines() error = %v, wantErr %v", err, tt.wantErr)
				return
			}

			if !cmp.Equal(got, tt.want) {
				t.Errorf("%q. ScanLines() = -got/+want %s", tt.name, cmp.Diff(got, tt.want))
			}
		})
	}
}

// errorReader mocks io.Reader but returns an error.
type errorReader struct{}

func (r *errorReader) Read(p []byte) (n int, err error) {
	return 0, fmt.Errorf("error")
}

func TestBatcher_read(t *testing.T) {
	type args struct {
		cancel bool
		r      io.Reader
		lines  chan []byte
		errC   chan error
	}
	tests := []struct {
		name    string
		args    args
		want    []string
		wantErr bool
	}{
		{
			name: "reading two lines produces 2 lines",
			args: args{
				r:     strings.NewReader("m1,t1=v1 f1=1\nm2,t2=v2 f2=2"),
				lines: make(chan []byte),
				errC:  make(chan error, 1),
			},
			want: []string{"m1,t1=v1 f1=1\n", "m2,t2=v2 f2=2"},
		},
		{
			name: "canceling returns no lines",
			args: args{
				cancel: true,
				r:      strings.NewReader("m1,t1=v1 f1=1"),
				lines:  make(chan []byte),
				errC:   make(chan error, 1),
			},
			want:    []string{},
			wantErr: true,
		},
		{
			name: "error from reader returns error",
			args: args{
				r:     &errorReader{},
				lines: make(chan []byte),
				errC:  make(chan error, 1),
			},
			want:    []string{},
			wantErr: true,
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			ctx := context.Background()
			var cancel context.CancelFunc
			if tt.args.cancel {
				ctx, cancel = context.WithCancel(ctx)
			}

			b := &Batcher{}
			got := []string{}

			go b.read(ctx, tt.args.r, tt.args.lines, tt.args.errC)
			if cancel != nil {
				cancel()
			} else {
				for line := range tt.args.lines {
					got = append(got, string(line))
				}
			}

			err := <-tt.args.errC
			if (err != nil) != tt.wantErr {
				t.Errorf("ScanLines.read() error = %v, wantErr %v", err, tt.wantErr)
				return
			}

			if !cmp.Equal(got, tt.want) {
				t.Errorf("%q. Batcher.read() = -got/+want %s", tt.name, cmp.Diff(got, tt.want))
			}
		})
	}
}

func TestBatcher_write(t *testing.T) {
	type fields struct {
		MaxFlushBytes    int
		MaxFlushInterval time.Duration
	}
	type args struct {
		cancel     bool
		writeError bool
		org        platform.ID
		bucket     platform.ID
		line       string
		lines      chan []byte
		errC       chan error
	}
	tests := []struct {
		name    string
		fields  fields
		args    args
		want    string
		wantErr bool
	}{
		{
			name: "sending a single line will send a line to the service",
			fields: fields{
				MaxFlushBytes: 1,
			},
			args: args{
				org:    platform.ID(1),
				bucket: platform.ID(2),
				line:   "m1,t1=v1 f1=1",
				lines:  make(chan []byte),
				errC:   make(chan error),
			},
			want: "m1,t1=v1 f1=1",
		},
		{
			name: "sending a single line and waiting for a timeout will send a line to the service",
			fields: fields{
				MaxFlushInterval: time.Millisecond,
			},
			args: args{
				org:    platform.ID(1),
				bucket: platform.ID(2),
				line:   "m1,t1=v1 f1=1",
				lines:  make(chan []byte),
				errC:   make(chan error),
			},
			want: "m1,t1=v1 f1=1",
		},
		{
			name: "write service returning error stops the write after timeout",
			fields: fields{
				MaxFlushInterval: time.Millisecond,
			},
			args: args{
				writeError: true,
				org:        platform.ID(1),
				bucket:     platform.ID(2),
				line:       "m1,t1=v1 f1=1",
				lines:      make(chan []byte),
				errC:       make(chan error),
			},
			wantErr: true,
		},
		{
			name: "canceling will write no data to service",
			fields: fields{
				MaxFlushBytes: 1,
			},
			args: args{
				cancel: true,
				org:    platform.ID(1),
				bucket: platform.ID(2),
				line:   "m1,t1=v1 f1=1",
				lines:  make(chan []byte),
				errC:   make(chan error),
			},
			wantErr: true,
		},
		{
			name: "write service returning error stops the write",
			fields: fields{
				MaxFlushBytes: 1,
			},
			args: args{
				writeError: true,
				org:        platform.ID(1),
				bucket:     platform.ID(2),
				line:       "m1,t1=v1 f1=1",
				lines:      make(chan []byte),
				errC:       make(chan error),
			},
			wantErr: true,
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			ctx := context.Background()
			var cancel context.CancelFunc
			if tt.args.cancel {
				ctx, cancel = context.WithCancel(ctx)
			}

			// mocking the write service here to either return an error
			// or get back all the bytes from the reader.
			var got string
			svc := &mock.WriteService{
				WriteF: func(ctx context.Context, org, bucket platform.ID, r io.Reader) error {
					if tt.args.writeError {
						return fmt.Errorf("error")
					}
					b, err := ioutil.ReadAll(r)
					got = string(b)
					return err
				},
			}

			b := &Batcher{
				MaxFlushBytes:    tt.fields.MaxFlushBytes,
				MaxFlushInterval: tt.fields.MaxFlushInterval,
				Service:          svc,
			}

			go b.write(ctx, tt.args.org, tt.args.bucket, tt.args.lines, tt.args.errC)

			if cancel != nil {
				cancel()
			} else {
				tt.args.lines <- []byte(tt.args.line)
				// if the max flush interval is not zero,  we are testing to see
				// if the data is flushed via the timer rather than forced by
				// closing the channel.
				if tt.fields.MaxFlushInterval != 0 {
					time.Sleep(tt.fields.MaxFlushInterval * 100)
				}
				close(tt.args.lines)
			}

			err := <-tt.args.errC
			if (err != nil) != tt.wantErr {
				t.Errorf("ScanLines.read() error = %v, wantErr %v", err, tt.wantErr)
				return
			}

			if !cmp.Equal(got, tt.want) {
				t.Errorf("%q. Batcher.write() = -got/+want %s", tt.name, cmp.Diff(got, tt.want))
			}
		})
	}
}

func TestBatcher_Write(t *testing.T) {
	type fields struct {
		MaxFlushBytes    int
		MaxFlushInterval time.Duration
	}
	type args struct {
		writeError bool
		org        platform.ID
		bucket     platform.ID
		r          io.Reader
	}
	tests := []struct {
		name        string
		fields      fields
		args        args
		want        string
		wantFlushes int
		wantErr     bool
	}{
		{
			name: "a line of line protocol is sent to the service",
			fields: fields{
				MaxFlushBytes: 1,
			},
			args: args{
				org:    platform.ID(1),
				bucket: platform.ID(2),
				r:      strings.NewReader("m1,t1=v1 f1=1"),
			},
			want:        "m1,t1=v1 f1=1",
			wantFlushes: 1,
		},
		{
			name: "multiple lines cause multiple flushes",
			fields: fields{
				MaxFlushBytes: len([]byte("m1,t1=v1 f1=1\n")),
			},
			args: args{
				org:    platform.ID(1),
				bucket: platform.ID(2),
				r:      strings.NewReader("m1,t1=v1 f1=1\nm2,t2=v2 f2=2\nm3,t3=v3 f3=3"),
			},
			want:        "m3,t3=v3 f3=3",
			wantFlushes: 3,
		},
		{
			name:   "errors during read return error",
			fields: fields{},
			args: args{
				org:    platform.ID(1),
				bucket: platform.ID(2),
				r:      &errorReader{},
			},
			wantErr: true,
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			// mocking the write service here to either return an error
			// or get back all the bytes from the reader.
			var (
				got        string
				gotFlushes int
			)
			svc := &mock.WriteService{
				WriteF: func(ctx context.Context, org, bucket platform.ID, r io.Reader) error {
					if tt.args.writeError {
						return fmt.Errorf("error")
					}
					b, err := ioutil.ReadAll(r)
					got = string(b)
					gotFlushes++
					return err
				},
			}

			b := &Batcher{
				MaxFlushBytes:    tt.fields.MaxFlushBytes,
				MaxFlushInterval: tt.fields.MaxFlushInterval,
				Service:          svc,
			}

			ctx := context.Background()
			if err := b.Write(ctx, tt.args.org, tt.args.bucket, tt.args.r); (err != nil) != tt.wantErr {
				t.Errorf("Batcher.Write() error = %v, wantErr %v", err, tt.wantErr)
			}

			if gotFlushes != tt.wantFlushes {
				t.Errorf("%q. Batcher.Write() flushes %d want %d", tt.name, gotFlushes, tt.wantFlushes)
			}
			if !cmp.Equal(got, tt.want) {
				t.Errorf("%q. Batcher.Write() = -got/+want %s", tt.name, cmp.Diff(got, tt.want))
			}
		})
	}
}

func TestBatcher_WriteTimeout(t *testing.T) {
	// mocking the write service here to either return an error
	// or get back all the bytes from the reader.
	var got string
	svc := &mock.WriteService{
		WriteF: func(ctx context.Context, org, bucket platform.ID, r io.Reader) error {
			b, err := ioutil.ReadAll(r)
			got = string(b)
			return err
		},
	}

	b := &Batcher{
		Service: svc,
	}

	// this mimics a reader like stdin that may never return data.
	r, _ := io.Pipe()

	ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond)
	defer cancel()

	if err := b.Write(ctx, platform.ID(1), platform.ID(2), r); err != context.DeadlineExceeded {
		t.Errorf("Batcher.Write() with timeout error = %v", err)
	}

	if got != "" {
		t.Errorf(" Batcher.Write() with timeout got %s", got)
	}
}
<commit_msg>chore(cmd/influx/write): add test for line batching<commit_after>package write

import (
	"bufio"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"strings"
	"testing"
	"time"

	"github.com/google/go-cmp/cmp"
	platform "github.com/influxdata/influxdb"
	"github.com/influxdata/influxdb/mock"
)

func TestScanLines(t *testing.T) {
	tests := []struct {
		name    string
		input   string
		want    []string
		wantErr bool
	}{
		{
			name:  "3 lines produced including their newlines",
			input: "m1,t1=v1 f1=1\nm2,t2=v2 f2=2\nm3,t3=v3 f3=3",
			want:  []string{"m1,t1=v1 f1=1\n", "m2,t2=v2 f2=2\n", "m3,t3=v3 f3=3"},
		},
		{
			name:  "single line without newline",
			input: "m1,t1=v1 f1=1",
			want:  []string{"m1,t1=v1 f1=1"},
		},
		{
			name:  "single line with newline",
			input: "m1,t1=v1 f1=1\n",
			want:  []string{"m1,t1=v1 f1=1\n"},
		},
		{
			name:  "no lines",
			input: "",
			want:  []string{},
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			scanner := bufio.NewScanner(strings.NewReader(tt.input))
			scanner.Split(ScanLines)
			got := []string{}
			for scanner.Scan() {
				got = append(got, scanner.Text())
			}
			err := scanner.Err()

			if (err != nil) != tt.wantErr {
				t.Errorf("ScanLines() error = %v, wantErr %v", err, tt.wantErr)
				return
			}

			if !cmp.Equal(got, tt.want) {
				t.Errorf("%q. ScanLines() = -got/+want %s", tt.name, cmp.Diff(got, tt.want))
			}
		})
	}
}

// errorReader mocks io.Reader but returns an error.
type errorReader struct{}

func (r *errorReader) Read(p []byte) (n int, err error) {
	return 0, fmt.Errorf("error")
}

func TestBatcher_read(t *testing.T) {
	type args struct {
		cancel bool
		r      io.Reader
		lines  chan []byte
		errC   chan error
	}
	tests := []struct {
		name    string
		args    args
		want    []string
		wantErr bool
	}{
		{
			name: "reading two lines produces 2 lines",
			args: args{
				r:     strings.NewReader("m1,t1=v1 f1=1\nm2,t2=v2 f2=2"),
				lines: make(chan []byte),
				errC:  make(chan error, 1),
			},
			want: []string{"m1,t1=v1 f1=1\n", "m2,t2=v2 f2=2"},
		},
		{
			name: "canceling returns no lines",
			args: args{
				cancel: true,
				r:      strings.NewReader("m1,t1=v1 f1=1"),
				lines:  make(chan []byte),
				errC:   make(chan error, 1),
			},
			want:    []string{},
			wantErr: true,
		},
		{
			name: "error from reader returns error",
			args: args{
				r:     &errorReader{},
				lines: make(chan []byte),
				errC:  make(chan error, 1),
			},
			want:    []string{},
			wantErr: true,
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			ctx := context.Background()
			var cancel context.CancelFunc
			if tt.args.cancel {
				ctx, cancel = context.WithCancel(ctx)
			}

			b := &Batcher{}
			got := []string{}

			go b.read(ctx, tt.args.r, tt.args.lines, tt.args.errC)
			if cancel != nil {
				cancel()
			} else {
				for line := range tt.args.lines {
					got = append(got, string(line))
				}
			}

			err := <-tt.args.errC
			if (err != nil) != tt.wantErr {
				t.Errorf("ScanLines.read() error = %v, wantErr %v", err, tt.wantErr)
				return
			}

			if !cmp.Equal(got, tt.want) {
				t.Errorf("%q. Batcher.read() = -got/+want %s", tt.name, cmp.Diff(got, tt.want))
			}
		})
	}
}

func TestBatcher_write(t *testing.T) {
	type fields struct {
		MaxFlushBytes    int
		MaxFlushInterval time.Duration
	}
	type args struct {
		cancel     bool
		writeError bool
		org        platform.ID
		bucket     platform.ID
		line       string
		lines      chan []byte
		errC       chan error
	}
	tests := []struct {
		name    string
		fields  fields
		args    args
		want    string
		wantErr bool
	}{
		{
			name: "sending a single line will send a line to the service",
			fields: fields{
				MaxFlushBytes: 1,
			},
			args: args{
				org:    platform.ID(1),
				bucket: platform.ID(2),
				line:   "m1,t1=v1 f1=1",
				lines:  make(chan []byte),
				errC:   make(chan error),
			},
			want: "m1,t1=v1 f1=1",
		},
		{
			name: "sending a single line and waiting for a timeout will send a line to the service",
			fields: fields{
				MaxFlushInterval: time.Millisecond,
			},
			args: args{
				org:    platform.ID(1),
				bucket: platform.ID(2),
				line:   "m1,t1=v1 f1=1",
				lines:  make(chan []byte),
				errC:   make(chan error),
			},
			want: "m1,t1=v1 f1=1",
		},
		{
			name: "write service returning error stops the write after timeout",
			fields: fields{
				MaxFlushInterval: time.Millisecond,
			},
			args: args{
				writeError: true,
				org:        platform.ID(1),
				bucket:     platform.ID(2),
				line:       "m1,t1=v1 f1=1",
				lines:      make(chan []byte),
				errC:       make(chan error),
			},
			wantErr: true,
		},
		{
			name: "canceling will write no data to service",
			fields: fields{
				MaxFlushBytes: 1,
			},
			args: args{
				cancel: true,
				org:    platform.ID(1),
				bucket: platform.ID(2),
				line:   "m1,t1=v1 f1=1",
				lines:  make(chan []byte),
				errC:   make(chan error),
			},
			wantErr: true,
		},
		{
			name: "write service returning error stops the write",
			fields: fields{
				MaxFlushBytes: 1,
			},
			args: args{
				writeError: true,
				org:        platform.ID(1),
				bucket:     platform.ID(2),
				line:       "m1,t1=v1 f1=1",
				lines:      make(chan []byte),
				errC:       make(chan error),
			},
			wantErr: true,
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			ctx := context.Background()
			var cancel context.CancelFunc
			if tt.args.cancel {
				ctx, cancel = context.WithCancel(ctx)
			}

			// mocking the write service here to either return an error
			// or get back all the bytes from the reader.
			var got string
			svc := &mock.WriteService{
				WriteF: func(ctx context.Context, org, bucket platform.ID, r io.Reader) error {
					if tt.args.writeError {
						return fmt.Errorf("error")
					}
					b, err := ioutil.ReadAll(r)
					got = string(b)
					return err
				},
			}

			b := &Batcher{
				MaxFlushBytes:    tt.fields.MaxFlushBytes,
				MaxFlushInterval: tt.fields.MaxFlushInterval,
				Service:          svc,
			}

			go b.write(ctx, tt.args.org, tt.args.bucket, tt.args.lines, tt.args.errC)

			if cancel != nil {
				cancel()
			} else {
				tt.args.lines <- []byte(tt.args.line)
				// if the max flush interval is not zero,  we are testing to see
				// if the data is flushed via the timer rather than forced by
				// closing the channel.
				if tt.fields.MaxFlushInterval != 0 {
					time.Sleep(tt.fields.MaxFlushInterval * 100)
				}
				close(tt.args.lines)
			}

			err := <-tt.args.errC
			if (err != nil) != tt.wantErr {
				t.Errorf("ScanLines.read() error = %v, wantErr %v", err, tt.wantErr)
				return
			}

			if !cmp.Equal(got, tt.want) {
				t.Errorf("%q. Batcher.write() = -got/+want %s", tt.name, cmp.Diff(got, tt.want))
			}
		})
	}
}

func TestBatcher_Write(t *testing.T) {
	type fields struct {
		MaxFlushBytes    int
		MaxFlushInterval time.Duration
	}
	type args struct {
		writeError bool
		org        platform.ID
		bucket     platform.ID
		r          io.Reader
	}
	tests := []struct {
		name        string
		fields      fields
		args        args
		want        string
		wantFlushes int
		wantErr     bool
	}{
		{
			name: "a line of line protocol is sent to the service",
			fields: fields{
				MaxFlushBytes: 1,
			},
			args: args{
				org:    platform.ID(1),
				bucket: platform.ID(2),
				r:      strings.NewReader("m1,t1=v1 f1=1"),
			},
			want:        "m1,t1=v1 f1=1",
			wantFlushes: 1,
		},
		{
			name: "multiple lines cause multiple flushes",
			fields: fields{
				MaxFlushBytes: len([]byte("m1,t1=v1 f1=1\n")),
			},
			args: args{
				org:    platform.ID(1),
				bucket: platform.ID(2),
				r:      strings.NewReader("m1,t1=v1 f1=1\nm2,t2=v2 f2=2\nm3,t3=v3 f3=3"),
			},
			want:        "m3,t3=v3 f3=3",
			wantFlushes: 3,
		},
		{
			name:   "errors during read return error",
			fields: fields{},
			args: args{
				org:    platform.ID(1),
				bucket: platform.ID(2),
				r:      &errorReader{},
			},
			wantErr: true,
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			// mocking the write service here to either return an error
			// or get back all the bytes from the reader.
			var (
				got        string
				gotFlushes int
			)
			svc := &mock.WriteService{
				WriteF: func(ctx context.Context, org, bucket platform.ID, r io.Reader) error {
					if tt.args.writeError {
						return fmt.Errorf("error")
					}
					b, err := ioutil.ReadAll(r)
					got = string(b)
					gotFlushes++
					return err
				},
			}

			b := &Batcher{
				MaxFlushBytes:    tt.fields.MaxFlushBytes,
				MaxFlushInterval: tt.fields.MaxFlushInterval,
				Service:          svc,
			}

			ctx := context.Background()
			if err := b.Write(ctx, tt.args.org, tt.args.bucket, tt.args.r); (err != nil) != tt.wantErr {
				t.Errorf("Batcher.Write() error = %v, wantErr %v", err, tt.wantErr)
			}

			if gotFlushes != tt.wantFlushes {
				t.Errorf("%q. Batcher.Write() flushes %d want %d", tt.name, gotFlushes, tt.wantFlushes)
			}
			if !cmp.Equal(got, tt.want) {
				t.Errorf("%q. Batcher.Write() = -got/+want %s", tt.name, cmp.Diff(got, tt.want))
			}
		})
	}
}

func TestBatcher_WriteTimeout(t *testing.T) {
	// mocking the write service here to either return an error
	// or get back all the bytes from the reader.
	var got string
	svc := &mock.WriteService{
		WriteF: func(ctx context.Context, org, bucket platform.ID, r io.Reader) error {
			b, err := ioutil.ReadAll(r)
			got = string(b)
			return err
		},
	}

	b := &Batcher{
		Service: svc,
	}

	// this mimics a reader like stdin that may never return data.
	r, _ := io.Pipe()

	ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond)
	defer cancel()

	if err := b.Write(ctx, platform.ID(1), platform.ID(2), r); err != context.DeadlineExceeded {
		t.Errorf("Batcher.Write() with timeout error = %v", err)
	}

	if got != "" {
		t.Errorf(" Batcher.Write() with timeout got %s", got)
	}
}

func TestBatcher_WriteWithoutService(t *testing.T) {
	b := Batcher{}
	err := b.Write(context.Background(), platform.ID(1), platform.ID(1), strings.NewReader("m1,t1=v1 f1=1"))
	if err == nil || !strings.Contains(err.Error(), "write service required") {
		t.Errorf(" Batcher.Write() error expected, but got %v", err)
	}
}
<|endoftext|>


<commit_before>package instancemetadata

import (
	"errors"
	"fmt"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/ec2metadata"
	"github.com/aws/aws-sdk-go/aws/session"
)

type MetadataFetcher interface {
	Available() bool
	GetMetadata(string) (string, error)
}

func New(debug bool) MetadataFetcher {
	sess := session.New()
	if debug {
		sess.Config.LogLevel = aws.LogLevel(aws.LogDebug)
	}
	return ec2metadata.New(sess)
}

type InstanceMetadata struct {
	Subnet           string
	Instance         string
	AvailabilityZone string
	Region           string
}

func FetchMetadata(mdf MetadataFetcher) (InstanceMetadata, error) {
	m := InstanceMetadata{}
	if !mdf.Available() {
		return m, errors.New("No metadata service")
	}
	az, err := mdf.GetMetadata("placement/availability-zone")
	if err != nil {
		return m, errors.New(fmt.Sprintf("Error getting AZ: %s", err.Error()))
	}
	m.AvailabilityZone = az
	m.Region = az[:len(az)-1]

	instanceId, err := mdf.GetMetadata("instance-id")
	if err != nil {
		return m, errors.New(fmt.Sprintf("Error getting instance-id: %s", err.Error()))
	}
	m.Instance = instanceId

	subnet, err := getSubnetId(mdf)
	if err != nil {
		return m, errors.New(fmt.Sprintf("Error getting metadata: %s", err.Error()))
	}
	m.Subnet = subnet

	return m, nil
}

func getSubnetId(mdf MetadataFetcher) (string, error) {
	mac, err := mdf.GetMetadata("mac")
	if err != nil {
		return "", err
	}
	return mdf.GetMetadata(fmt.Sprintf("network/interfaces/macs/%s/subnet-id", mac))
}
<commit_msg>Log useful info at boot<commit_after>package instancemetadata

import (
	"errors"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/ec2metadata"
	"github.com/aws/aws-sdk-go/aws/session"
)

type MetadataFetcher interface {
	Available() bool
	GetMetadata(string) (string, error)
}

func New(debug bool) MetadataFetcher {
	sess := session.New()
	if debug {
		sess.Config.LogLevel = aws.LogLevel(aws.LogDebug)
	}
	return ec2metadata.New(sess)
}

type InstanceMetadata struct {
	Subnet           string
	Instance         string
	AvailabilityZone string
	Region           string
}

func FetchMetadata(mdf MetadataFetcher) (InstanceMetadata, error) {
	m := InstanceMetadata{}
	if !mdf.Available() {
		return m, errors.New("No metadata service")
	}
	az, err := mdf.GetMetadata("placement/availability-zone")
	if err != nil {
		return m, errors.New(fmt.Sprintf("Error getting AZ: %s", err.Error()))
	}
	m.AvailabilityZone = az
	m.Region = az[:len(az)-1]

	instanceId, err := mdf.GetMetadata("instance-id")
	if err != nil {
		return m, errors.New(fmt.Sprintf("Error getting instance-id: %s", err.Error()))
	}
	m.Instance = instanceId

	subnet, err := getSubnetId(mdf)
	if err != nil {
		return m, errors.New(fmt.Sprintf("Error getting metadata: %s", err.Error()))
	}
	m.Subnet = subnet

	log.WithFields(log.Fields{
		"subnet_id":         subnet,
		"availability_zone": az,
		"instance_id":       instanceId,
		"region":            m.Region,
	}).Info("Got instance metadata")

	return m, nil
}

func getSubnetId(mdf MetadataFetcher) (string, error) {
	mac, err := mdf.GetMetadata("mac")
	if err != nil {
		return "", err
	}
	return mdf.GetMetadata(fmt.Sprintf("network/interfaces/macs/%s/subnet-id", mac))
}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fake

import (
	"context"
	"fmt"

	"github.com/google/trillian"
)

// LogClient implements trillian/client.VerifyingLogClient.
type LogClient struct {
	leaves []*trillian.LogLeaf
}

// NewFakeTrillianClient returns a client that mimicks a trillian log.
func NewFakeTrillianClient() *LogClient {
	return &LogClient{
		leaves: make([]*trillian.LogLeaf, 0),
	}
}

// AddLeaf adds a leaf to the log.
func (f *LogClient) AddLeaf(ctx context.Context, data []byte) error {
	f.leaves = append(f.leaves, &trillian.LogLeaf{
		LeafValue: data,
	})
	return nil
}

// GetByIndex returns the requested leaf.
func (f *LogClient) GetByIndex(ctx context.Context, index int64) (*trillian.LogLeaf, error) {
	if got, want := index, int64(len(f.leaves)); got > want {
		return nil, fmt.Errorf("Index out of range. Got %v, want <= %v", got, want)
	}
	if got, want := index, int64(0); got < want {
		return nil, fmt.Errorf("Index out of range. Got %v, want >= %v", got, want)
	}
	return f.leaves[index], nil
}

// ListByIndex returns the set of requested leaves.
func (f *LogClient) ListByIndex(ctx context.Context, start int64, count int64) ([]*trillian.LogLeaf, error) {
	if got, want := start+count, int64(len(f.leaves)); got > want {
		return nil, fmt.Errorf("Index out of range. Got %v, want <= %v", got, want)
	}
	if got, want := start, int64(0); got < want {
		return nil, fmt.Errorf("Index out of range. Got %v, want >= %v", got, want)
	}
	return f.leaves[start : start+count], nil
}

// UpdateRoot fetches the latest signed tree root.
func (f *LogClient) UpdateRoot(ctx context.Context) error {
	return nil
}

// Root returns the latest local copy of the signed log root.
func (f *LogClient) Root() trillian.SignedLogRoot {
	return trillian.SignedLogRoot{
		TreeSize: int64(len(f.leaves)),
	}
}
<commit_msg>Add methods to fake.logClient<commit_after>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fake

import (
	"context"
	"fmt"

	"github.com/google/trillian"
	"github.com/google/trillian/client"
)

// logClient implements trillian/client.VerifyingLogClient.
type logClient struct {
	leaves []*trillian.LogLeaf
}

// NewFakeTrillianClient returns a client that mimicks a trillian log.
func NewFakeTrillianClient() client.VerifyingLogClient {
	return &logClient{
		leaves: make([]*trillian.LogLeaf, 0),
	}
}

// AddLeaf adds a leaf to the log.
func (f *logClient) AddLeaf(ctx context.Context, data []byte) error {
	f.leaves = append(f.leaves, &trillian.LogLeaf{
		LeafValue: data,
	})
	return nil
}

// GetByIndex returns the requested leaf.
func (f *logClient) GetByIndex(ctx context.Context, index int64) (*trillian.LogLeaf, error) {
	if got, want := index, int64(len(f.leaves)); got > want {
		return nil, fmt.Errorf("Index out of range. Got %v, want <= %v", got, want)
	}
	if got, want := index, int64(0); got < want {
		return nil, fmt.Errorf("Index out of range. Got %v, want >= %v", got, want)
	}
	return f.leaves[index], nil
}

// ListByIndex returns the set of requested leaves.
func (f *logClient) ListByIndex(ctx context.Context, start int64, count int64) ([]*trillian.LogLeaf, error) {
	if got, want := start+count, int64(len(f.leaves)); got > want {
		return nil, fmt.Errorf("Index out of range. Got %v, want <= %v", got, want)
	}
	if got, want := start, int64(0); got < want {
		return nil, fmt.Errorf("Index out of range. Got %v, want >= %v", got, want)
	}
	return f.leaves[start : start+count], nil
}

// UpdateRoot fetches the latest signed tree root.
func (f *logClient) UpdateRoot(ctx context.Context) error {
	return nil
}

// Root returns the latest local copy of the signed log root.
func (f *logClient) Root() trillian.SignedLogRoot {
	return trillian.SignedLogRoot{
		TreeSize: int64(len(f.leaves)),
	}
}

// VerifyInclusion returns nil.
func (l *logClient) VerifyInclusion(ctx context.Context, data []byte) error {
	return nil
}

// VerifyInclusionAtIndex returns nil.
func (l *logClient) VerifyInclusionAtIndex(ctx context.Context, data []byte, index int64) error {
	return nil
}
<|endoftext|>


<commit_before>package models

import (
	"gopkg.in/yaml.v2"
	"io/ioutil"
)

type Setting struct {
	Database struct {
		Host       string
		DbName     string
		TokenTable string
		UserTable  string
	}
	Ssl struct {
		Key         string
		Certificate string
	}
	Router struct {
		Register string
		Login    string
		Validate string
	}
	Ip string
}

var Set Setting

func LoadSettings() error {
	text, err := ioutil.ReadFile("settings.yml")
	if err != nil {
		return err
	}

	if err = yaml.Unmarshal(text, &Set); err != nil {
		return err
	}

	return nil
}
<commit_msg>Import sort<commit_after>package models

import (
	"io/ioutil"

	"gopkg.in/yaml.v2"
)

type Setting struct {
	Database struct {
		Host       string
		DbName     string
		TokenTable string
		UserTable  string
	}
	Ssl struct {
		Key         string
		Certificate string
	}
	Router struct {
		Register string
		Login    string
		Validate string
	}
	Ip string
}

var Set Setting

func LoadSettings() error {
	text, err := ioutil.ReadFile("settings.yml")
	if err != nil {
		return err
	}

	if err = yaml.Unmarshal(text, &Set); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package collisions

import (
	"github.com/cj123/robot/initio"
	"log"
	"time"
)

type Reading int

const (
	// the directions
	DIR_LEFT = iota
	DIR_FRONT
	DIR_RIGHT
	DIR_UNKNOWN

	// type of reading
	READING_QUICK Reading = iota
	READING_FULL  Reading = iota

	// the increments for the servos
	PAN_STEP  = 10
	TILT_STEP = 20

	// an adjustment to take from the time calculated on a corner
	CORNER_ADJUST = 25 * time.Millisecond
)

var (
	// the readings from the sensors at given tilt/pan angles
	// as there are +90 -> -90 degrees on each centre, that gives 180*180
	// possible readings. How quick these will happen, we don't know let's see
	readings [180][180]int

	// the average readings of each sensor pan location (averaged tilts)
	averageReadings [180]int

	// are we in motion?
	inMotion bool

	takingMeasurements, foundCollision bool

	motors *initio.Motors

	panServo, tiltServo *initio.Servo
)

func init() {

	motors = initio.NewMotor()
	panServo = initio.NewServo(initio.Pan)
	tiltServo = initio.NewServo(initio.Tilt)

	inMotion = false

	// make all reading values -1 to begin with
	for i := 0; i < len(readings); i++ {
		averageReadings[i] = -1
		for j := 0; j < len(readings[i]); j++ {
			readings[i][j] = -1
		}
	}
}

// because we're using a 2D array, we need to adjust the index to be in the range
// 0 < 180 rather than -90 < 90.
func getIndex(i int) int {
	return i + 90
}

// make readings, populating the readings
func MakeReadings(readingType Reading) {

	// sonar
	sonar := initio.NewSonar()

	takingMeasurements = true

	// get the step values
	panStep, tiltStep := PAN_STEP, TILT_STEP

	// if it's a quick read, increase them
	if readingType == READING_QUICK {
		panStep *= 2
		tiltStep *= 3
	}

	start := time.Now()
	numReadings := 0

	for i := -90; i < 90; i += panStep {
		// move the servo pan position
		tiltServo.Set(i)

		for j := -60; j < 90; j += tiltStep {
			dist := sonar.GetDistance()

			//fmt.Printf("(%d, %d) = %d\n", i, j, dist)

			// store reading
			readings[getIndex(i)][getIndex(j)] = dist

			// move the servo tilt position
			panServo.Set(j)

			time.Sleep(50 * time.Millisecond)
			numReadings++
		}
	}

	end := time.Now()

	log.Printf("Total time taken: %fs\n", end.Sub(start).Seconds())
	log.Printf("Number of readings: %d\n", numReadings)

	takingMeasurements = false
}

// get the position for the maximum distance, as well as that distance
// note that maxKey is returned as servo position, not array index
// (use getIndex() to get the array index)
func GetMaximumDistance() (maxKey int, maxVal int) {

	// if not in motion
	if !inMotion {
		// do quick readings
		MakeReadings(READING_QUICK)
	} else {
		// do a full scan
		MakeReadings(READING_FULL)
	}

	// go through all the readings
	for i := 0; i < len(readings); i++ {
		sum := 0
		numTaken := 0

		for j := 0; j < len(readings[i]); j++ {
			// if it's set to not read here,
			if readings[i][j] == -1 {
				continue
			}

			// average all the y values into an array
			sum += readings[i][j]

			// for the denominator
			numTaken++
		}

		// avoid division by zero
		if numTaken != 0 {
			averageReadings[i] = sum / numTaken
		}
	}

	// find the maximum value
	maxVal = averageReadings[0]
	maxKey = 0

	for index, val := range averageReadings {
		//fmt.Printf("%d = %f\n", index, val)
		if val != -1 && val != 0 && val > maxVal {

			maxVal = val
			maxKey = index
		}
	}

	return maxKey - 90, maxVal
}

// constantly recheck the IR sensors,
func checkIR() {

	ir := initio.IR{}

	for {
		time.Sleep(10 * time.Microsecond)

		if takingMeasurements {
			log.Println("Taking measurements...")
			continue
		}

		log.Println("Checking for collisions")

		// check ir
		if ir.Left() || ir.Right() {

			log.Println("Found immediate collision. Stopping...")

			foundCollision = true

			motors.Stop() // stop immediately

			// reverse until the objects are gone
			motors.Reverse(0)

			for ir.Left() || ir.Right() {
				time.Sleep(10 * time.Microsecond)

				// block any other actions until we're out of danger
				motors.Reverse(0)
			}

			motors.Stop()

			log.Println("Object moved, continuing...")

			foundCollision = false
		}
	}
}

// start collision avoidance
func Start(run *bool) bool {

	if *run {
		log.Println("Starting collision avoidance...")
	} else {
		log.Println("Collision avoidance initialised, but disabled for now")
	}

	go func() {
		foundCollision = false

		// start the IR sensor checking
		checkIR()
	}()

	for {
		// if we found a collision
		if foundCollision || !*run {
			log.Println("Collision found, no readings to be taken")
			time.Sleep(10 * time.Millisecond)
			continue
		}

		maxKey, maxVal := GetMaximumDistance()
		direction := getDirection(maxKey)

		log.Printf("Maximum value: %d at key %d\n", maxVal, maxKey)
		log.Printf("So we should probably turn %s\n", getDirectionName(direction))

		log.Println("Moving servo to that position...")

		// set the servo to the turn value
		panServo.Set(maxKey)

		// set the servo height, because... OCD
		tiltServo.Set(initio.DEFAULT_VAL)

		// find the turn degrees required to do it
		switch direction {
		case DIR_FRONT:
			// go forwards
			motors.Forward(0) // speed still needs implementing
			inMotion = true
			time.Sleep(getTimeToMoveForwards(maxVal))
			break
		case DIR_LEFT:
			motors.SpinLeft(0)
			t := getTimeForTurn(maxKey)
			log.Println("I should turn for", t)
			inMotion = true
			time.Sleep(t)

			motors.Forward(0) // then keep going

			// get the time to move forward, but take a small amount off
			// because of the corner we just turned
			time.Sleep(getTimeToMoveForwards(maxVal) - CORNER_ADJUST)
			break
		case DIR_RIGHT:
			motors.SpinRight(0)
			t := getTimeForTurn(maxKey)
			log.Println("I should turn for", t)
			inMotion = true
			time.Sleep(t)

			motors.Forward(0) // then keep going

			// get the time to move forward, but take a small amount off
			// because of the corner we just turned
			time.Sleep(getTimeToMoveForwards(maxVal) - CORNER_ADJUST)
			break
		default:
			log.Println("broken")
		}

		// stop moving
		motors.Stop()

		inMotion = false
	}

	return true
}
<commit_msg>add support for reverse sensors, to prevent backwards-crashes<commit_after>package collisions

import (
	"github.com/cj123/robot/initio"
	"log"
	"time"
)

type Reading int

const (
	// the directions
	DIR_LEFT = iota
	DIR_FRONT
	DIR_RIGHT
	DIR_UNKNOWN

	// type of reading
	READING_QUICK Reading = iota
	READING_FULL  Reading = iota

	// the increments for the servos
	PAN_STEP  = 10
	TILT_STEP = 20

	// an adjustment to take from the time calculated on a corner
	CORNER_ADJUST = 25 * time.Millisecond
)

var (
	// the readings from the sensors at given tilt/pan angles
	// as there are +90 -> -90 degrees on each centre, that gives 180*180
	// possible readings. How quick these will happen, we don't know let's see
	readings [180][180]int

	// the average readings of each sensor pan location (averaged tilts)
	averageReadings [180]int

	// are we in motion?
	inMotion bool

	takingMeasurements, foundCollision bool

	motors *initio.Motors

	panServo, tiltServo *initio.Servo
)

func init() {

	motors = initio.NewMotor()
	panServo = initio.NewServo(initio.Pan)
	tiltServo = initio.NewServo(initio.Tilt)

	inMotion = false

	// make all reading values -1 to begin with
	for i := 0; i < len(readings); i++ {
		averageReadings[i] = -1
		for j := 0; j < len(readings[i]); j++ {
			readings[i][j] = -1
		}
	}
}

// because we're using a 2D array, we need to adjust the index to be in the range
// 0 < 180 rather than -90 < 90.
func getIndex(i int) int {
	return i + 90
}

// make readings, populating the readings
func MakeReadings(readingType Reading) {

	// sonar
	sonar := initio.NewSonar()

	takingMeasurements = true

	// get the step values
	panStep, tiltStep := PAN_STEP, TILT_STEP

	// if it's a quick read, increase them
	if readingType == READING_QUICK {
		panStep *= 2
		tiltStep *= 3
	}

	start := time.Now()
	numReadings := 0

	for i := -90; i < 90; i += panStep {
		// move the servo pan position
		panServo.Set(i)

		for j := -60; j < 90; j += tiltStep {
			dist := sonar.GetDistance()

			//fmt.Printf("(%d, %d) = %d\n", i, j, dist)

			// store reading
			readings[getIndex(i)][getIndex(j)] = dist

			// move the servo tilt position
			tiltServo.Set(j)

			time.Sleep(50 * time.Millisecond)
			numReadings++
		}
	}

	end := time.Now()

	log.Printf("Total time taken: %fs\n", end.Sub(start).Seconds())
	log.Printf("Number of readings: %d\n", numReadings)

	takingMeasurements = false
}

// get the position for the maximum distance, as well as that distance
// note that maxKey is returned as servo position, not array index
// (use getIndex() to get the array index)
func GetMaximumDistance() (maxKey int, maxVal int) {

	// if not in motion
	if !inMotion {
		// do quick readings
		MakeReadings(READING_QUICK)
	} else {
		// do a full scan
		MakeReadings(READING_FULL)
	}

	// go through all the readings
	for i := 0; i < len(readings); i++ {
		sum := 0
		numTaken := 0

		for j := 0; j < len(readings[i]); j++ {
			// if it's set to not read here,
			if readings[i][j] == -1 {
				continue
			}

			// average all the y values into an array
			sum += readings[i][j]

			// for the denominator
			numTaken++
		}

		// avoid division by zero
		if numTaken != 0 {
			averageReadings[i] = sum / numTaken
		}
	}

	// find the maximum value
	maxVal = averageReadings[0]
	maxKey = 0

	for index, val := range averageReadings {
		//fmt.Printf("%d = %f\n", index, val)
		if val != -1 && val != 0 && val > maxVal {

			maxVal = val
			maxKey = index
		}
	}

	return maxKey - 90, maxVal
}

// constantly recheck the IR sensors,
func checkIR() {

	ir := initio.IR{}

	for {
		time.Sleep(10 * time.Microsecond)

		if takingMeasurements {
			log.Println("Taking measurements...")
			continue
		}

		// check ir
		if ir.Left() || ir.Right() {

			log.Println("Found immediate collision. Stopping...")

			foundCollision = true

			motors.Stop() // stop immediately

			// reverse until the objects are gone
			motors.Reverse(0)

			for ir.Left() || ir.Right() {
				time.Sleep(10 * time.Microsecond)

				// block any other actions until we're out of danger
				motors.Reverse(0)
			}

			motors.Stop()

			log.Println("Object moved, continuing...")

			foundCollision = false
		}

		if ir.BackLeft() || ir.BackRight() {
			log.Println("Found collision behind robot. Stopping...")

			foundCollision = true

			motors.Stop() // stop now

			// move forward until obstacle is gone
			motors.Forward(0)

			for ir.BackLeft() || ir.BackRight() {
				time.Sleep(10 * time.Microsecond)

				// block other actions
				motors.Forward(0)
			}

			motors.Stop()

			log.Println("Lost object, continuing...")

			foundCollision = false
		}
	}
}

// start collision avoidance
func Start(run *bool) bool {

	if *run {
		log.Println("Starting collision avoidance...")
	} else {
		log.Println("Collision avoidance initialised, but disabled for now")
	}

	go func() {
		foundCollision = false

		// start the IR sensor checking
		checkIR()
	}()

	for {
		// if we found a collision
		if foundCollision || !*run {
			time.Sleep(10 * time.Millisecond)
			continue
		}

		maxKey, maxVal := GetMaximumDistance()
		direction := getDirection(maxKey)

		log.Printf("Maximum value: %d at key %d\n", maxVal, maxKey)
		log.Printf("So we should probably turn %s\n", getDirectionName(direction))

		log.Println("Moving servo to that position...")

		// set the servo to the turn value
		panServo.Set(maxKey)

		// set the servo height, because... OCD
		tiltServo.Set(initio.DEFAULT_VAL)

		// find the turn degrees required to do it
		switch direction {
		case DIR_FRONT:
			// go forwards
			motors.Forward(0) // speed still needs implementing
			inMotion = true
			time.Sleep(getTimeToMoveForwards(maxVal))
			break
		case DIR_LEFT:
			motors.SpinLeft(0)
			t := getTimeForTurn(maxKey)
			log.Println("I should turn for", t)
			inMotion = true
			time.Sleep(t)

			motors.Forward(0) // then keep going

			// get the time to move forward, but take a small amount off
			// because of the corner we just turned
			time.Sleep(getTimeToMoveForwards(maxVal) - CORNER_ADJUST)
			break
		case DIR_RIGHT:
			motors.SpinRight(0)
			t := getTimeForTurn(maxKey)
			log.Println("I should turn for", t)
			inMotion = true
			time.Sleep(t)

			motors.Forward(0) // then keep going

			// get the time to move forward, but take a small amount off
			// because of the corner we just turned
			time.Sleep(getTimeToMoveForwards(maxVal) - CORNER_ADJUST)
			break
		default:
			log.Println("broken")
		}

		// stop moving
		motors.Stop()

		inMotion = false
	}

	return true
}
<|endoftext|>


<commit_before>package integration

import (
	"code.cloudfoundry.org/cli/integration/helpers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
)

var _ = Describe("proxy", func() {
	var proxyURL, apiRegexp string

	BeforeEach(func() {
		proxyURL = "127.0.0.1:9999"
		apiRegexp = "(?:https:\\/\\/)*" + apiURL
	})

	Context("V2 Legacy", func() {
		It("handles a proxy", func() {
			session := helpers.CFWithEnv(map[string]string{"http_proxy": proxyURL}, "api", apiURL)
			Eventually(session).Should(Say("Error performing request: Get %s\\/v2\\/info: http: error connecting to proxy http:\\/\\/%s", apiRegexp, proxyURL))
			Eventually(session).Should(Exit(1))
		})
	})

	Context("V3", func() {
		It("handles a proxy", func() {
			session := helpers.CFWithEnv(map[string]string{"http_proxy": proxyURL}, "run-task", "app", "echo")
			Eventually(session.Err).Should(Say("Get %s: http: error connecting to proxy http:\\/\\/%s", apiRegexp, proxyURL))
			Eventually(session).Should(Exit(1))
		})
	})
})
<commit_msg>use https_proxy environment variable in proxy integration tests<commit_after>package integration

import (
	"code.cloudfoundry.org/cli/integration/helpers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
)

var _ = Describe("proxy", func() {
	var proxyURL, apiRegexp string

	BeforeEach(func() {
		proxyURL = "127.0.0.1:9999"
		apiRegexp = "(?:https:\\/\\/)*" + apiURL
	})

	Context("V2 Legacy", func() {
		It("handles a proxy", func() {
			session := helpers.CFWithEnv(map[string]string{"https_proxy": proxyURL}, "api", apiURL)
			Eventually(session).Should(Say("Error performing request: Get %s\\/v2\\/info: http: error connecting to proxy http:\\/\\/%s", apiRegexp, proxyURL))
			Eventually(session).Should(Exit(1))
		})
	})

	Context("V3", func() {
		It("handles a proxy", func() {
			session := helpers.CFWithEnv(map[string]string{"https_proxy": proxyURL}, "run-task", "app", "echo")
			Eventually(session.Err).Should(Say("Get %s: http: error connecting to proxy http:\\/\\/%s", apiRegexp, proxyURL))
			Eventually(session).Should(Exit(1))
		})
	})
})
<|endoftext|>


<commit_before>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package api

import (
	"fmt"
	"os"
	"strconv"
	"testing"
	"time"

	info "github.com/google/cadvisor/info/v1"
	"github.com/google/cadvisor/info/v2"
	"github.com/google/cadvisor/integration/framework"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

// Sanity check the container by:
// - Checking that the specified alias is a valid one for this container.
// - Verifying that stats are not empty.
func sanityCheck(alias string, containerInfo info.ContainerInfo, t *testing.T) {
	assert.Contains(t, containerInfo.Aliases, alias, "Alias %q should be in list of aliases %v", alias, containerInfo.Aliases)
	assert.NotEmpty(t, containerInfo.Stats, "Expected container to have stats")
}

// Sanity check the container by:
// - Checking that the specified alias is a valid one for this container.
// - Verifying that stats are not empty.
func sanityCheckV2(alias string, info v2.ContainerInfo, t *testing.T) {
	assert.Contains(t, info.Spec.Aliases, alias, "Alias %q should be in list of aliases %v", alias, info.Spec.Aliases)
	assert.NotEmpty(t, info.Stats, "Expected container to have stats")
}

// Waits up to 5s for a container with the specified alias to appear.
func waitForContainer(alias string, fm framework.Framework) {
	err := framework.RetryForDuration(func() error {
		ret, err := fm.Cadvisor().Client().DockerContainer(alias, &info.ContainerInfoRequest{
			NumStats: 1,
		})
		if err != nil {
			return err
		}
		if len(ret.Stats) != 1 {
			return fmt.Errorf("no stats returned for container %q", alias)
		}

		return nil
	}, 5*time.Second)
	require.NoError(fm.T(), err, "Timed out waiting for container %q to be available in cAdvisor: %v", alias, err)
}

// A Docker container in /docker/<ID>
func TestDockerContainerById(t *testing.T) {
	fm := framework.New(t)
	defer fm.Cleanup()

	containerId := fm.Docker().RunPause()

	// Wait for the container to show up.
	waitForContainer(containerId, fm)

	request := &info.ContainerInfoRequest{
		NumStats: 1,
	}
	containerInfo, err := fm.Cadvisor().Client().DockerContainer(containerId, request)
	require.NoError(t, err)

	sanityCheck(containerId, containerInfo, t)
}

// A Docker container in /docker/<name>
func TestDockerContainerByName(t *testing.T) {
	fm := framework.New(t)
	defer fm.Cleanup()

	containerName := fmt.Sprintf("test-docker-container-by-name-%d", os.Getpid())
	fm.Docker().Run(framework.DockerRunArgs{
		Image: "kubernetes/pause",
		Args:  []string{"--name", containerName},
	})

	// Wait for the container to show up.
	waitForContainer(containerName, fm)

	request := &info.ContainerInfoRequest{
		NumStats: 1,
	}
	containerInfo, err := fm.Cadvisor().Client().DockerContainer(containerName, request)
	require.NoError(t, err)

	sanityCheck(containerName, containerInfo, t)
}

// Find the first container with the specified alias in containers.
func findContainer(alias string, containers []info.ContainerInfo, t *testing.T) info.ContainerInfo {
	for _, cont := range containers {
		for _, a := range cont.Aliases {
			if alias == a {
				return cont
			}
		}
	}
	t.Fatalf("Failed to find container %q in %+v", alias, containers)
	return info.ContainerInfo{}
}

// All Docker containers through /docker
func TestGetAllDockerContainers(t *testing.T) {
	fm := framework.New(t)
	defer fm.Cleanup()

	// Wait for the containers to show up.
	containerId1 := fm.Docker().RunPause()
	containerId2 := fm.Docker().RunPause()
	waitForContainer(containerId1, fm)
	waitForContainer(containerId2, fm)

	request := &info.ContainerInfoRequest{
		NumStats: 1,
	}
	containersInfo, err := fm.Cadvisor().Client().AllDockerContainers(request)
	require.NoError(t, err)

	if len(containersInfo) < 2 {
		t.Fatalf("At least 2 Docker containers should exist, received %d: %+v", len(containersInfo), containersInfo)
	}
	sanityCheck(containerId1, findContainer(containerId1, containersInfo, t), t)
	sanityCheck(containerId2, findContainer(containerId2, containersInfo, t), t)
}

// Check expected properties of a Docker container.
func TestBasicDockerContainer(t *testing.T) {
	fm := framework.New(t)
	defer fm.Cleanup()

	containerName := fmt.Sprintf("test-basic-docker-container-%d", os.Getpid())
	containerId := fm.Docker().Run(framework.DockerRunArgs{
		Image: "kubernetes/pause",
		Args: []string{
			"--name", containerName,
		},
	})

	// Wait for the container to show up.
	waitForContainer(containerId, fm)

	request := &info.ContainerInfoRequest{
		NumStats: 1,
	}
	containerInfo, err := fm.Cadvisor().Client().DockerContainer(containerId, request)
	require.NoError(t, err)

	// Check that the contianer is known by both its name and ID.
	sanityCheck(containerId, containerInfo, t)
	sanityCheck(containerName, containerInfo, t)

	assert.Empty(t, containerInfo.Subcontainers, "Should not have subcontainers")
	assert.Len(t, containerInfo.Stats, 1, "Should have exactly one stat")
}

// TODO(vmarmol): Handle if CPU or memory is not isolated on this system.
// Check the ContainerSpec.
func TestDockerContainerSpec(t *testing.T) {
	fm := framework.New(t)
	defer fm.Cleanup()

	var (
		cpuShares   = uint64(2048)
		cpuMask     = "0"
		memoryLimit = uint64(1 << 30) // 1GB
		image       = "kubernetes/pause"
		env         = map[string]string{"test_var": "FOO"}
		labels      = map[string]string{"bar": "baz"}
	)

	containerId := fm.Docker().Run(framework.DockerRunArgs{
		Image: image,
		Args: []string{
			"--cpu-shares", strconv.FormatUint(cpuShares, 10),
			"--cpuset-cpus", cpuMask,
			"--memory", strconv.FormatUint(memoryLimit, 10),
			"--env", "TEST_VAR=FOO",
			"--label", "bar=baz",
		},
	})

	// Wait for the container to show up.
	waitForContainer(containerId, fm)

	request := &info.ContainerInfoRequest{
		NumStats: 1,
	}
	containerInfo, err := fm.Cadvisor().Client().DockerContainer(containerId, request)
	require.NoError(t, err)
	sanityCheck(containerId, containerInfo, t)

	assert := assert.New(t)

	assert.True(containerInfo.Spec.HasCpu, "CPU should be isolated")
	assert.Equal(cpuShares, containerInfo.Spec.Cpu.Limit, "Container should have %d shares, has %d", cpuShares, containerInfo.Spec.Cpu.Limit)
	assert.Equal(cpuMask, containerInfo.Spec.Cpu.Mask, "Cpu mask should be %q, but is %q", cpuMask, containerInfo.Spec.Cpu.Mask)
	assert.True(containerInfo.Spec.HasMemory, "Memory should be isolated")
	assert.Equal(memoryLimit, containerInfo.Spec.Memory.Limit, "Container should have memory limit of %d, has %d", memoryLimit, containerInfo.Spec.Memory.Limit)
	assert.True(containerInfo.Spec.HasNetwork, "Network should be isolated")
	assert.True(containerInfo.Spec.HasDiskIo, "Blkio should be isolated")

	assert.Equal(image, containerInfo.Spec.Image, "Spec should include container image")
	assert.Equal(env, containerInfo.Spec.Envs, "Spec should include environment variables")
	assert.Equal(labels, containerInfo.Spec.Labels, "Spec should include labels")
}

// Check the CPU ContainerStats.
func TestDockerContainerCpuStats(t *testing.T) {
	fm := framework.New(t)
	defer fm.Cleanup()

	// Wait for the container to show up.
	containerId := fm.Docker().RunBusybox("ping", "www.google.com")
	waitForContainer(containerId, fm)

	request := &info.ContainerInfoRequest{
		NumStats: 1,
	}
	containerInfo, err := fm.Cadvisor().Client().DockerContainer(containerId, request)
	if err != nil {
		t.Fatal(err)
	}
	sanityCheck(containerId, containerInfo, t)

	// Checks for CpuStats.
	checkCpuStats(t, containerInfo.Stats[0].Cpu)
}

// Check the memory ContainerStats.
func TestDockerContainerMemoryStats(t *testing.T) {
	fm := framework.New(t)
	defer fm.Cleanup()

	// Wait for the container to show up.
	containerId := fm.Docker().RunBusybox("ping", "www.google.com")
	waitForContainer(containerId, fm)

	request := &info.ContainerInfoRequest{
		NumStats: 1,
	}
	containerInfo, err := fm.Cadvisor().Client().DockerContainer(containerId, request)
	require.NoError(t, err)
	sanityCheck(containerId, containerInfo, t)

	// Checks for MemoryStats.
	checkMemoryStats(t, containerInfo.Stats[0].Memory)
}

// Check the network ContainerStats.
func TestDockerContainerNetworkStats(t *testing.T) {
	fm := framework.New(t)
	defer fm.Cleanup()

	// Wait for the container to show up.
	containerId := fm.Docker().RunBusybox("watch", "-n1", "wget", "http://www.google.com/")
	waitForContainer(containerId, fm)

	time.Sleep(10 * time.Second)
	request := &info.ContainerInfoRequest{
		NumStats: 1,
	}
	containerInfo, err := fm.Cadvisor().Client().DockerContainer(containerId, request)
	require.NoError(t, err)
	sanityCheck(containerId, containerInfo, t)

	// Checks for NetworkStats.
	stat := containerInfo.Stats[0]
	assert := assert.New(t)
	assert.NotEqual(0, stat.Network.TxBytes, "Network tx bytes should not be zero")
	assert.NotEqual(0, stat.Network.TxPackets, "Network tx packets should not be zero")
	assert.NotEqual(0, stat.Network.RxBytes, "Network rx bytes should not be zero")
	assert.NotEqual(0, stat.Network.RxPackets, "Network rx packets should not be zero")
	assert.NotEqual(stat.Network.RxBytes, stat.Network.TxBytes, "Network tx and rx bytes should not be equal")
	assert.NotEqual(stat.Network.RxPackets, stat.Network.TxPackets, "Network tx and rx packets should not be equal")
}

func TestDockerFilesystemStats(t *testing.T) {
	fm := framework.New(t)
	defer fm.Cleanup()

	storageDriver := fm.Docker().StorageDriver()
	if storageDriver == framework.DeviceMapper {
		// Filesystem stats not supported with devicemapper, yet
		return
	}

	const (
		ddUsage       = uint64(1 << 3) // 1 KB
		sleepDuration = 10 * time.Second
	)
	// Wait for the container to show up.
	// FIXME: Tests should be bundled and run on the remote host instead of being run over ssh.
	// Escaping bash over ssh is ugly.
	// Once github issue 1130 is fixed, this logic can be removed.
	dockerCmd := fmt.Sprintf("dd if=/dev/zero of=/file count=2 bs=%d & ping google.com", ddUsage)
	if fm.Hostname().Host != "localhost" {
		dockerCmd = fmt.Sprintf("'%s'", dockerCmd)
	}
	containerId := fm.Docker().RunBusybox("/bin/sh", "-c", dockerCmd)
	waitForContainer(containerId, fm)
	request := &v2.RequestOptions{
		IdType: v2.TypeDocker,
		Count:  1,
	}
	needsBaseUsageCheck := false
	switch storageDriver {
	case framework.Aufs, framework.Overlay, framework.Overlay2, framework.DeviceMapper:
		needsBaseUsageCheck = true
	}
	pass := false
	// We need to wait for the `dd` operation to complete.
	for i := 0; i < 10; i++ {
		containerInfo, err := fm.Cadvisor().ClientV2().Stats(containerId, request)
		if err != nil {
			t.Logf("%v stats unavailable - %v", time.Now().String(), err)
			t.Logf("retrying after %s...", sleepDuration.String())
			time.Sleep(sleepDuration)

			continue
		}
		require.Equal(t, len(containerInfo), 1)
		var info v2.ContainerInfo
		// There is only one container in containerInfo. Since it is a map with unknown key,
		// use the value blindly.
		for _, cInfo := range containerInfo {
			info = cInfo
		}
		sanityCheckV2(containerId, info, t)

		require.NotNil(t, info.Stats[0], "got info: %+v", info)
		require.NotNil(t, info.Stats[0].Filesystem, "got info: %+v", info)
		require.NotNil(t, info.Stats[0].Filesystem.TotalUsageBytes, "got info: %+v", info.Stats[0].Filesystem)
		if *info.Stats[0].Filesystem.TotalUsageBytes >= ddUsage {
			if !needsBaseUsageCheck {
				pass = true
				break
			}
			require.NotNil(t, info.Stats[0].Filesystem.BaseUsageBytes)
			if *info.Stats[0].Filesystem.BaseUsageBytes >= ddUsage {
				pass = true
				break
			}
		}
		t.Logf("expected total usage %d bytes to be greater than %d bytes", *info.Stats[0].Filesystem.TotalUsageBytes, ddUsage)
		if needsBaseUsageCheck {
			t.Logf("expected base %d bytes to be greater than %d bytes", *info.Stats[0].Filesystem.BaseUsageBytes, ddUsage)
		}
		t.Logf("retrying after %s...", sleepDuration.String())
		time.Sleep(sleepDuration)
	}

	if !pass {
		t.Fail()
	}
}
<commit_msg>Pick the right Network interface<commit_after>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package api

import (
	"fmt"
	"os"
	"strconv"
	"testing"
	"time"

	info "github.com/google/cadvisor/info/v1"
	"github.com/google/cadvisor/info/v2"
	"github.com/google/cadvisor/integration/framework"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

// Sanity check the container by:
// - Checking that the specified alias is a valid one for this container.
// - Verifying that stats are not empty.
func sanityCheck(alias string, containerInfo info.ContainerInfo, t *testing.T) {
	assert.Contains(t, containerInfo.Aliases, alias, "Alias %q should be in list of aliases %v", alias, containerInfo.Aliases)
	assert.NotEmpty(t, containerInfo.Stats, "Expected container to have stats")
}

// Sanity check the container by:
// - Checking that the specified alias is a valid one for this container.
// - Verifying that stats are not empty.
func sanityCheckV2(alias string, info v2.ContainerInfo, t *testing.T) {
	assert.Contains(t, info.Spec.Aliases, alias, "Alias %q should be in list of aliases %v", alias, info.Spec.Aliases)
	assert.NotEmpty(t, info.Stats, "Expected container to have stats")
}

// Waits up to 5s for a container with the specified alias to appear.
func waitForContainer(alias string, fm framework.Framework) {
	err := framework.RetryForDuration(func() error {
		ret, err := fm.Cadvisor().Client().DockerContainer(alias, &info.ContainerInfoRequest{
			NumStats: 1,
		})
		if err != nil {
			return err
		}
		if len(ret.Stats) != 1 {
			return fmt.Errorf("no stats returned for container %q", alias)
		}

		return nil
	}, 5*time.Second)
	require.NoError(fm.T(), err, "Timed out waiting for container %q to be available in cAdvisor: %v", alias, err)
}

// A Docker container in /docker/<ID>
func TestDockerContainerById(t *testing.T) {
	fm := framework.New(t)
	defer fm.Cleanup()

	containerId := fm.Docker().RunPause()

	// Wait for the container to show up.
	waitForContainer(containerId, fm)

	request := &info.ContainerInfoRequest{
		NumStats: 1,
	}
	containerInfo, err := fm.Cadvisor().Client().DockerContainer(containerId, request)
	require.NoError(t, err)

	sanityCheck(containerId, containerInfo, t)
}

// A Docker container in /docker/<name>
func TestDockerContainerByName(t *testing.T) {
	fm := framework.New(t)
	defer fm.Cleanup()

	containerName := fmt.Sprintf("test-docker-container-by-name-%d", os.Getpid())
	fm.Docker().Run(framework.DockerRunArgs{
		Image: "kubernetes/pause",
		Args:  []string{"--name", containerName},
	})

	// Wait for the container to show up.
	waitForContainer(containerName, fm)

	request := &info.ContainerInfoRequest{
		NumStats: 1,
	}
	containerInfo, err := fm.Cadvisor().Client().DockerContainer(containerName, request)
	require.NoError(t, err)

	sanityCheck(containerName, containerInfo, t)
}

// Find the first container with the specified alias in containers.
func findContainer(alias string, containers []info.ContainerInfo, t *testing.T) info.ContainerInfo {
	for _, cont := range containers {
		for _, a := range cont.Aliases {
			if alias == a {
				return cont
			}
		}
	}
	t.Fatalf("Failed to find container %q in %+v", alias, containers)
	return info.ContainerInfo{}
}

// All Docker containers through /docker
func TestGetAllDockerContainers(t *testing.T) {
	fm := framework.New(t)
	defer fm.Cleanup()

	// Wait for the containers to show up.
	containerId1 := fm.Docker().RunPause()
	containerId2 := fm.Docker().RunPause()
	waitForContainer(containerId1, fm)
	waitForContainer(containerId2, fm)

	request := &info.ContainerInfoRequest{
		NumStats: 1,
	}
	containersInfo, err := fm.Cadvisor().Client().AllDockerContainers(request)
	require.NoError(t, err)

	if len(containersInfo) < 2 {
		t.Fatalf("At least 2 Docker containers should exist, received %d: %+v", len(containersInfo), containersInfo)
	}
	sanityCheck(containerId1, findContainer(containerId1, containersInfo, t), t)
	sanityCheck(containerId2, findContainer(containerId2, containersInfo, t), t)
}

// Check expected properties of a Docker container.
func TestBasicDockerContainer(t *testing.T) {
	fm := framework.New(t)
	defer fm.Cleanup()

	containerName := fmt.Sprintf("test-basic-docker-container-%d", os.Getpid())
	containerId := fm.Docker().Run(framework.DockerRunArgs{
		Image: "kubernetes/pause",
		Args: []string{
			"--name", containerName,
		},
	})

	// Wait for the container to show up.
	waitForContainer(containerId, fm)

	request := &info.ContainerInfoRequest{
		NumStats: 1,
	}
	containerInfo, err := fm.Cadvisor().Client().DockerContainer(containerId, request)
	require.NoError(t, err)

	// Check that the contianer is known by both its name and ID.
	sanityCheck(containerId, containerInfo, t)
	sanityCheck(containerName, containerInfo, t)

	assert.Empty(t, containerInfo.Subcontainers, "Should not have subcontainers")
	assert.Len(t, containerInfo.Stats, 1, "Should have exactly one stat")
}

// TODO(vmarmol): Handle if CPU or memory is not isolated on this system.
// Check the ContainerSpec.
func TestDockerContainerSpec(t *testing.T) {
	fm := framework.New(t)
	defer fm.Cleanup()

	var (
		cpuShares   = uint64(2048)
		cpuMask     = "0"
		memoryLimit = uint64(1 << 30) // 1GB
		image       = "kubernetes/pause"
		env         = map[string]string{"test_var": "FOO"}
		labels      = map[string]string{"bar": "baz"}
	)

	containerId := fm.Docker().Run(framework.DockerRunArgs{
		Image: image,
		Args: []string{
			"--cpu-shares", strconv.FormatUint(cpuShares, 10),
			"--cpuset-cpus", cpuMask,
			"--memory", strconv.FormatUint(memoryLimit, 10),
			"--env", "TEST_VAR=FOO",
			"--label", "bar=baz",
		},
	})

	// Wait for the container to show up.
	waitForContainer(containerId, fm)

	request := &info.ContainerInfoRequest{
		NumStats: 1,
	}
	containerInfo, err := fm.Cadvisor().Client().DockerContainer(containerId, request)
	require.NoError(t, err)
	sanityCheck(containerId, containerInfo, t)

	assert := assert.New(t)

	assert.True(containerInfo.Spec.HasCpu, "CPU should be isolated")
	assert.Equal(cpuShares, containerInfo.Spec.Cpu.Limit, "Container should have %d shares, has %d", cpuShares, containerInfo.Spec.Cpu.Limit)
	assert.Equal(cpuMask, containerInfo.Spec.Cpu.Mask, "Cpu mask should be %q, but is %q", cpuMask, containerInfo.Spec.Cpu.Mask)
	assert.True(containerInfo.Spec.HasMemory, "Memory should be isolated")
	assert.Equal(memoryLimit, containerInfo.Spec.Memory.Limit, "Container should have memory limit of %d, has %d", memoryLimit, containerInfo.Spec.Memory.Limit)
	assert.True(containerInfo.Spec.HasNetwork, "Network should be isolated")
	assert.True(containerInfo.Spec.HasDiskIo, "Blkio should be isolated")

	assert.Equal(image, containerInfo.Spec.Image, "Spec should include container image")
	assert.Equal(env, containerInfo.Spec.Envs, "Spec should include environment variables")
	assert.Equal(labels, containerInfo.Spec.Labels, "Spec should include labels")
}

// Check the CPU ContainerStats.
func TestDockerContainerCpuStats(t *testing.T) {
	fm := framework.New(t)
	defer fm.Cleanup()

	// Wait for the container to show up.
	containerId := fm.Docker().RunBusybox("ping", "www.google.com")
	waitForContainer(containerId, fm)

	request := &info.ContainerInfoRequest{
		NumStats: 1,
	}
	containerInfo, err := fm.Cadvisor().Client().DockerContainer(containerId, request)
	if err != nil {
		t.Fatal(err)
	}
	sanityCheck(containerId, containerInfo, t)

	// Checks for CpuStats.
	checkCpuStats(t, containerInfo.Stats[0].Cpu)
}

// Check the memory ContainerStats.
func TestDockerContainerMemoryStats(t *testing.T) {
	fm := framework.New(t)
	defer fm.Cleanup()

	// Wait for the container to show up.
	containerId := fm.Docker().RunBusybox("ping", "www.google.com")
	waitForContainer(containerId, fm)

	request := &info.ContainerInfoRequest{
		NumStats: 1,
	}
	containerInfo, err := fm.Cadvisor().Client().DockerContainer(containerId, request)
	require.NoError(t, err)
	sanityCheck(containerId, containerInfo, t)

	// Checks for MemoryStats.
	checkMemoryStats(t, containerInfo.Stats[0].Memory)
}

// Check the network ContainerStats.
func TestDockerContainerNetworkStats(t *testing.T) {
	fm := framework.New(t)
	defer fm.Cleanup()

	// Wait for the container to show up.
	containerId := fm.Docker().RunBusybox("watch", "-n1", "wget", "http://www.google.com/")
	waitForContainer(containerId, fm)

	time.Sleep(10 * time.Second)
	request := &info.ContainerInfoRequest{
		NumStats: 1,
	}
	containerInfo, err := fm.Cadvisor().Client().DockerContainer(containerId, request)
	require.NoError(t, err)
	sanityCheck(containerId, containerInfo, t)

	stat := containerInfo.Stats[0]
	ifaceStats := stat.Network.InterfaceStats
	// macOS we have more than one interface, since traffic is
	// only on eth0 we need to pick that one
	if len(stat.Network.Interfaces) > 0 {
		for _, iface := range stat.Network.Interfaces {
			if iface.Name == "eth0" {
				ifaceStats = iface
			}
		}
	}

	// Checks for NetworkStats.
	assert := assert.New(t)
	assert.NotEqual(0, ifaceStats.TxBytes, "Network tx bytes should not be zero")
	assert.NotEqual(0, ifaceStats.TxPackets, "Network tx packets should not be zero")
	assert.NotEqual(0, ifaceStats.RxBytes, "Network rx bytes should not be zero")
	assert.NotEqual(0, ifaceStats.RxPackets, "Network rx packets should not be zero")
	assert.NotEqual(ifaceStats.RxBytes, ifaceStats.TxBytes, "Network tx and rx bytes should not be equal")
	assert.NotEqual(ifaceStats.RxPackets, ifaceStats.TxPackets, "Network tx and rx packets should not be equal")
}

func TestDockerFilesystemStats(t *testing.T) {
	fm := framework.New(t)
	defer fm.Cleanup()

	storageDriver := fm.Docker().StorageDriver()
	if storageDriver == framework.DeviceMapper {
		// Filesystem stats not supported with devicemapper, yet
		return
	}

	const (
		ddUsage       = uint64(1 << 3) // 1 KB
		sleepDuration = 10 * time.Second
	)
	// Wait for the container to show up.
	// FIXME: Tests should be bundled and run on the remote host instead of being run over ssh.
	// Escaping bash over ssh is ugly.
	// Once github issue 1130 is fixed, this logic can be removed.
	dockerCmd := fmt.Sprintf("dd if=/dev/zero of=/file count=2 bs=%d & ping google.com", ddUsage)
	if fm.Hostname().Host != "localhost" {
		dockerCmd = fmt.Sprintf("'%s'", dockerCmd)
	}
	containerId := fm.Docker().RunBusybox("/bin/sh", "-c", dockerCmd)
	waitForContainer(containerId, fm)
	request := &v2.RequestOptions{
		IdType: v2.TypeDocker,
		Count:  1,
	}
	needsBaseUsageCheck := false
	switch storageDriver {
	case framework.Aufs, framework.Overlay, framework.Overlay2, framework.DeviceMapper:
		needsBaseUsageCheck = true
	}
	pass := false
	// We need to wait for the `dd` operation to complete.
	for i := 0; i < 10; i++ {
		containerInfo, err := fm.Cadvisor().ClientV2().Stats(containerId, request)
		if err != nil {
			t.Logf("%v stats unavailable - %v", time.Now().String(), err)
			t.Logf("retrying after %s...", sleepDuration.String())
			time.Sleep(sleepDuration)

			continue
		}
		require.Equal(t, len(containerInfo), 1)
		var info v2.ContainerInfo
		// There is only one container in containerInfo. Since it is a map with unknown key,
		// use the value blindly.
		for _, cInfo := range containerInfo {
			info = cInfo
		}
		sanityCheckV2(containerId, info, t)

		require.NotNil(t, info.Stats[0], "got info: %+v", info)
		require.NotNil(t, info.Stats[0].Filesystem, "got info: %+v", info)
		require.NotNil(t, info.Stats[0].Filesystem.TotalUsageBytes, "got info: %+v", info.Stats[0].Filesystem)
		if *info.Stats[0].Filesystem.TotalUsageBytes >= ddUsage {
			if !needsBaseUsageCheck {
				pass = true
				break
			}
			require.NotNil(t, info.Stats[0].Filesystem.BaseUsageBytes)
			if *info.Stats[0].Filesystem.BaseUsageBytes >= ddUsage {
				pass = true
				break
			}
		}
		t.Logf("expected total usage %d bytes to be greater than %d bytes", *info.Stats[0].Filesystem.TotalUsageBytes, ddUsage)
		if needsBaseUsageCheck {
			t.Logf("expected base %d bytes to be greater than %d bytes", *info.Stats[0].Filesystem.BaseUsageBytes, ddUsage)
		}
		t.Logf("retrying after %s...", sleepDuration.String())
		time.Sleep(sleepDuration)
	}

	if !pass {
		t.Fail()
	}
}
<|endoftext|>


<commit_before>package build

import (
	"flag"
	"fmt"
	"strings"

	"github.com/tueftler/doget/command"
	"github.com/tueftler/doget/docker"
	"github.com/tueftler/doget/dockerfile"
)

// BuildCommand is a thin wrapper around transform > docker build > clean
type BuildCommand struct {
	command.Command
	flags     *flag.FlagSet
	transform command.Command
	docker    docker.Client
	clean     command.Command
}

// NewCommand creates new build command instance
func NewCommand(name string, transform command.Command, clean command.Command, client docker.Client) *BuildCommand {
	return &BuildCommand{
		flags:     flag.NewFlagSet(name, flag.ExitOnError),
		transform: transform,
		docker:    client,
		clean:     clean,
	}
}

func (b *BuildCommand) Usage() error {
  output, err := b.docker.Help()
  if err != nil {
    return err
  }

	fmt.Println("Usage: doget build [OPTIONS] PATH | URL | - \n")

	// Make these look like docker build --help output
	fmt.Println("  --doget-no-cache                Do not use cache")
	fmt.Println("  --doget-in                      Input (default: Dockerfile.in)")
	fmt.Println("  --doget-out                     Output (default: Dockerfile)")

	// Only print flags usage
	for _, line := range strings.Split(string(output), "\n") {
		if strings.HasPrefix(line, "  -") {
			fmt.Println(line)
		}
	}
	fmt.Println()

	return nil
}

// Run performs action of build command
func (b *BuildCommand) Run(parser *dockerfile.Parser, args []string) error {
	if 0 == len(args) || "-help" == args[0] || "--help" == args[0] {
		return b.Usage()
	}

	transformArgs, dockerArgs := split(args)

	if err := b.transform.Run(parser, transformArgs); err != nil {
		return err
	}

	if err := b.docker.Build(dockerArgs); err != nil {
		return err
	}

	if err := b.clean.Run(parser, []string{}); err != nil {
		return err
	}

	return nil
}

func split(args []string) ([]string, []string) {
	transformArgs := []string{}
	dockerArgs := []string{}
	for _, arg := range args {
		if strings.HasPrefix(arg, "--doget-") {
			for _, val := range strings.Split(strings.Replace(arg, "--doget", "", 1), "=") {
				transformArgs = append(transformArgs, val)
			}
		} else {
			dockerArgs = append(dockerArgs, arg)
		}
	}

	return transformArgs, dockerArgs
}
<commit_msg>Stress --doget-[OPT]=[VAL] usage<commit_after>package build

import (
	"flag"
	"fmt"
	"strings"

	"github.com/tueftler/doget/command"
	"github.com/tueftler/doget/docker"
	"github.com/tueftler/doget/dockerfile"
)

// BuildCommand is a thin wrapper around transform > docker build > clean
type BuildCommand struct {
	command.Command
	flags     *flag.FlagSet
	transform command.Command
	docker    docker.Client
	clean     command.Command
}

// NewCommand creates new build command instance
func NewCommand(name string, transform command.Command, clean command.Command, client docker.Client) *BuildCommand {
	return &BuildCommand{
		flags:     flag.NewFlagSet(name, flag.ExitOnError),
		transform: transform,
		docker:    client,
		clean:     clean,
	}
}

func (b *BuildCommand) Usage() error {
  output, err := b.docker.Help()
  if err != nil {
    return err
  }

	fmt.Println("Usage: doget build [OPTIONS] PATH | URL | - \n")

	// Make these look like docker build --help output
	fmt.Println("  --doget-no-cache=false          Do not use cache for traits")
	fmt.Println("  --doget-in=Dockerfile.in        Input")
	fmt.Println("  --doget-out=Dockerfile          Output, combine with --file")

	// Only print flags usage
	for _, line := range strings.Split(string(output), "\n") {
		if strings.HasPrefix(line, "  -") {
			fmt.Println(line)
		}
	}
	fmt.Println()

	return nil
}

// Run performs action of build command
func (b *BuildCommand) Run(parser *dockerfile.Parser, args []string) error {
	if 0 == len(args) || "-help" == args[0] || "--help" == args[0] {
		return b.Usage()
	}

	transformArgs, dockerArgs := split(args)

	if err := b.transform.Run(parser, transformArgs); err != nil {
		return err
	}

	if err := b.docker.Build(dockerArgs); err != nil {
		return err
	}

	if err := b.clean.Run(parser, []string{}); err != nil {
		return err
	}

	return nil
}

func split(args []string) ([]string, []string) {
	transformArgs := []string{}
	dockerArgs := []string{}
	for _, arg := range args {
		if strings.HasPrefix(arg, "--doget-") {
			for _, val := range strings.Split(strings.Replace(arg, "--doget", "", 1), "=") {
				transformArgs = append(transformArgs, val)
			}
		} else {
			dockerArgs = append(dockerArgs, arg)
		}
	}

	return transformArgs, dockerArgs
}
<|endoftext|>


<commit_before>// Copyright 2018 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build !cluster_proxy

package integration

import (
	"context"
	"fmt"
	"testing"
	"time"

	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
)

// TestV3WatchRestoreSnapshotUnsync tests whether slow follower can restore
// from leader snapshot, and still notify on watchers from an old revision
// that were created in synced watcher group in the first place.
// TODO: fix panic with gRPC proxy "panic: watcher current revision should not exceed current revision"
func TestV3WatchRestoreSnapshotUnsync(t *testing.T) {
	clus := NewClusterV3(t, &ClusterConfig{
		Size:                   3,
		SnapshotCount:          10,
		SnapshotCatchUpEntries: 5,
	})
	defer clus.Terminate(t)

	// spawn a watcher before shutdown, and put it in synced watcher
	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancel()
	wStream, errW := toGRPC(clus.Client(0)).Watch.Watch(ctx)
	if errW != nil {
		t.Fatal(errW)
	}
	if err := wStream.Send(&pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{
		CreateRequest: &pb.WatchCreateRequest{Key: []byte("foo"), StartRevision: 5}}}); err != nil {
		t.Fatalf("wStream.Send error: %v", err)
	}
	wresp, errR := wStream.Recv()
	if errR != nil {
		t.Errorf("wStream.Recv error: %v", errR)
	}
	if !wresp.Created {
		t.Errorf("wresp.Created got = %v, want = true", wresp.Created)
	}

	clus.Members[0].InjectPartition(t, clus.Members[1:]...)
	clus.waitLeader(t, clus.Members[1:])
	time.Sleep(2 * time.Second)

	kvc := toGRPC(clus.Client(1)).KV

	// to trigger snapshot from the leader to the stopped follower
	for i := 0; i < 15; i++ {
		_, err := kvc.Put(context.TODO(), &pb.PutRequest{Key: []byte("foo"), Value: []byte("bar")})
		if err != nil {
			t.Errorf("#%d: couldn't put key (%v)", i, err)
		}
	}

	// trigger snapshot send from leader to this slow follower
	// which then calls watchable store Restore
	clus.Members[0].RecoverPartition(t, clus.Members[1:]...)
	clus.WaitLeader(t)
	time.Sleep(2 * time.Second)

	// slow follower now applies leader snapshot
	// should be able to notify on old-revision watchers in unsynced
	// make sure restore watch operation correctly moves watchers
	// between synced and unsynced watchers
	errc := make(chan error)
	go func() {
		cresp, cerr := wStream.Recv()
		if cerr != nil {
			errc <- cerr
			return
		}
		// from start revision 5 to latest revision 16
		if len(cresp.Events) != 12 {
			errc <- fmt.Errorf("expected 12 events, got %+v", cresp.Events)
			return
		}
		errc <- nil
	}()
	select {
	case <-time.After(10 * time.Second):
		t.Fatal("took too long to receive events from restored watcher")
	case err := <-errc:
		if err != nil {
			t.Fatalf("wStream.Recv error: %v", err)
		}
	}
}
<commit_msg>integration: enable TestV3WatchRestoreSnapshotUnsync for gRPC proxy<commit_after>// Copyright 2018 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package integration

import (
	"context"
	"fmt"
	"testing"
	"time"

	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
)

// TestV3WatchRestoreSnapshotUnsync tests whether slow follower can restore
// from leader snapshot, and still notify on watchers from an old revision
// that were created in synced watcher group in the first place.
// TODO: fix panic with gRPC proxy "panic: watcher current revision should not exceed current revision"
func TestV3WatchRestoreSnapshotUnsync(t *testing.T) {
	clus := NewClusterV3(t, &ClusterConfig{
		Size:                   3,
		SnapshotCount:          10,
		SnapshotCatchUpEntries: 5,
	})
	defer clus.Terminate(t)

	// spawn a watcher before shutdown, and put it in synced watcher
	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancel()
	wStream, errW := toGRPC(clus.Client(0)).Watch.Watch(ctx)
	if errW != nil {
		t.Fatal(errW)
	}
	if err := wStream.Send(&pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{
		CreateRequest: &pb.WatchCreateRequest{Key: []byte("foo"), StartRevision: 5}}}); err != nil {
		t.Fatalf("wStream.Send error: %v", err)
	}
	wresp, errR := wStream.Recv()
	if errR != nil {
		t.Errorf("wStream.Recv error: %v", errR)
	}
	if !wresp.Created {
		t.Errorf("wresp.Created got = %v, want = true", wresp.Created)
	}

	clus.Members[0].InjectPartition(t, clus.Members[1:]...)
	clus.waitLeader(t, clus.Members[1:])
	time.Sleep(2 * time.Second)

	kvc := toGRPC(clus.Client(1)).KV

	// to trigger snapshot from the leader to the stopped follower
	for i := 0; i < 15; i++ {
		_, err := kvc.Put(context.TODO(), &pb.PutRequest{Key: []byte("foo"), Value: []byte("bar")})
		if err != nil {
			t.Errorf("#%d: couldn't put key (%v)", i, err)
		}
	}

	// trigger snapshot send from leader to this slow follower
	// which then calls watchable store Restore
	clus.Members[0].RecoverPartition(t, clus.Members[1:]...)
	clus.WaitLeader(t)
	time.Sleep(2 * time.Second)

	// slow follower now applies leader snapshot
	// should be able to notify on old-revision watchers in unsynced
	// make sure restore watch operation correctly moves watchers
	// between synced and unsynced watchers
	errc := make(chan error)
	go func() {
		cresp, cerr := wStream.Recv()
		if cerr != nil {
			errc <- cerr
			return
		}
		// from start revision 5 to latest revision 16
		if len(cresp.Events) != 12 {
			errc <- fmt.Errorf("expected 12 events, got %+v", cresp.Events)
			return
		}
		errc <- nil
	}()
	select {
	case <-time.After(10 * time.Second):
		t.Fatal("took too long to receive events from restored watcher")
	case err := <-errc:
		if err != nil {
			t.Fatalf("wStream.Recv error: %v", err)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 The etcd-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package framework

import (
	"fmt"
	"os"

	"github.com/coreos/etcd-operator/pkg/client"
	"github.com/coreos/etcd-operator/pkg/generated/clientset/versioned"
	"github.com/coreos/etcd-operator/pkg/util/constants"
	"github.com/coreos/etcd-operator/pkg/util/k8sutil"
	"github.com/coreos/etcd-operator/pkg/util/probe"
	"github.com/coreos/etcd-operator/test/e2e/e2eutil"

	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	appsv1beta1 "k8s.io/api/apps/v1beta1"
	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/util/intstr"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/clientcmd"
)

type Config struct {
	// program flags
	KubeConfig string
	KubeNS     string
	OldImage   string
	NewImage   string
}

type Framework struct {
	Config
	// global var
	KubeCli  kubernetes.Interface
	CRClient versioned.Interface
	S3Cli    *s3.S3
	S3Bucket string
}

func New(fc Config) (*Framework, error) {
	kc, err := clientcmd.BuildConfigFromFlags("", fc.KubeConfig)
	if err != nil {
		return nil, err
	}
	kubecli, err := kubernetes.NewForConfig(kc)
	if err != nil {
		return nil, err
	}

	f := &Framework{
		Config:   fc,
		KubeCli:  kubecli,
		CRClient: client.MustNew(kc),
	}
	err = f.setupAWS()
	return f, err
}

func (f *Framework) CreateOperator(name string) error {
	cmd := []string{"/usr/local/bin/etcd-operator"}
	image := f.OldImage
	d := &appsv1beta1.Deployment{
		ObjectMeta: metav1.ObjectMeta{
			Name:      name,
			Namespace: f.KubeNS,
		},
		Spec: appsv1beta1.DeploymentSpec{
			Strategy: appsv1beta1.DeploymentStrategy{
				Type: appsv1beta1.RollingUpdateDeploymentStrategyType,
				RollingUpdate: &appsv1beta1.RollingUpdateDeployment{
					MaxUnavailable: &intstr.IntOrString{Type: intstr.Int, IntVal: 1},
					MaxSurge:       &intstr.IntOrString{Type: intstr.Int, IntVal: 1},
				},
			},
			Selector: &metav1.LabelSelector{MatchLabels: operatorLabelSelector(name)},
			Template: v1.PodTemplateSpec{
				ObjectMeta: metav1.ObjectMeta{
					Labels: operatorLabelSelector(name),
				},
				Spec: v1.PodSpec{
					Containers: []v1.Container{{
						Name:            name,
						Image:           image,
						ImagePullPolicy: v1.PullAlways,
						Command:         cmd,
						Env: []v1.EnvVar{
							{
								Name:      constants.EnvOperatorPodNamespace,
								ValueFrom: &v1.EnvVarSource{FieldRef: &v1.ObjectFieldSelector{FieldPath: "metadata.namespace"}},
							},
							{
								Name:      constants.EnvOperatorPodName,
								ValueFrom: &v1.EnvVarSource{FieldRef: &v1.ObjectFieldSelector{FieldPath: "metadata.name"}},
							},
						},
						ReadinessProbe: &v1.Probe{
							Handler: v1.Handler{
								HTTPGet: &v1.HTTPGetAction{
									Path: probe.HTTPReadyzEndpoint,
									Port: intstr.IntOrString{Type: intstr.Int, IntVal: 8080},
								},
							},
							InitialDelaySeconds: 3,
							PeriodSeconds:       3,
							FailureThreshold:    3,
						},
					}},
				},
			},
		},
	}
	_, err := f.KubeCli.AppsV1beta1().Deployments(f.KubeNS).Create(d)
	if err != nil {
		return fmt.Errorf("failed to create deployment: %v", err)
	}
	return nil
}

func (f *Framework) DeleteOperator(name string) error {
	err := f.KubeCli.AppsV1beta1().Deployments(f.KubeNS).Delete(name, k8sutil.CascadeDeleteOptions(0))
	if err != nil {
		return err
	}

	// Wait until the etcd-operator pod is actually gone and not just terminating.
	// In upgrade tests, the next test shouldn't see any etcd operator pod.
	lo := metav1.ListOptions{
		LabelSelector: labels.SelectorFromSet(operatorLabelSelector(name)).String(),
	}
	_, err = e2eutil.WaitPodsDeletedCompletely(f.KubeCli, f.KubeNS, 3, lo)
	if err != nil {
		return err
	}
	// This is assumption coupled with endpoint resource lock.
	// TODO: change this if we change to use another kind of lock, e.g. configmap.
	return f.KubeCli.CoreV1().Endpoints(f.KubeNS).Delete("etcd-operator", metav1.NewDeleteOptions(0))
}

func (f *Framework) UpgradeOperator(name string) error {
	uf := func(d *appsv1beta1.Deployment) {
		d.Spec.Template.Spec.Containers[0].Image = f.NewImage
	}
	err := k8sutil.PatchDeployment(f.KubeCli, f.KubeNS, name, uf)
	if err != nil {
		return err
	}

	lo := metav1.ListOptions{
		LabelSelector: labels.SelectorFromSet(operatorLabelSelector(name)).String(),
	}
	_, err = e2eutil.WaitPodsWithImageDeleted(f.KubeCli, f.KubeNS, f.OldImage, 3, lo)
	if err != nil {
		return fmt.Errorf("failed to wait for pod with old image to get deleted: %v", err)
	}
	err = e2eutil.WaitUntilOperatorReady(f.KubeCli, f.KubeNS, name)
	return err
}

func (f *Framework) setupAWS() error {
	if err := os.Setenv("AWS_SHARED_CREDENTIALS_FILE", os.Getenv("AWS_CREDENTIAL")); err != nil {
		return err
	}
	if err := os.Setenv("AWS_CONFIG_FILE", os.Getenv("AWS_CONFIG")); err != nil {
		return err
	}
	sess, err := session.NewSessionWithOptions(session.Options{
		SharedConfigState: session.SharedConfigEnable,
	})
	if err != nil {
		return err
	}
	f.S3Cli = s3.New(sess)
	f.S3Bucket = "jenkins-etcd-operator"
	return nil
}

func operatorLabelSelector(name string) map[string]string {
	return e2eutil.NameLabelSelector(name)
}
<commit_msg>upgradetest/framework: Add reason for deleting Endpoints resource<commit_after>// Copyright 2017 The etcd-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package framework

import (
	"fmt"
	"os"

	"github.com/coreos/etcd-operator/pkg/client"
	"github.com/coreos/etcd-operator/pkg/generated/clientset/versioned"
	"github.com/coreos/etcd-operator/pkg/util/constants"
	"github.com/coreos/etcd-operator/pkg/util/k8sutil"
	"github.com/coreos/etcd-operator/pkg/util/probe"
	"github.com/coreos/etcd-operator/test/e2e/e2eutil"

	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	appsv1beta1 "k8s.io/api/apps/v1beta1"
	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/util/intstr"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/clientcmd"
)

type Config struct {
	// program flags
	KubeConfig string
	KubeNS     string
	OldImage   string
	NewImage   string
}

type Framework struct {
	Config
	// global var
	KubeCli  kubernetes.Interface
	CRClient versioned.Interface
	S3Cli    *s3.S3
	S3Bucket string
}

func New(fc Config) (*Framework, error) {
	kc, err := clientcmd.BuildConfigFromFlags("", fc.KubeConfig)
	if err != nil {
		return nil, err
	}
	kubecli, err := kubernetes.NewForConfig(kc)
	if err != nil {
		return nil, err
	}

	f := &Framework{
		Config:   fc,
		KubeCli:  kubecli,
		CRClient: client.MustNew(kc),
	}
	err = f.setupAWS()
	return f, err
}

func (f *Framework) CreateOperator(name string) error {
	cmd := []string{"/usr/local/bin/etcd-operator"}
	image := f.OldImage
	d := &appsv1beta1.Deployment{
		ObjectMeta: metav1.ObjectMeta{
			Name:      name,
			Namespace: f.KubeNS,
		},
		Spec: appsv1beta1.DeploymentSpec{
			Strategy: appsv1beta1.DeploymentStrategy{
				Type: appsv1beta1.RollingUpdateDeploymentStrategyType,
				RollingUpdate: &appsv1beta1.RollingUpdateDeployment{
					MaxUnavailable: &intstr.IntOrString{Type: intstr.Int, IntVal: 1},
					MaxSurge:       &intstr.IntOrString{Type: intstr.Int, IntVal: 1},
				},
			},
			Selector: &metav1.LabelSelector{MatchLabels: operatorLabelSelector(name)},
			Template: v1.PodTemplateSpec{
				ObjectMeta: metav1.ObjectMeta{
					Labels: operatorLabelSelector(name),
				},
				Spec: v1.PodSpec{
					Containers: []v1.Container{{
						Name:            name,
						Image:           image,
						ImagePullPolicy: v1.PullAlways,
						Command:         cmd,
						Env: []v1.EnvVar{
							{
								Name:      constants.EnvOperatorPodNamespace,
								ValueFrom: &v1.EnvVarSource{FieldRef: &v1.ObjectFieldSelector{FieldPath: "metadata.namespace"}},
							},
							{
								Name:      constants.EnvOperatorPodName,
								ValueFrom: &v1.EnvVarSource{FieldRef: &v1.ObjectFieldSelector{FieldPath: "metadata.name"}},
							},
						},
						ReadinessProbe: &v1.Probe{
							Handler: v1.Handler{
								HTTPGet: &v1.HTTPGetAction{
									Path: probe.HTTPReadyzEndpoint,
									Port: intstr.IntOrString{Type: intstr.Int, IntVal: 8080},
								},
							},
							InitialDelaySeconds: 3,
							PeriodSeconds:       3,
							FailureThreshold:    3,
						},
					}},
				},
			},
		},
	}
	_, err := f.KubeCli.AppsV1beta1().Deployments(f.KubeNS).Create(d)
	if err != nil {
		return fmt.Errorf("failed to create deployment: %v", err)
	}
	return nil
}

func (f *Framework) DeleteOperator(name string) error {
	err := f.KubeCli.AppsV1beta1().Deployments(f.KubeNS).Delete(name, k8sutil.CascadeDeleteOptions(0))
	if err != nil {
		return err
	}

	// Wait until the etcd-operator pod is actually gone and not just terminating.
	// In upgrade tests, the next test shouldn't see any etcd operator pod.
	lo := metav1.ListOptions{
		LabelSelector: labels.SelectorFromSet(operatorLabelSelector(name)).String(),
	}
	_, err = e2eutil.WaitPodsDeletedCompletely(f.KubeCli, f.KubeNS, 3, lo)
	if err != nil {
		return err
	}
	// The deleted operator will not actively release the Endpoints lock causing a non-leader candidate to timeout for the lease duration: 15s
	// Deleting the Endpoints resource simulates the leader actively releasing the lock so that the next candidate avoids the timeout.
	// TODO: change this if we change to use another kind of lock, e.g. configmap.
	return f.KubeCli.CoreV1().Endpoints(f.KubeNS).Delete("etcd-operator", metav1.NewDeleteOptions(0))
}

func (f *Framework) UpgradeOperator(name string) error {
	uf := func(d *appsv1beta1.Deployment) {
		d.Spec.Template.Spec.Containers[0].Image = f.NewImage
	}
	err := k8sutil.PatchDeployment(f.KubeCli, f.KubeNS, name, uf)
	if err != nil {
		return err
	}

	lo := metav1.ListOptions{
		LabelSelector: labels.SelectorFromSet(operatorLabelSelector(name)).String(),
	}
	_, err = e2eutil.WaitPodsWithImageDeleted(f.KubeCli, f.KubeNS, f.OldImage, 3, lo)
	if err != nil {
		return fmt.Errorf("failed to wait for pod with old image to get deleted: %v", err)
	}
	err = e2eutil.WaitUntilOperatorReady(f.KubeCli, f.KubeNS, name)
	return err
}

func (f *Framework) setupAWS() error {
	if err := os.Setenv("AWS_SHARED_CREDENTIALS_FILE", os.Getenv("AWS_CREDENTIAL")); err != nil {
		return err
	}
	if err := os.Setenv("AWS_CONFIG_FILE", os.Getenv("AWS_CONFIG")); err != nil {
		return err
	}
	sess, err := session.NewSessionWithOptions(session.Options{
		SharedConfigState: session.SharedConfigEnable,
	})
	if err != nil {
		return err
	}
	f.S3Cli = s3.New(sess)
	f.S3Bucket = "jenkins-etcd-operator"
	return nil
}

func operatorLabelSelector(name string) map[string]string {
	return e2eutil.NameLabelSelector(name)
}
<|endoftext|>


<commit_before>// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see <http://www.gnu.org/licenses/>.

package modules

import (
	"encoding/json"
	"fmt"
	"github.com/nmeum/marvin/irc"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

type Module interface {
	Name() string
	Help() string
	Load(*irc.Client) error
	Defaults()
}

type ModuleSet struct {
	client  *irc.Client
	modules []Module
	configs string
}

func NewModuleSet(client *irc.Client, configs string) *ModuleSet {
	return &ModuleSet{client: client, configs: configs}
}

func (m *ModuleSet) Register(module Module) {
	m.modules = append(m.modules, module)
}

func (m *ModuleSet) LoadAll() error {
	if err := os.MkdirAll(m.configs, 0755); err != nil {
		return err
	}

	for _, module := range m.modules {
		fn := fmt.Sprintf("%s.json", strings.ToLower(module.Name()))
		fp := filepath.Join(m.configs, fn)

		module.Defaults()
		data, err := ioutil.ReadFile(fp)
		if err == nil {
			if err := json.Unmarshal(data, &module); err != nil {
				return err
			}
		}

		if err := module.Load(m.client); err != nil {
			return err
		}
	}

	m.client.CmdHook("privmsg", m.helpCmd)
	m.client.CmdHook("privmsg", m.moduleCmd)

	return nil
}

func (m *ModuleSet) findModule(name string) Module {
	for _, module := range m.modules {
		if module.Name() == name {
			return module
		}
	}

	return nil
}

func (m *ModuleSet) helpCmd(client *irc.Client, msg irc.Message) error {
	if msg.Data != "!help" && len(m.modules) >= 1 {
		return nil
	}

	var names []string
	for _, module := range m.modules {
		names = append(names, module.Name())
	}

	help := fmt.Sprintf("The following modules are available: %s",
		strings.Join(names, ", "))

	return client.Write("NOTICE %s :%s", msg.Receiver, help)
}

func (m *ModuleSet) moduleCmd(client *irc.Client, msg irc.Message) error {
	splited := strings.Fields(msg.Data)
	if len(splited) < 2 || splited[0] != "!help" {
		return nil
	}

	name := strings.ToLower(splited[1])
	module := m.findModule(name)
	if module == nil {
		return client.Write("NOTICE %s :Module %q isn't installed",
			msg.Receiver, name)
	}

	return client.Write("NOTICE %s :%s module: %s",
		msg.Receiver, module.Name(), module.Help())
}
<commit_msg>Revert "Load configs from lowercased module name"<commit_after>// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see <http://www.gnu.org/licenses/>.

package modules

import (
	"encoding/json"
	"fmt"
	"github.com/nmeum/marvin/irc"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

type Module interface {
	Name() string
	Help() string
	Load(*irc.Client) error
	Defaults()
}

type ModuleSet struct {
	client  *irc.Client
	modules []Module
	configs string
}

func NewModuleSet(client *irc.Client, configs string) *ModuleSet {
	return &ModuleSet{client: client, configs: configs}
}

func (m *ModuleSet) Register(module Module) {
	m.modules = append(m.modules, module)
}

func (m *ModuleSet) LoadAll() error {
	if err := os.MkdirAll(m.configs, 0755); err != nil {
		return err
	}

	for _, module := range m.modules {
		fn := fmt.Sprintf("%s.json", module.Name())
		fp := filepath.Join(m.configs, fn)

		module.Defaults()
		data, err := ioutil.ReadFile(fp)
		if err == nil {
			if err := json.Unmarshal(data, &module); err != nil {
				return err
			}
		}

		if err := module.Load(m.client); err != nil {
			return err
		}
	}

	m.client.CmdHook("privmsg", m.helpCmd)
	m.client.CmdHook("privmsg", m.moduleCmd)

	return nil
}

func (m *ModuleSet) findModule(name string) Module {
	for _, module := range m.modules {
		if module.Name() == name {
			return module
		}
	}

	return nil
}

func (m *ModuleSet) helpCmd(client *irc.Client, msg irc.Message) error {
	if msg.Data != "!help" && len(m.modules) >= 1 {
		return nil
	}

	var names []string
	for _, module := range m.modules {
		names = append(names, module.Name())
	}

	help := fmt.Sprintf("The following modules are available: %s",
		strings.Join(names, ", "))

	return client.Write("NOTICE %s :%s", msg.Receiver, help)
}

func (m *ModuleSet) moduleCmd(client *irc.Client, msg irc.Message) error {
	splited := strings.Fields(msg.Data)
	if len(splited) < 2 || splited[0] != "!help" {
		return nil
	}

	name := strings.ToLower(splited[1])
	module := m.findModule(name)
	if module == nil {
		return client.Write("NOTICE %s :Module %q isn't installed",
			msg.Receiver, name)
	}

	return client.Write("NOTICE %s :%s module: %s",
		msg.Receiver, module.Name(), module.Help())
}
<|endoftext|>