<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package protobuf

import (
	"bytes"
	"errors"
	"fmt"
	"go/ast"
	"go/format"
	"go/parser"
	"go/printer"
	"go/token"
	"io/ioutil"
	"os"
	"reflect"
	"strings"

	customreflect "k8s.io/kubernetes/third_party/forked/golang/reflect"
)

func rewriteFile(name string, header []byte, rewriteFn func(*token.FileSet, *ast.File) error) error {
	fset := token.NewFileSet()
	src, err := ioutil.ReadFile(name)
	if err != nil {
		return err
	}
	file, err := parser.ParseFile(fset, name, src, parser.DeclarationErrors|parser.ParseComments)
	if err != nil {
		return err
	}

	if err := rewriteFn(fset, file); err != nil {
		return err
	}

	b := &bytes.Buffer{}
	b.Write(header)
	if err := printer.Fprint(b, fset, file); err != nil {
		return err
	}

	body, err := format.Source(b.Bytes())
	if err != nil {
		return err
	}

	f, err := os.OpenFile(name, os.O_WRONLY|os.O_TRUNC, 0644)
	if err != nil {
		return err
	}
	defer f.Close()
	if _, err := f.Write(body); err != nil {
		return err
	}
	return f.Close()
}

// ExtractFunc extracts information from the provided TypeSpec and returns true if the type should be
// removed from the destination file.
type ExtractFunc func(*ast.TypeSpec) bool

// OptionalFunc returns true if the provided local name is a type that has protobuf.nullable=true
// and should have its marshal functions adjusted to remove the 'Items' accessor.
type OptionalFunc func(name string) bool

func RewriteGeneratedGogoProtobufFile(name string, extractFn ExtractFunc, optionalFn OptionalFunc, header []byte) error {
	return rewriteFile(name, header, func(fset *token.FileSet, file *ast.File) error {
		cmap := ast.NewCommentMap(fset, file, file.Comments)

		// transform methods that point to optional maps or slices
		for _, d := range file.Decls {
			rewriteOptionalMethods(d, optionalFn)
		}

		// remove types that are already declared
		decls := []ast.Decl{}
		for _, d := range file.Decls {
			if dropExistingTypeDeclarations(d, extractFn) {
				continue
			}
			if dropEmptyImportDeclarations(d) {
				continue
			}
			decls = append(decls, d)
		}
		file.Decls = decls

		// remove unmapped comments
		file.Comments = cmap.Filter(file).Comments()
		return nil
	})
}

// rewriteOptionalMethods makes specific mutations to marshaller methods that belong to types identified
// as being "optional" (they may be nil on the wire). This allows protobuf to serialize a map or slice and
// properly discriminate between empty and nil (which is not possible in protobuf).
// TODO: move into upstream gogo-protobuf once https://github.com/gogo/protobuf/issues/181
//   has agreement
func rewriteOptionalMethods(decl ast.Decl, isOptional OptionalFunc) {
	switch t := decl.(type) {
	case *ast.FuncDecl:
		ident, ptr, ok := receiver(t)
		if !ok {
			return
		}

		// correct initialization of the form `m.Field = &OptionalType{}` to
		// `m.Field = OptionalType{}`
		if t.Name.Name == "Unmarshal" {
			ast.Walk(optionalAssignmentVisitor{fn: isOptional}, t.Body)
		}

		if !isOptional(ident.Name) {
			return
		}

		switch t.Name.Name {
		case "Unmarshal":
			ast.Walk(&optionalItemsVisitor{}, t.Body)
		case "MarshalTo", "Size":
			ast.Walk(&optionalItemsVisitor{}, t.Body)
			fallthrough
		case "Marshal":
			// if the method has a pointer receiver, set it back to a normal receiver
			if ptr {
				t.Recv.List[0].Type = ident
			}
		}
	}
}

type optionalAssignmentVisitor struct {
	fn OptionalFunc
}

// Visit walks the provided node, transforming field initializations of the form
// m.Field = &OptionalType{} -> m.Field = OptionalType{}
func (v optionalAssignmentVisitor) Visit(n ast.Node) ast.Visitor {
	switch t := n.(type) {
	case *ast.AssignStmt:
		if len(t.Lhs) == 1 && len(t.Rhs) == 1 {
			if !isFieldSelector(t.Lhs[0], "m", "") {
				return nil
			}
			unary, ok := t.Rhs[0].(*ast.UnaryExpr)
			if !ok || unary.Op != token.AND {
				return nil
			}
			composite, ok := unary.X.(*ast.CompositeLit)
			if !ok || composite.Type == nil || len(composite.Elts) != 0 {
				return nil
			}
			if ident, ok := composite.Type.(*ast.Ident); ok && v.fn(ident.Name) {
				t.Rhs[0] = composite
			}
		}
		return nil
	}
	return v
}

type optionalItemsVisitor struct{}

// Visit walks the provided node, looking for specific patterns to transform that match
// the effective outcome of turning struct{ map[x]y || []x } into map[x]y or []x.
func (v *optionalItemsVisitor) Visit(n ast.Node) ast.Visitor {
	switch t := n.(type) {
	case *ast.RangeStmt:
		if isFieldSelector(t.X, "m", "Items") {
			t.X = &ast.Ident{Name: "m"}
		}
	case *ast.AssignStmt:
		if len(t.Lhs) == 1 && len(t.Rhs) == 1 {
			switch lhs := t.Lhs[0].(type) {
			case *ast.IndexExpr:
				if isFieldSelector(lhs.X, "m", "Items") {
					lhs.X = &ast.StarExpr{X: &ast.Ident{Name: "m"}}
				}
			default:
				if isFieldSelector(t.Lhs[0], "m", "Items") {
					t.Lhs[0] = &ast.StarExpr{X: &ast.Ident{Name: "m"}}
				}
			}
			switch rhs := t.Rhs[0].(type) {
			case *ast.CallExpr:
				if ident, ok := rhs.Fun.(*ast.Ident); ok && ident.Name == "append" {
					ast.Walk(v, rhs)
					if len(rhs.Args) > 0 {
						switch arg := rhs.Args[0].(type) {
						case *ast.Ident:
							if arg.Name == "m" {
								rhs.Args[0] = &ast.StarExpr{X: &ast.Ident{Name: "m"}}
							}
						}
					}
					return nil
				}
			}
		}
	case *ast.IfStmt:
		if b, ok := t.Cond.(*ast.BinaryExpr); ok && b.Op == token.EQL {
			if isFieldSelector(b.X, "m", "Items") && isIdent(b.Y, "nil") {
				b.X = &ast.StarExpr{X: &ast.Ident{Name: "m"}}
			}
		}
	case *ast.IndexExpr:
		if isFieldSelector(t.X, "m", "Items") {
			t.X = &ast.Ident{Name: "m"}
			return nil
		}
	case *ast.CallExpr:
		changed := false
		for i := range t.Args {
			if isFieldSelector(t.Args[i], "m", "Items") {
				t.Args[i] = &ast.Ident{Name: "m"}
				changed = true
			}
		}
		if changed {
			return nil
		}
	}
	return v
}

func isFieldSelector(n ast.Expr, name, field string) bool {
	s, ok := n.(*ast.SelectorExpr)
	if !ok || s.Sel == nil || (field != "" && s.Sel.Name != field) {
		return false
	}
	return isIdent(s.X, name)
}

func isIdent(n ast.Expr, value string) bool {
	ident, ok := n.(*ast.Ident)
	return ok && ident.Name == value
}

func receiver(f *ast.FuncDecl) (ident *ast.Ident, pointer bool, ok bool) {
	if f.Recv == nil || len(f.Recv.List) != 1 {
		return nil, false, false
	}
	switch t := f.Recv.List[0].Type.(type) {
	case *ast.StarExpr:
		identity, ok := t.X.(*ast.Ident)
		if !ok {
			return nil, false, false
		}
		return identity, true, true
	case *ast.Ident:
		return t, false, true
	}
	return nil, false, false
}

// dropExistingTypeDeclarations removes any type declaration for which extractFn returns true. The function
// returns true if the entire declaration should be dropped.
func dropExistingTypeDeclarations(decl ast.Decl, extractFn ExtractFunc) bool {
	switch t := decl.(type) {
	case *ast.GenDecl:
		if t.Tok != token.TYPE {
			return false
		}
		specs := []ast.Spec{}
		for _, s := range t.Specs {
			switch spec := s.(type) {
			case *ast.TypeSpec:
				if extractFn(spec) {
					continue
				}
				specs = append(specs, spec)
			}
		}
		if len(specs) == 0 {
			return true
		}
		t.Specs = specs
	}
	return false
}

// dropEmptyImportDeclarations strips any generated but no-op imports from the generated code
// to prevent generation from being able to define side-effects.  The function returns true
// if the entire declaration should be dropped.
func dropEmptyImportDeclarations(decl ast.Decl) bool {
	switch t := decl.(type) {
	case *ast.GenDecl:
		if t.Tok != token.IMPORT {
			return false
		}
		specs := []ast.Spec{}
		for _, s := range t.Specs {
			switch spec := s.(type) {
			case *ast.ImportSpec:
				if spec.Name != nil && spec.Name.Name == "_" {
					continue
				}
				specs = append(specs, spec)
			}
		}
		if len(specs) == 0 {
			return true
		}
		t.Specs = specs
	}
	return false
}

func RewriteTypesWithProtobufStructTags(name string, structTags map[string]map[string]string) error {
	return rewriteFile(name, []byte{}, func(fset *token.FileSet, file *ast.File) error {
		allErrs := []error{}

		// set any new struct tags
		for _, d := range file.Decls {
			if errs := updateStructTags(d, structTags, []string{"protobuf"}); len(errs) > 0 {
				allErrs = append(allErrs, errs...)
			}
		}

		if len(allErrs) > 0 {
			var s string
			for _, err := range allErrs {
				s += err.Error() + "\n"
			}
			return errors.New(s)
		}
		return nil
	})
}

func updateStructTags(decl ast.Decl, structTags map[string]map[string]string, toCopy []string) []error {
	var errs []error
	t, ok := decl.(*ast.GenDecl)
	if !ok {
		return nil
	}
	if t.Tok != token.TYPE {
		return nil
	}

	for _, s := range t.Specs {
		spec, ok := s.(*ast.TypeSpec)
		if !ok {
			continue
		}
		typeName := spec.Name.Name
		fieldTags, ok := structTags[typeName]
		if !ok {
			continue
		}
		st, ok := spec.Type.(*ast.StructType)
		if !ok {
			continue
		}

		for i := range st.Fields.List {
			f := st.Fields.List[i]
			var name string
			if len(f.Names) == 0 {
				switch t := f.Type.(type) {
				case *ast.Ident:
					name = t.Name
				case *ast.SelectorExpr:
					name = t.Sel.Name
				default:
					errs = append(errs, fmt.Errorf("unable to get name for tag from struct %q, field %#v", spec.Name.Name, t))
					continue
				}
			} else {
				name = f.Names[0].Name
			}
			value, ok := fieldTags[name]
			if !ok {
				continue
			}
			var tags customreflect.StructTags
			if f.Tag != nil {
				oldTags, err := customreflect.ParseStructTags(strings.Trim(f.Tag.Value, "`"))
				if err != nil {
					errs = append(errs, fmt.Errorf("unable to read struct tag from struct %q, field %q: %v", spec.Name.Name, name, err))
					continue
				}
				tags = oldTags
			}
			for _, name := range toCopy {
				// don't overwrite existing tags
				if tags.Has(name) {
					continue
				}
				// append new tags
				if v := reflect.StructTag(value).Get(name); len(v) > 0 {
					tags = append(tags, customreflect.StructTag{Name: name, Value: v})
				}
			}
			if len(tags) == 0 {
				continue
			}
			if f.Tag == nil {
				f.Tag = &ast.BasicLit{}
			}
			f.Tag.Value = tags.String()
		}
	}
	return errs
}
<commit_msg>Unable to have optional message slice<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package protobuf

import (
	"bytes"
	"errors"
	"fmt"
	"go/ast"
	"go/format"
	"go/parser"
	"go/printer"
	"go/token"
	"io/ioutil"
	"os"
	"reflect"
	"strings"

	customreflect "k8s.io/kubernetes/third_party/forked/golang/reflect"
)

func rewriteFile(name string, header []byte, rewriteFn func(*token.FileSet, *ast.File) error) error {
	fset := token.NewFileSet()
	src, err := ioutil.ReadFile(name)
	if err != nil {
		return err
	}
	file, err := parser.ParseFile(fset, name, src, parser.DeclarationErrors|parser.ParseComments)
	if err != nil {
		return err
	}

	if err := rewriteFn(fset, file); err != nil {
		return err
	}

	b := &bytes.Buffer{}
	b.Write(header)
	if err := printer.Fprint(b, fset, file); err != nil {
		return err
	}

	body, err := format.Source(b.Bytes())
	if err != nil {
		return err
	}

	f, err := os.OpenFile(name, os.O_WRONLY|os.O_TRUNC, 0644)
	if err != nil {
		return err
	}
	defer f.Close()
	if _, err := f.Write(body); err != nil {
		return err
	}
	return f.Close()
}

// ExtractFunc extracts information from the provided TypeSpec and returns true if the type should be
// removed from the destination file.
type ExtractFunc func(*ast.TypeSpec) bool

// OptionalFunc returns true if the provided local name is a type that has protobuf.nullable=true
// and should have its marshal functions adjusted to remove the 'Items' accessor.
type OptionalFunc func(name string) bool

func RewriteGeneratedGogoProtobufFile(name string, extractFn ExtractFunc, optionalFn OptionalFunc, header []byte) error {
	return rewriteFile(name, header, func(fset *token.FileSet, file *ast.File) error {
		cmap := ast.NewCommentMap(fset, file, file.Comments)

		// transform methods that point to optional maps or slices
		for _, d := range file.Decls {
			rewriteOptionalMethods(d, optionalFn)
		}

		// remove types that are already declared
		decls := []ast.Decl{}
		for _, d := range file.Decls {
			if dropExistingTypeDeclarations(d, extractFn) {
				continue
			}
			if dropEmptyImportDeclarations(d) {
				continue
			}
			decls = append(decls, d)
		}
		file.Decls = decls

		// remove unmapped comments
		file.Comments = cmap.Filter(file).Comments()
		return nil
	})
}

// rewriteOptionalMethods makes specific mutations to marshaller methods that belong to types identified
// as being "optional" (they may be nil on the wire). This allows protobuf to serialize a map or slice and
// properly discriminate between empty and nil (which is not possible in protobuf).
// TODO: move into upstream gogo-protobuf once https://github.com/gogo/protobuf/issues/181
//   has agreement
func rewriteOptionalMethods(decl ast.Decl, isOptional OptionalFunc) {
	switch t := decl.(type) {
	case *ast.FuncDecl:
		ident, ptr, ok := receiver(t)
		if !ok {
			return
		}

		// correct initialization of the form `m.Field = &OptionalType{}` to
		// `m.Field = OptionalType{}`
		if t.Name.Name == "Unmarshal" {
			ast.Walk(optionalAssignmentVisitor{fn: isOptional}, t.Body)
		}

		if !isOptional(ident.Name) {
			return
		}

		switch t.Name.Name {
		case "Unmarshal":
			ast.Walk(&optionalItemsVisitor{}, t.Body)
		case "MarshalTo", "Size":
			ast.Walk(&optionalItemsVisitor{}, t.Body)
			fallthrough
		case "Marshal":
			// if the method has a pointer receiver, set it back to a normal receiver
			if ptr {
				t.Recv.List[0].Type = ident
			}
		}
	}
}

type optionalAssignmentVisitor struct {
	fn OptionalFunc
}

// Visit walks the provided node, transforming field initializations of the form
// m.Field = &OptionalType{} -> m.Field = OptionalType{}
func (v optionalAssignmentVisitor) Visit(n ast.Node) ast.Visitor {
	switch t := n.(type) {
	case *ast.AssignStmt:
		if len(t.Lhs) == 1 && len(t.Rhs) == 1 {
			if !isFieldSelector(t.Lhs[0], "m", "") {
				return nil
			}
			unary, ok := t.Rhs[0].(*ast.UnaryExpr)
			if !ok || unary.Op != token.AND {
				return nil
			}
			composite, ok := unary.X.(*ast.CompositeLit)
			if !ok || composite.Type == nil || len(composite.Elts) != 0 {
				return nil
			}
			if ident, ok := composite.Type.(*ast.Ident); ok && v.fn(ident.Name) {
				t.Rhs[0] = composite
			}
		}
		return nil
	}
	return v
}

type optionalItemsVisitor struct{}

// Visit walks the provided node, looking for specific patterns to transform that match
// the effective outcome of turning struct{ map[x]y || []x } into map[x]y or []x.
func (v *optionalItemsVisitor) Visit(n ast.Node) ast.Visitor {
	switch t := n.(type) {
	case *ast.RangeStmt:
		if isFieldSelector(t.X, "m", "Items") {
			t.X = &ast.Ident{Name: "m"}
		}
	case *ast.AssignStmt:
		if len(t.Lhs) == 1 && len(t.Rhs) == 1 {
			switch lhs := t.Lhs[0].(type) {
			case *ast.IndexExpr:
				if isFieldSelector(lhs.X, "m", "Items") {
					lhs.X = &ast.StarExpr{X: &ast.Ident{Name: "m"}}
				}
			default:
				if isFieldSelector(t.Lhs[0], "m", "Items") {
					t.Lhs[0] = &ast.StarExpr{X: &ast.Ident{Name: "m"}}
				}
			}
			switch rhs := t.Rhs[0].(type) {
			case *ast.CallExpr:
				if ident, ok := rhs.Fun.(*ast.Ident); ok && ident.Name == "append" {
					ast.Walk(v, rhs)
					if len(rhs.Args) > 0 {
						switch arg := rhs.Args[0].(type) {
						case *ast.Ident:
							if arg.Name == "m" {
								rhs.Args[0] = &ast.StarExpr{X: &ast.Ident{Name: "m"}}
							}
						}
					}
					return nil
				}
			}
		}
	case *ast.IfStmt:
		switch cond := t.Cond.(type) {
		case *ast.BinaryExpr:
			if cond.Op == token.EQL {
				if isFieldSelector(cond.X, "m", "Items") && isIdent(cond.Y, "nil") {
					cond.X = &ast.StarExpr{X: &ast.Ident{Name: "m"}}
				}
			}
		}
		if t.Init != nil {
			// Find form:
			// if err := m[len(m.Items)-1].Unmarshal(data[iNdEx:postIndex]); err != nil {
			// 	return err
			// }
			switch s := t.Init.(type) {
			case *ast.AssignStmt:
				if call, ok := s.Rhs[0].(*ast.CallExpr); ok {
					if sel, ok := call.Fun.(*ast.SelectorExpr); ok {
						if x, ok := sel.X.(*ast.IndexExpr); ok {
							// m[] -> (*m)[]
							if sel2, ok := x.X.(*ast.SelectorExpr); ok {
								if ident, ok := sel2.X.(*ast.Ident); ok && ident.Name == "m" {
									x.X = &ast.StarExpr{X: &ast.Ident{Name: "m"}}
								}
							}
							// len(m.Items) -> len(*m)
							if bin, ok := x.Index.(*ast.BinaryExpr); ok {
								if call2, ok := bin.X.(*ast.CallExpr); ok && len(call2.Args) == 1 {
									if isFieldSelector(call2.Args[0], "m", "Items") {
										call2.Args[0] = &ast.StarExpr{X: &ast.Ident{Name: "m"}}
									}
								}
							}
						}
					}
				}
			}
		}
	case *ast.IndexExpr:
		if isFieldSelector(t.X, "m", "Items") {
			t.X = &ast.Ident{Name: "m"}
			return nil
		}
	case *ast.CallExpr:
		changed := false
		for i := range t.Args {
			if isFieldSelector(t.Args[i], "m", "Items") {
				t.Args[i] = &ast.Ident{Name: "m"}
				changed = true
			}
		}
		if changed {
			return nil
		}
	}
	return v
}

func isFieldSelector(n ast.Expr, name, field string) bool {
	s, ok := n.(*ast.SelectorExpr)
	if !ok || s.Sel == nil || (field != "" && s.Sel.Name != field) {
		return false
	}
	return isIdent(s.X, name)
}

func isIdent(n ast.Expr, value string) bool {
	ident, ok := n.(*ast.Ident)
	return ok && ident.Name == value
}

func receiver(f *ast.FuncDecl) (ident *ast.Ident, pointer bool, ok bool) {
	if f.Recv == nil || len(f.Recv.List) != 1 {
		return nil, false, false
	}
	switch t := f.Recv.List[0].Type.(type) {
	case *ast.StarExpr:
		identity, ok := t.X.(*ast.Ident)
		if !ok {
			return nil, false, false
		}
		return identity, true, true
	case *ast.Ident:
		return t, false, true
	}
	return nil, false, false
}

// dropExistingTypeDeclarations removes any type declaration for which extractFn returns true. The function
// returns true if the entire declaration should be dropped.
func dropExistingTypeDeclarations(decl ast.Decl, extractFn ExtractFunc) bool {
	switch t := decl.(type) {
	case *ast.GenDecl:
		if t.Tok != token.TYPE {
			return false
		}
		specs := []ast.Spec{}
		for _, s := range t.Specs {
			switch spec := s.(type) {
			case *ast.TypeSpec:
				if extractFn(spec) {
					continue
				}
				specs = append(specs, spec)
			}
		}
		if len(specs) == 0 {
			return true
		}
		t.Specs = specs
	}
	return false
}

// dropEmptyImportDeclarations strips any generated but no-op imports from the generated code
// to prevent generation from being able to define side-effects.  The function returns true
// if the entire declaration should be dropped.
func dropEmptyImportDeclarations(decl ast.Decl) bool {
	switch t := decl.(type) {
	case *ast.GenDecl:
		if t.Tok != token.IMPORT {
			return false
		}
		specs := []ast.Spec{}
		for _, s := range t.Specs {
			switch spec := s.(type) {
			case *ast.ImportSpec:
				if spec.Name != nil && spec.Name.Name == "_" {
					continue
				}
				specs = append(specs, spec)
			}
		}
		if len(specs) == 0 {
			return true
		}
		t.Specs = specs
	}
	return false
}

func RewriteTypesWithProtobufStructTags(name string, structTags map[string]map[string]string) error {
	return rewriteFile(name, []byte{}, func(fset *token.FileSet, file *ast.File) error {
		allErrs := []error{}

		// set any new struct tags
		for _, d := range file.Decls {
			if errs := updateStructTags(d, structTags, []string{"protobuf"}); len(errs) > 0 {
				allErrs = append(allErrs, errs...)
			}
		}

		if len(allErrs) > 0 {
			var s string
			for _, err := range allErrs {
				s += err.Error() + "\n"
			}
			return errors.New(s)
		}
		return nil
	})
}

func updateStructTags(decl ast.Decl, structTags map[string]map[string]string, toCopy []string) []error {
	var errs []error
	t, ok := decl.(*ast.GenDecl)
	if !ok {
		return nil
	}
	if t.Tok != token.TYPE {
		return nil
	}

	for _, s := range t.Specs {
		spec, ok := s.(*ast.TypeSpec)
		if !ok {
			continue
		}
		typeName := spec.Name.Name
		fieldTags, ok := structTags[typeName]
		if !ok {
			continue
		}
		st, ok := spec.Type.(*ast.StructType)
		if !ok {
			continue
		}

		for i := range st.Fields.List {
			f := st.Fields.List[i]
			var name string
			if len(f.Names) == 0 {
				switch t := f.Type.(type) {
				case *ast.Ident:
					name = t.Name
				case *ast.SelectorExpr:
					name = t.Sel.Name
				default:
					errs = append(errs, fmt.Errorf("unable to get name for tag from struct %q, field %#v", spec.Name.Name, t))
					continue
				}
			} else {
				name = f.Names[0].Name
			}
			value, ok := fieldTags[name]
			if !ok {
				continue
			}
			var tags customreflect.StructTags
			if f.Tag != nil {
				oldTags, err := customreflect.ParseStructTags(strings.Trim(f.Tag.Value, "`"))
				if err != nil {
					errs = append(errs, fmt.Errorf("unable to read struct tag from struct %q, field %q: %v", spec.Name.Name, name, err))
					continue
				}
				tags = oldTags
			}
			for _, name := range toCopy {
				// don't overwrite existing tags
				if tags.Has(name) {
					continue
				}
				// append new tags
				if v := reflect.StructTag(value).Get(name); len(v) > 0 {
					tags = append(tags, customreflect.StructTag{Name: name, Value: v})
				}
			}
			if len(tags) == 0 {
				continue
			}
			if f.Tag == nil {
				f.Tag = &ast.BasicLit{}
			}
			f.Tag.Value = tags.String()
		}
	}
	return errs
}
<|endoftext|>


<commit_before>package sqlpaxos

import "net"
import "fmt"
import "net/rpc"
import "log"
import "time"
import "paxos"
import "sync"
import "os"
import "syscall"
import "encoding/gob"
import "math/rand"
import "strconv"
import "math"
import "barista"
import "encoding/json"
import "logger"
import "db"

const Debug=0

func DPrintf(format string, a ...interface{}) (n int, err error) {
        if Debug > 0 {
                log.Printf(format, a...)
        }
        return
}

type LastSeen struct {
  RequestId int
  Reply interface{}
}

type SQLPaxos struct {
  mu sync.Mutex
  l net.Listener
  me int
  dead bool // for testing
  unreliable bool // for testing
  px *paxos.Paxos

  // Your definitions here.
  ops map[int]Op // log of operations
  replies map[int]interface{} // the replies for this sequence number
  done map[int]bool // true if we can delete the data for this sequence number
  data map[string]string // the database
  lastSeen map[int64]LastSeen // the last request/reply for this client
  connections map[int64]*db.DBManager // connections per client. Limited to a single connection per client
  next int // the next sequence number to be executed
  logger *logger.Logger // logger to write paxos log to file
}

func (sp *SQLPaxos) execute(op Op) interface{} {

  testing := false
  if testing {
    args := op.Args
    reply := ExecReply{}

    // @TODO remove this
    if op.NoOp {
       return reply
    }

    // @TODO remove get & put
    key := args.(ExecArgs).Key
    if args.(ExecArgs).Type == Put {
       // execute the put

       prevValue, ok := sp.data[key]
       if ok {
          reply.Value = prevValue
       } else {
          reply.Value = ""
       }

       if args.(ExecArgs).DoHash {
          sp.data[key] = strconv.Itoa(int(hash(reply.Value + args.(ExecArgs).Value)))
       } else {
          sp.data[key] = args.(ExecArgs).Value
       }

       reply.Err = OK

    } else if args.(ExecArgs).Type == Get {
       // execute the get

       value, ok := sp.data[key]
       if ok {
          reply.Value = value
          reply.Err = OK
       } else {
          reply.Value = ""
          reply.Err = ErrNoKey
       }
    }

    return reply

  } else {
    // not testing

    // write op to file
    err := sp.WriteToLog(op)
    if err != nil {
      // log something
    }

    switch {
    case op.Type == Open:
      return sp.OpenHelper(op.Args.(OpenArgs), op.SeqNum)
    case op.Type == Close:
      return sp.CloseHelper(op.Args.(CloseArgs), op.SeqNum)
    case op.Type == Execute:
      return sp.ExecuteHelper(op.Args.(ExecArgs), op.SeqNum)
    }
  }
  return nil
}

func (sp *SQLPaxos) WriteToLog(op Op) error {
  b, err := json.Marshal(op)
  if err != nil {
    return err
  }
  return sp.logger.WriteToLog(b)
}

func (sp *SQLPaxos) ExecuteHelper(args ExecArgs, seqnum int) ExecReply {
  rows, columns, err := sp.UpdateDatabase(args.ClientId, args.Query, args.QueryParams, seqnum)
  if err != OK {
    // log something
    return ExecReply{Value:"", Err:err}
  }

  tuples := []*barista.Tuple{}
  for _, row := range rows {
    tuple := barista.Tuple{Cells: &row}
    tuples = append(tuples, &tuple)
  }

  result_set := new(barista.ResultSet)
  //result_set.Con = con. @TODO: this will not be populating this
  result_set.Tuples = &tuples
  result_set.FieldNames = &columns
  return ExecReply{Result:result_set, Err:OK}
}

func (sp *SQLPaxos) OpenHelper(args OpenArgs, seqnum int) OpenReply {
  reply := OpenReply{}
  _, ok := sp.connections[args.ClientId]
  if ok {
    reply.Err = ConnAlreadyOpen
  } else {
      manager := new(db.DBManager)
      reply.Err = errorToErr(manager.OpenConnection(args.User, args.Password, args.Database))
      sp.connections[args.ClientId] = manager
  }
  _, _, err := sp.UpdateDatabase(args.ClientId, "", nil, seqnum)
  if err != OK {
    // log something
  }
  return reply
}

func errorToErr(error error) Err {
  if error != nil {
    return Err(error.Error())
  } else {
    return OK
  }
}

func (sp *SQLPaxos) CloseHelper(args CloseArgs, seqnum int) CloseReply {
  _, _, err := sp.UpdateDatabase(args.ClientId, "", nil, seqnum)
  reply := CloseReply{}
  _, ok := sp.connections[args.ClientId]
  if !ok {
    reply.Err = ConnAlreadyClosed
  } else {
    reply.Err = errorToErr(sp.connections[args.ClientId].CloseConnection())
    delete(sp.connections, args.ClientId) //only delete on successful close?
  }
  if err != OK {
    // log something
  }
  return reply
}

// note that NoOps don't update the state table
func (sp *SQLPaxos) UpdateDatabase(clientId int64, query string, query_params [][]byte, seqnum int) ([][][]byte, []string, Err) {
  tx, err := sp.connections[clientId].BeginTxn()

  if err != nil || tx == nil {
     return make([][][]byte), make([]string), errorToErr(err)
  }

  rows, columns, error := sp.connections[clientId].QueryTxn(query, query_params, tx)

  update := "UPDATE SQLPaxosLog SET lastSeqNum=" + strconv.Itoa(seqnum) + ";"
  sp.connections[clientId].ExecTxn(update, query_params, tx)

  sp.connections[clientId].EndTxn(tx)

  return rows, columns, errorToErr(error)
}

func (sp *SQLPaxos) fillHoles(next int, seq int) interface{} {

  var reply interface{}

  // make sure there are no holes in the log before our operation
  for i := next; i <= seq; i++ {
     nwaits := 0
     for !sp.dead {
	if _, ok := sp.ops[i]; ok || sp.next > i {
      	   break
        }

        decided, v_i := sp.px.Status(i)
        if decided {
           // the operation in slot i has been decided
           sp.ops[i] = v_i.(Op)
           break
        } else {
           nwaits++
           sp.mu.Unlock()
           if nwaits == 5 || nwaits == 10 {
              // propose a no-op
              sp.px.Start(i, Op{NoOp: true})
           } else if nwaits > 10 {
              time.Sleep(100 * time.Millisecond)
           } else {
              time.Sleep(10 * time.Millisecond)
           }
           sp.mu.Lock()
        }
     }

     if i == sp.next {
        // the operation at slot i is next to be executed
	r, executed := sp.checkIfExecuted(sp.ops[i])
        if executed {
    	   sp.replies[i] = r
	} else {
	   r := sp.execute(sp.ops[i])
	   sp.replies[i] = r
	   sp.lastSeen[getOpClientId(sp.ops[i])] = LastSeen{ RequestId: getOpRequestId(sp.ops[i]), Reply: r }
	}
        sp.next++
     }

     if i == seq {
        reply = sp.replies[i]
     }
  }

  return reply
}

func getOpClientId(op Op) int64 {
  switch {
  case op.Type == Open:
    return op.Args.(OpenArgs).ClientId;
  case op.Type == Close:
    return op.Args.(CloseArgs).ClientId;
  case op.Type == Execute:
    return op.Args.(ExecArgs).ClientId;
  }
  return -1;
}

func getOpRequestId(op Op) int {
  switch {
  case op.Type == Open:
    return op.Args.(OpenArgs).RequestId;
  case op.Type == Close:
    return op.Args.(CloseArgs).RequestId;
  case op.Type == Execute:
    return op.Args.(ExecArgs).RequestId;
  }
  return -1;
}

// @TODO: update to support multiple types of operations
func (sp *SQLPaxos) checkIfExecuted(op Op) (interface{}, bool) {
  // need some casting here
  lastSeen, ok := sp.lastSeen[getOpClientId(op)]
  if ok {
     if lastSeen.RequestId == getOpRequestId(op) {
        return lastSeen.Reply, true
     } else if lastSeen.RequestId > getOpRequestId(op) {
        return nil, true // empty reply since this is an old request
     }
  }

  return nil, false
}

func (sp *SQLPaxos) reserveSlot(op Op) int {

  // propose this operation for slot seq
  seq := sp.px.Max() + 1
  v := op
  sp.px.Start(seq, v)

  nwaits := 0
  for !sp.dead {
     decided, v_a := sp.px.Status(seq)
     if decided && v_a != nil && getOpClientId(v_a.(Op)) == getOpClientId(v) &&
       getOpRequestId(v_a.(Op)) == getOpRequestId(v) {
        // we successfully claimed this slot for our operation
        if _, ok := sp.ops[seq]; !ok {
	   v.SeqNum = seq
           sp.ops[seq] = v
        }
        break
     } else if decided {
        // another proposer got this slot, so try to get our operation in a new slot
        seq = int(math.Max(float64(sp.px.Max() + 1), float64(seq + 1)))
        sp.px.Start(seq, v)
        nwaits = 0
     } else {
        nwaits++
  	sp.mu.Unlock()
        if nwaits == 5 || nwaits == 10 {
           // re-propose our operation
           sp.px.Start(seq, v)
     	} else if nwaits > 10 {
           time.Sleep(100 * time.Millisecond)
        } else {
           time.Sleep(10 * time.Millisecond)
     	}
  	sp.mu.Lock()
     }
  }
  v.SeqNum = seq // update sequence number
  return seq
}

func (sp *SQLPaxos) freeMemory(seq int) {

  sp.done[seq] = true
  minNotDone := seq + 1
  for i := seq; i >= 0; i-- {
     _, ok := sp.ops[i]
     if ok {
        if done, ok := sp.done[i]; ok && done || sp.ops[i].NoOp {
           delete(sp.ops, i)
           delete(sp.replies, i)
           delete(sp.done, i)
        } else {
           minNotDone = i
        }
     }
  }

  sp.px.Done(minNotDone - 1)
}

//@Make it work for multiple types of arguments
func (sp *SQLPaxos) commit(op Op) interface{} {

  sp.mu.Lock()
  defer sp.mu.Unlock()

  // first check if this request has already been executed
  reply, ok := sp.checkIfExecuted(op)
  if ok {
     return reply
  }

  // reserve a slot in the paxos log for this operation
  seq := sp.reserveSlot(op)

  next := sp.next
  if next > seq {
     // our operation has already been executed
     reply = sp.replies[seq]
  } else {
     // fill holes in the log and execute our operation
     reply = sp.fillHoles(next, seq)
  }

  // delete un-needed log entries to free up memory
  sp.freeMemory(seq)

  return reply
}

func (sp *SQLPaxos) ExecuteSQL(args *ExecArgs, reply *ExecReply) error {
  // execute this operation and store the response in r
  op := Op{Type:Execute, Args: *args}
  r := sp.commit(op)

  if r != nil {
     reply.Value = r.(ExecReply).Value
     reply.Err = r.(ExecReply).Err
  }

  return nil
}

// open the connection to the database
func (sp *SQLPaxos) Open(args *OpenArgs, reply *OpenReply) error {
  // execute this operation and store the response in r
  op := Op{Type:Open, Args: *args}
  r := sp.commit(op)

  if r != nil {
     reply.Err = r.(OpenReply).Err
  }

  return nil
}

// close the connection to the database
func (sp *SQLPaxos) Close(args *CloseArgs, reply *CloseReply) error {
  // execute this operation and store the response in r
  op := Op{Type:Close, Args: *args}
  r := sp.commit(op)

  if r != nil {
     reply.Err = r.(CloseReply).Err
  }

  return nil
}

// tell the server to shut itself down.
func (sp *SQLPaxos) kill() {
  sp.dead = true
  sp.l.Close()
  sp.px.Kill()
}

//
// servers[] contains the ports of the set of
// servers that will cooperate via Paxos to
// form the fault-tolerant key/value service.
// me is the index of the current server in servers[].
//
func StartServer(servers []string, me int) *SQLPaxos {
  // call gob.Register on structures you want
  // Go's RPC library to marshall/unmarshall.
  gob.Register(Op{})
  gob.Register(ExecArgs{})

  sp := new(SQLPaxos)
  sp.me = me

  // Your initialization code here.
  sp.ops = make(map[int]Op)
  sp.data = make(map[string]string)
  sp.replies = make(map[int]interface{})
  sp.done = make(map[int]bool)
  sp.lastSeen = make(map[int64]LastSeen)
  sp.next = 0
  sp.connections = make(map[int64]*db.DBManager)
  sp.logger = logger.Make("sqlpaxos_log.txt")

  rpcs := rpc.NewServer()
  rpcs.Register(sp)

  sp.px = paxos.Make(servers, me, rpcs)

  os.Remove(servers[me])
  l, e := net.Listen("unix", servers[me]);
  if e != nil {
    log.Fatal("listen error: ", e);
  }
  sp.l = l

  // please do not change any of the following code,
  // or do anything to subvert it.

  go func() {
    for sp.dead == false {
      conn, err := sp.l.Accept()
      if err == nil && sp.dead == false {
      if sp.unreliable && (rand.Int63() % 1000) < 100 {
          // discard the request.
          conn.Close()
        } else if sp.unreliable && (rand.Int63() % 1000) < 200 {
          // process the request but force discard of reply.
          c1 := conn.(*net.UnixConn)
          f, _ := c1.File()
          err := syscall.Shutdown(int(f.Fd()), syscall.SHUT_WR)
          if err != nil {
            fmt.Printf("shutdown: %v\n", err)
          }
          go rpcs.ServeConn(conn)
        } else {
          go rpcs.ServeConn(conn)
        }
      } else if err == nil {
        conn.Close()
      }
      if err != nil && sp.dead == false {
        fmt.Printf("SQLPaxos(%v) accept: %v\n", me, err.Error())
	sp.kill()
      }
    }
  }()

  return sp
}

<commit_msg>minor fix<commit_after>package sqlpaxos

import "net"
import "fmt"
import "net/rpc"
import "log"
import "time"
import "paxos"
import "sync"
import "os"
import "syscall"
import "encoding/gob"
import "math/rand"
import "strconv"
import "math"
import "barista"
import "encoding/json"
import "logger"
import "db"

const Debug=0

func DPrintf(format string, a ...interface{}) (n int, err error) {
        if Debug > 0 {
                log.Printf(format, a...)
        }
        return
}

type LastSeen struct {
  RequestId int
  Reply interface{}
}

type SQLPaxos struct {
  mu sync.Mutex
  l net.Listener
  me int
  dead bool // for testing
  unreliable bool // for testing
  px *paxos.Paxos

  // Your definitions here.
  ops map[int]Op // log of operations
  replies map[int]interface{} // the replies for this sequence number
  done map[int]bool // true if we can delete the data for this sequence number
  data map[string]string // the database
  lastSeen map[int64]LastSeen // the last request/reply for this client
  connections map[int64]*db.DBManager // connections per client. Limited to a single connection per client
  next int // the next sequence number to be executed
  logger *logger.Logger // logger to write paxos log to file
}

func (sp *SQLPaxos) execute(op Op) interface{} {

  testing := false
  if testing {
    args := op.Args
    reply := ExecReply{}

    // @TODO remove this
    if op.NoOp {
       return reply
    }

    // @TODO remove get & put
    key := args.(ExecArgs).Key
    if args.(ExecArgs).Type == Put {
       // execute the put

       prevValue, ok := sp.data[key]
       if ok {
          reply.Value = prevValue
       } else {
          reply.Value = ""
       }

       if args.(ExecArgs).DoHash {
          sp.data[key] = strconv.Itoa(int(hash(reply.Value + args.(ExecArgs).Value)))
       } else {
          sp.data[key] = args.(ExecArgs).Value
       }

       reply.Err = OK

    } else if args.(ExecArgs).Type == Get {
       // execute the get

       value, ok := sp.data[key]
       if ok {
          reply.Value = value
          reply.Err = OK
       } else {
          reply.Value = ""
          reply.Err = ErrNoKey
       }
    }

    return reply

  } else {
    // not testing

    // write op to file
    err := sp.WriteToLog(op)
    if err != nil {
      // log something
    }

    switch {
    case op.Type == Open:
      return sp.OpenHelper(op.Args.(OpenArgs), op.SeqNum)
    case op.Type == Close:
      return sp.CloseHelper(op.Args.(CloseArgs), op.SeqNum)
    case op.Type == Execute:
      return sp.ExecuteHelper(op.Args.(ExecArgs), op.SeqNum)
    }
  }
  return nil
}

func (sp *SQLPaxos) WriteToLog(op Op) error {
  b, err := json.Marshal(op)
  if err != nil {
    return err
  }
  return sp.logger.WriteToLog(b)
}

func (sp *SQLPaxos) ExecuteHelper(args ExecArgs, seqnum int) ExecReply {
  rows, columns, err := sp.UpdateDatabase(args.ClientId, args.Query, args.QueryParams, seqnum)
  if err != OK {
    // log something
    return ExecReply{Value:"", Err:err}
  }

  tuples := []*barista.Tuple{}
  for _, row := range rows {
    tuple := barista.Tuple{Cells: &row}
    tuples = append(tuples, &tuple)
  }

  result_set := new(barista.ResultSet)
  //result_set.Con = con. @TODO: this will not be populating this
  result_set.Tuples = &tuples
  result_set.FieldNames = &columns
  return ExecReply{Result:result_set, Err:OK}
}

func (sp *SQLPaxos) OpenHelper(args OpenArgs, seqnum int) OpenReply {
  reply := OpenReply{}
  _, ok := sp.connections[args.ClientId]
  if ok {
    reply.Err = ConnAlreadyOpen
  } else {
      manager := new(db.DBManager)
      reply.Err = errorToErr(manager.OpenConnection(args.User, args.Password, args.Database))
      sp.connections[args.ClientId] = manager
  }
  _, _, err := sp.UpdateDatabase(args.ClientId, "", nil, seqnum)
  if err != OK {
    // log something
  }
  return reply
}

func errorToErr(error error) Err {
  if error != nil {
    return Err(error.Error())
  } else {
    return OK
  }
}

func (sp *SQLPaxos) CloseHelper(args CloseArgs, seqnum int) CloseReply {
  _, _, err := sp.UpdateDatabase(args.ClientId, "", nil, seqnum)
  reply := CloseReply{}
  _, ok := sp.connections[args.ClientId]
  if !ok {
    reply.Err = ConnAlreadyClosed
  } else {
    reply.Err = errorToErr(sp.connections[args.ClientId].CloseConnection())
    delete(sp.connections, args.ClientId) //only delete on successful close?
  }
  if err != OK {
    // log something
  }
  return reply
}

// note that NoOps don't update the state table
func (sp *SQLPaxos) UpdateDatabase(clientId int64, query string, query_params [][]byte, seqnum int) ([][][]byte, []string, Err) {
  tx, err := sp.connections[clientId].BeginTxn()

  if err != nil || tx == nil {
     return make([][][]byte, 0), make([]string, 0), errorToErr(err)
  }

  rows, columns, error := sp.connections[clientId].QueryTxn(query, query_params, tx)

  update := "UPDATE SQLPaxosLog SET lastSeqNum=" + strconv.Itoa(seqnum) + ";"
  sp.connections[clientId].ExecTxn(update, query_params, tx)

  sp.connections[clientId].EndTxn(tx)

  return rows, columns, errorToErr(error)
}

func (sp *SQLPaxos) fillHoles(next int, seq int) interface{} {

  var reply interface{}

  // make sure there are no holes in the log before our operation
  for i := next; i <= seq; i++ {
     nwaits := 0
     for !sp.dead {
	if _, ok := sp.ops[i]; ok || sp.next > i {
      	   break
        }

        decided, v_i := sp.px.Status(i)
        if decided {
           // the operation in slot i has been decided
           sp.ops[i] = v_i.(Op)
           break
        } else {
           nwaits++
           sp.mu.Unlock()
           if nwaits == 5 || nwaits == 10 {
              // propose a no-op
              sp.px.Start(i, Op{NoOp: true})
           } else if nwaits > 10 {
              time.Sleep(100 * time.Millisecond)
           } else {
              time.Sleep(10 * time.Millisecond)
           }
           sp.mu.Lock()
        }
     }

     if i == sp.next {
        // the operation at slot i is next to be executed
	r, executed := sp.checkIfExecuted(sp.ops[i])
        if executed {
    	   sp.replies[i] = r
	} else {
	   r := sp.execute(sp.ops[i])
	   sp.replies[i] = r
	   sp.lastSeen[getOpClientId(sp.ops[i])] = LastSeen{ RequestId: getOpRequestId(sp.ops[i]), Reply: r }
	}
        sp.next++
     }

     if i == seq {
        reply = sp.replies[i]
     }
  }

  return reply
}

func getOpClientId(op Op) int64 {
  switch {
  case op.Type == Open:
    return op.Args.(OpenArgs).ClientId;
  case op.Type == Close:
    return op.Args.(CloseArgs).ClientId;
  case op.Type == Execute:
    return op.Args.(ExecArgs).ClientId;
  }
  return -1;
}

func getOpRequestId(op Op) int {
  switch {
  case op.Type == Open:
    return op.Args.(OpenArgs).RequestId;
  case op.Type == Close:
    return op.Args.(CloseArgs).RequestId;
  case op.Type == Execute:
    return op.Args.(ExecArgs).RequestId;
  }
  return -1;
}

// @TODO: update to support multiple types of operations
func (sp *SQLPaxos) checkIfExecuted(op Op) (interface{}, bool) {
  // need some casting here
  lastSeen, ok := sp.lastSeen[getOpClientId(op)]
  if ok {
     if lastSeen.RequestId == getOpRequestId(op) {
        return lastSeen.Reply, true
     } else if lastSeen.RequestId > getOpRequestId(op) {
        return nil, true // empty reply since this is an old request
     }
  }

  return nil, false
}

func (sp *SQLPaxos) reserveSlot(op Op) int {

  // propose this operation for slot seq
  seq := sp.px.Max() + 1
  v := op
  sp.px.Start(seq, v)

  nwaits := 0
  for !sp.dead {
     decided, v_a := sp.px.Status(seq)
     if decided && v_a != nil && getOpClientId(v_a.(Op)) == getOpClientId(v) &&
       getOpRequestId(v_a.(Op)) == getOpRequestId(v) {
        // we successfully claimed this slot for our operation
        if _, ok := sp.ops[seq]; !ok {
	   v.SeqNum = seq
           sp.ops[seq] = v
        }
        break
     } else if decided {
        // another proposer got this slot, so try to get our operation in a new slot
        seq = int(math.Max(float64(sp.px.Max() + 1), float64(seq + 1)))
        sp.px.Start(seq, v)
        nwaits = 0
     } else {
        nwaits++
  	sp.mu.Unlock()
        if nwaits == 5 || nwaits == 10 {
           // re-propose our operation
           sp.px.Start(seq, v)
     	} else if nwaits > 10 {
           time.Sleep(100 * time.Millisecond)
        } else {
           time.Sleep(10 * time.Millisecond)
     	}
  	sp.mu.Lock()
     }
  }
  v.SeqNum = seq // update sequence number
  return seq
}

func (sp *SQLPaxos) freeMemory(seq int) {

  sp.done[seq] = true
  minNotDone := seq + 1
  for i := seq; i >= 0; i-- {
     _, ok := sp.ops[i]
     if ok {
        if done, ok := sp.done[i]; ok && done || sp.ops[i].NoOp {
           delete(sp.ops, i)
           delete(sp.replies, i)
           delete(sp.done, i)
        } else {
           minNotDone = i
        }
     }
  }

  sp.px.Done(minNotDone - 1)
}

//@Make it work for multiple types of arguments
func (sp *SQLPaxos) commit(op Op) interface{} {

  sp.mu.Lock()
  defer sp.mu.Unlock()

  // first check if this request has already been executed
  reply, ok := sp.checkIfExecuted(op)
  if ok {
     return reply
  }

  // reserve a slot in the paxos log for this operation
  seq := sp.reserveSlot(op)

  next := sp.next
  if next > seq {
     // our operation has already been executed
     reply = sp.replies[seq]
  } else {
     // fill holes in the log and execute our operation
     reply = sp.fillHoles(next, seq)
  }

  // delete un-needed log entries to free up memory
  sp.freeMemory(seq)

  return reply
}

func (sp *SQLPaxos) ExecuteSQL(args *ExecArgs, reply *ExecReply) error {
  // execute this operation and store the response in r
  op := Op{Type:Execute, Args: *args}
  r := sp.commit(op)

  if r != nil {
     reply.Value = r.(ExecReply).Value
     reply.Err = r.(ExecReply).Err
  }

  return nil
}

// open the connection to the database
func (sp *SQLPaxos) Open(args *OpenArgs, reply *OpenReply) error {
  // execute this operation and store the response in r
  op := Op{Type:Open, Args: *args}
  r := sp.commit(op)

  if r != nil {
     reply.Err = r.(OpenReply).Err
  }

  return nil
}

// close the connection to the database
func (sp *SQLPaxos) Close(args *CloseArgs, reply *CloseReply) error {
  // execute this operation and store the response in r
  op := Op{Type:Close, Args: *args}
  r := sp.commit(op)

  if r != nil {
     reply.Err = r.(CloseReply).Err
  }

  return nil
}

// tell the server to shut itself down.
func (sp *SQLPaxos) kill() {
  sp.dead = true
  sp.l.Close()
  sp.px.Kill()
}

//
// servers[] contains the ports of the set of
// servers that will cooperate via Paxos to
// form the fault-tolerant key/value service.
// me is the index of the current server in servers[].
//
func StartServer(servers []string, me int) *SQLPaxos {
  // call gob.Register on structures you want
  // Go's RPC library to marshall/unmarshall.
  gob.Register(Op{})
  gob.Register(ExecArgs{})

  sp := new(SQLPaxos)
  sp.me = me

  // Your initialization code here.
  sp.ops = make(map[int]Op)
  sp.data = make(map[string]string)
  sp.replies = make(map[int]interface{})
  sp.done = make(map[int]bool)
  sp.lastSeen = make(map[int64]LastSeen)
  sp.next = 0
  sp.connections = make(map[int64]*db.DBManager)
  sp.logger = logger.Make("sqlpaxos_log.txt")

  rpcs := rpc.NewServer()
  rpcs.Register(sp)

  sp.px = paxos.Make(servers, me, rpcs)

  os.Remove(servers[me])
  l, e := net.Listen("unix", servers[me]);
  if e != nil {
    log.Fatal("listen error: ", e);
  }
  sp.l = l

  // please do not change any of the following code,
  // or do anything to subvert it.

  go func() {
    for sp.dead == false {
      conn, err := sp.l.Accept()
      if err == nil && sp.dead == false {
      if sp.unreliable && (rand.Int63() % 1000) < 100 {
          // discard the request.
          conn.Close()
        } else if sp.unreliable && (rand.Int63() % 1000) < 200 {
          // process the request but force discard of reply.
          c1 := conn.(*net.UnixConn)
          f, _ := c1.File()
          err := syscall.Shutdown(int(f.Fd()), syscall.SHUT_WR)
          if err != nil {
            fmt.Printf("shutdown: %v\n", err)
          }
          go rpcs.ServeConn(conn)
        } else {
          go rpcs.ServeConn(conn)
        }
      } else if err == nil {
        conn.Close()
      }
      if err != nil && sp.dead == false {
        fmt.Printf("SQLPaxos(%v) accept: %v\n", me, err.Error())
	sp.kill()
      }
    }
  }()

  return sp
}

<|endoftext|>


<commit_before>package flavors

import (
	"fmt"
	"net/http"
	"reflect"
	"testing"

	"github.com/rackspace/gophercloud"
	"github.com/rackspace/gophercloud/pagination"
	"github.com/rackspace/gophercloud/testhelper"
)

const tokenID = "blerb"

func serviceClient() *gophercloud.ServiceClient {
	return &gophercloud.ServiceClient{
		Provider: &gophercloud.ProviderClient{TokenID: tokenID},
		Endpoint: testhelper.Endpoint(),
	}
}

func TestListFlavors(t *testing.T) {
	testhelper.SetupHTTP()
	defer testhelper.TeardownHTTP()

	testhelper.Mux.HandleFunc("/flavors/detail", func(w http.ResponseWriter, r *http.Request) {
		testhelper.TestMethod(t, r, "GET")
		testhelper.TestHeader(t, r, "X-Auth-Token", tokenID)

		w.Header().Add("Content-Type", "application/json")
		r.ParseForm()
		marker := r.Form.Get("marker")
		switch marker {
		case "":
			fmt.Fprintf(w, `
					{
						"flavors": [
							{
								"id": "1",
								"name": "m1.tiny",
								"disk": 1,
								"ram": 512,
								"vcpus": 1
							},
							{
								"id": "2",
								"name": "m2.small",
								"disk": 10,
								"ram": 1024,
								"vcpus": 2
							}
						]
					}
				`)
		case "2":
			fmt.Fprintf(w, `{ "flavors": [] }`)
		default:
			t.Fatalf("Unexpected marker: [%s]", marker)
		}
	})

	client := serviceClient()
	pages := 0
	err := List(client, ListFilterOptions{}).EachPage(func(page pagination.Page) (bool, error) {
		pages++

		actual, err := ExtractFlavors(page)
		if err != nil {
			return false, err
		}

		expected := []Flavor{
			Flavor{ID: "1", Name: "m1.tiny", Disk: 1, RAM: 512, VCPUs: 1},
			Flavor{ID: "2", Name: "m2.small", Disk: 10, RAM: 1024, VCPUs: 2},
		}

		if !reflect.DeepEqual(expected, actual) {
			t.Errorf("Expected %#v, but was %#v", expected, actual)
		}

		return true, nil
	})
	if err != nil {
		t.Fatal(err)
	}
}
<commit_msg>Add flavors_links to the List result.<commit_after>package flavors

import (
	"fmt"
	"net/http"
	"reflect"
	"testing"

	"github.com/rackspace/gophercloud"
	"github.com/rackspace/gophercloud/pagination"
	"github.com/rackspace/gophercloud/testhelper"
)

const tokenID = "blerb"

func serviceClient() *gophercloud.ServiceClient {
	return &gophercloud.ServiceClient{
		Provider: &gophercloud.ProviderClient{TokenID: tokenID},
		Endpoint: testhelper.Endpoint(),
	}
}

func TestListFlavors(t *testing.T) {
	testhelper.SetupHTTP()
	defer testhelper.TeardownHTTP()

	testhelper.Mux.HandleFunc("/flavors/detail", func(w http.ResponseWriter, r *http.Request) {
		testhelper.TestMethod(t, r, "GET")
		testhelper.TestHeader(t, r, "X-Auth-Token", tokenID)

		w.Header().Add("Content-Type", "application/json")
		r.ParseForm()
		marker := r.Form.Get("marker")
		switch marker {
		case "":
			fmt.Fprintf(w, `
					{
						"flavors": [
							{
								"id": "1",
								"name": "m1.tiny",
								"disk": 1,
								"ram": 512,
								"vcpus": 1
							},
							{
								"id": "2",
								"name": "m2.small",
								"disk": 10,
								"ram": 1024,
								"vcpus": 2
							}
						],
						"flavors_links": [
							{
								"href": "%s/flavors?marker=2"
							}
						]
					}
				`, testhelper.Server.URL)
		case "2":
			fmt.Fprintf(w, `{ "flavors": [] }`)
		default:
			t.Fatalf("Unexpected marker: [%s]", marker)
		}
	})

	client := serviceClient()
	pages := 0
	err := List(client, ListFilterOptions{}).EachPage(func(page pagination.Page) (bool, error) {
		pages++

		actual, err := ExtractFlavors(page)
		if err != nil {
			return false, err
		}

		expected := []Flavor{
			Flavor{ID: "1", Name: "m1.tiny", Disk: 1, RAM: 512, VCPUs: 1},
			Flavor{ID: "2", Name: "m2.small", Disk: 10, RAM: 1024, VCPUs: 2},
		}

		if !reflect.DeepEqual(expected, actual) {
			t.Errorf("Expected %#v, but was %#v", expected, actual)
		}

		return true, nil
	})
	if err != nil {
		t.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package hoverfly_end_to_end_test

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"os/exec"
	"os"
	"strings"
	"github.com/phayes/freeport"
	"fmt"
	"github.com/dghubble/sling"
	"strconv"
	"io/ioutil"
)

var _ = Describe("When I use hoverfly-cli", func() {
	var (
		hoverflyCmd *exec.Cmd

		workingDir, _ = os.Getwd()
		adminPort = freeport.GetPort()
		adminPortAsString = strconv.Itoa(adminPort)

		proxyPort = freeport.GetPort()
	)

	Describe("with a running hoverfly", func() {

		BeforeEach(func() {
			hoverflyCmd = startHoverfly(adminPort, proxyPort, workingDir)
		})

		AfterEach(func() {
			hoverflyCmd.Process.Kill()
		})

		Describe("Managing Hoverflies data using the CLI", func() {

			BeforeEach(func() {
				DoRequest(sling.New().Post(fmt.Sprintf("http://localhost:%v/api/records", adminPort)).Body(strings.NewReader(`
					{
						"data": [{
							"request": {
								"path": "/api/bookings",
								"method": "POST",
								"destination": "www.my-test.com",
								"scheme": "http",
								"query": "",
								"body": "{\"flightId\": \"1\"}",
								"headers": {
									"Content-Type": [
										"application/json"
									]
								}
							},
							"response": {
								"status": 201,
								"body": "",
								"encodedBody": false,
								"headers": {
									"Location": [
										"http://localhost/api/bookings/1"
									]
								}
							}
						}]
					}`)))

				resp := DoRequest(sling.New().Get(fmt.Sprintf("http://localhost:%v/api/records", adminPort)))
				bytes, _ := ioutil.ReadAll(resp.Body)
				Expect(string(bytes)).ToNot(Equal(`{"data":null}`))
			})

			It("it can wipe data", func() {
				output, err := exec.Command(hoverctlBinary, "wipe", "--admin-port=" + adminPortAsString).Output()
				Expect(err).To(BeNil())
				Expect(output).To(ContainSubstring("Hoverfly has been wiped"))

				resp := DoRequest(sling.New().Get(fmt.Sprintf("http://localhost:%v/api/records", adminPort)))
				bytes, _ := ioutil.ReadAll(resp.Body)
				Expect(string(bytes)).To(Equal(`{"data":null}`))
			})

			It("can export the data", func() {
				output, err := exec.Command(hoverctlBinary, "export", "mogronalol/twitter", "--admin-port=" + adminPortAsString).Output()
				Expect(err).To(BeNil())
				Expect(output).To(ContainSubstring("mogronalol/twitter:latest exported successfully"))
				Expect(ioutil.ReadFile(hoverctlCacheDir + "/mogronalol.twitter.latest.json")).To(MatchJSON(`
					{
						"data": [{
							"request": {
								"path": "/api/bookings",
								"method": "POST",
								"destination": "www.my-test.com",
								"scheme": "http",
								"query": "",
								"body": "{\"flightId\": \"1\"}",
								"headers": {
									"Content-Type": [
										"application/json"
									]
								}
							},
							"response": {
								"status": 201,
								"body": "",
								"encodedBody": false,
								"headers": {
									"Location": [
										"http://localhost/api/bookings/1"
									]
								}
							}
						}]
					}`),
				)
			})
		})
	})
})<commit_msg>Added test for importing data<commit_after>package hoverfly_end_to_end_test

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"os/exec"
	"os"
	"strings"
	"github.com/phayes/freeport"
	"fmt"
	"github.com/dghubble/sling"
	"strconv"
	"io/ioutil"
)

var _ = Describe("When I use hoverfly-cli", func() {
	var (
		hoverflyCmd *exec.Cmd

		workingDir, _ = os.Getwd()
		adminPort = freeport.GetPort()
		adminPortAsString = strconv.Itoa(adminPort)

		proxyPort = freeport.GetPort()
	)

	Describe("with a running hoverfly", func() {

		BeforeEach(func() {
			hoverflyCmd = startHoverfly(adminPort, proxyPort, workingDir)
		})

		AfterEach(func() {
			hoverflyCmd.Process.Kill()
		})

		Describe("Managing Hoverflies data using the CLI", func() {

			BeforeEach(func() {
				DoRequest(sling.New().Post(fmt.Sprintf("http://localhost:%v/api/records", adminPort)).Body(strings.NewReader(`
					{
						"data": [{
							"request": {
								"path": "/api/bookings",
								"method": "POST",
								"destination": "www.my-test.com",
								"scheme": "http",
								"query": "",
								"body": "{\"flightId\": \"1\"}",
								"headers": {
									"Content-Type": [
										"application/json"
									]
								}
							},
							"response": {
								"status": 201,
								"body": "",
								"encodedBody": false,
								"headers": {
									"Location": [
										"http://localhost/api/bookings/1"
									]
								}
							}
						}]
					}`)))

				resp := DoRequest(sling.New().Get(fmt.Sprintf("http://localhost:%v/api/records", adminPort)))
				bytes, _ := ioutil.ReadAll(resp.Body)
				Expect(string(bytes)).ToNot(Equal(`{"data":null}`))
			})

			It("can export, wipe and then re-import the data", func() {

				// Export the data
				output, err := exec.Command(hoverctlBinary, "export", "mogronalol/twitter", "--admin-port=" + adminPortAsString).Output()
				Expect(err).To(BeNil())
				Expect(output).To(ContainSubstring("mogronalol/twitter:latest exported successfully"))
				Expect(ioutil.ReadFile(hoverctlCacheDir + "/mogronalol.twitter.latest.json")).To(MatchJSON(`
					{
						"data": [{
							"request": {
								"path": "/api/bookings",
								"method": "POST",
								"destination": "www.my-test.com",
								"scheme": "http",
								"query": "",
								"body": "{\"flightId\": \"1\"}",
								"headers": {
									"Content-Type": [
										"application/json"
									]
								}
							},
							"response": {
								"status": 201,
								"body": "",
								"encodedBody": false,
								"headers": {
									"Location": [
										"http://localhost/api/bookings/1"
									]
								}
							}
						}]
					}`),
				)

				// Wipe it
				output, err = exec.Command(hoverctlBinary, "wipe", "--admin-port=" + adminPortAsString).Output()
				Expect(err).To(BeNil())
				Expect(output).To(ContainSubstring("Hoverfly has been wiped"))

				resp := DoRequest(sling.New().Get(fmt.Sprintf("http://localhost:%v/api/records", adminPort)))
				bytes, _ := ioutil.ReadAll(resp.Body)
				Expect(string(bytes)).To(Equal(`{"data":null}`))

				// Re-import it
				output, err = exec.Command(hoverctlBinary, "import", "mogronalol/twitter", "--admin-port=" + adminPortAsString).Output()
				Expect(err).To(BeNil())
				Expect(output).To(ContainSubstring("mogronalol/twitter:latest imported successfully"))

				resp = DoRequest(sling.New().Get(fmt.Sprintf("http://localhost:%v/api/records", adminPort)))
				bytes, _ = ioutil.ReadAll(resp.Body)
				Expect(string(bytes)).To(MatchJSON(`
					{
						"data": [{
							"request": {
								"path": "/api/bookings",
								"method": "POST",
								"destination": "www.my-test.com",
								"scheme": "http",
								"query": "",
								"body": "{\"flightId\": \"1\"}",
								"headers": {
									"Content-Type": [
										"application/json"
									]
								}
							},
							"response": {
								"status": 201,
								"body": "",
								"encodedBody": false,
								"headers": {
									"Location": [
										"http://localhost/api/bookings/1"
									]
								}
							}
						}]
					}`))
			})

		})
	})
})<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package gce

import (
	"fmt"

	"github.com/gophercloud/gophercloud"
	"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
	"github.com/gophercloud/gophercloud/pagination"
	"k8s.io/klog/v2"
	"k8s.io/kops/protokube/pkg/gossip"
	"k8s.io/kops/upup/pkg/fi/cloudup/openstack"
)

type SeedProvider struct {
	computeClient *gophercloud.ServiceClient
	projectID     string
	clusterName   string
}

var _ gossip.SeedProvider = &SeedProvider{}

func (p *SeedProvider) GetSeeds() ([]string, error) {
	var seeds []string

	err := servers.List(p.computeClient, servers.ListOpts{
		TenantID: p.projectID,
	}).EachPage(func(page pagination.Page) (bool, error) {
		var s []servers.Server
		err := servers.ExtractServersInto(page, &s)
		if err != nil {
			return false, err
		}

		for _, server := range s {
			if clusterName, ok := server.Metadata[openstack.TagClusterName]; ok {
				var err error
				// find kopsNetwork from metadata, fallback to clustername
				ifName := clusterName
				if val, ok := server.Metadata[openstack.TagKopsNetwork]; ok {
					ifName = val
				}
				addr, err := openstack.GetServerFixedIP(&server, ifName)
				if err != nil {
					klog.Warningf("Failed to list seeds: %v", err)
					continue
				}
				seeds = append(seeds, addr)
			}
		}
		return true, nil
	})

	if err != nil {
		return seeds, fmt.Errorf("Failed to list servers while retrieving seeds: %v", err)
	}

	return seeds, nil
}

func NewSeedProvider(computeClient *gophercloud.ServiceClient, clusterName string, projectID string) (*SeedProvider, error) {
	return &SeedProvider{
		computeClient: computeClient,
		clusterName:   clusterName,
		projectID:     projectID,
	}, nil
}
<commit_msg>Update seeds.go<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package gce

import (
	"fmt"

	"github.com/gophercloud/gophercloud"
	"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
	"github.com/gophercloud/gophercloud/pagination"
	"k8s.io/klog/v2"
	"k8s.io/kops/protokube/pkg/gossip"
	"k8s.io/kops/upup/pkg/fi/cloudup/openstack"
)

type SeedProvider struct {
	computeClient *gophercloud.ServiceClient
	projectID     string
	clusterName   string
}

var _ gossip.SeedProvider = &SeedProvider{}

func (p *SeedProvider) GetSeeds() ([]string, error) {
	var seeds []string

	err := servers.List(p.computeClient, servers.ListOpts{
		TenantID: p.projectID,
	}).EachPage(func(page pagination.Page) (bool, error) {
		var s []servers.Server
		err := servers.ExtractServersInto(page, &s)
		if err != nil {
			return false, err
		}

		for _, server := range s {
			if clusterName, ok := server.Metadata[openstack.TagClusterName]; ok {
				// verify that the instance is from the same cluster
				if clusterName != p.clusterName {
					continue
				}

				var err error
				// find kopsNetwork from metadata, fallback to clustername
				ifName := clusterName
				if val, ok := server.Metadata[openstack.TagKopsNetwork]; ok {
					ifName = val
				}
				addr, err := openstack.GetServerFixedIP(&server, ifName)
				if err != nil {
					klog.Warningf("Failed to list seeds: %v", err)
					continue
				}
				seeds = append(seeds, addr)
			}
		}
		return true, nil
	})

	if err != nil {
		return seeds, fmt.Errorf("Failed to list servers while retrieving seeds: %v", err)
	}

	return seeds, nil
}

func NewSeedProvider(computeClient *gophercloud.ServiceClient, clusterName string, projectID string) (*SeedProvider, error) {
	return &SeedProvider{
		computeClient: computeClient,
		clusterName:   clusterName,
		projectID:     projectID,
	}, nil
}
<|endoftext|>


<commit_before>// Copyright 2016 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package nodecontainer

import (
	"fmt"
	"time"

	"github.com/fsouza/go-dockerclient"
	"github.com/tsuru/config"
	"github.com/tsuru/docker-cluster/cluster"
	"github.com/tsuru/monsterqueue"
	"github.com/tsuru/tsuru/queue"
)

const QueueTaskName = "run-bs"

// RegisterQueueTask registers the internal bs queue task for later execution.
func RegisterQueueTask(p DockerProvisioner) error {
	q, err := queue.Queue()
	if err != nil {
		return err
	}
	return q.RegisterTask(&runBs{provisioner: p})
}

type runBs struct {
	provisioner DockerProvisioner
}

func (t *runBs) Name() string {
	return QueueTaskName
}

func (t *runBs) Run(job monsterqueue.Job) {
	_, err := InitializeBS()
	if err != nil {
		job.Error(err)
		return
	}
	params := job.Parameters()
	dockerEndpoint := params["endpoint"].(string)
	node, err := t.provisioner.Cluster().GetNode(dockerEndpoint)
	if err != nil {
		job.Error(err)
		return
	}
	client, err := node.Client()
	if err != nil {
		job.Error(err)
		return
	}
	err = t.waitDocker(client)
	if err != nil {
		job.Error(err)
		return
	}
	node.CreationStatus = cluster.NodeCreationStatusCreated
	err = RecreateContainers(t.provisioner, nil, node)
	if err != nil {
		t.provisioner.Cluster().UpdateNode(node)
		job.Error(err)
		return
	}
	node.Metadata["LastSuccess"] = time.Now().Format(time.RFC3339)
	_, err = t.provisioner.Cluster().UpdateNode(node)
	if err != nil {
		job.Error(err)
		return
	}
	job.Success(nil)
}

func (t *runBs) waitDocker(client *docker.Client) error {
	timeout, _ := config.GetInt("docker:api-timeout")
	if timeout == 0 {
		timeout = 600
	}
	timeoutChan := time.After(time.Duration(timeout) * time.Second)
	pong := make(chan error, 1)
	exit := make(chan struct{})
	go func() {
		for {
			err := client.Ping()
			if err == nil {
				pong <- nil
				return
			}
			if e, ok := err.(*docker.Error); ok && e.Status > 499 {
				pong <- err
				return
			}
			select {
			case <-exit:
				return
			case <-time.After(time.Second):
			}
		}
	}()
	select {
	case err := <-pong:
		return err
	case <-timeoutChan:
		close(exit)
		return fmt.Errorf("Docker API at %q didn't respond after %d seconds", client.Endpoint(), timeout)
	}
}
<commit_msg>provision/docker: removes bs initialization from queue job<commit_after>// Copyright 2016 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package nodecontainer

import (
	"fmt"
	"time"

	"github.com/fsouza/go-dockerclient"
	"github.com/tsuru/config"
	"github.com/tsuru/docker-cluster/cluster"
	"github.com/tsuru/monsterqueue"
	"github.com/tsuru/tsuru/queue"
)

const QueueTaskName = "run-bs"

// RegisterQueueTask registers the internal bs queue task for later execution.
func RegisterQueueTask(p DockerProvisioner) error {
	q, err := queue.Queue()
	if err != nil {
		return err
	}
	return q.RegisterTask(&runBs{provisioner: p})
}

type runBs struct {
	provisioner DockerProvisioner
}

func (t *runBs) Name() string {
	return QueueTaskName
}

func (t *runBs) Run(job monsterqueue.Job) {
	params := job.Parameters()
	dockerEndpoint := params["endpoint"].(string)
	node, err := t.provisioner.Cluster().GetNode(dockerEndpoint)
	if err != nil {
		job.Error(err)
		return
	}
	client, err := node.Client()
	if err != nil {
		job.Error(err)
		return
	}
	err = t.waitDocker(client)
	if err != nil {
		job.Error(err)
		return
	}
	node.CreationStatus = cluster.NodeCreationStatusCreated
	err = RecreateContainers(t.provisioner, nil, node)
	if err != nil {
		t.provisioner.Cluster().UpdateNode(node)
		job.Error(err)
		return
	}
	node.Metadata["LastSuccess"] = time.Now().Format(time.RFC3339)
	_, err = t.provisioner.Cluster().UpdateNode(node)
	if err != nil {
		job.Error(err)
		return
	}
	job.Success(nil)
}

func (t *runBs) waitDocker(client *docker.Client) error {
	timeout, _ := config.GetInt("docker:api-timeout")
	if timeout == 0 {
		timeout = 600
	}
	timeoutChan := time.After(time.Duration(timeout) * time.Second)
	pong := make(chan error, 1)
	exit := make(chan struct{})
	go func() {
		for {
			err := client.Ping()
			if err == nil {
				pong <- nil
				return
			}
			if e, ok := err.(*docker.Error); ok && e.Status > 499 {
				pong <- err
				return
			}
			select {
			case <-exit:
				return
			case <-time.After(time.Second):
			}
		}
	}()
	select {
	case err := <-pong:
		return err
	case <-timeoutChan:
		close(exit)
		return fmt.Errorf("Docker API at %q didn't respond after %d seconds", client.Endpoint(), timeout)
	}
}
<|endoftext|>


<commit_before>package maestro_test

import (
	"code.google.com/p/gomock/gomock"
	. "launchpad.net/gocheck"
	"testing"

	"github.com/marmelab/gaudi/docker" // mock
	"github.com/marmelab/gaudi/maestro"
)

func Test(t *testing.T) { TestingT(t) }

type MaestroTestSuite struct{}

var _ = Suite(&MaestroTestSuite{})

func (s *MaestroTestSuite) TestInitFromStringShouldTrowAndErrorOnMalformedYmlContent(c *C) {
	m := maestro.Maestro{}

	c.Assert(func() {
		m.InitFromString(`
		applications:
			tabulated:
				type: varnish
`, "")
	}, PanicMatches, "YAML error: line 1: found character that cannot start any token")
}

func (s *MaestroTestSuite) TestInitFromStringShouldTrowAndErrorOnWrongContent(c *C) {
	m := maestro.Maestro{}

	c.Assert(func() { m.InitFromString("<oldFormat>Skrew you, i'm not yml</oldFormat>", "") }, PanicMatches, "No application to start")
}

func (s *MaestroTestSuite) TestInitFromStringShouldCreateAMaestro(c *C) {
	m := maestro.Maestro{}
	m.InitFromString(`
applications:
    app:
        type: php-fpm
        links: [db]
    db:
        type: mysql
        ports:
            3306: 9000
`, "")

	// Create a gomock controller, and arrange for it's finish to be called
	ctrl := gomock.NewController(c)
	defer ctrl.Finish()
	docker.MOCK().SetController(ctrl)
	docker.EXPECT().Inspect(gomock.Any()).Return([]byte("[{\"ID\": \"123\", \"State\":{\"Running\": false}, \"NetworkSettings\": {\"IPAddress\": \"\"}}]"), nil)

	c.Assert(len(m.Applications), Equals, 2)
	c.Assert(m.GetContainer("app").Name, Equals, "app")
	c.Assert(m.GetContainer("app").Type, Equals, "php-fpm")
	c.Assert(m.GetContainer("app").Dependencies[0].Name, Equals, "db")
	c.Assert(m.GetContainer("db").GetFirstPort(), Equals, "3306")
	c.Assert(m.GetContainer("db").IsRunning(), Equals, false)
}

func (s *MaestroTestSuite) TestStartApplicationShouldCleanAndBuildThem(c *C) {
	// Create a gomock controller, and arrange for it's finish to be called
	ctrl := gomock.NewController(c)
	defer ctrl.Finish()

	// Setup the docker mock package
	docker.MOCK().SetController(ctrl)
	docker.EXPECT().Kill(gomock.Any()).Return().Times(2)
	docker.EXPECT().Remove(gomock.Any()).Return().Times(2)
	docker.EXPECT().Build(gomock.Any(), gomock.Any()).Return().Times(2)
	docker.EXPECT().Start(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return("123").Times(2)
	docker.EXPECT().Inspect(gomock.Any()).Return([]byte("[{\"ID\": \"123\", \"State\":{\"Running\": false}, \"NetworkSettings\": {\"IPAddress\": \"172.17.0.10\"}}]"), nil).Times(4)

	m := maestro.Maestro{}
	m.InitFromString(`
applications:
    app:
        type: php-fpm
        links: [db]
    db:
        type: mysql
        ports:
            3306: 9000
`, "")

	c.Assert(len(m.Applications), Equals, 2)

	m.Start()
	c.Assert(m.GetContainer("db").IsRunning(), Equals, true)
	c.Assert(m.GetContainer("app").IsRunning(), Equals, true)
}

func (s *MaestroTestSuite) TestStartApplicationShouldStartThemByOrderOfDependencies(c *C) {
	// Create a gomock controller, and arrange for it's finish to be called
	ctrl := gomock.NewController(c)
	defer ctrl.Finish()

	// Setup the docker mock package
	docker.MOCK().SetController(ctrl)

	m := maestro.Maestro{}
	m.InitFromString(`
applications:
    lb:
        links: [front1, front2]
        type: varnish

    front1:
        links: [app]
        type: apache

    front2:
        links: [app]
        type: apache

    app:
        links: [db]
        type: php-fpm

    db:
      type: mysql
`, "")

	c.Assert(len(m.Applications), Equals, 5)

	docker.EXPECT().Kill(gomock.Any()).Return().Times(5)
	docker.EXPECT().Remove(gomock.Any()).Return().Times(5)
	docker.EXPECT().Build(gomock.Any(), gomock.Any()).Return().Times(5)

	gomock.InOrder(
		docker.EXPECT().Inspect("db").Return([]byte("[{\"ID\": \"100\", \"State\":{\"Running\": false}, \"NetworkSettings\": {\"IPAddress\": \"172.17.0.10\"}}]"), nil),
		docker.EXPECT().Start("db", gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return("100"),
		docker.EXPECT().Inspect("100").Return([]byte("[{\"ID\": \"100\", \"State\":{\"Running\": true}, \"NetworkSettings\": {\"IPAddress\": \"172.17.0.10\"}}]"), nil),

		docker.EXPECT().Inspect("app").Return([]byte("[{\"ID\": \"101\", \"State\":{\"Running\": false}, \"NetworkSettings\": {\"IPAddress\": \"172.17.0.10\"}}]"), nil),
		docker.EXPECT().Start("app", gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return("101"),
		docker.EXPECT().Inspect("101").Return([]byte("[{\"ID\": \"101\", \"State\":{\"Running\": true}, \"NetworkSettings\": {\"IPAddress\": \"172.17.0.10\"}}]"), nil),

		docker.EXPECT().Inspect("front1").Return([]byte("[{\"ID\": \"102\", \"State\":{\"Running\": false}, \"NetworkSettings\": {\"IPAddress\": \"172.17.0.10\"}}]"), nil),
		docker.EXPECT().Start("front1", gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return("102"),

		docker.EXPECT().Inspect("front2").Return([]byte("[{\"ID\": \"103\", \"State\":{\"Running\": false}, \"NetworkSettings\": {\"IPAddress\": \"172.17.0.10\"}}]"), nil),
		docker.EXPECT().Start("front2", gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return("103"),

		docker.EXPECT().Inspect("102").Return([]byte("[{\"ID\": \"102\", \"State\":{\"Running\": true}, \"NetworkSettings\": {\"IPAddress\": \"172.17.0.10\"}}]"), nil),
		docker.EXPECT().Inspect("103").Return([]byte("[{\"ID\": \"103\", \"State\":{\"Running\": true}, \"NetworkSettings\": {\"IPAddress\": \"172.17.0.10\"}}]"), nil),

		docker.EXPECT().Inspect("lb").Return([]byte("[{\"ID\": \"104\", \"State\":{\"Running\": false}, \"NetworkSettings\": {\"IPAddress\": \"172.17.0.10\"}}]"), nil),
		docker.EXPECT().Start("lb", gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return("104"),
		docker.EXPECT().Inspect("104").Return([]byte("[{\"ID\": \"104\", \"State\":{\"Running\": true}, \"NetworkSettings\": {\"IPAddress\": \"172.17.0.10\"}}]"), nil),
	)

	m.Start(true)
}
<commit_msg>Fix tests<commit_after>package maestro_test

import (
	"code.google.com/p/gomock/gomock"
	. "launchpad.net/gocheck"
	"testing"

	"github.com/marmelab/gaudi/docker" // mock
	"github.com/marmelab/gaudi/maestro"
)

func Test(t *testing.T) { TestingT(t) }

type MaestroTestSuite struct{}

var _ = Suite(&MaestroTestSuite{})

func (s *MaestroTestSuite) TestInitFromStringShouldTrowAndErrorOnMalformedYmlContent(c *C) {
	m := maestro.Maestro{}

	c.Assert(func() {
		m.InitFromString(`
		applications:
			tabulated:
				type: varnish
`, "")
	}, PanicMatches, "YAML error: line 1: found character that cannot start any token")
}

func (s *MaestroTestSuite) TestInitFromStringShouldTrowAndErrorOnWrongContent(c *C) {
	m := maestro.Maestro{}

	c.Assert(func() { m.InitFromString("<oldFormat>Skrew you, i'm not yml</oldFormat>", "") }, PanicMatches, "No application to start")
}

func (s *MaestroTestSuite) TestInitFromStringShouldCreateAMaestro(c *C) {
	m := maestro.Maestro{}
	m.InitFromString(`
applications:
    app:
        type: php-fpm
        links: [db]
    db:
        type: mysql
        ports:
            3306: 9000
`, "")

	// Create a gomock controller, and arrange for it's finish to be called
	ctrl := gomock.NewController(c)
	defer ctrl.Finish()
	docker.MOCK().SetController(ctrl)
	docker.EXPECT().Inspect(gomock.Any()).Return([]byte("[{\"ID\": \"123\", \"State\":{\"Running\": false}, \"NetworkSettings\": {\"IPAddress\": \"\"}}]"), nil)

	c.Assert(len(m.Applications), Equals, 2)
	c.Assert(m.GetContainer("app").Name, Equals, "app")
	c.Assert(m.GetContainer("app").Type, Equals, "php-fpm")
	c.Assert(m.GetContainer("app").Dependencies[0].Name, Equals, "db")
	c.Assert(m.GetContainer("db").GetFirstPort(), Equals, "3306")
	c.Assert(m.GetContainer("db").IsRunning(), Equals, false)
}

func (s *MaestroTestSuite) TestStartApplicationShouldCleanAndBuildThem(c *C) {
	// Create a gomock controller, and arrange for it's finish to be called
	ctrl := gomock.NewController(c)
	defer ctrl.Finish()

	// Setup the docker mock package
	docker.MOCK().SetController(ctrl)
	docker.EXPECT().Kill(gomock.Any()).Return().Times(2)
	docker.EXPECT().Remove(gomock.Any()).Return().Times(2)
	docker.EXPECT().Build(gomock.Any(), gomock.Any()).Return().Times(2)
	docker.EXPECT().Start(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return("123").Times(2)
	docker.EXPECT().Inspect(gomock.Any()).Return([]byte("[{\"ID\": \"123\", \"State\":{\"Running\": false}, \"NetworkSettings\": {\"IPAddress\": \"172.17.0.10\"}}]"), nil).Times(4)

	m := maestro.Maestro{}
	m.InitFromString(`
applications:
    app:
        type: php-fpm
        links: [db]
    db:
        type: mysql
        ports:
            3306: 9000
`, "")

	c.Assert(len(m.Applications), Equals, 2)

	m.Start()
	c.Assert(m.GetContainer("db").IsRunning(), Equals, true)
	c.Assert(m.GetContainer("app").IsRunning(), Equals, true)
}

func (s *MaestroTestSuite) TestStartApplicationShouldStartThemByOrderOfDependencies(c *C) {
	// Create a gomock controller, and arrange for it's finish to be called
	ctrl := gomock.NewController(c)
	defer ctrl.Finish()

	// Setup the docker mock package
	docker.MOCK().SetController(ctrl)

	m := maestro.Maestro{}
	m.InitFromString(`
applications:
    lb:
        links: [front1, front2]
        type: varnish

    front1:
        links: [app]
        type: apache

    front2:
        links: [app]
        type: apache

    app:
        links: [db]
        type: php-fpm

    db:
      type: mysql
`, "")

	c.Assert(len(m.Applications), Equals, 5)

	docker.EXPECT().Kill(gomock.Any()).Return().Times(5)
	docker.EXPECT().Remove(gomock.Any()).Return().Times(5)
	docker.EXPECT().Build(gomock.Any(), gomock.Any()).Return().Times(5)

	gomock.InOrder(
		docker.EXPECT().Inspect("db").Return([]byte("[{\"ID\": \"100\", \"State\":{\"Running\": false}, \"NetworkSettings\": {\"IPAddress\": \"172.17.0.10\"}}]"), nil),
		docker.EXPECT().Start("db", gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return("100"),
		docker.EXPECT().Inspect("100").Return([]byte("[{\"ID\": \"100\", \"State\":{\"Running\": true}, \"NetworkSettings\": {\"IPAddress\": \"172.17.0.10\"}}]"), nil),

		docker.EXPECT().Inspect("app").Return([]byte("[{\"ID\": \"101\", \"State\":{\"Running\": false}, \"NetworkSettings\": {\"IPAddress\": \"172.17.0.10\"}}]"), nil),
		docker.EXPECT().Start("app", gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return("101"),
		docker.EXPECT().Inspect("101").Return([]byte("[{\"ID\": \"101\", \"State\":{\"Running\": true}, \"NetworkSettings\": {\"IPAddress\": \"172.17.0.10\"}}]"), nil),

		docker.EXPECT().Inspect("front1").Return([]byte("[{\"ID\": \"102\", \"State\":{\"Running\": false}, \"NetworkSettings\": {\"IPAddress\": \"172.17.0.10\"}}]"), nil),
		docker.EXPECT().Start("front1", gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return("102"),

		docker.EXPECT().Inspect("front2").Return([]byte("[{\"ID\": \"103\", \"State\":{\"Running\": false}, \"NetworkSettings\": {\"IPAddress\": \"172.17.0.10\"}}]"), nil),
		docker.EXPECT().Start("front2", gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return("103"),

		docker.EXPECT().Inspect("102").Return([]byte("[{\"ID\": \"102\", \"State\":{\"Running\": true}, \"NetworkSettings\": {\"IPAddress\": \"172.17.0.10\"}}]"), nil),
		docker.EXPECT().Inspect("103").Return([]byte("[{\"ID\": \"103\", \"State\":{\"Running\": true}, \"NetworkSettings\": {\"IPAddress\": \"172.17.0.10\"}}]"), nil),

		docker.EXPECT().Inspect("lb").Return([]byte("[{\"ID\": \"104\", \"State\":{\"Running\": false}, \"NetworkSettings\": {\"IPAddress\": \"172.17.0.10\"}}]"), nil),
		docker.EXPECT().Start("lb", gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return("104"),
		docker.EXPECT().Inspect("104").Return([]byte("[{\"ID\": \"104\", \"State\":{\"Running\": true}, \"NetworkSettings\": {\"IPAddress\": \"172.17.0.10\"}}]"), nil),
	)

	m.Start()
}
<|endoftext|>


<commit_before>package ovextra

import (
	"encoding/json"
	"log"
	"time"
)

type UplinkSetCol struct {
	Type        string      `json:"type"`
	Members     []UplinkSet `json:"members"`
	Count       int         `json:"count"`
	Total       int         `json:"total"`
	NextPageURI string      `json:"nextPageUri"`
	Start       int         `json:"start"`
	PrevPageURI string      `json:"prevPageUri"`
	Category    string      `json:"category"`
	Modified    string      `json:"modified"`
	ETag        string      `json:"eTag"`
	Created     string      `json:"created"`
	URI         string      `json:"uri"`
}

type UplinkSet struct {
	Type                           string           `json:"type"`
	ConnectionMode                 string           `json:"connectionMode"`
	ManualLoginRedistributionState string           `json:"manualLoginRedistributionState"`
	NativeNetworkURI               string           `json:"nativeNetworkUri"`
	FcoeNetworkUris                []string         `json:"fcoeNetworkUris"`
	FcNetworkUris                  []string         `json:"fcNetworkUris"`
	PrimaryPortLocation            string           `json:"primaryPortLocation"`
	LogicalInterconnectURI         string           `json:"logicalInterconnectUri"`
	NetworkType                    string           `json:"networkType"`
	EthernetNetworkType            string           `json:"ethernetNetworkType"`
	PortConfigInfos                []PortConfigInfo `json:"portConfigInfos"`
	Reachability                   string           `json:"reachability"`
	NetworkUris                    []string         `json:"networkUris"`
	LacpTimer                      string           `json:"lacpTimer"`
	Description                    string           `json:"description"`
	Status                         string           `json:"status"`
	Name                           string           `json:"name"`
	State                          string           `json:"state"`
	Category                       string           `json:"category"`
	Modified                       string           `json:"modified"`
	ETag                           string           `json:"eTag"`
	Created                        string           `json:"created"`
	URI                            string           `json:"uri"`
	LIURI                          string           //manually add to be get LI name from LogicalInterconnectURI
}

type PortConfigInfo struct {
	DesiredSpeed     string `json:"desiredSpeed"`
	PortURI          string `json:"portUri"`
	ExpectedNeighbor string `json:"expectedNeighbor"`
	Location         struct {
		LocationEntries []struct {
			Value string `json:"value"`
			Type  string `json:"type"`
		} `json:"locationEntries"`
	} `json:"location"`
}

const (
	uplinkShowFormat = "" +
		"{{range .}}" +
		"{{if ne .ProductName \"Synergy 20Gb Interconnect Link Module\" }}" +
		"-------------\n" +
		"Interconnect: {{.Name}} ({{.ProductName}})\n" +
		"-------------\n" +
		"PortName\tConnectorType\tPortStatus\tPortType\tNeighbor\tNeighbor Port\tTransceiver\n" +
		"{{range .Ports}}" +
		"{{if or (eq .PortType \"Uplink\") (eq .PortType \"Stacking\") }}" +
		//"{{if eq .PortType Uplink }}" +
		"{{.PortName}}\t{{.ConnectorType}}\t{{.PortStatus}}\t{{.PortType}}\t{{.Neighbor.RemoteSystemName}}\t{{.Neighbor.RemotePortID}}\t{{.TransceiverPN}}\n" +
		"{{end}}" +
		"{{end}}" +
		"\n" +
		"{{end}}" +
		"{{end}}"
)

//GetUplinkSet is to retrive uplinkset information
func GetUplinkSet() LIUplinkSetMap {

	uplinkSetMapC := make(chan UplinkSetMap)
	liMapC := make(chan LIMap)

	go UplinkSetGetURI(uplinkSetMapC, "Name")
	go LIGetURI(liMapC, "Name")

	var liMap LIMap
	var uplinkSetMap UplinkSetMap

	for i := 0; i < 2; i++ {
		select {
		case uplinkSetMap = <-uplinkSetMapC:
		case liMap = <-liMapC:
		}
	}

	//Save LIG value to LI entrie field, LIGName is manually added in JSON struct. use LI's LIG URI as index to find among LIG Map
	for k := range liMap {
		liMap[k].LIGName = uplinkSetMap[liMap[k].LogicalInterconnectGroupURI].Name
	}

	var liUplinkSetMap LIUplinkSetMap

	return liUplinkSetMap

}

//UplinkSetGetURI is the function to get raw structs from all json next pages
func UplinkSetGetURI(x chan UplinkSetMap, key string) {

	log.Println("Rest Get UplinkSet Collection")

	defer timeTrack(time.Now(), "Rest Get UplinkSet Collection")

	c := NewCLIOVClient()

	uplinkSetMap := UplinkSetMap{}
	pages := make([]UplinkSetCol, 5)

	for i, uri := 0, UplinkSetURL; uri != ""; i++ {

		data, err := c.GetURI("", "", uri)
		if err != nil {
			log.Fatal(err)
		}

		err = json.Unmarshal(data, &pages[i])

		if err != nil {
			log.Fatal(err)
		}

		for k := range pages[i].Members {
			switch key {
			case "Name":
				uplinkSetMap[pages[i].Members[k].Name] = &pages[i].Members[k]
			case "URI":
				uplinkSetMap[pages[i].Members[k].URI] = &pages[i].Members[k]
			}
		}

		uri = pages[i].NextPageURI
	}

	x <- uplinkSetMap

}
<commit_msg>modify uplinkset mapping<commit_after>package ovextra

import (
	"encoding/json"
	"log"
	"time"
)

type UplinkSetCol struct {
	Type        string      `json:"type"`
	Members     []UplinkSet `json:"members"`
	Count       int         `json:"count"`
	Total       int         `json:"total"`
	NextPageURI string      `json:"nextPageUri"`
	Start       int         `json:"start"`
	PrevPageURI string      `json:"prevPageUri"`
	Category    string      `json:"category"`
	Modified    string      `json:"modified"`
	ETag        string      `json:"eTag"`
	Created     string      `json:"created"`
	URI         string      `json:"uri"`
}

type UplinkSet struct {
	Type                           string           `json:"type"`
	ConnectionMode                 string           `json:"connectionMode"`
	ManualLoginRedistributionState string           `json:"manualLoginRedistributionState"`
	NativeNetworkURI               string           `json:"nativeNetworkUri"`
	FcoeNetworkUris                []string         `json:"fcoeNetworkUris"`
	FcNetworkUris                  []string         `json:"fcNetworkUris"`
	PrimaryPortLocation            string           `json:"primaryPortLocation"`
	LogicalInterconnectURI         string           `json:"logicalInterconnectUri"`
	NetworkType                    string           `json:"networkType"`
	EthernetNetworkType            string           `json:"ethernetNetworkType"`
	PortConfigInfos                []PortConfigInfo `json:"portConfigInfos"`
	Reachability                   string           `json:"reachability"`
	NetworkUris                    []string         `json:"networkUris"`
	LacpTimer                      string           `json:"lacpTimer"`
	Description                    string           `json:"description"`
	Status                         string           `json:"status"`
	Name                           string           `json:"name"`
	State                          string           `json:"state"`
	Category                       string           `json:"category"`
	Modified                       string           `json:"modified"`
	ETag                           string           `json:"eTag"`
	Created                        string           `json:"created"`
	URI                            string           `json:"uri"`
	LIName                         string           //manually add to be get LI name from LogicalInterconnectURI
}

type PortConfigInfo struct {
	DesiredSpeed     string `json:"desiredSpeed"`
	PortURI          string `json:"portUri"`
	ExpectedNeighbor string `json:"expectedNeighbor"`
	Location         struct {
		LocationEntries []struct {
			Value string `json:"value"`
			Type  string `json:"type"`
		} `json:"locationEntries"`
	} `json:"location"`
}

const (
	uplinkShowFormat = "" +
		"{{range .}}" +
		"{{if ne .ProductName \"Synergy 20Gb Interconnect Link Module\" }}" +
		"-------------\n" +
		"Interconnect: {{.Name}} ({{.ProductName}})\n" +
		"-------------\n" +
		"PortName\tConnectorType\tPortStatus\tPortType\tNeighbor\tNeighbor Port\tTransceiver\n" +
		"{{range .Ports}}" +
		"{{if or (eq .PortType \"Uplink\") (eq .PortType \"Stacking\") }}" +
		//"{{if eq .PortType Uplink }}" +
		"{{.PortName}}\t{{.ConnectorType}}\t{{.PortStatus}}\t{{.PortType}}\t{{.Neighbor.RemoteSystemName}}\t{{.Neighbor.RemotePortID}}\t{{.TransceiverPN}}\n" +
		"{{end}}" +
		"{{end}}" +
		"\n" +
		"{{end}}" +
		"{{end}}"
)

//GetUplinkSet is to retrive uplinkset information
func GetUplinkSet() LIUplinkSetMap {

	uplinkSetMapC := make(chan UplinkSetMap)
	liMapC := make(chan LIMap)

	go UplinkSetGetURI(uplinkSetMapC, "Name")
	go LIGetURI(liMapC, "Name")

	var liMap LIMap
	var uplinkSetMap UplinkSetMap

	for i := 0; i < 2; i++ {
		select {
		case uplinkSetMap = <-uplinkSetMapC:
		case liMap = <-liMapC:
		}
	}

	for k := range uplinkSetMap {
		//left side is the new field LI name in uplinkset struct, right side is to use uplinkset's LI URI as index to find LI's name using LI Map

		uplinkSetMap[k].LIName = liMap[uplinkSetMap[k].LogicalInterconnectURI].Name

	}

	var liUplinkSetMap LIUplinkSetMap

	return liUplinkSetMap

}

//UplinkSetGetURI is the function to get raw structs from all json next pages
func UplinkSetGetURI(x chan UplinkSetMap, key string) {

	log.Println("Rest Get UplinkSet Collection")

	defer timeTrack(time.Now(), "Rest Get UplinkSet Collection")

	c := NewCLIOVClient()

	uplinkSetMap := UplinkSetMap{}
	pages := make([]UplinkSetCol, 5)

	for i, uri := 0, UplinkSetURL; uri != ""; i++ {

		data, err := c.GetURI("", "", uri)
		if err != nil {
			log.Fatal(err)
		}

		err = json.Unmarshal(data, &pages[i])

		if err != nil {
			log.Fatal(err)
		}

		for k := range pages[i].Members {
			switch key {
			case "Name":
				uplinkSetMap[pages[i].Members[k].Name] = &pages[i].Members[k]
			case "URI":
				uplinkSetMap[pages[i].Members[k].URI] = &pages[i].Members[k]
			}
		}

		uri = pages[i].NextPageURI
	}

	x <- uplinkSetMap

}
<|endoftext|>


<commit_before>// Package osimage provides a client for Operating System Images.
package osimage

import (
	"encoding/xml"

	"github.com/Azure/azure-sdk-for-go/management"
)

const (
	azureImageListURL    = "services/images"
	errInvalidImage      = "Can not find image %s in specified subscription, please specify another image name."
	errParamNotSpecified = "Parameter %s is not specified."
)

// NewClient is used to instantiate a new OSImageClient from an Azure client.
func NewClient(client management.Client) OSImageClient {
	return OSImageClient{client: client}
}

func (c OSImageClient) ListOSImages() (ListOSImagesResponse, error) {
	var l ListOSImagesResponse

	response, err := c.client.SendAzureGetRequest(azureImageListURL)
	if err != nil {
		return l, err
	}

	err = xml.Unmarshal(response, &l)
	return l, err
}
<commit_msg>management/osimage: implement AddOSImage<commit_after>// Package osimage provides a client for Operating System Images.
package osimage

import (
	"encoding/xml"

	"github.com/Azure/azure-sdk-for-go/management"
)

const (
	azureImageListURL    = "services/images"
	errInvalidImage      = "Can not find image %s in specified subscription, please specify another image name."
	errParamNotSpecified = "Parameter %s is not specified."
)

// NewClient is used to instantiate a new OSImageClient from an Azure client.
func NewClient(client management.Client) OSImageClient {
	return OSImageClient{client: client}
}

func (c OSImageClient) ListOSImages() (ListOSImagesResponse, error) {
	var l ListOSImagesResponse

	response, err := c.client.SendAzureGetRequest(azureImageListURL)
	if err != nil {
		return l, err
	}

	err = xml.Unmarshal(response, &l)
	return l, err
}

// AddOSImage adds an operating system image to the image repository that is associated with the specified subscription.
//
// See https://msdn.microsoft.com/en-us/library/azure/jj157192.aspx for details.
func (c OSImageClient) AddOSImage(osi *OSImage) (management.OperationID, error) {
	data, err := xml.Marshal(osi)
	if err != nil {
		return "", err
	}

	return c.client.SendAzurePostRequest(azureImageListURL, data)

}
<|endoftext|>


<commit_before>package awsup

import (
	"fmt"
	"github.com/golang/glog"
)

// I believe one vCPU ~ 3 ECUS, and 60 CPU credits would be needed to use one vCPU for an hour
const BurstableCreditsToECUS float32 = 3.0 / 60.0

type AWSMachineTypeInfo struct {
	Name           string
	MemoryGB       float32
	ECU            float32
	Cores          int
	EphemeralDisks []int
	Burstable      bool
}

type EphemeralDevice struct {
	DeviceName  string
	VirtualName string
	SizeGB      int
}

func (m *AWSMachineTypeInfo) EphemeralDevices() ([]*EphemeralDevice, error) {
	var disks []*EphemeralDevice
	for i, sizeGB := range m.EphemeralDisks {
		d := &EphemeralDevice{
			SizeGB: sizeGB,
		}

		if i >= 20 {
			// TODO: What drive letters do we use?
			glog.Fatalf("ephemeral devices for > 20 not yet implemented")
		}
		d.DeviceName = "/dev/sd" + string('c'+i)
		d.VirtualName = fmt.Sprintf("ephemeral%d", i)

		disks = append(disks, d)
	}
	return disks, nil
}

func GetMachineTypeInfo(machineType string) (*AWSMachineTypeInfo, error) {
	for i := range MachineTypes {
		m := &MachineTypes[i]
		if m.Name == machineType {
			return m, nil
		}
	}

	return nil, fmt.Errorf("instance type not handled: %q", machineType)
}

var MachineTypes []AWSMachineTypeInfo = []AWSMachineTypeInfo{
	// This is tedious, but seems simpler than trying to have some logic and then a lot of exceptions

	// t2 family
	{
		Name:           "t2.nano",
		MemoryGB:       0.5,
		ECU:            3 * BurstableCreditsToECUS,
		Cores:          1,
		EphemeralDisks: nil,
		Burstable:      true,
	},
	{
		Name:           "t2.micro",
		MemoryGB:       1,
		ECU:            6 * BurstableCreditsToECUS,
		Cores:          1,
		EphemeralDisks: nil,
		Burstable:      true,
	},
	{
		Name:           "t2.small",
		MemoryGB:       2,
		ECU:            12 * BurstableCreditsToECUS,
		Cores:          1,
		EphemeralDisks: nil,
		Burstable:      true,
	},
	{
		Name:           "t2.medium",
		MemoryGB:       4,
		ECU:            24 * BurstableCreditsToECUS,
		Cores:          2,
		EphemeralDisks: nil,
		Burstable:      true,
	},
	{
		Name:           "t2.large",
		MemoryGB:       8,
		ECU:            36 * BurstableCreditsToECUS,
		Cores:          2,
		EphemeralDisks: nil,
		Burstable:      true,
	},

	// m3 family
	{
		Name:           "m3.medium",
		MemoryGB:       3.75,
		ECU:            3,
		Cores:          1,
		EphemeralDisks: []int{4},
	},
	{
		Name:           "m3.large",
		MemoryGB:       7.5,
		ECU:            6.5,
		Cores:          2,
		EphemeralDisks: []int{32},
	},
	{
		Name:           "m3.xlarge",
		MemoryGB:       15,
		ECU:            13,
		Cores:          4,
		EphemeralDisks: []int{40, 40},
	},
	{
		Name:           "m3.2xlarge",
		MemoryGB:       30,
		ECU:            26,
		Cores:          8,
		EphemeralDisks: []int{80, 80},
	},
}
<commit_msg>upup: define m3, m4, c4 families<commit_after>package awsup

import (
	"fmt"
	"github.com/golang/glog"
)

// I believe one vCPU ~ 3 ECUS, and 60 CPU credits would be needed to use one vCPU for an hour
const BurstableCreditsToECUS float32 = 3.0 / 60.0

type AWSMachineTypeInfo struct {
	Name           string
	MemoryGB       float32
	ECU            float32
	Cores          int
	EphemeralDisks []int
	Burstable      bool
}

type EphemeralDevice struct {
	DeviceName  string
	VirtualName string
	SizeGB      int
}

func (m *AWSMachineTypeInfo) EphemeralDevices() ([]*EphemeralDevice, error) {
	var disks []*EphemeralDevice
	for i, sizeGB := range m.EphemeralDisks {
		d := &EphemeralDevice{
			SizeGB: sizeGB,
		}

		if i >= 20 {
			// TODO: What drive letters do we use?
			glog.Fatalf("ephemeral devices for > 20 not yet implemented")
		}
		d.DeviceName = "/dev/sd" + string('c'+i)
		d.VirtualName = fmt.Sprintf("ephemeral%d", i)

		disks = append(disks, d)
	}
	return disks, nil
}

func GetMachineTypeInfo(machineType string) (*AWSMachineTypeInfo, error) {
	for i := range MachineTypes {
		m := &MachineTypes[i]
		if m.Name == machineType {
			return m, nil
		}
	}

	return nil, fmt.Errorf("instance type not handled: %q", machineType)
}

var MachineTypes []AWSMachineTypeInfo = []AWSMachineTypeInfo{
	// This is tedious, but seems simpler than trying to have some logic and then a lot of exceptions

	// t2 family
	{
		Name:           "t2.nano",
		MemoryGB:       0.5,
		ECU:            3 * BurstableCreditsToECUS,
		Cores:          1,
		EphemeralDisks: nil,
		Burstable:      true,
	},
	{
		Name:           "t2.micro",
		MemoryGB:       1,
		ECU:            6 * BurstableCreditsToECUS,
		Cores:          1,
		EphemeralDisks: nil,
		Burstable:      true,
	},
	{
		Name:           "t2.small",
		MemoryGB:       2,
		ECU:            12 * BurstableCreditsToECUS,
		Cores:          1,
		EphemeralDisks: nil,
		Burstable:      true,
	},
	{
		Name:           "t2.medium",
		MemoryGB:       4,
		ECU:            24 * BurstableCreditsToECUS,
		Cores:          2,
		EphemeralDisks: nil,
		Burstable:      true,
	},
	{
		Name:           "t2.large",
		MemoryGB:       8,
		ECU:            36 * BurstableCreditsToECUS,
		Cores:          2,
		EphemeralDisks: nil,
		Burstable:      true,
	},

	// m3 family
	{
		Name:           "m3.medium",
		MemoryGB:       3.75,
		ECU:            3,
		Cores:          1,
		EphemeralDisks: []int{4},
	},
	{
		Name:           "m3.large",
		MemoryGB:       7.5,
		ECU:            6.5,
		Cores:          2,
		EphemeralDisks: []int{32},
	},
	{
		Name:           "m3.xlarge",
		MemoryGB:       15,
		ECU:            13,
		Cores:          4,
		EphemeralDisks: []int{40, 40},
	},
	{
		Name:           "m3.2xlarge",
		MemoryGB:       30,
		ECU:            26,
		Cores:          8,
		EphemeralDisks: []int{80, 80},
	},

	// m4 family
	{
		Name:           "m4.large",
		MemoryGB:       8,
		ECU:            6.5,
		Cores:          2,
		EphemeralDisks: nil,
	},
	{
		Name:           "m4.xlarge",
		MemoryGB:       16,
		ECU:            13,
		Cores:          4,
		EphemeralDisks: nil,
	},
	{
		Name:           "m4.2xlarge",
		MemoryGB:       32,
		ECU:            26,
		Cores:          8,
		EphemeralDisks: nil,
	},
	{
		Name:           "m4.4xlarge",
		MemoryGB:       64,
		ECU:            53.5,
		Cores:          16,
		EphemeralDisks: nil,
	},
	{
		Name:           "m4.10xlarge",
		MemoryGB:       160,
		ECU:            124.5,
		Cores:          40,
		EphemeralDisks: nil,
	},

	// c3 family
	{
		Name:           "c3.large",
		MemoryGB:       3.75,
		ECU:            7,
		Cores:          2,
		EphemeralDisks: []int{16, 16},
	},
	{
		Name:           "c3.xlarge",
		MemoryGB:       7.5,
		ECU:            14,
		Cores:          4,
		EphemeralDisks: []int{40, 40},
	},
	{
		Name:           "c3.2xlarge",
		MemoryGB:       15,
		ECU:            28,
		Cores:          8,
		EphemeralDisks: []int{80, 80},
	},
	{
		Name:           "c3.4xlarge",
		MemoryGB:       30,
		ECU:            55,
		Cores:          16,
		EphemeralDisks: []int{160, 160},
	},
	{
		Name:           "c3.8xlarge",
		MemoryGB:       60,
		ECU:            108,
		Cores:          32,
		EphemeralDisks: []int{320, 320},
	},

	// c4 family
	{
		Name:           "c4.large",
		MemoryGB:       3.75,
		ECU:            8,
		Cores:          2,
		EphemeralDisks: nil,
	},
	{
		Name:           "c4.xlarge",
		MemoryGB:       7.5,
		ECU:            16,
		Cores:          4,
		EphemeralDisks: nil,
	},
	{
		Name:           "c4.2xlarge",
		MemoryGB:       15,
		ECU:            31,
		Cores:          8,
		EphemeralDisks: nil,
	},
	{
		Name:           "c4.4xlarge",
		MemoryGB:       30,
		ECU:            62,
		Cores:          16,
		EphemeralDisks: nil,
	},
	{
		Name:           "c4.8xlarge",
		MemoryGB:       60,
		ECU:            132,
		Cores:          32,
		EphemeralDisks: nil,
	},
}
<|endoftext|>