<commit_before>package main

import (
	"bufio"
	"fmt"
	"log"
	"math"
	"os"
	"regexp"
	"runtime"
	"strings"
)

type ParseData map[string]int
type MultiSetData map[string]map[string]int

func CalcDotProduct(v1, v2 []float64) float64 {
	result := 0.0

	for index, _ := range v1 {
		result += v1[index] * v2[index]
	}

	return result
}

func CalcMagnitude(v []float64) float64 {
	if len(v) == 0 {
		return 0.0
	}

	var subtotal float64

	for _, somefloat := range v {
		subtotal += float64(somefloat * somefloat)
	}

	return math.Sqrt(subtotal)
}

func BuildVector(outerkey, innerkey string, outervalue, innervalue map[string]int) ([]float64, []float64) {
	maxlength := len(outervalue) * len(innervalue)

	outervector := make([]float64, 0, maxlength)
	innervector := make([]float64, 0, maxlength)

	commonkeys := make(map[string]bool)

	for outerstring, _ := range outervalue {
		commonkeys[outerstring] = true
	}

	for innerstring, _ := range innervalue {
		commonkeys[innerstring] = true
	}

	if len(commonkeys) == 0 {
		return outervector, innervector
	}

	for commonstring, commonbool := range commonkeys {
		if commonbool == true {
			outervector = append(outervector, float64(outervalue[commonstring]))
			innervector = append(innervector, float64(innervalue[commonstring]))
		}
	}

	return outervector, innervector
}

func CalcCosim(v1, v2 []float64) float64 {
	dotproduct := CalcDotProduct(v1, v2)

	if dotproduct > 0 {
		v1m := CalcMagnitude(v1)
		v2m := CalcMagnitude(v2)

		return dotproduct / (v1m * v2m)
	} else {
		return 0.0
	}
}

func CleanString(s string) string {
	reg, err := regexp.Compile("[^A-Za-z0-9 ]+")

	if err != nil {
		log.Fatal(err)
	}

	safe := reg.ReplaceAllString(s, "")
	safe = strings.ToLower(strings.Trim(safe, ""))
	safe = strings.TrimSpace(safe)

	return safe
}

func Parse(s ParseData) {
	scanner := bufio.NewScanner(os.Stdin)

	for scanner.Scan() {
		somestring := CleanString(scanner.Text())

		if len(somestring) == 0 {
			continue
		}

		t := strings.Split(somestring, " ")

		if len(t) <= 1 {
			continue
		}

		for i := 0; i < (len(t) - 1); i++ {
			k1 := t[i] + "##" + t[i+1]
			k2 := t[i+1] + "##" + t[i]

			s[k1] += 1
			s[k2] += 1
		}
	}
}

func BuildMultiSet(s ParseData, v MultiSetData) {
	for key, value := range s {
		keys := strings.Split(key, "##")

		if v[keys[0]] == nil {
			v[keys[0]] = make(map[string]int)
		} else {
			keys := strings.Split(key, "##")
			v[keys[0]][keys[1]] = value
		}

	}
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	p := make(ParseData)
	m := make(MultiSetData)
	fmt.Println("GOing to parse")
	Parse(p)
	fmt.Println("GOing to build multi set")
	BuildMultiSet(p, m)

	var keys []string

	if len(os.Args) > 1 {
		keys = os.Args[1:]
	}

	fmt.Println("GOing to find any matches")

	// for outerkey, outervalue := range m {
	for _, outerkey := range keys {
		for innerkey, innervalue := range m {
			// v1, v2 := BuildVector(outerkey, innerkey, outervalue, innervalue)
			v1, v2 := BuildVector(outerkey, innerkey, m[outerkey], innervalue)

			if cosign := CalcCosim(v1, v2); cosign > 0.78 {
				fmt.Println(outerkey, innerkey, "\t\t", math.Floor(cosign*100))
			}
		}
	}
}
<commit_msg>reduce loops... or did I...<commit_after>package main

import (
	"bufio"
	"fmt"
	"log"
	"math"
	"os"
	"regexp"
	"runtime"
	"strings"
)

type ParseData map[string]int
type MultiSetData map[string]map[string]int

func CalcDotProduct(v1, v2 []float64) float64 {
	result := 0.0

	for index, _ := range v1 {
		result += v1[index] * v2[index]
	}

	return result
}

func CalcMagnitude(v []float64) float64 {
	if len(v) == 0 {
		return 0.0
	}

	var subtotal float64

	for _, somefloat := range v {
		subtotal += float64(somefloat * somefloat)
	}

	return math.Sqrt(subtotal)
}

func BuildVector(outerkey, innerkey string, outervalue, innervalue map[string]int) ([]float64, []float64) {
	maxlength := len(outervalue) * len(innervalue)

	outervector := make([]float64, 0, maxlength)
	innervector := make([]float64, 0, maxlength)

	commonkeys := make(map[string]bool)

	for outerstring, _ := range outervalue {
		commonkeys[outerstring] = true
	}

	for innerstring, _ := range innervalue {
		commonkeys[innerstring] = true
	}

	if len(commonkeys) == 0 {
		return outervector, innervector
	}

	for commonstring, commonbool := range commonkeys {
		if commonbool == true {
			outervector = append(outervector, float64(outervalue[commonstring]))
			innervector = append(innervector, float64(innervalue[commonstring]))
		}
	}

	return outervector, innervector
}

func CalcCosim(v1, v2 []float64) float64 {
	dotproduct := CalcDotProduct(v1, v2)

	if dotproduct > 0 {
		v1m := CalcMagnitude(v1)
		v2m := CalcMagnitude(v2)

		return dotproduct / (v1m * v2m)
	} else {
		return 0.0
	}
}

func CleanString(s string) string {
	reg, err := regexp.Compile("[^A-Za-z0-9 ]+")

	if err != nil {
		log.Fatal(err)
	}

	safe := reg.ReplaceAllString(s, "")
	safe = strings.ToLower(strings.Trim(safe, ""))
	safe = strings.TrimSpace(safe)

	return safe
}

func Parse(m MultiSetData) {
	scanner := bufio.NewScanner(os.Stdin)

	for scanner.Scan() {
		somestring := CleanString(scanner.Text())

		if len(somestring) == 0 {
			continue
		}

		t := strings.Split(somestring, " ")

		if len(t) <= 1 {
			continue
		}

		for i := 0; i < (len(t) - 1); i++ {
			if m[t[i]] == nil {
				m[t[i]] = make(map[string]int)
			}
			m[t[i]][t[i+1]] += 1

			if m[t[i+1]] == nil {
				m[t[i+1]] = make(map[string]int)
			}
			m[t[i+1]][t[i]] += 1
		}
	}
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	m := make(MultiSetData)

	fmt.Println("GOing to parse")

	Parse(m)

	var keys []string

	if len(os.Args) > 1 {
		keys = os.Args[1:]
	}

	fmt.Println("GOing to find any matches")

	for _, outerkey := range keys {
		for innerkey, innervalue := range m {
			v1, v2 := BuildVector(outerkey, innerkey, m[outerkey], innervalue)

			if cosign := CalcCosim(v1, v2); cosign > 0.78 {
				fmt.Println(outerkey, innerkey, "\t\t", math.Floor(cosign*100))
			}
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 Harrison Shoebridge. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package engi

import "math"

// A rather basic camera
type Camera struct {
	pos, to  Point
	tracking *Entity // The entity that is currently being followed
}

func (cam *Camera) FollowEntity(entity *Entity) {
	cam.tracking = entity
	var space *SpaceComponent
	if !cam.tracking.GetComponent(&space) {
		return
	}

	cam.to = space.Position
	cam.centerCam(Width(), Height(), 1, space)
}

func (cam *Camera) Update(dt float32) {
	// maxPoints := 8
	if cam.tracking != nil {
		var space *SpaceComponent
		if !cam.tracking.GetComponent(&space) {
			return
		}
		cam.centerCam(Width(), Height(), 0.09, space)
	}
}

func (cam *Camera) centerCam(width, height, lerp float32, space *SpaceComponent) {
	cam.to.X += ((space.Position.X + space.Width/2) - (cam.to.X + width/2)) * lerp
	cam.to.Y += ((space.Position.Y + space.Height/2) - (cam.to.Y + height/2)) * lerp

	minX, minY := cam.to.X-100, cam.to.Y-100
	maxX, maxY := minX+200, minY+200

	if cam.pos.X < minX || cam.pos.X > maxX {
		cam.pos.X = Clamp(floorFloat32(minX), WorldBounds.Min.X, WorldBounds.Max.X-width)
	}

	if cam.pos.Y < minY || cam.pos.Y > maxY {
		cam.pos.Y = Clamp(floorFloat32(minY), WorldBounds.Min.Y, WorldBounds.Max.Y-height)
	}
}

func floorFloat32(i float32) float32 {
	return float32(math.Floor(float64(i)))
}
<commit_msg>camera: add changeable deadzone size<commit_after>// Copyright 2014 Harrison Shoebridge. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package engi

import "math"

// A rather basic camera
type Camera struct {
	DeadzoneSize Point
	pos, to      Point
	tracking     *Entity // The entity that is currently being followed
}

func (cam *Camera) FollowEntity(entity *Entity) {
	cam.tracking = entity
	var space *SpaceComponent
	if !cam.tracking.GetComponent(&space) {
		return
	}

	cam.to = space.Position
	cam.centerCam(Width(), Height(), 1, space)
}

func (cam *Camera) Update(dt float32) {
	if cam.tracking != nil {
		var space *SpaceComponent
		if !cam.tracking.GetComponent(&space) {
			return
		}
		cam.centerCam(Width(), Height(), 0.09, space)
	}
}

func (cam *Camera) centerCam(width, height, lerp float32, space *SpaceComponent) {
	cam.to.X += ((space.Position.X + space.Width/2) - (cam.to.X + width/2)) * lerp
	cam.to.Y += ((space.Position.Y + space.Height/2) - (cam.to.Y + height/2)) * lerp

	dWidth := cam.DeadzoneSize.X
	dHeight := cam.DeadzoneSize.Y

	if dWidth == 0 {
		dWidth = 200
	}

	if dHeight == 0 {
		dHeight = 200
	}

	minX, minY := cam.to.X-(dWidth/2), cam.to.Y-(dHeight/2)
	maxX, maxY := minX+dWidth, minY+dHeight

	if cam.pos.X < minX || cam.pos.X > maxX {
		cam.pos.X = Clamp(floorFloat32(minX), WorldBounds.Min.X, WorldBounds.Max.X-width)
	}

	if cam.pos.Y < minY || cam.pos.Y > maxY {
		cam.pos.Y = Clamp(floorFloat32(minY), WorldBounds.Min.Y, WorldBounds.Max.Y-height)
	}
}

func floorFloat32(i float32) float32 {
	return float32(math.Floor(float64(i)))
}
<|endoftext|>


<commit_before>package terraform

import (
	"strings"
	"testing"
)

func TestContextImport_basic(t *testing.T) {
	p := testProvider("aws")
	ctx := testContext2(t, &ContextOpts{
		Providers: map[string]ResourceProviderFactory{
			"aws": testProviderFuncFixed(p),
		},
	})

	p.ImportStateReturn = []*InstanceState{
		&InstanceState{
			ID:        "foo",
			Ephemeral: EphemeralState{Type: "aws_instance"},
		},
	}

	state, err := ctx.Import(&ImportOpts{
		Targets: []*ImportTarget{
			&ImportTarget{
				Addr: "aws_instance.foo",
				ID:   "bar",
			},
		},
	})
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	actual := strings.TrimSpace(state.String())
	expected := strings.TrimSpace(testImportStr)
	if actual != expected {
		t.Fatalf("bad: \n%s", actual)
	}
}

func TestContextImport_missingType(t *testing.T) {
	p := testProvider("aws")
	ctx := testContext2(t, &ContextOpts{
		Providers: map[string]ResourceProviderFactory{
			"aws": testProviderFuncFixed(p),
		},
	})

	p.ImportStateReturn = []*InstanceState{
		&InstanceState{
			ID: "foo",
		},
	}

	state, err := ctx.Import(&ImportOpts{
		Targets: []*ImportTarget{
			&ImportTarget{
				Addr: "aws_instance.foo",
				ID:   "bar",
			},
		},
	})
	if err == nil {
		t.Fatal("should error")
	}

	actual := strings.TrimSpace(state.String())
	expected := "<nil>"
	if actual != expected {
		t.Fatalf("bad: \n%s", actual)
	}
}

func TestContextImport_refresh(t *testing.T) {
	p := testProvider("aws")
	ctx := testContext2(t, &ContextOpts{
		Providers: map[string]ResourceProviderFactory{
			"aws": testProviderFuncFixed(p),
		},
	})

	p.ImportStateReturn = []*InstanceState{
		&InstanceState{
			ID:        "foo",
			Ephemeral: EphemeralState{Type: "aws_instance"},
		},
	}

	p.RefreshFn = func(info *InstanceInfo, s *InstanceState) (*InstanceState, error) {
		return &InstanceState{
			ID:         "foo",
			Attributes: map[string]string{"foo": "bar"},
		}, nil
	}

	state, err := ctx.Import(&ImportOpts{
		Targets: []*ImportTarget{
			&ImportTarget{
				Addr: "aws_instance.foo",
				ID:   "bar",
			},
		},
	})
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	actual := strings.TrimSpace(state.String())
	expected := strings.TrimSpace(testImportRefreshStr)
	if actual != expected {
		t.Fatalf("bad: \n%s", actual)
	}
}

func TestContextImport_module(t *testing.T) {
	p := testProvider("aws")
	ctx := testContext2(t, &ContextOpts{
		Providers: map[string]ResourceProviderFactory{
			"aws": testProviderFuncFixed(p),
		},
	})

	p.ImportStateReturn = []*InstanceState{
		&InstanceState{
			ID:        "foo",
			Ephemeral: EphemeralState{Type: "aws_instance"},
		},
	}

	state, err := ctx.Import(&ImportOpts{
		Targets: []*ImportTarget{
			&ImportTarget{
				Addr: "module.foo.aws_instance.foo",
				ID:   "bar",
			},
		},
	})
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	actual := strings.TrimSpace(state.String())
	expected := strings.TrimSpace(testImportModuleStr)
	if actual != expected {
		t.Fatalf("bad: \n%s", actual)
	}
}

func TestContextImport_moduleDepth2(t *testing.T) {
	p := testProvider("aws")
	ctx := testContext2(t, &ContextOpts{
		Providers: map[string]ResourceProviderFactory{
			"aws": testProviderFuncFixed(p),
		},
	})

	p.ImportStateReturn = []*InstanceState{
		&InstanceState{
			ID:        "foo",
			Ephemeral: EphemeralState{Type: "aws_instance"},
		},
	}

	state, err := ctx.Import(&ImportOpts{
		Targets: []*ImportTarget{
			&ImportTarget{
				Addr: "module.a.module.b.aws_instance.foo",
				ID:   "bar",
			},
		},
	})
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	actual := strings.TrimSpace(state.String())
	expected := strings.TrimSpace(testImportModuleDepth2Str)
	if actual != expected {
		t.Fatalf("bad: \n%s", actual)
	}
}

func TestContextImport_moduleExisting(t *testing.T) {
	p := testProvider("aws")
	ctx := testContext2(t, &ContextOpts{
		Providers: map[string]ResourceProviderFactory{
			"aws": testProviderFuncFixed(p),
		},

		State: &State{
			Modules: []*ModuleState{
				&ModuleState{
					Path: []string{"root", "foo"},
					Resources: map[string]*ResourceState{
						"aws_instance.bar": &ResourceState{
							Type: "aws_instance",
							Primary: &InstanceState{
								ID: "bar",
							},
						},
					},
				},
			},
		},
	})

	p.ImportStateReturn = []*InstanceState{
		&InstanceState{
			ID:        "foo",
			Ephemeral: EphemeralState{Type: "aws_instance"},
		},
	}

	state, err := ctx.Import(&ImportOpts{
		Targets: []*ImportTarget{
			&ImportTarget{
				Addr: "module.foo.aws_instance.foo",
				ID:   "bar",
			},
		},
	})
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	actual := strings.TrimSpace(state.String())
	expected := strings.TrimSpace(testImportModuleExistingStr)
	if actual != expected {
		t.Fatalf("bad: \n%s", actual)
	}
}

const testImportStr = `
aws_instance.foo:
  ID = foo
  provider = aws
`

const testImportModuleStr = `
<no state>
module.foo:
  aws_instance.foo:
    ID = foo
    provider = aws
`

const testImportModuleDepth2Str = `
<no state>
module.a.b:
  aws_instance.foo:
    ID = foo
    provider = aws
`

const testImportModuleExistingStr = `
module.foo:
  aws_instance.bar:
    ID = bar
  aws_instance.foo:
    ID = foo
    provider = aws
`

const testImportRefreshStr = `
aws_instance.foo:
  ID = foo
  provider = aws
  foo = bar
`
<commit_msg>terraform: test importing into a new module with other modules<commit_after>package terraform

import (
	"strings"
	"testing"
)

func TestContextImport_basic(t *testing.T) {
	p := testProvider("aws")
	ctx := testContext2(t, &ContextOpts{
		Providers: map[string]ResourceProviderFactory{
			"aws": testProviderFuncFixed(p),
		},
	})

	p.ImportStateReturn = []*InstanceState{
		&InstanceState{
			ID:        "foo",
			Ephemeral: EphemeralState{Type: "aws_instance"},
		},
	}

	state, err := ctx.Import(&ImportOpts{
		Targets: []*ImportTarget{
			&ImportTarget{
				Addr: "aws_instance.foo",
				ID:   "bar",
			},
		},
	})
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	actual := strings.TrimSpace(state.String())
	expected := strings.TrimSpace(testImportStr)
	if actual != expected {
		t.Fatalf("bad: \n%s", actual)
	}
}

func TestContextImport_missingType(t *testing.T) {
	p := testProvider("aws")
	ctx := testContext2(t, &ContextOpts{
		Providers: map[string]ResourceProviderFactory{
			"aws": testProviderFuncFixed(p),
		},
	})

	p.ImportStateReturn = []*InstanceState{
		&InstanceState{
			ID: "foo",
		},
	}

	state, err := ctx.Import(&ImportOpts{
		Targets: []*ImportTarget{
			&ImportTarget{
				Addr: "aws_instance.foo",
				ID:   "bar",
			},
		},
	})
	if err == nil {
		t.Fatal("should error")
	}

	actual := strings.TrimSpace(state.String())
	expected := "<nil>"
	if actual != expected {
		t.Fatalf("bad: \n%s", actual)
	}
}

func TestContextImport_refresh(t *testing.T) {
	p := testProvider("aws")
	ctx := testContext2(t, &ContextOpts{
		Providers: map[string]ResourceProviderFactory{
			"aws": testProviderFuncFixed(p),
		},
	})

	p.ImportStateReturn = []*InstanceState{
		&InstanceState{
			ID:        "foo",
			Ephemeral: EphemeralState{Type: "aws_instance"},
		},
	}

	p.RefreshFn = func(info *InstanceInfo, s *InstanceState) (*InstanceState, error) {
		return &InstanceState{
			ID:         "foo",
			Attributes: map[string]string{"foo": "bar"},
		}, nil
	}

	state, err := ctx.Import(&ImportOpts{
		Targets: []*ImportTarget{
			&ImportTarget{
				Addr: "aws_instance.foo",
				ID:   "bar",
			},
		},
	})
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	actual := strings.TrimSpace(state.String())
	expected := strings.TrimSpace(testImportRefreshStr)
	if actual != expected {
		t.Fatalf("bad: \n%s", actual)
	}
}

func TestContextImport_module(t *testing.T) {
	p := testProvider("aws")
	ctx := testContext2(t, &ContextOpts{
		Providers: map[string]ResourceProviderFactory{
			"aws": testProviderFuncFixed(p),
		},
	})

	p.ImportStateReturn = []*InstanceState{
		&InstanceState{
			ID:        "foo",
			Ephemeral: EphemeralState{Type: "aws_instance"},
		},
	}

	state, err := ctx.Import(&ImportOpts{
		Targets: []*ImportTarget{
			&ImportTarget{
				Addr: "module.foo.aws_instance.foo",
				ID:   "bar",
			},
		},
	})
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	actual := strings.TrimSpace(state.String())
	expected := strings.TrimSpace(testImportModuleStr)
	if actual != expected {
		t.Fatalf("bad: \n%s", actual)
	}
}

func TestContextImport_moduleDepth2(t *testing.T) {
	p := testProvider("aws")
	ctx := testContext2(t, &ContextOpts{
		Providers: map[string]ResourceProviderFactory{
			"aws": testProviderFuncFixed(p),
		},
	})

	p.ImportStateReturn = []*InstanceState{
		&InstanceState{
			ID:        "foo",
			Ephemeral: EphemeralState{Type: "aws_instance"},
		},
	}

	state, err := ctx.Import(&ImportOpts{
		Targets: []*ImportTarget{
			&ImportTarget{
				Addr: "module.a.module.b.aws_instance.foo",
				ID:   "bar",
			},
		},
	})
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	actual := strings.TrimSpace(state.String())
	expected := strings.TrimSpace(testImportModuleDepth2Str)
	if actual != expected {
		t.Fatalf("bad: \n%s", actual)
	}
}

func TestContextImport_moduleDiff(t *testing.T) {
	p := testProvider("aws")
	ctx := testContext2(t, &ContextOpts{
		Providers: map[string]ResourceProviderFactory{
			"aws": testProviderFuncFixed(p),
		},

		State: &State{
			Modules: []*ModuleState{
				&ModuleState{
					Path: []string{"root", "bar"},
					Resources: map[string]*ResourceState{
						"aws_instance.bar": &ResourceState{
							Type: "aws_instance",
							Primary: &InstanceState{
								ID: "bar",
							},
						},
					},
				},
			},
		},
	})

	p.ImportStateReturn = []*InstanceState{
		&InstanceState{
			ID:        "foo",
			Ephemeral: EphemeralState{Type: "aws_instance"},
		},
	}

	state, err := ctx.Import(&ImportOpts{
		Targets: []*ImportTarget{
			&ImportTarget{
				Addr: "module.foo.aws_instance.foo",
				ID:   "bar",
			},
		},
	})
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	actual := strings.TrimSpace(state.String())
	expected := strings.TrimSpace(testImportModuleDiffStr)
	if actual != expected {
		t.Fatalf("bad: \n%s", actual)
	}
}

func TestContextImport_moduleExisting(t *testing.T) {
	p := testProvider("aws")
	ctx := testContext2(t, &ContextOpts{
		Providers: map[string]ResourceProviderFactory{
			"aws": testProviderFuncFixed(p),
		},

		State: &State{
			Modules: []*ModuleState{
				&ModuleState{
					Path: []string{"root", "foo"},
					Resources: map[string]*ResourceState{
						"aws_instance.bar": &ResourceState{
							Type: "aws_instance",
							Primary: &InstanceState{
								ID: "bar",
							},
						},
					},
				},
			},
		},
	})

	p.ImportStateReturn = []*InstanceState{
		&InstanceState{
			ID:        "foo",
			Ephemeral: EphemeralState{Type: "aws_instance"},
		},
	}

	state, err := ctx.Import(&ImportOpts{
		Targets: []*ImportTarget{
			&ImportTarget{
				Addr: "module.foo.aws_instance.foo",
				ID:   "bar",
			},
		},
	})
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	actual := strings.TrimSpace(state.String())
	expected := strings.TrimSpace(testImportModuleExistingStr)
	if actual != expected {
		t.Fatalf("bad: \n%s", actual)
	}
}

const testImportStr = `
aws_instance.foo:
  ID = foo
  provider = aws
`

const testImportModuleStr = `
<no state>
module.foo:
  aws_instance.foo:
    ID = foo
    provider = aws
`

const testImportModuleDepth2Str = `
<no state>
module.a.b:
  aws_instance.foo:
    ID = foo
    provider = aws
`

const testImportModuleDiffStr = `
module.bar:
  aws_instance.bar:
    ID = bar
module.foo:
  aws_instance.foo:
    ID = foo
    provider = aws
`

const testImportModuleExistingStr = `
module.foo:
  aws_instance.bar:
    ID = bar
  aws_instance.foo:
    ID = foo
    provider = aws
`

const testImportRefreshStr = `
aws_instance.foo:
  ID = foo
  provider = aws
  foo = bar
`
<|endoftext|>


<commit_before>package completion

import (
	"os"
	"path"
	"regexp"
	"strings"

	"../dos"
	"../interpreter"
)

var rxEnvPattern = regexp.MustCompile("%[^%]+%")
var rxTilde = regexp.MustCompile("^~[/\\\\]")

func listUpFiles(str string) ([]string, error) {
	str = rxEnvPattern.ReplaceAllStringFunc(str, func(p string) string {
		if len(p) == 2 {
			return "%"
		} else if val := os.Getenv(p[1 : len(p)-1]); val != "" {
			return val
		} else if f, ok := interpreter.PercentFunc[p[1:len(p)-1]]; ok {
			return f()
		} else {
			return p
		}
	})

	str = rxTilde.ReplaceAllStringFunc(str, func(p string) string {
		if home := dos.GetHome(); home != "" {
			return home + "\\"
		} else {
			return p
		}
	})
	str = strings.Replace(strings.Replace(str, "\\", "/", -1), "\"", "", -1)
	var directory string
	str = strings.Replace(str, "\\", "/", -1)
	if strings.HasSuffix(str, "/") {
		directory = path.Join(str, ".")
	} else {
		directory = path.Dir(str)
	}
	cutprefix := 0
	if strings.HasPrefix(directory, "/") {
		wd, _ := os.Getwd()
		directory = wd[0:2] + directory
		cutprefix = 2
	}

	fd, fdErr := os.Open(directory)
	if fdErr != nil {
		return nil, fdErr
	}
	defer fd.Close()
	files, filesErr := fd.Readdir(-1)
	if filesErr != nil {
		return nil, filesErr
	}
	commons := make([]string, 0)
	STR := strings.ToUpper(path.Clean(str))
	for _, node1 := range files {
		name := path.Join(directory, node1.Name())
		if attr, attrErr := dos.GetFileAttributes(name); attrErr == nil && (attr&dos.FILE_ATTRIBUTE_HIDDEN) != 0 {
			continue
		}
		if node1.IsDir() {
			name += "/"
		}
		if cutprefix > 0 {
			name = name[2:]
		}
		nameUpr := strings.ToUpper(path.Clean(name))
		if strings.HasPrefix(nameUpr, STR) {
			commons = append(commons, name)
		}
	}
	return commons, nil
}
<commit_msg>Fix #46 complete empty-path to files starting .<commit_after>package completion

import (
	"os"
	"path"
	"regexp"
	"strings"

	"../dos"
	"../interpreter"
)

var rxEnvPattern = regexp.MustCompile("%[^%]+%")
var rxTilde = regexp.MustCompile("^~[/\\\\]")

func listUpFiles(str string) ([]string, error) {
	str = rxEnvPattern.ReplaceAllStringFunc(str, func(p string) string {
		if len(p) == 2 {
			return "%"
		} else if val := os.Getenv(p[1 : len(p)-1]); val != "" {
			return val
		} else if f, ok := interpreter.PercentFunc[p[1:len(p)-1]]; ok {
			return f()
		} else {
			return p
		}
	})

	str = rxTilde.ReplaceAllStringFunc(str, func(p string) string {
		if home := dos.GetHome(); home != "" {
			return home + "\\"
		} else {
			return p
		}
	})
	str = strings.Replace(strings.Replace(str, "\\", "/", -1), "\"", "", -1)
	var directory string
	str = strings.Replace(str, "\\", "/", -1)
	if strings.HasSuffix(str, "/") {
		directory = path.Join(str, ".")
	} else {
		directory = path.Dir(str)
	}

	// Drive letter
	cutprefix := 0
	if strings.HasPrefix(directory, "/") {
		wd, _ := os.Getwd()
		directory = wd[0:2] + directory
		cutprefix = 2
	}

	fd, fdErr := os.Open(directory)
	if fdErr != nil {
		return nil, fdErr
	}
	defer fd.Close()
	files, filesErr := fd.Readdir(-1)
	if filesErr != nil {
		return nil, filesErr
	}
	commons := make([]string, 0)
	if str != "" {
		str = path.Clean(str)
		// Since path.Clean("") -> ".", completed name to ".xxxx"
	}
	STR := strings.ToUpper(str)
	for _, node1 := range files {
		name := path.Join(directory, node1.Name())
		if attr, attrErr := dos.GetFileAttributes(name); attrErr == nil && (attr&dos.FILE_ATTRIBUTE_HIDDEN) != 0 {
			continue
		}
		if node1.IsDir() {
			name += "/"
		}
		if cutprefix > 0 {
			name = name[2:]
		}
		nameUpr := strings.ToUpper(path.Clean(name))
		if strings.HasPrefix(nameUpr, STR) {
			commons = append(commons, name)
		}
	}
	return commons, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2014 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"github.com/robertosgm/example/stringutil"
	"net/http"
	"log"
	"html"
	"crypto/tls"
	"crypto/x509"
	"io/ioutil"
)

func barHandler(w http.ResponseWriter, r *http.Request) {
        fmt.Fprintf(w, stringutil.Reverse("!selpmaxe oG ,olleH")+ "\n")
        fmt.Fprintf(w, "%q" + ":" + "%q" + "\n", r.Method, r.Proto)
        fmt.Fprintf(w, "%q" + ":" + "%q" + "\n", html.EscapeString(r.URL.Path), r.RemoteAddr)
}

func main() {
	caCert, err := ioutil.ReadFile("./rootCA.pem")
	if err != nil {
		log.Fatal(err)
	}
	caCertPool := x509.NewCertPool()
	caCertPool.AppendCertsFromPEM(caCert)
	cert, err := tls.LoadX509KeyPair("./server.crt", "./server.key")
	if err != nil {
		log.Fatal(err)
	}
	config := tls.Config{
		Certificates: []tls.Certificate{cert},
		ClientAuth: tls.RequireAndVerifyClientCert,
		ClientCAs: caCertPool,
	}

	s := &http.Server{
		Addr:           ":4443",
		TLSConfig:	&config,
	}
	http.Handle("/", http.HandlerFunc(barHandler))
	log.Fatal(s.ListenAndServeTLS("",""))
}
<commit_msg>Just some format.<commit_after>/*
Copyright 2014 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"github.com/robertosgm/example/stringutil"
	"html"
	"io/ioutil"
	"log"
	"net/http"
)

func barHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, stringutil.Reverse("!selpmaxe oG ,olleH")+"\n")
	fmt.Fprintf(w, "%q"+":"+"%q"+"\n", r.Method, r.Proto)
	fmt.Fprintf(w, "%q"+":"+"%q"+"\n", html.EscapeString(r.URL.Path), r.RemoteAddr)
}

func main() {
	caCert, err := ioutil.ReadFile("./rootCA.pem")
	if err != nil {
		log.Fatal(err)
	}
	caCertPool := x509.NewCertPool()
	caCertPool.AppendCertsFromPEM(caCert)
	cert, err := tls.LoadX509KeyPair("./server.crt", "./server.key")
	if err != nil {
		log.Fatal(err)
	}
	config := tls.Config{
		Certificates: []tls.Certificate{cert},
		ClientAuth:   tls.RequireAndVerifyClientCert,
		ClientCAs:    caCertPool,
	}

	s := &http.Server{
		Addr:      ":4443",
		TLSConfig: &config,
	}
	http.Handle("/", http.HandlerFunc(barHandler))
	log.Fatal(s.ListenAndServeTLS("", ""))
}
<|endoftext|>


<commit_before>package main

import "fmt"
//import "github.com/ropes/palindromes"
import "github.com/ropes/gotour"
import "code.google.com/p/go-tour/pic"

func main(){
    fmt.Println("Hellow!!! I like Go's install so far!  Very straitforward!")
    pic.Show(gotour.Pic)
}

<commit_msg>Hello go world<commit_after>package main

import "fmt"

//import "github.com/ropes/palindromes"
import "github.com/ropes/learngo/gotour"
import "code.google.com/p/go-tour/pic"

func main() {
	fmt.Println("Hellow!!! I like Go's install so far!  Very straitforward!")
	pic.Show(gotour.Pic)
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"io"
	"net"
	"os"
	"path/filepath"
	"runtime"
	"testing"
	"time"

	"github.com/taskcluster/slugid-go/slugid"
	tcclient "github.com/taskcluster/taskcluster-client-go"
	"github.com/taskcluster/taskcluster-client-go/queue"
)

type PayloadArtifact struct {
	Expires tcclient.Time `json:"expires,omitempty"`
	Name    string        `json:"name,omitempty"`
	Path    string        `json:"path"`
	Type    string        `json:"type"`
}

var (
	inAnHour       tcclient.Time
	testdataDir    string
	globalTestName string
)

func setup(t *testing.T, testName string) {
	// some basic setup...
	cwd, err := os.Getwd()
	if err != nil {
		t.Fatalf("Test failed during setup phase!")
	}
	testdataDir = filepath.Join(cwd, "testdata")

	// configure the worker
	config = &Config{
		AccessToken:                    os.Getenv("TASKCLUSTER_ACCESS_TOKEN"),
		CachesDir:                      filepath.Join(cwd, "caches"),
		Certificate:                    os.Getenv("TASKCLUSTER_CERTIFICATE"),
		CheckForNewDeploymentEverySecs: 0,
		CleanUpTaskDirs:                false,
		ClientID:                       os.Getenv("TASKCLUSTER_CLIENT_ID"),
		DeploymentID:                   "",
		DownloadsDir:                   filepath.Join(cwd, "downloads"),
		IdleTimeoutSecs:                60,
		InstanceID:                     "test-instance-id",
		InstanceType:                   "p3.enormous",
		LiveLogCertificate:             "",
		LiveLogExecutable:              "livelog",
		LiveLogGETPort:                 30582,
		LiveLogKey:                     "",
		LiveLogPUTPort:                 43264,
		LiveLogSecret:                  "xyz",
		NumberOfTasksToRun:             1,
		PrivateIP:                      net.ParseIP("87.65.43.21"),
		ProvisionerID:                  "test-provisioner",
		PublicIP:                       net.ParseIP("12.34.56.78"),
		RefreshUrlsPrematurelySecs:     310,
		Region: "outer-space",
		RequiredDiskSpaceMegabytes:     1024,
		RunTasksAsCurrentUser:          true,
		ShutdownMachineOnIdle:          false,
		ShutdownMachineOnInternalError: false,
		SigningKeyLocation:             filepath.Join("testdata", "private-opengpg-key"),
		Subdomain:                      "taskcluster-worker.net",
		TasksDir:                       filepath.Join(testdataDir, testName),
		WorkerGroup:                    "test-worker-group",
		WorkerID:                       "test-worker-id",
		WorkerType:                     slugid.Nice(),
		WorkerTypeMetadata: map[string]interface{}{
			"aws": map[string]string{
				"ami-id":            "test-ami",
				"availability-zone": "outer-space",
				"instance-id":       "test-instance-id",
				"instance-type":     "p3.enormous",
				"local-ipv4":        "87.65.43.21",
				"public-ipv4":       "12.34.56.78",
			},
			"generic-worker": map[string]string{
				"go-arch":    runtime.GOARCH,
				"go-os":      runtime.GOOS,
				"go-version": runtime.Version(),
				"release":    "test-release-url",
				"version":    version,
			},
			"machine-setup": map[string]string{
				"maintainer": "pmoore@mozilla.com",
				"script":     "test-script-url",
			},
		},
	}

	if os.Getenv("GW_TESTS_GENERATE_USERS") != "" {
		config.RunTasksAsCurrentUser = false
	}

	// Needed for tests that don't call RunWorker()
	// but test methods/functions directly
	taskContext = &TaskContext{
		TaskDir: testdataDir,
	}

	// useful for expiry dates of tasks
	inAnHour = tcclient.Time(time.Now().Add(time.Hour * 1))
	globalTestName = testName
}

func teardown(t *testing.T) {
	err := os.RemoveAll(taskContext.TaskDir)
	if err != nil {
		t.Fatalf("Not able to clean up after test: %v", err)
	}
}

func executeTask(t *testing.T, td *queue.TaskDefinitionRequest, payload GenericWorkerPayload) (taskID string, myQueue *queue.Queue) {
	// check we have all the env vars we need to run this test
	if config.ClientID == "" || config.AccessToken == "" {
		t.Skip("Skipping test since TASKCLUSTER_CLIENT_ID and/or TASKCLUSTER_ACCESS_TOKEN env vars not set")
	}
	creds := &tcclient.Credentials{
		ClientID:    config.ClientID,
		AccessToken: config.AccessToken,
		Certificate: config.Certificate,
	}
	myQueue = queue.New(creds)
	taskID = slugid.Nice()

	b, err := json.Marshal(&payload)
	if err != nil {
		t.Fatalf("Could not convert task payload to json")
	}

	payloadJSON := json.RawMessage{}
	err = json.Unmarshal(b, &payloadJSON)
	if err != nil {
		t.Fatalf("Could not convert json bytes of payload to json.RawMessage")
	}

	td.Payload = payloadJSON

	// submit task
	_, err = myQueue.CreateTask(taskID, td)
	if err != nil {
		t.Fatalf("Could not submit task: %v", err)
	}

	err = UpdateTasksResolvedFile(0)
	if err != nil {
		t.Fatalf("Test setup failure - could not write to tasks-resolved-count.txt file: %v", err)
	}
	exitCode := RunWorker()

	if exitCode != TASKS_COMPLETE {
		t.Fatalf("Something went wrong executing worker - got exit code %v but was expecting exit code %v", exitCode, TASKS_COMPLETE)
	}

	return
}

func testTask(t *testing.T) *queue.TaskDefinitionRequest {
	created := time.Now().UTC()
	// reset nanoseconds
	created = created.Add(time.Nanosecond * time.Duration(created.Nanosecond()*-1))
	// deadline in one hour' time
	deadline := created.Add(15 * time.Minute)
	// expiry in one day, in case we need test results
	expires := created.AddDate(0, 0, 1)
	return &queue.TaskDefinitionRequest{
		Created:      tcclient.Time(created),
		Deadline:     tcclient.Time(deadline),
		Expires:      tcclient.Time(expires),
		Extra:        json.RawMessage(`{}`),
		Dependencies: []string{},
		Requires:     "all-completed",
		Metadata: struct {
			Description string `json:"description"`
			Name        string `json:"name"`
			Owner       string `json:"owner"`
			Source      string `json:"source"`
		}{
			Description: "Test task",
			Name:        "[TC] Generic Worker CI - " + globalTestName,
			Owner:       "generic-worker-ci@mozilla.com",
			Source:      "https://github.com/taskcluster/generic-worker",
		},
		Payload:       json.RawMessage(``),
		ProvisionerID: config.ProvisionerID,
		Retries:       1,
		Routes:        []string{},
		SchedulerID:   "test-scheduler",
		Scopes:        []string{},
		Tags:          json.RawMessage(`{"CI":"generic-worker"}`),
		Priority:      "lowest",
		TaskGroupID:   taskGroupID,
		WorkerType:    config.WorkerType,
	}
}

func checkSHA256(t *testing.T, sha256Hex string, file string) {
	hasher := sha256.New()
	f, err := os.Open(file)
	if err != nil {
		t.Fatal(err)
	}
	defer f.Close()
	if _, err := io.Copy(hasher, f); err != nil {
		t.Fatal(err)
	}
	if actualSHA256Hex := hex.EncodeToString(hasher.Sum(nil)); actualSHA256Hex != sha256Hex {
		t.Errorf("Expected file %v to have SHA256 %v but it was %v", file, sha256Hex, actualSHA256Hex)
	}
}
<commit_msg>Fixed tests that don't submit a task<commit_after>package main

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"io"
	"net"
	"os"
	"path/filepath"
	"runtime"
	"testing"
	"time"

	"github.com/taskcluster/slugid-go/slugid"
	tcclient "github.com/taskcluster/taskcluster-client-go"
	"github.com/taskcluster/taskcluster-client-go/queue"
)

type PayloadArtifact struct {
	Expires tcclient.Time `json:"expires,omitempty"`
	Name    string        `json:"name,omitempty"`
	Path    string        `json:"path"`
	Type    string        `json:"type"`
}

var (
	inAnHour       tcclient.Time
	testdataDir    string
	globalTestName string
)

func setup(t *testing.T, testName string) {
	// some basic setup...
	cwd, err := os.Getwd()
	if err != nil {
		t.Fatalf("Test failed during setup phase!")
	}
	testdataDir = filepath.Join(cwd, "testdata")

	// configure the worker
	config = &Config{
		AccessToken:                    os.Getenv("TASKCLUSTER_ACCESS_TOKEN"),
		CachesDir:                      filepath.Join(cwd, "caches"),
		Certificate:                    os.Getenv("TASKCLUSTER_CERTIFICATE"),
		CheckForNewDeploymentEverySecs: 0,
		CleanUpTaskDirs:                false,
		ClientID:                       os.Getenv("TASKCLUSTER_CLIENT_ID"),
		DeploymentID:                   "",
		DownloadsDir:                   filepath.Join(cwd, "downloads"),
		IdleTimeoutSecs:                60,
		InstanceID:                     "test-instance-id",
		InstanceType:                   "p3.enormous",
		LiveLogCertificate:             "",
		LiveLogExecutable:              "livelog",
		LiveLogGETPort:                 30582,
		LiveLogKey:                     "",
		LiveLogPUTPort:                 43264,
		LiveLogSecret:                  "xyz",
		NumberOfTasksToRun:             1,
		PrivateIP:                      net.ParseIP("87.65.43.21"),
		ProvisionerID:                  "test-provisioner",
		PublicIP:                       net.ParseIP("12.34.56.78"),
		RefreshUrlsPrematurelySecs:     310,
		Region: "outer-space",
		RequiredDiskSpaceMegabytes:     1024,
		RunTasksAsCurrentUser:          true,
		ShutdownMachineOnIdle:          false,
		ShutdownMachineOnInternalError: false,
		SigningKeyLocation:             filepath.Join("testdata", "private-opengpg-key"),
		Subdomain:                      "taskcluster-worker.net",
		TasksDir:                       filepath.Join(testdataDir, testName),
		WorkerGroup:                    "test-worker-group",
		WorkerID:                       "test-worker-id",
		WorkerType:                     slugid.Nice(),
		WorkerTypeMetadata: map[string]interface{}{
			"aws": map[string]string{
				"ami-id":            "test-ami",
				"availability-zone": "outer-space",
				"instance-id":       "test-instance-id",
				"instance-type":     "p3.enormous",
				"local-ipv4":        "87.65.43.21",
				"public-ipv4":       "12.34.56.78",
			},
			"generic-worker": map[string]string{
				"go-arch":    runtime.GOARCH,
				"go-os":      runtime.GOOS,
				"go-version": runtime.Version(),
				"release":    "test-release-url",
				"version":    version,
			},
			"machine-setup": map[string]string{
				"maintainer": "pmoore@mozilla.com",
				"script":     "test-script-url",
			},
		},
	}

	if os.Getenv("GW_TESTS_GENERATE_USERS") != "" {
		config.RunTasksAsCurrentUser = false
	}

	// Needed for tests that don't call RunWorker()
	// but test methods/functions directly
	taskContext = &TaskContext{
		TaskDir: testdataDir,
	}

	// useful for expiry dates of tasks
	inAnHour = tcclient.Time(time.Now().Add(time.Hour * 1))
	globalTestName = testName
}

func teardown(t *testing.T) {
	// note for tests that don't submit a task, they will have
	// taskContext.TasksDir set to the testdata subfolder, and we don't
	// want to delete that, which is why we delete the TasksDir
	err := os.RemoveAll(Config.TasksDir)
	if err != nil {
		t.Fatalf("Not able to clean up after test: %v", err)
	}
}

func executeTask(t *testing.T, td *queue.TaskDefinitionRequest, payload GenericWorkerPayload) (taskID string, myQueue *queue.Queue) {
	// check we have all the env vars we need to run this test
	if config.ClientID == "" || config.AccessToken == "" {
		t.Skip("Skipping test since TASKCLUSTER_CLIENT_ID and/or TASKCLUSTER_ACCESS_TOKEN env vars not set")
	}
	creds := &tcclient.Credentials{
		ClientID:    config.ClientID,
		AccessToken: config.AccessToken,
		Certificate: config.Certificate,
	}
	myQueue = queue.New(creds)
	taskID = slugid.Nice()

	b, err := json.Marshal(&payload)
	if err != nil {
		t.Fatalf("Could not convert task payload to json")
	}

	payloadJSON := json.RawMessage{}
	err = json.Unmarshal(b, &payloadJSON)
	if err != nil {
		t.Fatalf("Could not convert json bytes of payload to json.RawMessage")
	}

	td.Payload = payloadJSON

	// submit task
	_, err = myQueue.CreateTask(taskID, td)
	if err != nil {
		t.Fatalf("Could not submit task: %v", err)
	}

	err = UpdateTasksResolvedFile(0)
	if err != nil {
		t.Fatalf("Test setup failure - could not write to tasks-resolved-count.txt file: %v", err)
	}
	exitCode := RunWorker()

	if exitCode != TASKS_COMPLETE {
		t.Fatalf("Something went wrong executing worker - got exit code %v but was expecting exit code %v", exitCode, TASKS_COMPLETE)
	}

	return
}

func testTask(t *testing.T) *queue.TaskDefinitionRequest {
	created := time.Now().UTC()
	// reset nanoseconds
	created = created.Add(time.Nanosecond * time.Duration(created.Nanosecond()*-1))
	// deadline in one hour' time
	deadline := created.Add(15 * time.Minute)
	// expiry in one day, in case we need test results
	expires := created.AddDate(0, 0, 1)
	return &queue.TaskDefinitionRequest{
		Created:      tcclient.Time(created),
		Deadline:     tcclient.Time(deadline),
		Expires:      tcclient.Time(expires),
		Extra:        json.RawMessage(`{}`),
		Dependencies: []string{},
		Requires:     "all-completed",
		Metadata: struct {
			Description string `json:"description"`
			Name        string `json:"name"`
			Owner       string `json:"owner"`
			Source      string `json:"source"`
		}{
			Description: "Test task",
			Name:        "[TC] Generic Worker CI - " + globalTestName,
			Owner:       "generic-worker-ci@mozilla.com",
			Source:      "https://github.com/taskcluster/generic-worker",
		},
		Payload:       json.RawMessage(``),
		ProvisionerID: config.ProvisionerID,
		Retries:       1,
		Routes:        []string{},
		SchedulerID:   "test-scheduler",
		Scopes:        []string{},
		Tags:          json.RawMessage(`{"CI":"generic-worker"}`),
		Priority:      "lowest",
		TaskGroupID:   taskGroupID,
		WorkerType:    config.WorkerType,
	}
}

func checkSHA256(t *testing.T, sha256Hex string, file string) {
	hasher := sha256.New()
	f, err := os.Open(file)
	if err != nil {
		t.Fatal(err)
	}
	defer f.Close()
	if _, err := io.Copy(hasher, f); err != nil {
		t.Fatal(err)
	}
	if actualSHA256Hex := hex.EncodeToString(hasher.Sum(nil)); actualSHA256Hex != sha256Hex {
		t.Errorf("Expected file %v to have SHA256 %v but it was %v", file, sha256Hex, actualSHA256Hex)
	}
}
<|endoftext|>


<commit_before>package helpers

import (
	"github.com/globalsign/mgo"
	"github.com/globalsign/mgo/bson"

	"reflect"

	"strings"

	"crypto/tls"

	"net"

	"fmt"
	"time"

	"github.com/Seklfreak/Robyul2/cache"
	"github.com/Seklfreak/Robyul2/models"
	"github.com/pkg/errors"
)

var (
	mDbSession  *mgo.Session
	mDbDatabase string
)

type mgoLogger struct {
}

func (mgol mgoLogger) Output(calldepth int, s string) error {
	cache.GetLogger().WithField("module", "mdb").Info(s)
	return nil
}

// ConnectDB connects to mongodb and stores the session
func ConnectMDB(url string, database string) {
	var err error

	log := cache.GetLogger()
	log.WithField("module", "mdb").Info("Connecting to " + url)

	mgoL := new(mgoLogger)
	mgo.SetLogger(mgoL)

	cache.GetLogger()

	newUrl := strings.TrimSuffix(url, "?ssl=true")
	newUrl = strings.Replace(newUrl, "ssl=true&", "", -1)

	dialInfo, err := mgo.ParseURL(newUrl)
	if err != nil {
		log.WithField("module", "mdb").Error(err.Error())
		panic(err)
	}

	// setup TLS if we use SSL
	if newUrl != url {
		tlsConfig := &tls.Config{}
		tlsConfig.InsecureSkipVerify = true

		dialInfo.DialServer = func(addr *mgo.ServerAddr) (net.Conn, error) {
			conn, err := tls.Dial("tcp", addr.String(), tlsConfig)
			return conn, err
		}
	}

	mDbSession, err = mgo.DialWithInfo(dialInfo)
	if err != nil {
		log.WithField("module", "mdb").Error(err.Error())
		panic(err)
	}

	mDbSession.SetMode(mgo.Monotonic, true)
	mDbSession.SetSafe(&mgo.Safe{WMode: "majority"})

	mDbDatabase = database

	log.WithField("module", "mdb").Info("Connected!")
}

// GetDB is a simple getter for the mongodb database.
func GetMDb() *mgo.Database {
	return mDbSession.DB(mDbDatabase)
}

// GetDB is a simple getter for the mongodb session.
func GetMDbSession() *mgo.Session {
	return mDbSession
}

func MDbInsert(collection models.MongoDbCollection, data interface{}) (rid bson.ObjectId, err error) {
	ptr := reflect.New(reflect.TypeOf(data))
	temp := ptr.Elem()
	temp.Set(reflect.ValueOf(data))

	v := temp.FieldByName("ID")

	if !v.IsValid() {
		return bson.ObjectId(""), errors.New("invalid data")
	}

	newID := v.String()
	if newID == "" {
		newID = string(bson.NewObjectId())
		v.SetString(newID)
	}

	start := time.Now()
	err = GetMDb().C(collection.String()).Insert(temp.Interface())
	took := time.Since(start)

	if cache.HasKeen() {
		go func() {
			defer Recover()

			err := cache.GetKeen().AddEvent("Robyul_MongoDB", &KeenMongoDbEvent{
				Seconds:    took.Seconds(),
				Type:       "insert",
				Method:     "MDbInsert()",
				Collection: stripRobyulDatabaseFromCollection(collection.String()),
				Data:       fmt.Sprintf("%+v", data),
			})
			if err != nil {
				cache.GetLogger().WithField("module", "mdb").Error("Error logging MongoDB request to keen: ", err.Error())
			}
		}()
	}

	if err != nil {
		return bson.ObjectId(""), err
	}

	return bson.ObjectId(newID), nil
}

func MDbInsertWithoutLogging(collection models.MongoDbCollection, data interface{}) (rid bson.ObjectId, err error) {
	ptr := reflect.New(reflect.TypeOf(data))
	temp := ptr.Elem()
	temp.Set(reflect.ValueOf(data))

	v := temp.FieldByName("ID")

	if !v.IsValid() {
		return bson.ObjectId(""), errors.New("invalid data")
	}

	newID := v.String()
	if newID == "" {
		newID = string(bson.NewObjectId())
		v.SetString(newID)
	}

	err = GetMDb().C(collection.String()).Insert(temp.Interface())

	if err != nil {
		return bson.ObjectId(""), err
	}

	return bson.ObjectId(newID), nil
}

func MDbUpdate(collection models.MongoDbCollection, id bson.ObjectId, data interface{}) (rid bson.ObjectId, err error) {
	if !id.Valid() {
		return bson.ObjectId(""), errors.New("invalid id")
	}

	start := time.Now()
	err = GetMDb().C(collection.String()).UpdateId(id, data)
	took := time.Since(start)

	if cache.HasKeen() {
		go func() {
			defer Recover()

			err := cache.GetKeen().AddEvent("Robyul_MongoDB", &KeenMongoDbEvent{
				Seconds:    took.Seconds(),
				Type:       "update",
				Method:     "MDbUpdate()",
				Collection: stripRobyulDatabaseFromCollection(collection.String()),
				Id:         MdbIdToHuman(id),
				Data:       fmt.Sprintf("%+v", data),
			})
			if err != nil {
				cache.GetLogger().WithField("module", "mdb").Error("Error logging MongoDB request to keen: ", err.Error())
			}
		}()
	}

	if err != nil {
		return bson.ObjectId(""), err
	}

	return id, nil
}

func MDbUpdateWithoutLogging(collection models.MongoDbCollection, id bson.ObjectId, data interface{}) (rid bson.ObjectId, err error) {
	if !id.Valid() {
		return bson.ObjectId(""), errors.New("invalid id")
	}

	err = GetMDb().C(collection.String()).UpdateId(id, data)

	if err != nil {
		return bson.ObjectId(""), err
	}

	return id, nil
}

func MDbUpsertID(collection models.MongoDbCollection, id bson.ObjectId, data interface{}) (err error) {
	if !id.Valid() {
		return errors.New("invalid id")
	}

	start := time.Now()
	_, err = GetMDb().C(collection.String()).UpsertId(id, data)
	took := time.Since(start)

	if cache.HasKeen() {
		go func() {
			defer Recover()

			err := cache.GetKeen().AddEvent("Robyul_MongoDB", &KeenMongoDbEvent{
				Seconds:    took.Seconds(),
				Type:       "upsert",
				Method:     "MDbUpsertID()",
				Collection: stripRobyulDatabaseFromCollection(collection.String()),
				Id:         MdbIdToHuman(id),
				Data:       fmt.Sprintf("%+v", data),
			})
			if err != nil {
				cache.GetLogger().WithField("module", "mdb").Error("Error logging MongoDB request to keen: ", err.Error())
			}
		}()
	}

	if err != nil {
		return err
	}

	return nil
}

func MDbUpsert(collection models.MongoDbCollection, selector interface{}, data interface{}) (err error) {
	start := time.Now()
	_, err = GetMDb().C(collection.String()).Upsert(selector, data)
	took := time.Since(start)

	if cache.HasKeen() {
		go func() {
			defer Recover()

			err := cache.GetKeen().AddEvent("Robyul_MongoDB", &KeenMongoDbEvent{
				Seconds:    took.Seconds(),
				Type:       "upsert",
				Method:     "MDbUpsert()",
				Collection: stripRobyulDatabaseFromCollection(collection.String()),
				Query:      fmt.Sprintf("%+v", selector),
				Data:       fmt.Sprintf("%+v", data),
			})
			if err != nil {
				cache.GetLogger().WithField("module", "mdb").Error("Error logging MongoDB request to keen: ", err.Error())
			}
		}()
	}

	return err
}

func MDbDelete(collection models.MongoDbCollection, id bson.ObjectId) (err error) {
	if !id.Valid() {
		return errors.New("invalid id")
	}

	start := time.Now()
	err = GetMDb().C(collection.String()).RemoveId(id)
	took := time.Since(start)

	if cache.HasKeen() {
		go func() {
			defer Recover()

			err := cache.GetKeen().AddEvent("Robyul_MongoDB", &KeenMongoDbEvent{
				Seconds:    took.Seconds(),
				Type:       "remove",
				Method:     "MDbDelete()",
				Collection: stripRobyulDatabaseFromCollection(collection.String()),
				Id:         MdbIdToHuman(id),
			})
			if err != nil {
				cache.GetLogger().WithField("module", "mdb").Error("Error logging MongoDB request to keen: ", err.Error())
			}
		}()
	}

	if err != nil {
		return err
	}

	return nil
}

func MdbCollection(collection models.MongoDbCollection) (query *mgo.Collection) {
	return GetMDb().C(collection.String())
}

func MDbIter(query *mgo.Query) (iter *mgo.Iter) {
	start := time.Now()
	iter = query.Iter()
	took := time.Since(start)
	if cache.HasKeen() {
		go func() {
			defer Recover()

			queryValue := reflect.ValueOf(*query)
			queryOp := queryValue.FieldByName("query").FieldByName("op")

			err := cache.GetKeen().AddEvent("Robyul_MongoDB", &KeenMongoDbEvent{
				Seconds:    took.Seconds(),
				Type:       "query",
				Method:     "MdbIter()",
				Collection: stripRobyulDatabaseFromCollection(queryOp.FieldByName("collection").String()),
				Query:      fmt.Sprintf("%+v", reflect.ValueOf(queryOp.FieldByName("query")).Interface()),
				Skip:       queryOp.FieldByName("skip").Int(),
				Limit:      queryOp.FieldByName("limit").Int(),
			})
			if err != nil {
				cache.GetLogger().WithField("module", "mdb").Error("Error logging MongoDB request to keen: ", err.Error())
			}
		}()
	}
	return
}

func MdbOne(query *mgo.Query, object interface{}) (err error) {
	start := time.Now()
	err = query.One(object)
	took := time.Since(start)
	if cache.HasKeen() {
		go func() {
			defer Recover()

			queryValue := reflect.ValueOf(*query)
			queryOp := queryValue.FieldByName("query").FieldByName("op")

			err := cache.GetKeen().AddEvent("Robyul_MongoDB", &KeenMongoDbEvent{
				Seconds:    took.Seconds(),
				Type:       "query",
				Method:     "MdbOne()",
				Collection: stripRobyulDatabaseFromCollection(queryOp.FieldByName("collection").String()),
				Query:      fmt.Sprintf("%+v", reflect.ValueOf(queryOp.FieldByName("query")).Interface()),
				Skip:       queryOp.FieldByName("skip").Int(),
				Limit:      queryOp.FieldByName("limit").Int(),
			})
			if err != nil {
				cache.GetLogger().WithField("module", "mdb").Error("Error logging MongoDB request to keen: ", err.Error())
			}
		}()
	}
	return
}

func MdbOneWithoutLogging(query *mgo.Query, object interface{}) (err error) {
	err = query.One(object)
	return
}

// Returns a human readable ID version of a ObjectID
// id	: the ObjectID to convert
func MdbIdToHuman(id bson.ObjectId) (text string) {
	return fmt.Sprintf(`%x`, string(id))
}

// Returns an ObjectID from a human readable ID
// text	: the human readable ID
func HumanToMdbId(text string) (id bson.ObjectId) {
	return bson.ObjectIdHex(text)
}

func stripRobyulDatabaseFromCollection(input string) (output string) {
	return strings.TrimPrefix(input, mDbDatabase+".")
}

type KeenMongoDbEvent struct {
	Seconds    float64
	Collection string
	Type       string
	Method     string
	Query      string `json:",omitempty"`
	Skip       int64  `json:",omitempty"`
	Limit      int64  `json:",omitempty"`
	Id         string `json:",omitempty"`
	Data       string `json:",omitempty"`
}
<commit_msg>[mdb] truncate max keen data length ✂<commit_after>package helpers

import (
	"github.com/globalsign/mgo"
	"github.com/globalsign/mgo/bson"

	"reflect"

	"strings"

	"crypto/tls"

	"net"

	"fmt"
	"time"

	"github.com/Seklfreak/Robyul2/cache"
	"github.com/Seklfreak/Robyul2/models"
	"github.com/pkg/errors"
)

var (
	mDbSession  *mgo.Session
	mDbDatabase string
)

type mgoLogger struct {
}

func (mgol mgoLogger) Output(calldepth int, s string) error {
	cache.GetLogger().WithField("module", "mdb").Info(s)
	return nil
}

// ConnectDB connects to mongodb and stores the session
func ConnectMDB(url string, database string) {
	var err error

	log := cache.GetLogger()
	log.WithField("module", "mdb").Info("Connecting to " + url)

	mgoL := new(mgoLogger)
	mgo.SetLogger(mgoL)

	cache.GetLogger()

	newUrl := strings.TrimSuffix(url, "?ssl=true")
	newUrl = strings.Replace(newUrl, "ssl=true&", "", -1)

	dialInfo, err := mgo.ParseURL(newUrl)
	if err != nil {
		log.WithField("module", "mdb").Error(err.Error())
		panic(err)
	}

	// setup TLS if we use SSL
	if newUrl != url {
		tlsConfig := &tls.Config{}
		tlsConfig.InsecureSkipVerify = true

		dialInfo.DialServer = func(addr *mgo.ServerAddr) (net.Conn, error) {
			conn, err := tls.Dial("tcp", addr.String(), tlsConfig)
			return conn, err
		}
	}

	mDbSession, err = mgo.DialWithInfo(dialInfo)
	if err != nil {
		log.WithField("module", "mdb").Error(err.Error())
		panic(err)
	}

	mDbSession.SetMode(mgo.Monotonic, true)
	mDbSession.SetSafe(&mgo.Safe{WMode: "majority"})

	mDbDatabase = database

	log.WithField("module", "mdb").Info("Connected!")
}

// GetDB is a simple getter for the mongodb database.
func GetMDb() *mgo.Database {
	return mDbSession.DB(mDbDatabase)
}

// GetDB is a simple getter for the mongodb session.
func GetMDbSession() *mgo.Session {
	return mDbSession
}

func MDbInsert(collection models.MongoDbCollection, data interface{}) (rid bson.ObjectId, err error) {
	ptr := reflect.New(reflect.TypeOf(data))
	temp := ptr.Elem()
	temp.Set(reflect.ValueOf(data))

	v := temp.FieldByName("ID")

	if !v.IsValid() {
		return bson.ObjectId(""), errors.New("invalid data")
	}

	newID := v.String()
	if newID == "" {
		newID = string(bson.NewObjectId())
		v.SetString(newID)
	}

	start := time.Now()
	err = GetMDb().C(collection.String()).Insert(temp.Interface())
	took := time.Since(start)

	if cache.HasKeen() {
		go func() {
			defer Recover()

			err := cache.GetKeen().AddEvent("Robyul_MongoDB", &KeenMongoDbEvent{
				Seconds:    took.Seconds(),
				Type:       "insert",
				Method:     "MDbInsert()",
				Collection: stripRobyulDatabaseFromCollection(collection.String()),
				Data:       truncateKeenValue(fmt.Sprintf("%+v", data)),
			})
			if err != nil {
				cache.GetLogger().WithField("module", "mdb").Error("Error logging MongoDB request to keen: ", err.Error())
			}
		}()
	}

	if err != nil {
		return bson.ObjectId(""), err
	}

	return bson.ObjectId(newID), nil
}

func MDbInsertWithoutLogging(collection models.MongoDbCollection, data interface{}) (rid bson.ObjectId, err error) {
	ptr := reflect.New(reflect.TypeOf(data))
	temp := ptr.Elem()
	temp.Set(reflect.ValueOf(data))

	v := temp.FieldByName("ID")

	if !v.IsValid() {
		return bson.ObjectId(""), errors.New("invalid data")
	}

	newID := v.String()
	if newID == "" {
		newID = string(bson.NewObjectId())
		v.SetString(newID)
	}

	err = GetMDb().C(collection.String()).Insert(temp.Interface())

	if err != nil {
		return bson.ObjectId(""), err
	}

	return bson.ObjectId(newID), nil
}

func MDbUpdate(collection models.MongoDbCollection, id bson.ObjectId, data interface{}) (rid bson.ObjectId, err error) {
	if !id.Valid() {
		return bson.ObjectId(""), errors.New("invalid id")
	}

	start := time.Now()
	err = GetMDb().C(collection.String()).UpdateId(id, data)
	took := time.Since(start)

	if cache.HasKeen() {
		go func() {
			defer Recover()

			err := cache.GetKeen().AddEvent("Robyul_MongoDB", &KeenMongoDbEvent{
				Seconds:    took.Seconds(),
				Type:       "update",
				Method:     "MDbUpdate()",
				Collection: stripRobyulDatabaseFromCollection(collection.String()),
				Id:         MdbIdToHuman(id),
				Data:       truncateKeenValue(fmt.Sprintf("%+v", data)),
			})
			if err != nil {
				cache.GetLogger().WithField("module", "mdb").Error("Error logging MongoDB request to keen: ", err.Error())
			}
		}()
	}

	if err != nil {
		return bson.ObjectId(""), err
	}

	return id, nil
}

func MDbUpdateWithoutLogging(collection models.MongoDbCollection, id bson.ObjectId, data interface{}) (rid bson.ObjectId, err error) {
	if !id.Valid() {
		return bson.ObjectId(""), errors.New("invalid id")
	}

	err = GetMDb().C(collection.String()).UpdateId(id, data)

	if err != nil {
		return bson.ObjectId(""), err
	}

	return id, nil
}

func MDbUpsertID(collection models.MongoDbCollection, id bson.ObjectId, data interface{}) (err error) {
	if !id.Valid() {
		return errors.New("invalid id")
	}

	start := time.Now()
	_, err = GetMDb().C(collection.String()).UpsertId(id, data)
	took := time.Since(start)

	if cache.HasKeen() {
		go func() {
			defer Recover()

			err := cache.GetKeen().AddEvent("Robyul_MongoDB", &KeenMongoDbEvent{
				Seconds:    took.Seconds(),
				Type:       "upsert",
				Method:     "MDbUpsertID()",
				Collection: stripRobyulDatabaseFromCollection(collection.String()),
				Id:         MdbIdToHuman(id),
				Data:       truncateKeenValue(fmt.Sprintf("%+v", data)),
			})
			if err != nil {
				cache.GetLogger().WithField("module", "mdb").Error("Error logging MongoDB request to keen: ", err.Error())
			}
		}()
	}

	if err != nil {
		return err
	}

	return nil
}

func MDbUpsert(collection models.MongoDbCollection, selector interface{}, data interface{}) (err error) {
	start := time.Now()
	_, err = GetMDb().C(collection.String()).Upsert(selector, data)
	took := time.Since(start)

	if cache.HasKeen() {
		go func() {
			defer Recover()

			err := cache.GetKeen().AddEvent("Robyul_MongoDB", &KeenMongoDbEvent{
				Seconds:    took.Seconds(),
				Type:       "upsert",
				Method:     "MDbUpsert()",
				Collection: stripRobyulDatabaseFromCollection(collection.String()),
				Query:      fmt.Sprintf("%+v", selector),
				Data:       truncateKeenValue(fmt.Sprintf("%+v", data)),
			})
			if err != nil {
				cache.GetLogger().WithField("module", "mdb").Error("Error logging MongoDB request to keen: ", err.Error())
			}
		}()
	}

	return err
}

func MDbDelete(collection models.MongoDbCollection, id bson.ObjectId) (err error) {
	if !id.Valid() {
		return errors.New("invalid id")
	}

	start := time.Now()
	err = GetMDb().C(collection.String()).RemoveId(id)
	took := time.Since(start)

	if cache.HasKeen() {
		go func() {
			defer Recover()

			err := cache.GetKeen().AddEvent("Robyul_MongoDB", &KeenMongoDbEvent{
				Seconds:    took.Seconds(),
				Type:       "remove",
				Method:     "MDbDelete()",
				Collection: stripRobyulDatabaseFromCollection(collection.String()),
				Id:         MdbIdToHuman(id),
			})
			if err != nil {
				cache.GetLogger().WithField("module", "mdb").Error("Error logging MongoDB request to keen: ", err.Error())
			}
		}()
	}

	if err != nil {
		return err
	}

	return nil
}

func MdbCollection(collection models.MongoDbCollection) (query *mgo.Collection) {
	return GetMDb().C(collection.String())
}

func MDbIter(query *mgo.Query) (iter *mgo.Iter) {
	start := time.Now()
	iter = query.Iter()
	took := time.Since(start)
	if cache.HasKeen() {
		go func() {
			defer Recover()

			queryValue := reflect.ValueOf(*query)
			queryOp := queryValue.FieldByName("query").FieldByName("op")

			err := cache.GetKeen().AddEvent("Robyul_MongoDB", &KeenMongoDbEvent{
				Seconds:    took.Seconds(),
				Type:       "query",
				Method:     "MdbIter()",
				Collection: stripRobyulDatabaseFromCollection(queryOp.FieldByName("collection").String()),
				Query:      fmt.Sprintf("%+v", reflect.ValueOf(queryOp.FieldByName("query")).Interface()),
				Skip:       queryOp.FieldByName("skip").Int(),
				Limit:      queryOp.FieldByName("limit").Int(),
			})
			if err != nil {
				cache.GetLogger().WithField("module", "mdb").Error("Error logging MongoDB request to keen: ", err.Error())
			}
		}()
	}
	return
}

func MdbOne(query *mgo.Query, object interface{}) (err error) {
	start := time.Now()
	err = query.One(object)
	took := time.Since(start)
	if cache.HasKeen() {
		go func() {
			defer Recover()

			queryValue := reflect.ValueOf(*query)
			queryOp := queryValue.FieldByName("query").FieldByName("op")

			err := cache.GetKeen().AddEvent("Robyul_MongoDB", &KeenMongoDbEvent{
				Seconds:    took.Seconds(),
				Type:       "query",
				Method:     "MdbOne()",
				Collection: stripRobyulDatabaseFromCollection(queryOp.FieldByName("collection").String()),
				Query:      fmt.Sprintf("%+v", reflect.ValueOf(queryOp.FieldByName("query")).Interface()),
				Skip:       queryOp.FieldByName("skip").Int(),
				Limit:      queryOp.FieldByName("limit").Int(),
			})
			if err != nil {
				cache.GetLogger().WithField("module", "mdb").Error("Error logging MongoDB request to keen: ", err.Error())
			}
		}()
	}
	return
}

func MdbOneWithoutLogging(query *mgo.Query, object interface{}) (err error) {
	err = query.One(object)
	return
}

// Returns a human readable ID version of a ObjectID
// id	: the ObjectID to convert
func MdbIdToHuman(id bson.ObjectId) (text string) {
	return fmt.Sprintf(`%x`, string(id))
}

// Returns an ObjectID from a human readable ID
// text	: the human readable ID
func HumanToMdbId(text string) (id bson.ObjectId) {
	return bson.ObjectIdHex(text)
}

func stripRobyulDatabaseFromCollection(input string) (output string) {
	return strings.TrimPrefix(input, mDbDatabase+".")
}

func truncateKeenValue(input string) string {
	if len(input) < 8000 {
		return input
	}
	return input[0:7999]
}

type KeenMongoDbEvent struct {
	Seconds    float64
	Collection string
	Type       string
	Method     string
	Query      string `json:",omitempty"`
	Skip       int64  `json:",omitempty"`
	Limit      int64  `json:",omitempty"`
	Id         string `json:",omitempty"`
	Data       string `json:",omitempty"`
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mocknetworking

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"regexp"
	"strings"

	"k8s.io/kops/upup/pkg/fi"

	"github.com/google/uuid"
	"github.com/gophercloud/gophercloud/openstack/networking/v2/ports"
)

type portListResponse struct {
	Ports []ports.Port `json:"ports"`
}

type portGetResponse struct {
	Port ports.Port `json:"port"`
}

type portCreateRequest struct {
	Port ports.CreateOpts `json:"port"`
}

type portUpdateRequest struct {
	Port ports.UpdateOpts `json:"port"`
}

func (m *MockClient) mockPorts() {
	re := regexp.MustCompile(`/ports/?`)

	handler := func(w http.ResponseWriter, r *http.Request) {
		m.mutex.Lock()
		defer m.mutex.Unlock()

		w.Header().Add("Content-Type", "application/json")

		portID := re.ReplaceAllString(r.URL.Path, "")
		switch r.Method {
		case http.MethodGet:
			if portID == "" {
				r.ParseForm()
				m.listPorts(w, r.Form)
			} else {
				m.getPort(w, portID)
			}
		case http.MethodPut:
			parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/")
			if len(parts) == 4 && parts[2] == "tags" {

				m.tagPort(w, r)
			} else if len(parts) == 2 {
				m.updatePort(w, r)
			}
		case http.MethodPost:
			m.createPort(w, r)
		case http.MethodDelete:
			m.deletePort(w, portID)
		default:
			w.WriteHeader(http.StatusBadRequest)
		}
	}
	m.Mux.HandleFunc("/ports/", handler)
	m.Mux.HandleFunc("/ports", handler)
}

func (m *MockClient) listPorts(w http.ResponseWriter, vals url.Values) {

	w.WriteHeader(http.StatusOK)

	ports := make([]ports.Port, 0)
	nameFilter := vals.Get("name")
	idFilter := vals.Get("id")
	networkFilter := vals.Get("network_id")
	deviceFilter := vals.Get("device_id")
	for _, p := range m.ports {
		if nameFilter != "" && nameFilter != p.Name {
			continue
		}
		if deviceFilter != "" && deviceFilter != p.DeviceID {
			continue
		}
		if networkFilter != "" && networkFilter != p.NetworkID {
			continue
		}
		if idFilter != "" && idFilter != p.ID {
			continue
		}
		ports = append(ports, p)
	}

	resp := portListResponse{
		Ports: ports,
	}
	respB, err := json.Marshal(resp)
	if err != nil {
		panic(fmt.Sprintf("failed to marshal %+v", resp))
	}
	_, err = w.Write(respB)
	if err != nil {
		panic("failed to write body")
	}
}

func (m *MockClient) getPort(w http.ResponseWriter, portID string) {
	if port, ok := m.ports[portID]; ok {
		resp := portGetResponse{
			Port: port,
		}
		respB, err := json.Marshal(resp)
		if err != nil {
			panic(fmt.Sprintf("failed to marshal %+v", resp))
		}
		_, err = w.Write(respB)
		if err != nil {
			panic("failed to write body")
		}
	} else {
		w.WriteHeader(http.StatusNotFound)
	}
}

func (m *MockClient) deletePort(w http.ResponseWriter, portID string) {
	if _, ok := m.ports[portID]; ok {
		delete(m.ports, portID)
		w.WriteHeader(http.StatusOK)
	} else {
		w.WriteHeader(http.StatusNotFound)
	}
}

func (m *MockClient) createPort(w http.ResponseWriter, r *http.Request) {
	var create portCreateRequest
	err := json.NewDecoder(r.Body).Decode(&create)
	if err != nil {
		panic("error decoding create port request")
	}

	fixedIPs := make([]ports.IP, 0)
	// The request type uses a []interface{map[string]interface{}]} to represent []ports.IP
	if rawIPs, ok := create.Port.FixedIPs.([]interface{}); ok {
		for _, rawFixedIP := range rawIPs {
			if rawIP, ok := rawFixedIP.(map[string]interface{}); ok {
				if subnetID, ok := rawIP["subnet_id"]; ok {
					if subnet, ok := subnetID.(string); ok {
						fixedIPs = append(fixedIPs, ports.IP{SubnetID: subnet})
					}
				}
			}
		}
	}
	w.WriteHeader(http.StatusAccepted)

	p := ports.Port{
		ID:             uuid.New().String(),
		Name:           create.Port.Name,
		NetworkID:      create.Port.NetworkID,
		SecurityGroups: *create.Port.SecurityGroups,
		DeviceID:       create.Port.DeviceID,
		FixedIPs:       fixedIPs,
	}
	m.ports[p.ID] = p

	resp := portGetResponse{
		Port: p,
	}
	respB, err := json.Marshal(resp)
	if err != nil {
		panic(fmt.Sprintf("failed to marshal %+v", resp))
	}
	_, err = w.Write(respB)
	if err != nil {
		panic("failed to write body")
	}
}

func (m *MockClient) tagPort(w http.ResponseWriter, r *http.Request) {
	path := r.URL.Path
	// /ports/<portid>/tags/<tag>
	parts := strings.Split(strings.Trim(path, "/"), "/")
	if len(parts) != 4 {
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	portID := parts[1]
	tag := parts[3]

	if _, ok := m.ports[portID]; !ok {
		w.WriteHeader(http.StatusNotFound)
		return
	}
	port := m.ports[portID]
	port.Tags = append(port.Tags, tag)
	m.ports[portID] = port

	w.WriteHeader(http.StatusCreated)
}

func (m *MockClient) updatePort(w http.ResponseWriter, r *http.Request) {
	var update portUpdateRequest
	err := json.NewDecoder(r.Body).Decode(&update)
	if err != nil {
		panic("error decoding update port request")
	}
	parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/")
	if len(parts) != 2 {
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	portID := parts[1]

	if _, ok := m.ports[portID]; !ok {
		w.WriteHeader(http.StatusNotFound)
		return
	}
	if _, ok := m.ports[portID]; !ok {
		w.WriteHeader(http.StatusNotFound)
		return
	}
	port := m.ports[portID]

	deviceID := update.Port.DeviceID
	if deviceID != nil {
		port.DeviceID = fi.StringValue(deviceID)
	}
	m.ports[portID] = port

	w.WriteHeader(http.StatusOK)

}
<commit_msg>Remove duplicate code<commit_after>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mocknetworking

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"regexp"
	"strings"

	"k8s.io/kops/upup/pkg/fi"

	"github.com/google/uuid"
	"github.com/gophercloud/gophercloud/openstack/networking/v2/ports"
)

type portListResponse struct {
	Ports []ports.Port `json:"ports"`
}

type portGetResponse struct {
	Port ports.Port `json:"port"`
}

type portCreateRequest struct {
	Port ports.CreateOpts `json:"port"`
}

type portUpdateRequest struct {
	Port ports.UpdateOpts `json:"port"`
}

func (m *MockClient) mockPorts() {
	re := regexp.MustCompile(`/ports/?`)

	handler := func(w http.ResponseWriter, r *http.Request) {
		m.mutex.Lock()
		defer m.mutex.Unlock()

		w.Header().Add("Content-Type", "application/json")

		portID := re.ReplaceAllString(r.URL.Path, "")
		switch r.Method {
		case http.MethodGet:
			if portID == "" {
				r.ParseForm()
				m.listPorts(w, r.Form)
			} else {
				m.getPort(w, portID)
			}
		case http.MethodPut:
			parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/")
			if len(parts) == 4 && parts[2] == "tags" {

				m.tagPort(w, r)
			} else if len(parts) == 2 {
				m.updatePort(w, r)
			}
		case http.MethodPost:
			m.createPort(w, r)
		case http.MethodDelete:
			m.deletePort(w, portID)
		default:
			w.WriteHeader(http.StatusBadRequest)
		}
	}
	m.Mux.HandleFunc("/ports/", handler)
	m.Mux.HandleFunc("/ports", handler)
}

func (m *MockClient) listPorts(w http.ResponseWriter, vals url.Values) {

	w.WriteHeader(http.StatusOK)

	ports := make([]ports.Port, 0)
	nameFilter := vals.Get("name")
	idFilter := vals.Get("id")
	networkFilter := vals.Get("network_id")
	deviceFilter := vals.Get("device_id")
	for _, p := range m.ports {
		if nameFilter != "" && nameFilter != p.Name {
			continue
		}
		if deviceFilter != "" && deviceFilter != p.DeviceID {
			continue
		}
		if networkFilter != "" && networkFilter != p.NetworkID {
			continue
		}
		if idFilter != "" && idFilter != p.ID {
			continue
		}
		ports = append(ports, p)
	}

	resp := portListResponse{
		Ports: ports,
	}
	respB, err := json.Marshal(resp)
	if err != nil {
		panic(fmt.Sprintf("failed to marshal %+v", resp))
	}
	_, err = w.Write(respB)
	if err != nil {
		panic("failed to write body")
	}
}

func (m *MockClient) getPort(w http.ResponseWriter, portID string) {
	if port, ok := m.ports[portID]; ok {
		resp := portGetResponse{
			Port: port,
		}
		respB, err := json.Marshal(resp)
		if err != nil {
			panic(fmt.Sprintf("failed to marshal %+v", resp))
		}
		_, err = w.Write(respB)
		if err != nil {
			panic("failed to write body")
		}
	} else {
		w.WriteHeader(http.StatusNotFound)
	}
}

func (m *MockClient) deletePort(w http.ResponseWriter, portID string) {
	if _, ok := m.ports[portID]; ok {
		delete(m.ports, portID)
		w.WriteHeader(http.StatusOK)
	} else {
		w.WriteHeader(http.StatusNotFound)
	}
}

func (m *MockClient) createPort(w http.ResponseWriter, r *http.Request) {
	var create portCreateRequest
	err := json.NewDecoder(r.Body).Decode(&create)
	if err != nil {
		panic("error decoding create port request")
	}

	fixedIPs := make([]ports.IP, 0)
	// The request type uses a []interface{map[string]interface{}]} to represent []ports.IP
	if rawIPs, ok := create.Port.FixedIPs.([]interface{}); ok {
		for _, rawFixedIP := range rawIPs {
			if rawIP, ok := rawFixedIP.(map[string]interface{}); ok {
				if subnetID, ok := rawIP["subnet_id"]; ok {
					if subnet, ok := subnetID.(string); ok {
						fixedIPs = append(fixedIPs, ports.IP{SubnetID: subnet})
					}
				}
			}
		}
	}
	w.WriteHeader(http.StatusAccepted)

	p := ports.Port{
		ID:             uuid.New().String(),
		Name:           create.Port.Name,
		NetworkID:      create.Port.NetworkID,
		SecurityGroups: *create.Port.SecurityGroups,
		DeviceID:       create.Port.DeviceID,
		FixedIPs:       fixedIPs,
	}
	m.ports[p.ID] = p

	resp := portGetResponse{
		Port: p,
	}
	respB, err := json.Marshal(resp)
	if err != nil {
		panic(fmt.Sprintf("failed to marshal %+v", resp))
	}
	_, err = w.Write(respB)
	if err != nil {
		panic("failed to write body")
	}
}

func (m *MockClient) tagPort(w http.ResponseWriter, r *http.Request) {
	path := r.URL.Path
	// /ports/<portid>/tags/<tag>
	parts := strings.Split(strings.Trim(path, "/"), "/")
	if len(parts) != 4 {
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	portID := parts[1]
	tag := parts[3]

	if _, ok := m.ports[portID]; !ok {
		w.WriteHeader(http.StatusNotFound)
		return
	}
	port := m.ports[portID]
	port.Tags = append(port.Tags, tag)
	m.ports[portID] = port

	w.WriteHeader(http.StatusCreated)
}

func (m *MockClient) updatePort(w http.ResponseWriter, r *http.Request) {
	var update portUpdateRequest
	err := json.NewDecoder(r.Body).Decode(&update)
	if err != nil {
		panic("error decoding update port request")
	}
	parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/")
	if len(parts) != 2 {
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	portID := parts[1]

	if _, ok := m.ports[portID]; !ok {
		w.WriteHeader(http.StatusNotFound)
		return
	}
	port := m.ports[portID]

	deviceID := update.Port.DeviceID
	if deviceID != nil {
		port.DeviceID = fi.StringValue(deviceID)
	}
	m.ports[portID] = port

	w.WriteHeader(http.StatusOK)

}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package framework

import (
	"fmt"
	"os"
	"path"
	"strings"
	"time"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/kubernetes/pkg/api/v1"
	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
)

func EtcdUpgrade(target_storage, target_version string) error {
	switch TestContext.Provider {
	case "gce":
		return etcdUpgradeGCE(target_storage, target_version)
	default:
		return fmt.Errorf("EtcdUpgrade() is not implemented for provider %s", TestContext.Provider)
	}
}

func MasterUpgrade(v string) error {
	switch TestContext.Provider {
	case "gce":
		return masterUpgradeGCE(v)
	case "gke":
		return masterUpgradeGKE(v)
	default:
		return fmt.Errorf("MasterUpgrade() is not implemented for provider %s", TestContext.Provider)
	}
}

func etcdUpgradeGCE(target_storage, target_version string) error {
	env := append(
		os.Environ(),
		"TEST_ETCD_VERSION="+target_version,
		"STORAGE_BACKEND="+target_storage,
		"TEST_ETCD_IMAGE=3.0.17")

	_, _, err := RunCmdEnv(env, gceUpgradeScript(), "-l", "-M")
	return err
}

func masterUpgradeGCE(rawV string) error {
	v := "v" + rawV
	_, _, err := RunCmd(gceUpgradeScript(), "-M", v)
	return err
}

func masterUpgradeGKE(v string) error {
	Logf("Upgrading master to %q", v)
	_, _, err := RunCmd("gcloud", "container",
		"clusters",
		fmt.Sprintf("--project=%s", TestContext.CloudConfig.ProjectID),
		fmt.Sprintf("--zone=%s", TestContext.CloudConfig.Zone),
		"upgrade",
		TestContext.CloudConfig.Cluster,
		"--master",
		fmt.Sprintf("--cluster-version=%s", v),
		"--quiet")
	return err
}

func NodeUpgrade(f *Framework, v string, img string) error {
	// Perform the upgrade.
	var err error
	switch TestContext.Provider {
	case "gce":
		err = nodeUpgradeGCE(v, img)
	case "gke":
		err = nodeUpgradeGKE(v, img)
	default:
		err = fmt.Errorf("NodeUpgrade() is not implemented for provider %s", TestContext.Provider)
	}
	if err != nil {
		return err
	}

	// Wait for it to complete and validate nodes are healthy.
	//
	// TODO(ihmccreery) We shouldn't have to wait for nodes to be ready in
	// GKE; the operation shouldn't return until they all are.
	Logf("Waiting up to %v for all nodes to be ready after the upgrade", RestartNodeReadyAgainTimeout)
	if _, err := CheckNodesReady(f.ClientSet, RestartNodeReadyAgainTimeout, TestContext.CloudConfig.NumNodes); err != nil {
		return err
	}
	return nil
}

func nodeUpgradeGCE(rawV, img string) error {
	v := "v" + rawV
	if img != "" {
		env := append(os.Environ(), "KUBE_NODE_OS_DISTRIBUTION="+img)
		_, _, err := RunCmdEnv(env, gceUpgradeScript(), "-N", "-o", v)
		return err
	}
	_, _, err := RunCmd(gceUpgradeScript(), "-N", v)
	return err
}

func cleanupNodeUpgradeGCE(tmplBefore string) {
	Logf("Cleaning up any unused node templates")
	tmplAfter, err := MigTemplate()
	if err != nil {
		Logf("Could not get node template post-upgrade; may have leaked template %s", tmplBefore)
		return
	}
	if tmplBefore == tmplAfter {
		// The node upgrade failed so there's no need to delete
		// anything.
		Logf("Node template %s is still in use; not cleaning up", tmplBefore)
		return
	}
	Logf("Deleting node template %s", tmplBefore)
	if _, _, err := retryCmd("gcloud", "compute", "instance-templates",
		fmt.Sprintf("--project=%s", TestContext.CloudConfig.ProjectID),
		"delete",
		tmplBefore); err != nil {
		Logf("gcloud compute instance-templates delete %s call failed with err: %v", tmplBefore, err)
		Logf("May have leaked instance template %q", tmplBefore)
	}
}

func nodeUpgradeGKE(v string, img string) error {
	Logf("Upgrading nodes to version %q and image %q", v, img)
	args := []string{
		"container",
		"clusters",
		fmt.Sprintf("--project=%s", TestContext.CloudConfig.ProjectID),
		fmt.Sprintf("--zone=%s", TestContext.CloudConfig.Zone),
		"upgrade",
		TestContext.CloudConfig.Cluster,
		fmt.Sprintf("--cluster-version=%s", v),
		"--quiet",
	}
	if len(img) > 0 {
		args = append(args, fmt.Sprintf("--image-type=%s", img))
	}
	_, _, err := RunCmd("gcloud", args...)
	return err
}

// CheckNodesReady waits up to nt for expect nodes accessed by c to be ready,
// returning an error if this doesn't happen in time. It returns the names of
// nodes it finds.
func CheckNodesReady(c clientset.Interface, nt time.Duration, expect int) ([]string, error) {
	// First, keep getting all of the nodes until we get the number we expect.
	var nodeList *v1.NodeList
	var errLast error
	start := time.Now()
	found := wait.Poll(Poll, nt, func() (bool, error) {
		// A rolling-update (GCE/GKE implementation of restart) can complete before the apiserver
		// knows about all of the nodes. Thus, we retry the list nodes call
		// until we get the expected number of nodes.
		nodeList, errLast = c.Core().Nodes().List(metav1.ListOptions{
			FieldSelector: fields.Set{"spec.unschedulable": "false"}.AsSelector().String()})
		if errLast != nil {
			return false, nil
		}
		if len(nodeList.Items) != expect {
			errLast = fmt.Errorf("expected to find %d nodes but found only %d (%v elapsed)",
				expect, len(nodeList.Items), time.Since(start))
			Logf("%v", errLast)
			return false, nil
		}
		return true, nil
	}) == nil
	nodeNames := make([]string, len(nodeList.Items))
	for i, n := range nodeList.Items {
		nodeNames[i] = n.ObjectMeta.Name
	}
	if !found {
		return nodeNames, fmt.Errorf("couldn't find %d nodes within %v; last error: %v",
			expect, nt, errLast)
	}
	Logf("Successfully found %d nodes", expect)

	// Next, ensure in parallel that all the nodes are ready. We subtract the
	// time we spent waiting above.
	timeout := nt - time.Since(start)
	result := make(chan bool, len(nodeList.Items))
	for _, n := range nodeNames {
		n := n
		go func() { result <- WaitForNodeToBeReady(c, n, timeout) }()
	}
	failed := false
	// TODO(mbforbes): Change to `for range` syntax once we support only Go
	// >= 1.4.
	for i := range nodeList.Items {
		_ = i
		if !<-result {
			failed = true
		}
	}
	if failed {
		return nodeNames, fmt.Errorf("at least one node failed to be ready")
	}
	return nodeNames, nil
}

// MigTemplate (GCE-only) returns the name of the MIG template that the
// nodes of the cluster use.
func MigTemplate() (string, error) {
	var errLast error
	var templ string
	key := "instanceTemplate"
	if wait.Poll(Poll, SingleCallTimeout, func() (bool, error) {
		// TODO(mikedanese): make this hit the compute API directly instead of
		// shelling out to gcloud.
		// An `instance-groups managed describe` call outputs what we want to stdout.
		output, _, err := retryCmd("gcloud", "compute", "instance-groups", "managed",
			fmt.Sprintf("--project=%s", TestContext.CloudConfig.ProjectID),
			"describe",
			fmt.Sprintf("--zone=%s", TestContext.CloudConfig.Zone),
			TestContext.CloudConfig.NodeInstanceGroup)
		if err != nil {
			errLast = fmt.Errorf("gcloud compute instance-groups managed describe call failed with err: %v", err)
			return false, nil
		}

		// The 'describe' call probably succeeded; parse the output and try to
		// find the line that looks like "instanceTemplate: url/to/<templ>" and
		// return <templ>.
		if val := ParseKVLines(output, key); len(val) > 0 {
			url := strings.Split(val, "/")
			templ = url[len(url)-1]
			Logf("MIG group %s using template: %s", TestContext.CloudConfig.NodeInstanceGroup, templ)
			return true, nil
		}
		errLast = fmt.Errorf("couldn't find %s in output to get MIG template. Output: %s", key, output)
		return false, nil
	}) != nil {
		return "", fmt.Errorf("MigTemplate() failed with last error: %v", errLast)
	}
	return templ, nil
}

func gceUpgradeScript() string {
	if len(TestContext.GCEUpgradeScript) == 0 {
		return path.Join(TestContext.RepoRoot, "cluster/gce/upgrade.sh")
	}
	return TestContext.GCEUpgradeScript
}
<commit_msg>Add ETCD env vars for downgrade<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package framework

import (
	"fmt"
	"os"
	"path"
	"strings"
	"time"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/kubernetes/pkg/api/v1"
	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
)

func EtcdUpgrade(target_storage, target_version string) error {
	switch TestContext.Provider {
	case "gce":
		return etcdUpgradeGCE(target_storage, target_version)
	default:
		return fmt.Errorf("EtcdUpgrade() is not implemented for provider %s", TestContext.Provider)
	}
}

func MasterUpgrade(v string) error {
	switch TestContext.Provider {
	case "gce":
		return masterUpgradeGCE(v)
	case "gke":
		return masterUpgradeGKE(v)
	default:
		return fmt.Errorf("MasterUpgrade() is not implemented for provider %s", TestContext.Provider)
	}
}

func etcdUpgradeGCE(target_storage, target_version string) error {
	env := append(
		os.Environ(),
		"TEST_ETCD_VERSION="+target_version,
		"STORAGE_BACKEND="+target_storage,
		"TEST_ETCD_IMAGE=3.0.17")

	_, _, err := RunCmdEnv(env, gceUpgradeScript(), "-l", "-M")
	return err
}

func masterUpgradeGCE(rawV string) error {
	env := os.Environ()
	// TODO: Remove these variables when they're no longer needed for downgrades.
	if TestContext.EtcdUpgradeVersion != "" && TestContext.EtcdUpgradeStorage != "" {
		env = append(env,
			"TEST_ETCD_VERSION="+TestContext.EtcdUpgradeVersion,
			"STORAGE_BACKEND="+TestContext.EtcdUpgradeStorage,
			"TEST_ETCD_IMAGE=3.0.17")
	}

	v := "v" + rawV
	_, _, err := RunCmdEnv(env, gceUpgradeScript(), "-M", v)
	return err
}

func masterUpgradeGKE(v string) error {
	Logf("Upgrading master to %q", v)
	_, _, err := RunCmd("gcloud", "container",
		"clusters",
		fmt.Sprintf("--project=%s", TestContext.CloudConfig.ProjectID),
		fmt.Sprintf("--zone=%s", TestContext.CloudConfig.Zone),
		"upgrade",
		TestContext.CloudConfig.Cluster,
		"--master",
		fmt.Sprintf("--cluster-version=%s", v),
		"--quiet")
	return err
}

func NodeUpgrade(f *Framework, v string, img string) error {
	// Perform the upgrade.
	var err error
	switch TestContext.Provider {
	case "gce":
		err = nodeUpgradeGCE(v, img)
	case "gke":
		err = nodeUpgradeGKE(v, img)
	default:
		err = fmt.Errorf("NodeUpgrade() is not implemented for provider %s", TestContext.Provider)
	}
	if err != nil {
		return err
	}

	// Wait for it to complete and validate nodes are healthy.
	//
	// TODO(ihmccreery) We shouldn't have to wait for nodes to be ready in
	// GKE; the operation shouldn't return until they all are.
	Logf("Waiting up to %v for all nodes to be ready after the upgrade", RestartNodeReadyAgainTimeout)
	if _, err := CheckNodesReady(f.ClientSet, RestartNodeReadyAgainTimeout, TestContext.CloudConfig.NumNodes); err != nil {
		return err
	}
	return nil
}

func nodeUpgradeGCE(rawV, img string) error {
	v := "v" + rawV
	if img != "" {
		env := append(os.Environ(), "KUBE_NODE_OS_DISTRIBUTION="+img)
		_, _, err := RunCmdEnv(env, gceUpgradeScript(), "-N", "-o", v)
		return err
	}
	_, _, err := RunCmd(gceUpgradeScript(), "-N", v)
	return err
}

func cleanupNodeUpgradeGCE(tmplBefore string) {
	Logf("Cleaning up any unused node templates")
	tmplAfter, err := MigTemplate()
	if err != nil {
		Logf("Could not get node template post-upgrade; may have leaked template %s", tmplBefore)
		return
	}
	if tmplBefore == tmplAfter {
		// The node upgrade failed so there's no need to delete
		// anything.
		Logf("Node template %s is still in use; not cleaning up", tmplBefore)
		return
	}
	Logf("Deleting node template %s", tmplBefore)
	if _, _, err := retryCmd("gcloud", "compute", "instance-templates",
		fmt.Sprintf("--project=%s", TestContext.CloudConfig.ProjectID),
		"delete",
		tmplBefore); err != nil {
		Logf("gcloud compute instance-templates delete %s call failed with err: %v", tmplBefore, err)
		Logf("May have leaked instance template %q", tmplBefore)
	}
}

func nodeUpgradeGKE(v string, img string) error {
	Logf("Upgrading nodes to version %q and image %q", v, img)
	args := []string{
		"container",
		"clusters",
		fmt.Sprintf("--project=%s", TestContext.CloudConfig.ProjectID),
		fmt.Sprintf("--zone=%s", TestContext.CloudConfig.Zone),
		"upgrade",
		TestContext.CloudConfig.Cluster,
		fmt.Sprintf("--cluster-version=%s", v),
		"--quiet",
	}
	if len(img) > 0 {
		args = append(args, fmt.Sprintf("--image-type=%s", img))
	}
	_, _, err := RunCmd("gcloud", args...)
	return err
}

// CheckNodesReady waits up to nt for expect nodes accessed by c to be ready,
// returning an error if this doesn't happen in time. It returns the names of
// nodes it finds.
func CheckNodesReady(c clientset.Interface, nt time.Duration, expect int) ([]string, error) {
	// First, keep getting all of the nodes until we get the number we expect.
	var nodeList *v1.NodeList
	var errLast error
	start := time.Now()
	found := wait.Poll(Poll, nt, func() (bool, error) {
		// A rolling-update (GCE/GKE implementation of restart) can complete before the apiserver
		// knows about all of the nodes. Thus, we retry the list nodes call
		// until we get the expected number of nodes.
		nodeList, errLast = c.Core().Nodes().List(metav1.ListOptions{
			FieldSelector: fields.Set{"spec.unschedulable": "false"}.AsSelector().String()})
		if errLast != nil {
			return false, nil
		}
		if len(nodeList.Items) != expect {
			errLast = fmt.Errorf("expected to find %d nodes but found only %d (%v elapsed)",
				expect, len(nodeList.Items), time.Since(start))
			Logf("%v", errLast)
			return false, nil
		}
		return true, nil
	}) == nil
	nodeNames := make([]string, len(nodeList.Items))
	for i, n := range nodeList.Items {
		nodeNames[i] = n.ObjectMeta.Name
	}
	if !found {
		return nodeNames, fmt.Errorf("couldn't find %d nodes within %v; last error: %v",
			expect, nt, errLast)
	}
	Logf("Successfully found %d nodes", expect)

	// Next, ensure in parallel that all the nodes are ready. We subtract the
	// time we spent waiting above.
	timeout := nt - time.Since(start)
	result := make(chan bool, len(nodeList.Items))
	for _, n := range nodeNames {
		n := n
		go func() { result <- WaitForNodeToBeReady(c, n, timeout) }()
	}
	failed := false
	// TODO(mbforbes): Change to `for range` syntax once we support only Go
	// >= 1.4.
	for i := range nodeList.Items {
		_ = i
		if !<-result {
			failed = true
		}
	}
	if failed {
		return nodeNames, fmt.Errorf("at least one node failed to be ready")
	}
	return nodeNames, nil
}

// MigTemplate (GCE-only) returns the name of the MIG template that the
// nodes of the cluster use.
func MigTemplate() (string, error) {
	var errLast error
	var templ string
	key := "instanceTemplate"
	if wait.Poll(Poll, SingleCallTimeout, func() (bool, error) {
		// TODO(mikedanese): make this hit the compute API directly instead of
		// shelling out to gcloud.
		// An `instance-groups managed describe` call outputs what we want to stdout.
		output, _, err := retryCmd("gcloud", "compute", "instance-groups", "managed",
			fmt.Sprintf("--project=%s", TestContext.CloudConfig.ProjectID),
			"describe",
			fmt.Sprintf("--zone=%s", TestContext.CloudConfig.Zone),
			TestContext.CloudConfig.NodeInstanceGroup)
		if err != nil {
			errLast = fmt.Errorf("gcloud compute instance-groups managed describe call failed with err: %v", err)
			return false, nil
		}

		// The 'describe' call probably succeeded; parse the output and try to
		// find the line that looks like "instanceTemplate: url/to/<templ>" and
		// return <templ>.
		if val := ParseKVLines(output, key); len(val) > 0 {
			url := strings.Split(val, "/")
			templ = url[len(url)-1]
			Logf("MIG group %s using template: %s", TestContext.CloudConfig.NodeInstanceGroup, templ)
			return true, nil
		}
		errLast = fmt.Errorf("couldn't find %s in output to get MIG template. Output: %s", key, output)
		return false, nil
	}) != nil {
		return "", fmt.Errorf("MigTemplate() failed with last error: %v", errLast)
	}
	return templ, nil
}

func gceUpgradeScript() string {
	if len(TestContext.GCEUpgradeScript) == 0 {
		return path.Join(TestContext.RepoRoot, "cluster/gce/upgrade.sh")
	}
	return TestContext.GCEUpgradeScript
}
<|endoftext|>