<commit_before>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package user_test

import (
	"fmt"
	"net/http"
	"net/url"
	"strings"
	"testing"

	"github.com/google/exposure-notifications-verification-server/internal/envstest"
	"github.com/google/exposure-notifications-verification-server/internal/project"
	"github.com/google/exposure-notifications-verification-server/pkg/controller"
	"github.com/google/exposure-notifications-verification-server/pkg/controller/user"
	"github.com/google/exposure-notifications-verification-server/pkg/database"
	"github.com/google/exposure-notifications-verification-server/pkg/rbac"
	"github.com/gorilla/sessions"
)

func TestHandleBulkPermissions(t *testing.T) {
	t.Parallel()

	ctx := project.TestContext(t)
	harness := envstest.NewServerConfig(t, testDatabaseInstance)

	c := user.New(harness.AuthProvider, harness.Cacher, harness.Database, harness.Renderer)
	handler := c.HandleBulkPermissions(database.BulkPermissionActionAdd)

	t.Run("middleware", func(t *testing.T) {
		t.Parallel()

		envstest.ExerciseSessionMissing(t, handler)
		envstest.ExerciseMembershipMissing(t, handler)
		envstest.ExercisePermissionMissing(t, handler)
	})

	t.Run("missing_permission", func(t *testing.T) {
		t.Parallel()

		_, testUser, realm := provisionUsers(t, harness.Database)

		session := &sessions.Session{
			Values: make(map[interface{}]interface{}),
		}

		ctx := ctx
		ctx = controller.WithSession(ctx, session)
		ctx = controller.WithMembership(ctx, &database.Membership{
			Realm:       realm,
			User:        testUser,
			Permissions: rbac.UserWrite,
		})

		w, r := envstest.BuildFormRequest(ctx, t, http.MethodPost, "/", &url.Values{
			"user_id":    []string{fmt.Sprintf("%d", testUser.ID)},
			"permission": []string{"1"},
		})
		handler.ServeHTTP(w, r)

		if got, want := w.Code, http.StatusSeeOther; got != want {
			t.Errorf("expected %d to be %d", got, want)
		}

		flash := controller.Flash(session)
		if got, want := strings.Join(flash.Errors(), ", "), "does not have all scopes which are being granted"; !strings.Contains(got, want) {
			t.Errorf("expected %q to include %q", got, want)
		}
	})

	t.Run("success", func(t *testing.T) {
		t.Parallel()

		admin, testUser, realm := provisionUsers(t, harness.Database)

		ctx := ctx
		ctx = controller.WithSession(ctx, &sessions.Session{})
		ctx = controller.WithMembership(ctx, &database.Membership{
			Realm:       realm,
			User:        admin,
			Permissions: rbac.UserWrite | 1,
		})

		w, r := envstest.BuildFormRequest(ctx, t, http.MethodPost, "/", &url.Values{
			"user_id":    []string{fmt.Sprintf("%d", testUser.ID)},
			"permission": []string{"1"},
		})
		handler.ServeHTTP(w, r)

		if got, want := w.Code, http.StatusSeeOther; got != want {
			t.Errorf("expected %d to be %d", got, want)
		}

		record, err := testUser.FindMembership(harness.Database, realm.ID)
		if err != nil {
			t.Fatal(err)
		}
		if !record.Can(1) {
			t.Errorf("expected %q to be able to %q", record.Permissions, 1)
		}
	})
}
<commit_msg>Fix failing test (#1920)<commit_after>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package user_test

import (
	"fmt"
	"net/http"
	"net/url"
	"strings"
	"testing"

	"github.com/google/exposure-notifications-verification-server/internal/envstest"
	"github.com/google/exposure-notifications-verification-server/internal/project"
	"github.com/google/exposure-notifications-verification-server/pkg/controller"
	"github.com/google/exposure-notifications-verification-server/pkg/controller/user"
	"github.com/google/exposure-notifications-verification-server/pkg/database"
	"github.com/google/exposure-notifications-verification-server/pkg/rbac"
	"github.com/gorilla/sessions"
)

func TestHandleBulkPermissions(t *testing.T) {
	t.Parallel()

	ctx := project.TestContext(t)
	harness := envstest.NewServerConfig(t, testDatabaseInstance)

	c := user.New(harness.AuthProvider, harness.Cacher, harness.Database, harness.Renderer)
	handler := c.HandleBulkPermissions(database.BulkPermissionActionAdd)

	t.Run("middleware", func(t *testing.T) {
		t.Parallel()

		envstest.ExerciseSessionMissing(t, handler)
		envstest.ExerciseMembershipMissing(t, handler)
		envstest.ExercisePermissionMissing(t, handler)
	})

	t.Run("missing_permission", func(t *testing.T) {
		t.Parallel()

		_, testUser, realm := provisionUsers(t, harness.Database)

		session := &sessions.Session{
			Values: make(map[interface{}]interface{}),
		}

		ctx := ctx
		ctx = controller.WithSession(ctx, session)
		ctx = controller.WithMembership(ctx, &database.Membership{
			Realm:       realm,
			User:        testUser,
			Permissions: rbac.UserWrite,
		})

		w, r := envstest.BuildFormRequest(ctx, t, http.MethodPost, "/", &url.Values{
			"user_id":    []string{fmt.Sprintf("%d", testUser.ID)},
			"permission": []string{fmt.Sprintf("%d", rbac.SettingsWrite)},
		})
		handler.ServeHTTP(w, r)

		if got, want := w.Code, http.StatusSeeOther; got != want {
			t.Errorf("expected %d to be %d", got, want)
		}

		flash := controller.Flash(session)
		if got, want := strings.Join(flash.Errors(), ", "), "does not have all scopes which are being granted"; !strings.Contains(got, want) {
			t.Errorf("expected %q to include %q", got, want)
		}
	})

	t.Run("success", func(t *testing.T) {
		t.Parallel()

		admin, testUser, realm := provisionUsers(t, harness.Database)

		ctx := ctx
		ctx = controller.WithSession(ctx, &sessions.Session{})
		ctx = controller.WithMembership(ctx, &database.Membership{
			Realm:       realm,
			User:        admin,
			Permissions: rbac.UserWrite | rbac.SettingsWrite,
		})

		w, r := envstest.BuildFormRequest(ctx, t, http.MethodPost, "/", &url.Values{
			"user_id":    []string{fmt.Sprintf("%d", testUser.ID)},
			"permission": []string{fmt.Sprintf("%d", rbac.SettingsWrite)},
		})
		handler.ServeHTTP(w, r)

		if got, want := w.Code, http.StatusSeeOther; got != want {
			t.Errorf("expected %d to be %d", got, want)
		}

		record, err := testUser.FindMembership(harness.Database, realm.ID)
		if err != nil {
			t.Fatal(err)
		}
		if !record.Can(rbac.SettingsWrite) {
			t.Errorf("expected %q to be able to %q", record.Permissions, 1)
		}
	})
}
<|endoftext|>


<commit_before>package main

// Test of initialization order of package-level vars.

var counter int

func next() int {
	c := counter
	counter++
	return c
}

func makeOrder1() [6]int {
	return [6]int{f1, b1, d1, e1, c1, a1}
}

func makeOrder2() [6]int {
	return [6]int{f2, b2, d2, e2, c2, a2}
}

var order1 = makeOrder1()

func main() {
	// order1 is a package-level variable
	if order1 != [6]int{5, 1, 3, 4, 2, 0} {
		panic(order1)
	}

	// order2 is a local variable
	var order2 = makeOrder2()
	if order2 != [6]int{11, 7, 9, 10, 8, 6} {
		panic(order2)
	}
}

var a1, b1 = next(), next()
var c1, d1 = next(), next()
var e1, f1 = next(), next()

var a2, b2 = next(), next()
var c2, d2 = next(), next()
var e2, f2 = next(), next()
<commit_msg>go.tools/ssa/interp: rewrite initorder test.<commit_after>package main

// Test of initialization order of package-level vars.

var counter int

func next() int {
	c := counter
	counter++
	return c
}

func next2() (x int, y int) {
	x = next()
	y = next()
	return
}

func makeOrder() int {
	_, _, _, _ = f, b, d, e
	return 0
}

func main() {
	// Initialization constraints:
	// - {f,b,c/d,e} < order  (ref graph traversal)
	// - order < {a}          (lexical order)
	// - b < c/d < e < f      (lexical order)
	// Solution: b c/d e f a
	abcdef := [6]int{a, b, c, d, e, f}
	if abcdef != [6]int{5, 0, 1, 2, 3, 4} {
		panic(abcdef)
	}
}

var order = makeOrder()

var a, b = next(), next()
var c, d = next2()
var e, f = next(), next()
<|endoftext|>


<commit_before>package config

import (
	"fmt"
	"os"

	"github.com/taskcluster/taskcluster-cli/client"
)

var (
	// Configuration contains the current configuration values.
	Configuration map[string]map[string]interface{}

	// OptionsDefinitions is a map of all the OptionDefinitions, by command.
	OptionsDefinitions = make(map[string]map[string]OptionDefinition)

	// Credentials is the client credentials, if present.
	Credentials *client.Credentials
)

// Setup is to be called from main
// this was originally the init() function
// but we want to make sure all other packages have been initialized
// before calling them, which Load() does
func Setup() {
	var err error

	// load configuration
	Configuration, err = Load()
	if err != nil {
		fmt.Fprintf(os.Stderr, "failed to load configuration file, error: %s\n", err)
		os.Exit(1)
	}

	// load credentials
	clientID, ok1 := Configuration["config"]["clientId"].(string)
	accessToken, ok2 := Configuration["config"]["accessToken"].(string)
	if ok1 && ok2 {
		certificate, _ := Configuration["config"]["certificate"].(string)
		authorizedScopes, _ := Configuration["config"]["authorizedScopes"].([]string)
		Credentials = &client.Credentials{
			ClientID:         clientID,
			AccessToken:      accessToken,
			Certificate:      certificate,
			AuthorizedScopes: authorizedScopes,
		}
	}
	if ok1 || ok2 {
		fmt.Println("Either ClientID or Access Token not set")
		os.Exit(1)
	}
}
<commit_msg>config: Print error message to stderr<commit_after>package config

import (
	"fmt"
	"os"

	"github.com/taskcluster/taskcluster-cli/client"
)

var (
	// Configuration contains the current configuration values.
	Configuration map[string]map[string]interface{}

	// OptionsDefinitions is a map of all the OptionDefinitions, by command.
	OptionsDefinitions = make(map[string]map[string]OptionDefinition)

	// Credentials is the client credentials, if present.
	Credentials *client.Credentials
)

// Setup is to be called from main
// this was originally the init() function
// but we want to make sure all other packages have been initialized
// before calling them, which Load() does
func Setup() {
	var err error

	// load configuration
	Configuration, err = Load()
	if err != nil {
		fmt.Fprintf(os.Stderr, "failed to load configuration file, error: %s\n", err)
		os.Exit(1)
	}

	// load credentials
	clientID, ok1 := Configuration["config"]["clientId"].(string)
	accessToken, ok2 := Configuration["config"]["accessToken"].(string)
	if ok1 && ok2 {
		certificate, _ := Configuration["config"]["certificate"].(string)
		authorizedScopes, _ := Configuration["config"]["authorizedScopes"].([]string)
		Credentials = &client.Credentials{
			ClientID:         clientID,
			AccessToken:      accessToken,
			Certificate:      certificate,
			AuthorizedScopes: authorizedScopes,
		}
	}
	if ok1 || ok2 {
		fmt.Fprintln(os.Stderr, "Either ClientID or Access Token not set")
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package gateway

import (
	"net/http"
	"testing"

	"github.com/stretchr/testify/assert"

	"github.com/TykTechnologies/tyk/apidef"
	"github.com/TykTechnologies/tyk/test"
	"github.com/TykTechnologies/tyk/user"
)

func (ts *Test) testPrepareVersioning() (string, string) {

	ts.Gw.BuildAndLoadAPI(func(spec *APISpec) {
		spec.UseKeylessAccess = false
		spec.VersionData.NotVersioned = false
		spec.VersionDefinition.Location = "header"
		spec.VersionDefinition.Key = "version"
		spec.Proxy.ListenPath = "/"
		spec.VersionData.Versions["expired"] = apidef.VersionInfo{
			Name:    "expired",
			Expires: "2006-01-02 15:04",
		}
		spec.VersionData.Versions["v2"] = apidef.VersionInfo{
			Name:             "v2",
			UseExtendedPaths: true,
			ExtendedPaths: apidef.ExtendedPathsSet{
				WhiteList: []apidef.EndPointMeta{
					{
						Path: "/mock",
						MethodActions: map[string]apidef.EndpointMethodMeta{
							http.MethodGet: {
								Action:  apidef.Reply,
								Code:    http.StatusOK,
								Data:    "testbody",
								Headers: map[string]string{"testheader": "testvalue"},
							},
						},
					},
				},
				URLRewrite: []apidef.URLRewriteMeta{
					{
						Path:         "/a",
						Method:       http.MethodGet,
						MatchPattern: "/a(.*)",
						RewriteTo:    "/b",
					},
					{
						Path:         "/c",
						Method:       http.MethodPost,
						MatchPattern: "/c(.*)",
						RewriteTo:    "/d",
					},
				},
				Ignored: []apidef.EndPointMeta{
					{
						Path: "/ignore",
					},
				},
			},
		}
	})

	keyWrongVersion := CreateSession(ts.Gw, func(s *user.SessionState) {
		s.AccessRights = map[string]user.AccessDefinition{"test": {
			APIID: "test", Versions: []string{"v3"},
		}}
	})

	keyKnownVersion := CreateSession(ts.Gw, func(s *user.SessionState) {
		s.AccessRights = map[string]user.AccessDefinition{"test": {
			APIID: "test", Versions: []string{"v1", "v2", "expired"},
		}}
	})

	return keyWrongVersion, keyKnownVersion
}

func TestVersioning(t *testing.T) {
	ts := StartTest(nil)
	defer ts.Close()

	keyWrongVersion, keyKnownVersion := ts.testPrepareVersioning()

	wrongVersionHeaders := map[string]string{
		"authorization": keyWrongVersion,
		"version":       "v3",
	}

	disallowedAccessHeaders := map[string]string{
		"authorization": keyWrongVersion,
		"version":       "v1",
	}

	knownVersionHeaders := map[string]string{
		"authorization": keyKnownVersion,
		"version":       "v1",
	}

	expiredVersionHeaders := map[string]string{
		"authorization": keyKnownVersion,
		"version":       "expired",
	}

	mockVersionHeaders := map[string]string{
		"authorization": keyKnownVersion,
		"version":       "v2",
	}

	ts.Run(t, []test.TestCase{
		{Path: "/", Code: 403, Headers: wrongVersionHeaders, BodyMatch: "This API version does not seem to exist"},
		{Path: "/", Code: 403, Headers: disallowedAccessHeaders, BodyMatch: "Access to this API has been disallowed"},
		{Path: "/", Code: 200, Headers: knownVersionHeaders},
		{Path: "/", Code: 403, Headers: expiredVersionHeaders, BodyMatch: string(VersionExpired)},
		{Path: "/mock", Code: 200, Headers: mockVersionHeaders, BodyMatch: "testbody", HeadersMatch: map[string]string{"testheader": "testvalue"}},
		{Path: "/ignore", Code: 200, Headers: mockVersionHeaders},
	}...)
}

func BenchmarkVersioning(b *testing.B) {
	b.ReportAllocs()

	ts := StartTest(nil)
	defer ts.Close()

	keyWrongVersion, keyKnownVersion := ts.testPrepareVersioning()
	wrongVersionHeaders := map[string]string{
		"authorization": keyWrongVersion,
		"version":       "v3",
	}

	disallowedAccessHeaders := map[string]string{
		"authorization": keyWrongVersion,
		"version":       "v1",
	}

	knownVersionHeaders := map[string]string{
		"authorization": keyKnownVersion,
		"version":       "v1",
	}

	expiredVersionHeaders := map[string]string{
		"authorization": keyKnownVersion,
		"version":       "expired",
	}
	mockVersionHeaders := map[string]string{
		"authorization": keyKnownVersion,
		"version":       "v2",
	}

	for i := 0; i < b.N; i++ {
		ts.Run(b, []test.TestCase{
			{Path: "/", Code: 403, Headers: wrongVersionHeaders, BodyMatch: "This API version does not seem to exist"},
			{Path: "/", Code: 403, Headers: disallowedAccessHeaders, BodyMatch: "Access to this API has been disallowed"},
			{Path: "/", Code: 200, Headers: knownVersionHeaders},
			{Path: "/", Code: 403, Headers: expiredVersionHeaders, BodyMatch: string(VersionExpired)},
			{Path: "/mock", Code: 200, Headers: mockVersionHeaders, BodyMatch: "testbody", HeadersMatch: map[string]string{"testheader": "testvalue"}},
			{Path: "/ignore", Code: 200, Headers: mockVersionHeaders},
		}...)
	}
}

func TestNotVersioned(t *testing.T) {
	g := StartTest(nil)
	defer g.Close()

	api := BuildAPI(func(spec *APISpec) {
		spec.Proxy.ListenPath = "/"
		spec.VersionData.NotVersioned = false
		spec.VersionData.Versions["Default"] = apidef.VersionInfo{
			Name:           "Default",
			OverrideTarget: "www.example.com",
		}
	})[0]

	t.Run("Versioning enabled, override target URL", func(t *testing.T) {
		g.Gw.LoadAPI(api)
		_, _ = g.Run(t, test.TestCase{Code: http.StatusInternalServerError})
	})

	t.Run("Versioning disabled, use original target URL", func(t *testing.T) {
		api.VersionData.NotVersioned = true
		g.Gw.LoadAPI(api)

		_, _ = g.Run(t, test.TestCase{Code: http.StatusOK})
	})
}

func TestNewVersioning(t *testing.T) {
	ts := StartTest(nil)
	defer ts.Close()

	versionedAPI := BuildAPI(func(a *APISpec) {
		a.APIID = "versioned"
		a.Name = "versioned"
		a.Proxy.ListenPath = "/new"
		a.UseKeylessAccess = false
		a.VersionData.DefaultVersion = ""
		a.VersionData.Versions = map[string]apidef.VersionInfo{
			"Default": {},
		}
	})[0]

	baseAPI := BuildAPI(func(a *APISpec) {
		a.APIID = "base"
		a.Proxy.ListenPath = "/default"
		a.UseKeylessAccess = true
		a.VersionData.NotVersioned = false
		a.VersionData.DefaultVersion = "v1"
		a.VersionData.Versions = map[string]apidef.VersionInfo{
			"v1": {
				Name: "v1",
			},
			"v2": {
				Name:  "v2",
				APIID: versionedAPI.APIID,
			},
		}
		a.VersionDefinition.Location = "url-param"
		a.VersionDefinition.Key = "version"
	})[0]

	ts.Gw.LoadAPI(baseAPI, versionedAPI)

	_, key := ts.CreateSession(func(s *user.SessionState) {
		s.AccessRights = map[string]user.AccessDefinition{versionedAPI.APIID: {
			APIID: versionedAPI.APIID,
		}}
	})

	headers := map[string]string{
		"Authorization": key,
	}

	_, _ = ts.Run(t, []test.TestCase{
		{Path: "/default", Code: http.StatusOK},
		// default listen path with version param should route to v2
		{Path: "/default?version=v2", Code: http.StatusUnauthorized},
		{Path: "/default?version=v2", Headers: headers, Code: http.StatusOK},
		// For not found version, it should fallback to default
		{Path: "/default?version=notFound", BodyMatch: string(VersionDoesNotExist), Code: http.StatusForbidden},
		// v2 should be accessible by its own listen path
		{Path: "/new", Code: http.StatusUnauthorized},
		{Path: "/new", Headers: headers, Code: http.StatusOK},
	}...)

	t.Run("versioned API is not versioned", func(t *testing.T) {
		assert.True(t, versionedAPI.VersionData.NotVersioned)
		t.Run("default is not specified", func(t *testing.T) {
			assert.Empty(t, versionedAPI.VersionData.DefaultVersion)
			_, _ = ts.Run(t, test.TestCase{Path: "/default?version=v2", Headers: headers, Code: http.StatusOK})
		})

		t.Run("default is invalid", func(t *testing.T) {
			versionedAPI.VersionData.DefaultVersion = "invalid"
			ts.Gw.LoadAPI(baseAPI, versionedAPI)
			_, _ = ts.Run(t, test.TestCase{Path: "/default?version=v2", Headers: headers, Code: http.StatusOK})
		})
	})

	t.Run("versioned API is versioned", func(t *testing.T) {
		versionedAPI.VersionData.NotVersioned = false
		t.Run("default is not specified", func(t *testing.T) {
			versionedAPI.VersionData.DefaultVersion = ""
			ts.Gw.LoadAPI(baseAPI, versionedAPI)
			assert.Empty(t, versionedAPI.VersionData.DefaultVersion)
			_, _ = ts.Run(t, test.TestCase{Path: "/default?version=v2", Headers: headers,
				BodyMatch: string(VersionNotFound), Code: http.StatusForbidden})
		})

		t.Run("default is invalid", func(t *testing.T) {
			versionedAPI.VersionData.DefaultVersion = "invalid"
			ts.Gw.LoadAPI(baseAPI, versionedAPI)
			_, _ = ts.Run(t, test.TestCase{Path: "/default?version=v2", Headers: headers,
				BodyMatch: string(VersionDoesNotExist), Code: http.StatusForbidden})
		})
	})
}
<commit_msg>Add strip path test when version location is url (#3777)<commit_after>package gateway

import (
	"net/http"
	"testing"

	"github.com/stretchr/testify/assert"

	"github.com/TykTechnologies/tyk/apidef"
	"github.com/TykTechnologies/tyk/test"
	"github.com/TykTechnologies/tyk/user"
)

func (ts *Test) testPrepareVersioning() (string, string) {

	ts.Gw.BuildAndLoadAPI(func(spec *APISpec) {
		spec.UseKeylessAccess = false
		spec.VersionData.NotVersioned = false
		spec.VersionDefinition.Location = "header"
		spec.VersionDefinition.Key = "version"
		spec.Proxy.ListenPath = "/"
		spec.VersionData.Versions["expired"] = apidef.VersionInfo{
			Name:    "expired",
			Expires: "2006-01-02 15:04",
		}
		spec.VersionData.Versions["v2"] = apidef.VersionInfo{
			Name:             "v2",
			UseExtendedPaths: true,
			ExtendedPaths: apidef.ExtendedPathsSet{
				WhiteList: []apidef.EndPointMeta{
					{
						Path: "/mock",
						MethodActions: map[string]apidef.EndpointMethodMeta{
							http.MethodGet: {
								Action:  apidef.Reply,
								Code:    http.StatusOK,
								Data:    "testbody",
								Headers: map[string]string{"testheader": "testvalue"},
							},
						},
					},
				},
				URLRewrite: []apidef.URLRewriteMeta{
					{
						Path:         "/a",
						Method:       http.MethodGet,
						MatchPattern: "/a(.*)",
						RewriteTo:    "/b",
					},
					{
						Path:         "/c",
						Method:       http.MethodPost,
						MatchPattern: "/c(.*)",
						RewriteTo:    "/d",
					},
				},
				Ignored: []apidef.EndPointMeta{
					{
						Path: "/ignore",
					},
				},
			},
		}
	})

	keyWrongVersion := CreateSession(ts.Gw, func(s *user.SessionState) {
		s.AccessRights = map[string]user.AccessDefinition{"test": {
			APIID: "test", Versions: []string{"v3"},
		}}
	})

	keyKnownVersion := CreateSession(ts.Gw, func(s *user.SessionState) {
		s.AccessRights = map[string]user.AccessDefinition{"test": {
			APIID: "test", Versions: []string{"v1", "v2", "expired"},
		}}
	})

	return keyWrongVersion, keyKnownVersion
}

func TestVersioning(t *testing.T) {
	ts := StartTest(nil)
	defer ts.Close()

	keyWrongVersion, keyKnownVersion := ts.testPrepareVersioning()

	wrongVersionHeaders := map[string]string{
		"authorization": keyWrongVersion,
		"version":       "v3",
	}

	disallowedAccessHeaders := map[string]string{
		"authorization": keyWrongVersion,
		"version":       "v1",
	}

	knownVersionHeaders := map[string]string{
		"authorization": keyKnownVersion,
		"version":       "v1",
	}

	expiredVersionHeaders := map[string]string{
		"authorization": keyKnownVersion,
		"version":       "expired",
	}

	mockVersionHeaders := map[string]string{
		"authorization": keyKnownVersion,
		"version":       "v2",
	}

	ts.Run(t, []test.TestCase{
		{Path: "/", Code: 403, Headers: wrongVersionHeaders, BodyMatch: "This API version does not seem to exist"},
		{Path: "/", Code: 403, Headers: disallowedAccessHeaders, BodyMatch: "Access to this API has been disallowed"},
		{Path: "/", Code: 200, Headers: knownVersionHeaders},
		{Path: "/", Code: 403, Headers: expiredVersionHeaders, BodyMatch: string(VersionExpired)},
		{Path: "/mock", Code: 200, Headers: mockVersionHeaders, BodyMatch: "testbody", HeadersMatch: map[string]string{"testheader": "testvalue"}},
		{Path: "/ignore", Code: 200, Headers: mockVersionHeaders},
	}...)
}

func BenchmarkVersioning(b *testing.B) {
	b.ReportAllocs()

	ts := StartTest(nil)
	defer ts.Close()

	keyWrongVersion, keyKnownVersion := ts.testPrepareVersioning()
	wrongVersionHeaders := map[string]string{
		"authorization": keyWrongVersion,
		"version":       "v3",
	}

	disallowedAccessHeaders := map[string]string{
		"authorization": keyWrongVersion,
		"version":       "v1",
	}

	knownVersionHeaders := map[string]string{
		"authorization": keyKnownVersion,
		"version":       "v1",
	}

	expiredVersionHeaders := map[string]string{
		"authorization": keyKnownVersion,
		"version":       "expired",
	}
	mockVersionHeaders := map[string]string{
		"authorization": keyKnownVersion,
		"version":       "v2",
	}

	for i := 0; i < b.N; i++ {
		ts.Run(b, []test.TestCase{
			{Path: "/", Code: 403, Headers: wrongVersionHeaders, BodyMatch: "This API version does not seem to exist"},
			{Path: "/", Code: 403, Headers: disallowedAccessHeaders, BodyMatch: "Access to this API has been disallowed"},
			{Path: "/", Code: 200, Headers: knownVersionHeaders},
			{Path: "/", Code: 403, Headers: expiredVersionHeaders, BodyMatch: string(VersionExpired)},
			{Path: "/mock", Code: 200, Headers: mockVersionHeaders, BodyMatch: "testbody", HeadersMatch: map[string]string{"testheader": "testvalue"}},
			{Path: "/ignore", Code: 200, Headers: mockVersionHeaders},
		}...)
	}
}

func TestNotVersioned(t *testing.T) {
	g := StartTest(nil)
	defer g.Close()

	api := BuildAPI(func(spec *APISpec) {
		spec.Proxy.ListenPath = "/"
		spec.VersionData.NotVersioned = false
		spec.VersionData.Versions["Default"] = apidef.VersionInfo{
			Name:           "Default",
			OverrideTarget: "www.example.com",
		}
	})[0]

	t.Run("Versioning enabled, override target URL", func(t *testing.T) {
		g.Gw.LoadAPI(api)
		_, _ = g.Run(t, test.TestCase{Code: http.StatusInternalServerError})
	})

	t.Run("Versioning disabled, use original target URL", func(t *testing.T) {
		api.VersionData.NotVersioned = true
		g.Gw.LoadAPI(api)

		_, _ = g.Run(t, test.TestCase{Code: http.StatusOK})
	})
}

func TestNewVersioning(t *testing.T) {
	ts := StartTest(nil)
	defer ts.Close()

	versionedAPI := BuildAPI(func(a *APISpec) {
		a.APIID = "versioned"
		a.Name = "versioned"
		a.Proxy.ListenPath = "/new"
		a.UseKeylessAccess = false
		a.VersionData.DefaultVersion = ""
		a.VersionData.Versions = map[string]apidef.VersionInfo{
			"Default": {},
		}
	})[0]

	baseAPI := BuildAPI(func(a *APISpec) {
		a.APIID = "base"
		a.Proxy.ListenPath = "/default"
		a.UseKeylessAccess = true
		a.VersionData.NotVersioned = false
		a.VersionData.DefaultVersion = "v1"
		a.VersionData.Versions = map[string]apidef.VersionInfo{
			"v1": {
				Name: "v1",
			},
			"v2": {
				Name:  "v2",
				APIID: versionedAPI.APIID,
			},
		}
		a.VersionDefinition.Location = "url-param"
		a.VersionDefinition.Key = "version"
	})[0]

	ts.Gw.LoadAPI(baseAPI, versionedAPI)

	_, key := ts.CreateSession(func(s *user.SessionState) {
		s.AccessRights = map[string]user.AccessDefinition{versionedAPI.APIID: {
			APIID: versionedAPI.APIID,
		}}
	})

	headers := map[string]string{
		"Authorization": key,
	}

	_, _ = ts.Run(t, []test.TestCase{
		{Path: "/default", Code: http.StatusOK},
		// default listen path with version param should route to v2
		{Path: "/default?version=v2", Code: http.StatusUnauthorized},
		{Path: "/default?version=v2", Headers: headers, Code: http.StatusOK},
		// For not found version, it should fallback to default
		{Path: "/default?version=notFound", BodyMatch: string(VersionDoesNotExist), Code: http.StatusForbidden},
		// v2 should be accessible by its own listen path
		{Path: "/new", Code: http.StatusUnauthorized},
		{Path: "/new", Headers: headers, Code: http.StatusOK},
	}...)

	t.Run("versioned API is not versioned", func(t *testing.T) {
		assert.True(t, versionedAPI.VersionData.NotVersioned)
		t.Run("default is not specified", func(t *testing.T) {
			assert.Empty(t, versionedAPI.VersionData.DefaultVersion)
			_, _ = ts.Run(t, test.TestCase{Path: "/default?version=v2", Headers: headers, Code: http.StatusOK})
		})

		t.Run("default is invalid", func(t *testing.T) {
			versionedAPI.VersionData.DefaultVersion = "invalid"
			ts.Gw.LoadAPI(baseAPI, versionedAPI)
			_, _ = ts.Run(t, test.TestCase{Path: "/default?version=v2", Headers: headers, Code: http.StatusOK})
		})
	})

	t.Run("versioned API is versioned", func(t *testing.T) {
		versionedAPI.VersionData.NotVersioned = false
		t.Run("default is not specified", func(t *testing.T) {
			versionedAPI.VersionData.DefaultVersion = ""
			ts.Gw.LoadAPI(baseAPI, versionedAPI)
			assert.Empty(t, versionedAPI.VersionData.DefaultVersion)
			_, _ = ts.Run(t, test.TestCase{Path: "/default?version=v2", Headers: headers,
				BodyMatch: string(VersionNotFound), Code: http.StatusForbidden})
		})

		t.Run("default is invalid", func(t *testing.T) {
			versionedAPI.VersionData.DefaultVersion = "invalid"
			ts.Gw.LoadAPI(baseAPI, versionedAPI)
			_, _ = ts.Run(t, test.TestCase{Path: "/default?version=v2", Headers: headers,
				BodyMatch: string(VersionDoesNotExist), Code: http.StatusForbidden})
		})
	})
}

func TestVersioning_StripPath(t *testing.T) {
	ts := StartTest(nil)
	defer ts.Close()

	const versionKey = "version"

	api := BuildAPI(func(spec *APISpec) {
		spec.Proxy.ListenPath = "/"
		spec.VersionData.NotVersioned = false
		spec.VersionData.DefaultVersion = "Default"
		spec.VersionData.Versions = map[string]apidef.VersionInfo{
			"Default": {},
			"v1":      {},
		}
		spec.VersionDefinition.Location = urlLocation
		spec.VersionDefinition.Key = versionKey
		spec.VersionDefinition.StripPath = false
	})[0]

	ts.Gw.LoadAPI(api)

	_, _ = ts.Run(t, test.TestCase{Path: "/v1/", BodyMatch: `"URI":"/v1/"`, Code: http.StatusOK})

	api.VersionDefinition.StripPath = true
	ts.Gw.LoadAPI(api)
	_, _ = ts.Run(t, test.TestCase{Path: "/v1/", BodyMatch: `"URI":"/"`, Code: http.StatusOK})
}
<|endoftext|>


<commit_before>package tuntap

import (
	"os"
	"unsafe"
	"syscall"
)

func createInterface(file *os.File, ifPattern string, kind DevKind) (string, error) {
	var req ifReq
	req.Flags = iffOneQueue
	copy(req.Name[:15], ifPattern)
	switch kind {
	case DevTun:
		req.Flags |= iffTun
	case DevTap:
		req.Flags |= iffTap
	default:
		panic("Unknown interface type")
	}
	_, _, err := syscall.Syscall(syscall.SYS_IOCTL, file.Fd(), uintptr(syscall.TUNSETIFF), uintptr(unsafe.Pointer(&req)))
	if err != 0 {
		return "", err
	}
	return string(req.Name[:]), nil
}
<commit_msg>fix name bug. the buffer from the kernel contains trailing null byets<commit_after>package tuntap

import (
	"os"
	"strings"
	"syscall"
	"unsafe"
)

func createInterface(file *os.File, ifPattern string, kind DevKind) (string, error) {
	var req ifReq
	req.Flags = iffOneQueue
	copy(req.Name[:15], ifPattern)
	switch kind {
	case DevTun:
		req.Flags |= iffTun
	case DevTap:
		req.Flags |= iffTap
	default:
		panic("Unknown interface type")
	}
	_, _, err := syscall.Syscall(syscall.SYS_IOCTL, file.Fd(), uintptr(syscall.TUNSETIFF), uintptr(unsafe.Pointer(&req)))
	if err != 0 {
		return "", err
	}
	return strings.TrimRight(string(req.Name[:]), "\x00"), nil
}
<|endoftext|>


<commit_before>package influxql

import (
	"encoding/json"
	"errors"
	"hash/fnv"
	"sort"
)

// TagSet is a fundamental concept within the query system. It represents a composite series,
// composed of multiple individual series that share a set of tag attributes.
type TagSet struct {
	Tags       map[string]string
	Filters    []Expr
	SeriesKeys []string
	Key        []byte
}

// AddFilter adds a series-level filter to the Tagset.
func (t *TagSet) AddFilter(key string, filter Expr) {
	t.SeriesKeys = append(t.SeriesKeys, key)
	t.Filters = append(t.Filters, filter)
}

// Row represents a single row returned from the execution of a statement.
type Row struct {
	Name    string            `json:"name,omitempty"`
	Tags    map[string]string `json:"tags,omitempty"`
	Columns []string          `json:"columns,omitempty"`
	Values  [][]interface{}   `json:"values,omitempty"`
	Err     error             `json:"err,omitempty"`
}

// tagsHash returns a hash of tag key/value pairs.
func (r *Row) tagsHash() uint64 {
	h := fnv.New64a()
	keys := r.tagsKeys()
	for _, k := range keys {
		h.Write([]byte(k))
		h.Write([]byte(r.Tags[k]))
	}
	return h.Sum64()
}

// tagKeys returns a sorted list of tag keys.
func (r *Row) tagsKeys() []string {
	a := make([]string, 0, len(r.Tags))
	for k := range r.Tags {
		a = append(a, k)
	}
	sort.Strings(a)
	return a
}

// Rows represents a list of rows that can be sorted consistently by name/tag.
type Rows []*Row

func (p Rows) Len() int { return len(p) }

func (p Rows) Less(i, j int) bool {
	// Sort by name first.
	if p[i].Name != p[j].Name {
		return p[i].Name < p[j].Name
	}

	// Sort by tag set hash. Tags don't have a meaningful sort order so we
	// just compute a hash and sort by that instead. This allows the tests
	// to receive rows in a predictable order every time.
	return p[i].tagsHash() < p[j].tagsHash()
}

func (p Rows) Swap(i, j int) { p[i], p[j] = p[j], p[i] }

// Result represents a resultset returned from a single statement.
type Result struct {
	// StatementID is just the statement's position in the query. It's used
	// to combine statement results if they're being buffered in memory.
	StatementID int `json:"-"`
	Series      Rows
	Err         error
}

// MarshalJSON encodes the result into JSON.
func (r *Result) MarshalJSON() ([]byte, error) {
	// Define a struct that outputs "error" as a string.
	var o struct {
		Series []*Row `json:"series,omitempty"`
		Err    string `json:"error,omitempty"`
	}

	// Copy fields to output struct.
	o.Series = r.Series
	if r.Err != nil {
		o.Err = r.Err.Error()
	}

	return json.Marshal(&o)
}

// UnmarshalJSON decodes the data into the Result struct
func (r *Result) UnmarshalJSON(b []byte) error {
	var o struct {
		Series []*Row `json:"series,omitempty"`
		Err    string `json:"error,omitempty"`
	}

	err := json.Unmarshal(b, &o)
	if err != nil {
		return err
	}
	r.Series = o.Series
	if o.Err != "" {
		r.Err = errors.New(o.Err)
	}
	return nil
}

func GetProcessor(expr Expr, startIndex int) (Processor, int) {
	switch expr := expr.(type) {
	case *VarRef:
		return newEchoProcessor(startIndex), startIndex + 1
	case *Call:
		return newEchoProcessor(startIndex), startIndex + 1
	case *BinaryExpr:
		return getBinaryProcessor(expr, startIndex)
	case *ParenExpr:
		return GetProcessor(expr.Expr, startIndex)
	case *NumberLiteral:
		return newLiteralProcessor(expr.Val), startIndex
	case *StringLiteral:
		return newLiteralProcessor(expr.Val), startIndex
	case *BooleanLiteral:
		return newLiteralProcessor(expr.Val), startIndex
	case *TimeLiteral:
		return newLiteralProcessor(expr.Val), startIndex
	case *DurationLiteral:
		return newLiteralProcessor(expr.Val), startIndex
	}
	panic("unreachable")
}

type Processor func(values []interface{}) interface{}

func newEchoProcessor(index int) Processor {
	return func(values []interface{}) interface{} {
		return values[index]
	}
}

func newLiteralProcessor(val interface{}) Processor {
	return func(values []interface{}) interface{} {
		return val
	}
}

func getBinaryProcessor(expr *BinaryExpr, startIndex int) (Processor, int) {
	lhs, index := GetProcessor(expr.LHS, startIndex)
	rhs, index := GetProcessor(expr.RHS, index)

	return newBinaryExprEvaluator(expr.Op, lhs, rhs), index
}

func newBinaryExprEvaluator(op Token, lhs, rhs Processor) Processor {
	switch op {
	case ADD:
		return func(values []interface{}) interface{} {
			l := lhs(values)
			r := rhs(values)
			if lv, ok := l.(float64); ok {
				if rv, ok := r.(float64); ok {
					if rv != 0 {
						return lv + rv
					}
				}
			}
			return nil
		}
	case SUB:
		return func(values []interface{}) interface{} {
			l := lhs(values)
			r := rhs(values)
			if lv, ok := l.(float64); ok {
				if rv, ok := r.(float64); ok {
					if rv != 0 {
						return lv - rv
					}
				}
			}
			return nil
		}
	case MUL:
		return func(values []interface{}) interface{} {
			l := lhs(values)
			r := rhs(values)
			if lv, ok := l.(float64); ok {
				if rv, ok := r.(float64); ok {
					if rv != 0 {
						return lv * rv
					}
				}
			}
			return nil
		}
	case DIV:
		return func(values []interface{}) interface{} {
			l := lhs(values)
			r := rhs(values)
			if lv, ok := l.(float64); ok {
				if rv, ok := r.(float64); ok {
					if rv != 0 {
						return lv / rv
					}
				}
			}
			return nil
		}
	default:
		// we shouldn't get here, but give them back nils if it goes this way
		return func(values []interface{}) interface{} {
			return nil
		}
	}
}
<commit_msg>Force all values to Float64 for math<commit_after>package influxql

import (
	"encoding/json"
	"errors"
	"hash/fnv"
	"sort"
)

// TagSet is a fundamental concept within the query system. It represents a composite series,
// composed of multiple individual series that share a set of tag attributes.
type TagSet struct {
	Tags       map[string]string
	Filters    []Expr
	SeriesKeys []string
	Key        []byte
}

// AddFilter adds a series-level filter to the Tagset.
func (t *TagSet) AddFilter(key string, filter Expr) {
	t.SeriesKeys = append(t.SeriesKeys, key)
	t.Filters = append(t.Filters, filter)
}

// Row represents a single row returned from the execution of a statement.
type Row struct {
	Name    string            `json:"name,omitempty"`
	Tags    map[string]string `json:"tags,omitempty"`
	Columns []string          `json:"columns,omitempty"`
	Values  [][]interface{}   `json:"values,omitempty"`
	Err     error             `json:"err,omitempty"`
}

// tagsHash returns a hash of tag key/value pairs.
func (r *Row) tagsHash() uint64 {
	h := fnv.New64a()
	keys := r.tagsKeys()
	for _, k := range keys {
		h.Write([]byte(k))
		h.Write([]byte(r.Tags[k]))
	}
	return h.Sum64()
}

// tagKeys returns a sorted list of tag keys.
func (r *Row) tagsKeys() []string {
	a := make([]string, 0, len(r.Tags))
	for k := range r.Tags {
		a = append(a, k)
	}
	sort.Strings(a)
	return a
}

// Rows represents a list of rows that can be sorted consistently by name/tag.
type Rows []*Row

func (p Rows) Len() int { return len(p) }

func (p Rows) Less(i, j int) bool {
	// Sort by name first.
	if p[i].Name != p[j].Name {
		return p[i].Name < p[j].Name
	}

	// Sort by tag set hash. Tags don't have a meaningful sort order so we
	// just compute a hash and sort by that instead. This allows the tests
	// to receive rows in a predictable order every time.
	return p[i].tagsHash() < p[j].tagsHash()
}

func (p Rows) Swap(i, j int) { p[i], p[j] = p[j], p[i] }

// Result represents a resultset returned from a single statement.
type Result struct {
	// StatementID is just the statement's position in the query. It's used
	// to combine statement results if they're being buffered in memory.
	StatementID int `json:"-"`
	Series      Rows
	Err         error
}

// MarshalJSON encodes the result into JSON.
func (r *Result) MarshalJSON() ([]byte, error) {
	// Define a struct that outputs "error" as a string.
	var o struct {
		Series []*Row `json:"series,omitempty"`
		Err    string `json:"error,omitempty"`
	}

	// Copy fields to output struct.
	o.Series = r.Series
	if r.Err != nil {
		o.Err = r.Err.Error()
	}

	return json.Marshal(&o)
}

// UnmarshalJSON decodes the data into the Result struct
func (r *Result) UnmarshalJSON(b []byte) error {
	var o struct {
		Series []*Row `json:"series,omitempty"`
		Err    string `json:"error,omitempty"`
	}

	err := json.Unmarshal(b, &o)
	if err != nil {
		return err
	}
	r.Series = o.Series
	if o.Err != "" {
		r.Err = errors.New(o.Err)
	}
	return nil
}

func GetProcessor(expr Expr, startIndex int) (Processor, int) {
	switch expr := expr.(type) {
	case *VarRef:
		return newEchoProcessor(startIndex), startIndex + 1
	case *Call:
		return newEchoProcessor(startIndex), startIndex + 1
	case *BinaryExpr:
		return getBinaryProcessor(expr, startIndex)
	case *ParenExpr:
		return GetProcessor(expr.Expr, startIndex)
	case *NumberLiteral:
		return newLiteralProcessor(expr.Val), startIndex
	case *StringLiteral:
		return newLiteralProcessor(expr.Val), startIndex
	case *BooleanLiteral:
		return newLiteralProcessor(expr.Val), startIndex
	case *TimeLiteral:
		return newLiteralProcessor(expr.Val), startIndex
	case *DurationLiteral:
		return newLiteralProcessor(expr.Val), startIndex
	}
	panic("unreachable")
}

type Processor func(values []interface{}) interface{}

func newEchoProcessor(index int) Processor {
	return func(values []interface{}) interface{} {
		return values[index]
	}
}

func newLiteralProcessor(val interface{}) Processor {
	return func(values []interface{}) interface{} {
		return val
	}
}

func getBinaryProcessor(expr *BinaryExpr, startIndex int) (Processor, int) {
	lhs, index := GetProcessor(expr.LHS, startIndex)
	rhs, index := GetProcessor(expr.RHS, index)

	return newBinaryExprEvaluator(expr.Op, lhs, rhs), index
}

func newBinaryExprEvaluator(op Token, lhs, rhs Processor) Processor {
	switch op {
	case ADD:
		return func(values []interface{}) interface{} {
			l := lhs(values)
			r := rhs(values)
			if lf, rf, ok := processorValuesAsFloat64(l, r); ok {
				return lf + rf
			}
			return nil
		}
	case SUB:
		return func(values []interface{}) interface{} {
			l := lhs(values)
			r := rhs(values)
			if lf, rf, ok := processorValuesAsFloat64(l, r); ok {
				return lf - rf
			}
			return nil
		}
	case MUL:
		return func(values []interface{}) interface{} {
			l := lhs(values)
			r := rhs(values)
			if lf, rf, ok := processorValuesAsFloat64(l, r); ok {
				return lf * rf
			}
			return nil
		}
	case DIV:
		return func(values []interface{}) interface{} {
			l := lhs(values)
			r := rhs(values)
			if lf, rf, ok := processorValuesAsFloat64(l, r); ok {
				return lf / rf
			}
			return nil
		}
	default:
		// we shouldn't get here, but give them back nils if it goes this way
		return func(values []interface{}) interface{} {
			return nil
		}
	}
}

func processorValuesAsFloat64(lhs interface{}, rhs interface{}) (float64, float64, bool) {
	var lf float64
	var rf float64
	var ok bool

	lf, ok = lhs.(float64)
	if !ok {
		var li int64
		if li, ok = lhs.(int64); !ok {
			return 0, 0, false
		}
		lf = float64(li)
	}
	rf, ok = rhs.(float64)
	if !ok {
		var ri int64
		if ri, ok = rhs.(int64); !ok {
			return 0, 0, false
		}
		rf = float64(ri)
	}
	return lf, rf, true
}
<|endoftext|>


<commit_before>package funcs

import (
	"bytes"
	"fmt"
	"io/ioutil"

	"net"
	"strconv"
	"strings"
	"time"

	"github.com/shanghai-edu/multissh/g"
	"golang.org/x/crypto/ssh"
)

func connect(user, password, host, key string, port int, cipherList []string) (*ssh.Session, error) {
	var (
		auth         []ssh.AuthMethod
		addr         string
		clientConfig *ssh.ClientConfig
		client       *ssh.Client
		config       ssh.Config
		session      *ssh.Session
		err          error
	)
	// get auth method
	auth = make([]ssh.AuthMethod, 0)
	if key == "" {
		auth = append(auth, ssh.Password(password))
	} else {
		pemBytes, err := ioutil.ReadFile(key)
		if err != nil {
			return nil, err
		}

		var signer ssh.Signer
		if password == "" {
			signer, err = ssh.ParsePrivateKey(pemBytes)
		} else {
			signer, err = ssh.ParsePrivateKeyWithPassphrase(pemBytes, []byte(password))
		}
		if err != nil {
			return nil, err
		}
		auth = append(auth, ssh.PublicKeys(signer))
	}

	if len(cipherList) == 0 {
		config = ssh.Config{
			Ciphers: []string{"aes128-ctr", "aes192-ctr", "aes256-ctr", "aes128-gcm@openssh.com", "arcfour256", "arcfour128", "aes128-cbc", "3des-cbc", "aes192-cbc", "aes256-cbc"},
		}
	} else {
		config = ssh.Config{
			Ciphers: cipherList,
		}
	}

	clientConfig = &ssh.ClientConfig{
		User:    user,
		Auth:    auth,
		Timeout: 30 * time.Second,
		Config:  config,
		HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error {
			return nil
		},
	}

	// connet to ssh
	addr = fmt.Sprintf("%s:%d", host, port)

	if client, err = ssh.Dial("tcp", addr, clientConfig); err != nil {
		return nil, err
	}

	// create session
	if session, err = client.NewSession(); err != nil {
		return nil, err
	}

	modes := ssh.TerminalModes{
		ssh.ECHO:          0,     // disable echoing
		ssh.TTY_OP_ISPEED: 14400, // input speed = 14.4kbaud
		ssh.TTY_OP_OSPEED: 14400, // output speed = 14.4kbaud
	}

	if err := session.RequestPty("xterm", 80, 40, modes); err != nil {
		return nil, err
	}

	return session, nil
}

func Dossh(username, password, host, key string, cmdlist []string, port, timeout int, cipherList []string, linuxMode bool, ch chan g.SSHResult) {
	chSSH := make(chan g.SSHResult)
	if linuxMode {
		go dossh_run(username, password, host, key, cmdlist, port, cipherList, chSSH)
	} else {
		go dossh_session(username, password, host, key, cmdlist, port, cipherList, chSSH)
	}
	var res g.SSHResult

	select {
	case <-time.After(time.Duration(timeout) * time.Second):
		res.Host = host
		res.Success = false
		res.Result = ("SSH run timeout：" + strconv.Itoa(timeout) + " second.")
		ch <- res
	case res = <-chSSH:
		ch <- res
	}
	return
}

func dossh_session(username, password, host, key string, cmdlist []string, port int, cipherList []string, ch chan g.SSHResult) {
	session, err := connect(username, password, host, key, port, cipherList)
	var sshResult g.SSHResult
	sshResult.Host = host

	if err != nil {
		sshResult.Success = false
		sshResult.Result = fmt.Sprintf("<%s>", err.Error())
		ch <- sshResult
		return
	}
	defer session.Close()

	cmdlist = append(cmdlist, "exit")

	stdinBuf, _ := session.StdinPipe()

	var outbt, errbt bytes.Buffer
	session.Stdout = &outbt

	session.Stderr = &errbt
	err = session.Shell()
	if err != nil {
		sshResult.Success = false
		sshResult.Result = fmt.Sprintf("<%s>", err.Error())
		ch <- sshResult
		return
	}
	for _, c := range cmdlist {
		c = c + "\n"
		stdinBuf.Write([]byte(c))
	}
	session.Wait()
	if errbt.String() != "" {
		sshResult.Success = false
		sshResult.Result = errbt.String()
		ch <- sshResult
	} else {
		sshResult.Success = true
		sshResult.Result = outbt.String()
		ch <- sshResult
	}

	return
}

func dossh_run(username, password, host, key string, cmdlist []string, port int, cipherList []string, ch chan g.SSHResult) {
	session, err := connect(username, password, host, key, port, cipherList)
	var sshResult g.SSHResult
	sshResult.Host = host

	if err != nil {
		sshResult.Success = false
		sshResult.Result = fmt.Sprintf("<%s>", err.Error())
		ch <- sshResult
		return
	}
	defer session.Close()

	cmdlist = append(cmdlist, "exit")
	newcmd := strings.Join(cmdlist, "&&")

	var outbt, errbt bytes.Buffer
	session.Stdout = &outbt

	session.Stderr = &errbt
	err = session.Run(newcmd)
	if err != nil {
		sshResult.Success = false
		sshResult.Result = fmt.Sprintf("<%s>", err.Error())
		ch <- sshResult
		return
	}

	if errbt.String() != "" {
		sshResult.Success = false
		sshResult.Result = errbt.String()
		ch <- sshResult
	} else {
		sshResult.Success = true
		sshResult.Result = outbt.String()
		ch <- sshResult
	}

	return
}
<commit_msg>fix:Adapt to some certain types of devices(switch).<commit_after>package funcs

import (
	"bytes"
	"fmt"
	"io/ioutil"

	"net"
	"strconv"
	"strings"
	"time"

	"github.com/shanghai-edu/multissh/g"
	"golang.org/x/crypto/ssh"
)

func connect(user, password, host, key string, port int, cipherList []string) (*ssh.Session, error) {
	var (
		auth         []ssh.AuthMethod
		addr         string
		clientConfig *ssh.ClientConfig
		client       *ssh.Client
		config       ssh.Config
		session      *ssh.Session
		err          error
	)
	// get auth method
	auth = make([]ssh.AuthMethod, 0)
	if key == "" {
		auth = append(auth, ssh.Password(password))
	} else {
		pemBytes, err := ioutil.ReadFile(key)
		if err != nil {
			return nil, err
		}

		var signer ssh.Signer
		if password == "" {
			signer, err = ssh.ParsePrivateKey(pemBytes)
		} else {
			signer, err = ssh.ParsePrivateKeyWithPassphrase(pemBytes, []byte(password))
		}
		if err != nil {
			return nil, err
		}
		auth = append(auth, ssh.PublicKeys(signer))
	}

	if len(cipherList) == 0 {
		config = ssh.Config{
			Ciphers:      []string{"aes128-ctr", "aes192-ctr", "aes256-ctr", "aes128-gcm@openssh.com", "arcfour256", "arcfour128", "aes128-cbc", "3des-cbc", "aes192-cbc", "aes256-cbc"},
			KeyExchanges: []string{"diffie-hellman-group-exchange-sha1", "diffie-hellman-group1-sha1", "diffie-hellman-group-exchange-sha256"},
		}
	} else {
		config = ssh.Config{
			Ciphers: cipherList,
		}
	}

	clientConfig = &ssh.ClientConfig{
		User:    user,
		Auth:    auth,
		Timeout: 30 * time.Second,
		Config:  config,
		HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error {
			return nil
		},
	}

	// connet to ssh
	addr = fmt.Sprintf("%s:%d", host, port)

	if client, err = ssh.Dial("tcp", addr, clientConfig); err != nil {
		return nil, err
	}

	// create session
	if session, err = client.NewSession(); err != nil {
		return nil, err
	}

	modes := ssh.TerminalModes{
		ssh.ECHO:          0,     // disable echoing
		ssh.TTY_OP_ISPEED: 14400, // input speed = 14.4kbaud
		ssh.TTY_OP_OSPEED: 14400, // output speed = 14.4kbaud
	}

	if err := session.RequestPty("xterm", 80, 40, modes); err != nil {
		return nil, err
	}

	return session, nil
}

func Dossh(username, password, host, key string, cmdlist []string, port, timeout int, cipherList []string, linuxMode bool, ch chan g.SSHResult) {
	chSSH := make(chan g.SSHResult)
	if linuxMode {
		go dossh_run(username, password, host, key, cmdlist, port, cipherList, chSSH)
	} else {
		go dossh_session(username, password, host, key, cmdlist, port, cipherList, chSSH)
	}
	var res g.SSHResult

	select {
	case <-time.After(time.Duration(timeout) * time.Second):
		res.Host = host
		res.Success = false
		res.Result = ("SSH run timeout：" + strconv.Itoa(timeout) + " second.")
		ch <- res
	case res = <-chSSH:
		ch <- res
	}
	return
}

func dossh_session(username, password, host, key string, cmdlist []string, port int, cipherList []string, ch chan g.SSHResult) {
	session, err := connect(username, password, host, key, port, cipherList)
	var sshResult g.SSHResult
	sshResult.Host = host

	if err != nil {
		sshResult.Success = false
		sshResult.Result = fmt.Sprintf("<%s>", err.Error())
		ch <- sshResult
		return
	}
	defer session.Close()

	cmdlist = append(cmdlist, "exit")

	stdinBuf, _ := session.StdinPipe()

	var outbt, errbt bytes.Buffer
	session.Stdout = &outbt

	session.Stderr = &errbt
	err = session.Shell()
	if err != nil {
		sshResult.Success = false
		sshResult.Result = fmt.Sprintf("<%s>", err.Error())
		ch <- sshResult
		return
	}
	for _, c := range cmdlist {
		c = c + "\n"
		stdinBuf.Write([]byte(c))
	}
	session.Wait()
	if errbt.String() != "" {
		sshResult.Success = false
		sshResult.Result = errbt.String()
		ch <- sshResult
	} else {
		sshResult.Success = true
		sshResult.Result = outbt.String()
		ch <- sshResult
	}

	return
}

func dossh_run(username, password, host, key string, cmdlist []string, port int, cipherList []string, ch chan g.SSHResult) {
	session, err := connect(username, password, host, key, port, cipherList)
	var sshResult g.SSHResult
	sshResult.Host = host

	if err != nil {
		sshResult.Success = false
		sshResult.Result = fmt.Sprintf("<%s>", err.Error())
		ch <- sshResult
		return
	}
	defer session.Close()

	cmdlist = append(cmdlist, "exit")
	newcmd := strings.Join(cmdlist, "&&")

	var outbt, errbt bytes.Buffer
	session.Stdout = &outbt

	session.Stderr = &errbt
	err = session.Run(newcmd)
	if err != nil {
		sshResult.Success = false
		sshResult.Result = fmt.Sprintf("<%s>", err.Error())
		ch <- sshResult
		return
	}

	if errbt.String() != "" {
		sshResult.Success = false
		sshResult.Result = errbt.String()
		ch <- sshResult
	} else {
		sshResult.Success = true
		sshResult.Result = outbt.String()
		ch <- sshResult
	}

	return
}
<|endoftext|>


<commit_before>// Copyright 2014 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build integration

package facade

import (
	"time"

	"github.com/control-center/serviced/dao"
	"github.com/control-center/serviced/datastore"
	"github.com/control-center/serviced/domain/addressassignment"
	"github.com/control-center/serviced/domain/host"
	"github.com/control-center/serviced/domain/pool"
	"github.com/control-center/serviced/domain/service"
	"github.com/control-center/serviced/domain/servicedefinition"
	"github.com/zenoss/glog"
	. "gopkg.in/check.v1"
)

func (s *FacadeTest) Test_HostCRUD(t *C) {
	testid := "deadb10f"
	poolid := "pool-id"
	defer s.Facade.RemoveHost(s.CTX, testid)

	//fill host with required values
	h, err := host.Build("", "65535", poolid, "", []string{}...)
	h.ID = testid
	if err != nil {
		t.Fatalf("Unexpected error building host: %v", err)
	}
	glog.Infof("Facade test add host %v", h)
	err = s.Facade.AddHost(s.CTX, h)
	//should fail since pool doesn't exist
	if err == nil {
		t.Errorf("Expected error: %v", err)
	}

	//create pool for test
	rp := pool.New(poolid)
	if err := s.Facade.AddResourcePool(s.CTX, rp); err != nil {
		t.Fatalf("Could not add pool for test: %v", err)
	}
	defer s.Facade.RemoveResourcePool(s.CTX, poolid)

	err = s.Facade.AddHost(s.CTX, h)
	if err != nil {
		t.Errorf("Unexpected error adding host: %v", err)
	}

	//Test re-add fails
	err = s.Facade.AddHost(s.CTX, h)
	if err == nil {
		t.Errorf("Expected already exists error: %v", err)
	}

	h2, err := s.Facade.GetHost(s.CTX, testid)
	if err != nil {
		t.Errorf("Unexpected error: %v", err)
	}
	if h2 == nil {
		t.Error("Unexpected nil host")

	} else if !host.HostEquals(t, h, h2) {
		t.Error("Hosts did not match")
	}

	//Test update
	h.Memory = 1024
	err = s.Facade.UpdateHost(s.CTX, h)
	h2, err = s.Facade.GetHost(s.CTX, testid)
	if err != nil {
		t.Errorf("Unexpected error: %v", err)
	}
	if !host.HostEquals(t, h, h2) {
		t.Error("Hosts did not match")
	}

	//test delete
	err = s.Facade.RemoveHost(s.CTX, testid)
	h2, err = s.Facade.GetHost(s.CTX, testid)
	if err != nil && !datastore.IsErrNoSuchEntity(err) {
		t.Errorf("Unexpected error: %v", err)
	}
}

func (s *FacadeTest) TestRestoreHosts(c *C) {
	// create pool for testing
	resourcePool := pool.New("poolid")
	s.Facade.AddResourcePool(s.CTX, resourcePool)
	defer s.Facade.RemoveResourcePool(s.CTX, "poolid")

	// success
	hosts1 := []host.Host{
		{
			ID:      "deadb11f",
			PoolID:  "poolid",
			Name:    "h1",
			IPAddr:  "192.168.0.1",
			RPCPort: 65535,
			IPs: []host.HostIPResource{
				{
					HostID:    "deadb11f",
					IPAddress: "192.168.0.1",
				},
			},
			CreatedAt: time.Time{},
			UpdatedAt: time.Time{},
		},
	}
	defer s.Facade.RemoveHost(s.CTX, "deadb11f")
	err := s.Facade.RestoreHosts(s.CTX, hosts1)
	c.Assert(err, IsNil)
	actual, err := s.Facade.GetHosts(s.CTX)
	c.Assert(err, IsNil)
	for i := range actual {
		actual[i].DatabaseVersion = 0
		actual[i].CreatedAt = time.Time{}
		actual[i].UpdatedAt = time.Time{}
	}
	c.Assert(actual, DeepEquals, hosts1)

	// different host with the same ip
	hosts2 := []host.Host{
		{
			ID:      "deadb11e",
			PoolID:  "poolid",
			Name:    "h2",
			IPAddr:  "192.168.1.1",
			RPCPort: 65535,
			IPs: []host.HostIPResource{
				{
					HostID:    "deadb11e",
					IPAddress: "192.168.1.1",
				}, {
					HostID:    "deadb11e",
					IPAddress: "192.168.0.1",
				},
			},
			CreatedAt: time.Time{},
			UpdatedAt: time.Time{},
		},
	}
	err = s.Facade.RestoreHosts(s.CTX, hosts2)
	c.Assert(err, IsNil)
	hosts2[0].DatabaseVersion++
	actual, err = s.Facade.GetHosts(s.CTX)
	c.Assert(err, IsNil)
	for i := range actual {
		actual[i].DatabaseVersion = 0
		actual[i].CreatedAt = time.Time{}
		actual[i].UpdatedAt = time.Time{}
	}
	c.Assert(actual, DeepEquals, hosts1)

	// host in different pool
	resourcePool = pool.New("poolid2")
	s.Facade.AddResourcePool(s.CTX, resourcePool)
	defer s.Facade.RemoveResourcePool(s.CTX, "poolid2")
	hosts3 := []host.Host{
		{
			ID:      "deadb11f",
			PoolID:  "poolid2",
			Name:    "h1",
			IPAddr:  "192.168.0.1",
			RPCPort: 65535,
			IPs: []host.HostIPResource{
				{
					HostID:    "deadb11f",
					IPAddress: "192.168.0.1",
				},
			},
			CreatedAt: time.Time{},
			UpdatedAt: time.Time{},
		},
	}
	err = s.Facade.RestoreHosts(s.CTX, hosts3)
	c.Assert(err, NotNil)
	actual, err = s.Facade.GetHosts(s.CTX)
	c.Assert(err, IsNil)
	for i := range actual {
		actual[i].DatabaseVersion = 0
		actual[i].CreatedAt = time.Time{}
		actual[i].UpdatedAt = time.Time{}
	}
	c.Assert(actual, DeepEquals, hosts1)
}

func (s *FacadeTest) Test_HostRemove(t *C) {
	//create pool for testing
	resoucePool := pool.New("poolid")
	s.Facade.AddResourcePool(s.CTX, resoucePool)
	defer s.Facade.RemoveResourcePool(s.CTX, "poolid")

	//add host1
	h1 := host.Host{
		ID:      "deadb11f",
		PoolID:  "poolid",
		Name:    "h1",
		IPAddr:  "192.168.0.1",
		RPCPort: 65535,
		IPs: []host.HostIPResource{
			host.HostIPResource{
				HostID:    "deadb11f",
				IPAddress: "192.168.0.1",
			},
		},
	}
	err := s.Facade.AddHost(s.CTX, &h1)
	if err != nil {
		t.Fatalf("Failed to add host %+v: %s", h1, err)
	}

	//add host2
	h2 := host.Host{
		ID:      "deadb12f",
		PoolID:  "poolid",
		Name:    "h2",
		IPAddr:  "192.168.0.2",
		RPCPort: 65535,
		IPs: []host.HostIPResource{
			host.HostIPResource{
				HostID:    "deadb12f",
				IPAddress: "192.168.0.2",
			},
		},
	}
	err = s.Facade.AddHost(s.CTX, &h2)
	if err != nil {
		t.Fatalf("Failed to add host %+v: %s", h2, err)
	}
	defer s.Facade.RemoveHost(s.CTX, "host2")

	//add service with address assignment
	s1, _ := service.NewService()
	s1.Name = "name"
	s1.PoolID = "poolid"
	s1.DeploymentID = "deployment_id"
	s1.Launch = "manual"
	s1.Endpoints = []service.ServiceEndpoint{
		service.ServiceEndpoint{},
	}
	s1.Endpoints[0].Name = "name"
	s1.Endpoints[0].AddressConfig = servicedefinition.AddressResourceConfig{Port: 123, Protocol: "tcp"}
	aa := addressassignment.AddressAssignment{ID: "id", HostID: h1.ID}
	s1.Endpoints[0].SetAssignment(aa)
	err = s.Facade.AddService(s.CTX, *s1)
	if err != nil {
		t.Fatalf("Failed to add service %+v: %s", s1, err)
	}
	defer s.Facade.RemoveService(s.CTX, s1.ID)

	request := dao.AssignmentRequest{ServiceID: s1.ID, IPAddress: h1.IPAddr, AutoAssignment: false}
	if err = s.Facade.AssignIPs(s.CTX, request); err != nil {
		t.Fatalf("Failed assigning ip to service: %s", err)
	}

	var serviceRequest dao.ServiceRequest
	services, _ := s.Facade.GetServices(s.CTX, serviceRequest)
	if len(services) <= 0 {
		t.Fatalf("Expected one service in context")
	}
	if len(services[0].Endpoints) <= 0 {
		t.Fatalf("Expected service with one endpoint in context")
	}
	ep := services[0].Endpoints[0]
	if ep.AddressAssignment.IPAddr != h1.IPAddr && ep.AddressAssignment.HostID != h1.ID {
		t.Fatalf("Incorrect IPAddress and HostID before remove host")
	}

	//remove host1
	err = s.Facade.RemoveHost(s.CTX, h1.ID)
	if err != nil {
		t.Fatalf("Failed to remove host: %s", err)
	}
	services, _ = s.Facade.GetServices(s.CTX, serviceRequest)

	if len(services) <= 0 {
		t.Fatalf("Expected one service in context")
	}

	if len(services[0].Endpoints) <= 0 {
		t.Fatalf("Expected service with one endpoint in context")
	}

	ep = services[0].Endpoints[0]
	if ep.AddressAssignment.IPAddr == h2.IPAddr || ep.AddressAssignment.HostID == h2.ID {
		t.Fatalf("Incorrect IPAddress and HostID after remove host")
	}
}
<commit_msg>fixed facade test<commit_after>// Copyright 2014 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build integration

package facade

import (
	"time"

	"github.com/control-center/serviced/dao"
	"github.com/control-center/serviced/datastore"
	"github.com/control-center/serviced/domain/addressassignment"
	"github.com/control-center/serviced/domain/host"
	"github.com/control-center/serviced/domain/pool"
	"github.com/control-center/serviced/domain/service"
	"github.com/control-center/serviced/domain/servicedefinition"
	"github.com/stretchr/testify/mock"
	"github.com/zenoss/glog"
	. "gopkg.in/check.v1"
)

func (s *FacadeTest) Test_HostCRUD(t *C) {
	testid := "deadb10f"
	poolid := "pool-id"
	defer s.Facade.RemoveHost(s.CTX, testid)

	//fill host with required values
	h, err := host.Build("", "65535", poolid, "", []string{}...)
	h.ID = testid
	if err != nil {
		t.Fatalf("Unexpected error building host: %v", err)
	}
	glog.Infof("Facade test add host %v", h)
	err = s.Facade.AddHost(s.CTX, h)
	//should fail since pool doesn't exist
	if err == nil {
		t.Errorf("Expected error: %v", err)
	}

	//create pool for test
	rp := pool.New(poolid)
	if err := s.Facade.AddResourcePool(s.CTX, rp); err != nil {
		t.Fatalf("Could not add pool for test: %v", err)
	}
	defer s.Facade.RemoveResourcePool(s.CTX, poolid)

	err = s.Facade.AddHost(s.CTX, h)
	if err != nil {
		t.Errorf("Unexpected error adding host: %v", err)
	}

	//Test re-add fails
	err = s.Facade.AddHost(s.CTX, h)
	if err == nil {
		t.Errorf("Expected already exists error: %v", err)
	}

	h2, err := s.Facade.GetHost(s.CTX, testid)
	if err != nil {
		t.Errorf("Unexpected error: %v", err)
	}
	if h2 == nil {
		t.Error("Unexpected nil host")

	} else if !host.HostEquals(t, h, h2) {
		t.Error("Hosts did not match")
	}

	//Test update
	h.Memory = 1024
	err = s.Facade.UpdateHost(s.CTX, h)
	h2, err = s.Facade.GetHost(s.CTX, testid)
	if err != nil {
		t.Errorf("Unexpected error: %v", err)
	}
	if !host.HostEquals(t, h, h2) {
		t.Error("Hosts did not match")
	}

	//test delete
	err = s.Facade.RemoveHost(s.CTX, testid)
	h2, err = s.Facade.GetHost(s.CTX, testid)
	if err != nil && !datastore.IsErrNoSuchEntity(err) {
		t.Errorf("Unexpected error: %v", err)
	}
}

func (s *FacadeTest) TestRestoreHosts(c *C) {
	// create pool for testing
	resourcePool := pool.New("poolid")
	s.Facade.AddResourcePool(s.CTX, resourcePool)
	defer s.Facade.RemoveResourcePool(s.CTX, "poolid")

	// success
	hosts1 := []host.Host{
		{
			ID:      "deadb11f",
			PoolID:  "poolid",
			Name:    "h1",
			IPAddr:  "192.168.0.1",
			RPCPort: 65535,
			IPs: []host.HostIPResource{
				{
					HostID:    "deadb11f",
					IPAddress: "192.168.0.1",
				},
			},
			CreatedAt: time.Time{},
			UpdatedAt: time.Time{},
		},
	}
	defer s.Facade.RemoveHost(s.CTX, "deadb11f")
	err := s.Facade.RestoreHosts(s.CTX, hosts1)
	c.Assert(err, IsNil)
	actual, err := s.Facade.GetHosts(s.CTX)
	c.Assert(err, IsNil)
	for i := range actual {
		actual[i].DatabaseVersion = 0
		actual[i].CreatedAt = time.Time{}
		actual[i].UpdatedAt = time.Time{}
	}
	c.Assert(actual, DeepEquals, hosts1)

	// different host with the same ip
	hosts2 := []host.Host{
		{
			ID:      "deadb11e",
			PoolID:  "poolid",
			Name:    "h2",
			IPAddr:  "192.168.1.1",
			RPCPort: 65535,
			IPs: []host.HostIPResource{
				{
					HostID:    "deadb11e",
					IPAddress: "192.168.1.1",
				}, {
					HostID:    "deadb11e",
					IPAddress: "192.168.0.1",
				},
			},
			CreatedAt: time.Time{},
			UpdatedAt: time.Time{},
		},
	}
	err = s.Facade.RestoreHosts(s.CTX, hosts2)
	c.Assert(err, IsNil)
	hosts2[0].DatabaseVersion++
	actual, err = s.Facade.GetHosts(s.CTX)
	c.Assert(err, IsNil)
	for i := range actual {
		actual[i].DatabaseVersion = 0
		actual[i].CreatedAt = time.Time{}
		actual[i].UpdatedAt = time.Time{}
	}
	c.Assert(actual, DeepEquals, hosts1)

	// host in different pool
	resourcePool = pool.New("poolid2")
	s.Facade.AddResourcePool(s.CTX, resourcePool)
	defer s.Facade.RemoveResourcePool(s.CTX, "poolid2")
	hosts3 := []host.Host{
		{
			ID:      "deadb11f",
			PoolID:  "poolid2",
			Name:    "h1",
			IPAddr:  "192.168.0.1",
			RPCPort: 65535,
			IPs: []host.HostIPResource{
				{
					HostID:    "deadb11f",
					IPAddress: "192.168.0.1",
				},
			},
			CreatedAt: time.Time{},
			UpdatedAt: time.Time{},
		},
	}
	err = s.Facade.RestoreHosts(s.CTX, hosts3)
	c.Assert(err, NotNil)
	actual, err = s.Facade.GetHosts(s.CTX)
	c.Assert(err, IsNil)
	for i := range actual {
		actual[i].DatabaseVersion = 0
		actual[i].CreatedAt = time.Time{}
		actual[i].UpdatedAt = time.Time{}
	}
	c.Assert(actual, DeepEquals, hosts1)
}

func (s *FacadeTest) Test_HostRemove(t *C) {
	//create pool for testing
	resoucePool := pool.New("poolid")
	s.Facade.AddResourcePool(s.CTX, resoucePool)
	defer s.Facade.RemoveResourcePool(s.CTX, "poolid")

	//add host1
	h1 := host.Host{
		ID:      "deadb11f",
		PoolID:  "poolid",
		Name:    "h1",
		IPAddr:  "192.168.0.1",
		RPCPort: 65535,
		IPs: []host.HostIPResource{
			host.HostIPResource{
				HostID:    "deadb11f",
				IPAddress: "192.168.0.1",
			},
		},
	}
	err := s.Facade.AddHost(s.CTX, &h1)
	if err != nil {
		t.Fatalf("Failed to add host %+v: %s", h1, err)
	}

	//add host2
	h2 := host.Host{
		ID:      "deadb12f",
		PoolID:  "poolid",
		Name:    "h2",
		IPAddr:  "192.168.0.2",
		RPCPort: 65535,
		IPs: []host.HostIPResource{
			host.HostIPResource{
				HostID:    "deadb12f",
				IPAddress: "192.168.0.2",
			},
		},
	}
	err = s.Facade.AddHost(s.CTX, &h2)
	if err != nil {
		t.Fatalf("Failed to add host %+v: %s", h2, err)
	}
	defer s.Facade.RemoveHost(s.CTX, "host2")

	//add service with address assignment
	s.On("Destroy", mock.Anything).Return(nil)
	s1, _ := service.NewService()
	s1.Name = "name"
	s1.PoolID = "poolid"
	s1.DeploymentID = "deployment_id"
	s1.Launch = "manual"
	s1.Endpoints = []service.ServiceEndpoint{
		service.ServiceEndpoint{},
	}
	s1.Endpoints[0].Name = "name"
	s1.Endpoints[0].AddressConfig = servicedefinition.AddressResourceConfig{Port: 123, Protocol: "tcp"}
	aa := addressassignment.AddressAssignment{ID: "id", HostID: h1.ID}
	s1.Endpoints[0].SetAssignment(aa)
	err = s.Facade.AddService(s.CTX, *s1)
	if err != nil {
		t.Fatalf("Failed to add service %+v: %s", s1, err)
	}
	defer s.Facade.RemoveService(s.CTX, s1.ID)

	request := dao.AssignmentRequest{ServiceID: s1.ID, IPAddress: h1.IPAddr, AutoAssignment: false}
	if err = s.Facade.AssignIPs(s.CTX, request); err != nil {
		t.Fatalf("Failed assigning ip to service: %s", err)
	}

	var serviceRequest dao.ServiceRequest
	services, _ := s.Facade.GetServices(s.CTX, serviceRequest)
	if len(services) <= 0 {
		t.Fatalf("Expected one service in context")
	}
	if len(services[0].Endpoints) <= 0 {
		t.Fatalf("Expected service with one endpoint in context")
	}
	ep := services[0].Endpoints[0]
	if ep.AddressAssignment.IPAddr != h1.IPAddr && ep.AddressAssignment.HostID != h1.ID {
		t.Fatalf("Incorrect IPAddress and HostID before remove host")
	}

	//remove host1
	err = s.Facade.RemoveHost(s.CTX, h1.ID)
	if err != nil {
		t.Fatalf("Failed to remove host: %s", err)
	}
	services, _ = s.Facade.GetServices(s.CTX, serviceRequest)

	if len(services) <= 0 {
		t.Fatalf("Expected one service in context")
	}

	if len(services[0].Endpoints) <= 0 {
		t.Fatalf("Expected service with one endpoint in context")
	}

	ep = services[0].Endpoints[0]
	if ep.AddressAssignment.IPAddr == h2.IPAddr || ep.AddressAssignment.HostID == h2.ID {
		t.Fatalf("Incorrect IPAddress and HostID after remove host")
	}
}
<|endoftext|>


<commit_before>package bluez

import (
	"reflect"

	"github.com/godbus/dbus/v5"
	"github.com/muka/go-bluetooth/util"
	log "github.com/sirupsen/logrus"
)

type WatchableClient interface {
	Client() *Client
	Path() dbus.ObjectPath
	ToProps() Properties
	GetWatchPropertiesChannel() chan *dbus.Signal
	SetWatchPropertiesChannel(chan *dbus.Signal)
}

// WatchProperties updates on property changes
func WatchProperties(wprop WatchableClient) (chan *PropertyChanged, error) {

	channel, err := wprop.Client().Register(wprop.Path(), PropertiesInterface)
	if err != nil {
		return nil, err
	}

	wprop.SetWatchPropertiesChannel(channel)
	ch := make(chan *PropertyChanged)

	go (func() {
		for {

			if channel == nil {
				break
			}

			sig := <-channel

			if sig == nil {
				return
			}

			if sig.Name != PropertiesChanged {
				continue
			}
			if sig.Path != wprop.Path() {
				continue
			}

			iface := sig.Body[0].(string)
			changes := sig.Body[1].(map[string]dbus.Variant)

			for field, val := range changes {

				// updates [*]Properties struct when a property change
				s := reflect.ValueOf(wprop.ToProps()).Elem()
				// exported field
				f := s.FieldByName(field)
				if f.IsValid() {
					// A Value can be changed only if it is
					// addressable and was not obtained by
					// the use of unexported struct fields.
					if f.CanSet() {
						x := reflect.ValueOf(val.Value())
						wprop.ToProps().Lock()
						// map[*]variant -> map[*]interface{}
						ok, err := util.AssignMapVariantToInterface(f, x)
						if err != nil {
							log.Errorf("Failed to set %s: %s", f.String(), err)
							continue
						}
						// direct assignment
						if !ok {
							f.Set(x)
						}
						wprop.ToProps().Unlock()
					}
				}

				propChanged := &PropertyChanged{
					Interface: iface,
					Name:      field,
					Value:     val.Value(),
				}
				ch <- propChanged
			}

		}
	})()

	return ch, nil
}

func UnwatchProperties(wprop WatchableClient, ch chan *PropertyChanged) error {
	if wprop.GetWatchPropertiesChannel() != nil {
		wprop.GetWatchPropertiesChannel() <- nil
		err := wprop.Client().Unregister(wprop.Path(), PropertiesInterface, wprop.GetWatchPropertiesChannel())
		if err != nil {
			return err
		}
	}
	ch <- nil
	close(ch)
	return nil
}
<commit_msg>Recover from panic in UnwatchProperties<commit_after>package bluez

import (
	"reflect"

	"github.com/godbus/dbus/v5"
	"github.com/muka/go-bluetooth/util"
	log "github.com/sirupsen/logrus"
)

type WatchableClient interface {
	Client() *Client
	Path() dbus.ObjectPath
	ToProps() Properties
	GetWatchPropertiesChannel() chan *dbus.Signal
	SetWatchPropertiesChannel(chan *dbus.Signal)
}

// WatchProperties updates on property changes
func WatchProperties(wprop WatchableClient) (chan *PropertyChanged, error) {

	channel, err := wprop.Client().Register(wprop.Path(), PropertiesInterface)
	if err != nil {
		return nil, err
	}

	wprop.SetWatchPropertiesChannel(channel)
	ch := make(chan *PropertyChanged)

	go (func() {
		defer func() {
			if err := recover(); err != nil {
				log.Warnf("Recovering from panic in SetWatchPropertiesChannel: %s", err)
			}
		}()

		for {

			if channel == nil {
				break
			}

			sig := <-channel

			if sig == nil {
				return
			}

			if sig.Name != PropertiesChanged {
				continue
			}
			if sig.Path != wprop.Path() {
				continue
			}

			iface := sig.Body[0].(string)
			changes := sig.Body[1].(map[string]dbus.Variant)

			for field, val := range changes {

				// updates [*]Properties struct when a property change
				s := reflect.ValueOf(wprop.ToProps()).Elem()
				// exported field
				f := s.FieldByName(field)
				if f.IsValid() {
					// A Value can be changed only if it is
					// addressable and was not obtained by
					// the use of unexported struct fields.
					if f.CanSet() {
						x := reflect.ValueOf(val.Value())
						wprop.ToProps().Lock()
						// map[*]variant -> map[*]interface{}
						ok, err := util.AssignMapVariantToInterface(f, x)
						if err != nil {
							log.Errorf("Failed to set %s: %s", f.String(), err)
							continue
						}
						// direct assignment
						if !ok {
							f.Set(x)
						}
						wprop.ToProps().Unlock()
					}
				}

				propChanged := &PropertyChanged{
					Interface: iface,
					Name:      field,
					Value:     val.Value(),
				}
				ch <- propChanged
			}

		}
	})()

	return ch, nil
}

func UnwatchProperties(wprop WatchableClient, ch chan *PropertyChanged) error {
	defer func() {
		if err := recover(); err != nil {
			log.Warnf("Recovering from panic in UnwatchProperties: %s", err)
		}
	}()
	if wprop.GetWatchPropertiesChannel() != nil {
		wprop.GetWatchPropertiesChannel() <- nil
		err := wprop.Client().Unregister(wprop.Path(), PropertiesInterface, wprop.GetWatchPropertiesChannel())
		if err != nil {
			return err
		}
	}
	ch <- nil
	close(ch)
	return nil
}
<|endoftext|>


<commit_before>// +build e2e

/*
Copyright 2018 Knative Authors LLC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package test

import (
	"fmt"
	"sync"
	"testing"

	knativetest "github.com/knative/pkg/test"
	"github.com/knative/pkg/test/logging"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"github.com/tektoncd/pipeline/pkg/apis/pipeline"
	tb "github.com/tektoncd/pipeline/test/builder"
)

// TestDuplicatePodTaskRun creates 10 builds and checks that each of them has only one build pod.
func TestDuplicatePodTaskRun(t *testing.T) {
	logger := logging.GetContextLogger(t.Name())
	c, namespace := setup(t, logger)
	t.Parallel()

	knativetest.CleanupOnInterrupt(func() { tearDown(t, logger, c, namespace) }, logger)
	defer tearDown(t, logger, c, namespace)

	var wg sync.WaitGroup
	for i := 0; i < 25; i++ {
		wg.Add(1)
		taskrunName := fmt.Sprintf("duplicate-pod-taskrun-%d", i)
		logger.Infof("Creating taskrun %q.", taskrunName)

		taskrun := tb.TaskRun(taskrunName, namespace, tb.TaskRunSpec(
			tb.TaskRunTaskSpec(tb.Step("echo", "busybox",
				tb.Command("/bin/echo"),
				tb.Args("simple"),
			)),
		))
		if _, err := c.TaskRunClient.Create(taskrun); err != nil {
			t.Fatalf("Error creating taskrun: %v", err)
		}
		go func(t *testing.T) {
			defer wg.Done()

			if err := WaitForTaskRunState(c, taskrunName, TaskRunSucceed(taskrunName), "TaskRunDuplicatePodTaskRunFailed"); err != nil {
				t.Fatalf("Error waiting for TaskRun to finish: %s", err)
			}

			pods, err := c.KubeClient.Kube.CoreV1().Pods(namespace).List(metav1.ListOptions{
				LabelSelector: fmt.Sprintf("%s=%s", pipeline.GroupName+pipeline.TaskRunLabelKey, taskrunName),
			})
			if err != nil {
				t.Fatalf("Error getting TaskRun pod list: %v", err)
			}
			if n := len(pods.Items); n != 1 {
				t.Fatalf("Error matching the number of build pods: expecting 1 pod, got %d", n)
			}
		}(t)
	}
	wg.Wait()
}
<commit_msg>Don't call `t.Fatal` from a goroutine 💀<commit_after>// +build e2e

/*
Copyright 2018 Knative Authors LLC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package test

import (
	"fmt"
	"sync"
	"testing"

	knativetest "github.com/knative/pkg/test"
	"github.com/knative/pkg/test/logging"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"github.com/tektoncd/pipeline/pkg/apis/pipeline"
	tb "github.com/tektoncd/pipeline/test/builder"
)

// TestDuplicatePodTaskRun creates 10 builds and checks that each of them has only one build pod.
func TestDuplicatePodTaskRun(t *testing.T) {
	logger := logging.GetContextLogger(t.Name())
	c, namespace := setup(t, logger)
	t.Parallel()

	knativetest.CleanupOnInterrupt(func() { tearDown(t, logger, c, namespace) }, logger)
	defer tearDown(t, logger, c, namespace)

	var wg sync.WaitGroup
	for i := 0; i < 25; i++ {
		wg.Add(1)
		taskrunName := fmt.Sprintf("duplicate-pod-taskrun-%d", i)
		logger.Infof("Creating taskrun %q.", taskrunName)

		taskrun := tb.TaskRun(taskrunName, namespace, tb.TaskRunSpec(
			tb.TaskRunTaskSpec(tb.Step("echo", "busybox",
				tb.Command("/bin/echo"),
				tb.Args("simple"),
			)),
		))
		if _, err := c.TaskRunClient.Create(taskrun); err != nil {
			t.Fatalf("Error creating taskrun: %v", err)
		}
		go func(t *testing.T) {
			defer wg.Done()

			if err := WaitForTaskRunState(c, taskrunName, TaskRunSucceed(taskrunName), "TaskRunDuplicatePodTaskRunFailed"); err != nil {
				t.Errorf("Error waiting for TaskRun to finish: %s", err)
				return
			}

			pods, err := c.KubeClient.Kube.CoreV1().Pods(namespace).List(metav1.ListOptions{
				LabelSelector: fmt.Sprintf("%s=%s", pipeline.GroupName+pipeline.TaskRunLabelKey, taskrunName),
			})
			if err != nil {
				t.Errorf("Error getting TaskRun pod list: %v", err)
				return
			}
			if n := len(pods.Items); n != 1 {
				t.Errorf("Error matching the number of build pods: expecting 1 pod, got %d", n)
				return
			}
		}(t)
	}
	wg.Wait()
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package assets

import (
	"bytes"
	"fmt"
	"net/url"
	"os"
	"path"
	"regexp"
	"strings"

	"github.com/blang/semver"
	"github.com/golang/glog"

	"k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/pkg/apis/kops/util"
	"k8s.io/kops/pkg/featureflag"
	"k8s.io/kops/pkg/kubemanifest"
	"k8s.io/kops/pkg/values"
	"k8s.io/kops/util/pkg/hashing"
	"k8s.io/kops/util/pkg/vfs"
)

// RewriteManifests controls whether we rewrite manifests
// Because manifest rewriting converts everything to and from YAML, we normalize everything by doing so
var RewriteManifests = featureflag.New("RewriteManifests", featureflag.Bool(true))

// AssetBuilder discovers and remaps assets.
type AssetBuilder struct {
	ContainerAssets []*ContainerAsset
	FileAssets      []*FileAsset
	AssetsLocation  *kops.Assets
	// TODO we'd like to use cloudup.Phase here, but that introduces a go cyclic dependency
	Phase string

	// KubernetesVersion is the version of kubernetes we are installing
	KubernetesVersion semver.Version
}

// ContainerAsset models a container's location.
type ContainerAsset struct {
	// DockerImage will be the name of the container we should run.
	// This is used to copy a container to a ContainerRegistry.
	DockerImage string
	// CanonicalLocation will be the source location of the container.
	CanonicalLocation string
}

// FileAsset models a file's location.
type FileAsset struct {
	// FileURL is the URL of a file that is accessed by a Kubernetes cluster.
	FileURL *url.URL
	// CanonicalFileURL is the source URL of a file. This is used to copy a file to a FileRepository.
	CanonicalFileURL *url.URL
	// SHAValue is the SHA hash of the FileAsset.
	SHAValue string
}

// NewAssetBuilder creates a new AssetBuilder.
func NewAssetBuilder(cluster *kops.Cluster, phase string) *AssetBuilder {
	a := &AssetBuilder{
		AssetsLocation: cluster.Spec.Assets,
		Phase:          phase,
	}

	version, err := util.ParseKubernetesVersion(cluster.Spec.KubernetesVersion)
	if err != nil {
		// This should have already been validated
		glog.Fatalf("unexpected error from ParseKubernetesVersion %s: %v", cluster.Spec.KubernetesVersion, err)
	}
	a.KubernetesVersion = *version

	return a
}

// RemapManifest transforms a kubernetes manifest.
// Whenever we are building a Task that includes a manifest, we should pass it through RemapManifest first.
// This will:
// * rewrite the images if they are being redirected to a mirror, and ensure the image is uploaded
func (a *AssetBuilder) RemapManifest(data []byte) ([]byte, error) {
	if !RewriteManifests.Enabled() {
		return data, nil
	}

	manifests, err := kubemanifest.LoadManifestsFrom(data)
	if err != nil {
		return nil, err
	}

	var yamlSeparator = []byte("\n---\n\n")
	var remappedManifests [][]byte
	for _, manifest := range manifests {
		if err := manifest.RemapImages(a.RemapImage); err != nil {
			return nil, fmt.Errorf("error remapping images: %v", err)
		}

		y, err := manifest.ToYAML()
		if err != nil {
			return nil, fmt.Errorf("error re-marshalling manifest: %v", err)
		}

		remappedManifests = append(remappedManifests, y)
	}

	return bytes.Join(remappedManifests, yamlSeparator), nil
}

// RemapImage normalizes a containers location if a user sets the AssetsLocation ContainerRegistry location.
func (a *AssetBuilder) RemapImage(image string) (string, error) {
	asset := &ContainerAsset{}

	asset.DockerImage = image

	// The k8s.gcr.io prefix is an alias, but for CI builds we run from a docker load,
	// and we only double-tag from 1.10 onwards.
	// For versions prior to 1.10, remap k8s.gcr.io to the old name.
	// This also means that we won't start using the aliased names on existing clusters,
	// which could otherwise be surprising to users.
	if !util.IsKubernetesGTE("1.10", a.KubernetesVersion) && strings.HasPrefix(image, "k8s.gcr.io/") {
		image = "gcr.io/google_containers/" + strings.TrimPrefix(image, "k8s.gcr.io/")
	}

	if strings.HasPrefix(image, "kope/dns-controller:") {
		// To use user-defined DNS Controller:
		// 1. DOCKER_REGISTRY=[your docker hub repo] make dns-controller-push
		// 2. export DNSCONTROLLER_IMAGE=[your docker hub repo]
		// 3. make kops and create/apply cluster
		override := os.Getenv("DNSCONTROLLER_IMAGE")
		if override != "" {
			image = override
		}
	}

	if a.AssetsLocation != nil && a.AssetsLocation.ContainerProxy != nil {
		containerProxy := strings.TrimRight(*a.AssetsLocation.ContainerProxy, "/")
		normalized := image

		// If the image name contains only a single / we need to determine if the image is located on docker-hub or if it's using a convenient URL like k8s.gcr.io/<image-name>
		// In case of a hub image it should be sufficient to just prepend the proxy url, producing eg docker-proxy.example.com/weaveworks/weave-kube
		if strings.Count(normalized, "/") <= 1 && !strings.ContainsAny(strings.Split(normalized, "/")[0], ".:") {
			normalized = containerProxy + "/" + normalized
		} else {
			var re = regexp.MustCompile(`^[^/]+`)
			normalized = re.ReplaceAllString(normalized, containerProxy)
		}

		asset.DockerImage = normalized
		asset.CanonicalLocation = image

		// Run the new image
		image = asset.DockerImage
	}

	if a.AssetsLocation != nil && a.AssetsLocation.ContainerRegistry != nil {
		registryMirror := *a.AssetsLocation.ContainerRegistry
		normalized := image

		// Remove the 'standard' kubernetes image prefix, just for sanity
		if !util.IsKubernetesGTE("1.10", a.KubernetesVersion) && strings.HasPrefix(normalized, "gcr.io/google_containers/") {
			normalized = strings.TrimPrefix(normalized, "gcr.io/google_containers/")
		} else {
			normalized = strings.TrimPrefix(normalized, "k8s.gcr.io/")
		}

		// When assembling the cluster spec, kops may call the option more then once until the config converges
		// This means that this function may me called more than once on the same image
		// It this is pass is the second one, the image will already have been normalized with the containerRegistry settings
		// If this is the case, passing though the process again will re-prepend the container registry again
		// and again, causing the spec to never converge and the config build to fail.
		if !strings.HasPrefix(normalized, registryMirror+"/") {
			// We can't nest arbitrarily
			// Some risk of collisions, but also -- and __ in the names appear to be blocked by docker hub
			normalized = strings.Replace(normalized, "/", "-", -1)
			asset.DockerImage = registryMirror + "/" + normalized
		}

		asset.CanonicalLocation = image

		// Run the new image
		image = asset.DockerImage
	}

	a.ContainerAssets = append(a.ContainerAssets, asset)
	return image, nil
}

// RemapFileAndSHA returns a remapped url for the file, if AssetsLocation is defined.
// It also returns the SHA hash of the file.
func (a *AssetBuilder) RemapFileAndSHA(fileURL *url.URL) (*url.URL, *hashing.Hash, error) {
	if fileURL == nil {
		return nil, nil, fmt.Errorf("unable to remap a nil URL")
	}

	fileAsset := &FileAsset{
		FileURL: fileURL,
	}

	if a.AssetsLocation != nil && a.AssetsLocation.FileRepository != nil {
		fileAsset.CanonicalFileURL = fileURL

		normalizedFileURL, err := a.normalizeURL(fileURL)
		if err != nil {
			return nil, nil, err
		}

		fileAsset.FileURL = normalizedFileURL

		glog.V(4).Infof("adding remapped file: %+v", fileAsset)
	}

	h, err := a.findHash(fileAsset)
	if err != nil {
		return nil, nil, err
	}
	fileAsset.SHAValue = h.Hex()

	a.FileAssets = append(a.FileAssets, fileAsset)
	glog.V(8).Infof("adding file: %+v", fileAsset)

	return fileAsset.FileURL, h, nil
}

// TODO - remove this method as CNI does now have a SHA file

// RemapFileAndSHAValue is used exclusively to remap the cni tarball, as the tarball does not have a sha file in object storage.
func (a *AssetBuilder) RemapFileAndSHAValue(fileURL *url.URL, shaValue string) (*url.URL, error) {
	if fileURL == nil {
		return nil, fmt.Errorf("unable to remap a nil URL")
	}

	fileAsset := &FileAsset{
		FileURL:  fileURL,
		SHAValue: shaValue,
	}

	if a.AssetsLocation != nil && a.AssetsLocation.FileRepository != nil {
		fileAsset.CanonicalFileURL = fileURL

		normalizedFile, err := a.normalizeURL(fileURL)
		if err != nil {
			return nil, err
		}

		fileAsset.FileURL = normalizedFile
		glog.V(4).Infof("adding remapped file: %q", fileAsset.FileURL.String())
	}

	a.FileAssets = append(a.FileAssets, fileAsset)

	return fileAsset.FileURL, nil
}

// FindHash returns the hash value of a FileAsset.
func (a *AssetBuilder) findHash(file *FileAsset) (*hashing.Hash, error) {

	// If the phase is "assets" we use the CanonicalFileURL,
	// but during other phases we use the hash from the FileRepository or the base kops path.
	// We do not want to just test for CanonicalFileURL as it is defined in
	// other phases, but is not used to test for the SHA.
	// This prevents a chicken and egg problem where the file is not yet in the FileRepository.
	//
	// assets phase -> get the sha file from the source / CanonicalFileURL
	// any other phase -> get the sha file from the kops base location or the FileRepository
	//
	// TLDR; we use the file.CanonicalFileURL during assets phase, and use file.FileUrl the
	// rest of the time. If not we get a chicken and the egg problem where we are reading the sha file
	// before it exists.
	u := file.FileURL
	if a.Phase == "assets" && file.CanonicalFileURL != nil {
		u = file.CanonicalFileURL
	}

	if u == nil {
		return nil, fmt.Errorf("file url is not defined")
	}

	for _, ext := range []string{".sha1"} {
		hashURL := u.String() + ext
		b, err := vfs.Context.ReadFile(hashURL)
		if err != nil {
			glog.Infof("error reading hash file %q: %v", hashURL, err)
			continue
		}
		hashString := strings.TrimSpace(string(b))
		glog.V(2).Infof("Found hash %q for %q", hashString, u)

		return hashing.FromString(hashString)
	}

	if a.AssetsLocation != nil && a.AssetsLocation.FileRepository != nil {
		return nil, fmt.Errorf("you may have not staged your files correctly, please execute kops update cluster using the assets phase")
	}
	return nil, fmt.Errorf("cannot determine hash for %q (have you specified a valid file location?)", u)
}

func (a *AssetBuilder) normalizeURL(file *url.URL) (*url.URL, error) {

	if a.AssetsLocation == nil || a.AssetsLocation.FileRepository == nil {
		return nil, fmt.Errorf("assetLocation and fileRepository cannot be nil to normalize an file asset URL")
	}

	f := values.StringValue(a.AssetsLocation.FileRepository)

	if f == "" {
		return nil, fmt.Errorf("assetsLocation fileRepository cannot be an empty string")
	}

	fileRepo, err := url.Parse(f)
	if err != nil {
		return nil, fmt.Errorf("unable to parse file repository URL %q: %v", values.StringValue(a.AssetsLocation.FileRepository), err)
	}

	fileRepo.Path = path.Join(fileRepo.Path, file.Path)

	return fileRepo, nil
}
<commit_msg>Recognize shasum format for hashes<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package assets

import (
	"bytes"
	"fmt"
	"net/url"
	"os"
	"path"
	"regexp"
	"strings"

	"github.com/blang/semver"
	"github.com/golang/glog"

	"k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/pkg/apis/kops/util"
	"k8s.io/kops/pkg/featureflag"
	"k8s.io/kops/pkg/kubemanifest"
	"k8s.io/kops/pkg/values"
	"k8s.io/kops/util/pkg/hashing"
	"k8s.io/kops/util/pkg/vfs"
)

// RewriteManifests controls whether we rewrite manifests
// Because manifest rewriting converts everything to and from YAML, we normalize everything by doing so
var RewriteManifests = featureflag.New("RewriteManifests", featureflag.Bool(true))

// AssetBuilder discovers and remaps assets.
type AssetBuilder struct {
	ContainerAssets []*ContainerAsset
	FileAssets      []*FileAsset
	AssetsLocation  *kops.Assets
	// TODO we'd like to use cloudup.Phase here, but that introduces a go cyclic dependency
	Phase string

	// KubernetesVersion is the version of kubernetes we are installing
	KubernetesVersion semver.Version
}

// ContainerAsset models a container's location.
type ContainerAsset struct {
	// DockerImage will be the name of the container we should run.
	// This is used to copy a container to a ContainerRegistry.
	DockerImage string
	// CanonicalLocation will be the source location of the container.
	CanonicalLocation string
}

// FileAsset models a file's location.
type FileAsset struct {
	// FileURL is the URL of a file that is accessed by a Kubernetes cluster.
	FileURL *url.URL
	// CanonicalFileURL is the source URL of a file. This is used to copy a file to a FileRepository.
	CanonicalFileURL *url.URL
	// SHAValue is the SHA hash of the FileAsset.
	SHAValue string
}

// NewAssetBuilder creates a new AssetBuilder.
func NewAssetBuilder(cluster *kops.Cluster, phase string) *AssetBuilder {
	a := &AssetBuilder{
		AssetsLocation: cluster.Spec.Assets,
		Phase:          phase,
	}

	version, err := util.ParseKubernetesVersion(cluster.Spec.KubernetesVersion)
	if err != nil {
		// This should have already been validated
		glog.Fatalf("unexpected error from ParseKubernetesVersion %s: %v", cluster.Spec.KubernetesVersion, err)
	}
	a.KubernetesVersion = *version

	return a
}

// RemapManifest transforms a kubernetes manifest.
// Whenever we are building a Task that includes a manifest, we should pass it through RemapManifest first.
// This will:
// * rewrite the images if they are being redirected to a mirror, and ensure the image is uploaded
func (a *AssetBuilder) RemapManifest(data []byte) ([]byte, error) {
	if !RewriteManifests.Enabled() {
		return data, nil
	}

	manifests, err := kubemanifest.LoadManifestsFrom(data)
	if err != nil {
		return nil, err
	}

	var yamlSeparator = []byte("\n---\n\n")
	var remappedManifests [][]byte
	for _, manifest := range manifests {
		if err := manifest.RemapImages(a.RemapImage); err != nil {
			return nil, fmt.Errorf("error remapping images: %v", err)
		}

		y, err := manifest.ToYAML()
		if err != nil {
			return nil, fmt.Errorf("error re-marshalling manifest: %v", err)
		}

		remappedManifests = append(remappedManifests, y)
	}

	return bytes.Join(remappedManifests, yamlSeparator), nil
}

// RemapImage normalizes a containers location if a user sets the AssetsLocation ContainerRegistry location.
func (a *AssetBuilder) RemapImage(image string) (string, error) {
	asset := &ContainerAsset{}

	asset.DockerImage = image

	// The k8s.gcr.io prefix is an alias, but for CI builds we run from a docker load,
	// and we only double-tag from 1.10 onwards.
	// For versions prior to 1.10, remap k8s.gcr.io to the old name.
	// This also means that we won't start using the aliased names on existing clusters,
	// which could otherwise be surprising to users.
	if !util.IsKubernetesGTE("1.10", a.KubernetesVersion) && strings.HasPrefix(image, "k8s.gcr.io/") {
		image = "gcr.io/google_containers/" + strings.TrimPrefix(image, "k8s.gcr.io/")
	}

	if strings.HasPrefix(image, "kope/dns-controller:") {
		// To use user-defined DNS Controller:
		// 1. DOCKER_REGISTRY=[your docker hub repo] make dns-controller-push
		// 2. export DNSCONTROLLER_IMAGE=[your docker hub repo]
		// 3. make kops and create/apply cluster
		override := os.Getenv("DNSCONTROLLER_IMAGE")
		if override != "" {
			image = override
		}
	}

	if a.AssetsLocation != nil && a.AssetsLocation.ContainerProxy != nil {
		containerProxy := strings.TrimRight(*a.AssetsLocation.ContainerProxy, "/")
		normalized := image

		// If the image name contains only a single / we need to determine if the image is located on docker-hub or if it's using a convenient URL like k8s.gcr.io/<image-name>
		// In case of a hub image it should be sufficient to just prepend the proxy url, producing eg docker-proxy.example.com/weaveworks/weave-kube
		if strings.Count(normalized, "/") <= 1 && !strings.ContainsAny(strings.Split(normalized, "/")[0], ".:") {
			normalized = containerProxy + "/" + normalized
		} else {
			var re = regexp.MustCompile(`^[^/]+`)
			normalized = re.ReplaceAllString(normalized, containerProxy)
		}

		asset.DockerImage = normalized
		asset.CanonicalLocation = image

		// Run the new image
		image = asset.DockerImage
	}

	if a.AssetsLocation != nil && a.AssetsLocation.ContainerRegistry != nil {
		registryMirror := *a.AssetsLocation.ContainerRegistry
		normalized := image

		// Remove the 'standard' kubernetes image prefix, just for sanity
		if !util.IsKubernetesGTE("1.10", a.KubernetesVersion) && strings.HasPrefix(normalized, "gcr.io/google_containers/") {
			normalized = strings.TrimPrefix(normalized, "gcr.io/google_containers/")
		} else {
			normalized = strings.TrimPrefix(normalized, "k8s.gcr.io/")
		}

		// When assembling the cluster spec, kops may call the option more then once until the config converges
		// This means that this function may me called more than once on the same image
		// It this is pass is the second one, the image will already have been normalized with the containerRegistry settings
		// If this is the case, passing though the process again will re-prepend the container registry again
		// and again, causing the spec to never converge and the config build to fail.
		if !strings.HasPrefix(normalized, registryMirror+"/") {
			// We can't nest arbitrarily
			// Some risk of collisions, but also -- and __ in the names appear to be blocked by docker hub
			normalized = strings.Replace(normalized, "/", "-", -1)
			asset.DockerImage = registryMirror + "/" + normalized
		}

		asset.CanonicalLocation = image

		// Run the new image
		image = asset.DockerImage
	}

	a.ContainerAssets = append(a.ContainerAssets, asset)
	return image, nil
}

// RemapFileAndSHA returns a remapped url for the file, if AssetsLocation is defined.
// It also returns the SHA hash of the file.
func (a *AssetBuilder) RemapFileAndSHA(fileURL *url.URL) (*url.URL, *hashing.Hash, error) {
	if fileURL == nil {
		return nil, nil, fmt.Errorf("unable to remap a nil URL")
	}

	fileAsset := &FileAsset{
		FileURL: fileURL,
	}

	if a.AssetsLocation != nil && a.AssetsLocation.FileRepository != nil {
		fileAsset.CanonicalFileURL = fileURL

		normalizedFileURL, err := a.normalizeURL(fileURL)
		if err != nil {
			return nil, nil, err
		}

		fileAsset.FileURL = normalizedFileURL

		glog.V(4).Infof("adding remapped file: %+v", fileAsset)
	}

	h, err := a.findHash(fileAsset)
	if err != nil {
		return nil, nil, err
	}
	fileAsset.SHAValue = h.Hex()

	a.FileAssets = append(a.FileAssets, fileAsset)
	glog.V(8).Infof("adding file: %+v", fileAsset)

	return fileAsset.FileURL, h, nil
}

// TODO - remove this method as CNI does now have a SHA file

// RemapFileAndSHAValue is used exclusively to remap the cni tarball, as the tarball does not have a sha file in object storage.
func (a *AssetBuilder) RemapFileAndSHAValue(fileURL *url.URL, shaValue string) (*url.URL, error) {
	if fileURL == nil {
		return nil, fmt.Errorf("unable to remap a nil URL")
	}

	fileAsset := &FileAsset{
		FileURL:  fileURL,
		SHAValue: shaValue,
	}

	if a.AssetsLocation != nil && a.AssetsLocation.FileRepository != nil {
		fileAsset.CanonicalFileURL = fileURL

		normalizedFile, err := a.normalizeURL(fileURL)
		if err != nil {
			return nil, err
		}

		fileAsset.FileURL = normalizedFile
		glog.V(4).Infof("adding remapped file: %q", fileAsset.FileURL.String())
	}

	a.FileAssets = append(a.FileAssets, fileAsset)

	return fileAsset.FileURL, nil
}

// FindHash returns the hash value of a FileAsset.
func (a *AssetBuilder) findHash(file *FileAsset) (*hashing.Hash, error) {

	// If the phase is "assets" we use the CanonicalFileURL,
	// but during other phases we use the hash from the FileRepository or the base kops path.
	// We do not want to just test for CanonicalFileURL as it is defined in
	// other phases, but is not used to test for the SHA.
	// This prevents a chicken and egg problem where the file is not yet in the FileRepository.
	//
	// assets phase -> get the sha file from the source / CanonicalFileURL
	// any other phase -> get the sha file from the kops base location or the FileRepository
	//
	// TLDR; we use the file.CanonicalFileURL during assets phase, and use file.FileUrl the
	// rest of the time. If not we get a chicken and the egg problem where we are reading the sha file
	// before it exists.
	u := file.FileURL
	if a.Phase == "assets" && file.CanonicalFileURL != nil {
		u = file.CanonicalFileURL
	}

	if u == nil {
		return nil, fmt.Errorf("file url is not defined")
	}

	for _, ext := range []string{".sha1"} {
		hashURL := u.String() + ext
		b, err := vfs.Context.ReadFile(hashURL)
		if err != nil {
			glog.Infof("error reading hash file %q: %v", hashURL, err)
			continue
		}
		hashString := strings.TrimSpace(string(b))
		glog.V(2).Infof("Found hash %q for %q", hashString, u)

		// Accept a hash string that is `<hash> <filename>`
		fields := strings.Fields(hashString)
		return hashing.FromString(fields[0])
	}

	if a.AssetsLocation != nil && a.AssetsLocation.FileRepository != nil {
		return nil, fmt.Errorf("you may have not staged your files correctly, please execute kops update cluster using the assets phase")
	}
	return nil, fmt.Errorf("cannot determine hash for %q (have you specified a valid file location?)", u)
}

func (a *AssetBuilder) normalizeURL(file *url.URL) (*url.URL, error) {

	if a.AssetsLocation == nil || a.AssetsLocation.FileRepository == nil {
		return nil, fmt.Errorf("assetLocation and fileRepository cannot be nil to normalize an file asset URL")
	}

	f := values.StringValue(a.AssetsLocation.FileRepository)

	if f == "" {
		return nil, fmt.Errorf("assetsLocation fileRepository cannot be an empty string")
	}

	fileRepo, err := url.Parse(f)
	if err != nil {
		return nil, fmt.Errorf("unable to parse file repository URL %q: %v", values.StringValue(a.AssetsLocation.FileRepository), err)
	}

	fileRepo.Path = path.Join(fileRepo.Path, file.Path)

	return fileRepo, nil
}
<|endoftext|>