<commit_before>// Copyright © 2017 Ricardo Aravena <raravena@branch.io>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package exec

import (
	"fmt"
	glssh "github.com/gliderlabs/ssh"
	"golang.org/x/crypto/ssh"
	"golang.org/x/crypto/ssh/agent"
	"golang.org/x/crypto/ssh/testdata"
	"io"
	"io/ioutil"
	"math/rand"
	"net"
	"os"
	"reflect"
	"testing"
)

type mockSSHKey struct {
	keyname string
	content []byte
	privkey agent.AddedKey
	pubkey  ssh.PublicKey
}

var (
	testPrivateKeys map[string]interface{}
	testSigners     map[string]ssh.Signer
	testPublicKeys  map[string]ssh.PublicKey
	sshAgentSocket  string
)

func init() {
	var err error

	n := len(testdata.PEMBytes)
	testSigners = make(map[string]ssh.Signer, n)
	testPrivateKeys = make(map[string]interface{}, n)
	testPublicKeys = make(map[string]ssh.PublicKey, n)

	for t, k := range testdata.PEMBytes {
		testPrivateKeys[t], err = ssh.ParseRawPrivateKey(k)
		if err != nil {
			panic(fmt.Sprintf("Unable to parse test key %s: %v", t, err))
		}
		testSigners[t], err = ssh.NewSignerFromKey(testPrivateKeys[t])
		if err != nil {
			panic(fmt.Sprintf("Unable to create signer for test key %s: %v", t, err))
		}
		testPublicKeys[t] = testSigners[t].PublicKey()
	}

	randomStr := fmt.Sprintf("%v", rand.Intn(5000))
	socketFile := "/tmp/gosocket" + randomStr + ".sock"
	setupSshAgent(socketFile)
	startSSHServer()
}

func setupSshAgent(socketFile string) {
	done := make(chan string, 1)
	a := agent.NewKeyring()
	go func(done chan<- string) {
		ln, err := net.Listen("unix", socketFile)
		if err != nil {
			panic(fmt.Sprintf("Couldn't create socket for tests %v", err))
		}
		// Need to wait until the socket is setup
		firstTime := true
		for {
			if firstTime == true {
				done <- socketFile
				firstTime = false
			}
			c, err := ln.Accept()
			defer c.Close()
			if err != nil {
				panic(fmt.Sprintf("Couldn't accept connection to agent tests %v", err))
			}
			go func(c io.ReadWriter) {
				err := agent.ServeAgent(a, c)
				if err != nil {
					panic(fmt.Sprintf("Couldn't serve ssh agent for tests %v", err))
				}

			}(c)
		}

	}(done)
	sshAgentSocket = <-done
}

func addKeytoSSHAgent(key agent.AddedKey) {
	aConn, _ := net.Dial("unix", sshAgentSocket)
	sshAgent := agent.NewClient(aConn)
	sshAgent.Add(key)
}

func removeKeyfromSSHAgent(key ssh.PublicKey) {
	aConn, _ := net.Dial("unix", sshAgentSocket)
	sshAgent := agent.NewClient(aConn)
	sshAgent.Remove(key)
}

func startSSHServer() {
	done := make(chan bool, 1)
	go func(done chan<- bool) {
		glssh.Handle(func(s glssh.Session) {
			authorizedKey := ssh.MarshalAuthorizedKey(s.PublicKey())
			io.WriteString(s, fmt.Sprintf("public key used by %s:\n", s.User()))
			s.Write(authorizedKey)
		})

		publicKeyOption := glssh.PublicKeyAuth(func(ctx glssh.Context, key glssh.PublicKey) bool {
			for _, pubk := range testPublicKeys {
				if glssh.KeysEqual(key, pubk) {
					return true
				}
			}
			return false // use glssh.KeysEqual() to compare against known keys
		})

		fmt.Println("starting ssh server on port 2222...")
		done <- true
		panic(glssh.ListenAndServe(":2222", nil, publicKeyOption))
	}(done)
	<-done
}

func TestMakeSigner(t *testing.T) {
	tests := []struct {
		name     string
		key      mockSSHKey
		expected ssh.Signer
	}{
		{name: "Basic key signer with valid rsa key",
			key: mockSSHKey{
				keyname: "/tmp/mockkey",
				content: testdata.PEMBytes["rsa"],
			},
			expected: testSigners["rsa"],
		},
		{name: "Basic key signer with valid dsa key",
			key: mockSSHKey{
				keyname: "/tmp/mockkey",
				content: testdata.PEMBytes["dsa"],
			},
			expected: testSigners["dsa"],
		},
		{name: "Basic key signer with valid ecdsa key",
			key: mockSSHKey{
				keyname: "/tmp/mockkey",
				content: testdata.PEMBytes["ecdsa"],
			},
			expected: testSigners["ecdsa"],
		},
		{name: "Basic key signer with valid user key",
			key: mockSSHKey{
				keyname: "/tmp/mockkey",
				content: testdata.PEMBytes["user"],
			},
			expected: testSigners["user"],
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			// Write content of the key to the keyname file
			ioutil.WriteFile(tt.key.keyname, tt.key.content, 0644)
			returned, _ := makeSigner(tt.key.keyname)
			if !reflect.DeepEqual(returned, tt.expected) {
				t.Errorf("Value received: %v expected %v", returned, tt.expected)
			}
			os.Remove(tt.key.keyname)
		})
	}
}

func TestMakeKeyring(t *testing.T) {
	tests := []struct {
		name     string
		useagent bool
		key      mockSSHKey
		expected ssh.AuthMethod
	}{
		{name: "Basic key ring with valid rsa key",
			useagent: false,
			key: mockSSHKey{
				keyname: "/tmp/mockkey",
				content: testdata.PEMBytes["rsa"],
			},
			expected: ssh.PublicKeys(testSigners["rsa"]),
		},
		{name: "Basic key ring with valid dsa key",
			useagent: false,
			key: mockSSHKey{
				keyname: "/tmp/mockkey",
				content: testdata.PEMBytes["dsa"],
			},
			expected: ssh.PublicKeys(testSigners["dsa"]),
		},
		{name: "Basic key ring with valid ecdsa key",
			useagent: false,
			key: mockSSHKey{
				keyname: "/tmp/mockkey",
				content: testdata.PEMBytes["ecdsa"],
			},
			expected: ssh.PublicKeys(testSigners["ecdsa"]),
		},
		{name: "Basic key ring with valid user key",
			useagent: false,
			key: mockSSHKey{
				keyname: "/tmp/mockkey",
				content: testdata.PEMBytes["user"],
			},
			expected: ssh.PublicKeys(testSigners["user"]),
		},
		{name: "Basic key ring agent with valid rsa key",
			useagent: true,
			key: mockSSHKey{
				keyname: "",
				content: testdata.PEMBytes["rsa"],
				privkey: agent.AddedKey{PrivateKey: testPrivateKeys["rsa"]},
				pubkey:  testPublicKeys["rsa"],
			},
			expected: ssh.PublicKeys(testSigners["rsa"]),
		},
		{name: "Basic key ring agent with valid dsa key",
			useagent: true,
			key: mockSSHKey{
				keyname: "",
				content: testdata.PEMBytes["dsa"],
				privkey: agent.AddedKey{PrivateKey: testPrivateKeys["dsa"]},
				pubkey:  testPublicKeys["dsa"],
			},
			expected: ssh.PublicKeys(testSigners["dsa"]),
		},
		{name: "Basic key ring agent with valid ecdsa key",
			useagent: true,
			key: mockSSHKey{
				keyname: "",
				content: testdata.PEMBytes["ecdsa"],
				privkey: agent.AddedKey{PrivateKey: testPrivateKeys["ecdsa"]},
				pubkey:  testPublicKeys["ecdsa"],
			},
			expected: ssh.PublicKeys(testSigners["ecdsa"]),
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if tt.useagent == true {
				addKeytoSSHAgent(tt.key.privkey)
			}
			// Write content of the key to the keyname file
			if tt.key.keyname != "" {
				ioutil.WriteFile(tt.key.keyname, tt.key.content, 0644)
			}
			returned := makeKeyring(tt.key.keyname, tt.useagent)
			// DeepEqual always returns false for functions unless nil
			// hence converting to string to compare
			check1 := reflect.ValueOf(returned).String()
			check2 := reflect.ValueOf(tt.expected).String()
			if !reflect.DeepEqual(check1, check2) {
				t.Errorf("Value received: %v expected %v", returned, tt.expected)
			}
			if tt.useagent == true {
				removeKeyfromSSHAgent(tt.key.pubkey)
			}
			if tt.key.keyname != "" {
				os.Remove(tt.key.keyname)
			}
		})
	}
}

func TestRun(t *testing.T) {
	tests := []struct {
		name     string
		machines []string
		port     string
		user     string
		cmd      string
		key      mockSSHKey
		useagent bool
		expected bool
	}{
		{name: "Basic with valid rsa key",
			machines: []string{"localhost"},
			port:     "2222",
			cmd:      "ls",
			user:     "testuser",
			key: mockSSHKey{
				keyname: "/tmp/mockkey",
				content: testdata.PEMBytes["rsa"],
			},
			useagent: false,
			expected: true,
		},
		{name: "Basic with valid rsa key wrong hostname",
			machines: []string{"bogushost"},
			port:     "2222",
			cmd:      "ls",
			user:     "testuser",
			key: mockSSHKey{
				keyname: "/tmp/mockkey",
				content: testdata.PEMBytes["rsa"],
			},
			useagent: false,
			expected: false,
		},
		{name: "Basic with valid rsa key wrong port",
			machines: []string{"localhost"},
			port:     "2223",
			cmd:      "ls",
			user:     "testuser",
			key: mockSSHKey{
				keyname: "/tmp/mockkey",
				content: testdata.PEMBytes["rsa"],
			},
			useagent: false,
			expected: false,
		},
		{name: "Basic with valid rsa key Google endpoint",
			machines: []string{"www.google.com"},
			port:     "22",
			cmd:      "ls",
			user:     "testuser",
			key: mockSSHKey{
				keyname: "/tmp/mockkey",
				content: testdata.PEMBytes["rsa"],
			},
			useagent: false,
			expected: false,
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if tt.useagent == true {
				addKeytoSSHAgent(tt.key.privkey)
			}
			// Write content of the key to the keyname file
			if tt.key.keyname != "" {
				ioutil.WriteFile(tt.key.keyname, tt.key.content, 0644)
			}
			returned := Run(Machines(tt.machines),
				User(tt.user),
				Port(tt.port),
				Cmd(tt.cmd),
				Key(tt.key.keyname),
				UseAgent(tt.useagent))

			if !(returned == tt.expected) {
				t.Errorf("Value received: %v expected %v", returned, tt.expected)
			}

			if tt.useagent == true {
				removeKeyfromSSHAgent(tt.key.pubkey)
			}
			if tt.key.keyname != "" {
				os.Remove(tt.key.keyname)
			}
		})
	}
}

func TestTearDown(t *testing.T) {
	tests := []struct {
		name string
		id   string
	}{
		{name: "Teardown SSH Agent",
			id: "sshAgentTdown"},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if tt.id == "sshAgentTdown" {
				os.Remove(sshAgentSocket)
			}

		})

	}
}
<commit_msg>Clean up (#17)<commit_after>// Copyright © 2017 Ricardo Aravena <raravena@branch.io>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package exec

import (
	"fmt"
	glssh "github.com/gliderlabs/ssh"
	"golang.org/x/crypto/ssh"
	"golang.org/x/crypto/ssh/agent"
	"golang.org/x/crypto/ssh/testdata"
	"io"
	"io/ioutil"
	"math/rand"
	"net"
	"os"
	"reflect"
	"testing"
)

type mockSSHKey struct {
	keyname string
	content []byte
	privkey agent.AddedKey
	pubkey  ssh.PublicKey
}

var (
	testPrivateKeys map[string]interface{}
	testSigners     map[string]ssh.Signer
	testPublicKeys  map[string]ssh.PublicKey
	sshAgentSocket  string
)

func init() {
	var err error

	n := len(testdata.PEMBytes)
	testSigners = make(map[string]ssh.Signer, n)
	testPrivateKeys = make(map[string]interface{}, n)
	testPublicKeys = make(map[string]ssh.PublicKey, n)

	for t, k := range testdata.PEMBytes {
		testPrivateKeys[t], err = ssh.ParseRawPrivateKey(k)
		if err != nil {
			panic(fmt.Sprintf("Unable to parse test key %s: %v", t, err))
		}
		testSigners[t], err = ssh.NewSignerFromKey(testPrivateKeys[t])
		if err != nil {
			panic(fmt.Sprintf("Unable to create signer for test key %s: %v", t, err))
		}
		testPublicKeys[t] = testSigners[t].PublicKey()
	}

	randomStr := fmt.Sprintf("%v", rand.Intn(5000))
	socketFile := "/tmp/gosocket" + randomStr + ".sock"
	setupSshAgent(socketFile)
	startSSHServer()
}

func setupSshAgent(socketFile string) {
	done := make(chan string, 1)
	a := agent.NewKeyring()
	go func(done chan<- string) {
		ln, err := net.Listen("unix", socketFile)
		if err != nil {
			panic(fmt.Sprintf("Couldn't create socket for tests %v", err))
		}
		// Need to wait until the socket is setup
		firstTime := true
		for {
			if firstTime == true {
				done <- socketFile
				firstTime = false
			}
			c, err := ln.Accept()
			defer c.Close()
			if err != nil {
				panic(fmt.Sprintf("Couldn't accept connection to agent tests %v", err))
			}
			go func(c io.ReadWriter) {
				err := agent.ServeAgent(a, c)
				if err != nil {
					panic(fmt.Sprintf("Couldn't serve ssh agent for tests %v", err))
				}

			}(c)
		}

	}(done)
	sshAgentSocket = <-done
}

func addKeytoSSHAgent(key agent.AddedKey) {
	aConn, _ := net.Dial("unix", sshAgentSocket)
	sshAgent := agent.NewClient(aConn)
	sshAgent.Add(key)
}

func removeKeyfromSSHAgent(key ssh.PublicKey) {
	aConn, _ := net.Dial("unix", sshAgentSocket)
	sshAgent := agent.NewClient(aConn)
	sshAgent.Remove(key)
}

func startSSHServer() {
	done := make(chan bool, 1)
	go func(done chan<- bool) {
		glssh.Handle(func(s glssh.Session) {
			authorizedKey := ssh.MarshalAuthorizedKey(s.PublicKey())
			io.WriteString(s, fmt.Sprintf("public key used by %s:\n", s.User()))
			s.Write(authorizedKey)
		})

		publicKeyOption := glssh.PublicKeyAuth(func(ctx glssh.Context, key glssh.PublicKey) bool {
			for _, pubk := range testPublicKeys {
				if glssh.KeysEqual(key, pubk) {
					return true
				}
			}
			return false
		})

		fmt.Println("starting ssh server on port 2222...")
		done <- true
		panic(glssh.ListenAndServe(":2222", nil, publicKeyOption))
	}(done)
	<-done
}

func TestMakeSigner(t *testing.T) {
	tests := []struct {
		name     string
		key      mockSSHKey
		expected ssh.Signer
	}{
		{name: "Basic key signer with valid rsa key",
			key: mockSSHKey{
				keyname: "/tmp/mockkey",
				content: testdata.PEMBytes["rsa"],
			},
			expected: testSigners["rsa"],
		},
		{name: "Basic key signer with valid dsa key",
			key: mockSSHKey{
				keyname: "/tmp/mockkey",
				content: testdata.PEMBytes["dsa"],
			},
			expected: testSigners["dsa"],
		},
		{name: "Basic key signer with valid ecdsa key",
			key: mockSSHKey{
				keyname: "/tmp/mockkey",
				content: testdata.PEMBytes["ecdsa"],
			},
			expected: testSigners["ecdsa"],
		},
		{name: "Basic key signer with valid user key",
			key: mockSSHKey{
				keyname: "/tmp/mockkey",
				content: testdata.PEMBytes["user"],
			},
			expected: testSigners["user"],
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			// Write content of the key to the keyname file
			ioutil.WriteFile(tt.key.keyname, tt.key.content, 0644)
			returned, _ := makeSigner(tt.key.keyname)
			if !reflect.DeepEqual(returned, tt.expected) {
				t.Errorf("Value received: %v expected %v", returned, tt.expected)
			}
			os.Remove(tt.key.keyname)
		})
	}
}

func TestMakeKeyring(t *testing.T) {
	tests := []struct {
		name     string
		useagent bool
		key      mockSSHKey
		expected ssh.AuthMethod
	}{
		{name: "Basic key ring with valid rsa key",
			useagent: false,
			key: mockSSHKey{
				keyname: "/tmp/mockkey",
				content: testdata.PEMBytes["rsa"],
			},
			expected: ssh.PublicKeys(testSigners["rsa"]),
		},
		{name: "Basic key ring with valid dsa key",
			useagent: false,
			key: mockSSHKey{
				keyname: "/tmp/mockkey",
				content: testdata.PEMBytes["dsa"],
			},
			expected: ssh.PublicKeys(testSigners["dsa"]),
		},
		{name: "Basic key ring with valid ecdsa key",
			useagent: false,
			key: mockSSHKey{
				keyname: "/tmp/mockkey",
				content: testdata.PEMBytes["ecdsa"],
			},
			expected: ssh.PublicKeys(testSigners["ecdsa"]),
		},
		{name: "Basic key ring with valid user key",
			useagent: false,
			key: mockSSHKey{
				keyname: "/tmp/mockkey",
				content: testdata.PEMBytes["user"],
			},
			expected: ssh.PublicKeys(testSigners["user"]),
		},
		{name: "Basic key ring agent with valid rsa key",
			useagent: true,
			key: mockSSHKey{
				keyname: "",
				content: testdata.PEMBytes["rsa"],
				privkey: agent.AddedKey{PrivateKey: testPrivateKeys["rsa"]},
				pubkey:  testPublicKeys["rsa"],
			},
			expected: ssh.PublicKeys(testSigners["rsa"]),
		},
		{name: "Basic key ring agent with valid dsa key",
			useagent: true,
			key: mockSSHKey{
				keyname: "",
				content: testdata.PEMBytes["dsa"],
				privkey: agent.AddedKey{PrivateKey: testPrivateKeys["dsa"]},
				pubkey:  testPublicKeys["dsa"],
			},
			expected: ssh.PublicKeys(testSigners["dsa"]),
		},
		{name: "Basic key ring agent with valid ecdsa key",
			useagent: true,
			key: mockSSHKey{
				keyname: "",
				content: testdata.PEMBytes["ecdsa"],
				privkey: agent.AddedKey{PrivateKey: testPrivateKeys["ecdsa"]},
				pubkey:  testPublicKeys["ecdsa"],
			},
			expected: ssh.PublicKeys(testSigners["ecdsa"]),
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if tt.useagent == true {
				addKeytoSSHAgent(tt.key.privkey)
			}
			// Write content of the key to the keyname file
			if tt.key.keyname != "" {
				ioutil.WriteFile(tt.key.keyname, tt.key.content, 0644)
			}
			returned := makeKeyring(tt.key.keyname, tt.useagent)
			// DeepEqual always returns false for functions unless nil
			// hence converting to string to compare
			check1 := reflect.ValueOf(returned).String()
			check2 := reflect.ValueOf(tt.expected).String()
			if !reflect.DeepEqual(check1, check2) {
				t.Errorf("Value received: %v expected %v", returned, tt.expected)
			}
			if tt.useagent == true {
				removeKeyfromSSHAgent(tt.key.pubkey)
			}
			if tt.key.keyname != "" {
				os.Remove(tt.key.keyname)
			}
		})
	}
}

func TestRun(t *testing.T) {
	tests := []struct {
		name     string
		machines []string
		port     string
		user     string
		cmd      string
		key      mockSSHKey
		useagent bool
		expected bool
	}{
		{name: "Basic with valid rsa key",
			machines: []string{"localhost"},
			port:     "2222",
			cmd:      "ls",
			user:     "testuser",
			key: mockSSHKey{
				keyname: "/tmp/mockkey",
				content: testdata.PEMBytes["rsa"],
			},
			useagent: false,
			expected: true,
		},
		{name: "Basic with valid rsa key wrong hostname",
			machines: []string{"bogushost"},
			port:     "2222",
			cmd:      "ls",
			user:     "testuser",
			key: mockSSHKey{
				keyname: "/tmp/mockkey",
				content: testdata.PEMBytes["rsa"],
			},
			useagent: false,
			expected: false,
		},
		{name: "Basic with valid rsa key wrong port",
			machines: []string{"localhost"},
			port:     "2223",
			cmd:      "ls",
			user:     "testuser",
			key: mockSSHKey{
				keyname: "/tmp/mockkey",
				content: testdata.PEMBytes["rsa"],
			},
			useagent: false,
			expected: false,
		},
		{name: "Basic with valid rsa key Google endpoint",
			machines: []string{"www.google.com"},
			port:     "22",
			cmd:      "ls",
			user:     "testuser",
			key: mockSSHKey{
				keyname: "/tmp/mockkey",
				content: testdata.PEMBytes["rsa"],
			},
			useagent: false,
			expected: false,
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if tt.useagent == true {
				addKeytoSSHAgent(tt.key.privkey)
			}
			// Write content of the key to the keyname file
			if tt.key.keyname != "" {
				ioutil.WriteFile(tt.key.keyname, tt.key.content, 0644)
			}
			returned := Run(Machines(tt.machines),
				User(tt.user),
				Port(tt.port),
				Cmd(tt.cmd),
				Key(tt.key.keyname),
				UseAgent(tt.useagent))

			if !(returned == tt.expected) {
				t.Errorf("Value received: %v expected %v", returned, tt.expected)
			}

			if tt.useagent == true {
				removeKeyfromSSHAgent(tt.key.pubkey)
			}
			if tt.key.keyname != "" {
				os.Remove(tt.key.keyname)
			}
		})
	}
}

func TestTearDown(t *testing.T) {
	tests := []struct {
		name string
		id   string
	}{
		{name: "Teardown SSH Agent",
			id: "sshAgentTdown"},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if tt.id == "sshAgentTdown" {
				os.Remove(sshAgentSocket)
			}

		})

	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package expfmt

import (
	"fmt"
	"io"
	"math"
	"mime"
	"net/http"

	dto "github.com/prometheus/client_model/go"

	"github.com/matttproud/golang_protobuf_extensions/pbutil"
	"github.com/prometheus/common/model"
)

// Decoder types decode an input stream into metric families.
type Decoder interface {
	Decode(*dto.MetricFamily) error
}

type DecodeOptions struct {
	// Timestamp is added to each value from the stream that has no explicit timestamp set.
	Timestamp model.Time
}

// NewDecor returns a new decoder based on the HTTP header.
func NewDecoder(r io.Reader, h http.Header) (Decoder, error) {
	ct := h.Get(hdrContentType)

	mediatype, params, err := mime.ParseMediaType(ct)
	if err != nil {
		return nil, fmt.Errorf("invalid Content-Type header %q: %s", ct, err)
	}

	const (
		protoType = ProtoType + "/" + ProtoSubType
		textType  = "text/plain"
	)

	switch mediatype {
	case protoType:
		if p := params["proto"]; p != ProtoProtocol {
			return nil, fmt.Errorf("unrecognized protocol message %s", p)
		}
		if e := params["encoding"]; e != "delimited" {
			return nil, fmt.Errorf("unsupported encoding %s", e)
		}
		return &protoDecoder{r: r}, nil

	case textType:
		if v, ok := params["version"]; ok && v != "0.0.4" {
			return nil, fmt.Errorf("unrecognized protocol version %s", v)
		}
		return &textDecoder{r: r}, nil

	default:
		return nil, fmt.Errorf("unsupported media type %q, expected %q or %q", mediatype, protoType, textType)
	}
}

// protoDecoder implements the Decoder interface for protocol buffers.
type protoDecoder struct {
	r io.Reader
}

// Decode implements the Decoder interface.
func (d *protoDecoder) Decode(v *dto.MetricFamily) error {
	_, err := pbutil.ReadDelimited(d.r, v)
	return err
}

// textDecoder implements the Decoder interface for the text protcol.
type textDecoder struct {
	r    io.Reader
	p    TextParser
	fams []*dto.MetricFamily
}

// Decode implements the Decoder interface.
func (d *textDecoder) Decode(v *dto.MetricFamily) error {
	// TODO(fabxc): Wrap this as a line reader to make streaming safer.
	if len(d.fams) == 0 {
		// No cached metric families, read everything and parse metrics.
		fams, err := d.p.TextToMetricFamilies(d.r)
		if err != nil {
			return err
		}
		if len(fams) == 0 {
			return io.EOF
		}
		for _, f := range fams {
			d.fams = append(d.fams, f)
		}
	}
	*v = *d.fams[len(d.fams)-1]
	d.fams = d.fams[:len(d.fams)-1]
	return nil
}

type SampleDecoder struct {
	Dec  Decoder
	Opts *DecodeOptions

	f dto.MetricFamily
}

func (sd *SampleDecoder) Decode(s *model.Vector) error {
	if err := sd.Dec.Decode(&sd.f); err != nil {
		return err
	}
	*s = extractSamples(&sd.f, sd.Opts)
	return nil
}

// Extract samples builds a slice of samples from the provided metric families.
func ExtractSamples(o *DecodeOptions, fams ...*dto.MetricFamily) model.Vector {
	var all model.Vector
	for _, f := range fams {
		all = append(all, extractSamples(f, o)...)
	}
	return all
}

func extractSamples(f *dto.MetricFamily, o *DecodeOptions) model.Vector {
	switch *f.Type {
	case dto.MetricType_COUNTER:
		return extractCounter(o, f)
	case dto.MetricType_GAUGE:
		return extractGauge(o, f)
	case dto.MetricType_SUMMARY:
		return extractSummary(o, f)
	case dto.MetricType_UNTYPED:
		return extractUntyped(o, f)
	case dto.MetricType_HISTOGRAM:
		return extractHistogram(o, f)
	}
	panic("expfmt.extractSamples: unknown metric family type")
}

func extractCounter(o *DecodeOptions, f *dto.MetricFamily) model.Vector {
	samples := make(model.Vector, 0, len(f.Metric))

	for _, m := range f.Metric {
		if m.Counter == nil {
			continue
		}

		lset := make(model.LabelSet, len(m.Label)+1)
		for _, p := range m.Label {
			lset[model.LabelName(p.GetName())] = model.LabelValue(p.GetValue())
		}
		lset[model.MetricNameLabel] = model.LabelValue(f.GetName())

		smpl := &model.Sample{
			Metric: model.Metric(lset),
			Value:  model.SampleValue(m.Counter.GetValue()),
		}

		if m.TimestampMs != nil {
			smpl.Timestamp = model.TimeFromUnixNano(*m.TimestampMs * 1000000)
		} else {
			smpl.Timestamp = o.Timestamp
		}

		samples = append(samples, smpl)
	}

	return samples
}

func extractGauge(o *DecodeOptions, f *dto.MetricFamily) model.Vector {
	samples := make(model.Vector, 0, len(f.Metric))

	for _, m := range f.Metric {
		if m.Gauge == nil {
			continue
		}

		lset := make(model.LabelSet, len(m.Label)+1)
		for _, p := range m.Label {
			lset[model.LabelName(p.GetName())] = model.LabelValue(p.GetValue())
		}
		lset[model.MetricNameLabel] = model.LabelValue(f.GetName())

		smpl := &model.Sample{
			Metric: model.Metric(lset),
			Value:  model.SampleValue(m.Gauge.GetValue()),
		}

		if m.TimestampMs != nil {
			smpl.Timestamp = model.TimeFromUnixNano(*m.TimestampMs * 1000000)
		} else {
			smpl.Timestamp = o.Timestamp
		}

		samples = append(samples, smpl)
	}

	return samples
}

func extractUntyped(o *DecodeOptions, f *dto.MetricFamily) model.Vector {
	samples := make(model.Vector, 0, len(f.Metric))

	for _, m := range f.Metric {
		if m.Untyped == nil {
			continue
		}

		lset := make(model.LabelSet, len(m.Label)+1)
		for _, p := range m.Label {
			lset[model.LabelName(p.GetName())] = model.LabelValue(p.GetValue())
		}
		lset[model.MetricNameLabel] = model.LabelValue(f.GetName())

		smpl := &model.Sample{
			Metric: model.Metric(lset),
			Value:  model.SampleValue(m.Untyped.GetValue()),
		}

		if m.TimestampMs != nil {
			smpl.Timestamp = model.TimeFromUnixNano(*m.TimestampMs * 1000000)
		} else {
			smpl.Timestamp = o.Timestamp
		}

		samples = append(samples, smpl)
	}

	return samples
}

func extractSummary(o *DecodeOptions, f *dto.MetricFamily) model.Vector {
	samples := make(model.Vector, 0, len(f.Metric))

	for _, m := range f.Metric {
		if m.Summary == nil {
			continue
		}

		timestamp := o.Timestamp
		if m.TimestampMs != nil {
			timestamp = model.TimeFromUnixNano(*m.TimestampMs * 1000000)
		}

		for _, q := range m.Summary.Quantile {
			lset := make(model.LabelSet, len(m.Label)+2)
			for _, p := range m.Label {
				lset[model.LabelName(p.GetName())] = model.LabelValue(p.GetValue())
			}
			// BUG(matt): Update other names to "quantile".
			lset[model.LabelName(model.QuantileLabel)] = model.LabelValue(fmt.Sprint(q.GetQuantile()))
			lset[model.MetricNameLabel] = model.LabelValue(f.GetName())

			samples = append(samples, &model.Sample{
				Metric:    model.Metric(lset),
				Value:     model.SampleValue(q.GetValue()),
				Timestamp: timestamp,
			})
		}

		if m.Summary.SampleSum != nil {
			lset := make(model.LabelSet, len(m.Label)+1)
			for _, p := range m.Label {
				lset[model.LabelName(p.GetName())] = model.LabelValue(p.GetValue())
			}
			lset[model.MetricNameLabel] = model.LabelValue(f.GetName() + "_sum")

			samples = append(samples, &model.Sample{
				Metric:    model.Metric(lset),
				Value:     model.SampleValue(m.Summary.GetSampleSum()),
				Timestamp: timestamp,
			})
		}

		if m.Summary.SampleCount != nil {
			lset := make(model.LabelSet, len(m.Label)+1)
			for _, p := range m.Label {
				lset[model.LabelName(p.GetName())] = model.LabelValue(p.GetValue())
			}
			lset[model.MetricNameLabel] = model.LabelValue(f.GetName() + "_count")

			samples = append(samples, &model.Sample{
				Metric:    model.Metric(lset),
				Value:     model.SampleValue(m.Summary.GetSampleCount()),
				Timestamp: timestamp,
			})
		}
	}

	return samples
}

func extractHistogram(o *DecodeOptions, f *dto.MetricFamily) model.Vector {
	samples := make(model.Vector, 0, len(f.Metric))

	for _, m := range f.Metric {
		if m.Histogram == nil {
			continue
		}

		timestamp := o.Timestamp
		if m.TimestampMs != nil {
			timestamp = model.TimeFromUnixNano(*m.TimestampMs * 1000000)
		}

		infSeen := false

		for _, q := range m.Histogram.Bucket {
			lset := make(model.LabelSet, len(m.Label)+2)
			for _, p := range m.Label {
				lset[model.LabelName(p.GetName())] = model.LabelValue(p.GetValue())
			}
			lset[model.LabelName(model.BucketLabel)] = model.LabelValue(fmt.Sprint(q.GetUpperBound()))
			lset[model.MetricNameLabel] = model.LabelValue(f.GetName() + "_bucket")

			if math.IsInf(q.GetUpperBound(), +1) {
				infSeen = true
			}

			samples = append(samples, &model.Sample{
				Metric:    model.Metric(lset),
				Value:     model.SampleValue(q.GetCumulativeCount()),
				Timestamp: timestamp,
			})
		}

		if m.Histogram.SampleSum != nil {
			lset := make(model.LabelSet, len(m.Label)+1)
			for _, p := range m.Label {
				lset[model.LabelName(p.GetName())] = model.LabelValue(p.GetValue())
			}
			lset[model.MetricNameLabel] = model.LabelValue(f.GetName() + "_sum")

			samples = append(samples, &model.Sample{
				Metric:    model.Metric(lset),
				Value:     model.SampleValue(m.Histogram.GetSampleSum()),
				Timestamp: timestamp,
			})
		}

		if m.Histogram.SampleCount != nil {
			lset := make(model.LabelSet, len(m.Label)+1)
			for _, p := range m.Label {
				lset[model.LabelName(p.GetName())] = model.LabelValue(p.GetValue())
			}
			lset[model.MetricNameLabel] = model.LabelValue(f.GetName() + "_count")

			count := &model.Sample{
				Metric:    model.Metric(lset),
				Value:     model.SampleValue(m.Histogram.GetSampleCount()),
				Timestamp: timestamp,
			}
			samples = append(samples, count)

			if !infSeen {
				// Append a infinity bucket sample.
				lset := make(model.LabelSet, len(m.Label)+2)
				for _, p := range m.Label {
					lset[model.LabelName(p.GetName())] = model.LabelValue(p.GetValue())
				}
				lset[model.LabelName(model.BucketLabel)] = model.LabelValue("+Inf")
				lset[model.MetricNameLabel] = model.LabelValue(f.GetName() + "_bucket")

				samples = append(samples, &model.Sample{
					Metric:    model.Metric(lset),
					Value:     count.Value,
					Timestamp: timestamp,
				})
			}
		}
	}

	return samples
}
<commit_msg>Fix typo in doc comment.<commit_after>// Copyright 2015 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package expfmt

import (
	"fmt"
	"io"
	"math"
	"mime"
	"net/http"

	dto "github.com/prometheus/client_model/go"

	"github.com/matttproud/golang_protobuf_extensions/pbutil"
	"github.com/prometheus/common/model"
)

// Decoder types decode an input stream into metric families.
type Decoder interface {
	Decode(*dto.MetricFamily) error
}

type DecodeOptions struct {
	// Timestamp is added to each value from the stream that has no explicit timestamp set.
	Timestamp model.Time
}

// NewDecoder returns a new decoder based on the HTTP header.
func NewDecoder(r io.Reader, h http.Header) (Decoder, error) {
	ct := h.Get(hdrContentType)

	mediatype, params, err := mime.ParseMediaType(ct)
	if err != nil {
		return nil, fmt.Errorf("invalid Content-Type header %q: %s", ct, err)
	}

	const (
		protoType = ProtoType + "/" + ProtoSubType
		textType  = "text/plain"
	)

	switch mediatype {
	case protoType:
		if p := params["proto"]; p != ProtoProtocol {
			return nil, fmt.Errorf("unrecognized protocol message %s", p)
		}
		if e := params["encoding"]; e != "delimited" {
			return nil, fmt.Errorf("unsupported encoding %s", e)
		}
		return &protoDecoder{r: r}, nil

	case textType:
		if v, ok := params["version"]; ok && v != "0.0.4" {
			return nil, fmt.Errorf("unrecognized protocol version %s", v)
		}
		return &textDecoder{r: r}, nil

	default:
		return nil, fmt.Errorf("unsupported media type %q, expected %q or %q", mediatype, protoType, textType)
	}
}

// protoDecoder implements the Decoder interface for protocol buffers.
type protoDecoder struct {
	r io.Reader
}

// Decode implements the Decoder interface.
func (d *protoDecoder) Decode(v *dto.MetricFamily) error {
	_, err := pbutil.ReadDelimited(d.r, v)
	return err
}

// textDecoder implements the Decoder interface for the text protcol.
type textDecoder struct {
	r    io.Reader
	p    TextParser
	fams []*dto.MetricFamily
}

// Decode implements the Decoder interface.
func (d *textDecoder) Decode(v *dto.MetricFamily) error {
	// TODO(fabxc): Wrap this as a line reader to make streaming safer.
	if len(d.fams) == 0 {
		// No cached metric families, read everything and parse metrics.
		fams, err := d.p.TextToMetricFamilies(d.r)
		if err != nil {
			return err
		}
		if len(fams) == 0 {
			return io.EOF
		}
		for _, f := range fams {
			d.fams = append(d.fams, f)
		}
	}
	*v = *d.fams[len(d.fams)-1]
	d.fams = d.fams[:len(d.fams)-1]
	return nil
}

type SampleDecoder struct {
	Dec  Decoder
	Opts *DecodeOptions

	f dto.MetricFamily
}

func (sd *SampleDecoder) Decode(s *model.Vector) error {
	if err := sd.Dec.Decode(&sd.f); err != nil {
		return err
	}
	*s = extractSamples(&sd.f, sd.Opts)
	return nil
}

// Extract samples builds a slice of samples from the provided metric families.
func ExtractSamples(o *DecodeOptions, fams ...*dto.MetricFamily) model.Vector {
	var all model.Vector
	for _, f := range fams {
		all = append(all, extractSamples(f, o)...)
	}
	return all
}

func extractSamples(f *dto.MetricFamily, o *DecodeOptions) model.Vector {
	switch *f.Type {
	case dto.MetricType_COUNTER:
		return extractCounter(o, f)
	case dto.MetricType_GAUGE:
		return extractGauge(o, f)
	case dto.MetricType_SUMMARY:
		return extractSummary(o, f)
	case dto.MetricType_UNTYPED:
		return extractUntyped(o, f)
	case dto.MetricType_HISTOGRAM:
		return extractHistogram(o, f)
	}
	panic("expfmt.extractSamples: unknown metric family type")
}

func extractCounter(o *DecodeOptions, f *dto.MetricFamily) model.Vector {
	samples := make(model.Vector, 0, len(f.Metric))

	for _, m := range f.Metric {
		if m.Counter == nil {
			continue
		}

		lset := make(model.LabelSet, len(m.Label)+1)
		for _, p := range m.Label {
			lset[model.LabelName(p.GetName())] = model.LabelValue(p.GetValue())
		}
		lset[model.MetricNameLabel] = model.LabelValue(f.GetName())

		smpl := &model.Sample{
			Metric: model.Metric(lset),
			Value:  model.SampleValue(m.Counter.GetValue()),
		}

		if m.TimestampMs != nil {
			smpl.Timestamp = model.TimeFromUnixNano(*m.TimestampMs * 1000000)
		} else {
			smpl.Timestamp = o.Timestamp
		}

		samples = append(samples, smpl)
	}

	return samples
}

func extractGauge(o *DecodeOptions, f *dto.MetricFamily) model.Vector {
	samples := make(model.Vector, 0, len(f.Metric))

	for _, m := range f.Metric {
		if m.Gauge == nil {
			continue
		}

		lset := make(model.LabelSet, len(m.Label)+1)
		for _, p := range m.Label {
			lset[model.LabelName(p.GetName())] = model.LabelValue(p.GetValue())
		}
		lset[model.MetricNameLabel] = model.LabelValue(f.GetName())

		smpl := &model.Sample{
			Metric: model.Metric(lset),
			Value:  model.SampleValue(m.Gauge.GetValue()),
		}

		if m.TimestampMs != nil {
			smpl.Timestamp = model.TimeFromUnixNano(*m.TimestampMs * 1000000)
		} else {
			smpl.Timestamp = o.Timestamp
		}

		samples = append(samples, smpl)
	}

	return samples
}

func extractUntyped(o *DecodeOptions, f *dto.MetricFamily) model.Vector {
	samples := make(model.Vector, 0, len(f.Metric))

	for _, m := range f.Metric {
		if m.Untyped == nil {
			continue
		}

		lset := make(model.LabelSet, len(m.Label)+1)
		for _, p := range m.Label {
			lset[model.LabelName(p.GetName())] = model.LabelValue(p.GetValue())
		}
		lset[model.MetricNameLabel] = model.LabelValue(f.GetName())

		smpl := &model.Sample{
			Metric: model.Metric(lset),
			Value:  model.SampleValue(m.Untyped.GetValue()),
		}

		if m.TimestampMs != nil {
			smpl.Timestamp = model.TimeFromUnixNano(*m.TimestampMs * 1000000)
		} else {
			smpl.Timestamp = o.Timestamp
		}

		samples = append(samples, smpl)
	}

	return samples
}

func extractSummary(o *DecodeOptions, f *dto.MetricFamily) model.Vector {
	samples := make(model.Vector, 0, len(f.Metric))

	for _, m := range f.Metric {
		if m.Summary == nil {
			continue
		}

		timestamp := o.Timestamp
		if m.TimestampMs != nil {
			timestamp = model.TimeFromUnixNano(*m.TimestampMs * 1000000)
		}

		for _, q := range m.Summary.Quantile {
			lset := make(model.LabelSet, len(m.Label)+2)
			for _, p := range m.Label {
				lset[model.LabelName(p.GetName())] = model.LabelValue(p.GetValue())
			}
			// BUG(matt): Update other names to "quantile".
			lset[model.LabelName(model.QuantileLabel)] = model.LabelValue(fmt.Sprint(q.GetQuantile()))
			lset[model.MetricNameLabel] = model.LabelValue(f.GetName())

			samples = append(samples, &model.Sample{
				Metric:    model.Metric(lset),
				Value:     model.SampleValue(q.GetValue()),
				Timestamp: timestamp,
			})
		}

		if m.Summary.SampleSum != nil {
			lset := make(model.LabelSet, len(m.Label)+1)
			for _, p := range m.Label {
				lset[model.LabelName(p.GetName())] = model.LabelValue(p.GetValue())
			}
			lset[model.MetricNameLabel] = model.LabelValue(f.GetName() + "_sum")

			samples = append(samples, &model.Sample{
				Metric:    model.Metric(lset),
				Value:     model.SampleValue(m.Summary.GetSampleSum()),
				Timestamp: timestamp,
			})
		}

		if m.Summary.SampleCount != nil {
			lset := make(model.LabelSet, len(m.Label)+1)
			for _, p := range m.Label {
				lset[model.LabelName(p.GetName())] = model.LabelValue(p.GetValue())
			}
			lset[model.MetricNameLabel] = model.LabelValue(f.GetName() + "_count")

			samples = append(samples, &model.Sample{
				Metric:    model.Metric(lset),
				Value:     model.SampleValue(m.Summary.GetSampleCount()),
				Timestamp: timestamp,
			})
		}
	}

	return samples
}

func extractHistogram(o *DecodeOptions, f *dto.MetricFamily) model.Vector {
	samples := make(model.Vector, 0, len(f.Metric))

	for _, m := range f.Metric {
		if m.Histogram == nil {
			continue
		}

		timestamp := o.Timestamp
		if m.TimestampMs != nil {
			timestamp = model.TimeFromUnixNano(*m.TimestampMs * 1000000)
		}

		infSeen := false

		for _, q := range m.Histogram.Bucket {
			lset := make(model.LabelSet, len(m.Label)+2)
			for _, p := range m.Label {
				lset[model.LabelName(p.GetName())] = model.LabelValue(p.GetValue())
			}
			lset[model.LabelName(model.BucketLabel)] = model.LabelValue(fmt.Sprint(q.GetUpperBound()))
			lset[model.MetricNameLabel] = model.LabelValue(f.GetName() + "_bucket")

			if math.IsInf(q.GetUpperBound(), +1) {
				infSeen = true
			}

			samples = append(samples, &model.Sample{
				Metric:    model.Metric(lset),
				Value:     model.SampleValue(q.GetCumulativeCount()),
				Timestamp: timestamp,
			})
		}

		if m.Histogram.SampleSum != nil {
			lset := make(model.LabelSet, len(m.Label)+1)
			for _, p := range m.Label {
				lset[model.LabelName(p.GetName())] = model.LabelValue(p.GetValue())
			}
			lset[model.MetricNameLabel] = model.LabelValue(f.GetName() + "_sum")

			samples = append(samples, &model.Sample{
				Metric:    model.Metric(lset),
				Value:     model.SampleValue(m.Histogram.GetSampleSum()),
				Timestamp: timestamp,
			})
		}

		if m.Histogram.SampleCount != nil {
			lset := make(model.LabelSet, len(m.Label)+1)
			for _, p := range m.Label {
				lset[model.LabelName(p.GetName())] = model.LabelValue(p.GetValue())
			}
			lset[model.MetricNameLabel] = model.LabelValue(f.GetName() + "_count")

			count := &model.Sample{
				Metric:    model.Metric(lset),
				Value:     model.SampleValue(m.Histogram.GetSampleCount()),
				Timestamp: timestamp,
			}
			samples = append(samples, count)

			if !infSeen {
				// Append a infinity bucket sample.
				lset := make(model.LabelSet, len(m.Label)+2)
				for _, p := range m.Label {
					lset[model.LabelName(p.GetName())] = model.LabelValue(p.GetValue())
				}
				lset[model.LabelName(model.BucketLabel)] = model.LabelValue("+Inf")
				lset[model.MetricNameLabel] = model.LabelValue(f.GetName() + "_bucket")

				samples = append(samples, &model.Sample{
					Metric:    model.Metric(lset),
					Value:     count.Value,
					Timestamp: timestamp,
				})
			}
		}
	}

	return samples
}
<|endoftext|>


<commit_before>// Copyright 2018-2019 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package k8sTest

import (
	"context"
	"fmt"

	. "github.com/cilium/cilium/test/ginkgo-ext"
	"github.com/cilium/cilium/test/helpers"

	. "github.com/onsi/gomega"
)

var _ = Describe("K8sHealthTest", func() {

	var (
		kubectl        *helpers.Kubectl
		ciliumFilename string
	)

	BeforeAll(func() {
		kubectl = helpers.CreateKubectl(helpers.K8s1VMName(), logger)

		ciliumFilename = helpers.TimestampFilename("cilium.yaml")
		DeployCiliumAndDNS(kubectl, ciliumFilename)
	})

	AfterFailed(func() {
		kubectl.CiliumReport("cilium endpoint list")
	})

	JustAfterEach(func() {
		kubectl.ValidateNoErrorsInLogs(CurrentGinkgoTestDescription().Duration)
	})

	AfterEach(func() {
		ExpectAllPodsTerminated(kubectl)
	})

	AfterAll(func() {
		UninstallCiliumFromManifest(kubectl, ciliumFilename)
		kubectl.CloseSSHClient()
	})

	getCilium := func(node string) (pod, ip string) {
		pod, err := kubectl.GetCiliumPodOnNodeWithLabel(node)
		Expect(err).Should(BeNil())

		res, err := kubectl.Get(
			helpers.CiliumNamespace,
			fmt.Sprintf("pod %s", pod)).Filter("{.status.podIP}")
		Expect(err).Should(BeNil())
		ip = res.String()

		return pod, ip
	}

	checkIP := func(pod, ip string) {
		jsonpath := "{.nodes[*].host.primary-address.ip}"
		ciliumCmd := fmt.Sprintf("cilium-health status -o jsonpath='%s'", jsonpath)

		err := kubectl.CiliumExecUntilMatch(pod, ciliumCmd, ip)
		ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Never saw cilium-health ip %s in pod %s", ip, pod)
	}

	It("checks cilium-health status between nodes", func() {
		SkipIfIntegration(helpers.CIIntegrationFlannel)
		SkipIfIntegration(helpers.CIIntegrationEKS)

		cilium1, cilium1IP := getCilium(helpers.K8s1)
		cilium2, cilium2IP := getCilium(helpers.K8s2)

		By("checking that cilium API exposes health instances")
		checkIP(cilium1, cilium1IP)
		checkIP(cilium1, cilium2IP)
		checkIP(cilium2, cilium1IP)
		checkIP(cilium2, cilium2IP)

		By("checking that `cilium-health --probe` succeeds")
		healthCmd := "cilium-health status --probe -o json"
		status := kubectl.CiliumExecMustSucceed(context.TODO(), cilium1, healthCmd)
		Expect(status.Stdout()).ShouldNot(ContainSubstring("error"))

		apiPaths := []string{
			"endpoint.icmp",
			"endpoint.http",
			"host.primary-address.icmp",
			"host.primary-address.http",
		}
		for node := 0; node <= 1; node++ {
			healthCmd := "cilium-health status -o json"
			status := kubectl.CiliumExecMustSucceed(context.TODO(), cilium1, healthCmd, "Cannot retrieve health status")
			for _, path := range apiPaths {
				filter := fmt.Sprintf("{.nodes[%d].%s}", node, path)
				By("checking API response for %q", filter)
				data, err := status.Filter(filter)
				Expect(err).To(BeNil(), "cannot retrieve filter %q from health output", filter)
				Expect(data.String()).Should(Not((BeEmpty())))
				statusFilter := fmt.Sprintf("{.nodes[%d].%s.status}", node, path)
				By("checking API status response for %q", statusFilter)
				data, err = status.Filter(statusFilter)
				Expect(err).To(BeNil(), "cannot retrieve filter %q from health output", statusFilter)
				Expect(data.String()).Should(BeEmpty())
			}
		}
	}, 30)
})
<commit_msg>Revert "CI: On EKS, skip cilium-health test"<commit_after>// Copyright 2018-2019 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package k8sTest

import (
	"context"
	"fmt"

	. "github.com/cilium/cilium/test/ginkgo-ext"
	"github.com/cilium/cilium/test/helpers"

	. "github.com/onsi/gomega"
)

var _ = Describe("K8sHealthTest", func() {

	var (
		kubectl        *helpers.Kubectl
		ciliumFilename string
	)

	BeforeAll(func() {
		kubectl = helpers.CreateKubectl(helpers.K8s1VMName(), logger)

		ciliumFilename = helpers.TimestampFilename("cilium.yaml")
		DeployCiliumAndDNS(kubectl, ciliumFilename)
	})

	AfterFailed(func() {
		kubectl.CiliumReport("cilium endpoint list")
	})

	JustAfterEach(func() {
		kubectl.ValidateNoErrorsInLogs(CurrentGinkgoTestDescription().Duration)
	})

	AfterEach(func() {
		ExpectAllPodsTerminated(kubectl)
	})

	AfterAll(func() {
		UninstallCiliumFromManifest(kubectl, ciliumFilename)
		kubectl.CloseSSHClient()
	})

	getCilium := func(node string) (pod, ip string) {
		pod, err := kubectl.GetCiliumPodOnNodeWithLabel(node)
		Expect(err).Should(BeNil())

		res, err := kubectl.Get(
			helpers.CiliumNamespace,
			fmt.Sprintf("pod %s", pod)).Filter("{.status.podIP}")
		Expect(err).Should(BeNil())
		ip = res.String()

		return pod, ip
	}

	checkIP := func(pod, ip string) {
		jsonpath := "{.nodes[*].host.primary-address.ip}"
		ciliumCmd := fmt.Sprintf("cilium-health status -o jsonpath='%s'", jsonpath)

		err := kubectl.CiliumExecUntilMatch(pod, ciliumCmd, ip)
		ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Never saw cilium-health ip %s in pod %s", ip, pod)
	}

	It("checks cilium-health status between nodes", func() {
		SkipIfIntegration(helpers.CIIntegrationFlannel)

		cilium1, cilium1IP := getCilium(helpers.K8s1)
		cilium2, cilium2IP := getCilium(helpers.K8s2)

		By("checking that cilium API exposes health instances")
		checkIP(cilium1, cilium1IP)
		checkIP(cilium1, cilium2IP)
		checkIP(cilium2, cilium1IP)
		checkIP(cilium2, cilium2IP)

		By("checking that `cilium-health --probe` succeeds")
		healthCmd := "cilium-health status --probe -o json"
		status := kubectl.CiliumExecMustSucceed(context.TODO(), cilium1, healthCmd)
		Expect(status.Stdout()).ShouldNot(ContainSubstring("error"))

		apiPaths := []string{
			"endpoint.icmp",
			"endpoint.http",
			"host.primary-address.icmp",
			"host.primary-address.http",
		}
		for node := 0; node <= 1; node++ {
			healthCmd := "cilium-health status -o json"
			status := kubectl.CiliumExecMustSucceed(context.TODO(), cilium1, healthCmd, "Cannot retrieve health status")
			for _, path := range apiPaths {
				filter := fmt.Sprintf("{.nodes[%d].%s}", node, path)
				By("checking API response for %q", filter)
				data, err := status.Filter(filter)
				Expect(err).To(BeNil(), "cannot retrieve filter %q from health output", filter)
				Expect(data.String()).Should(Not((BeEmpty())))
				statusFilter := fmt.Sprintf("{.nodes[%d].%s.status}", node, path)
				By("checking API status response for %q", statusFilter)
				data, err = status.Filter(statusFilter)
				Expect(err).To(BeNil(), "cannot retrieve filter %q from health output", statusFilter)
				Expect(data.String()).Should(BeEmpty())
			}
		}
	}, 30)
})
<|endoftext|>


<commit_before>/*
Copyright 2018 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// kube_checks contains functions which poll Kubernetes objects until
// they get into the state desired by the caller or time out.

package test

import (
	"context"
	"fmt"
	"time"

	"go.opencensus.io/trace"
	corev1 "k8s.io/api/core/v1"
	apiv1beta1 "k8s.io/api/extensions/v1beta1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/wait"
)

const (
	interval   = 1 * time.Second
	podTimeout = 8 * time.Minute
)

// WaitForDeploymentState polls the status of the Deployment called name
// from client every interval until inState returns `true` indicating it
// is done, returns an error or timeout. desc will be used to name the metric
// that is emitted to track how long it took for name to get into the state checked by inState.
func WaitForDeploymentState(client *KubeClient, name string, inState func(d *apiv1beta1.Deployment) (bool, error), desc string, namespace string, timeout time.Duration) error {
	d := client.Kube.ExtensionsV1beta1().Deployments(namespace)
	metricName := fmt.Sprintf("WaitForDeploymentState/%s/%s", name, desc)
	_, span := trace.StartSpan(context.Background(), metricName)
	defer span.End()

	return wait.PollImmediate(interval, timeout, func() (bool, error) {
		d, err := d.Get(name, metav1.GetOptions{})
		if err != nil {
			return true, err
		}
		return inState(d)
	})
}

// WaitForPodListState polls the status of the PodList
// from client every interval until inState returns `true` indicating it
// is done, returns an error or timeout. desc will be used to name the metric
// that is emitted to track how long it took to get into the state checked by inState.
func WaitForPodListState(client *KubeClient, inState func(p *corev1.PodList) (bool, error), desc string, namespace string) error {
	p := client.Kube.CoreV1().Pods(namespace)
	metricName := fmt.Sprintf("WaitForPodListState/%s", desc)
	_, span := trace.StartSpan(context.Background(), metricName)
	defer span.End()

	return wait.PollImmediate(interval, podTimeout, func() (bool, error) {
		p, err := p.List(metav1.ListOptions{})
		if err != nil {
			return true, err
		}
		return inState(p)
	})
}
<commit_msg>Move crd checks from serving in to test (#236)<commit_after>/*
Copyright 2018 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// kube_checks contains functions which poll Kubernetes objects until
// they get into the state desired by the caller or time out.

package test

import (
	"context"
	"fmt"
	"time"

	"go.opencensus.io/trace"
	corev1 "k8s.io/api/core/v1"
	apiv1beta1 "k8s.io/api/extensions/v1beta1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/wait"
	k8styped "k8s.io/client-go/kubernetes/typed/core/v1"
)

const (
	interval   = 1 * time.Second
	podTimeout = 8 * time.Minute
)

// WaitForDeploymentState polls the status of the Deployment called name
// from client every interval until inState returns `true` indicating it
// is done, returns an error or timeout. desc will be used to name the metric
// that is emitted to track how long it took for name to get into the state checked by inState.
func WaitForDeploymentState(client *KubeClient, name string, inState func(d *apiv1beta1.Deployment) (bool, error), desc string, namespace string, timeout time.Duration) error {
	d := client.Kube.ExtensionsV1beta1().Deployments(namespace)
	metricName := fmt.Sprintf("WaitForDeploymentState/%s/%s", name, desc)
	_, span := trace.StartSpan(context.Background(), metricName)
	defer span.End()

	return wait.PollImmediate(interval, timeout, func() (bool, error) {
		d, err := d.Get(name, metav1.GetOptions{})
		if err != nil {
			return true, err
		}
		return inState(d)
	})
}

// WaitForPodListState polls the status of the PodList
// from client every interval until inState returns `true` indicating it
// is done, returns an error or timeout. desc will be used to name the metric
// that is emitted to track how long it took to get into the state checked by inState.
func WaitForPodListState(client *KubeClient, inState func(p *corev1.PodList) (bool, error), desc string, namespace string) error {
	p := client.Kube.CoreV1().Pods(namespace)
	metricName := fmt.Sprintf("WaitForPodListState/%s", desc)
	_, span := trace.StartSpan(context.Background(), metricName)
	defer span.End()

	return wait.PollImmediate(interval, podTimeout, func() (bool, error) {
		p, err := p.List(metav1.ListOptions{})
		if err != nil {
			return true, err
		}
		return inState(p)
	})
}

// GetConfigMap gets the knative serving config map.
func GetConfigMap(client *KubeClient) k8styped.ConfigMapInterface {
	return client.Kube.CoreV1().ConfigMaps("knative-serving")
}

// Returns a func that evaluates if a deployment has scaled to 0 pods
func DeploymentScaledToZeroFunc() func(d *apiv1beta1.Deployment) (bool, error) {
	return func(d *apiv1beta1.Deployment) (bool, error) {
		return d.Status.ReadyReplicas == 0, nil
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	//"log"
	"regexp"
	"strings"
)

func main() {
	t := `package tpl

import (
	"fmt"
	"bytes"
)

func Index() string {
	_buffer := new(bytes.Buffer)

	Text

	for i:=0; i<10; i++ {
		{{string(i)}}
	}

	var t string = "2"
	fmt.Println(t)
	{{t}}

	print:=func() {
		text
	}
	print()

	test1(_buffer)
	test2(_buffer, t)

	return _buffer.String()
}

func test1(_buffer bytes.Buffer) {
	test1
}

func test2(_buffer bytes.Buffer, t string) {
	test{{t}}
}`
	fmt.Println(generate(t))
	/*
		package tpl

		import (
			"fmt"
			"bytes"
		)

		func Index() string {
			_buffer := new(bytes.Buffer)

			_buffer.WriteString("Text")

			for i:=0; i<10; i++ {
				_buffer.WriteString("")
				_buffer.WriteString(string(i))
				_buffer.WriteString("")
			}

			var t string = "2"
			fmt.Println(t)
			_buffer.WriteString(t)

			print:=func() {
				_buffer.WriteString("text")
			}
			print()

			test1(_buffer)
			test2(_buffer,t)

			return _buffer.String()
		}

		func test1(_buffer bytes.Buffer) {
			_buffer.WriteString("test1")
		}

		func test2(_buffer bytes.Buffer, t string) {
			_buffer.WriteString("test")
			_buffer.WriteString(t)
			_buffer.WriteString("")
		}
	*/
}

func generate(in string) (string, error) {
	var _buffer bytes.Buffer

	isHTML := regexp.MustCompile(`^\s*<.*>\s*$`)
	space := regexp.MustCompile(`^\s*$`)
	symbolLeft := "{{"
	symbolRight := "}}"
	symbolLeftLen := len(symbolLeft)
	symbolRightLen := len(symbolRight)
	// 分隔符
	symbol := regexp.MustCompile(symbolLeft + ".*" + symbolRight)

	a := regexp.MustCompile(`^\s*`)
	z := regexp.MustCompile(`\s*$`)

	var htmlBUF string

	r := new(readLine)
	for {
		buf, ok := r.read(in)

		switch {
		case isHTML.MatchString(buf):
			// 去首尾空
			buf = a.ReplaceAllLiteralString(buf, "")
			buf = z.ReplaceAllLiteralString(buf, "")
			// 转义双引号
			buf = strings.Replace(buf, `"`, `\"`, -1)
			// 替换分隔符
			if symbol.MatchString(buf) {
				// 找到本行全部分隔符
				symbolBUF := symbol.FindAllString(buf, -1)
				for _, v := range symbolBUF {
					// 去掉两边分隔符
					vBUF := v[symbolLeftLen : len(v)-symbolRightLen]
					// 组合插入
					vBUF = "\")\n_buffer.WriteString(" + vBUF + ")\n_buffer.WriteString(\""
					// 替换
					buf = strings.Replace(buf, v, vBUF, -1)
					fmt.Println(v)
				}
			}

			// 将本行添加到缓存中，输出为一行
			// 就是将两行
			// _buffer.WriteString('xxx')
			// _buffer.WriteString('xxx')
			// 简化为一行
			// _buffer.WriteString('xxxxxx')
			htmlBUF += buf
		case space.MatchString(buf):
			// 空行，跳过
		default:
			// 检查是否有html需要输出
			if htmlBUF != "" {
				// 输出html
				_buffer.WriteString("_buffer.WriteString(\"" + htmlBUF + "\")\n")
				// 清空缓存
				htmlBUF = ""
			}
			_buffer.WriteString(buf + "\n")
		}

		if ok {
			break
		}
	}
	return _buffer.String(), nil
}

type readLine struct {
	start int
	end   int
}

// 按行输出字符串
func (r *readLine) read(str string) (string, bool) {
	// 找到换行符位置
	n := strings.Index(str[r.start:], "\n")
	if n == -1 {
		// 无法找到换行符，已经到达最后一行
		// 返回ok
		line := string(str[r.start:])
		r.start = 0
		r.end = 0
		return line, true
	}
	r.end = r.start + n
	line := string(str[r.start:r.end])
	r.start = r.end + 1 // 跳过换行符,下次循环时从换行符后开始

	return line, false
}
<commit_msg>添加自动格式化<commit_after>package main

import (
	"bytes"
	"fmt"
	"go/format"
	"log"
	"regexp"
	"strings"
)

func main() {
	t := `package tpl

import (
	"fmt"
	"bytes"
)

func Index() string {
	_buffer := new(bytes.Buffer)

	Text

	for i:=0; i<10; i++ {
		{{string(i)}}
	}

	var t string = "2"
	fmt.Println(t)

	print:=func() {
		text
	}
	print()

	test1(_buffer)
	test2(_buffer, t)

	return _buffer.String()
}

func test1(_buffer bytes.Buffer) {
	test1
}

func test2(_buffer bytes.Buffer, t string) {
	test{{t}}
}`
	buf, err := format.Source([]byte(generate(t)))
	if err != nil {
		log.Println("Format error:", err)
		return
	}
	fmt.Println(string(buf))
	/*
		package tpl

		import (
			"fmt"
			"bytes"
		)

		func Index() string {
			_buffer := new(bytes.Buffer)

			_buffer.WriteString("Text")

			for i:=0; i<10; i++ {
				_buffer.WriteString("")
				_buffer.WriteString(string(i))
				_buffer.WriteString("")
			}

			var t string = "2"
			fmt.Println(t)

			print:=func() {
				_buffer.WriteString("text")
			}
			print()

			test1(_buffer)
			test2(_buffer,t)

			return _buffer.String()
		}

		func test1(_buffer bytes.Buffer) {
			_buffer.WriteString("test1")
		}

		func test2(_buffer bytes.Buffer, t string) {
			_buffer.WriteString("test")
			_buffer.WriteString(t)
			_buffer.WriteString("")
		}
	*/
}

func generate(in string) string {
	var _buffer bytes.Buffer

	isHTML := regexp.MustCompile(`^\s*<.*>\s*$`)
	symbolLeft := "{{"
	symbolRight := "}}"
	symbolLeftLen := len(symbolLeft)
	symbolRightLen := len(symbolRight)
	// 分隔符
	symbol := regexp.MustCompile(symbolLeft + ".*" + symbolRight)

	a := regexp.MustCompile(`^\s*`)
	z := regexp.MustCompile(`\s*$`)

	var htmlBUF string

	r := new(readLine)
	for {
		// 按行读取处理
		buf, ok := r.read(in)

		// 检查本行是否为HTML
		if isHTML.MatchString(buf) {
			// 去首尾空
			buf = a.ReplaceAllLiteralString(buf, "")
			buf = z.ReplaceAllLiteralString(buf, "")
			// 转义双引号
			buf = strings.Replace(buf, `"`, `\"`, -1)
			// 检查是否有分隔符需要替换
			if symbol.MatchString(buf) {
				// 替换分隔符
				// 找到本行全部分隔符
				symbolBUF := symbol.FindAllString(buf, -1)
				for _, v := range symbolBUF {
					// 去掉两边分隔符
					vBUF := v[symbolLeftLen : len(v)-symbolRightLen]
					// 组合插入
					vBUF = "\")\n_buffer.WriteString(" + vBUF + ")\n_buffer.WriteString(\""
					// 替换
					buf = strings.Replace(buf, v, vBUF, -1)
				}
			}
			// 将本行添加到缓存中，输出为一行
			// 就是将两行
			// _buffer.WriteString('xxx')
			// _buffer.WriteString('xxx')
			// 简化为一行
			// _buffer.WriteString('xxxxxx')
			htmlBUF += buf
		} else {
			// 本行为Golang
			// 检查是否有html需要输出
			if htmlBUF != "" {
				// 输出html
				_buffer.WriteString("_buffer.WriteString(\"" + htmlBUF + "\")\n")
				// 清空缓存
				htmlBUF = ""
			}
			_buffer.WriteString(buf + "\n")
		}
		// 检查是否已经读取完毕
		if ok {
			break
		}
	}
	return _buffer.String()
}

type readLine struct {
	// 用于记录每行的起始位置
	start int
	end   int
}

// 按行输出字符串
func (r *readLine) read(str string) (string, bool) {
	// 找到换行符位置
	n := strings.Index(str[r.start:], "\n")
	if n == -1 {
		// 无法找到换行符，已经到达最后一行
		// 返回ok
		line := string(str[r.start:])
		r.start = 0
		r.end = 0
		return line, true
	}
	// 行的结束位置
	r.end = r.start + n
	// 提取一行
	line := string(str[r.start:r.end])
	// 下一行的开始位置
	r.start = r.end + 1

	return line, false
}
<|endoftext|>


<commit_before><commit_msg>stackdriver: add tests from regex matching<commit_after><|endoftext|>


<commit_before>package docker

import (
	"fmt"
	"strings"

	"citadelapp.io/citadel"
	"github.com/samalba/dockerclient"
)

type Service struct {
	data      *citadel.ServiceData
	namespace string
	docker    *dockerclient.DockerClient
}

// New returns a new Service interacting with a docker daemon.  The namespace
// is the image namespace required to create new containers
func New(namespace, dockerUrl string, data *citadel.ServiceData) (citadel.Service, error) {
	client, err := dockerclient.NewDockerClient(dockerUrl)
	if err != nil {
		return nil, err
	}

	return &Service{
		data:      data,
		docker:    client,
		namespace: namespace,
	}, nil
}

func (d *Service) Data() *citadel.ServiceData {
	return d.data
}

func (d *Service) List(t *citadel.Task) ([]*citadel.ServiceData, error) {
	containers, err := d.docker.ListContainers(false)
	if err != nil {
		return nil, err
	}
	out := []*citadel.ServiceData{}

	for _, c := range containers {
		service, err := d.containerToService(c.Id, c.Names[0])
		if err != nil {
			return nil, err
		} else if service != nil {
			out = append(out, service)
		}
	}

	return out, nil
}

func (d *Service) Run(t *citadel.Task) (*citadel.RunResult, error) {
	if t.Service.Type == "docker" {
		return nil, fmt.Errorf("cannot start docker service")
	}

	config := &dockerclient.ContainerConfig{
		Image:  fmt.Sprintf("%s/%s:latest", d.namespace, t.Service.Type),
		Memory: t.Service.Memory,
		Cpuset: cpusToString(t.Service.Cpus),
	}

	id, err := d.docker.CreateContainer(config)
	if err != nil {
		return nil, err
	}

	if err := d.docker.StartContainer(id); err != nil {
		return nil, err
	}

	// FIXME: return a report of the run with state information
	return nil, nil
}

func (d *Service) Stop(t *citadel.Task) (*citadel.StopResult, error) {
	if t.Service.Type == "docker" {
		return nil, fmt.Errorf("cannot stop docker service")
	}

	if err := d.docker.StopContainer(t.Service.Name, 5); err != nil {
		return nil, err
	}

	// FIXME:
	return nil, nil
}

func (d *Service) containerToService(id, name string) (*citadel.ServiceData, error) {
	c, err := d.docker.InspectContainer(id)
	if err != nil {
		return nil, err
	}

	return &citadel.ServiceData{
		Type:   c.Image,
		Name:   name,
		Cpus:   stringToCpus(c.Config.Cpuset),
		Memory: c.Config.Memory,
		Addr:   c.NetworkSettings.IpAddress,
	}, nil
}

func cpusToString(cpus int) string {
	out := []string{}
	for i := 0; i < cpus; i++ {
		out = append(out, fmt.Sprint(i))
	}
	return strings.Join(out, ",")
}

func stringToCpus(cpuset string) int {
	return len(strings.Split(cpuset, ","))
}
<commit_msg>Poperly clean image name<commit_after>package docker

import (
	"fmt"
	"path/filepath"
	"strings"

	"citadelapp.io/citadel"
	"github.com/samalba/dockerclient"
)

type Service struct {
	data      *citadel.ServiceData
	namespace string
	docker    *dockerclient.DockerClient
}

// New returns a new Service interacting with a docker daemon.  The namespace
// is the image namespace required to create new containers
func New(namespace, dockerUrl string, data *citadel.ServiceData) (citadel.Service, error) {
	client, err := dockerclient.NewDockerClient(dockerUrl)
	if err != nil {
		return nil, err
	}

	return &Service{
		data:      data,
		docker:    client,
		namespace: namespace,
	}, nil
}

func (d *Service) Data() *citadel.ServiceData {
	return d.data
}

func (d *Service) List(t *citadel.Task) ([]*citadel.ServiceData, error) {
	containers, err := d.docker.ListContainers(false)
	if err != nil {
		return nil, err
	}
	out := []*citadel.ServiceData{}

	for _, c := range containers {
		service, err := d.containerToService(c.Id, c.Names[0], c.Image)
		if err != nil {
			return nil, err
		} else if service != nil {
			out = append(out, service)
		}
	}

	return out, nil
}

func (d *Service) Run(t *citadel.Task) (*citadel.RunResult, error) {
	if t.Service.Type == "docker" {
		return nil, fmt.Errorf("cannot start docker service")
	}

	config := &dockerclient.ContainerConfig{
		Image:  fmt.Sprintf("%s/%s:latest", d.namespace, t.Service.Type),
		Memory: t.Service.Memory,
		Cpuset: cpusToString(t.Service.Cpus),
	}

	id, err := d.docker.CreateContainer(config)
	if err != nil {
		return nil, err
	}

	if err := d.docker.StartContainer(id); err != nil {
		return nil, err
	}

	// FIXME: return a report of the run with state information
	return nil, nil
}

func (d *Service) Stop(t *citadel.Task) (*citadel.StopResult, error) {
	if t.Service.Type == "docker" {
		return nil, fmt.Errorf("cannot stop docker service")
	}

	if err := d.docker.StopContainer(t.Service.Name, 5); err != nil {
		return nil, err
	}

	// FIXME:
	return nil, nil
}

func (d *Service) containerToService(id, name, image string) (*citadel.ServiceData, error) {
	c, err := d.docker.InspectContainer(id)
	if err != nil {
		return nil, err
	}

	return &citadel.ServiceData{
		Type:   cleanImageName(image),
		Name:   name,
		Cpus:   stringToCpus(c.Config.Cpuset),
		Memory: c.Config.Memory,
		Addr:   c.NetworkSettings.IpAddress,
	}, nil
}

func cpusToString(cpus int) string {
	out := []string{}
	for i := 0; i < cpus; i++ {
		out = append(out, fmt.Sprint(i))
	}
	return strings.Join(out, ",")
}

func stringToCpus(cpuset string) int {
	return len(strings.Split(cpuset, ","))
}

func cleanImageName(image string) string {
	_, image = filepath.Split(image)
	return strings.Split(image, ":")[0]
}
<|endoftext|>


<commit_before>// Package snapcraft implements the Pipe interface providing Snapcraft bindings.
package snapcraft

import (
	"errors"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/apex/log"
	"github.com/goreleaser/goreleaser/context"
	"golang.org/x/sync/errgroup"
	yaml "gopkg.in/yaml.v2"
)

// ErrNoSnapcraft is shown when snapcraft cannot be found in $PATH
var ErrNoSnapcraft = errors.New("snapcraft not present in $PATH")

// SnapcraftMetadata to generate the snap package
type SnapcraftMetadata struct {
	Name          string
	Version       string
	Summary       string
	Description   string
	Grade         string `yaml:",omitempty"`
	Confinement   string `yaml:",omitempty"`
	Architectures []string
	Apps          map[string]AppsMetadata
}

// AppsMetadata for the binaries that will be in the snap package
type AppsMetadata struct {
	Command string
	//	Plugs []string
	//	Daemon string
}

// Pipe for snapcraft packaging
type Pipe struct{}

// Description of the pipe
func (Pipe) Description() string {
	return "Creating Linux packages with snapcraft"
}

// Run the pipe
func (Pipe) Run(ctx *context.Context) error {
	if ctx.Config.Snapcraft.Summary == "" {
		log.Info("no snapcraft summary defined, skipping")
		return nil
	}
	if ctx.Config.Snapcraft.Summary == "" {
		log.Info("no snapcraft description defined, skipping")
		return nil
	}
	_, err := exec.LookPath("snapcraft")
	if err != nil {
		return ErrNoSnapcraft
	}

	var g errgroup.Group
	for platform, groups := range ctx.Binaries {
		if !strings.Contains(platform, "linux") {
			log.WithField("platform", platform).Debug("skipped non-linux builds for snapcraft")
			continue
		}
		arch := archFor(platform)
		for folder, binaries := range groups {
			g.Go(func() error {
				return create(ctx, folder, arch, binaries)
			})
		}
	}
	return g.Wait()
}

func archFor(key string) string {
	switch {
	case strings.Contains(key, "amd64"):
		return "amd64"
	case strings.Contains(key, "386"):
		return "i386"
	case strings.Contains(key, "arm64"):
		return "arm64"
	case strings.Contains(key, "arm6"):
		return "armhf"
	}
	return key
}

func create(ctx *context.Context, folder, arch string, binaries []context.Binary) error {
	// prime is the directory that then will be compressed to make the .snap package.
	folderDir := filepath.Join(ctx.Config.Dist, folder)
	primeDir := filepath.Join(folderDir, "prime")
	metaDir := filepath.Join(primeDir, "meta")
	if err := os.MkdirAll(metaDir, 0755); err != nil {
		return err
	}

	var file = filepath.Join(primeDir, "meta", "snap.yaml")
	log.WithField("file", file).Info("creating snap metadata")

	var metadata = &SnapcraftMetadata{
		Name:          ctx.Config.ProjectName,
		Version:       ctx.Version,
		Summary:       ctx.Config.Snapcraft.Summary,
		Description:   ctx.Config.Snapcraft.Description,
		Grade:         ctx.Config.Snapcraft.Grade,
		Confinement:   ctx.Config.Snapcraft.Confinement,
		Architectures: []string{arch},
		Apps:          make(map[string]AppsMetadata),
	}

	for _, binary := range binaries {
		log.WithField("path", binary.Path).
			WithField("name", binary.Name).
			Info("passed binary to snapcraft")
		metadata.Apps[binary.Name] = AppsMetadata{Command: binary.Name}

		destBinaryPath := filepath.Join(primeDir, filepath.Base(binary.Path))
		os.Link(binary.Path, destBinaryPath)
	}
	out, err := yaml.Marshal(metadata)
	if err != nil {
		return err
	}

	if err = ioutil.WriteFile(file, out, 0644); err != nil {
		return err
	}

	snap := metadata.Name + "_" + metadata.Version + "_" + arch + ".snap"
	cmd := exec.Command("snapcraft", "snap", "prime", "--output", snap)
	cmd.Dir = folderDir
	if out, err = cmd.CombinedOutput(); err != nil {
		return errors.New(string(out))
	}
	ctx.AddArtifact(filepath.Join(folderDir, snap))
	return nil
}
<commit_msg>Change the log to Error<commit_after>// Package snapcraft implements the Pipe interface providing Snapcraft bindings.
package snapcraft

import (
	"errors"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/apex/log"
	"github.com/goreleaser/goreleaser/context"
	"golang.org/x/sync/errgroup"
	yaml "gopkg.in/yaml.v2"
)

// ErrNoSnapcraft is shown when snapcraft cannot be found in $PATH
var ErrNoSnapcraft = errors.New("snapcraft not present in $PATH")

// SnapcraftMetadata to generate the snap package
type SnapcraftMetadata struct {
	Name          string
	Version       string
	Summary       string
	Description   string
	Grade         string `yaml:",omitempty"`
	Confinement   string `yaml:",omitempty"`
	Architectures []string
	Apps          map[string]AppsMetadata
}

// AppsMetadata for the binaries that will be in the snap package
type AppsMetadata struct {
	Command string
	//	Plugs []string
	//	Daemon string
}

// Pipe for snapcraft packaging
type Pipe struct{}

// Description of the pipe
func (Pipe) Description() string {
	return "Creating Linux packages with snapcraft"
}

// Run the pipe
func (Pipe) Run(ctx *context.Context) error {
	if ctx.Config.Snapcraft.Summary == "" {
		log.Error("no snapcraft summary defined, skipping")
		return nil
	}
	if ctx.Config.Snapcraft.Summary == "" {
		log.Error("no snapcraft description defined, skipping")
		return nil
	}
	_, err := exec.LookPath("snapcraft")
	if err != nil {
		return ErrNoSnapcraft
	}

	var g errgroup.Group
	for platform, groups := range ctx.Binaries {
		if !strings.Contains(platform, "linux") {
			log.WithField("platform", platform).Debug("skipped non-linux builds for snapcraft")
			continue
		}
		arch := archFor(platform)
		for folder, binaries := range groups {
			g.Go(func() error {
				return create(ctx, folder, arch, binaries)
			})
		}
	}
	return g.Wait()
}

func archFor(key string) string {
	switch {
	case strings.Contains(key, "amd64"):
		return "amd64"
	case strings.Contains(key, "386"):
		return "i386"
	case strings.Contains(key, "arm64"):
		return "arm64"
	case strings.Contains(key, "arm6"):
		return "armhf"
	}
	return key
}

func create(ctx *context.Context, folder, arch string, binaries []context.Binary) error {
	// prime is the directory that then will be compressed to make the .snap package.
	folderDir := filepath.Join(ctx.Config.Dist, folder)
	primeDir := filepath.Join(folderDir, "prime")
	metaDir := filepath.Join(primeDir, "meta")
	if err := os.MkdirAll(metaDir, 0755); err != nil {
		return err
	}

	var file = filepath.Join(primeDir, "meta", "snap.yaml")
	log.WithField("file", file).Info("creating snap metadata")

	var metadata = &SnapcraftMetadata{
		Name:          ctx.Config.ProjectName,
		Version:       ctx.Version,
		Summary:       ctx.Config.Snapcraft.Summary,
		Description:   ctx.Config.Snapcraft.Description,
		Grade:         ctx.Config.Snapcraft.Grade,
		Confinement:   ctx.Config.Snapcraft.Confinement,
		Architectures: []string{arch},
		Apps:          make(map[string]AppsMetadata),
	}

	for _, binary := range binaries {
		log.WithField("path", binary.Path).
			WithField("name", binary.Name).
			Info("passed binary to snapcraft")
		metadata.Apps[binary.Name] = AppsMetadata{Command: binary.Name}

		destBinaryPath := filepath.Join(primeDir, filepath.Base(binary.Path))
		os.Link(binary.Path, destBinaryPath)
	}
	out, err := yaml.Marshal(metadata)
	if err != nil {
		return err
	}

	if err = ioutil.WriteFile(file, out, 0644); err != nil {
		return err
	}

	snap := metadata.Name + "_" + metadata.Version + "_" + arch + ".snap"
	cmd := exec.Command("snapcraft", "snap", "prime", "--output", snap)
	cmd.Dir = folderDir
	if out, err = cmd.CombinedOutput(); err != nil {
		return errors.New(string(out))
	}
	ctx.AddArtifact(filepath.Join(folderDir, snap))
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package user

// Info describes a user that has been authenticated to the system.
type Info interface {
	// GetName returns the name that uniquely identifies this user among all
	// other active users.
	GetName() string
	// GetUID returns a unique value for a particular user that will change
	// if the user is removed from the system and another user is added with
	// the same name.
	GetUID() string
	// GetGroups returns the names of the groups the user is a member of
	GetGroups() []string

	// GetExtra can contain any additional information that the authenticator
	// thought was interesting.  One example would be scopes on a token.
	// Keys in this map should be namespaced to the authenticator or
	// authenticator/authorizer pair making use of them.
	// For instance: "example.org/foo" instead of "foo"
	// This is a map[string][]string because it needs to be serializeable into
	// a SubjectAccessReviewSpec.authorization.k8s.io for proper authorization
	// delegation flows
	// In order to faithfully round-trip through an impersonation flow, these keys
	// MUST be lowercase.
	GetExtra() map[string][]string
}

// DefaultInfo provides a simple user information exchange object
// for components that implement the UserInfo interface.
type DefaultInfo struct {
	Name   string
	UID    string
	Groups []string
	Extra  map[string][]string
}

func (i *DefaultInfo) GetName() string {
	return i.Name
}

func (i *DefaultInfo) GetUID() string {
	return i.UID
}

func (i *DefaultInfo) GetGroups() []string {
	return i.Groups
}

func (i *DefaultInfo) GetExtra() map[string][]string {
	return i.Extra
}

// well-known user and group names
const (
	SystemPrivilegedGroup = "system:masters"
	NodesGroup            = "system:nodes"
	AllUnauthenticated    = "system:unauthenticated"
	AllAuthenticated      = "system:authenticated"

	Anonymous     = "system:anonymous"
	APIServerUser = "system:apiserver"

	// core kubernetes process identities
	KubeProxy             = "system:kube-proxy"
	KubeControllerManager = "system:kube-controller-manager"
	KubeScheduler         = "system:kube-scheduler"
)
<commit_msg>add bootstrap policy for monitoring roles<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package user

// Info describes a user that has been authenticated to the system.
type Info interface {
	// GetName returns the name that uniquely identifies this user among all
	// other active users.
	GetName() string
	// GetUID returns a unique value for a particular user that will change
	// if the user is removed from the system and another user is added with
	// the same name.
	GetUID() string
	// GetGroups returns the names of the groups the user is a member of
	GetGroups() []string

	// GetExtra can contain any additional information that the authenticator
	// thought was interesting.  One example would be scopes on a token.
	// Keys in this map should be namespaced to the authenticator or
	// authenticator/authorizer pair making use of them.
	// For instance: "example.org/foo" instead of "foo"
	// This is a map[string][]string because it needs to be serializeable into
	// a SubjectAccessReviewSpec.authorization.k8s.io for proper authorization
	// delegation flows
	// In order to faithfully round-trip through an impersonation flow, these keys
	// MUST be lowercase.
	GetExtra() map[string][]string
}

// DefaultInfo provides a simple user information exchange object
// for components that implement the UserInfo interface.
type DefaultInfo struct {
	Name   string
	UID    string
	Groups []string
	Extra  map[string][]string
}

func (i *DefaultInfo) GetName() string {
	return i.Name
}

func (i *DefaultInfo) GetUID() string {
	return i.UID
}

func (i *DefaultInfo) GetGroups() []string {
	return i.Groups
}

func (i *DefaultInfo) GetExtra() map[string][]string {
	return i.Extra
}

// well-known user and group names
const (
	SystemPrivilegedGroup = "system:masters"
	NodesGroup            = "system:nodes"
	MonitoringGroup       = "system:monitoring"
	AllUnauthenticated    = "system:unauthenticated"
	AllAuthenticated      = "system:authenticated"

	Anonymous     = "system:anonymous"
	APIServerUser = "system:apiserver"

	// core kubernetes process identities
	KubeProxy             = "system:kube-proxy"
	KubeControllerManager = "system:kube-controller-manager"
	KubeScheduler         = "system:kube-scheduler"
)
<|endoftext|>


<commit_before>/*
Copyright 2018 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package discover to discover unused devices.
package discover

import (
	"encoding/json"
	"fmt"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/coreos/pkg/capnslog"
	"github.com/rook/rook/pkg/clusterd"
	"github.com/rook/rook/pkg/operator/k8sutil"
	"github.com/rook/rook/pkg/util/sys"

	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

var (
	logger          = capnslog.NewPackageLogger("github.com/rook/rook", "rook-discover")
	AppName         = "rook-discover"
	NodeAttr        = "rook.io/node"
	LocalDiskCMData = "devices"
	LocalDiskCMName = "local-device-%s"
	nodeName        string
	namespace       string
	lastDevice      string
	cmName          string
	cm              *v1.ConfigMap
)

func Run(context *clusterd.Context, probeInterval time.Duration) error {
	if context == nil {
		return fmt.Errorf("nil context")
	}
	logger.Infof("device discovery interval is %s", probeInterval.String())
	nodeName = os.Getenv(k8sutil.NodeNameEnvVar)
	namespace = os.Getenv(k8sutil.PodNamespaceEnvVar)
	cmName = k8sutil.TruncateNodeName(LocalDiskCMName, nodeName)
	sigc := make(chan os.Signal, 1)
	signal.Notify(sigc, syscall.SIGTERM)
	err := updateDeviceCM(context)
	if err != nil {
		logger.Infof("failed to update device configmap: %v", err)
		return err
	}
	for {
		select {
		case <-sigc:
			logger.Infof("shutdown signal received, exiting...")
			return nil
		case <-time.After(probeInterval):
			updateDeviceCM(context)
		}
	}
}

func updateDeviceCM(context *clusterd.Context) error {
	logger.Infof("updating device configmap")
	devices, err := probeDevices(context)
	if err != nil {
		logger.Infof("failed to probe devices: %v", err)
		return err
	}
	deviceJson, err := json.Marshal(devices)
	if err != nil {
		logger.Infof("failed to marshal: %v", err)
		return err
	}
	deviceStr := string(deviceJson)
	if cm == nil {
		cm, err = context.Clientset.CoreV1().ConfigMaps(namespace).Get(cmName, metav1.GetOptions{})
	}
	if err == nil {
		lastDevice = cm.Data[LocalDiskCMData]
		logger.Debugf("last devices %s", lastDevice)
	} else {
		if !errors.IsNotFound(err) {
			logger.Infof("failed to get configmap: %v", err)
			return err
		}

		data := make(map[string]string, 1)
		data[LocalDiskCMData] = deviceStr
		// the map doesn't exist yet, create it now
		cm = &v1.ConfigMap{
			ObjectMeta: metav1.ObjectMeta{
				Name:      cmName,
				Namespace: namespace,
				Labels: map[string]string{
					k8sutil.AppAttr: AppName,
					NodeAttr:        nodeName,
				},
			},
			Data: data,
		}
		cm, err = context.Clientset.CoreV1().ConfigMaps(namespace).Create(cm)
		if err != nil {
			logger.Infof("failed to create configmap: %v", err)
			return fmt.Errorf("failed to create local device map %s: %+v", cmName, err)
		}
		lastDevice = deviceStr
	}
	if deviceStr != lastDevice {
		data := make(map[string]string, 1)
		data[LocalDiskCMData] = deviceStr
		cm.Data = data
		cm, err = context.Clientset.CoreV1().ConfigMaps(namespace).Update(cm)
		if err != nil {
			logger.Infof("failed to update configmap %s: %v", cmName, err)
			return err
		}
	}
	return nil
}

func probeDevices(context *clusterd.Context) ([]sys.LocalDisk, error) {
	devices := make([]sys.LocalDisk, 0)
	localDevices, err := clusterd.DiscoverDevices(context.Executor)
	if err != nil {
		return devices, fmt.Errorf("failed initial hardware discovery. %+v", err)
	}
	for _, device := range localDevices {
		if device == nil {
			continue
		}
		if device.Type == sys.PartType {
			continue
		}

		partitions, _, err := sys.GetDevicePartitions(device.Name, context.Executor)
		if err != nil {
			logger.Infof("failed to check device partitions %s: %v", device.Name, err)
			continue
		}

		// check if there is a file system on the device
		fs, err := sys.GetDeviceFilesystems(device.Name, context.Executor)
		if err != nil {
			logger.Infof("failed to check device filesystem %s: %v", device.Name, err)
			continue
		}
		device.Partitions = partitions
		device.Filesystem = fs
		device.Empty = clusterd.GetDeviceEmpty(device)

		devices = append(devices, *device)
	}

	logger.Infof("available devices: %+v", devices)
	return devices, nil
}
<commit_msg>discover: smarter device list change calculation<commit_after>/*
Copyright 2018 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package discover to discover unused devices.
package discover

import (
	"encoding/json"
	"fmt"
	"os"
	"os/signal"
	"reflect"
	"syscall"
	"time"

	"github.com/coreos/pkg/capnslog"
	"github.com/rook/rook/pkg/clusterd"
	"github.com/rook/rook/pkg/operator/k8sutil"
	"github.com/rook/rook/pkg/util/sys"

	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

var (
	logger          = capnslog.NewPackageLogger("github.com/rook/rook", "rook-discover")
	AppName         = "rook-discover"
	NodeAttr        = "rook.io/node"
	LocalDiskCMData = "devices"
	LocalDiskCMName = "local-device-%s"
	nodeName        string
	namespace       string
	lastDevice      string
	cmName          string
	cm              *v1.ConfigMap
)

func Run(context *clusterd.Context, probeInterval time.Duration) error {
	if context == nil {
		return fmt.Errorf("nil context")
	}
	logger.Infof("device discovery interval is %s", probeInterval.String())
	nodeName = os.Getenv(k8sutil.NodeNameEnvVar)
	namespace = os.Getenv(k8sutil.PodNamespaceEnvVar)
	cmName = k8sutil.TruncateNodeName(LocalDiskCMName, nodeName)
	sigc := make(chan os.Signal, 1)
	signal.Notify(sigc, syscall.SIGTERM)
	err := updateDeviceCM(context)
	if err != nil {
		logger.Infof("failed to update device configmap: %v", err)
		return err
	}
	for {
		select {
		case <-sigc:
			logger.Infof("shutdown signal received, exiting...")
			return nil
		case <-time.After(probeInterval):
			updateDeviceCM(context)
		}
	}
}

func deviceListsEqual(a, b string) (bool, error) {
	var d0 []sys.LocalDisk
	var d1 []sys.LocalDisk

	err := json.Unmarshal([]byte(a), &d0)
	if err != nil {
		return false, fmt.Errorf("Cannot unmarshal devices: %v", err)
	}

	err = json.Unmarshal([]byte(b), &d1)
	if err != nil {
		return false, fmt.Errorf("Cannot unmarshal devices: %v", err)
	}

	for i := range d0 {
		d0[i].UUID = ""
	}

	for i := range d1 {
		d1[i].UUID = ""
	}

	return reflect.DeepEqual(d0, d1), nil
}

func updateDeviceCM(context *clusterd.Context) error {
	logger.Infof("updating device configmap")
	devices, err := probeDevices(context)
	if err != nil {
		logger.Infof("failed to probe devices: %v", err)
		return err
	}
	deviceJson, err := json.Marshal(devices)
	if err != nil {
		logger.Infof("failed to marshal: %v", err)
		return err
	}
	deviceStr := string(deviceJson)
	if cm == nil {
		cm, err = context.Clientset.CoreV1().ConfigMaps(namespace).Get(cmName, metav1.GetOptions{})
	}
	if err == nil {
		lastDevice = cm.Data[LocalDiskCMData]
		logger.Debugf("last devices %s", lastDevice)
	} else {
		if !errors.IsNotFound(err) {
			logger.Infof("failed to get configmap: %v", err)
			return err
		}

		data := make(map[string]string, 1)
		data[LocalDiskCMData] = deviceStr
		// the map doesn't exist yet, create it now
		cm = &v1.ConfigMap{
			ObjectMeta: metav1.ObjectMeta{
				Name:      cmName,
				Namespace: namespace,
				Labels: map[string]string{
					k8sutil.AppAttr: AppName,
					NodeAttr:        nodeName,
				},
			},
			Data: data,
		}
		cm, err = context.Clientset.CoreV1().ConfigMaps(namespace).Create(cm)
		if err != nil {
			logger.Infof("failed to create configmap: %v", err)
			return fmt.Errorf("failed to create local device map %s: %+v", cmName, err)
		}
		lastDevice = deviceStr
	}
	devicesEqual, err := deviceListsEqual(deviceStr, lastDevice)
	if err != nil {
		return fmt.Errorf("failed to compare device lists: %v", err)
	}
	if !devicesEqual {
		data := make(map[string]string, 1)
		data[LocalDiskCMData] = deviceStr
		cm.Data = data
		cm, err = context.Clientset.CoreV1().ConfigMaps(namespace).Update(cm)
		if err != nil {
			logger.Infof("failed to update configmap %s: %v", cmName, err)
			return err
		}
	}
	return nil
}

func probeDevices(context *clusterd.Context) ([]sys.LocalDisk, error) {
	devices := make([]sys.LocalDisk, 0)
	localDevices, err := clusterd.DiscoverDevices(context.Executor)
	if err != nil {
		return devices, fmt.Errorf("failed initial hardware discovery. %+v", err)
	}
	for _, device := range localDevices {
		if device == nil {
			continue
		}
		if device.Type == sys.PartType {
			continue
		}

		partitions, _, err := sys.GetDevicePartitions(device.Name, context.Executor)
		if err != nil {
			logger.Infof("failed to check device partitions %s: %v", device.Name, err)
			continue
		}

		// check if there is a file system on the device
		fs, err := sys.GetDeviceFilesystems(device.Name, context.Executor)
		if err != nil {
			logger.Infof("failed to check device filesystem %s: %v", device.Name, err)
			continue
		}
		device.Partitions = partitions
		device.Filesystem = fs
		device.Empty = clusterd.GetDeviceEmpty(device)

		devices = append(devices, *device)
	}

	logger.Infof("available devices: %+v", devices)
	return devices, nil
}
<|endoftext|>


<commit_before>package cloudinit

import (
	"github.com/rancher/os/config"
	"github.com/rancher/os/pkg/compose"
	"github.com/rancher/os/pkg/init/docker"
	"github.com/rancher/os/pkg/log"
	"github.com/rancher/os/pkg/sysinit"
	"github.com/rancher/os/pkg/util"
)

func CloudInit(cfg *config.CloudConfig) (*config.CloudConfig, error) {
	stateConfig := config.LoadConfigWithPrefix(config.StateDir)
	cfg.Rancher.CloudInit.Datasources = stateConfig.Rancher.CloudInit.Datasources

	if stateConfig.Rancher.Network.DHCPTimeout > 0 {
		cfg.Rancher.Network.DHCPTimeout = stateConfig.Rancher.Network.DHCPTimeout
		if err := config.Set("rancher.network.dhcp_timeout", stateConfig.Rancher.Network.DHCPTimeout); err != nil {
			log.Error(err)
		}
	}

	if len(stateConfig.Rancher.Network.WifiNetworks) > 0 {
		cfg.Rancher.Network.WifiNetworks = stateConfig.Rancher.Network.WifiNetworks
		if err := config.Set("rancher.network.wifi_networks", stateConfig.Rancher.Network.WifiNetworks); err != nil {
			log.Error(err)
		}
	}

	if len(stateConfig.Rancher.Network.Interfaces) > 0 {
		cfg.Rancher.Network = stateConfig.Rancher.Network
		if err := config.Set("rancher.network", stateConfig.Rancher.Network); err != nil {
			log.Error(err)
		}
	}

	hypervisor := util.GetHypervisor()
	if hypervisor == "" {
		log.Infof("ros init: No Detected Hypervisor")
	} else {
		log.Infof("ros init: Detected Hypervisor: %s", hypervisor)
	}
	if hypervisor == "vmware" {
		// add vmware to the end - we don't want to over-ride an choices the user has made
		cfg.Rancher.CloudInit.Datasources = append(cfg.Rancher.CloudInit.Datasources, hypervisor)
	}

	if err := config.Set("rancher.cloud_init.datasources", cfg.Rancher.CloudInit.Datasources); err != nil {
		log.Error(err)
	}

	log.Infof("init, runCloudInitServices(%v)", cfg.Rancher.CloudInit.Datasources)
	if err := runCloudInitServices(cfg); err != nil {
		log.Error(err)
	}

	// It'd be nice to push to rsyslog before this, but we don't have network
	log.AddRSyslogHook()

	return config.LoadConfig(), nil
}

func runCloudInitServices(cfg *config.CloudConfig) error {
	c, err := docker.Start(cfg)
	if err != nil {
		return err
	}

	defer docker.Stop(c)

	_, err = config.ChainCfgFuncs(cfg,
		[]config.CfgFuncData{
			{"cloudinit loadImages", sysinit.LoadBootstrapImages},
			{"cloudinit Services", runCloudInitServiceSet},
		})
	return err
}

func runCloudInitServiceSet(cfg *config.CloudConfig) (*config.CloudConfig, error) {
	log.Info("Running cloud-init services")
	_, err := compose.RunServiceSet("cloud-init", cfg, cfg.Rancher.CloudInitServices)
	return cfg, err
}
<commit_msg>Disable cloud-int if necessary<commit_after>package cloudinit

import (
	"path/filepath"
	"strings"

	"github.com/rancher/os/config"
	"github.com/rancher/os/pkg/compose"
	"github.com/rancher/os/pkg/init/docker"
	"github.com/rancher/os/pkg/log"
	"github.com/rancher/os/pkg/sysinit"
	"github.com/rancher/os/pkg/util"
)

func CloudInit(cfg *config.CloudConfig) (*config.CloudConfig, error) {
	stateConfig := config.LoadConfigWithPrefix(config.StateDir)
	cfg.Rancher.CloudInit.Datasources = stateConfig.Rancher.CloudInit.Datasources

	hypervisor := util.GetHypervisor()
	if hypervisor == "" {
		log.Infof("ros init: No Detected Hypervisor")
	} else {
		log.Infof("ros init: Detected Hypervisor: %s", hypervisor)
	}
	if hypervisor == "vmware" {
		// add vmware to the end - we don't want to over-ride an choices the user has made
		cfg.Rancher.CloudInit.Datasources = append(cfg.Rancher.CloudInit.Datasources, hypervisor)
	}

	if len(cfg.Rancher.CloudInit.Datasources) == 0 {
		log.Info("No specific datasources, ignore cloudinit")
		return cfg, nil
	}
	if onlyConfigDrive(cfg.Rancher.CloudInit.Datasources) {
		configDev := util.ResolveDevice("LABEL=config-2")
		if configDev == "" {
			// Check v9fs: https://www.kernel.org/doc/Documentation/filesystems/9p.txt
			matches, _ := filepath.Glob("/sys/bus/virtio/drivers/9pnet_virtio/virtio*/mount_tag")
			if len(matches) == 0 {
				log.Info("Configdrive was enabled but has no configdrive device or filesystem, ignore cloudinit")
				return cfg, nil
			}
		}
	}

	if err := config.Set("rancher.cloud_init.datasources", cfg.Rancher.CloudInit.Datasources); err != nil {
		log.Error(err)
	}

	if stateConfig.Rancher.Network.DHCPTimeout > 0 {
		cfg.Rancher.Network.DHCPTimeout = stateConfig.Rancher.Network.DHCPTimeout
		if err := config.Set("rancher.network.dhcp_timeout", stateConfig.Rancher.Network.DHCPTimeout); err != nil {
			log.Error(err)
		}
	}

	if len(stateConfig.Rancher.Network.WifiNetworks) > 0 {
		cfg.Rancher.Network.WifiNetworks = stateConfig.Rancher.Network.WifiNetworks
		if err := config.Set("rancher.network.wifi_networks", stateConfig.Rancher.Network.WifiNetworks); err != nil {
			log.Error(err)
		}
	}

	if len(stateConfig.Rancher.Network.Interfaces) > 0 {
		cfg.Rancher.Network = stateConfig.Rancher.Network
		if err := config.Set("rancher.network", stateConfig.Rancher.Network); err != nil {
			log.Error(err)
		}
	}

	log.Infof("init, runCloudInitServices(%v)", cfg.Rancher.CloudInit.Datasources)
	if err := runCloudInitServices(cfg); err != nil {
		log.Error(err)
	}

	// It'd be nice to push to rsyslog before this, but we don't have network
	log.AddRSyslogHook()

	return config.LoadConfig(), nil
}

func runCloudInitServices(cfg *config.CloudConfig) error {
	c, err := docker.Start(cfg)
	if err != nil {
		return err
	}

	defer docker.Stop(c)

	_, err = config.ChainCfgFuncs(cfg,
		[]config.CfgFuncData{
			{"cloudinit loadImages", sysinit.LoadBootstrapImages},
			{"cloudinit Services", runCloudInitServiceSet},
		})
	return err
}

func runCloudInitServiceSet(cfg *config.CloudConfig) (*config.CloudConfig, error) {
	log.Info("Running cloud-init services")
	_, err := compose.RunServiceSet("cloud-init", cfg, cfg.Rancher.CloudInitServices)
	return cfg, err
}

func onlyConfigDrive(datasources []string) bool {
	if len(datasources) != 1 {
		return false
	}
	for _, ds := range datasources {
		parts := strings.SplitN(ds, ":", 2)
		if parts[0] == "configdrive" {
			return true
		}
	}
	return false
}
<|endoftext|>