<commit_before>// +build !bootstrap

// Contains functions related to dispatching work to remote processes.
// Right now those processes must be on the same box because they use
// the local temporary directories, but in the future this might form
// a foundation for doing real distributed work.

package build

import (
	"encoding/json"
	"fmt"
	"io"
	"os/exec"
	"path"
	"strings"
	"sync"

	"github.com/golang/protobuf/jsonpb"
	"github.com/google/shlex"

	pb "build/proto/worker"
	"core"
)

// A workerServer is the structure we use to maintain information about a remote work server.
type workerServer struct {
	requests      chan *pb.BuildRequest
	responses     map[string]chan *pb.BuildResponse
	responseMutex sync.Mutex
	process       *exec.Cmd
	stderr        *stderrLogger
	closing       bool
}

// workerMap contains all the remote workers we've started so far.
var workerMap = map[string]*workerServer{}
var workerMutex sync.Mutex

// buildMaybeRemotely builds a target, either sending it to a remote worker if needed,
// or locally if not.
func buildMaybeRemotely(state *core.BuildState, target *core.BuildTarget, inputHash []byte) ([]byte, error) {
	worker, workerArgs, localCmd := workerCommandAndArgs(state, target)
	if worker == "" {
		return runBuildCommand(state, target, localCmd, inputHash)
	}
	// The scheme here is pretty minimal; remote workers currently have quite a bit less info than
	// local ones get. Over time we'll probably evolve it to add more information.
	opts, err := shlex.Split(workerArgs)
	if err != nil {
		return nil, err
	}
	log.Debug("Sending remote build request for %s to %s; opts %s", target.Label, worker, workerArgs)
	resp, err := buildRemotely(state, worker, &pb.BuildRequest{
		Rule:    target.Label.String(),
		Labels:  target.Labels,
		TempDir: path.Join(core.RepoRoot, target.TmpDir()),
		Srcs:    target.AllSourcePaths(state.Graph),
		Opts:    opts,
	})
	if err != nil {
		return nil, err
	}
	out := strings.Join(resp.Messages, "\n")
	if !resp.Success {
		return nil, fmt.Errorf("Error building target %s: %s", target.Label, out)
	}
	// Okay, now we might need to do something locally too...
	if localCmd != "" {
		out2, err := runBuildCommand(state, target, localCmd, inputHash)
		return append([]byte(out+"\n"), out2...), err
	}
	return []byte(out), nil
}

// buildRemotely runs a single build request and returns its response.
func buildRemotely(state *core.BuildState, worker string, req *pb.BuildRequest) (*pb.BuildResponse, error) {
	w, err := getOrStartWorker(state, worker)
	if err != nil {
		return nil, err
	}
	w.requests <- req
	ch := make(chan *pb.BuildResponse, 1)
	w.responseMutex.Lock()
	w.responses[req.Rule] = ch
	w.responseMutex.Unlock()
	response := <-ch
	return response, nil
}

// EnsureWorkerStarted ensures that a worker server is started and has responded saying it's ready.
func EnsureWorkerStarted(state *core.BuildState, worker string, label core.BuildLabel) error {
	resp, err := buildRemotely(state, worker, &pb.BuildRequest{
		Rule: label.String(),
		Test: true,
	})
	if err == nil && !resp.Success {
		return fmt.Errorf(strings.Join(resp.Messages, "\n"))
	}
	return err
}

// getOrStartWorker either retrieves an existing worker process or starts a new one.
func getOrStartWorker(state *core.BuildState, worker string) (*workerServer, error) {
	workerMutex.Lock()
	defer workerMutex.Unlock()
	if w, present := workerMap[worker]; present {
		return w, nil
	}
	// Need to create a new process
	cmd := core.ExecCommand(worker)
	cmd.Env = core.GeneralBuildEnvironment(state.Config)
	stdin, err := cmd.StdinPipe()
	if err != nil {
		return nil, err
	}
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	stderr := &stderrLogger{}
	cmd.Stderr = stderr
	if err := cmd.Start(); err != nil {
		return nil, err
	}
	w := &workerServer{
		requests:  make(chan *pb.BuildRequest),
		responses: map[string]chan *pb.BuildResponse{},
		process:   cmd,
		stderr:    stderr,
	}
	go w.sendRequests(stdin)
	go w.readResponses(stdout)
	go w.wait()
	workerMap[worker] = w
	state.Stats.NumWorkerProcesses = len(workerMap)
	return w, nil
}

// sendRequests sends requests to a running worker server.
func (w *workerServer) sendRequests(stdin io.Writer) {
	m := &jsonpb.Marshaler{OrigName: true}
	for request := range w.requests {
		if err := m.Marshal(stdin, request); err != nil {
			log.Error("Failed to write request: %s", err)
			continue
		}
		stdin.Write([]byte{'\n'}) // Newline delimit them as a nicety.
	}
}

// readResponses reads the responses from a running worker server and dispatches them appropriately.
func (w *workerServer) readResponses(stdout io.Reader) {
	decoder := json.NewDecoder(stdout)
	u := &jsonpb.Unmarshaler{AllowUnknownFields: true}
	for {
		response := pb.BuildResponse{}
		if err := u.UnmarshalNext(decoder, &response); err != nil {
			w.Error("Failed to read response: %s", err)
			break
		}
		w.responseMutex.Lock()
		ch, present := w.responses[response.Rule]
		delete(w.responses, response.Rule)
		w.responseMutex.Unlock()
		if present {
			log.Debug("Got response from remote worker for %s, success: %v", response.Rule, response.Success)
			ch <- &response
		} else {
			w.Error("Couldn't find response channel for %s", response.Rule)
		}
	}
}

// wait waits for the process to terminate. If it dies unexpectedly this handles various failures.
func (w *workerServer) wait() {
	if err := w.process.Wait(); err != nil && !w.closing {
		log.Error("Worker process died unexpectedly: %s", err)
		w.responseMutex.Lock()
		for label, ch := range w.responses {
			ch <- &pb.BuildResponse{
				Rule:     label,
				Messages: []string{fmt.Sprintf("Worker failed: %s\n%s", err, string(w.stderr.History))},
			}
		}
		w.responseMutex.Unlock()

	}
}

func (w *workerServer) Error(msg string, args ...interface{}) {
	if !w.closing {
		log.Error(msg, args...)
	}
}

// stderrLogger is used to log any errors from our worker tools.
type stderrLogger struct {
	buffer  []byte
	History []byte
	// suppress will silence any further logging messages when set.
	Suppress bool
}

// Write implements the io.Writer interface
func (l *stderrLogger) Write(msg []byte) (int, error) {
	l.buffer = append(l.buffer, msg...)
	if len(l.buffer) > 0 && l.buffer[len(l.buffer)-1] == '\n' {
		if !l.Suppress {
			log.Error("Error from remote worker: %s", strings.TrimSpace(string(l.buffer)))
		}
		l.History = append(l.History, l.buffer...)
		l.buffer = nil
	}
	return len(msg), nil
}

// StopWorkers stops any running worker processes.
func StopWorkers() {
	for name, worker := range workerMap {
		log.Debug("Terminating build worker %s", name)
		worker.closing = true         // suppress any error messages from worker
		worker.stderr.Suppress = true // Make sure we don't print anything as they die.
		core.KillProcess(worker.process)
	}
	workerMap = map[string]*workerServer{}
}
<commit_msg>More robust handling when we fail to write a request to a worker.<commit_after>// +build !bootstrap

// Contains functions related to dispatching work to remote processes.
// Right now those processes must be on the same box because they use
// the local temporary directories, but in the future this might form
// a foundation for doing real distributed work.

package build

import (
	"encoding/json"
	"fmt"
	"io"
	"os/exec"
	"path"
	"strings"
	"sync"

	"github.com/golang/protobuf/jsonpb"
	"github.com/google/shlex"

	pb "build/proto/worker"
	"core"
)

// A workerServer is the structure we use to maintain information about a remote work server.
type workerServer struct {
	requests      chan *pb.BuildRequest
	responses     map[string]chan *pb.BuildResponse
	responseMutex sync.Mutex
	process       *exec.Cmd
	stderr        *stderrLogger
	closing       bool
}

// workerMap contains all the remote workers we've started so far.
var workerMap = map[string]*workerServer{}
var workerMutex sync.Mutex

// buildMaybeRemotely builds a target, either sending it to a remote worker if needed,
// or locally if not.
func buildMaybeRemotely(state *core.BuildState, target *core.BuildTarget, inputHash []byte) ([]byte, error) {
	worker, workerArgs, localCmd := workerCommandAndArgs(state, target)
	if worker == "" {
		return runBuildCommand(state, target, localCmd, inputHash)
	}
	// The scheme here is pretty minimal; remote workers currently have quite a bit less info than
	// local ones get. Over time we'll probably evolve it to add more information.
	opts, err := shlex.Split(workerArgs)
	if err != nil {
		return nil, err
	}
	log.Debug("Sending remote build request for %s to %s; opts %s", target.Label, worker, workerArgs)
	resp, err := buildRemotely(state, worker, &pb.BuildRequest{
		Rule:    target.Label.String(),
		Labels:  target.Labels,
		TempDir: path.Join(core.RepoRoot, target.TmpDir()),
		Srcs:    target.AllSourcePaths(state.Graph),
		Opts:    opts,
	})
	if err != nil {
		return nil, err
	}
	out := strings.Join(resp.Messages, "\n")
	if !resp.Success {
		return nil, fmt.Errorf("Error building target %s: %s", target.Label, out)
	}
	// Okay, now we might need to do something locally too...
	if localCmd != "" {
		out2, err := runBuildCommand(state, target, localCmd, inputHash)
		return append([]byte(out+"\n"), out2...), err
	}
	return []byte(out), nil
}

// buildRemotely runs a single build request and returns its response.
func buildRemotely(state *core.BuildState, worker string, req *pb.BuildRequest) (*pb.BuildResponse, error) {
	w, err := getOrStartWorker(state, worker)
	if err != nil {
		return nil, err
	}
	w.requests <- req
	ch := make(chan *pb.BuildResponse, 1)
	w.responseMutex.Lock()
	w.responses[req.Rule] = ch
	w.responseMutex.Unlock()
	response := <-ch
	return response, nil
}

// EnsureWorkerStarted ensures that a worker server is started and has responded saying it's ready.
func EnsureWorkerStarted(state *core.BuildState, worker string, label core.BuildLabel) error {
	resp, err := buildRemotely(state, worker, &pb.BuildRequest{
		Rule: label.String(),
		Test: true,
	})
	if err == nil && !resp.Success {
		return fmt.Errorf(strings.Join(resp.Messages, "\n"))
	}
	return err
}

// getOrStartWorker either retrieves an existing worker process or starts a new one.
func getOrStartWorker(state *core.BuildState, worker string) (*workerServer, error) {
	workerMutex.Lock()
	defer workerMutex.Unlock()
	if w, present := workerMap[worker]; present {
		return w, nil
	}
	// Need to create a new process
	cmd := core.ExecCommand(worker)
	cmd.Env = core.GeneralBuildEnvironment(state.Config)
	stdin, err := cmd.StdinPipe()
	if err != nil {
		return nil, err
	}
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	stderr := &stderrLogger{}
	cmd.Stderr = stderr
	if err := cmd.Start(); err != nil {
		return nil, err
	}
	w := &workerServer{
		requests:  make(chan *pb.BuildRequest),
		responses: map[string]chan *pb.BuildResponse{},
		process:   cmd,
		stderr:    stderr,
	}
	go w.sendRequests(stdin)
	go w.readResponses(stdout)
	go w.wait()
	workerMap[worker] = w
	state.Stats.NumWorkerProcesses = len(workerMap)
	return w, nil
}

// sendRequests sends requests to a running worker server.
func (w *workerServer) sendRequests(stdin io.Writer) {
	m := &jsonpb.Marshaler{OrigName: true}
	for request := range w.requests {
		if err := m.Marshal(stdin, request); err != nil {
			log.Error("Failed to write request: %s", err)
			w.dispatchResponse(&pb.BuildResponse{
				Rule:     request.Rule,
				Success:  false,
				Messages: []string{err.Error()},
			})
			continue
		}
		stdin.Write([]byte{'\n'}) // Newline delimit them as a nicety.
	}
}

// readResponses reads the responses from a running worker server and dispatches them appropriately.
func (w *workerServer) readResponses(stdout io.Reader) {
	decoder := json.NewDecoder(stdout)
	u := &jsonpb.Unmarshaler{AllowUnknownFields: true}
	for {
		response := pb.BuildResponse{}
		if err := u.UnmarshalNext(decoder, &response); err != nil {
			w.Error("Failed to read response: %s", err)
			break
		}
		w.dispatchResponse(&response)
	}
}

// dispatchResponse sends a single response on the appropriate channel.
func (w *workerServer) dispatchResponse(response *pb.BuildResponse) {
	w.responseMutex.Lock()
	ch, present := w.responses[response.Rule]
	delete(w.responses, response.Rule)
	w.responseMutex.Unlock()
	if present {
		log.Debug("Got response from remote worker for %s, success: %v", response.Rule, response.Success)
		ch <- response
	} else {
		w.Error("Couldn't find response channel for %s", response.Rule)
	}
}

// wait waits for the process to terminate. If it dies unexpectedly this handles various failures.
func (w *workerServer) wait() {
	if err := w.process.Wait(); err != nil && !w.closing {
		log.Error("Worker process died unexpectedly: %s", err)
		w.responseMutex.Lock()
		defer w.responseMutex.Unlock()
		for label, ch := range w.responses {
			ch <- &pb.BuildResponse{
				Rule:     label,
				Messages: []string{fmt.Sprintf("Worker failed: %s\n%s", err, string(w.stderr.History))},
			}
		}
	}
}

func (w *workerServer) Error(msg string, args ...interface{}) {
	if !w.closing {
		log.Error(msg, args...)
	}
}

// stderrLogger is used to log any errors from our worker tools.
type stderrLogger struct {
	buffer  []byte
	History []byte
	// suppress will silence any further logging messages when set.
	Suppress bool
}

// Write implements the io.Writer interface
func (l *stderrLogger) Write(msg []byte) (int, error) {
	l.buffer = append(l.buffer, msg...)
	if len(l.buffer) > 0 && l.buffer[len(l.buffer)-1] == '\n' {
		if !l.Suppress {
			log.Error("Error from remote worker: %s", strings.TrimSpace(string(l.buffer)))
		}
		l.History = append(l.History, l.buffer...)
		l.buffer = nil
	}
	return len(msg), nil
}

// StopWorkers stops any running worker processes.
func StopWorkers() {
	for name, worker := range workerMap {
		log.Debug("Terminating build worker %s", name)
		worker.closing = true         // suppress any error messages from worker
		worker.stderr.Suppress = true // Make sure we don't print anything as they die.
		core.KillProcess(worker.process)
	}
	workerMap = map[string]*workerServer{}
}
<|endoftext|>


<commit_before>package consensus

import (
	"testing"

	"github.com/NebulousLabs/Sia/types"
)

// TestEarliestChildTimestamp probes the earliestChildTimestamp method of the
// block node type.
func TestEarliestChildTimestamp(t *testing.T) {
	// Check the earliest timestamp generated when the block node has no
	// parent.
	bn1 := &blockNode{block: types.Block{Timestamp: 1}}
	if bn1.earliestChildTimestamp() != 1 {
		t.Error("earliest child timestamp has been calculated incorrectly.")
	}

	// Set up a series of targets, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11
	bn2 := &blockNode{block: types.Block{Timestamp: 2}, parent: bn1}
	bn3 := &blockNode{block: types.Block{Timestamp: 3}, parent: bn2}
	bn4 := &blockNode{block: types.Block{Timestamp: 4}, parent: bn3}
	bn5 := &blockNode{block: types.Block{Timestamp: 5}, parent: bn4}
	bn6 := &blockNode{block: types.Block{Timestamp: 6}, parent: bn5}
	bn7 := &blockNode{block: types.Block{Timestamp: 7}, parent: bn6}
	bn8 := &blockNode{block: types.Block{Timestamp: 8}, parent: bn7}
	bn9 := &blockNode{block: types.Block{Timestamp: 9}, parent: bn8}
	bn10 := &blockNode{block: types.Block{Timestamp: 10}, parent: bn9}
	bn11 := &blockNode{block: types.Block{Timestamp: 11}, parent: bn10}

	// Median should be '1' for bn6.
	if bn6.earliestChildTimestamp() != 1 {
		t.Error("incorrect child timestamp")
	}
	// Median should be '2' for bn7.
	if bn7.earliestChildTimestamp() != 2 {
		t.Error("incorrect child timestamp")
	}
	// Median should be '6' for bn11.
	if bn11.earliestChildTimestamp() != 6 {
		t.Error("incorrect child timestamp")
	}

	// Mix up the sorting:
	//           7, 5, 5, 2, 3, 9, 12, 1, 8, 6, 14
	// sorted11: 1, 2, 3, 5, 5, 6, 7, 8, 9, 12, 14
	// sorted10: 1, 2, 3, 5, 5, 6, 7, 7, 8, 9, 12
	// sorted9:  1, 2, 3, 5, 5, 7, 7, 7, 8, 9, 12
	bn1.block.Timestamp = 7
	bn2.block.Timestamp = 5
	bn3.block.Timestamp = 5
	bn4.block.Timestamp = 2
	bn5.block.Timestamp = 3
	bn6.block.Timestamp = 9
	bn7.block.Timestamp = 12
	bn8.block.Timestamp = 1
	bn9.block.Timestamp = 8
	bn10.block.Timestamp = 6
	bn11.block.Timestamp = 14

	// Median of bn11 should be '6'.
	if bn11.earliestChildTimestamp() != 6 {
		t.Error("incorrect child timestamp")
	}
	// Median of bn10 should be '6'.
	if bn10.earliestChildTimestamp() != 6 {
		t.Error("incorrect child timestamp")
	}
	// Median of bn9 should be '7'.
	if bn9.earliestChildTimestamp() != 7 {
		t.Error("incorrect child timestamp")
	}
}
<commit_msg>add unit test for heavierThan<commit_after>package consensus

import (
	"testing"

	"github.com/NebulousLabs/Sia/types"
)

// TestEarliestChildTimestamp probes the earliestChildTimestamp method of the
// block node type.
func TestEarliestChildTimestamp(t *testing.T) {
	// Check the earliest timestamp generated when the block node has no
	// parent.
	bn1 := &blockNode{block: types.Block{Timestamp: 1}}
	if bn1.earliestChildTimestamp() != 1 {
		t.Error("earliest child timestamp has been calculated incorrectly.")
	}

	// Set up a series of targets, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11
	bn2 := &blockNode{block: types.Block{Timestamp: 2}, parent: bn1}
	bn3 := &blockNode{block: types.Block{Timestamp: 3}, parent: bn2}
	bn4 := &blockNode{block: types.Block{Timestamp: 4}, parent: bn3}
	bn5 := &blockNode{block: types.Block{Timestamp: 5}, parent: bn4}
	bn6 := &blockNode{block: types.Block{Timestamp: 6}, parent: bn5}
	bn7 := &blockNode{block: types.Block{Timestamp: 7}, parent: bn6}
	bn8 := &blockNode{block: types.Block{Timestamp: 8}, parent: bn7}
	bn9 := &blockNode{block: types.Block{Timestamp: 9}, parent: bn8}
	bn10 := &blockNode{block: types.Block{Timestamp: 10}, parent: bn9}
	bn11 := &blockNode{block: types.Block{Timestamp: 11}, parent: bn10}

	// Median should be '1' for bn6.
	if bn6.earliestChildTimestamp() != 1 {
		t.Error("incorrect child timestamp")
	}
	// Median should be '2' for bn7.
	if bn7.earliestChildTimestamp() != 2 {
		t.Error("incorrect child timestamp")
	}
	// Median should be '6' for bn11.
	if bn11.earliestChildTimestamp() != 6 {
		t.Error("incorrect child timestamp")
	}

	// Mix up the sorting:
	//           7, 5, 5, 2, 3, 9, 12, 1, 8, 6, 14
	// sorted11: 1, 2, 3, 5, 5, 6, 7, 8, 9, 12, 14
	// sorted10: 1, 2, 3, 5, 5, 6, 7, 7, 8, 9, 12
	// sorted9:  1, 2, 3, 5, 5, 7, 7, 7, 8, 9, 12
	bn1.block.Timestamp = 7
	bn2.block.Timestamp = 5
	bn3.block.Timestamp = 5
	bn4.block.Timestamp = 2
	bn5.block.Timestamp = 3
	bn6.block.Timestamp = 9
	bn7.block.Timestamp = 12
	bn8.block.Timestamp = 1
	bn9.block.Timestamp = 8
	bn10.block.Timestamp = 6
	bn11.block.Timestamp = 14

	// Median of bn11 should be '6'.
	if bn11.earliestChildTimestamp() != 6 {
		t.Error("incorrect child timestamp")
	}
	// Median of bn10 should be '6'.
	if bn10.earliestChildTimestamp() != 6 {
		t.Error("incorrect child timestamp")
	}
	// Median of bn9 should be '7'.
	if bn9.earliestChildTimestamp() != 7 {
		t.Error("incorrect child timestamp")
	}
}

// TestHeavierThan probes the heavierThan method of the blockNode.
func TestHeavierThan(t *testing.T) {
	// Create a light node.
	bnLight := new(blockNode)
	bnLight.depth[0] = 64
	bnLight.childTarget[0] = 200

	// Create a node that's heavier, but not enough to beat the surpass
	// threshold.
	bnMiddle := new(blockNode)
	bnMiddle.depth[0] = 60
	bnMiddle.childTarget[0] = 200

	// Create a node that's heavy enough to break the surpass threshold.
	bnHeavy := new(blockNode)
	bnHeavy.depth[0] = 16
	bnHeavy.childTarget[0] = 200

	// bnLight should not be heavier than bnHeavy.
	if bnLight.heavierThan(bnHeavy) {
		t.Error("light heavier than heavy")
	}
	// bnLight should not be heavier than middle.
	if bnLight.heavierThan(bnMiddle) {
		t.Error("light heavier than middle")
	}
	// bnLight should not be heavier than itself.
	if bnLight.heavierThan(bnLight) {
		t.Error("light heavier than itself")
	}

	// bnMiddle should not be heavier than bnLight.
	if bnMiddle.heavierThan(bnLight) {
		t.Error("middle heaver than light - surpass threshold should not have been broken")
	}
	// bnHeavy should be heaver than bnLight.
	if !bnHeavy.heavierThan(bnLight) {
		t.Error("heavy is not heavier than light")
	}
	// bnHeavy should be heavier than bnMiddle.
	if !bnHeavy.heavierThan(bnMiddle) {
		t.Error("heavy is not heavier than middle")
	}
}
<|endoftext|>


<commit_before>package relay

import (
	"context"
	"time"

	circuit "github.com/libp2p/go-libp2p-circuit"
	discovery "github.com/libp2p/go-libp2p-discovery"
	ma "github.com/multiformats/go-multiaddr"
)

var (
	AdvertiseBootDelay = 30 * time.Second
)

// Advertise advertises this node as a libp2p relay.
func Advertise(ctx context.Context, advertise discovery.Advertiser) {
	go func() {
		select {
		case <-time.After(AdvertiseBootDelay):
			discovery.Advertise(ctx, advertise, RelayRendezvous)
		case <-ctx.Done():
		}
	}()
}

// Filter filters out all relay addresses.
func Filter(addrs []ma.Multiaddr) []ma.Multiaddr {
	raddrs := make([]ma.Multiaddr, 0, len(addrs))
	for _, addr := range addrs {
		_, err := addr.ValueForProtocol(circuit.P_CIRCUIT)
		if err == nil {
			continue
		}
		raddrs = append(raddrs, addr)
	}
	return raddrs
}
<commit_msg>increase relay advertising boot delay to 15min<commit_after>package relay

import (
	"context"
	"time"

	circuit "github.com/libp2p/go-libp2p-circuit"
	discovery "github.com/libp2p/go-libp2p-discovery"
	ma "github.com/multiformats/go-multiaddr"
)

var (
	// this is purposefully long to require some node stability before advertising as a relay
	AdvertiseBootDelay = 15 * time.Minute
)

// Advertise advertises this node as a libp2p relay.
func Advertise(ctx context.Context, advertise discovery.Advertiser) {
	go func() {
		select {
		case <-time.After(AdvertiseBootDelay):
			discovery.Advertise(ctx, advertise, RelayRendezvous)
		case <-ctx.Done():
		}
	}()
}

// Filter filters out all relay addresses.
func Filter(addrs []ma.Multiaddr) []ma.Multiaddr {
	raddrs := make([]ma.Multiaddr, 0, len(addrs))
	for _, addr := range addrs {
		_, err := addr.ValueForProtocol(circuit.P_CIRCUIT)
		if err == nil {
			continue
		}
		raddrs = append(raddrs, addr)
	}
	return raddrs
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"strings"

	"github.com/ghodss/yaml"
)

// To add new e2e test support, you need to:
//   1) Transform e2e performance test result into *PerfData* in k8s/kubernetes/test/e2e/perftype,
//   and print the PerfData in e2e test log.
//   2) Add corresponding bucket, job and test into *TestConfig*.

// TestDescription contains test name, output file prefix and parser function.
type TestDescription struct {
	Name             string
	OutputFilePrefix string
	Parser           func(data []byte, buildNumber int, testResult *BuildData)
}

// TestDescriptions is a map job->component->description.
type TestDescriptions map[string]map[string][]TestDescription

// Tests is a map from test label to test description.
type Tests struct {
	Prefix       string
	Descriptions TestDescriptions
	BuildsCount  int
}

// Jobs is a map from job name to all supported tests in the job.
type Jobs map[string]Tests

// Buckets is a map from bucket url to all supported jobs in the bucket.
type Buckets map[string]Jobs

var (
	// performanceDescriptions contains metrics exported by a --ginko.focus=[Feature:Performance]
	// e2e test
	performanceDescriptions = TestDescriptions{
		"E2E": {
			"DensityResources": []TestDescription{{
				Name:             "density",
				OutputFilePrefix: "ResourceUsageSummary",
				Parser:           parseResourceUsageData,
			}},
			"DensityPodStartup": []TestDescription{{
				Name:             "density",
				OutputFilePrefix: "PodStartupLatency_PodStartupLatency",
				Parser:           parsePerfData,
			}},
			"DensitySaturationPodStartup": []TestDescription{{
				Name:             "density",
				OutputFilePrefix: "PodStartupLatency_SaturationPodStartupLatency",
				Parser:           parsePerfData,
			}},
			"LoadResources": []TestDescription{{
				Name:             "load",
				OutputFilePrefix: "ResourceUsageSummary",
				Parser:           parseResourceUsageData,
			}},
		},
		"APIServer": {
			"DensityResponsiveness": []TestDescription{{
				Name:             "density",
				OutputFilePrefix: "APIResponsiveness",
				Parser:           parsePerfData,
			}},
			"DensityRequestCount": []TestDescription{{
				Name:             "density",
				OutputFilePrefix: "APIResponsiveness",
				Parser:           parseRequestCountData,
			}},
			"DensityResponsiveness_Prometheus": []TestDescription{{
				Name:             "density",
				OutputFilePrefix: "APIResponsivenessPrometheus",
				Parser:           parsePerfData,
			}},
			"DensityRequestCount_Prometheus": []TestDescription{{
				Name:             "density",
				OutputFilePrefix: "APIResponsivenessPrometheus",
				Parser:           parseRequestCountData,
			}},
			"DensityRequestCountByClient": []TestDescription{{
				Name:             "density",
				OutputFilePrefix: "MetricsForE2E",
				Parser:           parseApiserverRequestCount,
			}},
			"DensityInitEventsCount": []TestDescription{{
				Name:             "density",
				OutputFilePrefix: "MetricsForE2E",
				Parser:           parseApiserverInitEventsCount,
			}},
			"LoadResponsiveness": []TestDescription{{
				Name:             "load",
				OutputFilePrefix: "APIResponsiveness",
				Parser:           parsePerfData,
			}},
			"LoadRequestCount": []TestDescription{{
				Name:             "load",
				OutputFilePrefix: "APIResponsiveness",
				Parser:           parseRequestCountData,
			}},
			"LoadResponsiveness_Prometheus": []TestDescription{{
				Name:             "load",
				OutputFilePrefix: "APIResponsivenessPrometheus",
				Parser:           parsePerfData,
			}},
			"LoadRequestCount_Prometheus": []TestDescription{{
				Name:             "load",
				OutputFilePrefix: "APIResponsivenessPrometheus",
				Parser:           parseRequestCountData,
			}},
			"LoadRequestCountByClient": []TestDescription{{
				Name:             "load",
				OutputFilePrefix: "MetricsForE2E",
				Parser:           parseApiserverRequestCount,
			}},
			"LoadInitEventsCount": []TestDescription{{
				Name:             "load",
				OutputFilePrefix: "MetricsForE2E",
				Parser:           parseApiserverInitEventsCount,
			}},
		},
		"Scheduler": {
			"SchedulingLatency": []TestDescription{{
				Name:             "density",
				OutputFilePrefix: "SchedulingMetrics",
				Parser:           parseSchedulingLatency,
			}},
			"SchedulingThroughput": []TestDescription{{
				Name:             "density",
				OutputFilePrefix: "SchedulingThroughput",
				Parser:           parseSchedulingThroughputCL,
			}},
		},
		"Etcd": {
			"DensityBackendCommitDuration": []TestDescription{{
				Name:             "density",
				OutputFilePrefix: "EtcdMetrics",
				Parser:           parseHistogramMetric("backendCommitDuration"),
			}},
			"DensitySnapshotSaveTotalDuration": []TestDescription{{
				Name:             "density",
				OutputFilePrefix: "EtcdMetrics",
				Parser:           parseHistogramMetric("snapshotSaveTotalDuration"),
			}},
			"DensityPeerRoundTripTime": []TestDescription{{
				Name:             "density",
				OutputFilePrefix: "EtcdMetrics",
				Parser:           parseHistogramMetric("peerRoundTripTime"),
			}},
			"DensityWalFsyncDuration": []TestDescription{{
				Name:             "density",
				OutputFilePrefix: "EtcdMetrics",
				Parser:           parseHistogramMetric("walFsyncDuration"),
			}},
			"LoadBackendCommitDuration": []TestDescription{{
				Name:             "load",
				OutputFilePrefix: "EtcdMetrics",
				Parser:           parseHistogramMetric("backendCommitDuration"),
			}},
			"LoadSnapshotSaveTotalDuration": []TestDescription{{
				Name:             "load",
				OutputFilePrefix: "EtcdMetrics",
				Parser:           parseHistogramMetric("snapshotSaveTotalDuration"),
			}},
			"LoadPeerRoundTripTime": []TestDescription{{
				Name:             "load",
				OutputFilePrefix: "EtcdMetrics",
				Parser:           parseHistogramMetric("peerRoundTripTime"),
			}},
			"LoadWalFsyncDuration": []TestDescription{{
				Name:             "load",
				OutputFilePrefix: "EtcdMetrics",
				Parser:           parseHistogramMetric("walFsyncDuration"),
			}},
		},
		"Network": {
			"Load_NetworkProgrammingLatency": []TestDescription{{
				Name:             "load",
				OutputFilePrefix: "NetworkProgrammingLatency",
				Parser:           parsePerfData,
			}},

			"Load_NetworkLatency": []TestDescription{
				{
					// TODO(oxddr): remove this around Sep '19 when we stop showing old data
					Name:             "load",
					OutputFilePrefix: "in_cluster_network_latency",
					Parser:           parsePerfData,
				}, {
					Name:             "load",
					OutputFilePrefix: "InClusterNetworkLatency",
					Parser:           parsePerfData,
				}},

			"Density_NetworkLatency": []TestDescription{
				{
					// TODO(oxddr): remove this around Sep '19 when we stop showing old data
					Name:             "density",
					OutputFilePrefix: "in_cluster_network_latency",
					Parser:           parsePerfData,
				}, {
					Name:             "density",
					OutputFilePrefix: "InClusterNetworkLatency",
					Parser:           parsePerfData,
				}},
		},
		"DNS": {
			"Load_DNSLookupLatency": []TestDescription{{
				Name:             "load",
				OutputFilePrefix: "DnsLookupLatency",
				Parser:           parsePerfData,
			}},
			"Density_DNSLookupLatency": []TestDescription{{
				Name:             "density",
				OutputFilePrefix: "DnsLookupLatency",
				Parser:           parsePerfData,
			}},
		},
	}

	// benchmarkDescriptions contains metrics exported by test/integration/scheduler_perf
	benchmarkDescriptions = TestDescriptions{
		"Scheduler": {
			"BenchmarkResults": []TestDescription{{
				Name:             "benchmark",
				OutputFilePrefix: "BenchmarkResults",
				Parser:           parsePerfData,
			}},
		},
	}

	dnsBenchmarkDescriptions = TestDescriptions{
		"dns": {
			"Latency": []TestDescription{{
				Name:             "dns",
				OutputFilePrefix: "Latency",
				Parser:           parsePerfData,
			}},
			"LatencyPerc": []TestDescription{{
				Name:             "dns",
				OutputFilePrefix: "LatencyPerc",
				Parser:           parsePerfData,
			}},
			"Queries": []TestDescription{{
				Name:             "dns",
				OutputFilePrefix: "Queries",
				Parser:           parsePerfData,
			}},
			"Qps": []TestDescription{{
				Name:             "dns",
				OutputFilePrefix: "Qps",
				Parser:           parsePerfData,
			}},
		},
	}

	storageDescriptions = TestDescriptions{
		"APIServer": {
			"Responsiveness": []TestDescription{
				{
					Name:             "pod-with-ephemeral-volume-startup-latency",
					OutputFilePrefix: "APIResponsiveness",
					Parser:           parsePerfData,
				},
				{
					Name:             "storage-",
					OutputFilePrefix: "APIResponsiveness",
					Parser:           parsePerfData,
				},
			},
			"RequestCount": []TestDescription{
				{
					Name:             "pod-with-ephemeral-volume-startup-latency",
					OutputFilePrefix: "APIResponsiveness",
					Parser:           parseRequestCountData,
				},
				{
					Name:             "storage-",
					OutputFilePrefix: "APIResponsiveness",
					Parser:           parseRequestCountData,
				},
			},
		},
		"E2E": {
			"PodStartup": []TestDescription{
				{
					Name:             "pod-with-ephemeral-volume-startup-latency",
					OutputFilePrefix: "PodStartupLatency_PodWithMultiVolumeStartupLatency",
					Parser:           parsePerfData,
				},
				{
					Name:             "storage-",
					OutputFilePrefix: "PodStartupLatency_PodWithVolumesStartupLatency",
					Parser:           parsePerfData,
				},
			},
		},
	}

	jobTypeToDescriptions = map[string]TestDescriptions{
		"performance":  performanceDescriptions,
		"benchmark":    benchmarkDescriptions,
		"dnsBenchmark": dnsBenchmarkDescriptions,
		"storage":      storageDescriptions,
	}
)

func getProwConfigOrDie(configPaths []string) Jobs {
	jobs, err := getProwConfig(configPaths)
	if err != nil {
		panic(err)
	}
	return jobs
}

// Minimal subset of the prow config definition at k8s.io/test-infra/prow/config
type config struct {
	Periodics []periodic `json:"periodics"`
}
type periodic struct {
	Name string   `json:"name"`
	Tags []string `json:"tags"`
}

func urlConfigRead(url string) ([]byte, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, fmt.Errorf("error fetching prow config from %s: %v", url, err)
	}
	defer resp.Body.Close()
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("error reading prow config from %s: %v", url, err)
	}
	return b, nil
}

func fileConfigRead(path string) ([]byte, error) {
	return ioutil.ReadFile(path)
}

func getProwConfig(configPaths []string) (Jobs, error) {
	jobs := Jobs{}

	for _, configPath := range configPaths {
		fmt.Fprintf(os.Stderr, "Fetching config %s\n", configPath)
		// Perfdash supports only yamls.
		if !strings.HasSuffix(configPath, ".yaml") {
			fmt.Fprintf(os.Stderr, "%s is not an yaml file!\n", configPath)
			continue
		}
		var content []byte
		var err error
		switch {
		case strings.HasPrefix(configPath, "http://"), strings.HasPrefix(configPath, "https://"):
			content, err = urlConfigRead(configPath)
		default:
			content, err = fileConfigRead(configPath)
		}
		if err != nil {
			return nil, err
		}
		conf := &config{}
		if err := yaml.Unmarshal(content, conf); err != nil {
			return nil, fmt.Errorf("error unmarshaling prow config from %s: %v", configPath, err)
		}
		for _, periodic := range conf.Periodics {
			var thisPeriodicConfig Tests
			for _, tag := range periodic.Tags {
				if strings.HasPrefix(tag, "perfDashPrefix:") {
					split := strings.SplitN(tag, ":", 2)
					thisPeriodicConfig.Prefix = strings.TrimSpace(split[1])
					continue
				}
				if strings.HasPrefix(tag, "perfDashJobType:") {
					split := strings.SplitN(tag, ":", 2)
					jobType := strings.TrimSpace(split[1])
					var exists bool
					if thisPeriodicConfig.Descriptions, exists = jobTypeToDescriptions[jobType]; !exists {
						fmt.Fprintf(os.Stderr, "warning: unknown job type - %s\n", jobType)
					}
					continue
				}
				if strings.HasPrefix(tag, "perfDashBuildsCount:") {
					split := strings.SplitN(tag, ":", 2)
					i, err := strconv.Atoi(strings.TrimSpace(split[1]))
					if err != nil {
						fmt.Fprintf(os.Stderr, "warning: unparsable builds count - %v\n", split[1])
					}
					if i < 1 {
						fmt.Fprintf(os.Stderr, "warning: non-positive builds count - %v\n", i)
						continue
					}
					thisPeriodicConfig.BuildsCount = i
					continue
				}
				if strings.HasPrefix(tag, "perfDash") {
					fmt.Fprintf(os.Stderr, "warning: unknown perfdash tag name: %q\n", tag)
				}
			}
			if thisPeriodicConfig.Prefix == "" && thisPeriodicConfig.Descriptions == nil {
				continue
			}
			if thisPeriodicConfig.Prefix == "" || thisPeriodicConfig.Descriptions == nil {
				return nil, fmt.Errorf("invalid perfdash config of periodic %q: none or both of prefix and job type must be specified", periodic.Name)
			}
			jobs[periodic.Name] = thisPeriodicConfig
		}
	}
	fmt.Printf("Read configs with %d jobs\n", len(jobs))
	return jobs, nil
}
<commit_msg>Skip fetching metrics by perfdash for misconfigured jobs<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"strings"

	"github.com/ghodss/yaml"
)

// To add new e2e test support, you need to:
//   1) Transform e2e performance test result into *PerfData* in k8s/kubernetes/test/e2e/perftype,
//   and print the PerfData in e2e test log.
//   2) Add corresponding bucket, job and test into *TestConfig*.

// TestDescription contains test name, output file prefix and parser function.
type TestDescription struct {
	Name             string
	OutputFilePrefix string
	Parser           func(data []byte, buildNumber int, testResult *BuildData)
}

// TestDescriptions is a map job->component->description.
type TestDescriptions map[string]map[string][]TestDescription

// Tests is a map from test label to test description.
type Tests struct {
	Prefix       string
	Descriptions TestDescriptions
	BuildsCount  int
}

// Jobs is a map from job name to all supported tests in the job.
type Jobs map[string]Tests

// Buckets is a map from bucket url to all supported jobs in the bucket.
type Buckets map[string]Jobs

var (
	// performanceDescriptions contains metrics exported by a --ginko.focus=[Feature:Performance]
	// e2e test
	performanceDescriptions = TestDescriptions{
		"E2E": {
			"DensityResources": []TestDescription{{
				Name:             "density",
				OutputFilePrefix: "ResourceUsageSummary",
				Parser:           parseResourceUsageData,
			}},
			"DensityPodStartup": []TestDescription{{
				Name:             "density",
				OutputFilePrefix: "PodStartupLatency_PodStartupLatency",
				Parser:           parsePerfData,
			}},
			"DensitySaturationPodStartup": []TestDescription{{
				Name:             "density",
				OutputFilePrefix: "PodStartupLatency_SaturationPodStartupLatency",
				Parser:           parsePerfData,
			}},
			"LoadResources": []TestDescription{{
				Name:             "load",
				OutputFilePrefix: "ResourceUsageSummary",
				Parser:           parseResourceUsageData,
			}},
		},
		"APIServer": {
			"DensityResponsiveness": []TestDescription{{
				Name:             "density",
				OutputFilePrefix: "APIResponsiveness",
				Parser:           parsePerfData,
			}},
			"DensityRequestCount": []TestDescription{{
				Name:             "density",
				OutputFilePrefix: "APIResponsiveness",
				Parser:           parseRequestCountData,
			}},
			"DensityResponsiveness_Prometheus": []TestDescription{{
				Name:             "density",
				OutputFilePrefix: "APIResponsivenessPrometheus",
				Parser:           parsePerfData,
			}},
			"DensityRequestCount_Prometheus": []TestDescription{{
				Name:             "density",
				OutputFilePrefix: "APIResponsivenessPrometheus",
				Parser:           parseRequestCountData,
			}},
			"DensityRequestCountByClient": []TestDescription{{
				Name:             "density",
				OutputFilePrefix: "MetricsForE2E",
				Parser:           parseApiserverRequestCount,
			}},
			"DensityInitEventsCount": []TestDescription{{
				Name:             "density",
				OutputFilePrefix: "MetricsForE2E",
				Parser:           parseApiserverInitEventsCount,
			}},
			"LoadResponsiveness": []TestDescription{{
				Name:             "load",
				OutputFilePrefix: "APIResponsiveness",
				Parser:           parsePerfData,
			}},
			"LoadRequestCount": []TestDescription{{
				Name:             "load",
				OutputFilePrefix: "APIResponsiveness",
				Parser:           parseRequestCountData,
			}},
			"LoadResponsiveness_Prometheus": []TestDescription{{
				Name:             "load",
				OutputFilePrefix: "APIResponsivenessPrometheus",
				Parser:           parsePerfData,
			}},
			"LoadRequestCount_Prometheus": []TestDescription{{
				Name:             "load",
				OutputFilePrefix: "APIResponsivenessPrometheus",
				Parser:           parseRequestCountData,
			}},
			"LoadRequestCountByClient": []TestDescription{{
				Name:             "load",
				OutputFilePrefix: "MetricsForE2E",
				Parser:           parseApiserverRequestCount,
			}},
			"LoadInitEventsCount": []TestDescription{{
				Name:             "load",
				OutputFilePrefix: "MetricsForE2E",
				Parser:           parseApiserverInitEventsCount,
			}},
		},
		"Scheduler": {
			"SchedulingLatency": []TestDescription{{
				Name:             "density",
				OutputFilePrefix: "SchedulingMetrics",
				Parser:           parseSchedulingLatency,
			}},
			"SchedulingThroughput": []TestDescription{{
				Name:             "density",
				OutputFilePrefix: "SchedulingThroughput",
				Parser:           parseSchedulingThroughputCL,
			}},
		},
		"Etcd": {
			"DensityBackendCommitDuration": []TestDescription{{
				Name:             "density",
				OutputFilePrefix: "EtcdMetrics",
				Parser:           parseHistogramMetric("backendCommitDuration"),
			}},
			"DensitySnapshotSaveTotalDuration": []TestDescription{{
				Name:             "density",
				OutputFilePrefix: "EtcdMetrics",
				Parser:           parseHistogramMetric("snapshotSaveTotalDuration"),
			}},
			"DensityPeerRoundTripTime": []TestDescription{{
				Name:             "density",
				OutputFilePrefix: "EtcdMetrics",
				Parser:           parseHistogramMetric("peerRoundTripTime"),
			}},
			"DensityWalFsyncDuration": []TestDescription{{
				Name:             "density",
				OutputFilePrefix: "EtcdMetrics",
				Parser:           parseHistogramMetric("walFsyncDuration"),
			}},
			"LoadBackendCommitDuration": []TestDescription{{
				Name:             "load",
				OutputFilePrefix: "EtcdMetrics",
				Parser:           parseHistogramMetric("backendCommitDuration"),
			}},
			"LoadSnapshotSaveTotalDuration": []TestDescription{{
				Name:             "load",
				OutputFilePrefix: "EtcdMetrics",
				Parser:           parseHistogramMetric("snapshotSaveTotalDuration"),
			}},
			"LoadPeerRoundTripTime": []TestDescription{{
				Name:             "load",
				OutputFilePrefix: "EtcdMetrics",
				Parser:           parseHistogramMetric("peerRoundTripTime"),
			}},
			"LoadWalFsyncDuration": []TestDescription{{
				Name:             "load",
				OutputFilePrefix: "EtcdMetrics",
				Parser:           parseHistogramMetric("walFsyncDuration"),
			}},
		},
		"Network": {
			"Load_NetworkProgrammingLatency": []TestDescription{{
				Name:             "load",
				OutputFilePrefix: "NetworkProgrammingLatency",
				Parser:           parsePerfData,
			}},

			"Load_NetworkLatency": []TestDescription{
				{
					// TODO(oxddr): remove this around Sep '19 when we stop showing old data
					Name:             "load",
					OutputFilePrefix: "in_cluster_network_latency",
					Parser:           parsePerfData,
				}, {
					Name:             "load",
					OutputFilePrefix: "InClusterNetworkLatency",
					Parser:           parsePerfData,
				}},

			"Density_NetworkLatency": []TestDescription{
				{
					// TODO(oxddr): remove this around Sep '19 when we stop showing old data
					Name:             "density",
					OutputFilePrefix: "in_cluster_network_latency",
					Parser:           parsePerfData,
				}, {
					Name:             "density",
					OutputFilePrefix: "InClusterNetworkLatency",
					Parser:           parsePerfData,
				}},
		},
		"DNS": {
			"Load_DNSLookupLatency": []TestDescription{{
				Name:             "load",
				OutputFilePrefix: "DnsLookupLatency",
				Parser:           parsePerfData,
			}},
			"Density_DNSLookupLatency": []TestDescription{{
				Name:             "density",
				OutputFilePrefix: "DnsLookupLatency",
				Parser:           parsePerfData,
			}},
		},
	}

	// benchmarkDescriptions contains metrics exported by test/integration/scheduler_perf
	benchmarkDescriptions = TestDescriptions{
		"Scheduler": {
			"BenchmarkResults": []TestDescription{{
				Name:             "benchmark",
				OutputFilePrefix: "BenchmarkResults",
				Parser:           parsePerfData,
			}},
		},
	}

	dnsBenchmarkDescriptions = TestDescriptions{
		"dns": {
			"Latency": []TestDescription{{
				Name:             "dns",
				OutputFilePrefix: "Latency",
				Parser:           parsePerfData,
			}},
			"LatencyPerc": []TestDescription{{
				Name:             "dns",
				OutputFilePrefix: "LatencyPerc",
				Parser:           parsePerfData,
			}},
			"Queries": []TestDescription{{
				Name:             "dns",
				OutputFilePrefix: "Queries",
				Parser:           parsePerfData,
			}},
			"Qps": []TestDescription{{
				Name:             "dns",
				OutputFilePrefix: "Qps",
				Parser:           parsePerfData,
			}},
		},
	}

	storageDescriptions = TestDescriptions{
		"APIServer": {
			"Responsiveness": []TestDescription{
				{
					Name:             "pod-with-ephemeral-volume-startup-latency",
					OutputFilePrefix: "APIResponsiveness",
					Parser:           parsePerfData,
				},
				{
					Name:             "storage-",
					OutputFilePrefix: "APIResponsiveness",
					Parser:           parsePerfData,
				},
			},
			"RequestCount": []TestDescription{
				{
					Name:             "pod-with-ephemeral-volume-startup-latency",
					OutputFilePrefix: "APIResponsiveness",
					Parser:           parseRequestCountData,
				},
				{
					Name:             "storage-",
					OutputFilePrefix: "APIResponsiveness",
					Parser:           parseRequestCountData,
				},
			},
		},
		"E2E": {
			"PodStartup": []TestDescription{
				{
					Name:             "pod-with-ephemeral-volume-startup-latency",
					OutputFilePrefix: "PodStartupLatency_PodWithMultiVolumeStartupLatency",
					Parser:           parsePerfData,
				},
				{
					Name:             "storage-",
					OutputFilePrefix: "PodStartupLatency_PodWithVolumesStartupLatency",
					Parser:           parsePerfData,
				},
			},
		},
	}

	jobTypeToDescriptions = map[string]TestDescriptions{
		"performance":  performanceDescriptions,
		"benchmark":    benchmarkDescriptions,
		"dnsBenchmark": dnsBenchmarkDescriptions,
		"storage":      storageDescriptions,
	}
)

func getProwConfigOrDie(configPaths []string) Jobs {
	jobs, err := getProwConfig(configPaths)
	if err != nil {
		panic(err)
	}
	return jobs
}

// Minimal subset of the prow config definition at k8s.io/test-infra/prow/config
type config struct {
	Periodics []periodic `json:"periodics"`
}
type periodic struct {
	Name string   `json:"name"`
	Tags []string `json:"tags"`
}

func urlConfigRead(url string) ([]byte, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, fmt.Errorf("error fetching prow config from %s: %v", url, err)
	}
	defer resp.Body.Close()
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("error reading prow config from %s: %v", url, err)
	}
	return b, nil
}

func fileConfigRead(path string) ([]byte, error) {
	return ioutil.ReadFile(path)
}

func getProwConfig(configPaths []string) (Jobs, error) {
	jobs := Jobs{}

	for _, configPath := range configPaths {
		fmt.Fprintf(os.Stderr, "Fetching config %s\n", configPath)
		// Perfdash supports only yamls.
		if !strings.HasSuffix(configPath, ".yaml") {
			fmt.Fprintf(os.Stderr, "%s is not an yaml file!\n", configPath)
			continue
		}
		var content []byte
		var err error
		switch {
		case strings.HasPrefix(configPath, "http://"), strings.HasPrefix(configPath, "https://"):
			content, err = urlConfigRead(configPath)
		default:
			content, err = fileConfigRead(configPath)
		}
		if err != nil {
			return nil, err
		}
		conf := &config{}
		if err := yaml.Unmarshal(content, conf); err != nil {
			return nil, fmt.Errorf("error unmarshaling prow config from %s: %v", configPath, err)
		}
		for _, periodic := range conf.Periodics {
			config, err := parsePeriodicConfig(periodic)
			if err != nil {
				fmt.Fprintf(os.Stderr, "warning: failed to parse config of %q due to: %v\n",
					periodic.Name, err)
				continue
			}
			shouldUse, err := validatePeriodicConfig(config)
			if err != nil {
				fmt.Fprintf(os.Stderr, "warning: failed to validate config of %q due to: %v\n",
					periodic.Name, err)
				continue
			}
			if shouldUse {
				jobs[periodic.Name] = config
			}
		}
	}
	fmt.Printf("Read configs with %d jobs\n", len(jobs))
	return jobs, nil
}

func parsePeriodicConfig(periodic periodic) (Tests, error) {
	var thisPeriodicConfig Tests
	for _, tag := range periodic.Tags {
		if strings.HasPrefix(tag, "perfDashPrefix:") {
			split := strings.SplitN(tag, ":", 2)
			thisPeriodicConfig.Prefix = strings.TrimSpace(split[1])
			continue
		}
		if strings.HasPrefix(tag, "perfDashJobType:") {
			split := strings.SplitN(tag, ":", 2)
			jobType := strings.TrimSpace(split[1])
			var exists bool
			if thisPeriodicConfig.Descriptions, exists = jobTypeToDescriptions[jobType]; !exists {
				return Tests{}, fmt.Errorf("unknown job type - %s", jobType)
			}
			continue
		}
		if strings.HasPrefix(tag, "perfDashBuildsCount:") {
			split := strings.SplitN(tag, ":", 2)
			i, err := strconv.Atoi(strings.TrimSpace(split[1]))
			if err != nil {
				return Tests{}, fmt.Errorf("unparsable builds count - %v", split[1])
			}
			if i < 1 {
				return Tests{}, fmt.Errorf("non-positive builds count - %v", i)
			}
			thisPeriodicConfig.BuildsCount = i
			continue
		}
		if strings.HasPrefix(tag, "perfDash") {
			return Tests{}, fmt.Errorf("unknown perfdash tag name: %q", tag)
		}
	}
	return thisPeriodicConfig, nil
}

func validatePeriodicConfig(config Tests) (shouldUse bool, err error) {
	if config.Prefix == "" && config.Descriptions == nil {
		// this is expected case for jobs which are not expected to be visible in perfdash
		return false, nil
	}
	if config.Prefix == "" || config.Descriptions == nil {
		return false, fmt.Errorf("nonee or both of prefix and job type must be specified")
	}
	return true, nil
}
<|endoftext|>


<commit_before>// Copyright 2017 The casbin Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package persist

import (
	"strings"

	"github.com/casbin/casbin/model"
)

func loadPolicyLine(line string, model model.Model) {
	if line == "" {
		return
	}

	tokens := strings.Split(line, ", ")

	key := tokens[0]
	sec := key[:1]
	model[sec][key].Policy = append(model[sec][key].Policy, tokens[1:])
}

// Adapter represents the abstract adapter interface for policy persistence.
// FileAdapter, DBAdapter inherits this interface.
type Adapter interface {
	LoadPolicy(model model.Model)
	SavePolicy(model model.Model)
}
<commit_msg>Allow # comment in policy file.<commit_after>// Copyright 2017 The casbin Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package persist

import (
	"strings"

	"github.com/casbin/casbin/model"
)

func loadPolicyLine(line string, model model.Model) {
	if line == "" {
		return
	}

	if strings.HasPrefix(line, "#") {
		return
	}

	tokens := strings.Split(line, ", ")

	key := tokens[0]
	sec := key[:1]
	model[sec][key].Policy = append(model[sec][key].Policy, tokens[1:])
}

// Adapter represents the abstract adapter interface for policy persistence.
// FileAdapter, DBAdapter inherits this interface.
type Adapter interface {
	LoadPolicy(model model.Model)
	SavePolicy(model model.Model)
}
<|endoftext|>


<commit_before>package ApkInfGo

import (
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"regexp"
	"runtime"
	"strconv"
	"strings"
)

type ApkInfoSt struct {
	Name               string
	VersionCode        uint32
	VersionName        string
	Label              string
	Icon               string
	SdkVersion         uint16
	TargetSdkVersion   uint16
	NativeCode         string
	FileSize           int64
	FilePath           string
	Cert               ApkCertSt
	LaunchableActivity string
}

type Conf struct {
	aapt string
	cert *ConfCert
}

func ApkInfo(aaptApp string) *Conf {
	c := &Conf{aapt: aaptApp, cert: nil}
	return c
}

func (c *Conf) CertKeyTool(keytoolApp string) *Conf {
	app := ApkCertificate(keytoolApp)
	c.cert = app
	return c
}

func (c *Conf) File(apk string) *ApkInfoSt {
	d := parse(c, apk)
	if d != nil {
		file, _ := os.Stat(apk)
		d.FileSize = file.Size()
	}
	return d
}

func getLineSeparator() string {
	if runtime.GOOS == "windows" {
		return "\r\n"
	}
	return "\n"
}

func parse(c *Conf, apk string) *ApkInfoSt {
	out, err := exec.Command(c.aapt, "dump", "badging", apk).Output()
	if err != nil {
		log.Printf("err: %q, file: %q", err, apk)
		return nil
	}
	//log.Printf("apk file - %q\n", apk)
	data := strings.Split(string(out), getLineSeparator())
	info := ApkInfoSt{FilePath: apk}
	for _, s := range data {
		arr := strings.Split(s, ":")
		if len(arr) != 2 {
			//log.Printf("error split - %q\n", s)
			continue
		}
		switch arr[0] {
		case "package":
			//log.Printf("package - %q\n", arr[1])
			re := regexp.MustCompile("name='([^']+)?' versionCode='(\\d*)?' versionName='([^']+)?'")
			packageInfo := re.FindStringSubmatch(arr[1])
			info.Name = packageInfo[1]
			info.VersionName = packageInfo[3]
			versionCode, _ := strconv.ParseUint(packageInfo[2], 0, 32)
			info.VersionCode = uint32(versionCode)
		case "launchable-activity":
			re := regexp.MustCompile("name='([^']+)?'")
			launchInfo := re.FindStringSubmatch(arr[1])
			info.LaunchableActivity = launchInfo[1]
		case "sdkVersion":
			//log.Printf("sdkVersion - %q\n", arr[1])
			sdkVersion, _ := strconv.ParseUint(strings.Trim(arr[1], "'"), 0, 16)
			info.SdkVersion = uint16(sdkVersion)
		case "targetSdkVersion":
			//log.Printf("targetSdkVersion - %q\n", arr[1])
			targetSdkVersion, _ := strconv.ParseUint(strings.Trim(arr[1], "'"), 0, 16)
			info.TargetSdkVersion = uint16(targetSdkVersion)
		case "native-code":
			nativeCode := strings.Trim(strings.TrimSpace(arr[1]), "'")
			info.NativeCode = nativeCode
		case "application":
			//log.Printf("application - %q\n", arr[1])
			re2 := regexp.MustCompile("label='(.*)?' icon='([^']+)?'")
			d := re2.FindStringSubmatch(arr[1])
			info.Label = d[1]
			info.Icon = d[2]
		}
	}
	if c.cert != nil {
		info.Cert = *c.cert.File(apk)
	}
	return &info
}

func (c *Conf) Folder(dirname string, recurcive bool) *[]ApkInfoSt {
	files, err := ioutil.ReadDir(dirname)
	if err != nil {
		log.Printf("err: %q", err)
		return nil
	}
	//log.Printf("apk folder - %q\n", dirname)
	infoArr := make([]ApkInfoSt, 0)
	re := regexp.MustCompile(".*\\.apk$")
	for _, file := range files {
		if re.MatchString(file.Name()) {
			dir := dirname + string(os.PathSeparator) + file.Name()
			a := parse(c, dir)
			if a != nil {
				a.FileSize = file.Size()
				infoArr = append(infoArr, *a)
			}
		} else if file.IsDir() && recurcive {
			dir := dirname + string(os.PathSeparator) + file.Name()
			//log.Printf("apk subfolder - %q\n", dir)
			arr := (c).Folder(dir, true)
			if arr != nil {
				for _, a := range *arr {
					infoArr = append(infoArr, a)
				}
			}
		}
	}
	return &infoArr
}
<commit_msg>added label search in two lines<commit_after>package ApkInfGo

import (
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"regexp"
	"runtime"
	"strconv"
	"strings"
)

type ApkInfoSt struct {
	Name               string
	VersionCode        uint32
	VersionName        string
	Label              string
	Icon               string
	SdkVersion         uint16
	TargetSdkVersion   uint16
	NativeCode         string
	FileSize           int64
	FilePath           string
	Cert               ApkCertSt
	LaunchableActivity string
}

type Conf struct {
	aapt string
	cert *ConfCert
}

func ApkInfo(aaptApp string) *Conf {
	c := &Conf{aapt: aaptApp, cert: nil}
	return c
}

func (c *Conf) CertKeyTool(keytoolApp string) *Conf {
	app := ApkCertificate(keytoolApp)
	c.cert = app
	return c
}

func (c *Conf) File(apk string) *ApkInfoSt {
	d := parse(c, apk)
	if d != nil {
		file, _ := os.Stat(apk)
		d.FileSize = file.Size()
	}
	return d
}

func getLineSeparator() string {
	if runtime.GOOS == "windows" {
		return "\r\n"
	}
	return "\n"
}

func parse(c *Conf, apk string) *ApkInfoSt {
	out, err := exec.Command(c.aapt, "dump", "badging", apk).Output()
	if err != nil {
		log.Printf("err: %q, file: %q", err, apk)
		return nil
	}
	//log.Printf("apk file - %q\n", apk)
	data := strings.Split(string(out), getLineSeparator())
	info := ApkInfoSt{FilePath: apk}
	label2 := ""
	for _, s := range data {
		arr := strings.Split(s, ":")
		if len(arr) != 2 {
			//log.Printf("error split - %q\n", s)
			continue
		}
		switch arr[0] {
		case "package":
			//log.Printf("package - %q\n", arr[1])
			re := regexp.MustCompile("name='([^']+)?' versionCode='(\\d*)?' versionName='([^']+)?'")
			packageInfo := re.FindStringSubmatch(arr[1])
			info.Name = packageInfo[1]
			info.VersionName = packageInfo[3]
			versionCode, _ := strconv.ParseUint(packageInfo[2], 0, 32)
			info.VersionCode = uint32(versionCode)
		case "launchable-activity":
			re := regexp.MustCompile("name='([^']+)?'\\s+label='(.*)?'\\s+icon='([^']*)?'")
			launchInfo := re.FindStringSubmatch(arr[1])
			info.LaunchableActivity = launchInfo[1]
			info.Label = launchInfo[2]
		case "sdkVersion":
			//log.Printf("sdkVersion - %q\n", arr[1])
			sdkVersion, _ := strconv.ParseUint(strings.Trim(arr[1], "'"), 0, 16)
			info.SdkVersion = uint16(sdkVersion)
		case "targetSdkVersion":
			//log.Printf("targetSdkVersion - %q\n", arr[1])
			targetSdkVersion, _ := strconv.ParseUint(strings.Trim(arr[1], "'"), 0, 16)
			info.TargetSdkVersion = uint16(targetSdkVersion)
		case "native-code":
			nativeCode := strings.Trim(strings.TrimSpace(arr[1]), "'")
			info.NativeCode = nativeCode
		case "application":
			//log.Printf("application - %q\n", arr[1])
			re2 := regexp.MustCompile("label='(.*)?' icon='([^']+)?'")
			d := re2.FindStringSubmatch(arr[1])
			label2 = d[1]
			info.Icon = d[2]
		}
	}
	// if label empty
	if len(info.Label) == 0 {
		info.Label = label2
	}

	if c.cert != nil {
		info.Cert = *c.cert.File(apk)
	}
	return &info
}

func (c *Conf) Folder(dirname string, recurcive bool) *[]ApkInfoSt {
	files, err := ioutil.ReadDir(dirname)
	if err != nil {
		log.Printf("err: %q", err)
		return nil
	}
	//log.Printf("apk folder - %q\n", dirname)
	infoArr := make([]ApkInfoSt, 0)
	re := regexp.MustCompile(".*\\.apk$")
	for _, file := range files {
		if re.MatchString(file.Name()) {
			dir := dirname + string(os.PathSeparator) + file.Name()
			a := parse(c, dir)
			if a != nil {
				a.FileSize = file.Size()
				infoArr = append(infoArr, *a)
			}
		} else if file.IsDir() && recurcive {
			dir := dirname + string(os.PathSeparator) + file.Name()
			//log.Printf("apk subfolder - %q\n", dir)
			arr := (c).Folder(dir, true)
			if arr != nil {
				for _, a := range *arr {
					infoArr = append(infoArr, a)
				}
			}
		}
	}
	return &infoArr
}
<|endoftext|>


<commit_before>package pixelutils

import (
	"github.com/Zwobot/go-resample/resample"
	"image"
	"image/draw"
)

func Copy(dst draw.Image, src image.Image, sr, dr image.Rectangle) {
	subRect := sr.Sub(sr.Min)
	subImg := image.NewRGBA(sr)
	draw.Draw(subImg, subRect, src, sr.Min, draw.Over)
	resizeImg, _ := resample.Resize(image.Point{dr.Dx(), dr.Dy()}, subImg)
	draw.Draw(dst, dr, resizeImg, image.Point{0, 0}, draw.Over)
}
<commit_msg>Add SubImager<commit_after>package pixelutils

import (
	"github.com/Zwobot/go-resample/resample"
	"image"
	"image/draw"
)

func Copy(dst draw.Image, src image.Image, sr, dr image.Rectangle) {
	subRect := sr.Sub(sr.Min)
	subImg := image.NewRGBA(sr)
	draw.Draw(subImg, subRect, src, sr.Min, draw.Over)
	resizeImg, _ := resample.Resize(image.Point{dr.Dx(), dr.Dy()}, subImg)
	draw.Draw(dst, dr, resizeImg, image.Point{0, 0}, draw.Over)
}

type SubImager interface {
	draw.Image
	SubImage(r image.Rectangle) image.Image
}

func SubImage(img SubImager, r image.Rectangle) draw.Image {
	return img.SubImage(r).(draw.Image)
}
<|endoftext|>


<commit_before>package radosAPI

import (
	"encoding/json"
	"errors"
	"net/url"
	"time"

	"github.com/QuentinPerez/go-encodeUrl"
)

// UsageConfig usage request
type UsageConfig struct {
	UID         string     `url:"uid,ifStringIsNotEmpty"`     // The user for which the information is requested. If not specified will apply to all users
	Start       *time.Time `url:"start,ifTimeIsNotNilCeph"`   // Date and (optional) time that specifies the start time of the requested data
	End         *time.Time `url:"end,ifTimeIsNotNilCeph"`     // Date and (optional) time that specifies the end time of the requested data (non-inclusive)
	ShowEntries bool       `url:"show-entries,ifBoolIsFalse"` // Specifies whether data entries should be returned.
	ShowSummary bool       `url:"show-summary,ifBoolIsFalse"` // Specifies whether data summary should be returned
	RemoveAll   bool       `url:"remove-all,ifBoolIsTrue"`    // Required when uid is not specified, in order to acknowledge multi user data removal.
}

// GetUsage requests bandwidth usage information.
//
// !! caps: usage=read !!
//
// @UID
// @Start
// @End
// @ShowEntries
// @ShowSummary
//
func (api *API) GetUsage(conf *UsageConfig) (*Usage, error) {
	var (
		ret    = &Usage{}
		values = url.Values{}
		errs   []error
	)

	values.Add("format", "json")
	if conf != nil {
		values, errs = encurl.Translate(conf)
		if len(errs) > 0 {
			return nil, errs[0]
		}
	}
	body, _, err := api.get("/admin/usage", values)
	if err != nil {
		return nil, err
	}
	if err = json.Unmarshal(body, &ret); err != nil {
		return nil, err
	}
	return ret, nil
}

// DeleteUsage removes usage information. With no dates specified, removes all usage information
//
// !! caps: usage=write !!
//
// @UID
// @Start
// @End
// @RemoveAll
//
func (api *API) DeleteUsage(conf *UsageConfig) error {
	var (
		values = url.Values{}
		errs   []error
	)

	values.Add("format", "json")
	if conf != nil {
		values, errs = encurl.Translate(conf)
		if len(errs) > 0 {
			return errs[0]
		}
	}
	_, _, err := api.delete("/admin/usage", values)
	if err != nil {
		return err
	}
	return nil
}

// GetUser gets user information. If no user is specified returns the list of all users along with suspension information
//
// !! caps: users=read !!
//
// @uid
//
func (api *API) GetUser(uid ...string) (*User, error) {
	ret := &User{}
	values := url.Values{}

	values.Add("format", "json")
	if len(uid) != 0 {
		values.Add("uid", uid[0])
	}
	body, _, err := api.get("/admin/user", values)
	if err != nil {
		return nil, err
	}
	if err = json.Unmarshal(body, &ret); err != nil {
		return nil, err
	}
	return ret, nil
}

// UserConfig user request
type UserConfig struct {
	UID         string `url:"uid,ifStringIsNotEmpty"`          // The user ID to be created
	DisplayName string `url:"display-name,ifStringIsNotEmpty"` // The display name of the user to be created
	Email       string `url:"email,ifStringIsNotEmpty"`        // The email address associated with the user
	KeyType     string `url:"key-type,ifStringIsNotEmpty"`     // Key type to be generated, options are: swift, s3 (default)
	AccessKey   string `url:"access-key,ifStringIsNotEmpty"`   // Specify access key
	SecretKey   string `url:"secret-key,ifStringIsNotEmpty"`   // Specify secret key
	UserCaps    string `url:"user-caps,ifStringIsNotEmpty"`    // User capabilities
	GenerateKey bool   `url:"generate-key,ifBoolIsTrue"`       // Generate a new key pair and add to the existing keyring
	MaxBuckets  int    `url:"max-buckets,itoa"`                // Specify the maximum number of buckets the user can own
	Suspended   bool   `url:"suspended,ifBoolIsTrue"`          // Specify whether the user should be suspended
	PurgeData   bool   `url:"suspended,ifBoolIsTrue"`          // Specify whether the user should be suspended
}

// CreateUser creates a new user. By Default, a S3 key pair will be created automatically and returned in the response.
// If only one of access-key or secret-key is provided, the omitted key will be automatically generated.
// By default, a generated key is added to the keyring without replacing an existing key pair.
// If access-key is specified and refers to an existing key owned by the user then it will be modified
//
// !! caps: users=write !!
//
// @UID
// @DisplayName
// @Email
// @KeyType
// @AccessKey
// @SecretKey
// @UserCaps
// @GenerateKey
// @MaxBuckets
// @Suspended
//
func (api *API) CreateUser(conf *UserConfig) (*User, error) {
	if conf == nil {
		return nil, errors.New("UserConfig must be not nil")
	}
	if conf.UID == "" {
		return nil, errors.New("UID field is required")
	}
	if conf.DisplayName == "" {
		return nil, errors.New("DisplayName field is required")
	}

	var (
		ret    = &User{}
		values = url.Values{}
		errs   []error
	)

	values.Add("format", "json")
	if conf != nil {
		values, errs = encurl.Translate(conf)
		if len(errs) > 0 {
			return nil, errs[0]
		}
	}
	body, _, err := api.put("/admin/user", values)
	if err != nil {
		return nil, err
	}
	if err = json.Unmarshal(body, &ret); err != nil {
		return nil, err
	}
	return ret, nil
}

// RemoveUser removes an existing user.
//
// !! caps: users=write !!
//
// @UID
// @PurgeData
//
func (api *API) RemoveUser(conf *UserConfig) error {
	if conf == nil {
		return errors.New("UserConfig must be not nil")
	}
	if conf.UID == "" {
		return errors.New("UID field is required")
	}
	var (
		values = url.Values{}
		errs   []error
	)

	values.Add("format", "json")
	if conf != nil {
		values, errs = encurl.Translate(conf)
		if len(errs) > 0 {
			return errs[0]
		}
	}
	_, _, err := api.delete("/admin/user", values)
	if err != nil {
		return err
	}
	return nil
}
<commit_msg>Remove pointer check<commit_after>package radosAPI

import (
	"encoding/json"
	"errors"
	"net/url"
	"time"

	"github.com/QuentinPerez/go-encodeUrl"
)

// UsageConfig usage request
type UsageConfig struct {
	UID         string     `url:"uid,ifStringIsNotEmpty"`     // The user for which the information is requested. If not specified will apply to all users
	Start       *time.Time `url:"start,ifTimeIsNotNilCeph"`   // Date and (optional) time that specifies the start time of the requested data
	End         *time.Time `url:"end,ifTimeIsNotNilCeph"`     // Date and (optional) time that specifies the end time of the requested data (non-inclusive)
	ShowEntries bool       `url:"show-entries,ifBoolIsFalse"` // Specifies whether data entries should be returned.
	ShowSummary bool       `url:"show-summary,ifBoolIsFalse"` // Specifies whether data summary should be returned
	RemoveAll   bool       `url:"remove-all,ifBoolIsTrue"`    // Required when uid is not specified, in order to acknowledge multi user data removal.
}

// GetUsage requests bandwidth usage information.
//
// !! caps: usage=read !!
//
// @UID
// @Start
// @End
// @ShowEntries
// @ShowSummary
//
func (api *API) GetUsage(conf UsageConfig) (*Usage, error) {
	var (
		ret    = &Usage{}
		values = url.Values{}
		errs   []error
	)

	values.Add("format", "json")
	values, errs = encurl.Translate(conf)
	if len(errs) > 0 {
		return nil, errs[0]
	}
	body, _, err := api.get("/admin/usage", values)
	if err != nil {
		return nil, err
	}
	if err = json.Unmarshal(body, &ret); err != nil {
		return nil, err
	}
	return ret, nil
}

// DeleteUsage removes usage information. With no dates specified, removes all usage information
//
// !! caps: usage=write !!
//
// @UID
// @Start
// @End
// @RemoveAll
//
func (api *API) DeleteUsage(conf UsageConfig) error {
	var (
		values = url.Values{}
		errs   []error
	)

	values.Add("format", "json")
	values, errs = encurl.Translate(conf)
	if len(errs) > 0 {
		return errs[0]
	}
	_, _, err := api.delete("/admin/usage", values)
	if err != nil {
		return err
	}
	return nil
}

// GetUser gets user information. If no user is specified returns the list of all users along with suspension information
//
// !! caps: users=read !!
//
// @uid
//
func (api *API) GetUser(uid ...string) (*User, error) {
	ret := &User{}
	values := url.Values{}

	values.Add("format", "json")
	if len(uid) != 0 {
		values.Add("uid", uid[0])
	}
	body, _, err := api.get("/admin/user", values)
	if err != nil {
		return nil, err
	}
	if err = json.Unmarshal(body, &ret); err != nil {
		return nil, err
	}
	return ret, nil
}

// UserConfig user request
type UserConfig struct {
	UID         string `url:"uid,ifStringIsNotEmpty"`          // The user ID to be created
	DisplayName string `url:"display-name,ifStringIsNotEmpty"` // The display name of the user to be created
	Email       string `url:"email,ifStringIsNotEmpty"`        // The email address associated with the user
	KeyType     string `url:"key-type,ifStringIsNotEmpty"`     // Key type to be generated, options are: swift, s3 (default)
	AccessKey   string `url:"access-key,ifStringIsNotEmpty"`   // Specify access key
	SecretKey   string `url:"secret-key,ifStringIsNotEmpty"`   // Specify secret key
	UserCaps    string `url:"user-caps,ifStringIsNotEmpty"`    // User capabilities
	GenerateKey bool   `url:"generate-key,ifBoolIsTrue"`       // Generate a new key pair and add to the existing keyring
	MaxBuckets  int    `url:"max-buckets,itoa"`                // Specify the maximum number of buckets the user can own
	Suspended   bool   `url:"suspended,ifBoolIsTrue"`          // Specify whether the user should be suspended
	PurgeData   bool   `url:"suspended,ifBoolIsTrue"`          // Specify whether the user should be suspended
}

// CreateUser creates a new user. By Default, a S3 key pair will be created automatically and returned in the response.
// If only one of access-key or secret-key is provided, the omitted key will be automatically generated.
// By default, a generated key is added to the keyring without replacing an existing key pair.
// If access-key is specified and refers to an existing key owned by the user then it will be modified
//
// !! caps: users=write !!
//
// @UID
// @DisplayName
// @Email
// @KeyType
// @AccessKey
// @SecretKey
// @UserCaps
// @GenerateKey
// @MaxBuckets
// @Suspended
//
func (api *API) CreateUser(conf UserConfig) (*User, error) {
	if conf.UID == "" {
		return nil, errors.New("UID field is required")
	}
	if conf.DisplayName == "" {
		return nil, errors.New("DisplayName field is required")
	}

	var (
		ret    = &User{}
		values = url.Values{}
		errs   []error
	)

	values.Add("format", "json")
	values, errs = encurl.Translate(conf)
	if len(errs) > 0 {
		return nil, errs[0]
	}
	body, _, err := api.put("/admin/user", values)
	if err != nil {
		return nil, err
	}
	if err = json.Unmarshal(body, &ret); err != nil {
		return nil, err
	}
	return ret, nil
}

// RemoveUser removes an existing user.
//
// !! caps: users=write !!
//
// @UID
// @PurgeData
//
func (api *API) RemoveUser(conf UserConfig) error {
	if conf.UID == "" {
		return errors.New("UID field is required")
	}
	var (
		values = url.Values{}
		errs   []error
	)

	values.Add("format", "json")
	values, errs = encurl.Translate(conf)
	if len(errs) > 0 {
		return errs[0]
	}
	_, _, err := api.delete("/admin/user", values)
	if err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package cli

import (
	"errors"
	"fmt"
	"io"
	"os"

	"github.com/apprenda/kismatic/pkg/data"
	"github.com/apprenda/kismatic/pkg/install"
	"github.com/apprenda/kismatic/pkg/util"
	"github.com/spf13/cobra"
)

type upgradeOpts struct {
	generatedAssetsDir string
	verbose            bool
	outputFormat       string
	skipPreflight      bool
	online             bool
	planFile           string
	restartServices    bool
	partialAllowed     bool
}

// NewCmdUpgrade returns the upgrade command
func NewCmdUpgrade(out io.Writer) *cobra.Command {
	var opts upgradeOpts
	cmd := &cobra.Command{
		Use:   "upgrade",
		Short: "Upgrade your Kubernetes cluster",
		Long: `Upgrade your Kubernetes cluster.

The upgrade process is applied to each node, one node at a time. If a private docker registry
is being used, the new container images will be pushed by Kismatic before starting to upgrade
nodes.

Nodes in the cluster are upgraded in the following order:

1. Etcd nodes
2. Master nodes
3. Worker nodes (regardless of specialization)
`,
		RunE: func(cmd *cobra.Command, args []string) error {
			return cmd.Help()
		},
	}

	cmd.PersistentFlags().StringVar(&opts.generatedAssetsDir, "generated-assets-dir", "generated", "path to the directory where assets generated during the installation process will be stored")
	cmd.PersistentFlags().BoolVar(&opts.verbose, "verbose", false, "enable verbose logging from the installation")
	cmd.PersistentFlags().StringVarP(&opts.outputFormat, "output", "o", "simple", "installation output format (options \"simple\"|\"raw\")")
	cmd.PersistentFlags().BoolVar(&opts.skipPreflight, "skip-preflight", false, "skip upgrade pre-flight checks")
	cmd.PersistentFlags().BoolVar(&opts.restartServices, "restart-services", false, "force restart cluster services (Use with care)")
	cmd.PersistentFlags().BoolVar(&opts.partialAllowed, "partial-ok", false, "allow the upgrade of ready nodes, and skip nodes that have been deemed unready for upgrade")
	addPlanFileFlag(cmd.PersistentFlags(), &opts.planFile)

	// Subcommands
	cmd.AddCommand(NewCmdUpgradeOffline(out, &opts))
	cmd.AddCommand(NewCmdUpgradeOnline(out, &opts))
	return cmd
}

// NewCmdUpgradeOffline returns the command for running offline upgrades
func NewCmdUpgradeOffline(out io.Writer, opts *upgradeOpts) *cobra.Command {
	cmd := cobra.Command{
		Use:   "offline",
		Short: "Perform an offline upgrade of your Kubernetes cluster",
		Long: `Perform an offline upgrade of your Kubernetes cluster.

The offline upgrade is available for those clusters in which safety and availabilty are not a concern.
In this mode, the safety and availability checks will not be performed, nor will the nodes in the cluster
be drained of workloads.

Performing an offline upgrade could result in loss of critical data and reduced service
availability. For this reason, this method should not be used for clusters that are housing
production workloads.
`,
		RunE: func(cmd *cobra.Command, args []string) error {
			return doUpgrade(out, opts)
		},
	}
	return &cmd
}

// NewCmdUpgradeOnline returns the command for running online upgrades
func NewCmdUpgradeOnline(out io.Writer, opts *upgradeOpts) *cobra.Command {
	cmd := cobra.Command{
		Use:   "online",
		Short: "Perform an online upgrade of your Kubernetes cluster",
		Long: `Perform an online upgrade of your Kubernetes cluster.

During an online upgrade, Kismatic will run safety and availability checks (see table below) against the
existing cluster before performing the upgrade. If any unsafe condition is detected, a report will
be printed, and the upgrade will not proceed.

If the node under upgrade is a Kubernetes node, it is cordoned and drained of workloads
before any changes are applied.
`,
		RunE: func(cmd *cobra.Command, args []string) error {
			opts.online = true
			return doUpgrade(out, opts)
		},
	}
	return &cmd
}

func doUpgrade(out io.Writer, opts *upgradeOpts) error {
	planFile := opts.planFile
	planner := install.FilePlanner{File: planFile}
	executorOpts := install.ExecutorOptions{
		GeneratedAssetsDirectory: opts.generatedAssetsDir,
		RestartServices:          opts.restartServices,
		OutputFormat:             opts.outputFormat,
		Verbose:                  opts.verbose,
	}
	executor, err := install.NewExecutor(out, os.Stderr, executorOpts)
	if err != nil {
		return err
	}
	util.PrintHeader(out, "Computing upgrade plan", '=')

	// Read plan file
	if !planner.PlanExists() {
		util.PrettyPrintErr(out, "Reading plan file")
		return fmt.Errorf("plan file %q does not exist", planFile)
	}
	util.PrettyPrintOk(out, "Reading plan file")
	plan, err := planner.Read()
	if err != nil {
		util.PrettyPrintErr(out, "Reading plan file")
		return fmt.Errorf("error reading plan file %q: %v", planFile, err)
	}

	// Validate SSH connectivity to nodes
	if ok, errs := install.ValidatePlanSSHConnections(plan); !ok {
		util.PrettyPrintErr(out, "Validate SSH connectivity to nodes")
		util.PrintValidationErrors(out, errs)
		return fmt.Errorf("SSH connectivity validation errors found")
	}
	util.PrettyPrintOk(out, "Validate SSH connectivity to nodes")

	// Figure out which nodes to upgrade
	cv, err := install.ListVersions(plan)
	if err != nil {
		return fmt.Errorf("error listing cluster versions: %v", err)
	}
	var toUpgrade []install.ListableNode
	var toSkip []install.ListableNode
	for _, n := range cv.Nodes {
		if install.IsOlderVersion(n.Version) {
			toUpgrade = append(toUpgrade, n)
		} else {
			toSkip = append(toSkip, n)
		}
	}

	// Print the nodes that will be skipped
	if len(toSkip) > 0 {
		util.PrintHeader(out, "Skipping nodes", '=')
		for _, n := range toSkip {
			util.PrettyPrintOk(out, "- %q is at the target version %q", n.Node.Host, n.Version)
		}
		fmt.Fprintln(out)
	}

	if plan.ConfigureDockerRegistry() && plan.Cluster.DisconnectedInstallation {
		util.PrintHeader(out, "Upgrade Docker Registry", '=')
		if err := executor.UpgradeDockerRegistry(*plan); err != nil {
			return fmt.Errorf("Failed to upgrade docker registry: %v", err)
		}
	}

	if plan.ConfigureDockerRegistry() && plan.Cluster.DisconnectedInstallation {
		util.PrintHeader(out, "Upgrade Docker Registry", '=')
		if err := executor.UpgradeDockerRegistry(*plan); err != nil {
			return fmt.Errorf("Failed to upgrade docker registry: %v", err)
		}
	}

	// Print message if there's no work to do
	if len(toUpgrade) == 0 {
		fmt.Fprintln(out, "All nodes are at the target version. Skipping node upgrades.")
	} else {
		if err = upgradeNodes(out, *plan, *opts, toUpgrade, executor); err != nil {
			return err
		}
	}

	if opts.partialAllowed {
		util.PrintColor(out, util.Green, `

Partial upgrade complete.

Cluster level services are still left to upgrade. These can only be upgraded
when performing a full upgrade. When you are ready, you may use "kismatic upgrade"
without the "--partial-ok" flag to perform a full upgrade.

`)
		return nil
	}

	// Upgrade the cluster services
	util.PrintHeader(out, "Upgrade Cluster Services", '=')
	if err := executor.UpgradeClusterServices(*plan); err != nil {
		return fmt.Errorf("Failed to upgrade cluster services: %v", err)
	}

	if err := executor.RunSmokeTest(plan); err != nil {
		return fmt.Errorf("Smoke test failed: %v", err)
	}

	fmt.Fprintln(out)
	util.PrintColor(out, util.Green, "Upgrade complete\n")
	fmt.Fprintln(out)
	return nil
}

func upgradeNodes(out io.Writer, plan install.Plan, opts upgradeOpts, nodesNeedUpgrade []install.ListableNode, executor install.Executor) error {
	// Run safety checks if doing an online upgrade
	unsafeNodes := []install.ListableNode{}
	if opts.online {
		util.PrintHeader(out, "Validate Online Upgrade", '=')
		// Use the first master node for running kubectl
		client, err := plan.GetSSHClient(plan.Master.Nodes[0].Host)
		if err != nil {
			return fmt.Errorf("error getting SSH client: %v", err)
		}
		kubeClient := data.RemoteKubectl{SSHClient: client}
		for _, node := range nodesNeedUpgrade {
			util.PrettyPrint(out, "Node %q", node.Node.Host)
			errs := install.DetectNodeUpgradeSafety(plan, node.Node, kubeClient)
			if len(errs) != 0 {
				util.PrintError(out)
				fmt.Fprintln(out)
				for _, err := range errs {
					fmt.Println("-", err.Error())
				}
				unsafeNodes = append(unsafeNodes, node)
			} else {
				util.PrintOkln(out)
			}
		}
		// If we found any unsafe nodes, and we are not doing a partial upgrade, exit.
		if len(unsafeNodes) > 0 && !opts.partialAllowed {
			return errors.New("Unable to perform an online upgrade due to the unsafe conditions detected.")
		}
		// Block the upgrade if partial is allowed but there is an etcd or master node
		// that cannot be upgraded
		if opts.partialAllowed {
			for _, n := range unsafeNodes {
				for _, r := range n.Roles {
					if r == "master" || r == "etcd" {
						return errors.New("Unable to perform an online upgrade due to the unsafe conditions detected.")
					}
				}
			}
		}
	}

	// Run upgrade preflight on the nodes that are to be upgraded
	unreadyNodes := []install.ListableNode{}
	if !opts.skipPreflight {
		for _, node := range nodesNeedUpgrade {
			util.PrintHeader(out, fmt.Sprintf("Preflight Checks: %s %s", node.Node.Host, node.Roles), '=')
			if err := executor.RunUpgradePreFlightCheck(&plan, node); err != nil {
				// return fmt.Errorf("Upgrade preflight check failed: %v", err)
				unreadyNodes = append(unreadyNodes, node)
			}
		}
	}

	// Block upgrade if we found unready nodes, and we are not doing a partial upgrade
	if len(unreadyNodes) > 0 && !opts.partialAllowed {
		return errors.New("Errors found during preflight checks")
	}

	// Block the upgrade if partial is allowed but there is an etcd or master node
	// that cannot be upgraded
	if opts.partialAllowed {
		for _, n := range unreadyNodes {
			for _, r := range n.Roles {
				if r == "master" || r == "etcd" {
					return errors.New("Errors found during preflight checks")
				}
			}
		}
	}

	// Filter out the nodes that are unsafe/unready
	toUpgrade := []install.ListableNode{}
	for _, n := range nodesNeedUpgrade {
		upgrade := true
		for _, unsafe := range unsafeNodes {
			if unsafe.Node == n.Node {
				upgrade = false
			}
		}
		for _, unready := range unreadyNodes {
			if unready.Node == n.Node {
				upgrade = false
			}
		}
		if upgrade {
			toUpgrade = append(toUpgrade, n)
		}
	}

	// get all etcd nodes
	etcdToUpgrade := install.NodesWithRoles(toUpgrade, "etcd")
	// it's safe to upgrade one node etcd cluster from 2.3 to 3.1
	// it will always be required for this version because all prior ket versions had a etcd2
	if len(etcdToUpgrade) > 1 {
		// Run the upgrade on the nodes to Etcd v3.0.x
		if err := executor.UpgradeEtcd2Nodes(plan, etcdToUpgrade); err != nil {
			return fmt.Errorf("Failed to upgrade etcd2 nodes: %v", err)
		}
	}

	// Run the upgrade on the nodes that need it
	if err := executor.UpgradeNodes(plan, toUpgrade, opts.online); err != nil {
		return fmt.Errorf("Failed to upgrade nodes: %v", err)
	}
	return nil
}
<commit_msg>Remove duplicate ConfigureDockerRegistry after a bad merge<commit_after>package cli

import (
	"errors"
	"fmt"
	"io"
	"os"

	"github.com/apprenda/kismatic/pkg/data"
	"github.com/apprenda/kismatic/pkg/install"
	"github.com/apprenda/kismatic/pkg/util"
	"github.com/spf13/cobra"
)

type upgradeOpts struct {
	generatedAssetsDir string
	verbose            bool
	outputFormat       string
	skipPreflight      bool
	online             bool
	planFile           string
	restartServices    bool
	partialAllowed     bool
}

// NewCmdUpgrade returns the upgrade command
func NewCmdUpgrade(out io.Writer) *cobra.Command {
	var opts upgradeOpts
	cmd := &cobra.Command{
		Use:   "upgrade",
		Short: "Upgrade your Kubernetes cluster",
		Long: `Upgrade your Kubernetes cluster.

The upgrade process is applied to each node, one node at a time. If a private docker registry
is being used, the new container images will be pushed by Kismatic before starting to upgrade
nodes.

Nodes in the cluster are upgraded in the following order:

1. Etcd nodes
2. Master nodes
3. Worker nodes (regardless of specialization)
`,
		RunE: func(cmd *cobra.Command, args []string) error {
			return cmd.Help()
		},
	}

	cmd.PersistentFlags().StringVar(&opts.generatedAssetsDir, "generated-assets-dir", "generated", "path to the directory where assets generated during the installation process will be stored")
	cmd.PersistentFlags().BoolVar(&opts.verbose, "verbose", false, "enable verbose logging from the installation")
	cmd.PersistentFlags().StringVarP(&opts.outputFormat, "output", "o", "simple", "installation output format (options \"simple\"|\"raw\")")
	cmd.PersistentFlags().BoolVar(&opts.skipPreflight, "skip-preflight", false, "skip upgrade pre-flight checks")
	cmd.PersistentFlags().BoolVar(&opts.restartServices, "restart-services", false, "force restart cluster services (Use with care)")
	cmd.PersistentFlags().BoolVar(&opts.partialAllowed, "partial-ok", false, "allow the upgrade of ready nodes, and skip nodes that have been deemed unready for upgrade")
	addPlanFileFlag(cmd.PersistentFlags(), &opts.planFile)

	// Subcommands
	cmd.AddCommand(NewCmdUpgradeOffline(out, &opts))
	cmd.AddCommand(NewCmdUpgradeOnline(out, &opts))
	return cmd
}

// NewCmdUpgradeOffline returns the command for running offline upgrades
func NewCmdUpgradeOffline(out io.Writer, opts *upgradeOpts) *cobra.Command {
	cmd := cobra.Command{
		Use:   "offline",
		Short: "Perform an offline upgrade of your Kubernetes cluster",
		Long: `Perform an offline upgrade of your Kubernetes cluster.

The offline upgrade is available for those clusters in which safety and availabilty are not a concern.
In this mode, the safety and availability checks will not be performed, nor will the nodes in the cluster
be drained of workloads.

Performing an offline upgrade could result in loss of critical data and reduced service
availability. For this reason, this method should not be used for clusters that are housing
production workloads.
`,
		RunE: func(cmd *cobra.Command, args []string) error {
			return doUpgrade(out, opts)
		},
	}
	return &cmd
}

// NewCmdUpgradeOnline returns the command for running online upgrades
func NewCmdUpgradeOnline(out io.Writer, opts *upgradeOpts) *cobra.Command {
	cmd := cobra.Command{
		Use:   "online",
		Short: "Perform an online upgrade of your Kubernetes cluster",
		Long: `Perform an online upgrade of your Kubernetes cluster.

During an online upgrade, Kismatic will run safety and availability checks (see table below) against the
existing cluster before performing the upgrade. If any unsafe condition is detected, a report will
be printed, and the upgrade will not proceed.

If the node under upgrade is a Kubernetes node, it is cordoned and drained of workloads
before any changes are applied.
`,
		RunE: func(cmd *cobra.Command, args []string) error {
			opts.online = true
			return doUpgrade(out, opts)
		},
	}
	return &cmd
}

func doUpgrade(out io.Writer, opts *upgradeOpts) error {
	planFile := opts.planFile
	planner := install.FilePlanner{File: planFile}
	executorOpts := install.ExecutorOptions{
		GeneratedAssetsDirectory: opts.generatedAssetsDir,
		RestartServices:          opts.restartServices,
		OutputFormat:             opts.outputFormat,
		Verbose:                  opts.verbose,
	}
	executor, err := install.NewExecutor(out, os.Stderr, executorOpts)
	if err != nil {
		return err
	}
	util.PrintHeader(out, "Computing upgrade plan", '=')

	// Read plan file
	if !planner.PlanExists() {
		util.PrettyPrintErr(out, "Reading plan file")
		return fmt.Errorf("plan file %q does not exist", planFile)
	}
	util.PrettyPrintOk(out, "Reading plan file")
	plan, err := planner.Read()
	if err != nil {
		util.PrettyPrintErr(out, "Reading plan file")
		return fmt.Errorf("error reading plan file %q: %v", planFile, err)
	}

	// Validate SSH connectivity to nodes
	if ok, errs := install.ValidatePlanSSHConnections(plan); !ok {
		util.PrettyPrintErr(out, "Validate SSH connectivity to nodes")
		util.PrintValidationErrors(out, errs)
		return fmt.Errorf("SSH connectivity validation errors found")
	}
	util.PrettyPrintOk(out, "Validate SSH connectivity to nodes")

	// Figure out which nodes to upgrade
	cv, err := install.ListVersions(plan)
	if err != nil {
		return fmt.Errorf("error listing cluster versions: %v", err)
	}
	var toUpgrade []install.ListableNode
	var toSkip []install.ListableNode
	for _, n := range cv.Nodes {
		if install.IsOlderVersion(n.Version) {
			toUpgrade = append(toUpgrade, n)
		} else {
			toSkip = append(toSkip, n)
		}
	}

	// Print the nodes that will be skipped
	if len(toSkip) > 0 {
		util.PrintHeader(out, "Skipping nodes", '=')
		for _, n := range toSkip {
			util.PrettyPrintOk(out, "- %q is at the target version %q", n.Node.Host, n.Version)
		}
		fmt.Fprintln(out)
	}

	if plan.ConfigureDockerRegistry() && plan.Cluster.DisconnectedInstallation {
		util.PrintHeader(out, "Upgrade Docker Registry", '=')
		if err := executor.UpgradeDockerRegistry(*plan); err != nil {
			return fmt.Errorf("Failed to upgrade docker registry: %v", err)
		}
	}

	// Print message if there's no work to do
	if len(toUpgrade) == 0 {
		fmt.Fprintln(out, "All nodes are at the target version. Skipping node upgrades.")
	} else {
		if err = upgradeNodes(out, *plan, *opts, toUpgrade, executor); err != nil {
			return err
		}
	}

	if opts.partialAllowed {
		util.PrintColor(out, util.Green, `

Partial upgrade complete.

Cluster level services are still left to upgrade. These can only be upgraded
when performing a full upgrade. When you are ready, you may use "kismatic upgrade"
without the "--partial-ok" flag to perform a full upgrade.

`)
		return nil
	}

	// Upgrade the cluster services
	util.PrintHeader(out, "Upgrade Cluster Services", '=')
	if err := executor.UpgradeClusterServices(*plan); err != nil {
		return fmt.Errorf("Failed to upgrade cluster services: %v", err)
	}

	if err := executor.RunSmokeTest(plan); err != nil {
		return fmt.Errorf("Smoke test failed: %v", err)
	}

	fmt.Fprintln(out)
	util.PrintColor(out, util.Green, "Upgrade complete\n")
	fmt.Fprintln(out)
	return nil
}

func upgradeNodes(out io.Writer, plan install.Plan, opts upgradeOpts, nodesNeedUpgrade []install.ListableNode, executor install.Executor) error {
	// Run safety checks if doing an online upgrade
	unsafeNodes := []install.ListableNode{}
	if opts.online {
		util.PrintHeader(out, "Validate Online Upgrade", '=')
		// Use the first master node for running kubectl
		client, err := plan.GetSSHClient(plan.Master.Nodes[0].Host)
		if err != nil {
			return fmt.Errorf("error getting SSH client: %v", err)
		}
		kubeClient := data.RemoteKubectl{SSHClient: client}
		for _, node := range nodesNeedUpgrade {
			util.PrettyPrint(out, "Node %q", node.Node.Host)
			errs := install.DetectNodeUpgradeSafety(plan, node.Node, kubeClient)
			if len(errs) != 0 {
				util.PrintError(out)
				fmt.Fprintln(out)
				for _, err := range errs {
					fmt.Println("-", err.Error())
				}
				unsafeNodes = append(unsafeNodes, node)
			} else {
				util.PrintOkln(out)
			}
		}
		// If we found any unsafe nodes, and we are not doing a partial upgrade, exit.
		if len(unsafeNodes) > 0 && !opts.partialAllowed {
			return errors.New("Unable to perform an online upgrade due to the unsafe conditions detected.")
		}
		// Block the upgrade if partial is allowed but there is an etcd or master node
		// that cannot be upgraded
		if opts.partialAllowed {
			for _, n := range unsafeNodes {
				for _, r := range n.Roles {
					if r == "master" || r == "etcd" {
						return errors.New("Unable to perform an online upgrade due to the unsafe conditions detected.")
					}
				}
			}
		}
	}

	// Run upgrade preflight on the nodes that are to be upgraded
	unreadyNodes := []install.ListableNode{}
	if !opts.skipPreflight {
		for _, node := range nodesNeedUpgrade {
			util.PrintHeader(out, fmt.Sprintf("Preflight Checks: %s %s", node.Node.Host, node.Roles), '=')
			if err := executor.RunUpgradePreFlightCheck(&plan, node); err != nil {
				// return fmt.Errorf("Upgrade preflight check failed: %v", err)
				unreadyNodes = append(unreadyNodes, node)
			}
		}
	}

	// Block upgrade if we found unready nodes, and we are not doing a partial upgrade
	if len(unreadyNodes) > 0 && !opts.partialAllowed {
		return errors.New("Errors found during preflight checks")
	}

	// Block the upgrade if partial is allowed but there is an etcd or master node
	// that cannot be upgraded
	if opts.partialAllowed {
		for _, n := range unreadyNodes {
			for _, r := range n.Roles {
				if r == "master" || r == "etcd" {
					return errors.New("Errors found during preflight checks")
				}
			}
		}
	}

	// Filter out the nodes that are unsafe/unready
	toUpgrade := []install.ListableNode{}
	for _, n := range nodesNeedUpgrade {
		upgrade := true
		for _, unsafe := range unsafeNodes {
			if unsafe.Node == n.Node {
				upgrade = false
			}
		}
		for _, unready := range unreadyNodes {
			if unready.Node == n.Node {
				upgrade = false
			}
		}
		if upgrade {
			toUpgrade = append(toUpgrade, n)
		}
	}

	// get all etcd nodes
	etcdToUpgrade := install.NodesWithRoles(toUpgrade, "etcd")
	// it's safe to upgrade one node etcd cluster from 2.3 to 3.1
	// it will always be required for this version because all prior ket versions had a etcd2
	if len(etcdToUpgrade) > 1 {
		// Run the upgrade on the nodes to Etcd v3.0.x
		if err := executor.UpgradeEtcd2Nodes(plan, etcdToUpgrade); err != nil {
			return fmt.Errorf("Failed to upgrade etcd2 nodes: %v", err)
		}
	}

	// Run the upgrade on the nodes that need it
	if err := executor.UpgradeNodes(plan, toUpgrade, opts.online); err != nil {
		return fmt.Errorf("Failed to upgrade nodes: %v", err)
	}
	return nil
}
<|endoftext|>


<commit_before>package client

type FakeAction struct {
	Action string
	Value  interface{}
}

// Fake implements Interface. Meant to be embedded into a struct to get a default
// implementation. This makes faking out just the method you want to test easier.
type Fake struct {
	// Fake by default keeps a simple list of the methods that have been called.
	Actions []FakeAction
}

func (c *Fake) Builds(namespace string) BuildInterface {
	return &FakeBuilds{Fake: c, Namespace: namespace}
}

func (c *Fake) BuildConfigs(namespace string) BuildConfigInterface {
	return &FakeBuildConfigs{Fake: c, Namespace: namespace}
}

func (c *Fake) Images(namespace string) ImageInterface {
	return &FakeImages{Fake: c, Namespace: namespace}
}

func (c *Fake) ImageRepositories(namespace string) ImageRepositoryInterface {
	return &FakeImageRepositories{Fake: c, Namespace: namespace}
}

func (c *Fake) ImageRepositoryMappings(namespace string) ImageRepositoryMappingInterface {
	return &FakeImageRepositoryMappings{Fake: c, Namespace: namespace}
}

func (c *Fake) Deployments(namespace string) DeploymentInterface {
	return &FakeDeployments{Fake: c, Namespace: namespace}
}

func (c *Fake) DeploymentConfigs(namespace string) DeploymentConfigInterface {
	return &FakeDeploymentConfigs{Fake: c, Namespace: namespace}
}

func (c *Fake) Routes(namespace string) RouteInterface {
	return &FakeRoutes{Fake: c, Namespace: namespace}
}

func (c *Fake) Users(namespace string) UserInterface {
	return &FakeUsers{Fake: c}
}

func (c *Fake) UserIdentityMappings(namespace string) UserIdentityMappingInterface {
	return &FakeUserIdentityMappings{Fake: c}
}
<commit_msg>Fixed wrong User() and UserIdentityMappings in Fake client<commit_after>package client

type FakeAction struct {
	Action string
	Value  interface{}
}

// Fake implements Interface. Meant to be embedded into a struct to get a default
// implementation. This makes faking out just the method you want to test easier.
type Fake struct {
	// Fake by default keeps a simple list of the methods that have been called.
	Actions []FakeAction
}

func (c *Fake) Builds(namespace string) BuildInterface {
	return &FakeBuilds{Fake: c, Namespace: namespace}
}

func (c *Fake) BuildConfigs(namespace string) BuildConfigInterface {
	return &FakeBuildConfigs{Fake: c, Namespace: namespace}
}

func (c *Fake) Images(namespace string) ImageInterface {
	return &FakeImages{Fake: c, Namespace: namespace}
}

func (c *Fake) ImageRepositories(namespace string) ImageRepositoryInterface {
	return &FakeImageRepositories{Fake: c, Namespace: namespace}
}

func (c *Fake) ImageRepositoryMappings(namespace string) ImageRepositoryMappingInterface {
	return &FakeImageRepositoryMappings{Fake: c, Namespace: namespace}
}

func (c *Fake) Deployments(namespace string) DeploymentInterface {
	return &FakeDeployments{Fake: c, Namespace: namespace}
}

func (c *Fake) DeploymentConfigs(namespace string) DeploymentConfigInterface {
	return &FakeDeploymentConfigs{Fake: c, Namespace: namespace}
}

func (c *Fake) Routes(namespace string) RouteInterface {
	return &FakeRoutes{Fake: c, Namespace: namespace}
}

func (c *Fake) Users() UserInterface {
	return &FakeUsers{Fake: c}
}

func (c *Fake) UserIdentityMappings() UserIdentityMappingInterface {
	return &FakeUserIdentityMappings{Fake: c}
}
<|endoftext|>