<commit_before>package router

import (
	"testing"
	"time"

	wt "github.com/weaveworks/weave/testing"
)

// TODO test gossip unicast; atm we only test topology gossip, which
// does not employ unicast.

type mockChannelConnection struct {
	RemoteConnection
	dest *Router
}

// Construct a "passive" Router, i.e. without any goroutines, except
// for Routes and GossipSenders.
func NewTestRouter(name PeerName) *Router {
	router := NewRouter(Config{}, name, "")
	// need to create a dummy channel otherwise tests hang on nil
	// channels when the Router invoked ConnectionMaker.Refresh
	router.ConnectionMaker.actionChan = make(chan ConnectionMakerAction, ChannelSize)
	router.Routes.Start()
	return router
}

func (conn *mockChannelConnection) SendProtocolMsg(protocolMsg ProtocolMsg) {
	if err := conn.dest.handleGossip(protocolMsg.tag, protocolMsg.msg); err != nil {
		panic(err)
	}
}

func sendPendingGossip(routers ...*Router) {
	// Loop until all routers report they didn't send anything
	for sentSomething := true; sentSomething; {
		sentSomething = false
		for _, router := range routers {
			sentSomething = router.sendPendingGossip() || sentSomething
		}
	}
}

func (router *Router) AddTestChannelConnection(r *Router) {
	fromName := router.Ourself.Peer.Name
	toName := r.Ourself.Peer.Name

	fromPeer := NewPeer(fromName, "", router.Ourself.Peer.UID, 0)
	toPeer := NewPeer(toName, "", r.Ourself.Peer.UID, 0)

	r.Peers.FetchWithDefault(fromPeer)    // Has side-effect of incrementing refcount
	router.Peers.FetchWithDefault(toPeer) //

	conn := &mockChannelConnection{RemoteConnection{router.Ourself.Peer, toPeer, "", false, true}, r}
	router.Ourself.handleAddConnection(conn)
	router.Ourself.handleConnectionEstablished(conn)
}

func (router *Router) DeleteTestChannelConnection(r *Router) {
	fromName := router.Ourself.Peer.Name
	toName := r.Ourself.Peer.Name

	fromPeer, _ := r.Peers.Fetch(fromName)
	toPeer, _ := router.Peers.Fetch(toName)

	r.Peers.Dereference(fromPeer)
	router.Peers.Dereference(toPeer)

	conn, _ := router.Ourself.ConnectionTo(toName)
	router.Ourself.handleDeleteConnection(conn)
}

func TestGossipTopology(t *testing.T) {
	wt.RunWithTimeout(t, 5*time.Second, func() {
		implTestGossipTopology(t)
	})
}

// Create a Peer representing the receiver router, with connections to
// the routers supplied as arguments, carrying across all UID and
// version information.
func (router *Router) tp(routers ...*Router) *Peer {
	peer := NewPeer(router.Ourself.Peer.Name, "", router.Ourself.Peer.UID, 0)
	connections := make(map[PeerName]Connection)
	for _, r := range routers {
		p := NewPeer(r.Ourself.Peer.Name, "", r.Ourself.Peer.UID, r.Ourself.Peer.version)
		connections[r.Ourself.Peer.Name] = newMockConnection(peer, p)
	}
	peer.version = router.Ourself.Peer.version
	peer.connections = connections
	return peer
}

// Check that the topology of router matches the peers and all of their connections
func checkTopology(t *testing.T, router *Router, wantedPeers ...*Peer) {
	router.Peers.RLock()
	checkTopologyPeers(t, true, router.Peers.allPeers(), wantedPeers...)
	router.Peers.RUnlock()
}

func implTestGossipTopology(t *testing.T) {
	// Create some peers that will talk to each other
	peer1Name, _ := PeerNameFromString("01:00:00:01:00:00")
	peer2Name, _ := PeerNameFromString("02:00:00:02:00:00")
	peer3Name, _ := PeerNameFromString("03:00:00:03:00:00")
	r1 := NewTestRouter(peer1Name)
	r2 := NewTestRouter(peer2Name)
	r3 := NewTestRouter(peer3Name)

	// Check state when they have no connections
	checkTopology(t, r1, r1.tp())
	checkTopology(t, r2, r2.tp())

	// Now try adding some connections
	r1.AddTestChannelConnection(r2)
	sendPendingGossip(r1, r2)
	checkTopology(t, r1, r1.tp(r2), r2.tp())
	checkTopology(t, r2, r1.tp(r2), r2.tp())
	r2.AddTestChannelConnection(r1)
	sendPendingGossip(r1, r2)
	checkTopology(t, r1, r1.tp(r2), r2.tp(r1))
	checkTopology(t, r2, r1.tp(r2), r2.tp(r1))

	// Currently, the connection from 2 to 3 is one-way only
	r2.AddTestChannelConnection(r3)
	sendPendingGossip(r1, r2, r3)
	checkTopology(t, r1, r1.tp(r2), r2.tp(r1, r3), r3.tp())
	checkTopology(t, r2, r1.tp(r2), r2.tp(r1, r3), r3.tp())
	// When r2 gossiped to r3, 1 was unreachable from r3 so it got removed from the
	// list of peers, but remains referenced in the connection from 1 to 3.
	checkTopology(t, r3, r2.tp(r1, r3), r3.tp())

	// Add a connection from 3 to 1 and now r1 is reachable.
	r3.AddTestChannelConnection(r1)
	sendPendingGossip(r1, r2, r3)
	checkTopology(t, r1, r1.tp(r2), r2.tp(r1, r3), r3.tp(r1))
	checkTopology(t, r2, r1.tp(r2), r2.tp(r1, r3), r3.tp(r1))
	checkTopology(t, r3, r1.tp(), r2.tp(r1, r3), r3.tp(r1))

	r1.AddTestChannelConnection(r3)
	sendPendingGossip(r1, r2, r3)
	checkTopology(t, r1, r1.tp(r2, r3), r2.tp(r1, r3), r3.tp(r1))
	checkTopology(t, r2, r1.tp(r2, r3), r2.tp(r1, r3), r3.tp(r1))
	checkTopology(t, r3, r1.tp(r2, r3), r2.tp(r1, r3), r3.tp(r1))

	// Drop the connection from 2 to 3
	r2.DeleteTestChannelConnection(r3)
	sendPendingGossip(r1, r2, r3)
	checkTopology(t, r1, r1.tp(r2, r3), r2.tp(r1), r3.tp(r1))
	checkTopology(t, r2, r1.tp(r2, r3), r2.tp(r1))
	checkTopology(t, r3, r1.tp(r2, r3), r2.tp(r1), r3.tp(r1))

	// Drop the connection from 1 to 3
	r1.DeleteTestChannelConnection(r3)
	sendPendingGossip(r1, r2, r3)
	checkTopology(t, r1, r1.tp(r2), r2.tp(r1), r3.tp(r1))

	checkTopology(t, r1, r1.tp(r2), r2.tp(r1), r3.tp(r1))
	checkTopology(t, r2, r1.tp(r2), r2.tp(r1))
	// r3 still thinks r1 has a connection to it
	checkTopology(t, r3, r1.tp(r2, r3), r2.tp(r1), r3.tp(r1))

	// On a timer, r3 will gossip to r1
	r3.SendAllGossip()
	sendPendingGossip(r1, r2, r3)
	checkTopology(t, r1, r1.tp(r2), r2.tp(r1), r3.tp(r1))
}
<commit_msg>refactor: simplify test router creation<commit_after>package router

import (
	"testing"
	"time"

	wt "github.com/weaveworks/weave/testing"
)

// TODO test gossip unicast; atm we only test topology gossip, which
// does not employ unicast.

type mockChannelConnection struct {
	RemoteConnection
	dest *Router
}

// Construct a "passive" Router, i.e. without any goroutines, except
// for Routes and GossipSenders.
func NewTestRouter(name string) *Router {
	peerName, _ := PeerNameFromString(name)
	router := NewRouter(Config{}, peerName, "")
	// need to create a dummy channel otherwise tests hang on nil
	// channels when the Router invoked ConnectionMaker.Refresh
	router.ConnectionMaker.actionChan = make(chan ConnectionMakerAction, ChannelSize)
	router.Routes.Start()
	return router
}

func (conn *mockChannelConnection) SendProtocolMsg(protocolMsg ProtocolMsg) {
	if err := conn.dest.handleGossip(protocolMsg.tag, protocolMsg.msg); err != nil {
		panic(err)
	}
}

func sendPendingGossip(routers ...*Router) {
	// Loop until all routers report they didn't send anything
	for sentSomething := true; sentSomething; {
		sentSomething = false
		for _, router := range routers {
			sentSomething = router.sendPendingGossip() || sentSomething
		}
	}
}

func (router *Router) AddTestChannelConnection(r *Router) {
	fromName := router.Ourself.Peer.Name
	toName := r.Ourself.Peer.Name

	fromPeer := NewPeer(fromName, "", router.Ourself.Peer.UID, 0)
	toPeer := NewPeer(toName, "", r.Ourself.Peer.UID, 0)

	r.Peers.FetchWithDefault(fromPeer)    // Has side-effect of incrementing refcount
	router.Peers.FetchWithDefault(toPeer) //

	conn := &mockChannelConnection{RemoteConnection{router.Ourself.Peer, toPeer, "", false, true}, r}
	router.Ourself.handleAddConnection(conn)
	router.Ourself.handleConnectionEstablished(conn)
}

func (router *Router) DeleteTestChannelConnection(r *Router) {
	fromName := router.Ourself.Peer.Name
	toName := r.Ourself.Peer.Name

	fromPeer, _ := r.Peers.Fetch(fromName)
	toPeer, _ := router.Peers.Fetch(toName)

	r.Peers.Dereference(fromPeer)
	router.Peers.Dereference(toPeer)

	conn, _ := router.Ourself.ConnectionTo(toName)
	router.Ourself.handleDeleteConnection(conn)
}

func TestGossipTopology(t *testing.T) {
	wt.RunWithTimeout(t, 5*time.Second, func() {
		implTestGossipTopology(t)
	})
}

// Create a Peer representing the receiver router, with connections to
// the routers supplied as arguments, carrying across all UID and
// version information.
func (router *Router) tp(routers ...*Router) *Peer {
	peer := NewPeer(router.Ourself.Peer.Name, "", router.Ourself.Peer.UID, 0)
	connections := make(map[PeerName]Connection)
	for _, r := range routers {
		p := NewPeer(r.Ourself.Peer.Name, "", r.Ourself.Peer.UID, r.Ourself.Peer.version)
		connections[r.Ourself.Peer.Name] = newMockConnection(peer, p)
	}
	peer.version = router.Ourself.Peer.version
	peer.connections = connections
	return peer
}

// Check that the topology of router matches the peers and all of their connections
func checkTopology(t *testing.T, router *Router, wantedPeers ...*Peer) {
	router.Peers.RLock()
	checkTopologyPeers(t, true, router.Peers.allPeers(), wantedPeers...)
	router.Peers.RUnlock()
}

func implTestGossipTopology(t *testing.T) {
	// Create some peers that will talk to each other
	r1 := NewTestRouter("01:00:00:01:00:00")
	r2 := NewTestRouter("02:00:00:02:00:00")
	r3 := NewTestRouter("03:00:00:03:00:00")

	// Check state when they have no connections
	checkTopology(t, r1, r1.tp())
	checkTopology(t, r2, r2.tp())

	// Now try adding some connections
	r1.AddTestChannelConnection(r2)
	sendPendingGossip(r1, r2)
	checkTopology(t, r1, r1.tp(r2), r2.tp())
	checkTopology(t, r2, r1.tp(r2), r2.tp())
	r2.AddTestChannelConnection(r1)
	sendPendingGossip(r1, r2)
	checkTopology(t, r1, r1.tp(r2), r2.tp(r1))
	checkTopology(t, r2, r1.tp(r2), r2.tp(r1))

	// Currently, the connection from 2 to 3 is one-way only
	r2.AddTestChannelConnection(r3)
	sendPendingGossip(r1, r2, r3)
	checkTopology(t, r1, r1.tp(r2), r2.tp(r1, r3), r3.tp())
	checkTopology(t, r2, r1.tp(r2), r2.tp(r1, r3), r3.tp())
	// When r2 gossiped to r3, 1 was unreachable from r3 so it got removed from the
	// list of peers, but remains referenced in the connection from 1 to 3.
	checkTopology(t, r3, r2.tp(r1, r3), r3.tp())

	// Add a connection from 3 to 1 and now r1 is reachable.
	r3.AddTestChannelConnection(r1)
	sendPendingGossip(r1, r2, r3)
	checkTopology(t, r1, r1.tp(r2), r2.tp(r1, r3), r3.tp(r1))
	checkTopology(t, r2, r1.tp(r2), r2.tp(r1, r3), r3.tp(r1))
	checkTopology(t, r3, r1.tp(), r2.tp(r1, r3), r3.tp(r1))

	r1.AddTestChannelConnection(r3)
	sendPendingGossip(r1, r2, r3)
	checkTopology(t, r1, r1.tp(r2, r3), r2.tp(r1, r3), r3.tp(r1))
	checkTopology(t, r2, r1.tp(r2, r3), r2.tp(r1, r3), r3.tp(r1))
	checkTopology(t, r3, r1.tp(r2, r3), r2.tp(r1, r3), r3.tp(r1))

	// Drop the connection from 2 to 3
	r2.DeleteTestChannelConnection(r3)
	sendPendingGossip(r1, r2, r3)
	checkTopology(t, r1, r1.tp(r2, r3), r2.tp(r1), r3.tp(r1))
	checkTopology(t, r2, r1.tp(r2, r3), r2.tp(r1))
	checkTopology(t, r3, r1.tp(r2, r3), r2.tp(r1), r3.tp(r1))

	// Drop the connection from 1 to 3
	r1.DeleteTestChannelConnection(r3)
	sendPendingGossip(r1, r2, r3)
	checkTopology(t, r1, r1.tp(r2), r2.tp(r1), r3.tp(r1))

	checkTopology(t, r1, r1.tp(r2), r2.tp(r1), r3.tp(r1))
	checkTopology(t, r2, r1.tp(r2), r2.tp(r1))
	// r3 still thinks r1 has a connection to it
	checkTopology(t, r3, r1.tp(r2, r3), r2.tp(r1), r3.tp(r1))

	// On a timer, r3 will gossip to r1
	r3.SendAllGossip()
	sendPendingGossip(r1, r2, r3)
	checkTopology(t, r1, r1.tp(r2), r2.tp(r1), r3.tp(r1))
}
<|endoftext|>


<commit_before>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package cas provides remote-apis-sdks client with luci integration.
package cas

import (
	"context"
	"runtime"
	"strings"

	"github.com/bazelbuild/remote-apis-sdks/go/pkg/client"
	"google.golang.org/grpc/credentials"

	"go.chromium.org/luci/auth"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/hardcoded/chromeinfra"
)

// NewClient returns luci auth configured Client for RBE-CAS.
func NewClient(ctx context.Context, instance string, opts auth.Options, readOnly bool) (*client.Client, error) {
	project := strings.Split(instance, "/")[1]
	var role string
	if readOnly {
		role = "cas-read-only"
	} else {
		role = "cas-read-write"
	}

	// Construct auth.Options.
	opts.ActAsServiceAccount = role + "@" + project + ".iam.gserviceaccount.com"
	opts.ActViaLUCIRealm = "@internal:" + project + "/" + role
	opts.Scopes = []string{"https://www.googleapis.com/auth/cloud-platform"}

	if strings.HasSuffix(project, "-dev") || strings.HasSuffix(project, "-staging") {
		// use dev token server for dev/staging projects.
		opts.TokenServerHost = chromeinfra.TokenServerDevHost
	}

	a := auth.NewAuthenticator(ctx, auth.SilentLogin, opts)
	creds, err := a.PerRPCCredentials()
	if err != nil {
		return nil, errors.Annotate(err, "failed to get PerRPCCredentials").Err()
	}

	cl, err := client.NewClient(ctx, instance,
		client.DialParams{
			Service:            "remotebuildexecution.googleapis.com:443",
			TransportCredsOnly: true,
		}, ClientOptions(creds)...)
	if err != nil {
		return nil, errors.Annotate(err, "failed to create client").Err()
	}

	return cl, nil
}

// ClientOptions returns CAS client options.
func ClientOptions(creds credentials.PerRPCCredentials) []client.Opt {
	casConcurrency := runtime.NumCPU() * 2
	if runtime.GOOS == "windows" {
		// This is for better file write performance on Windows (http://b/171672371#comment6).
		casConcurrency = runtime.NumCPU()
	}

	return []client.Opt{
		&client.PerRPCCreds{Creds: creds},
		client.CASConcurrency(casConcurrency),
		client.UtilizeLocality(true),
		&client.TreeSymlinkOpts{Preserved: true, FollowsTarget: false},
		// Set restricted permission for written files.
		client.DirMode(0700),
		client.ExecutableMode(0700),
		client.RegularMode(0600),
		client.CompressedBytestreamThreshold(0),
	}
}

func ContextWithMetadata(ctx context.Context, toolName string) (context.Context, error) {
	ctx, err := client.ContextWithMetadata(ctx, &client.ContextMetadata{
		ToolName: toolName,
	})
	if err != nil {
		return nil, errors.Annotate(err, "failed to attach metadata").Err()
	}

	m, err := client.GetContextMetadata(ctx)
	if err != nil {
		return nil, errors.Annotate(err, "failed to extract metadata").Err()
	}

	logging.Infof(ctx, "context metadata: %#+v", *m)

	return ctx, nil
}
<commit_msg>[cas] Refactor NewClient<commit_after>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package cas provides remote-apis-sdks client with luci integration.
package cas

import (
	"context"
	"fmt"
	"runtime"
	"strings"

	"github.com/bazelbuild/remote-apis-sdks/go/pkg/client"
	"google.golang.org/grpc/credentials"

	"go.chromium.org/luci/auth"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/hardcoded/chromeinfra"
)

// NewClient returns luci auth configured Client for RBE-CAS.
func NewClient(ctx context.Context, instance string, opts auth.Options, readOnly bool) (*client.Client, error) {
	project := strings.Split(instance, "/")[1]
	var role string
	if readOnly {
		role = "cas-read-only"
	} else {
		role = "cas-read-write"
	}

	// Construct auth.Options.
	opts.ActAsServiceAccount = fmt.Sprintf("%s@%s.iam.gserviceaccount.com", role, project)
	opts.ActViaLUCIRealm = fmt.Sprintf("@internal:%s/%s", project, role)
	opts.Scopes = []string{"https://www.googleapis.com/auth/cloud-platform"}

	if strings.HasSuffix(project, "-dev") || strings.HasSuffix(project, "-staging") {
		// use dev token server for dev/staging projects.
		opts.TokenServerHost = chromeinfra.TokenServerDevHost
	}

	creds, err := auth.NewAuthenticator(ctx, auth.SilentLogin, opts).PerRPCCredentials()
	if err != nil {
		return nil, errors.Annotate(err, "failed to get PerRPCCredentials").Err()
	}
	dialParams := client.DialParams{
		Service:            "remotebuildexecution.googleapis.com:443",
		TransportCredsOnly: true,
	}

	cl, err := client.NewClient(ctx, instance, dialParams, ClientOptions(creds)...)
	if err != nil {
		return nil, errors.Annotate(err, "failed to create client").Err()
	}
	return cl, nil
}

// ClientOptions returns CAS client options.
func ClientOptions(creds credentials.PerRPCCredentials) []client.Opt {
	casConcurrency := runtime.NumCPU() * 2
	if runtime.GOOS == "windows" {
		// This is for better file write performance on Windows (http://b/171672371#comment6).
		casConcurrency = runtime.NumCPU()
	}

	return []client.Opt{
		&client.PerRPCCreds{Creds: creds},
		client.CASConcurrency(casConcurrency),
		client.UtilizeLocality(true),
		&client.TreeSymlinkOpts{Preserved: true, FollowsTarget: false},
		// Set restricted permission for written files.
		client.DirMode(0700),
		client.ExecutableMode(0700),
		client.RegularMode(0600),
		client.CompressedBytestreamThreshold(0),
	}
}

func ContextWithMetadata(ctx context.Context, toolName string) (context.Context, error) {
	ctx, err := client.ContextWithMetadata(ctx, &client.ContextMetadata{
		ToolName: toolName,
	})
	if err != nil {
		return nil, errors.Annotate(err, "failed to attach metadata").Err()
	}

	m, err := client.GetContextMetadata(ctx)
	if err != nil {
		return nil, errors.Annotate(err, "failed to extract metadata").Err()
	}

	logging.Infof(ctx, "context metadata: %#+v", *m)

	return ctx, nil
}
<|endoftext|>


<commit_before>package client

import "encoding/xml"

type TransitionProgress float64 // 0.0 .. 1.0

// Zero value is false
func (value TransitionProgress) InProgress() bool {
	return value > 0.0
}

const TransitionPosMax = 4096

type Transition struct {
	ID int `xml:"id,attr"`

	ArmMode         int `xml:"ArmMode"`
	TransPos        int `xml:"TransPos"`
	AutoTransInProg int `xml:"AutoTransInProg"`
	TransInProg     int `xml:"TransInProg"`
}

func (transition Transition) InProgress() bool {
	return transition.AutoTransInProg > 0
}

// Return float 0.0 .. 1.0
func (transition Transition) Progress() TransitionProgress {
	if transition.InProgress() {
		return TransitionProgress(float64(transition.TransPos) / float64(TransitionPosMax))
	} else {
		// XXX: assume transition is always > 0.0
		return TransitionProgress(0.0)
	}
}

type Transitions map[int]Transition

func (col *Transitions) UnmarshalXML(d *xml.Decoder, e xml.StartElement) error {
	return unmarshalXMLItem(col, d, e)
}

func (col Transitions) MarshalJSON() ([]byte, error) {
	return marshalJSONMap(col)
}
<commit_msg>client: handle manual Transitions<commit_after>package client

import "encoding/xml"

type TransitionProgress float64 // 0.0 .. 1.0

// Zero value is false
func (value TransitionProgress) InProgress() bool {
	return value > 0.0
}

const TransitionPosMax = 4096

type Transition struct {
	ID int `xml:"id,attr"`

	ArmMode         int `xml:"ArmMode"`
	TransPos        int `xml:"TransPos"`
	AutoTransInProg int `xml:"AutoTransInProg"`
	TransInProg     int `xml:"TransInProg"`
}

// Workaroud missing TransInProg=0 update after Cut
func (transition Transition) InProgress() bool {
	return (transition.AutoTransInProg > 0) || (transition.TransInProg > 0 && transition.TransPos > 0)
}

// Return float 0.0 .. 1.0
func (transition Transition) Progress() TransitionProgress {
	return TransitionProgress(float64(transition.TransPos) / float64(TransitionPosMax))
}

type Transitions map[int]Transition

func (col *Transitions) UnmarshalXML(d *xml.Decoder, e xml.StartElement) error {
	return unmarshalXMLItem(col, d, e)
}

func (col Transitions) MarshalJSON() ([]byte, error) {
	return marshalJSONMap(col)
}
<|endoftext|>


<commit_before>package cloudatcost

type StandardResponse struct {
	Status   string `json:"status"`
	Time     int    `json:"time"`
	API      string `json:"api"`
	Serverid string `json:"serverid"`
	Result   string `json:"result"`
}

type ConsoleResponse struct {
	Status   string `json:"status"`
	Time     int    `json:"time"`
	API      string `json:"api"`
	Serverid string `json:"serverid"`
	Console  string `json:"console"`
}

type PowerOperationResponse struct {
	Status   string `json:"status"`
	Time     int    `json:"time"`
	API      string `json:"api"`
	Action   string `json:"action"`
	Serverid string `json:"serverid"`
	Taskid   int64  `json:"taskid"`
	Result   string `json:"result"`
}
type RunModeResponse struct {
	Status   string `json:"status"`
	Time     int    `json:"time"`
	API      string `json:"api"`
	Serverid string `json:"serverid"`
	Result   string `json:"result"`
}

type ListTask struct {
	Cid        string `json:"cid"`
	Idf        string `json:"idf"`
	Serverid   string `json:"serverid"`
	Action     string `json:"action"`
	Status     string `json:"status"`
	Starttime  string `json:"starttime"`
	Finishtime string `json:"finishtime"`
}

type ListTasksResponse struct {
	Status string     `json:"status"`
	Time   int        `json:"time"`
	API    string     `json:"api"`
	Cid    string     `json:"cid"`
	Action string     `json:"action"`
	Data   []ListTask `json:"data"`
}

type ListTemplate struct {
	Ce_id  string `json:"ce_id"`
	Name   string `json:"name"`
}

type ListTemplatesResponse struct {
	Status string         `json:"status"`
	Time   int            `json:"time"`
	API    string         `json:"api"`
	Action string         `json:"action"`
	Data   []ListTemplate `json:"data"`
}

type ListServer struct {
	Sid        string      `json:"sid"`
	ID         string      `json:"id"`
	Packageid  string      `json:"packageid"`
	Servername string      `json:"servername"`
	Lable      interface{} `json:"lable"`
	Vmname     string      `json:"vmname"`
	IP         string      `json:"ip"`
	Netmask    string      `json:"netmask"`
	Gateway    string      `json:"gateway"`
	Portgroup  string      `json:"portgroup"`
	Hostname   string      `json:"hostname"`
	Rootpass   string      `json:"rootpass"`
	Vncport    string      `json:"vncport"`
	Vncpass    string      `json:"vncpass"`
	Servertype string      `json:"servertype"`
	Template   string      `json:"template"`
	CPU        string      `json:"cpu"`
	Cpuusage   string      `json:"cpuusage"`
	RAM        string      `json:"ram"`
	Ramusage   string      `json:"ramusage"`
	Storage    string      `json:"storage"`
	Hdusage    string      `json:"hdusage"`
	Sdate      string      `json:"sdate"`
	Status     string      `json:"status"`
	PanelNote  string      `json:"panel_note"`
	Mode       string      `json:"mode"`
	UID        string      `json:"uid"`
}

type ListServersResponse struct {
	Status string       `json:"status"`
	Time   int          `json:"time"`
	API    string       `json:"api"`
	Action string       `json:"action"`
	Data   []ListServer `json:"data"`
}

type CloudProServerResponse struct {
	Status string `json:"status"`
	Time   int    `json:"time"`
	API    string `json:"api"`
	Action string `json:"action"`
	Taskid int    `json:"taskid"`
	Result string `json:"result"`
}

type CloudProResourcesTotalResponse struct {
	CPU     string `json:"cpu_total"`
	Ram     string `json:"ram_total"`
	Stroage string `json:"storage_total"`
}

type CloudProResourcesUsedResponse struct {
	CPU     string `json:"cpu_used"`
	Ram     string `json:"ram_used"`
	Stroage string `json:"storage_used"`
}

type CloudProResourcesData struct {
	Total []CloudProResourcesTotalResponse `json:"total"`
	Used  CloudProResourcesUsedResponse    `json:"used"`
}

type CloudProResourcesResponse struct {
	Status string                `json:"status"`
	Time   int                   `json:"time"`
	API    string                `json:"api"`
	Action string                `json:"action"`
	Data   CloudProResourcesData `json:"data"`
}

type CreateServerOptions struct {
	Cpu     string `json:"cpu"`
	Ram     string `json:"ram"`
	Storage string `json:"storage"`
	OS      string `json:"os"`
	Datacenter string `json:"datacenter"`
}
<commit_msg>bugfixes<commit_after>package cloudatcost

type StandardResponse struct {
	Status   string `json:"status"`
	Time     int    `json:"time"`
	API      string `json:"api"`
	Serverid string `json:"serverid"`
	Result   string `json:"result"`
}

type ConsoleResponse struct {
	Status   string `json:"status"`
	Time     int    `json:"time"`
	API      string `json:"api"`
	Serverid string `json:"serverid"`
	Console  string `json:"console"`
}

type PowerOperationResponse struct {
	Status   string `json:"status"`
	Time     int    `json:"time"`
	API      string `json:"api"`
	Action   string `json:"action"`
	Serverid string `json:"serverid"`
	Taskid   int64  `json:"taskid"`
	Result   string `json:"result"`
}
type RunModeResponse struct {
	Status   string `json:"status"`
	Time     int    `json:"time"`
	API      string `json:"api"`
	Serverid string `json:"serverid"`
	Result   string `json:"result"`
}

type ListTask struct {
	Cid        string `json:"cid"`
	Idf        string `json:"idf"`
	Serverid   string `json:"serverid"`
	Action     string `json:"action"`
	Status     string `json:"status"`
	Starttime  string `json:"starttime"`
	Finishtime string `json:"finishtime"`
}

type ListTasksResponse struct {
	Status string     `json:"status"`
	Time   int        `json:"time"`
	API    string     `json:"api"`
	Cid    string     `json:"cid"`
	Action string     `json:"action"`
	Data   []ListTask `json:"data"`
}

type ListTemplate struct {
	Ce_id  string `json:"ce_id"`
	Name   string `json:"name"`
}

type ListTemplatesResponse struct {
	Status string         `json:"status"`
	Time   int            `json:"time"`
	API    string         `json:"api"`
	Action string         `json:"action"`
	Data   []ListTemplate `json:"data"`
}

type ListServer struct {
	Sid        string      `json:"sid"`
	ID         string      `json:"id"`
	Packageid  string      `json:"packageid"`
	Servername string      `json:"servername"`
	Lable      interface{} `json:"lable"`
	Vmname     string      `json:"vmname"`
	IP         string      `json:"ip"`
	Netmask    string      `json:"netmask"`
	Gateway    string      `json:"gateway"`
	Portgroup  string      `json:"portgroup"`
	Hostname   string      `json:"hostname"`
	Rootpass   string      `json:"rootpass"`
	Vncport    string      `json:"vncport"`
	Vncpass    string      `json:"vncpass"`
	Servertype string      `json:"servertype"`
	Template   string      `json:"template"`
	CPU        string      `json:"cpu"`
	Cpuusage   string      `json:"cpuusage"`
	RAM        string      `json:"ram"`
	Ramusage   string      `json:"ramusage"`
	Storage    string      `json:"storage"`
	Hdusage    string      `json:"hdusage"`
	Sdate      string      `json:"sdate"`
	Status     string      `json:"status"`
	PanelNote  string      `json:"panel_note"`
	Mode       string      `json:"mode"`
	UID        string      `json:"uid"`
}

type ListServersResponse struct {
	Status string       `json:"status"`
	Time   int          `json:"time"`
	API    string       `json:"api"`
	Action string       `json:"action"`
	Data   []ListServer `json:"data"`
}

type CloudProServerResponse struct {
	Status string `json:"status"`
	Time   int    `json:"time"`
	API    string `json:"api"`
	Action string `json:"action"`
	Taskid int    `json:"taskid"`
	Result string `json:"result"`
}

type CloudProResourcesTotalResponse struct {
	CPU     int `json:"cpu_total"`
	Ram     int `json:"ram_total"`
	Storage int `json:"storage_total"`
}

type CloudProResourcesUsedResponse struct {
	CPU     int `json:"cpu_used"`
	Ram     int `json:"ram_used"`
	Storage int `json:"storage_used"`
}

type CloudProResourcesData struct {
	Total CloudProResourcesTotalResponse `json:"total"`
	Used  CloudProResourcesUsedResponse    `json:"used"`
}

type CloudProResourcesResponse struct {
	Status string                `json:"status"`
	Time   int                   `json:"time"`
	API    string                `json:"api"`
	Action string                `json:"action"`
	Data   CloudProResourcesData `json:"data"`
}

type CreateServerOptions struct {
	Cpu     string `json:"cpu"`
	Ram     string `json:"ram"`
	Storage string `json:"storage"`
	OS      string `json:"os"`
	Datacenter string `json:"datacenter"`
}
<|endoftext|>


<commit_before>package cluster

import (
//  dockerapi "github.com/fsouza/go-dockerclient"
)

type ExistContainer struct {
  ID string
  Name string
  Index int
  Running bool
}

func NewExistContainer(id string, name string, index int, running bool) ExistContainer {
  return ExistContainer{
    ID: id,
    Name: name,
    Index: index,
    Running: running,
  }
}

type Application struct {
  Cluster map[string]interface{} `yaml:"cluster,inline"`
  Docker Docker
}

type Docker struct {
  Hosts []string
}

type Container struct {
  Name string
  Image string
  Hostname string
  Ports []string
  Environment []string
  Links []string
  Volumes []string
  Command string

  Post string
  Privileged bool

  Scale int
  // Links []string
  // Ports map[dockerapi.Port][]dockerapi.PortBinding

  Exist []ExistContainer

  Changed bool

  // Application level
  Cluster map[string]interface{}  `yaml:"cluster,inline"`
  Docker Docker
}

// check for containers and application mixing  `yaml:",inline"`<commit_msg>check yml config<commit_after>package cluster

import (
//  dockerapi "github.com/fsouza/go-dockerclient"
)

type ExistContainer struct {
  ID string
  Name string
  Index int
  Running bool
}

func NewExistContainer(id string, name string, index int, running bool) ExistContainer {
  return ExistContainer{
    ID: id,
    Name: name,
    Index: index,
    Running: running,
  }
}

type Application struct {
  Cluster map[string]interface{}
  // `yaml:"cluster,inline"`
  Docker Docker
}

type Docker struct {
  Hosts []string
}

type Container struct {
  Name string
  Image string
  Hostname string
  Ports []string
  Environment []string
  Links []string
  Volumes []string
  Command string

  Post string
  Privileged bool

  Scale int
  // Links []string
  // Ports map[dockerapi.Port][]dockerapi.PortBinding

  Exist []ExistContainer

  Changed bool

  // Application level
  Cluster map[string]interface{}
  // `yaml:"cluster,inline"`
  Docker Docker
}

// check for containers and application mixing  `yaml:",inline"`<|endoftext|>


<commit_before>// Copyright 2014 docker-cluster authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cluster

import (
	"errors"
	"github.com/fsouza/go-dockerclient"
	"sync"
	"sync/atomic"
)

// Arbitrary options to be sent to the scheduler. This options will
// be only read and interpreted by the scheduler itself.
type SchedulerOptions interface{}

// Scheduler represents a scheduling strategy, that will be used when creating
// new containers.
type Scheduler interface {
	// Schedule creates a new container, returning the ID of the node where
	// the container is running, and the container, or an error.
	Schedule(c *Cluster, opts docker.CreateContainerOptions, schedulerOpts SchedulerOptions) (Node, error)
}

type node struct {
	*docker.Client
	addr string
}

type roundRobin struct {
	lastUsed int64
	mut      sync.RWMutex
	init     int32
}

func (s *roundRobin) Schedule(c *Cluster, opts docker.CreateContainerOptions, schedulerOpts SchedulerOptions) (Node, error) {
	s.mut.RLock()
	defer s.mut.RUnlock()
	nodes, _ := c.Nodes()
	if len(nodes) == 0 {
		return Node{}, errors.New("No nodes available")
	}
	if atomic.CompareAndSwapInt32(&s.init, 0, 1) {
		atomic.StoreInt64(&s.lastUsed, -1)
	}
	value := atomic.AddInt64(&s.lastUsed, 1)
	index := value % int64(len(nodes))
	return nodes[index], nil
}
<commit_msg>cluster/scheduler: remove unused rwmutex from roundRobin<commit_after>// Copyright 2014 docker-cluster authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cluster

import (
	"errors"
	"github.com/fsouza/go-dockerclient"
	"sync/atomic"
)

// Arbitrary options to be sent to the scheduler. This options will
// be only read and interpreted by the scheduler itself.
type SchedulerOptions interface{}

// Scheduler represents a scheduling strategy, that will be used when creating
// new containers.
type Scheduler interface {
	// Schedule creates a new container, returning the ID of the node where
	// the container is running, and the container, or an error.
	Schedule(c *Cluster, opts docker.CreateContainerOptions, schedulerOpts SchedulerOptions) (Node, error)
}

type node struct {
	*docker.Client
	addr string
}

type roundRobin struct {
	lastUsed int64
	init     int32
}

func (s *roundRobin) Schedule(c *Cluster, opts docker.CreateContainerOptions, schedulerOpts SchedulerOptions) (Node, error) {
	nodes, _ := c.Nodes()
	if len(nodes) == 0 {
		return Node{}, errors.New("No nodes available")
	}
	if atomic.CompareAndSwapInt32(&s.init, 0, 1) {
		atomic.StoreInt64(&s.lastUsed, -1)
	}
	value := atomic.AddInt64(&s.lastUsed, 1)
	index := value % int64(len(nodes))
	return nodes[index], nil
}
<|endoftext|>


<commit_before>/*
 * MinIO Client (C) 2019 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"fmt"
	"strings"
	"time"

	"github.com/fatih/color"
	"github.com/minio/cli"
	json "github.com/minio/mc/pkg/colorjson"
	"github.com/minio/mc/pkg/probe"
	"github.com/minio/minio/pkg/console"
	"github.com/minio/minio/pkg/madmin"
)

const logTimeFormat string = "15:04:05 MST 01/02/2006"

var adminConsoleFlags = []cli.Flag{
	cli.IntFlag{
		Name:  "limit, l",
		Usage: "show last n log entries",
		Value: 10,
	},
	cli.StringFlag{
		Name:  "type, t",
		Usage: "list error logs by type. Valid options are '[minio, application, all]'",
		Value: "all",
	},
}

var adminConsoleCmd = cli.Command{
	Name:            "console",
	Usage:           "show console logs for MinIO server",
	Action:          mainAdminConsole,
	Before:          setGlobalsFromContext,
	Flags:           append(adminConsoleFlags, globalFlags...),
	HideHelpCommand: true,
	CustomHelpTemplate: `NAME:
  {{.HelpName}} - {{.Usage}}

USAGE:
  {{.HelpName}} [FLAGS] TARGET [NODENAME]

FLAGS:
  {{range .VisibleFlags}}{{.}}
  {{end}}
EXAMPLES:
  1. Show console logs for a MinIO server with alias 'play'
     {{.Prompt}} {{.HelpName}} play

  2. Show last 5 log entries for node 'node1' on MinIO server with alias 'cluster1'
     {{.Prompt}} {{.HelpName}} --limit 5 cluster1 node1

  3. Show application error logs on MinIO server with alias 'play'
     {{.Prompt}} {{.HelpName}} --type application play
`,
}

func checkAdminLogSyntax(ctx *cli.Context) {
	if len(ctx.Args()) == 0 || len(ctx.Args()) > 3 {
		cli.ShowCommandHelpAndExit(ctx, "console", 1) // last argument is exit code
	}
}

// Extend madmin.LogInfo to add String() and JSON() methods
type logMessage struct {
	madmin.LogInfo
}

// JSON - jsonify loginfo
func (l logMessage) JSON() string {
	logJSON, err := json.MarshalIndent(&l, "", " ")
	fatalIf(probe.NewError(err), "Unable to marshal into JSON.")

	return string(logJSON)

}
func getLogTime(lt string) string {
	tm, err := time.Parse(time.RFC3339Nano, lt)
	if err != nil {
		return lt
	}
	return tm.Format(logTimeFormat)
}

// String - return colorized loginfo as string.
func (l logMessage) String() string {
	var hostStr string
	var b = &strings.Builder{}

	if l.NodeName != "" {
		hostStr = fmt.Sprintf("%s ", colorizedNodeName(l.NodeName))
	}
	log := l.LogInfo
	if log.ConsoleMsg != "" {
		if strings.HasPrefix(log.ConsoleMsg, "\n") {
			fmt.Fprintf(b, "%s\n", hostStr)
			log.ConsoleMsg = strings.TrimPrefix(log.ConsoleMsg, "\n")
		}
		fmt.Fprintf(b, "%s %s", hostStr, log.ConsoleMsg)
		return b.String()
	}
	traceLength := len(l.Trace.Source)

	apiString := "API: " + l.API.Name + "("
	if l.API.Args != nil && l.API.Args.Bucket != "" {
		apiString = apiString + "bucket=" + l.API.Args.Bucket
	}
	if l.API.Args != nil && l.API.Args.Object != "" {
		apiString = apiString + ", object=" + l.API.Args.Object
	}
	apiString += ")"

	var msg = console.Colorize("LogMessage", l.Trace.Message)

	fmt.Fprintf(b, "\n%s %s", hostStr, console.Colorize("Api", apiString))
	fmt.Fprintf(b, "\n%s Time: %s", hostStr, getLogTime(l.Time))
	fmt.Fprintf(b, "\n%s DeploymentID: %s", hostStr, l.DeploymentID)
	if l.RequestID != "" {
		fmt.Fprintf(b, "\n%s RequestID: %s", hostStr, l.RequestID)
	}
	if l.RemoteHost != "" {
		fmt.Fprintf(b, "\n%s RemoteHost: %s", hostStr, l.RemoteHost)
	}
	if l.UserAgent != "" {
		fmt.Fprintf(b, "\n%s UserAgent: %s", hostStr, l.UserAgent)
	}
	fmt.Fprintf(b, "\n%s Error: %s", hostStr, msg)

	for key, value := range l.Trace.Variables {
		if value != "" {
			fmt.Fprintf(b, "\n%s %s=%s", hostStr, key, value)
		}
	}
	for i, element := range l.Trace.Source {
		fmt.Fprintf(b, "\n%s %8v: %s", hostStr, traceLength-i, element)

	}
	logMsg := strings.TrimPrefix(b.String(), "\n")
	return fmt.Sprintf("%s\n", logMsg)
}

// mainAdminConsole - the entry function of console command
func mainAdminConsole(ctx *cli.Context) error {
	// Check for command syntax
	checkAdminLogSyntax(ctx)
	console.SetColor("LogMessage", color.New(color.Bold, color.FgRed))
	console.SetColor("Api", color.New(color.Bold, color.FgWhite))
	for _, c := range colors {
		console.SetColor(fmt.Sprintf("Node%d", c), color.New(c))
	}
	aliasedURL := ctx.Args().Get(0)
	var node string
	if len(ctx.Args()) > 1 {
		node = ctx.Args().Get(1)
	}
	var limit int
	if ctx.IsSet("limit") {
		limit = ctx.Int("limit")
		if limit <= 0 {
			fatalIf(errInvalidArgument().Trace(ctx.Args()...), "please set a proper limit, for example: '--limit 5' to display last 5 logs, omit this flag to display all available logs")
		}
	}
	var logType string
	logType = strings.ToLower(ctx.String("type"))
	if logType != "minio" && logType != "application" && logType != "all" {
		fatalIf(errInvalidArgument().Trace(ctx.Args()...), "Invalid value for --type flag. Valid options are [minio, application, all]")
	}
	// Create a new MinIO Admin Client
	client, err := newAdminClient(aliasedURL)
	if err != nil {
		fatalIf(err.Trace(aliasedURL), "Cannot initialize admin client.")
		return nil
	}
	doneCh := make(chan struct{})
	defer close(doneCh)

	// Start listening on all console log activity.
	logCh := client.GetLogs(node, limit, logType, doneCh)
	for logInfo := range logCh {
		if logInfo.Err != nil {
			fatalIf(probe.NewError(logInfo.Err), "Cannot listen to console logs")
		}
		// drop nodeName from output if specified as cli arg
		if node != "" {
			logInfo.NodeName = ""
		}
		printMsg(logMessage{logInfo})
	}
	return nil
}
<commit_msg>admin/console: Fix a crash in console string generation (#3049)<commit_after>/*
 * MinIO Client (C) 2019 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"fmt"
	"strings"
	"time"

	"github.com/fatih/color"
	"github.com/minio/cli"
	json "github.com/minio/mc/pkg/colorjson"
	"github.com/minio/mc/pkg/probe"
	"github.com/minio/minio/pkg/console"
	"github.com/minio/minio/pkg/madmin"
)

const logTimeFormat string = "15:04:05 MST 01/02/2006"

var adminConsoleFlags = []cli.Flag{
	cli.IntFlag{
		Name:  "limit, l",
		Usage: "show last n log entries",
		Value: 10,
	},
	cli.StringFlag{
		Name:  "type, t",
		Usage: "list error logs by type. Valid options are '[minio, application, all]'",
		Value: "all",
	},
}

var adminConsoleCmd = cli.Command{
	Name:            "console",
	Usage:           "show console logs for MinIO server",
	Action:          mainAdminConsole,
	Before:          setGlobalsFromContext,
	Flags:           append(adminConsoleFlags, globalFlags...),
	HideHelpCommand: true,
	CustomHelpTemplate: `NAME:
  {{.HelpName}} - {{.Usage}}

USAGE:
  {{.HelpName}} [FLAGS] TARGET [NODENAME]

FLAGS:
  {{range .VisibleFlags}}{{.}}
  {{end}}
EXAMPLES:
  1. Show console logs for a MinIO server with alias 'play'
     {{.Prompt}} {{.HelpName}} play

  2. Show last 5 log entries for node 'node1' on MinIO server with alias 'cluster1'
     {{.Prompt}} {{.HelpName}} --limit 5 cluster1 node1

  3. Show application error logs on MinIO server with alias 'play'
     {{.Prompt}} {{.HelpName}} --type application play
`,
}

func checkAdminLogSyntax(ctx *cli.Context) {
	if len(ctx.Args()) == 0 || len(ctx.Args()) > 3 {
		cli.ShowCommandHelpAndExit(ctx, "console", 1) // last argument is exit code
	}
}

// Extend madmin.LogInfo to add String() and JSON() methods
type logMessage struct {
	madmin.LogInfo
}

// JSON - jsonify loginfo
func (l logMessage) JSON() string {
	logJSON, err := json.MarshalIndent(&l, "", " ")
	fatalIf(probe.NewError(err), "Unable to marshal into JSON.")

	return string(logJSON)

}
func getLogTime(lt string) string {
	tm, err := time.Parse(time.RFC3339Nano, lt)
	if err != nil {
		return lt
	}
	return tm.Format(logTimeFormat)
}

// String - return colorized loginfo as string.
func (l logMessage) String() string {
	var hostStr string
	var b = &strings.Builder{}
	if l.NodeName != "" {
		hostStr = fmt.Sprintf("%s ", colorizedNodeName(l.NodeName))
	}
	log := l.LogInfo
	if log.ConsoleMsg != "" {
		if strings.HasPrefix(log.ConsoleMsg, "\n") {
			fmt.Fprintf(b, "%s\n", hostStr)
			log.ConsoleMsg = strings.TrimPrefix(log.ConsoleMsg, "\n")
		}
		fmt.Fprintf(b, "%s %s", hostStr, log.ConsoleMsg)
		return b.String()
	}
	if l.API != nil {
		apiString := "API: " + l.API.Name + "("
		if l.API.Args != nil && l.API.Args.Bucket != "" {
			apiString = apiString + "bucket=" + l.API.Args.Bucket
		}
		if l.API.Args != nil && l.API.Args.Object != "" {
			apiString = apiString + ", object=" + l.API.Args.Object
		}
		apiString += ")"
		fmt.Fprintf(b, "\n%s %s", hostStr, console.Colorize("API", apiString))
	}
	if l.Time != "" {
		fmt.Fprintf(b, "\n%s Time: %s", hostStr, getLogTime(l.Time))
	}
	if l.DeploymentID != "" {
		fmt.Fprintf(b, "\n%s DeploymentID: %s", hostStr, l.DeploymentID)
	}
	if l.RequestID != "" {
		fmt.Fprintf(b, "\n%s RequestID: %s", hostStr, l.RequestID)
	}
	if l.RemoteHost != "" {
		fmt.Fprintf(b, "\n%s RemoteHost: %s", hostStr, l.RemoteHost)
	}
	if l.UserAgent != "" {
		fmt.Fprintf(b, "\n%s UserAgent: %s", hostStr, l.UserAgent)
	}
	if l.Trace.Message != "" {
		fmt.Fprintf(b, "\n%s Error: %s", hostStr, console.Colorize("LogMessage", l.Trace.Message))
	}
	for key, value := range l.Trace.Variables {
		if value != "" {
			fmt.Fprintf(b, "\n%s %s=%s", hostStr, key, value)
		}
	}
	traceLength := len(l.Trace.Source)
	for i, element := range l.Trace.Source {
		fmt.Fprintf(b, "\n%s %8v: %s", hostStr, traceLength-i, element)
	}

	logMsg := strings.TrimPrefix(b.String(), "\n")
	return fmt.Sprintf("%s\n", logMsg)
}

// mainAdminConsole - the entry function of console command
func mainAdminConsole(ctx *cli.Context) error {
	// Check for command syntax
	checkAdminLogSyntax(ctx)
	console.SetColor("LogMessage", color.New(color.Bold, color.FgRed))
	console.SetColor("Api", color.New(color.Bold, color.FgWhite))
	for _, c := range colors {
		console.SetColor(fmt.Sprintf("Node%d", c), color.New(c))
	}
	aliasedURL := ctx.Args().Get(0)
	var node string
	if len(ctx.Args()) > 1 {
		node = ctx.Args().Get(1)
	}
	var limit int
	if ctx.IsSet("limit") {
		limit = ctx.Int("limit")
		if limit <= 0 {
			fatalIf(errInvalidArgument().Trace(ctx.Args()...), "please set a proper limit, for example: '--limit 5' to display last 5 logs, omit this flag to display all available logs")
		}
	}
	var logType string
	logType = strings.ToLower(ctx.String("type"))
	if logType != "minio" && logType != "application" && logType != "all" {
		fatalIf(errInvalidArgument().Trace(ctx.Args()...), "Invalid value for --type flag. Valid options are [minio, application, all]")
	}
	// Create a new MinIO Admin Client
	client, err := newAdminClient(aliasedURL)
	if err != nil {
		fatalIf(err.Trace(aliasedURL), "Cannot initialize admin client.")
		return nil
	}
	doneCh := make(chan struct{})
	defer close(doneCh)

	// Start listening on all console log activity.
	logCh := client.GetLogs(node, limit, logType, doneCh)
	for logInfo := range logCh {
		if logInfo.Err != nil {
			fatalIf(probe.NewError(logInfo.Err), "Cannot listen to console logs")
		}
		// drop nodeName from output if specified as cli arg
		if node != "" {
			logInfo.NodeName = ""
		}
		printMsg(logMessage{logInfo})
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"compress/gzip"
	"flag"
	"fmt"
	"go/format"
	"io"
	"os"
	"path/filepath"
	"text/template"
)

var tpl = template.Must(template.New("assets").Parse(`package auto

import (
	"bytes"
	"compress/gzip"
	"encoding/hex"
	"io/ioutil"
)

var Assets = make(map[string][]byte)

func init() {
	var bs []byte
	var gr *gzip.Reader
{{range $asset := .assets}}
	bs, _ = hex.DecodeString("{{$asset.HexData}}")
	gr, _ = gzip.NewReader(bytes.NewBuffer(bs))
	bs, _ = ioutil.ReadAll(gr)
	Assets["{{$asset.Name}}"] = bs
{{end}}
}
`))

type asset struct {
	Name    string
	HexData string
}

var assets []asset

func walkerFor(basePath string) filepath.WalkFunc {
	return func(name string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if info.Mode().IsRegular() {
			fd, err := os.Open(name)
			if err != nil {
				return err
			}

			var buf bytes.Buffer
			gw := gzip.NewWriter(&buf)
			io.Copy(gw, fd)
			fd.Close()
			gw.Flush()
			gw.Close()

			name, _ = filepath.Rel(basePath, name)
			assets = append(assets, asset{
				Name:    name,
				HexData: fmt.Sprintf("%x", buf.Bytes()),
			})
		}

		return nil
	}
}

func main() {
	flag.Parse()

	filepath.Walk(flag.Arg(0), walkerFor(flag.Arg(0)))
	var buf bytes.Buffer
	tpl.Execute(&buf, map[string][]asset{"assets": assets})
	bs, err := format.Source(buf.Bytes())
	if err != nil {
		panic(err)
	}
	os.Stdout.Write(bs)
}
<commit_msg>Normalize asset paths (fixes #103)<commit_after>package main

import (
	"bytes"
	"compress/gzip"
	"flag"
	"fmt"
	"go/format"
	"io"
	"os"
	"path/filepath"
	"text/template"
)

var tpl = template.Must(template.New("assets").Parse(`package auto

import (
	"bytes"
	"compress/gzip"
	"encoding/hex"
	"io/ioutil"
)

var Assets = make(map[string][]byte)

func init() {
	var bs []byte
	var gr *gzip.Reader
{{range $asset := .assets}}
	bs, _ = hex.DecodeString("{{$asset.HexData}}")
	gr, _ = gzip.NewReader(bytes.NewBuffer(bs))
	bs, _ = ioutil.ReadAll(gr)
	Assets["{{$asset.Name}}"] = bs
{{end}}
}
`))

type asset struct {
	Name    string
	HexData string
}

var assets []asset

func walkerFor(basePath string) filepath.WalkFunc {
	return func(name string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if info.Mode().IsRegular() {
			fd, err := os.Open(name)
			if err != nil {
				return err
			}

			var buf bytes.Buffer
			gw := gzip.NewWriter(&buf)
			io.Copy(gw, fd)
			fd.Close()
			gw.Flush()
			gw.Close()

			name, _ = filepath.Rel(basePath, name)
			assets = append(assets, asset{
				Name:    filepath.ToSlash(name),
				HexData: fmt.Sprintf("%x", buf.Bytes()),
			})
		}

		return nil
	}
}

func main() {
	flag.Parse()

	filepath.Walk(flag.Arg(0), walkerFor(flag.Arg(0)))
	var buf bytes.Buffer
	tpl.Execute(&buf, map[string][]asset{"assets": assets})
	bs, err := format.Source(buf.Bytes())
	if err != nil {
		panic(err)
	}
	os.Stdout.Write(bs)
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/BytemarkHosting/bytemark-client/lib"
	"github.com/BytemarkHosting/bytemark-client/lib/brain"
	"github.com/BytemarkHosting/bytemark-client/lib/prettyprint"
	"github.com/BytemarkHosting/bytemark-client/util/log"
	"github.com/urfave/cli"
	"os"
)

func init() {

	commands = append(commands, cli.Command{
		Name:        "show",
		Action:      cli.ShowSubcommandHelp,
		Usage:       `displays information about the given server, group, or account`,
		UsageText:   "bytemark show account|server|group|user [flags] <name>",
		Description: `displays information about the given server, group, or account`,
		Subcommands: []cli.Command{{
			Name:      "account",
			Usage:     `displays information about the given account`,
			UsageText: "bytemark show account [--json] [name]",
			Description: `This command displays information about the given account, including contact details and how many servers are in it across its groups.
If no account is specified, it uses your default account.

If the --json flag is specified, prints a complete overview of the account in JSON format, including all groups and their servers.`,
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "json",
					Usage: "Output account details as a JSON object",
				},
			},
			Action: With(OptionalArgs("account"), AccountProvider("account"), func(c *Context) error {
				return c.IfNotMarshalJSON(c.Account, func() error {
					def, err := global.Client.GetDefaultAccount()
					if err != nil {
						return err
					}
					if def.BrainID == c.Account.BrainID {
						c.Account.IsDefaultAccount = true
					}
					err = c.Account.PrettyPrint(os.Stderr, prettyprint.Full)
					if err != nil {
						return err
					}
					log.Output()
					log.Output()

					for _, g := range c.Account.Groups {
						for _, vm := range g.VirtualMachines {
							err := vm.PrettyPrint(os.Stderr, prettyprint.Medium)
							log.Output()
							log.Output()
							if err != nil {
								return err
							}
						}
					}
					return nil
				})
			}),
		}, {
			Name:      "group",
			Usage:     "outputs info about a group",
			UsageText: "bytemark show group [--json] [name]",
			Description: `This command displays information about how many servers are in the given group.
If the --json flag is specified, prints a complete overview of the group in JSON format, including all servers.`,
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "json",
					Usage: "Output group details as a JSON object",
				},
				cli.GenericFlag{
					Name:  "group",
					Usage: "The name of the group to show",
					Value: new(GroupNameFlag),
				},
			},
			Action: With(OptionalArgs("group"), GroupProvider("group"), func(c *Context) error {
				return c.IfNotMarshalJSON(c.Group, func() error {
					s := ""
					if len(c.Group.VirtualMachines) != 1 {
						s = "s"
					}
					log.Outputf("%s - Group containing %d cloud server%s\r\n", c.Group.Name, len(c.Group.VirtualMachines), s)

					log.Output()
					for _, vm := range c.Group.VirtualMachines {

						err := vm.PrettyPrint(os.Stderr, prettyprint.Medium)
						log.Output()
						log.Output()
						if err != nil {
							return err
						}
					}

					return nil
				})
			}),
		}, {
			Name:        "server",
			Usage:       "displays details about a server",
			UsageText:   "bytemark show server [--json] <name>",
			Description: `Displays a collection of details about the server, including its full hostname, CPU and memory allocation, power status, disc capacities and IP addresses.`,
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "json",
					Usage: "Output server details as a JSON object.",
				},
				cli.GenericFlag{
					Name:  "server",
					Usage: "the server to display",
					Value: new(VirtualMachineNameFlag),
				},
			},
			Action: With(OptionalArgs("server"), RequiredFlags("server"), VirtualMachineProvider("server"), func(c *Context) error {
				return c.IfNotMarshalJSON(c.VirtualMachine, func() error {
					return c.VirtualMachine.PrettyPrint(os.Stderr, prettyprint.Full)
				})
			}),
		}, {
			Name:        "user",
			Usage:       "displays info about a user",
			UsageText:   "bytemark show user <name>",
			Description: `Currently the only details are what SSH keys are authorised for this user`,
			Action: With(OptionalArgs("user"), RequiredFlags("user"), UserProvider("user"), func(c *Context) error {
				log.Outputf("User %s:\n\nAuthorized keys:\n", c.User.Username)
				for _, k := range c.User.AuthorizedKeys {
					log.Output(k)
				}
				return nil
			}),
		}, {
			Name:      "privileges",
			Usage:     "shows privileges for a given account, group, server, or user",
			UsageText: "bytemark show privileges",
			Description: `Displays a list of all the privileges for a given account, group, server or user. If none are specified, shows the privileges for your user.

Setting --recursive will cause a lot of extra requests to be made and may take a long time to run.

Privileges will be output in no particular order.`,
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "json",
					Usage: "Output privileges as a JSON array.",
				},
				cli.BoolFlag{
					Name:  "recursive",
					Usage: "for account & group, will also find all privileges for all groups in the account and virtual machines in the group",
				},
				cli.StringFlag{
					Name:  "user",
					Usage: "The user to show the privileges of",
				},
				cli.StringFlag{
					Name:  "account",
					Usage: "The account to show the privileges of",
				},
				cli.GenericFlag{
					Name:  "group",
					Usage: "The group to show the privileges of",
					Value: new(GroupNameFlag),
				},
				cli.GenericFlag{
					Name:  "server",
					Usage: "The server to show the privileges of",
					Value: new(VirtualMachineNameFlag),
				},
			},
			Action: With(AuthProvider, func(c *Context) (err error) {
				acc := c.String("account")
				group := c.GroupName("group")
				server := c.VirtualMachineName("server")

				privs := make(brain.Privileges, 0)
				if acc != "" {
					if c.Bool("recursive") {
						newPrivs, err := recursiveFindPrivilegesForAccount(acc)
						if err != nil {
							return err
						}
						privs = append(privs, newPrivs...)
					} else {
						newPrivs, err := global.Client.GetPrivilegesForAccount(acc)
						if err != nil {
							return err
						}
						privs = append(privs, newPrivs...)
					}
				}

				if group.Group != "" {
					if c.Bool("recursive") {
						newPrivs, err := recursiveFindPrivilegesForGroup(group)
						if err != nil {
							return err
						}
						privs = append(privs, newPrivs...)
					} else {
						newPrivs, err := global.Client.GetPrivilegesForGroup(group)
						if err != nil {
							return err
						}
						privs = append(privs, newPrivs...)
					}
				}

				if server.VirtualMachine != "" {
					newPrivs, err := global.Client.GetPrivilegesForVirtualMachine(server)
					if err != nil {
						return err
					}
					privs = append(privs, newPrivs...)
				}
				if len(privs) == 0 {

					privs, err = global.Client.GetPrivileges(c.String("user"))
					if err != nil {
						return
					}
				}

				return c.IfNotMarshalJSON(privs, func() error {
					for _, p := range privs {
						log.Outputf("%s\r\n", p.String())
					}
					return nil
				})
			}),
		}},
	})
}

func recursiveFindPrivilegesForAccount(account string) (privs brain.Privileges, err error) {
	privs, err = global.Client.GetPrivilegesForAccount(account)
	if err != nil {
		return
	}
	acc, err := global.Client.GetAccount(account)
	if err != nil {
		return
	}

	for _, group := range acc.Groups {
		newPrivs, err := recursiveFindPrivilegesForGroup(lib.GroupName{
			Group:   group.Name,
			Account: account,
		})
		if err != nil {
			return privs, err
		}
		privs = append(privs, newPrivs...)
	}
	return
}

func recursiveFindPrivilegesForGroup(name lib.GroupName) (privs brain.Privileges, err error) {
	privs, err = global.Client.GetPrivilegesForGroup(name)
	if err != nil {
		return
	}
	group, err := global.Client.GetGroup(&name)
	if err != nil {
		return
	}
	for _, vm := range group.VirtualMachines {
		vmName := lib.VirtualMachineName{
			VirtualMachine: vm.Name,
			Group:          name.Group,
			Account:        name.Account,
		}
		newPrivs, err := global.Client.GetPrivilegesForVirtualMachine(vmName)
		if err != nil {
			return privs, err
		}
		privs = append(privs, newPrivs...)
	}
	return
}
<commit_msg>improve show privileges command<commit_after>package main

import (
	"github.com/BytemarkHosting/bytemark-client/lib"
	"github.com/BytemarkHosting/bytemark-client/lib/brain"
	"github.com/BytemarkHosting/bytemark-client/lib/prettyprint"
	"github.com/BytemarkHosting/bytemark-client/util/log"
	"github.com/urfave/cli"
	"os"
)

func init() {

	commands = append(commands, cli.Command{
		Name:        "show",
		Action:      cli.ShowSubcommandHelp,
		Usage:       `displays information about the given server, group, or account`,
		UsageText:   "bytemark show account|server|group|user [flags] <name>",
		Description: `displays information about the given server, group, or account`,
		Subcommands: []cli.Command{{
			Name:      "account",
			Usage:     `displays information about the given account`,
			UsageText: "bytemark show account [--json] [name]",
			Description: `This command displays information about the given account, including contact details and how many servers are in it across its groups.
If no account is specified, it uses your default account.

If the --json flag is specified, prints a complete overview of the account in JSON format, including all groups and their servers.`,
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "json",
					Usage: "Output account details as a JSON object",
				},
			},
			Action: With(OptionalArgs("account"), AccountProvider("account"), func(c *Context) error {
				return c.IfNotMarshalJSON(c.Account, func() error {
					def, err := global.Client.GetDefaultAccount()
					if err != nil {
						return err
					}
					if def.BrainID == c.Account.BrainID {
						c.Account.IsDefaultAccount = true
					}
					err = c.Account.PrettyPrint(os.Stderr, prettyprint.Full)
					if err != nil {
						return err
					}
					log.Output()
					log.Output()

					for _, g := range c.Account.Groups {
						for _, vm := range g.VirtualMachines {
							err := vm.PrettyPrint(os.Stderr, prettyprint.Medium)
							log.Output()
							log.Output()
							if err != nil {
								return err
							}
						}
					}
					return nil
				})
			}),
		}, {
			Name:      "group",
			Usage:     "outputs info about a group",
			UsageText: "bytemark show group [--json] [name]",
			Description: `This command displays information about how many servers are in the given group.
If the --json flag is specified, prints a complete overview of the group in JSON format, including all servers.`,
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "json",
					Usage: "Output group details as a JSON object",
				},
				cli.GenericFlag{
					Name:  "group",
					Usage: "The name of the group to show",
					Value: new(GroupNameFlag),
				},
			},
			Action: With(OptionalArgs("group"), GroupProvider("group"), func(c *Context) error {
				return c.IfNotMarshalJSON(c.Group, func() error {
					s := ""
					if len(c.Group.VirtualMachines) != 1 {
						s = "s"
					}
					log.Outputf("%s - Group containing %d cloud server%s\r\n", c.Group.Name, len(c.Group.VirtualMachines), s)

					log.Output()
					for _, vm := range c.Group.VirtualMachines {

						err := vm.PrettyPrint(os.Stderr, prettyprint.Medium)
						log.Output()
						log.Output()
						if err != nil {
							return err
						}
					}

					return nil
				})
			}),
		}, {
			Name:        "server",
			Usage:       "displays details about a server",
			UsageText:   "bytemark show server [--json] <name>",
			Description: `Displays a collection of details about the server, including its full hostname, CPU and memory allocation, power status, disc capacities and IP addresses.`,
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "json",
					Usage: "Output server details as a JSON object.",
				},
				cli.GenericFlag{
					Name:  "server",
					Usage: "the server to display",
					Value: new(VirtualMachineNameFlag),
				},
			},
			Action: With(OptionalArgs("server"), RequiredFlags("server"), VirtualMachineProvider("server"), func(c *Context) error {
				return c.IfNotMarshalJSON(c.VirtualMachine, func() error {
					return c.VirtualMachine.PrettyPrint(os.Stderr, prettyprint.Full)
				})
			}),
		}, {
			Name:        "user",
			Usage:       "displays info about a user",
			UsageText:   "bytemark show user <name>",
			Description: `Currently the only details are what SSH keys are authorised for this user`,
			Action: With(OptionalArgs("user"), RequiredFlags("user"), UserProvider("user"), func(c *Context) error {
				log.Outputf("User %s:\n\nAuthorized keys:\n", c.User.Username)
				for _, k := range c.User.AuthorizedKeys {
					log.Output(k)
				}
				return nil
			}),
		}, {
			Name:      "privileges",
			Usage:     "shows privileges for a given account, group, server, or user",
			UsageText: "bytemark show privileges",
			Description: `Displays a list of all the privileges for a given account, group, server or user. If none are specified, shows the privileges for your user.

Setting --recursive will cause a lot of extra requests to be made and may take a long time to run.

Privileges will be output in no particular order.`,
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "json",
					Usage: "Output privileges as a JSON array.",
				},
				cli.BoolFlag{
					Name:  "recursive",
					Usage: "for account & group, will also find all privileges for all groups in the account and virtual machines in the group",
				},
				cli.StringFlag{
					Name:  "user",
					Usage: "The user to show the privileges of",
				},
				cli.StringFlag{
					Name:  "account",
					Usage: "The account to show the privileges of",
				},
				cli.GenericFlag{
					Name:  "group",
					Usage: "The group to show the privileges of",
					Value: new(GroupNameFlag),
				},
				cli.GenericFlag{
					Name:  "server",
					Usage: "The server to show the privileges of",
					Value: new(VirtualMachineNameFlag),
				},
			},
			Action: With(AuthProvider, func(c *Context) (err error) {
				account := c.String("account")
				group := c.GroupName("group")
				server := c.VirtualMachineName("server")

				privs := make(brain.Privileges, 0)
				if account != "" {
					newPrivs, err := findPrivilegesForAccount(account, c.Bool("recursive"))
					if err != nil {
						return err
					}
					privs = append(privs, newPrivs...)
				}

				if group.Group != "" {
					newPrivs, err := findPrivilegesForGroup(group, c.Bool("recursive"))
					if err != nil {
						return err
					}
					privs = append(privs, newPrivs...)
				}

				if server.VirtualMachine != "" {
					newPrivs, err := global.Client.GetPrivilegesForVirtualMachine(server)
					if err != nil {
						return err
					}
					privs = append(privs, newPrivs...)
				}
				if c.String("user") != "" || (server.VirtualMachine == "" && group.Group == "" && account == "") {

					privs, err = global.Client.GetPrivileges(c.String("user"))
					if err != nil {
						return
					}
				}

				return c.IfNotMarshalJSON(privs, func() error {
					for _, p := range privs {
						log.Outputf("%s\r\n", p.String())
					}
					return nil
				})
			}),
		}},
	})
}

func findPrivilegesForAccount(account string, recurse bool) (privs brain.Privileges, err error) {
	privs, err = global.Client.GetPrivilegesForAccount(account)
	if !recurse || err != nil {
		return
	}
	acc, err := global.Client.GetAccount(account)
	if err != nil {
		return
	}

	for _, group := range acc.Groups {
		newPrivs, err := findPrivilegesForGroup(lib.GroupName{
			Group:   group.Name,
			Account: account,
		}, recurse) // recurse is always true at this point but maybe I'd like to make two flags? recurse-account and recurse-group?
		if err != nil {
			return privs, err
		}
		privs = append(privs, newPrivs...)
	}
	return
}

func findPrivilegesForGroup(name lib.GroupName, recurse bool) (privs brain.Privileges, err error) {
	privs, err = global.Client.GetPrivilegesForGroup(name)
	if !recurse || err != nil {
		return
	}
	group, err := global.Client.GetGroup(&name)
	if err != nil {
		return
	}
	for _, vm := range group.VirtualMachines {
		vmName := lib.VirtualMachineName{
			VirtualMachine: vm.Name,
			Group:          name.Group,
			Account:        name.Account,
		}
		newPrivs, err := global.Client.GetPrivilegesForVirtualMachine(vmName)
		if err != nil {
			return privs, err
		}
		privs = append(privs, newPrivs...)
	}
	return
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/rand"
	"crypto/rsa"
	"fmt"
	"net"
	"os"
	"path/filepath"

	"github.com/concourse/atc/atccmd"
	"github.com/concourse/flag"
	"github.com/concourse/tsa/tsacmd"
	flags "github.com/jessevdk/go-flags"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/sigmon"

	"github.com/concourse/bin/bindata"
)

type WebCommand struct {
	*atccmd.RunCommand

	*tsacmd.TSACommand `group:"TSA Configuration" namespace:"tsa"`
}

const cliArtifactsBindata = "cli-artifacts"

func (WebCommand) lessenRequirements(command *flags.Command) {
	// defaults to address from external URL
	command.FindOptionByLongName("tsa-peer-ip").Required = false

	// defaults to atc external URL
	command.FindOptionByLongName("tsa-atc-url").Required = false

	// defaults to atc session signing key
	command.FindOptionByLongName("tsa-session-signing-key").Required = false
}

func (cmd *WebCommand) Execute(args []string) error {
	runner, err := cmd.Runner(args)
	if err != nil {
		return err
	}

	return <-ifrit.Invoke(sigmon.New(runner)).Wait()
}

func (cmd *WebCommand) Runner(args []string) (ifrit.Runner, error) {
	err := bindata.RestoreAssets(os.TempDir(), cliArtifactsBindata)
	if err == nil {
		cmd.RunCommand.CLIArtifactsDir = flag.Dir(filepath.Join(os.TempDir(), cliArtifactsBindata))
	}

	cmd.populateTSAFlagsFromATCFlags()

	atcRunner, shouldSkipTSA, err := cmd.RunCommand.Runner(args)
	if err != nil {
		return nil, err
	}

	if shouldSkipTSA {
		return atcRunner, nil
	}

	tsaRunner, err := cmd.TSACommand.Runner(args)
	if err != nil {
		return nil, err
	}

	return grouper.NewParallel(os.Interrupt, grouper.Members{
		{Name: "atc", Runner: atcRunner},
		{Name: "tsa", Runner: tsaRunner},
	}), nil
}

func (cmd *WebCommand) populateTSAFlagsFromATCFlags() error {
	cmd.TSACommand.SessionSigningKey = cmd.RunCommand.Auth.AuthFlags.SigningKey

	if cmd.RunCommand.Auth.AuthFlags.SigningKey.PrivateKey == nil &&
		cmd.TSACommand.SessionSigningKey.PrivateKey == nil {
		signingKey, err := rsa.GenerateKey(rand.Reader, 2048)
		if err != nil {
			return fmt.Errorf("failed to generate session signing key: %s", err)
		}

		cmd.RunCommand.Auth.AuthFlags.SigningKey = &flag.PrivateKey{PrivateKey: signingKey}
		cmd.TSACommand.SessionSigningKey = &flag.PrivateKey{PrivateKey: signingKey}
	}

	if len(cmd.TSACommand.ATCURLs) == 0 {
		cmd.TSACommand.ATCURLs = []flag.URL{cmd.RunCommand.PeerURL()}
	}

	host, _, err := net.SplitHostPort(cmd.RunCommand.PeerURL().URL.Host)
	if err != nil {
		return err
	}

	cmd.TSACommand.PeerIP = host

	return nil
}
<commit_msg>change function name, leave field public<commit_after>package main

import (
	"crypto/rand"
	"crypto/rsa"
	"fmt"
	"net"
	"os"
	"path/filepath"

	"github.com/concourse/atc/atccmd"
	"github.com/concourse/flag"
	"github.com/concourse/tsa/tsacmd"
	flags "github.com/jessevdk/go-flags"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/sigmon"

	"github.com/concourse/bin/bindata"
)

type WebCommand struct {
	*atccmd.RunCommand

	*tsacmd.TSACommand `group:"TSA Configuration" namespace:"tsa"`
}

const cliArtifactsBindata = "cli-artifacts"

func (WebCommand) lessenRequirements(command *flags.Command) {
	// defaults to address from external URL
	command.FindOptionByLongName("tsa-peer-ip").Required = false

	// defaults to atc external URL
	command.FindOptionByLongName("tsa-atc-url").Required = false

	// defaults to atc session signing key
	command.FindOptionByLongName("tsa-session-signing-key").Required = false
}

func (cmd *WebCommand) Execute(args []string) error {
	runner, err := cmd.Runner(args)
	if err != nil {
		return err
	}

	return <-ifrit.Invoke(sigmon.New(runner)).Wait()
}

func (cmd *WebCommand) Runner(args []string) (ifrit.Runner, error) {
	err := bindata.RestoreAssets(os.TempDir(), cliArtifactsBindata)
	if err == nil {
		cmd.RunCommand.CLIArtifactsDir = flag.Dir(filepath.Join(os.TempDir(), cliArtifactsBindata))
	}

	cmd.populateTSAFlagsFromATCFlags()

	atcRunner, shouldSkipTSA, err := cmd.RunCommand.Runner(args)
	if err != nil {
		return nil, err
	}

	if shouldSkipTSA {
		return atcRunner, nil
	}

	tsaRunner, err := cmd.TSACommand.Runner(args)
	if err != nil {
		return nil, err
	}

	return grouper.NewParallel(os.Interrupt, grouper.Members{
		{Name: "atc", Runner: atcRunner},
		{Name: "tsa", Runner: tsaRunner},
	}), nil
}

func (cmd *WebCommand) populateTSAFlagsFromATCFlags() error {
	cmd.TSACommand.SessionSigningKey = cmd.RunCommand.Auth.AuthFlags.SigningKey

	if cmd.RunCommand.Auth.AuthFlags.SigningKey.PrivateKey == nil &&
		cmd.TSACommand.SessionSigningKey.PrivateKey == nil {
		signingKey, err := rsa.GenerateKey(rand.Reader, 2048)
		if err != nil {
			return fmt.Errorf("failed to generate session signing key: %s", err)
		}

		cmd.RunCommand.Auth.AuthFlags.SigningKey = &flag.PrivateKey{PrivateKey: signingKey}
		cmd.TSACommand.SessionSigningKey = &flag.PrivateKey{PrivateKey: signingKey}
	}

	if len(cmd.TSACommand.ATCURLs) == 0 {
		cmd.TSACommand.ATCURLs = []flag.URL{cmd.RunCommand.PeerURLOrDefault()}
	}

	host, _, err := net.SplitHostPort(cmd.RunCommand.PeerURLOrDefault().URL.Host)
	if err != nil {
		return err
	}

	cmd.TSACommand.PeerIP = host

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"math"
	"os"
	"path/filepath"
	"time"

	"github.com/cloudfoundry-incubator/cf-lager"
	"github.com/cloudfoundry-incubator/executor"
	"github.com/cloudfoundry-incubator/executor/depot"
	"github.com/cloudfoundry-incubator/executor/depot/event"
	"github.com/cloudfoundry-incubator/executor/depot/metrics"
	"github.com/cloudfoundry-incubator/executor/depot/store"
	garden "github.com/cloudfoundry-incubator/garden/api"
	GardenClient "github.com/cloudfoundry-incubator/garden/client"
	GardenConnection "github.com/cloudfoundry-incubator/garden/client/connection"
	"github.com/cloudfoundry/gunk/timeprovider"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/sigmon"

	cf_debug_server "github.com/cloudfoundry-incubator/cf-debug-server"
	"github.com/cloudfoundry-incubator/executor/cmd/executor/configuration"
	"github.com/cloudfoundry-incubator/executor/depot/tallyman"
	"github.com/cloudfoundry-incubator/executor/depot/transformer"
	"github.com/cloudfoundry-incubator/executor/depot/uploader"
	"github.com/cloudfoundry-incubator/executor/http/server"
	"github.com/cloudfoundry/dropsonde"
	"github.com/cloudfoundry/dropsonde/emitter/logemitter"
	"github.com/pivotal-golang/archiver/compressor"
	"github.com/pivotal-golang/archiver/extractor"
	"github.com/pivotal-golang/cacheddownloader"
	"github.com/pivotal-golang/lager"
	"github.com/pivotal-golang/timer"
)

var listenAddr = flag.String(
	"listenAddr",
	"0.0.0.0:1700",
	"host:port to serve API requests on",
)

var containerOwnerName = flag.String(
	"containerOwnerName",
	"executor",
	"name to track containers created by this executor; they will be reaped on start",
)

var gardenNetwork = flag.String(
	"gardenNetwork",
	"unix",
	"network mode for garden server (tcp, unix)",
)

var gardenAddr = flag.String(
	"gardenAddr",
	"/tmp/garden.sock",
	"network address for garden server",
)

var memoryMBFlag = flag.String(
	"memoryMB",
	configuration.Automatic,
	"the amount of memory the executor has available in megabytes",
)

var diskMBFlag = flag.String(
	"diskMB",
	configuration.Automatic,
	"the amount of disk the executor has available in megabytes",
)

var tempDir = flag.String(
	"tempDir",
	"/tmp",
	"location to store temporary assets",
)

var loggregatorServer = flag.String(
	"loggregatorServer",
	"",
	"loggregator server to emit logs to",
)

var loggregatorSecret = flag.String(
	"loggregatorSecret",
	"",
	"secret for the loggregator server",
)

var drainTimeout = flag.Duration(
	"drainTimeout",
	15*time.Minute,
	"time to give running tasks to drain before exiting",
)

var registryPruningInterval = flag.Duration(
	"pruneInterval",
	time.Minute,
	"amount of time during which a container can remain in the allocated state",
)

var containerInodeLimit = flag.Uint64(
	"containerInodeLimit",
	200000,
	"max number of inodes per container",
)

var containerMaxCpuShares = flag.Uint64(
	"containerMaxCpuShares",
	0,
	"cpu shares allocatable to a container",
)

var cachePath = flag.String(
	"cachePath",
	"/tmp/cache",
	"location to cache assets",
)

var maxCacheSizeInBytes = flag.Uint64(
	"maxCacheSizeInBytes",
	10*1024*1024*1024,
	"maximum size of the cache (in bytes) - you should include a healthy amount of overhead",
)

var metricsReportInterval = flag.Duration(
	"metricsReportInterval",
	1*time.Minute,
	"interval on which to report metrics",
)

var maxConcurrentDownloads = flag.Uint(
	"maxConcurrentDownloads",
	20,
	"maximum in-flight downloads",
)

var gardenSyncInterval = flag.Duration(
	"gardenSyncInterval",
	30*time.Second,
	"interval on which to poll Garden's containers to ensure usage is in sync",
)

var dropsondeOrigin = flag.String(
	"dropsondeOrigin",
	"executor",
	"Origin identifier for dropsonde-emitted metrics.",
)

var dropsondeDestination = flag.String(
	"dropsondeDestination",
	"localhost:3457",
	"Destination for dropsonde-emitted metrics.",
)

func main() {
	flag.Parse()

	logger := cf_lager.New("executor")

	initializeDropsonde(logger)

	if *containerMaxCpuShares == 0 {
		logger.Error("max-cpu-shares-invalid", nil)
		os.Exit(1)
	}

	logger.Info("starting")

	cf_debug_server.Run()

	gardenClient := GardenClient.New(GardenConnection.New(*gardenNetwork, *gardenAddr))
	waitForGarden(logger, gardenClient)

	containersFetcher := &executorContainers{
		gardenClient: gardenClient,
		owner:        *containerOwnerName,
	}

	destroyContainers(gardenClient, containersFetcher, logger)

	workDir := setupWorkDir(logger, *tempDir)

	transformer := initializeTransformer(
		logger,
		*cachePath,
		workDir,
		*maxCacheSizeInBytes,
		*maxConcurrentDownloads,
	)

	tallyman := tallyman.NewTallyman()

	hub := event.NewHub()

	gardenStore, allocationStore := initializeStores(
		logger,
		gardenClient,
		transformer,
		*containerOwnerName,
		*containerMaxCpuShares,
		*containerInodeLimit,
		*loggregatorServer,
		*loggregatorSecret,
		*registryPruningInterval,
		tallyman,
		hub,
	)

	depotClient := depot.NewClient(
		fetchCapacity(logger, gardenClient),
		gardenStore,
		allocationStore,
		tallyman,
		hub,
		logger,
	)

	group := grouper.NewOrdered(os.Interrupt, grouper.Members{
		{"api-server", &server.Server{
			Address:     *listenAddr,
			Logger:      logger,
			DepotClient: depotClient,
		}},
		{"metrics-reporter", &metrics.Reporter{
			ExecutorSource: depotClient,
			Interval:       *metricsReportInterval,
			Logger:         logger.Session("metrics-reporter"),
		}},
		{"garden-syncer", gardenStore.TrackContainers(*gardenSyncInterval)},
		{"hub-drainer", drainHub(hub)},
	})

	monitor := ifrit.Invoke(sigmon.New(group))

	logger.Info("started")

	err := <-monitor.Wait()
	if err != nil {
		logger.Error("exited-with-failure", err)
		os.Exit(1)
	}

	logger.Info("exited")
}

func drainHub(hub event.Hub) ifrit.Runner {
	return ifrit.RunFunc(func(signals <-chan os.Signal, ready chan<- struct{}) error {
		close(ready)
		<-signals
		hub.Close()
		return nil
	})
}

func setupWorkDir(logger lager.Logger, tempDir string) string {
	workDir := filepath.Join(tempDir, "executor-work")

	err := os.RemoveAll(workDir)
	if err != nil {
		logger.Error("working-dir.cleanup-failed", err)
		os.Exit(1)
	}

	err = os.MkdirAll(workDir, 0755)
	if err != nil {
		logger.Error("working-dir.create-failed", err)
		os.Exit(1)
	}

	return workDir
}

func initializeDropsonde(logger lager.Logger) {
	err := dropsonde.Initialize(*dropsondeDestination, *dropsondeOrigin)
	if err != nil {
		logger.Error("failed to initialize dropsonde: %v", err)
	}
}

func initializeStores(
	logger lager.Logger,
	gardenClient garden.Client,
	transformer *transformer.Transformer,
	containerOwnerName string,
	containerMaxCpuShares uint64,
	containerInodeLimit uint64,
	loggregatorServer string,
	loggregatorSecret string,
	registryPruningInterval time.Duration,
	tallyman *tallyman.Tallyman,
	emitter store.EventEmitter,
) (*store.GardenStore, *store.AllocationStore) {
	os.Setenv("LOGGREGATOR_SHARED_SECRET", loggregatorSecret)
	logEmitter, err := logemitter.NewEmitter(loggregatorServer, "", "", false)
	if err != nil {
		panic(err)
	}

	gardenStore := store.NewGardenStore(
		logger.Session("garden-store"),
		gardenClient,
		containerOwnerName,
		containerMaxCpuShares,
		containerInodeLimit,
		logEmitter,
		transformer,
		timer.NewTimer(),
		tallyman,
		emitter,
	)

	allocationStore := store.NewAllocationStore(
		timeprovider.NewTimeProvider(),
		registryPruningInterval,
		tallyman,
		emitter,
	)

	return gardenStore, allocationStore
}

func initializeTransformer(
	logger lager.Logger,
	cachePath, workDir string,
	maxCacheSizeInBytes uint64,
	maxConcurrentDownloads uint,
) *transformer.Transformer {
	cache := cacheddownloader.New(cachePath, workDir, int64(maxCacheSizeInBytes), 10*time.Minute, int(math.MaxInt8))
	uploader := uploader.New(10*time.Minute, logger)
	extractor := extractor.NewDetectable()
	compressor := compressor.NewTgz()

	return transformer.NewTransformer(
		cache,
		uploader,
		extractor,
		compressor,
		make(chan struct{}, maxConcurrentDownloads),
		logger,
		workDir,
	)
}

func waitForGarden(logger lager.Logger, gardenClient GardenClient.Client) {
	err := gardenClient.Ping()

	for err != nil {
		logger.Error("failed-to-make-connection", err)
		time.Sleep(time.Second)
		err = gardenClient.Ping()
	}
}

func fetchCapacity(logger lager.Logger, gardenClient GardenClient.Client) executor.ExecutorResources {
	capacity, err := configuration.ConfigureCapacity(gardenClient, *memoryMBFlag, *diskMBFlag)
	if err != nil {
		logger.Error("failed-to-configure-capacity", err)
		os.Exit(1)
	}

	logger.Info("initial-capacity", lager.Data{
		"capacity": capacity,
	})

	return capacity
}

func destroyContainers(gardenClient garden.Client, containersFetcher *executorContainers, logger lager.Logger) {
	logger.Info("executor-fetching-containers-to-destroy")
	containers, err := containersFetcher.Containers()
	if err != nil {
		logger.Fatal("executor-failed-to-get-containers", err)
		return
	} else {
		logger.Info("executor-fetched-containers-to-destroy", lager.Data{"num-containers": len(containers)})
	}

	for _, container := range containers {
		logger.Info("executor-destroying-container", lager.Data{"container-handle": container.Handle()})
		err := gardenClient.Destroy(container.Handle())
		if err != nil {
			logger.Fatal("executor-failed-to-destroy-container", err, lager.Data{
				"handle": container.Handle(),
			})
		} else {
			logger.Info("executor-destroyed-stray-container", lager.Data{
				"handle": container.Handle(),
			})
		}
	}
}
<commit_msg>Change max concurrent downloads from 20 to 5. [#80393556]<commit_after>package main

import (
	"flag"
	"math"
	"os"
	"path/filepath"
	"time"

	"github.com/cloudfoundry-incubator/cf-lager"
	"github.com/cloudfoundry-incubator/executor"
	"github.com/cloudfoundry-incubator/executor/depot"
	"github.com/cloudfoundry-incubator/executor/depot/event"
	"github.com/cloudfoundry-incubator/executor/depot/metrics"
	"github.com/cloudfoundry-incubator/executor/depot/store"
	garden "github.com/cloudfoundry-incubator/garden/api"
	GardenClient "github.com/cloudfoundry-incubator/garden/client"
	GardenConnection "github.com/cloudfoundry-incubator/garden/client/connection"
	"github.com/cloudfoundry/gunk/timeprovider"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/sigmon"

	cf_debug_server "github.com/cloudfoundry-incubator/cf-debug-server"
	"github.com/cloudfoundry-incubator/executor/cmd/executor/configuration"
	"github.com/cloudfoundry-incubator/executor/depot/tallyman"
	"github.com/cloudfoundry-incubator/executor/depot/transformer"
	"github.com/cloudfoundry-incubator/executor/depot/uploader"
	"github.com/cloudfoundry-incubator/executor/http/server"
	"github.com/cloudfoundry/dropsonde"
	"github.com/cloudfoundry/dropsonde/emitter/logemitter"
	"github.com/pivotal-golang/archiver/compressor"
	"github.com/pivotal-golang/archiver/extractor"
	"github.com/pivotal-golang/cacheddownloader"
	"github.com/pivotal-golang/lager"
	"github.com/pivotal-golang/timer"
)

var listenAddr = flag.String(
	"listenAddr",
	"0.0.0.0:1700",
	"host:port to serve API requests on",
)

var containerOwnerName = flag.String(
	"containerOwnerName",
	"executor",
	"name to track containers created by this executor; they will be reaped on start",
)

var gardenNetwork = flag.String(
	"gardenNetwork",
	"unix",
	"network mode for garden server (tcp, unix)",
)

var gardenAddr = flag.String(
	"gardenAddr",
	"/tmp/garden.sock",
	"network address for garden server",
)

var memoryMBFlag = flag.String(
	"memoryMB",
	configuration.Automatic,
	"the amount of memory the executor has available in megabytes",
)

var diskMBFlag = flag.String(
	"diskMB",
	configuration.Automatic,
	"the amount of disk the executor has available in megabytes",
)

var tempDir = flag.String(
	"tempDir",
	"/tmp",
	"location to store temporary assets",
)

var loggregatorServer = flag.String(
	"loggregatorServer",
	"",
	"loggregator server to emit logs to",
)

var loggregatorSecret = flag.String(
	"loggregatorSecret",
	"",
	"secret for the loggregator server",
)

var drainTimeout = flag.Duration(
	"drainTimeout",
	15*time.Minute,
	"time to give running tasks to drain before exiting",
)

var registryPruningInterval = flag.Duration(
	"pruneInterval",
	time.Minute,
	"amount of time during which a container can remain in the allocated state",
)

var containerInodeLimit = flag.Uint64(
	"containerInodeLimit",
	200000,
	"max number of inodes per container",
)

var containerMaxCpuShares = flag.Uint64(
	"containerMaxCpuShares",
	0,
	"cpu shares allocatable to a container",
)

var cachePath = flag.String(
	"cachePath",
	"/tmp/cache",
	"location to cache assets",
)

var maxCacheSizeInBytes = flag.Uint64(
	"maxCacheSizeInBytes",
	10*1024*1024*1024,
	"maximum size of the cache (in bytes) - you should include a healthy amount of overhead",
)

var metricsReportInterval = flag.Duration(
	"metricsReportInterval",
	1*time.Minute,
	"interval on which to report metrics",
)

var maxConcurrentDownloads = flag.Uint(
	"maxConcurrentDownloads",
	5,
	"maximum in-flight downloads",
)

var gardenSyncInterval = flag.Duration(
	"gardenSyncInterval",
	30*time.Second,
	"interval on which to poll Garden's containers to ensure usage is in sync",
)

var dropsondeOrigin = flag.String(
	"dropsondeOrigin",
	"executor",
	"Origin identifier for dropsonde-emitted metrics.",
)

var dropsondeDestination = flag.String(
	"dropsondeDestination",
	"localhost:3457",
	"Destination for dropsonde-emitted metrics.",
)

func main() {
	flag.Parse()

	logger := cf_lager.New("executor")

	initializeDropsonde(logger)

	if *containerMaxCpuShares == 0 {
		logger.Error("max-cpu-shares-invalid", nil)
		os.Exit(1)
	}

	logger.Info("starting")

	cf_debug_server.Run()

	gardenClient := GardenClient.New(GardenConnection.New(*gardenNetwork, *gardenAddr))
	waitForGarden(logger, gardenClient)

	containersFetcher := &executorContainers{
		gardenClient: gardenClient,
		owner:        *containerOwnerName,
	}

	destroyContainers(gardenClient, containersFetcher, logger)

	workDir := setupWorkDir(logger, *tempDir)

	transformer := initializeTransformer(
		logger,
		*cachePath,
		workDir,
		*maxCacheSizeInBytes,
		*maxConcurrentDownloads,
	)

	tallyman := tallyman.NewTallyman()

	hub := event.NewHub()

	gardenStore, allocationStore := initializeStores(
		logger,
		gardenClient,
		transformer,
		*containerOwnerName,
		*containerMaxCpuShares,
		*containerInodeLimit,
		*loggregatorServer,
		*loggregatorSecret,
		*registryPruningInterval,
		tallyman,
		hub,
	)

	depotClient := depot.NewClient(
		fetchCapacity(logger, gardenClient),
		gardenStore,
		allocationStore,
		tallyman,
		hub,
		logger,
	)

	group := grouper.NewOrdered(os.Interrupt, grouper.Members{
		{"api-server", &server.Server{
			Address:     *listenAddr,
			Logger:      logger,
			DepotClient: depotClient,
		}},
		{"metrics-reporter", &metrics.Reporter{
			ExecutorSource: depotClient,
			Interval:       *metricsReportInterval,
			Logger:         logger.Session("metrics-reporter"),
		}},
		{"garden-syncer", gardenStore.TrackContainers(*gardenSyncInterval)},
		{"hub-drainer", drainHub(hub)},
	})

	monitor := ifrit.Invoke(sigmon.New(group))

	logger.Info("started")

	err := <-monitor.Wait()
	if err != nil {
		logger.Error("exited-with-failure", err)
		os.Exit(1)
	}

	logger.Info("exited")
}

func drainHub(hub event.Hub) ifrit.Runner {
	return ifrit.RunFunc(func(signals <-chan os.Signal, ready chan<- struct{}) error {
		close(ready)
		<-signals
		hub.Close()
		return nil
	})
}

func setupWorkDir(logger lager.Logger, tempDir string) string {
	workDir := filepath.Join(tempDir, "executor-work")

	err := os.RemoveAll(workDir)
	if err != nil {
		logger.Error("working-dir.cleanup-failed", err)
		os.Exit(1)
	}

	err = os.MkdirAll(workDir, 0755)
	if err != nil {
		logger.Error("working-dir.create-failed", err)
		os.Exit(1)
	}

	return workDir
}

func initializeDropsonde(logger lager.Logger) {
	err := dropsonde.Initialize(*dropsondeDestination, *dropsondeOrigin)
	if err != nil {
		logger.Error("failed to initialize dropsonde: %v", err)
	}
}

func initializeStores(
	logger lager.Logger,
	gardenClient garden.Client,
	transformer *transformer.Transformer,
	containerOwnerName string,
	containerMaxCpuShares uint64,
	containerInodeLimit uint64,
	loggregatorServer string,
	loggregatorSecret string,
	registryPruningInterval time.Duration,
	tallyman *tallyman.Tallyman,
	emitter store.EventEmitter,
) (*store.GardenStore, *store.AllocationStore) {
	os.Setenv("LOGGREGATOR_SHARED_SECRET", loggregatorSecret)
	logEmitter, err := logemitter.NewEmitter(loggregatorServer, "", "", false)
	if err != nil {
		panic(err)
	}

	gardenStore := store.NewGardenStore(
		logger.Session("garden-store"),
		gardenClient,
		containerOwnerName,
		containerMaxCpuShares,
		containerInodeLimit,
		logEmitter,
		transformer,
		timer.NewTimer(),
		tallyman,
		emitter,
	)

	allocationStore := store.NewAllocationStore(
		timeprovider.NewTimeProvider(),
		registryPruningInterval,
		tallyman,
		emitter,
	)

	return gardenStore, allocationStore
}

func initializeTransformer(
	logger lager.Logger,
	cachePath, workDir string,
	maxCacheSizeInBytes uint64,
	maxConcurrentDownloads uint,
) *transformer.Transformer {
	cache := cacheddownloader.New(cachePath, workDir, int64(maxCacheSizeInBytes), 10*time.Minute, int(math.MaxInt8))
	uploader := uploader.New(10*time.Minute, logger)
	extractor := extractor.NewDetectable()
	compressor := compressor.NewTgz()

	return transformer.NewTransformer(
		cache,
		uploader,
		extractor,
		compressor,
		make(chan struct{}, maxConcurrentDownloads),
		logger,
		workDir,
	)
}

func waitForGarden(logger lager.Logger, gardenClient GardenClient.Client) {
	err := gardenClient.Ping()

	for err != nil {
		logger.Error("failed-to-make-connection", err)
		time.Sleep(time.Second)
		err = gardenClient.Ping()
	}
}

func fetchCapacity(logger lager.Logger, gardenClient GardenClient.Client) executor.ExecutorResources {
	capacity, err := configuration.ConfigureCapacity(gardenClient, *memoryMBFlag, *diskMBFlag)
	if err != nil {
		logger.Error("failed-to-configure-capacity", err)
		os.Exit(1)
	}

	logger.Info("initial-capacity", lager.Data{
		"capacity": capacity,
	})

	return capacity
}

func destroyContainers(gardenClient garden.Client, containersFetcher *executorContainers, logger lager.Logger) {
	logger.Info("executor-fetching-containers-to-destroy")
	containers, err := containersFetcher.Containers()
	if err != nil {
		logger.Fatal("executor-failed-to-get-containers", err)
		return
	} else {
		logger.Info("executor-fetched-containers-to-destroy", lager.Data{"num-containers": len(containers)})
	}

	for _, container := range containers {
		logger.Info("executor-destroying-container", lager.Data{"container-handle": container.Handle()})
		err := gardenClient.Destroy(container.Handle())
		if err != nil {
			logger.Fatal("executor-failed-to-destroy-container", err, lager.Data{
				"handle": container.Handle(),
			})
		} else {
			logger.Info("executor-destroyed-stray-container", lager.Data{
				"handle": container.Handle(),
			})
		}
	}
}
<|endoftext|>