<commit_before>// Copyright 2014 docker-cluster authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cluster

import (
	"errors"
	"fmt"
	"net"
	"net/url"
	"sync"

	"github.com/fsouza/go-dockerclient"
	"github.com/tsuru/docker-cluster/log"
)

type Container struct {
	Id   string `bson:"_id"`
	Host string
}

// CreateContainer creates a container in the specified node. If no node is
// specified, it will create the container in a node selected by the scheduler.
//
// It returns the container, or an error, in case of failures.
func (c *Cluster) CreateContainer(opts docker.CreateContainerOptions, nodes ...string) (string, *docker.Container, error) {
	return c.CreateContainerSchedulerOpts(opts, nil, nodes...)
}

// Similar to CreateContainer but allows arbritary options to be passed to
// the scheduler.
func (c *Cluster) CreateContainerSchedulerOpts(opts docker.CreateContainerOptions, schedulerOpts SchedulerOptions, nodes ...string) (string, *docker.Container, error) {
	var (
		addr      string
		container *docker.Container
		err       error
	)
	useScheduler := len(nodes) == 0
	maxTries := 5
	for ; maxTries > 0; maxTries-- {
		if useScheduler {
			node, scheduleErr := c.scheduler.Schedule(c, opts, schedulerOpts)
			if scheduleErr != nil {
				if err != nil {
					scheduleErr = fmt.Errorf("Error in scheduler after previous errors (%s) trying to create container: %s", err.Error(), scheduleErr.Error())
				}
				return addr, nil, scheduleErr
			}
			addr = node.Address
		} else {
			addr = nodes[0]
		}
		if addr == "" {
			return addr, nil, errors.New("CreateContainer needs a non empty node addr")
		}
		err = nil
		if c.Hook != nil {
			dbNode, retrieveErr := c.storage().RetrieveNode(addr)
			if retrieveErr != nil {
				retrieveErr = fmt.Errorf("could not retrieve node from cluster: %s", retrieveErr)
				return addr, nil, retrieveErr
			}
			err = c.Hook.BeforeCreateContainer(dbNode)
			if err != nil {
				log.Errorf("Error in before create container hook in node %q: %s. Trying again in another node...", addr, err)
			}
		}
		if err == nil {
			container, err = c.createContainerInNode(opts, addr)
			if err == nil {
				c.handleNodeSuccess(addr)
				break
			}
			log.Errorf("Error trying to create container in node %q: %s. Trying again in another node...", addr, err.Error())
		}
		shouldIncrementFailures := false
		isCreateContainerErr := false
		baseErr := err
		if nodeErr, ok := baseErr.(DockerNodeError); ok {
			isCreateContainerErr = nodeErr.cmd == "createContainer"
			baseErr = nodeErr.BaseError()
		}
		if urlErr, ok := baseErr.(*url.Error); ok {
			baseErr = urlErr.Err
		}
		_, isNetErr := baseErr.(*net.OpError)
		if isNetErr || isCreateContainerErr || baseErr == docker.ErrConnectionRefused {
			shouldIncrementFailures = true
		}
		c.handleNodeError(addr, err, shouldIncrementFailures)
		if !useScheduler {
			return addr, nil, err
		}
	}
	if err != nil {
		return addr, nil, fmt.Errorf("CreateContainer: maximum number of tries exceeded, last error: %s", err.Error())
	}
	err = c.storage().StoreContainer(container.ID, addr)
	return addr, container, err
}

func (c *Cluster) createContainerInNode(opts docker.CreateContainerOptions, nodeAddress string) (*docker.Container, error) {
	registryServer, _ := parseImageRegistry(opts.Config.Image)
	if registryServer != "" {
		err := c.PullImage(docker.PullImageOptions{
			Repository: opts.Config.Image,
		}, docker.AuthConfiguration{}, nodeAddress)
		if err != nil {
			return nil, err
		}
	}
	node, err := c.getNodeByAddr(nodeAddress)
	if err != nil {
		return nil, err
	}
	cont, err := node.CreateContainer(opts)
	return cont, wrapErrorWithCmd(node, err, "createContainer")
}

// InspectContainer returns information about a container by its ID, getting
// the information from the right node.
func (c *Cluster) InspectContainer(id string) (*docker.Container, error) {
	node, err := c.getNodeForContainer(id)
	if err != nil {
		return nil, err
	}
	cont, err := node.InspectContainer(id)
	return cont, wrapError(node, err)
}

// KillContainer kills a container, returning an error in case of failure.
func (c *Cluster) KillContainer(opts docker.KillContainerOptions) error {
	node, err := c.getNodeForContainer(opts.ID)
	if err != nil {
		return err
	}
	return wrapError(node, node.KillContainer(opts))
}

// ListContainers returns a slice of all containers in the cluster matching the
// given criteria.
func (c *Cluster) ListContainers(opts docker.ListContainersOptions) ([]docker.APIContainers, error) {
	nodes, err := c.Nodes()
	if err != nil {
		return nil, err
	}
	var wg sync.WaitGroup
	result := make(chan []docker.APIContainers, len(nodes))
	errs := make(chan error, len(nodes))
	for _, n := range nodes {
		wg.Add(1)
		client, _ := c.getNodeByAddr(n.Address)
		go func(n node) {
			defer wg.Done()
			if containers, err := n.ListContainers(opts); err != nil {
				errs <- wrapError(n, err)
			} else {
				result <- containers
			}
		}(client)
	}
	wg.Wait()
	var group []docker.APIContainers
	for {
		select {
		case containers := <-result:
			group = append(group, containers...)
		case err = <-errs:
		default:
			return group, err
		}
	}
}

// RemoveContainer removes a container from the cluster.
func (c *Cluster) RemoveContainer(opts docker.RemoveContainerOptions) error {
	return c.removeFromStorage(opts)
}

func (c *Cluster) removeFromStorage(opts docker.RemoveContainerOptions) error {
	node, err := c.getNodeForContainer(opts.ID)
	if err != nil {
		return err
	}
	err = node.RemoveContainer(opts)
	if err != nil {
		_, isNoSuchContainer := err.(*docker.NoSuchContainer)
		if !isNoSuchContainer {
			return wrapError(node, err)
		}
	}
	return c.storage().RemoveContainer(opts.ID)
}

func (c *Cluster) StartContainer(id string, hostConfig *docker.HostConfig) error {
	node, err := c.getNodeForContainer(id)
	if err != nil {
		return err
	}
	return wrapError(node, node.StartContainer(id, hostConfig))
}

// StopContainer stops a container, killing it after the given timeout, if it
// fails to stop nicely.
func (c *Cluster) StopContainer(id string, timeout uint) error {
	node, err := c.getNodeForContainer(id)
	if err != nil {
		return err
	}
	return wrapError(node, node.StopContainer(id, timeout))
}

// RestartContainer restarts a container, killing it after the given timeout,
// if it fails to stop nicely.
func (c *Cluster) RestartContainer(id string, timeout uint) error {
	node, err := c.getNodeForContainer(id)
	if err != nil {
		return err
	}
	return wrapError(node, node.RestartContainer(id, timeout))
}

// PauseContainer changes the container to the paused state.
func (c *Cluster) PauseContainer(id string) error {
	node, err := c.getNodeForContainer(id)
	if err != nil {
		return err
	}
	return wrapError(node, node.PauseContainer(id))
}

// UnpauseContainer removes the container from the paused state.
func (c *Cluster) UnpauseContainer(id string) error {
	node, err := c.getNodeForContainer(id)
	if err != nil {
		return err
	}
	return wrapError(node, node.UnpauseContainer(id))
}

// WaitContainer blocks until the given container stops, returning the exit
// code of the container command.
func (c *Cluster) WaitContainer(id string) (int, error) {
	node, err := c.getNodeForContainer(id)
	if err != nil {
		return -1, err
	}
	node.setPersistentClient()
	code, err := node.WaitContainer(id)
	return code, wrapError(node, err)
}

// AttachToContainer attaches to a container, using the given options.
func (c *Cluster) AttachToContainer(opts docker.AttachToContainerOptions) error {
	node, err := c.getNodeForContainer(opts.Container)
	if err != nil {
		return err
	}
	node.setPersistentClient()
	return wrapError(node, node.AttachToContainer(opts))
}

// Logs retrieves the logs of the specified container.
func (c *Cluster) Logs(opts docker.LogsOptions) error {
	node, err := c.getNodeForContainer(opts.Container)
	if err != nil {
		return err
	}
	return wrapError(node, node.Logs(opts))
}

// CommitContainer commits a container and returns the image id.
func (c *Cluster) CommitContainer(opts docker.CommitContainerOptions) (*docker.Image, error) {
	node, err := c.getNodeForContainer(opts.Container)
	if err != nil {
		return nil, err
	}
	image, err := node.CommitContainer(opts)
	if err != nil {
		return nil, wrapError(node, err)
	}
	key := imageKey(opts.Repository, opts.Tag)
	if key != "" {
		err = c.storage().StoreImage(key, image.ID, node.addr)
		if err != nil {
			return nil, err
		}
	}
	return image, nil
}

// ExportContainer exports a container as a tar and writes
// the result in out.
func (c *Cluster) ExportContainer(opts docker.ExportContainerOptions) error {
	node, err := c.getNodeForContainer(opts.ID)
	if err != nil {
		return err
	}
	return wrapError(node, node.ExportContainer(opts))
}

// TopContainer returns information about running processes inside a container
// by its ID, getting the information from the right node.
func (c *Cluster) TopContainer(id string, psArgs string) (docker.TopResult, error) {
	node, err := c.getNodeForContainer(id)
	if err != nil {
		return docker.TopResult{}, err
	}
	result, err := node.TopContainer(id, psArgs)
	return result, wrapError(node, err)
}

func (c *Cluster) getNodeForContainer(container string) (node, error) {
	return c.getNode(func(s Storage) (string, error) {
		return s.RetrieveContainer(container)
	})
}

func (c *Cluster) CreateExec(opts docker.CreateExecOptions) (*docker.Exec, error) {
	node, err := c.getNodeForContainer(opts.Container)
	if err != nil {
		return nil, err
	}
	exec, err := node.CreateExec(opts)
	return exec, wrapError(node, err)
}

func (c *Cluster) StartExec(execId, containerId string, opts docker.StartExecOptions) error {
	node, err := c.getNodeForContainer(containerId)
	if err != nil {
		return err
	}
	node.setPersistentClient()
	return wrapError(node, node.StartExec(execId, opts))
}

func (c *Cluster) ResizeExecTTY(execId, containerId string, height, width int) error {
	node, err := c.getNodeForContainer(containerId)
	if err != nil {
		return err
	}
	return wrapError(node, node.ResizeExecTTY(execId, height, width))
}

func (c *Cluster) InspectExec(execId, containerId string) (*docker.ExecInspect, error) {
	node, err := c.getNodeForContainer(containerId)
	if err != nil {
		return nil, err
	}
	execInspect, err := node.InspectExec(execId)
	if err != nil {
		return nil, wrapError(node, err)
	}
	return execInspect, nil
}
<commit_msg>cluster: use persistent client on commit<commit_after>// Copyright 2014 docker-cluster authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cluster

import (
	"errors"
	"fmt"
	"net"
	"net/url"
	"sync"

	"github.com/fsouza/go-dockerclient"
	"github.com/tsuru/docker-cluster/log"
)

type Container struct {
	Id   string `bson:"_id"`
	Host string
}

// CreateContainer creates a container in the specified node. If no node is
// specified, it will create the container in a node selected by the scheduler.
//
// It returns the container, or an error, in case of failures.
func (c *Cluster) CreateContainer(opts docker.CreateContainerOptions, nodes ...string) (string, *docker.Container, error) {
	return c.CreateContainerSchedulerOpts(opts, nil, nodes...)
}

// Similar to CreateContainer but allows arbritary options to be passed to
// the scheduler.
func (c *Cluster) CreateContainerSchedulerOpts(opts docker.CreateContainerOptions, schedulerOpts SchedulerOptions, nodes ...string) (string, *docker.Container, error) {
	var (
		addr      string
		container *docker.Container
		err       error
	)
	useScheduler := len(nodes) == 0
	maxTries := 5
	for ; maxTries > 0; maxTries-- {
		if useScheduler {
			node, scheduleErr := c.scheduler.Schedule(c, opts, schedulerOpts)
			if scheduleErr != nil {
				if err != nil {
					scheduleErr = fmt.Errorf("Error in scheduler after previous errors (%s) trying to create container: %s", err.Error(), scheduleErr.Error())
				}
				return addr, nil, scheduleErr
			}
			addr = node.Address
		} else {
			addr = nodes[0]
		}
		if addr == "" {
			return addr, nil, errors.New("CreateContainer needs a non empty node addr")
		}
		err = nil
		if c.Hook != nil {
			dbNode, retrieveErr := c.storage().RetrieveNode(addr)
			if retrieveErr != nil {
				retrieveErr = fmt.Errorf("could not retrieve node from cluster: %s", retrieveErr)
				return addr, nil, retrieveErr
			}
			err = c.Hook.BeforeCreateContainer(dbNode)
			if err != nil {
				log.Errorf("Error in before create container hook in node %q: %s. Trying again in another node...", addr, err)
			}
		}
		if err == nil {
			container, err = c.createContainerInNode(opts, addr)
			if err == nil {
				c.handleNodeSuccess(addr)
				break
			}
			log.Errorf("Error trying to create container in node %q: %s. Trying again in another node...", addr, err.Error())
		}
		shouldIncrementFailures := false
		isCreateContainerErr := false
		baseErr := err
		if nodeErr, ok := baseErr.(DockerNodeError); ok {
			isCreateContainerErr = nodeErr.cmd == "createContainer"
			baseErr = nodeErr.BaseError()
		}
		if urlErr, ok := baseErr.(*url.Error); ok {
			baseErr = urlErr.Err
		}
		_, isNetErr := baseErr.(*net.OpError)
		if isNetErr || isCreateContainerErr || baseErr == docker.ErrConnectionRefused {
			shouldIncrementFailures = true
		}
		c.handleNodeError(addr, err, shouldIncrementFailures)
		if !useScheduler {
			return addr, nil, err
		}
	}
	if err != nil {
		return addr, nil, fmt.Errorf("CreateContainer: maximum number of tries exceeded, last error: %s", err.Error())
	}
	err = c.storage().StoreContainer(container.ID, addr)
	return addr, container, err
}

func (c *Cluster) createContainerInNode(opts docker.CreateContainerOptions, nodeAddress string) (*docker.Container, error) {
	registryServer, _ := parseImageRegistry(opts.Config.Image)
	if registryServer != "" {
		err := c.PullImage(docker.PullImageOptions{
			Repository: opts.Config.Image,
		}, docker.AuthConfiguration{}, nodeAddress)
		if err != nil {
			return nil, err
		}
	}
	node, err := c.getNodeByAddr(nodeAddress)
	if err != nil {
		return nil, err
	}
	cont, err := node.CreateContainer(opts)
	return cont, wrapErrorWithCmd(node, err, "createContainer")
}

// InspectContainer returns information about a container by its ID, getting
// the information from the right node.
func (c *Cluster) InspectContainer(id string) (*docker.Container, error) {
	node, err := c.getNodeForContainer(id)
	if err != nil {
		return nil, err
	}
	cont, err := node.InspectContainer(id)
	return cont, wrapError(node, err)
}

// KillContainer kills a container, returning an error in case of failure.
func (c *Cluster) KillContainer(opts docker.KillContainerOptions) error {
	node, err := c.getNodeForContainer(opts.ID)
	if err != nil {
		return err
	}
	return wrapError(node, node.KillContainer(opts))
}

// ListContainers returns a slice of all containers in the cluster matching the
// given criteria.
func (c *Cluster) ListContainers(opts docker.ListContainersOptions) ([]docker.APIContainers, error) {
	nodes, err := c.Nodes()
	if err != nil {
		return nil, err
	}
	var wg sync.WaitGroup
	result := make(chan []docker.APIContainers, len(nodes))
	errs := make(chan error, len(nodes))
	for _, n := range nodes {
		wg.Add(1)
		client, _ := c.getNodeByAddr(n.Address)
		go func(n node) {
			defer wg.Done()
			if containers, err := n.ListContainers(opts); err != nil {
				errs <- wrapError(n, err)
			} else {
				result <- containers
			}
		}(client)
	}
	wg.Wait()
	var group []docker.APIContainers
	for {
		select {
		case containers := <-result:
			group = append(group, containers...)
		case err = <-errs:
		default:
			return group, err
		}
	}
}

// RemoveContainer removes a container from the cluster.
func (c *Cluster) RemoveContainer(opts docker.RemoveContainerOptions) error {
	return c.removeFromStorage(opts)
}

func (c *Cluster) removeFromStorage(opts docker.RemoveContainerOptions) error {
	node, err := c.getNodeForContainer(opts.ID)
	if err != nil {
		return err
	}
	err = node.RemoveContainer(opts)
	if err != nil {
		_, isNoSuchContainer := err.(*docker.NoSuchContainer)
		if !isNoSuchContainer {
			return wrapError(node, err)
		}
	}
	return c.storage().RemoveContainer(opts.ID)
}

func (c *Cluster) StartContainer(id string, hostConfig *docker.HostConfig) error {
	node, err := c.getNodeForContainer(id)
	if err != nil {
		return err
	}
	return wrapError(node, node.StartContainer(id, hostConfig))
}

// StopContainer stops a container, killing it after the given timeout, if it
// fails to stop nicely.
func (c *Cluster) StopContainer(id string, timeout uint) error {
	node, err := c.getNodeForContainer(id)
	if err != nil {
		return err
	}
	return wrapError(node, node.StopContainer(id, timeout))
}

// RestartContainer restarts a container, killing it after the given timeout,
// if it fails to stop nicely.
func (c *Cluster) RestartContainer(id string, timeout uint) error {
	node, err := c.getNodeForContainer(id)
	if err != nil {
		return err
	}
	return wrapError(node, node.RestartContainer(id, timeout))
}

// PauseContainer changes the container to the paused state.
func (c *Cluster) PauseContainer(id string) error {
	node, err := c.getNodeForContainer(id)
	if err != nil {
		return err
	}
	return wrapError(node, node.PauseContainer(id))
}

// UnpauseContainer removes the container from the paused state.
func (c *Cluster) UnpauseContainer(id string) error {
	node, err := c.getNodeForContainer(id)
	if err != nil {
		return err
	}
	return wrapError(node, node.UnpauseContainer(id))
}

// WaitContainer blocks until the given container stops, returning the exit
// code of the container command.
func (c *Cluster) WaitContainer(id string) (int, error) {
	node, err := c.getNodeForContainer(id)
	if err != nil {
		return -1, err
	}
	node.setPersistentClient()
	code, err := node.WaitContainer(id)
	return code, wrapError(node, err)
}

// AttachToContainer attaches to a container, using the given options.
func (c *Cluster) AttachToContainer(opts docker.AttachToContainerOptions) error {
	node, err := c.getNodeForContainer(opts.Container)
	if err != nil {
		return err
	}
	node.setPersistentClient()
	return wrapError(node, node.AttachToContainer(opts))
}

// Logs retrieves the logs of the specified container.
func (c *Cluster) Logs(opts docker.LogsOptions) error {
	node, err := c.getNodeForContainer(opts.Container)
	if err != nil {
		return err
	}
	return wrapError(node, node.Logs(opts))
}

// CommitContainer commits a container and returns the image id.
func (c *Cluster) CommitContainer(opts docker.CommitContainerOptions) (*docker.Image, error) {
	node, err := c.getNodeForContainer(opts.Container)
	if err != nil {
		return nil, err
	}
	node.setPersistentClient()
	image, err := node.CommitContainer(opts)
	if err != nil {
		return nil, wrapError(node, err)
	}
	key := imageKey(opts.Repository, opts.Tag)
	if key != "" {
		err = c.storage().StoreImage(key, image.ID, node.addr)
		if err != nil {
			return nil, err
		}
	}
	return image, nil
}

// ExportContainer exports a container as a tar and writes
// the result in out.
func (c *Cluster) ExportContainer(opts docker.ExportContainerOptions) error {
	node, err := c.getNodeForContainer(opts.ID)
	if err != nil {
		return err
	}
	return wrapError(node, node.ExportContainer(opts))
}

// TopContainer returns information about running processes inside a container
// by its ID, getting the information from the right node.
func (c *Cluster) TopContainer(id string, psArgs string) (docker.TopResult, error) {
	node, err := c.getNodeForContainer(id)
	if err != nil {
		return docker.TopResult{}, err
	}
	result, err := node.TopContainer(id, psArgs)
	return result, wrapError(node, err)
}

func (c *Cluster) getNodeForContainer(container string) (node, error) {
	return c.getNode(func(s Storage) (string, error) {
		return s.RetrieveContainer(container)
	})
}

func (c *Cluster) CreateExec(opts docker.CreateExecOptions) (*docker.Exec, error) {
	node, err := c.getNodeForContainer(opts.Container)
	if err != nil {
		return nil, err
	}
	exec, err := node.CreateExec(opts)
	return exec, wrapError(node, err)
}

func (c *Cluster) StartExec(execId, containerId string, opts docker.StartExecOptions) error {
	node, err := c.getNodeForContainer(containerId)
	if err != nil {
		return err
	}
	node.setPersistentClient()
	return wrapError(node, node.StartExec(execId, opts))
}

func (c *Cluster) ResizeExecTTY(execId, containerId string, height, width int) error {
	node, err := c.getNodeForContainer(containerId)
	if err != nil {
		return err
	}
	return wrapError(node, node.ResizeExecTTY(execId, height, width))
}

func (c *Cluster) InspectExec(execId, containerId string) (*docker.ExecInspect, error) {
	node, err := c.getNodeForContainer(containerId)
	if err != nil {
		return nil, err
	}
	execInspect, err := node.InspectExec(execId)
	if err != nil {
		return nil, wrapError(node, err)
	}
	return execInspect, nil
}
<|endoftext|>


<commit_before>package cluster

import (
	"time"

	"github.com/AsynkronIT/protoactor-go/actor"
	"github.com/AsynkronIT/protoactor-go/eventstream"
	"github.com/AsynkronIT/protoactor-go/log"
	"github.com/AsynkronIT/protoactor-go/remote"
)

var (
	pidCacheActorPid *actor.PID
	memberStatusSub  *eventstream.Subscription
)

func spawnPidCacheActor() {
	props := actor.FromProducer(newPidCacheActor())
	pidCacheActorPid, _ = actor.SpawnNamed(props, "PidCache")
}

func stopPidCacheActor() {
	pidCacheActorPid.GracefulStop()
}

func newPidCacheActor() actor.Producer {
	return func() actor.Actor {
		return &pidCachePartitionActor{}
	}
}

func subscribePidCacheMemberStatusEventStream() {
	memberStatusSub = eventstream.
		Subscribe(pidCacheActorPid.Tell).
		WithPredicate(func(m interface{}) bool {
			_, ok := m.(MemberStatusEvent)
			return ok
		})
}

func unsubPidCacheMemberStatusEventStream() {
	eventstream.Unsubscribe(memberStatusSub)
}

type pidCachePartitionActor struct {
	Cache                       map[string]*actor.PID
	ReverseCache                map[string]string
	ReverseCacheByMemberAddress map[string]keySet
}

type keySet map[string]bool

func (s keySet) add(val string)    { s[val] = true }
func (s keySet) remove(val string) { delete(s, val) }

type pidCacheRequest struct {
	name string
	kind string
}

func (p *pidCacheRequest) Hash() string {
	return p.name
}

type removePidCacheRequest struct {
	name string
}

func (p *removePidCacheRequest) Hash() string {
	return p.name
}

type pidCacheResponse struct {
	pid    *actor.PID
	status remote.ResponseStatusCode
}

func (a *pidCachePartitionActor) Receive(ctx actor.Context) {
	switch msg := ctx.Message().(type) {
	case *actor.Started:
		a.Cache = make(map[string]*actor.PID)
		a.ReverseCache = make(map[string]string)
		a.ReverseCacheByMemberAddress = make(map[string]keySet)

	case *pidCacheRequest:
		if pid, ok := a.Cache[msg.name]; ok {
			//name was in cache, exit early
			ctx.Respond(&pidCacheResponse{pid: pid})
			return
		}
		name := msg.name
		kind := msg.kind

		address := getMember(name, kind)
		if address == "" {
			//No available member found
			ctx.Respond(&pidCacheResponse{status: remote.ResponseStatusCodeUNAVAILABLE})
			return
		}

		remotePartition := partitionForKind(address, kind)

		//re-package the request as a remote.ActorPidRequest
		req := &remote.ActorPidRequest{
			Kind: kind,
			Name: name,
		}
		//ask the DHT partition for this name to give us a PID
		f := remotePartition.RequestFuture(req, 5*time.Second)
		ctx.AwaitFuture(f, func(r interface{}, err error) {
			if err != nil {
				ctx.Respond(&pidCacheResponse{status: remote.ResponseStatusCodeERROR})
				return
			}
			response, ok := r.(*remote.ActorPidResponse)
			if !ok {
				ctx.Respond(&pidCacheResponse{status: remote.ResponseStatusCodeERROR})
				return
			}

			statusCode := remote.ResponseStatusCode(response.StatusCode)
			switch statusCode {
			case remote.ResponseStatusCodeOK:
				key := response.Pid.String()

				a.Cache[name] = response.Pid
				//make a lookup from pid to name
				a.ReverseCache[key] = name
				//add to member address map
				if ks, ok := a.ReverseCacheByMemberAddress[response.Pid.Address]; ok {
					ks.add(key)
				} else {
					a.ReverseCacheByMemberAddress[response.Pid.Address] = keySet{key: true}
				}

				//watch the pid so we know if the node or pid dies
				ctx.Watch(response.Pid)
				//tell the original requester that we have a response
				ctx.Respond(&pidCacheResponse{response.Pid, statusCode})
			default:
				//forward to requester
				ctx.Respond(&pidCacheResponse{response.Pid, statusCode})
			}
		})

	case *MemberLeftEvent:
		address := msg.Name()
		a.removeCacheByMemberAddress(address)
	case *MemberRejoinedEvent:
		address := msg.Name()
		a.removeCacheByMemberAddress(address)
	case *actor.Terminated:
		a.removeCacheByPid(msg.Who)
	case *removePidCacheRequest:
		a.removeCacheByName(msg.name)
	}
}

func (a *pidCachePartitionActor) removeCacheByPid(pid *actor.PID) {
	key := pid.String()
	//get the virtual name from the pid
	name, ok := a.ReverseCache[key]
	if !ok {
		//we don't have it, just ignore
		return
	}
	//drop both lookups as this actor is now dead
	delete(a.Cache, name)
	delete(a.ReverseCache, key)
	if ks, ok := a.ReverseCacheByMemberAddress[pid.Address]; ok {
		ks.remove(key)
	}
}

func (a *pidCachePartitionActor) removeCacheByName(name string) {
	if pid, ok := a.Cache[name]; ok {
		key := pid.String()
		delete(a.Cache, name)
		delete(a.ReverseCache, key)
		if ks, ok := a.ReverseCacheByMemberAddress[pid.Address]; ok {
			ks.remove(key)
		}
	}
}

func (a *pidCachePartitionActor) removeCacheByMemberAddress(address string) {
	if ks, ok := a.ReverseCacheByMemberAddress[address]; ok {
		for k := range ks {
			if n, ok := a.ReverseCache[k]; ok {
				delete(a.Cache, n)
				delete(a.ReverseCache, k)
			}
		}
		delete(a.ReverseCacheByMemberAddress, address)
		plog.Error("PID caches removed from PidCache", log.Object("address", address))
	}
}
<commit_msg>Remove unnecessary reverseCacheByMemberAddress, following the logic of Partition actor.<commit_after>package cluster

import (
	"time"

	"github.com/AsynkronIT/protoactor-go/actor"
	"github.com/AsynkronIT/protoactor-go/eventstream"
	"github.com/AsynkronIT/protoactor-go/log"
	"github.com/AsynkronIT/protoactor-go/remote"
)

var (
	pidCacheActorPid *actor.PID
	memberStatusSub  *eventstream.Subscription
)

func spawnPidCacheActor() {
	props := actor.FromProducer(newPidCacheActor())
	pidCacheActorPid, _ = actor.SpawnNamed(props, "PidCache")
}

func stopPidCacheActor() {
	pidCacheActorPid.GracefulStop()
}

func newPidCacheActor() actor.Producer {
	return func() actor.Actor {
		return &pidCachePartitionActor{}
	}
}

func subscribePidCacheMemberStatusEventStream() {
	memberStatusSub = eventstream.
		Subscribe(pidCacheActorPid.Tell).
		WithPredicate(func(m interface{}) bool {
			_, ok := m.(MemberStatusEvent)
			return ok
		})
}

func unsubPidCacheMemberStatusEventStream() {
	eventstream.Unsubscribe(memberStatusSub)
}

type pidCachePartitionActor struct {
	Cache        map[string]*actor.PID
	ReverseCache map[string]string
}

type keySet map[string]bool

func (s keySet) add(val string)    { s[val] = true }
func (s keySet) remove(val string) { delete(s, val) }

type pidCacheRequest struct {
	name string
	kind string
}

func (p *pidCacheRequest) Hash() string {
	return p.name
}

type removePidCacheRequest struct {
	name string
}

func (p *removePidCacheRequest) Hash() string {
	return p.name
}

type pidCacheResponse struct {
	pid    *actor.PID
	status remote.ResponseStatusCode
}

func (a *pidCachePartitionActor) Receive(ctx actor.Context) {
	switch msg := ctx.Message().(type) {
	case *actor.Started:
		a.Cache = make(map[string]*actor.PID)
		a.ReverseCache = make(map[string]string)

	case *pidCacheRequest:
		if pid, ok := a.Cache[msg.name]; ok {
			//name was in cache, exit early
			ctx.Respond(&pidCacheResponse{pid: pid})
			return
		}
		name := msg.name
		kind := msg.kind

		address := getMember(name, kind)
		if address == "" {
			//No available member found
			ctx.Respond(&pidCacheResponse{status: remote.ResponseStatusCodeUNAVAILABLE})
			return
		}

		remotePartition := partitionForKind(address, kind)

		//re-package the request as a remote.ActorPidRequest
		req := &remote.ActorPidRequest{
			Kind: kind,
			Name: name,
		}
		//ask the DHT partition for this name to give us a PID
		f := remotePartition.RequestFuture(req, 5*time.Second)
		ctx.AwaitFuture(f, func(r interface{}, err error) {
			if err != nil {
				ctx.Respond(&pidCacheResponse{status: remote.ResponseStatusCodeERROR})
				return
			}
			response, ok := r.(*remote.ActorPidResponse)
			if !ok {
				ctx.Respond(&pidCacheResponse{status: remote.ResponseStatusCodeERROR})
				return
			}

			statusCode := remote.ResponseStatusCode(response.StatusCode)
			switch statusCode {
			case remote.ResponseStatusCodeOK:
				key := response.Pid.String()

				a.Cache[name] = response.Pid
				//make a lookup from pid to name
				a.ReverseCache[key] = name

				//watch the pid so we know if the node or pid dies
				ctx.Watch(response.Pid)
				//tell the original requester that we have a response
				ctx.Respond(&pidCacheResponse{response.Pid, statusCode})
			default:
				//forward to requester
				ctx.Respond(&pidCacheResponse{response.Pid, statusCode})
			}
		})

	case *MemberLeftEvent:
		address := msg.Name()
		a.removeCacheByMemberAddress(address)
	case *MemberRejoinedEvent:
		address := msg.Name()
		a.removeCacheByMemberAddress(address)
	case *actor.Terminated:
		a.removeCacheByPid(msg.Who)
	case *removePidCacheRequest:
		a.removeCacheByName(msg.name)
	}
}

func (a *pidCachePartitionActor) removeCacheByPid(pid *actor.PID) {
	key := pid.String()
	//get the virtual name from the pid
	name, ok := a.ReverseCache[key]
	if !ok {
		//we don't have it, just ignore
		return
	}
	//drop both lookups as this actor is now dead
	delete(a.Cache, name)
	delete(a.ReverseCache, key)
}

func (a *pidCachePartitionActor) removeCacheByName(name string) {
	if pid, ok := a.Cache[name]; ok {
		key := pid.String()
		delete(a.Cache, name)
		delete(a.ReverseCache, key)
	}
}

func (a *pidCachePartitionActor) removeCacheByMemberAddress(address string) {
	var cnt int
	for name, pid := range a.Cache {
		if pid.Address == address {
			delete(a.Cache, name)
			delete(a.ReverseCache, pid.String())
			cnt++
		}
	}

	if cnt > 0 {
		plog.Error("PID caches removed from PidCache", log.Int("Count", cnt), log.Object("address", address))
	}
}
<|endoftext|>


<commit_before>// Package kvite is a simple embedded K/V store backed by SQLite
package kvite

import (
	"database/sql"
	"errors"
	"fmt"
	_ "github.com/mattn/go-sqlite3" //import sqlite3 for driver
)

type (
	// DB is a wrapper around the underlying SQLite database.
	DB struct {
		db           *sql.DB
		table        string
		putQuery     string
		deleteQuery  string
		getQuery     string
		foreachQuery string
		bucketsQuery string
	}

	// Tx wraps most interactions with the datastore.
	Tx struct {
		db      *DB
		tx      *sql.Tx
		managed bool
	}

	//Bucket represents a collection of key/value pairs inside the database.
	Bucket struct {
		name string
		tx   *Tx
	}
)

// Open opens a KVite datastore. The returned DB is safe for concurrent use by multiple goroutines.
// It is rarely necessary to close a DB.
func Open(filename, table string) (*DB, error) {
	db, err := sql.Open("sqlite3", filename)
	if err != nil {
		return nil, err
	}

	if table == "" {
		table = "kvite"
	}

	tx, err := db.Begin()
	defer tx.Rollback()
	query := fmt.Sprintf("create TABLE IF NOT EXISTS '%s' (key text not null, bucket text not null, value blob not null)", table)
	if _, err := tx.Exec(query); err != nil {
		return nil, err
	}
	query = fmt.Sprintf("create INDEX IF NOT EXISTS '%s_kvite_index' ON '%s' (key, bucket)", table, table)
	if _, err := tx.Exec(query); err != nil {
		return nil, err
	}

	if err := tx.Commit(); err != nil {
		return nil, err
	}

	return &DB{
		db:           db,
		table:        table,
		getQuery:     fmt.Sprintf("SELECT value FROM '%s' WHERE key = ? and bucket = ?", table),
		deleteQuery:  fmt.Sprintf("DELETE FROM '%s' WHERE key = ? AND bucket = ?", table),
		putQuery:     fmt.Sprintf("INSERT OR REPLACE INTO '%s' (key, value, bucket) VALUES (?, ?, ?)", table),
		foreachQuery: fmt.Sprintf("SELECT key, value FROM '%s' WHERE bucket = ?", table),
		bucketsQuery: fmt.Sprintf("SELECT DISTINCT bucket from '%s'", table),
	}, nil
}

// Close closes the database, releasing any open resources.
// It is rare to Close a DB, as the DB handle is meant to be long-lived and shared between many goroutines.
func (db *DB) Close() error {
	return db.db.Close()
}

// Begin starts a transaction.
func (db *DB) Begin() (*Tx, error) {
	tx, err := db.db.Begin()
	if err != nil {
		return nil, err
	}
	t := &Tx{
		db: db,
		tx: tx,
	}
	return t, nil

}

// Buckets returns all the buckets
func (db *DB) Buckets() ([]string, error) {
	rows, err := db.db.Query(db.bucketsQuery)
	if err != nil {
		return nil, err
	}

	buckets := make([]string, 0, 32)
	for rows.Next() {
		var name string
		if err := rows.Scan(&name); err != nil {
			return nil, err
		}
		buckets = append(buckets, name)
	}
	if err := rows.Err(); err != nil {
		return nil, err
	}

	return buckets, nil
}

// Transaction executes a function within the context of a  managed transaction.
// If no error is returned from the function then the transaction is committed.
// If an error is returned then the entire transaction is rolled back.
// Rollback and Commit cannot be used inside of the function
func (db *DB) Transaction(fn func(*Tx) error) error {
	tx, err := db.Begin()
	if err != nil {
		return err
	}

	// Make sure the transaction rolls back in the event of a panic.
	defer func() {
		if tx.db != nil {
			tx.Rollback()
		}
	}()

	tx.managed = true
	err = fn(tx)
	tx.managed = false
	if err != nil {
		_ = tx.Rollback()
		return err
	}

	return tx.Commit()
}

// Commit commits the transaction.
func (tx *Tx) Commit() error {
	if tx.managed {
		return errors.New("managed tx commit not allowed")
	}
	if tx.db == nil {
		// should we return an error here?
		return nil
	}

	err := tx.tx.Commit()
	tx.db = nil
	return err
}

// Rollback aborts the transaction.
func (tx *Tx) Rollback() error {
	if tx.managed {
		return errors.New("managed tx commit not allowed")
	}
	return tx.tx.Rollback()
}

func (tx *Tx) newBucket(name string) *Bucket {
	return &Bucket{
		tx:   tx,
		name: name,
	}
}

// Bucket gets a bucket by name.  Buckets can be created on the fly and do not "exist" until they have keys.
func (tx *Tx) Bucket(name string) (*Bucket, error) {
	return tx.newBucket(name), nil
}

// CreateBucket is provided for compatibility. It just calls Bucket.
func (tx *Tx) CreateBucket(name string) (*Bucket, error) {
	return tx.Bucket(name)

}

// CreateBucketIfNotExists is provided for compatibility. It just calls Bucket.
func (tx *Tx) CreateBucketIfNotExists(name string) (*Bucket, error) {
	return tx.Bucket(name)
}

// Put sets the value for a key in the bucket. If the key exists, then its previous value will be overwritten.
func (b *Bucket) Put(key string, value []byte) error {
	_, err := b.tx.tx.Exec(b.tx.db.putQuery, key, value, b.name)
	if err != nil {
		return err
	}
	return nil
}

// Delete removes a key from the bucket. If the key does not exist then nothing is done and a nil error is returned.
func (b *Bucket) Delete(key string) error {
	_, err := b.tx.tx.Exec(b.tx.db.deleteQuery, key, b.name)
	if err != nil {
		return err
	}
	return nil
}

// Get retrieves the value for a key in the bucket. Returns a nil value if the key does not exist
func (b *Bucket) Get(key string) ([]byte, error) {
	var value []byte

	if err := b.tx.tx.QueryRow(b.tx.db.getQuery, key, b.name).Scan(&value); err != nil {
		if err == sql.ErrNoRows {
			return nil, nil
		}
		return nil, err
	}

	return value, nil
}

//ForEach executes a function for each key/value pair in a bucket. If the provided function returns an error then the iteration is stopped and the error is returned to the caller.
func (b *Bucket) ForEach(fn func(k string, v []byte) error) error {
	rows, err := b.tx.tx.Query(b.tx.db.foreachQuery, b.name)
	if err != nil {
		return err
	}
	for rows.Next() {
		var key string
		var value []byte
		if err := rows.Scan(&key, &value); err != nil {
			return err
		}
		if err := fn(key, value); err != nil {
			return err
		}
	}
	return rows.Err()
}
<commit_msg>clean up some return code, no need to check if err is nil, just return it<commit_after>// Package kvite is a simple embedded K/V store backed by SQLite
package kvite

import (
	"database/sql"
	"errors"
	"fmt"
	_ "github.com/mattn/go-sqlite3" //import sqlite3 for driver
)

type (
	// DB is a wrapper around the underlying SQLite database.
	DB struct {
		db           *sql.DB
		table        string
		putQuery     string
		deleteQuery  string
		getQuery     string
		foreachQuery string
		bucketsQuery string
	}

	// Tx wraps most interactions with the datastore.
	Tx struct {
		db      *DB
		tx      *sql.Tx
		managed bool
	}

	//Bucket represents a collection of key/value pairs inside the database.
	Bucket struct {
		name string
		tx   *Tx
	}
)

// Open opens a KVite datastore. The returned DB is safe for concurrent use by multiple goroutines.
// It is rarely necessary to close a DB.
func Open(filename, table string) (*DB, error) {
	db, err := sql.Open("sqlite3", filename)
	if err != nil {
		return nil, err
	}

	if table == "" {
		table = "kvite"
	}

	tx, err := db.Begin()
	defer tx.Rollback()
	query := fmt.Sprintf("create TABLE IF NOT EXISTS '%s' (key text not null, bucket text not null, value blob not null)", table)
	if _, err := tx.Exec(query); err != nil {
		return nil, err
	}
	query = fmt.Sprintf("create INDEX IF NOT EXISTS '%s_kvite_index' ON '%s' (key, bucket)", table, table)
	if _, err := tx.Exec(query); err != nil {
		return nil, err
	}

	if err := tx.Commit(); err != nil {
		return nil, err
	}

	return &DB{
		db:           db,
		table:        table,
		getQuery:     fmt.Sprintf("SELECT value FROM '%s' WHERE key = ? and bucket = ?", table),
		deleteQuery:  fmt.Sprintf("DELETE FROM '%s' WHERE key = ? AND bucket = ?", table),
		putQuery:     fmt.Sprintf("INSERT OR REPLACE INTO '%s' (key, value, bucket) VALUES (?, ?, ?)", table),
		foreachQuery: fmt.Sprintf("SELECT key, value FROM '%s' WHERE bucket = ?", table),
		bucketsQuery: fmt.Sprintf("SELECT DISTINCT bucket from '%s'", table),
	}, nil
}

// Close closes the database, releasing any open resources.
// It is rare to Close a DB, as the DB handle is meant to be long-lived and shared between many goroutines.
func (db *DB) Close() error {
	return db.db.Close()
}

// Begin starts a transaction.
func (db *DB) Begin() (*Tx, error) {
	tx, err := db.db.Begin()
	if err != nil {
		return nil, err
	}
	t := &Tx{
		db: db,
		tx: tx,
	}
	return t, nil

}

// Buckets returns all the buckets
func (db *DB) Buckets() ([]string, error) {
	rows, err := db.db.Query(db.bucketsQuery)
	if err != nil {
		return nil, err
	}

	buckets := make([]string, 0, 32)
	for rows.Next() {
		var name string
		if err := rows.Scan(&name); err != nil {
			return nil, err
		}
		buckets = append(buckets, name)
	}
	if err := rows.Err(); err != nil {
		return nil, err
	}

	return buckets, nil
}

// Transaction executes a function within the context of a  managed transaction.
// If no error is returned from the function then the transaction is committed.
// If an error is returned then the entire transaction is rolled back.
// Rollback and Commit cannot be used inside of the function
func (db *DB) Transaction(fn func(*Tx) error) error {
	tx, err := db.Begin()
	if err != nil {
		return err
	}

	// Make sure the transaction rolls back in the event of a panic.
	defer func() {
		if tx.db != nil {
			tx.Rollback()
		}
	}()

	tx.managed = true
	err = fn(tx)
	tx.managed = false
	if err != nil {
		_ = tx.Rollback()
		return err
	}

	return tx.Commit()
}

// Commit commits the transaction.
func (tx *Tx) Commit() error {
	if tx.managed {
		return errors.New("managed tx commit not allowed")
	}
	if tx.db == nil {
		// should we return an error here?
		return nil
	}

	err := tx.tx.Commit()
	tx.db = nil
	return err
}

// Rollback aborts the transaction.
func (tx *Tx) Rollback() error {
	if tx.managed {
		return errors.New("managed tx commit not allowed")
	}
	return tx.tx.Rollback()
}

func (tx *Tx) newBucket(name string) *Bucket {
	return &Bucket{
		tx:   tx,
		name: name,
	}
}

// Bucket gets a bucket by name.  Buckets can be created on the fly and do not "exist" until they have keys.
func (tx *Tx) Bucket(name string) (*Bucket, error) {
	return tx.newBucket(name), nil
}

// CreateBucket is provided for compatibility. It just calls Bucket.
func (tx *Tx) CreateBucket(name string) (*Bucket, error) {
	return tx.Bucket(name)

}

// CreateBucketIfNotExists is provided for compatibility. It just calls Bucket.
func (tx *Tx) CreateBucketIfNotExists(name string) (*Bucket, error) {
	return tx.Bucket(name)
}

// Put sets the value for a key in the bucket. If the key exists, then its previous value will be overwritten.
func (b *Bucket) Put(key string, value []byte) error {
	_, err := b.tx.tx.Exec(b.tx.db.putQuery, key, value, b.name)
	return err
}

// Delete removes a key from the bucket. If the key does not exist then nothing is done and a nil error is returned.
func (b *Bucket) Delete(key string) error {
	_, err := b.tx.tx.Exec(b.tx.db.deleteQuery, key, b.name)
	return err
}

// Get retrieves the value for a key in the bucket. Returns a nil value if the key does not exist
func (b *Bucket) Get(key string) ([]byte, error) {
	var value []byte

	if err := b.tx.tx.QueryRow(b.tx.db.getQuery, key, b.name).Scan(&value); err != nil {
		if err == sql.ErrNoRows {
			return nil, nil
		}
		return nil, err
	}

	return value, nil
}

//ForEach executes a function for each key/value pair in a bucket. If the provided function returns an error then the iteration is stopped and the error is returned to the caller.
func (b *Bucket) ForEach(fn func(k string, v []byte) error) error {
	rows, err := b.tx.tx.Query(b.tx.db.foreachQuery, b.name)
	if err != nil {
		return err
	}
	for rows.Next() {
		var key string
		var value []byte
		if err := rows.Scan(&key, &value); err != nil {
			return err
		}
		if err := fn(key, value); err != nil {
			return err
		}
	}
	return rows.Err()
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/BytemarkHosting/bytemark-client/lib/brain"
	"github.com/BytemarkHosting/bytemark-client/lib/prettyprint"
	"github.com/BytemarkHosting/bytemark-client/util/log"
	"github.com/urfave/cli"
)

func listServersInGroup(g *brain.Group) {
	for _, vm := range g.VirtualMachines {
		if vm.Deleted {
			log.Output(vm.Hostname + " (deleted)")
		} else {
			log.Output(vm.Hostname)
		}
	}
}

func init() {
	commands = append(commands, cli.Command{
		Name:      "list",
		Usage:     "scripting-friendly lists of your assets at Bytemark",
		UsageText: "bytemark list accounts|discs|groups|keys|servers",
		Description: `scripting-friendly lists of your assets at Bytemark

This commmand will list the kind of object you request, one per line. Perfect for piping into a bash while loop!`,
		Action: cli.ShowSubcommandHelp,
		Subcommands: []cli.Command{{
			Name:        "accounts",
			Usage:       "list all the accounts you're able to see",
			UsageText:   "bytemark list accounts",
			Description: `This will list all the accounts that your authentication token has some form of access to.`,
			Action: With(AuthProvider, func(c *Context) error {
				accounts, err := global.Client.GetAccounts()

				if err != nil {
					return err
				}

				for _, group := range accounts {
					log.Output(group.Name)
				}
				return nil
			}),
		}, {
			Name:        "discs",
			Usage:       "list all the discs attached to a given virtual machine",
			UsageText:   "bytemark list discs <virtual machine>",
			Description: `This command lists all the discs attached to the given virtual machine. They're presented in the following format: 'LABEL: SIZE GRADE', where size is an integer number of megabytes. Add the --human flag to output the size in GiB (rounded down to the nearest GiB)`,
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "human",
					Usage: "output disc size in GiB, suffixed",
				},
			},
			Action: With(VirtualMachineProvider, AuthProvider, func(c *Context) (err error) {
				for _, disc := range c.VirtualMachine.Discs {
					if c.Bool("human") {
						log.Outputf("%s: %dGiB %s\r\n", disc.Label, (disc.Size / 1024), disc.StorageGrade)
					} else {
						log.Outputf("%s: %d %s\r\n", disc.Label, disc.Size, disc.StorageGrade)
					}
				}
				return
			}),
		}, {
			Name:        "groups",
			Usage:       "list all the groups in an account",
			UsageText:   "bytemark list groups [account]",
			Description: `This command lists all the groups in the given account, or in your default account if not specified.`,
			Flags: []cli.Flag{
				cli.GenericFlag{
					Name:  "account",
					Usage: "the account to list the groups of",
					Value: new(AccountNameFlag),
				},
			},
			Action: With(OptionalArgs("account"), AccountProvider("account"), AuthProvider, func(c *Context) (err error) {
				for _, group := range c.Account.Groups {
					log.Output(group.Name)
				}
				return
			}),
		}, {
			Name:        "keys",
			Usage:       "list all the SSH public keys associated with a user",
			UsageText:   "bytemark list keys [user]",
			Description: "Lists all the SSH public keys associated with a user, defaulting to your log-in user.",
			Action: func(c *cli.Context) error {
				username := global.Config.GetIgnoreErr("user")
				if len(c.Args()) == 1 {
					username = c.Args().First()
				}

				err := EnsureAuth()
				if err != nil {
					return err
				}

				user, err := global.Client.GetUser(username)
				if err != nil {
					return err
				}

				for _, k := range user.AuthorizedKeys {
					log.Output(k)
				}

				return nil
			},
		}, {
			Name:      "servers",
			Usage:     "list all the servers in an account",
			UsageText: "bytemark list servers [account]",
			Description: `This command lists all the servers in the given account, or in your default account if not specified.
Deleted servers are included in the list, with ' (deleted)' appended.`,
			Flags: []cli.Flag{
				cli.GenericFlag{
					Name:  "account",
					Usage: "the account to list the groups of",
					Value: new(AccountNameFlag),
				},
			},
			Action: With(OptionalArgs("account"), AccountProvider("account"), AuthProvider, func(c *Context) (err error) {
				for _, g := range c.Account.Groups {
					listServersInGroup(g)
				}
				return
			}),
		}, {
			Name:        "backups",
			Usage:       "list all the backups of a server or disc",
			UsageText:   "bytemark list backups <server name> [disc label]",
			Description: "Lists all the backups of all the discs in the given server, or if you also give a disc label, just the backups of that disc.",
			Action: With(VirtualMachineNameProvider, func(c *Context) (err error) {
				label, _ := c.NextArg()
				var backups brain.Backups

				if label != "" {
					backups, err = global.Client.GetBackups(*c.VirtualMachineName, label)
					if err != nil {
						return
					}
				} else {
					err = VirtualMachineProvider(c)
					if err != nil {
						return
					}
					for _, disc := range c.VirtualMachine.Discs {
						snaps, err := global.Client.GetBackups(*c.VirtualMachineName, disc.Label)
						if err != nil {
							return err
						}
						// TODO(telyn): loop over snaps, attach disc as ParentDisc
						backups = append(backups, snaps...)
					}
				}
				return backups.PrettyPrint(global.App.Writer, prettyprint.Full)
			}),
		}},
	})
}
<commit_msg>fix --account Usage on list servers<commit_after>package main

import (
	"github.com/BytemarkHosting/bytemark-client/lib/brain"
	"github.com/BytemarkHosting/bytemark-client/lib/prettyprint"
	"github.com/BytemarkHosting/bytemark-client/util/log"
	"github.com/urfave/cli"
)

func listServersInGroup(g *brain.Group) {
	for _, vm := range g.VirtualMachines {
		if vm.Deleted {
			log.Output(vm.Hostname + " (deleted)")
		} else {
			log.Output(vm.Hostname)
		}
	}
}

func init() {
	commands = append(commands, cli.Command{
		Name:      "list",
		Usage:     "scripting-friendly lists of your assets at Bytemark",
		UsageText: "bytemark list accounts|discs|groups|keys|servers",
		Description: `scripting-friendly lists of your assets at Bytemark

This commmand will list the kind of object you request, one per line. Perfect for piping into a bash while loop!`,
		Action: cli.ShowSubcommandHelp,
		Subcommands: []cli.Command{{
			Name:        "accounts",
			Usage:       "list all the accounts you're able to see",
			UsageText:   "bytemark list accounts",
			Description: `This will list all the accounts that your authentication token has some form of access to.`,
			Action: With(AuthProvider, func(c *Context) error {
				accounts, err := global.Client.GetAccounts()

				if err != nil {
					return err
				}

				for _, group := range accounts {
					log.Output(group.Name)
				}
				return nil
			}),
		}, {
			Name:        "discs",
			Usage:       "list all the discs attached to a given virtual machine",
			UsageText:   "bytemark list discs <virtual machine>",
			Description: `This command lists all the discs attached to the given virtual machine. They're presented in the following format: 'LABEL: SIZE GRADE', where size is an integer number of megabytes. Add the --human flag to output the size in GiB (rounded down to the nearest GiB)`,
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "human",
					Usage: "output disc size in GiB, suffixed",
				},
			},
			Action: With(VirtualMachineProvider, AuthProvider, func(c *Context) (err error) {
				for _, disc := range c.VirtualMachine.Discs {
					if c.Bool("human") {
						log.Outputf("%s: %dGiB %s\r\n", disc.Label, (disc.Size / 1024), disc.StorageGrade)
					} else {
						log.Outputf("%s: %d %s\r\n", disc.Label, disc.Size, disc.StorageGrade)
					}
				}
				return
			}),
		}, {
			Name:        "groups",
			Usage:       "list all the groups in an account",
			UsageText:   "bytemark list groups [account]",
			Description: `This command lists all the groups in the given account, or in your default account if not specified.`,
			Flags: []cli.Flag{
				cli.GenericFlag{
					Name:  "account",
					Usage: "the account to list the groups of",
					Value: new(AccountNameFlag),
				},
			},
			Action: With(OptionalArgs("account"), AccountProvider("account"), AuthProvider, func(c *Context) (err error) {
				for _, group := range c.Account.Groups {
					log.Output(group.Name)
				}
				return
			}),
		}, {
			Name:        "keys",
			Usage:       "list all the SSH public keys associated with a user",
			UsageText:   "bytemark list keys [user]",
			Description: "Lists all the SSH public keys associated with a user, defaulting to your log-in user.",
			Action: func(c *cli.Context) error {
				username := global.Config.GetIgnoreErr("user")
				if len(c.Args()) == 1 {
					username = c.Args().First()
				}

				err := EnsureAuth()
				if err != nil {
					return err
				}

				user, err := global.Client.GetUser(username)
				if err != nil {
					return err
				}

				for _, k := range user.AuthorizedKeys {
					log.Output(k)
				}

				return nil
			},
		}, {
			Name:      "servers",
			Usage:     "list all the servers in an account",
			UsageText: "bytemark list servers [account]",
			Description: `This command lists all the servers in the given account, or in your default account if not specified.
Deleted servers are included in the list, with ' (deleted)' appended.`,
			Flags: []cli.Flag{
				cli.GenericFlag{
					Name:  "account",
					Usage: "the account to list the servers of",
					Value: new(AccountNameFlag),
				},
			},
			Action: With(OptionalArgs("account"), AccountProvider("account"), AuthProvider, func(c *Context) (err error) {
				for _, g := range c.Account.Groups {
					listServersInGroup(g)
				}
				return
			}),
		}, {
			Name:        "backups",
			Usage:       "list all the backups of a server or disc",
			UsageText:   "bytemark list backups <server name> [disc label]",
			Description: "Lists all the backups of all the discs in the given server, or if you also give a disc label, just the backups of that disc.",
			Action: With(VirtualMachineNameProvider, func(c *Context) (err error) {
				label, _ := c.NextArg()
				var backups brain.Backups

				if label != "" {
					backups, err = global.Client.GetBackups(*c.VirtualMachineName, label)
					if err != nil {
						return
					}
				} else {
					err = VirtualMachineProvider(c)
					if err != nil {
						return
					}
					for _, disc := range c.VirtualMachine.Discs {
						snaps, err := global.Client.GetBackups(*c.VirtualMachineName, disc.Label)
						if err != nil {
							return err
						}
						// TODO(telyn): loop over snaps, attach disc as ParentDisc
						backups = append(backups, snaps...)
					}
				}
				return backups.PrettyPrint(global.App.Writer, prettyprint.Full)
			}),
		}},
	})
}
<|endoftext|>


<commit_before>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package deploy

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os/exec"
	"path/filepath"

	"github.com/openconfig/kne/deploy"
	log "github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
	"gopkg.in/yaml.v3"
)

func New() *cobra.Command {
	deployCmd := &cobra.Command{
		Use:   "deploy <deployment yaml>",
		Short: "Deploy cluster.",
		RunE:  deployFn,
	}
	return deployCmd
}

type ClusterSpec struct {
	Kind string    `yaml:"kind"`
	Spec yaml.Node `yaml:"spec"`
}

type IngressSpec struct {
	Kind string    `yaml:"kind"`
	Spec yaml.Node `yaml:"spec"`
}

type CNISpec struct {
	Kind string    `yaml:"kind"`
	Spec yaml.Node `yaml:"spec"`
}

type ControllerSpec struct {
	Kind string    `yaml:"kind"`
	Spec yaml.Node `yaml:"spec"`
}

type DeploymentConfig struct {
	Cluster     ClusterSpec       `yaml:"cluster"`
	Ingress     IngressSpec       `yaml:"ingress"`
	CNI         CNISpec           `yaml:"cni"`
	Controllers []*ControllerSpec `yaml:"controllers"`
}

func newDeployment(cfgPath string) (*deploy.Deployment, error) {
	p, err := filepath.Abs(cfgPath)
	if err != nil {
		return nil, err
	}
	log.Infof("Reading deployment config: %q", p)
	b, err := ioutil.ReadFile(p)
	if err != nil {
		return nil, err
	}
	basePath := filepath.Dir(p)
	cfg := &DeploymentConfig{}
	decoder := yaml.NewDecoder(bytes.NewBuffer(b))
	decoder.KnownFields(true)
	if err := decoder.Decode(cfg); err != nil {
		return nil, err
	}

	d := &deploy.Deployment{}
	switch cfg.Cluster.Kind {
	case "Kind":
		log.Infof("Using kind scenario")
		v := &deploy.KindSpec{}
		if err := cfg.Cluster.Spec.Decode(v); err != nil {
			return nil, err
		}
		// make sure manifests are correct relative to configuration.
		for i, s := range v.AdditionalManifests {
			v.AdditionalManifests[i] = cleanPath(s, basePath)
		}

		// make sure kind config file is relative to configuration.
		v.KindConfigFile = cleanPath(v.KindConfigFile, basePath)

		d.Cluster = v
	default:
		return nil, fmt.Errorf("cluster type not supported: %s", cfg.Cluster.Kind)
	}
	switch cfg.CNI.Kind {
	case "Meshnet":
		v := &deploy.MeshnetSpec{}
		if err := cfg.CNI.Spec.Decode(v); err != nil {
			return nil, err
		}
		v.ManifestDir = cleanPath(v.ManifestDir, basePath)
		d.CNI = v
	default:
		return nil, fmt.Errorf("CNI type not supported: %s", cfg.CNI.Kind)
	}
	switch cfg.Ingress.Kind {
	case "MetalLB":
		v := &deploy.MetalLBSpec{}
		if err := cfg.Ingress.Spec.Decode(v); err != nil {
			return nil, err
		}
		v.ManifestDir = cleanPath(v.ManifestDir, basePath)
		d.Ingress = v
	default:
		return nil, fmt.Errorf("ingress type not supported: %s", cfg.Ingress.Kind)
	}
	if len(cfg.Controllers) != 0 {
		d.Controllers = []deploy.Controller{}
	}
	for _, c := range cfg.Controllers {
		switch c.Kind {
		case "IxiaTG":
			v := &deploy.IxiaTGSpec{}
			if err := c.Spec.Decode(v); err != nil {
				return nil, err
			}
			v.ManifestDir = cleanPath(v.ManifestDir, basePath)
			d.Controllers = append(d.Controllers, v)
		default:
			return nil, fmt.Errorf("controller type not supported: %s", c.Kind)
		}
	}
	return d, nil
}

func cleanPath(path, basePath string) string {
	if filepath.IsAbs(path) {
		return path
	}
	return filepath.Join(basePath, path)
}

func deployFn(cmd *cobra.Command, args []string) error {
	if len(args) != 1 {
		return fmt.Errorf("%s: missing args", cmd.Use)
	}
	if _, err := exec.LookPath("kubectl"); err != nil {
		return fmt.Errorf("install kubectl before running deploy: %v", err)
	}
	kubecfg, err := cmd.Flags().GetString("kubecfg")
	if err != nil {
		return err
	}
	d, err := newDeployment(args[0])
	if err != nil {
		return err
	}
	if err := d.Deploy(cmd.Context(), kubecfg); err != nil {
		return err
	}
	log.Infof("Deployment complete, ready for topology")
	return nil
}
<commit_msg>Only update kind config path if it was defined (#170)<commit_after>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package deploy

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os/exec"
	"path/filepath"

	"github.com/openconfig/kne/deploy"
	log "github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
	"gopkg.in/yaml.v3"
)

func New() *cobra.Command {
	deployCmd := &cobra.Command{
		Use:   "deploy <deployment yaml>",
		Short: "Deploy cluster.",
		RunE:  deployFn,
	}
	return deployCmd
}

type ClusterSpec struct {
	Kind string    `yaml:"kind"`
	Spec yaml.Node `yaml:"spec"`
}

type IngressSpec struct {
	Kind string    `yaml:"kind"`
	Spec yaml.Node `yaml:"spec"`
}

type CNISpec struct {
	Kind string    `yaml:"kind"`
	Spec yaml.Node `yaml:"spec"`
}

type ControllerSpec struct {
	Kind string    `yaml:"kind"`
	Spec yaml.Node `yaml:"spec"`
}

type DeploymentConfig struct {
	Cluster     ClusterSpec       `yaml:"cluster"`
	Ingress     IngressSpec       `yaml:"ingress"`
	CNI         CNISpec           `yaml:"cni"`
	Controllers []*ControllerSpec `yaml:"controllers"`
}

func newDeployment(cfgPath string) (*deploy.Deployment, error) {
	p, err := filepath.Abs(cfgPath)
	if err != nil {
		return nil, err
	}
	log.Infof("Reading deployment config: %q", p)
	b, err := ioutil.ReadFile(p)
	if err != nil {
		return nil, err
	}
	basePath := filepath.Dir(p)
	cfg := &DeploymentConfig{}
	decoder := yaml.NewDecoder(bytes.NewBuffer(b))
	decoder.KnownFields(true)
	if err := decoder.Decode(cfg); err != nil {
		return nil, err
	}

	d := &deploy.Deployment{}
	switch cfg.Cluster.Kind {
	case "Kind":
		log.Infof("Using kind scenario")
		v := &deploy.KindSpec{}
		if err := cfg.Cluster.Spec.Decode(v); err != nil {
			return nil, err
		}
		// make sure manifests are correct relative to configuration.
		for i, s := range v.AdditionalManifests {
			v.AdditionalManifests[i] = cleanPath(s, basePath)
		}

		// make sure kind config file is relative to configuration.
		if v.KindConfigFile != "" {
			v.KindConfigFile = cleanPath(v.KindConfigFile, basePath)
		}

		d.Cluster = v
	default:
		return nil, fmt.Errorf("cluster type not supported: %s", cfg.Cluster.Kind)
	}
	switch cfg.CNI.Kind {
	case "Meshnet":
		v := &deploy.MeshnetSpec{}
		if err := cfg.CNI.Spec.Decode(v); err != nil {
			return nil, err
		}
		v.ManifestDir = cleanPath(v.ManifestDir, basePath)
		d.CNI = v
	default:
		return nil, fmt.Errorf("CNI type not supported: %s", cfg.CNI.Kind)
	}
	switch cfg.Ingress.Kind {
	case "MetalLB":
		v := &deploy.MetalLBSpec{}
		if err := cfg.Ingress.Spec.Decode(v); err != nil {
			return nil, err
		}
		v.ManifestDir = cleanPath(v.ManifestDir, basePath)
		d.Ingress = v
	default:
		return nil, fmt.Errorf("ingress type not supported: %s", cfg.Ingress.Kind)
	}
	if len(cfg.Controllers) != 0 {
		d.Controllers = []deploy.Controller{}
	}
	for _, c := range cfg.Controllers {
		switch c.Kind {
		case "IxiaTG":
			v := &deploy.IxiaTGSpec{}
			if err := c.Spec.Decode(v); err != nil {
				return nil, err
			}
			v.ManifestDir = cleanPath(v.ManifestDir, basePath)
			d.Controllers = append(d.Controllers, v)
		default:
			return nil, fmt.Errorf("controller type not supported: %s", c.Kind)
		}
	}
	return d, nil
}

func cleanPath(path, basePath string) string {
	if filepath.IsAbs(path) {
		return path
	}
	return filepath.Join(basePath, path)
}

func deployFn(cmd *cobra.Command, args []string) error {
	if len(args) != 1 {
		return fmt.Errorf("%s: missing args", cmd.Use)
	}
	if _, err := exec.LookPath("kubectl"); err != nil {
		return fmt.Errorf("install kubectl before running deploy: %v", err)
	}
	kubecfg, err := cmd.Flags().GetString("kubecfg")
	if err != nil {
		return err
	}
	d, err := newDeployment(args[0])
	if err != nil {
		return err
	}
	if err := d.Deploy(cmd.Context(), kubecfg); err != nil {
		return err
	}
	log.Infof("Deployment complete, ready for topology")
	return nil
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"testing"

	"github.com/exercism/cli/config"
	"github.com/spf13/pflag"
	"github.com/spf13/viper"
	"github.com/stretchr/testify/assert"
)

func TestDownloadWithoutToken(t *testing.T) {
	cfg := config.Configuration{
		UserViperConfig: viper.New(),
	}

	err := runDownload(cfg, pflag.NewFlagSet("fake", pflag.PanicOnError), []string{})
	if assert.Error(t, err) {
		assert.Regexp(t, "Welcome to Exercism", err.Error())
	}
}

func TestDownloadWithoutWorkspace(t *testing.T) {
	v := viper.New()
	v.Set("token", "abc123")
	cfg := config.Configuration{
		UserViperConfig: v,
	}

	err := runDownload(cfg, pflag.NewFlagSet("fake", pflag.PanicOnError), []string{})
	if assert.Error(t, err) {
		assert.Regexp(t, "re-run the configure", err.Error())
	}
}

func TestDownloadWithoutBaseURL(t *testing.T) {
	v := viper.New()
	v.Set("token", "abc123")
	v.Set("workspace", "/home/whatever")
	cfg := config.Configuration{
		UserViperConfig: v,
	}

	err := runDownload(cfg, pflag.NewFlagSet("fake", pflag.PanicOnError), []string{})
	if assert.Error(t, err) {
		assert.Regexp(t, "re-run the configure", err.Error())
	}
}

func TestDownloadWithoutFlags(t *testing.T) {
	v := viper.New()
	v.Set("token", "abc123")
	v.Set("workspace", "/home/username")
	v.Set("apibaseurl", "http://example.com")

	cfg := config.Configuration{
		UserViperConfig: v,
	}

	flags := pflag.NewFlagSet("fake", pflag.PanicOnError)
	setupDownloadFlags(flags)

	err := runDownload(cfg, flags, []string{})
	if assert.Error(t, err) {
		assert.Regexp(t, "need an --exercise name or a solution --uuid", err.Error())
	}
}

func TestDownload(t *testing.T) {
	oldOut := Out
	oldErr := Err
	Out = ioutil.Discard
	Err = ioutil.Discard
	defer func() {
		Out = oldOut
		Err = oldErr
	}()

	testCases := []struct {
		requestor       string
		expectedDir     string
		flag, flagValue string
	}{
		{
			requestor:   requestorSelf,
			expectedDir: "",
			flag:        "exercise",
			flagValue:   "bogus-exercise",
		},
		{
			requestor:   requestorSelf,
			expectedDir: "",
			flag:        "uuid",
			flagValue:   "bogus-id",
		},
		{
			requestor:   requestorOther,
			expectedDir: filepath.Join("users", "alice"),
			flag:        "uuid",
			flagValue:   "bogus-id",
		},
	}

	for _, tc := range testCases {
		tmpDir, err := ioutil.TempDir("", "download-cmd")
		assert.NoError(t, err)

		ts := fakeDownloadServer(tc.requestor)
		defer ts.Close()

		v := viper.New()
		v.Set("workspace", tmpDir)
		v.Set("apibaseurl", ts.URL)
		v.Set("token", "abc123")

		cfg := config.Configuration{
			UserViperConfig: v,
		}
		flags := pflag.NewFlagSet("fake", pflag.PanicOnError)
		setupDownloadFlags(flags)
		flags.Set(tc.flag, tc.flagValue)

		err = runDownload(cfg, flags, []string{})
		assert.NoError(t, err)

		assertDownloadedCorrectFiles(t, filepath.Join(tmpDir, tc.expectedDir), tc.requestor)
	}
}

func fakeDownloadServer(requestor string) *httptest.Server {
	mux := http.NewServeMux()
	server := httptest.NewServer(mux)

	path1 := "file-1.txt"
	mux.HandleFunc("/"+path1, func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprint(w, "this is file 1")
	})

	path2 := "subdir/file-2.txt"
	mux.HandleFunc("/"+path2, func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprint(w, "this is file 2")
	})

	path3 := "file-3.txt"
	mux.HandleFunc("/"+path3, func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprint(w, "")
	})

	payloadBody := fmt.Sprintf(payloadTemplate, requestor, server.URL+"/", path1, path2, path3)
	mux.HandleFunc("/solutions/latest", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprint(w, payloadBody)
	})
	mux.HandleFunc("/solutions/bogus-id", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprint(w, payloadBody)
	})

	return server
}

func assertDownloadedCorrectFiles(t *testing.T, targetDir, requestor string) {
	metadata := `{"track":"bogus-track","exercise":"bogus-exercise","id":"bogus-id","url":"","handle":"alice","is_requester":%s,"auto_approve":false}`
	expectedFiles := []struct {
		desc     string
		path     string
		contents string
	}{
		{
			desc:     "a file in the exercise root directory",
			path:     filepath.Join(targetDir, "bogus-track", "bogus-exercise", "file-1.txt"),
			contents: "this is file 1",
		},
		{
			desc:     "a file in a subdirectory",
			path:     filepath.Join(targetDir, "bogus-track", "bogus-exercise", "subdir", "file-2.txt"),
			contents: "this is file 2",
		},
		{
			desc:     "the solution metadata file",
			path:     filepath.Join(targetDir, "bogus-track", "bogus-exercise", ".solution.json"),
			contents: fmt.Sprintf(metadata, requestor),
		},
	}

	for _, file := range expectedFiles {
		t.Run(file.desc, func(t *testing.T) {
			b, err := ioutil.ReadFile(file.path)
			assert.NoError(t, err)
			assert.Equal(t, file.contents, string(b))
		})
	}

	path := filepath.Join(targetDir, "bogus-track", "bogus-exercise", "file-3.txt")
	_, err := os.Lstat(path)
	assert.True(t, os.IsNotExist(err), "It should not write the file if empty.")
}

const requestorSelf = "true"
const requestorOther = "false"

const payloadTemplate = `
{
	"solution": {
		"id": "bogus-id",
		"user": {
			"handle": "alice",
			"is_requester": %s
		},
		"exercise": {
			"id": "bogus-exercise",
			"instructions_url": "http://example.com/bogus-exercise",
			"auto_approve": false,
			"track": {
				"id": "bogus-track",
				"language": "Bogus Language"
			}
		},
		"file_download_base_url": "%s",
		"files": [
		"%s",
		"%s",
		"%s"
		],
		"iteration": {
			"submitted_at": "2017-08-21t10:11:12.130z"
		}
	}
}
`
<commit_msg>Tweak flag setup in download test (#660)<commit_after>package cmd

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"testing"

	"github.com/exercism/cli/config"
	"github.com/spf13/pflag"
	"github.com/spf13/viper"
	"github.com/stretchr/testify/assert"
)

func TestDownloadWithoutToken(t *testing.T) {
	cfg := config.Configuration{
		UserViperConfig: viper.New(),
	}

	err := runDownload(cfg, pflag.NewFlagSet("fake", pflag.PanicOnError), []string{})
	if assert.Error(t, err) {
		assert.Regexp(t, "Welcome to Exercism", err.Error())
	}
}

func TestDownloadWithoutWorkspace(t *testing.T) {
	v := viper.New()
	v.Set("token", "abc123")
	cfg := config.Configuration{
		UserViperConfig: v,
	}

	err := runDownload(cfg, pflag.NewFlagSet("fake", pflag.PanicOnError), []string{})
	if assert.Error(t, err) {
		assert.Regexp(t, "re-run the configure", err.Error())
	}
}

func TestDownloadWithoutBaseURL(t *testing.T) {
	v := viper.New()
	v.Set("token", "abc123")
	v.Set("workspace", "/home/whatever")
	cfg := config.Configuration{
		UserViperConfig: v,
	}

	err := runDownload(cfg, pflag.NewFlagSet("fake", pflag.PanicOnError), []string{})
	if assert.Error(t, err) {
		assert.Regexp(t, "re-run the configure", err.Error())
	}
}

func TestDownloadWithoutFlags(t *testing.T) {
	v := viper.New()
	v.Set("token", "abc123")
	v.Set("workspace", "/home/username")
	v.Set("apibaseurl", "http://example.com")

	cfg := config.Configuration{
		UserViperConfig: v,
	}

	flags := pflag.NewFlagSet("fake", pflag.PanicOnError)
	setupDownloadFlags(flags)

	err := runDownload(cfg, flags, []string{})
	if assert.Error(t, err) {
		assert.Regexp(t, "need an --exercise name or a solution --uuid", err.Error())
	}
}

func TestDownload(t *testing.T) {
	oldOut := Out
	oldErr := Err
	Out = ioutil.Discard
	Err = ioutil.Discard
	defer func() {
		Out = oldOut
		Err = oldErr
	}()

	testCases := []struct {
		requestor   string
		expectedDir string
		flags       map[string]string
	}{
		{
			requestor:   requestorSelf,
			expectedDir: "",
			flags:       map[string]string{"exercise": "bogus-exercise"},
		},
		{
			requestor:   requestorSelf,
			expectedDir: "",
			flags:       map[string]string{"uuid": "bogus-id"},
		},
		{
			requestor:   requestorOther,
			expectedDir: filepath.Join("users", "alice"),
			flags:       map[string]string{"uuid": "bogus-id"},
		},
	}

	for _, tc := range testCases {
		tmpDir, err := ioutil.TempDir("", "download-cmd")
		assert.NoError(t, err)

		ts := fakeDownloadServer(tc.requestor)
		defer ts.Close()

		v := viper.New()
		v.Set("workspace", tmpDir)
		v.Set("apibaseurl", ts.URL)
		v.Set("token", "abc123")

		cfg := config.Configuration{
			UserViperConfig: v,
		}
		flags := pflag.NewFlagSet("fake", pflag.PanicOnError)
		setupDownloadFlags(flags)
		for name, value := range tc.flags {
			flags.Set(name, value)
		}

		err = runDownload(cfg, flags, []string{})
		assert.NoError(t, err)

		assertDownloadedCorrectFiles(t, filepath.Join(tmpDir, tc.expectedDir), tc.requestor)
	}
}

func fakeDownloadServer(requestor string) *httptest.Server {
	mux := http.NewServeMux()
	server := httptest.NewServer(mux)

	path1 := "file-1.txt"
	mux.HandleFunc("/"+path1, func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprint(w, "this is file 1")
	})

	path2 := "subdir/file-2.txt"
	mux.HandleFunc("/"+path2, func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprint(w, "this is file 2")
	})

	path3 := "file-3.txt"
	mux.HandleFunc("/"+path3, func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprint(w, "")
	})

	payloadBody := fmt.Sprintf(payloadTemplate, requestor, server.URL+"/", path1, path2, path3)
	mux.HandleFunc("/solutions/latest", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprint(w, payloadBody)
	})
	mux.HandleFunc("/solutions/bogus-id", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprint(w, payloadBody)
	})

	return server
}

func assertDownloadedCorrectFiles(t *testing.T, targetDir, requestor string) {
	metadata := `{"track":"bogus-track","exercise":"bogus-exercise","id":"bogus-id","url":"","handle":"alice","is_requester":%s,"auto_approve":false}`
	expectedFiles := []struct {
		desc     string
		path     string
		contents string
	}{
		{
			desc:     "a file in the exercise root directory",
			path:     filepath.Join(targetDir, "bogus-track", "bogus-exercise", "file-1.txt"),
			contents: "this is file 1",
		},
		{
			desc:     "a file in a subdirectory",
			path:     filepath.Join(targetDir, "bogus-track", "bogus-exercise", "subdir", "file-2.txt"),
			contents: "this is file 2",
		},
		{
			desc:     "the solution metadata file",
			path:     filepath.Join(targetDir, "bogus-track", "bogus-exercise", ".solution.json"),
			contents: fmt.Sprintf(metadata, requestor),
		},
	}

	for _, file := range expectedFiles {
		t.Run(file.desc, func(t *testing.T) {
			b, err := ioutil.ReadFile(file.path)
			assert.NoError(t, err)
			assert.Equal(t, file.contents, string(b))
		})
	}

	path := filepath.Join(targetDir, "bogus-track", "bogus-exercise", "file-3.txt")
	_, err := os.Lstat(path)
	assert.True(t, os.IsNotExist(err), "It should not write the file if empty.")
}

const requestorSelf = "true"
const requestorOther = "false"

const payloadTemplate = `
{
	"solution": {
		"id": "bogus-id",
		"user": {
			"handle": "alice",
			"is_requester": %s
		},
		"exercise": {
			"id": "bogus-exercise",
			"instructions_url": "http://example.com/bogus-exercise",
			"auto_approve": false,
			"track": {
				"id": "bogus-track",
				"language": "Bogus Language"
			}
		},
		"file_download_base_url": "%s",
		"files": [
		"%s",
		"%s",
		"%s"
		],
		"iteration": {
			"submitted_at": "2017-08-21t10:11:12.130z"
		}
	}
}
`
<|endoftext|>


<commit_before>/*
Fixhub runs golint on a GitHub repository.
*/
package main

import (
	"bytes"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"

	"github.com/dsymonds/fixhub/fixhub"
	"github.com/golang/lint"
)

var (
	personalAccessTokenFile = flag.String("personal_access_token_file", filepath.Join(os.Getenv("HOME"), ".fixhub-token"), "a file to load a GitHub personal access token from")
	rev                     = flag.String("rev", "master", "revision of the repo to check")
)

const (
	sizeLimit = 1 << 20 // 1 MB
)

func main() {
	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, "usage: fixhub [options] owner/repo")
		flag.PrintDefaults()
	}
	flag.Parse()

	if flag.NArg() != 1 {
		flag.Usage()
		os.Exit(1)
	}
	parts := strings.Split(flag.Arg(0), "/")
	if len(parts) != 2 {
		flag.Usage()
		os.Exit(1)
	}
	owner, repo := parts[0], parts[1]

	var accessToken string
	if pat, err := ioutil.ReadFile(*personalAccessTokenFile); err == nil {
		// security check
		fi, err := os.Stat(*personalAccessTokenFile)
		if err != nil {
			log.Fatalf("os.Stat(%q): %v", *personalAccessTokenFile, err)
		}
		if fi.Mode()&0077 != 0 { // check that no group/world perm bits are set
			log.Fatalf("%s is too accessible; run `chmod go= %s` to fix", *personalAccessTokenFile, *personalAccessTokenFile)
		}

		accessToken = string(bytes.TrimSpace(pat))
	}

	client, err := fixhub.NewClient(owner, repo, accessToken)
	if err != nil {
		log.Fatal(err)
	}

	ref, err := client.ResolveRef(*rev)
	if err != nil {
		log.Fatalf("GetCommit(%q): %v", *rev, err)
	}
	log.Printf("rev %q is %s", *rev, ref)

	tree, err := client.GetTree(ref)
	if err != nil {
		log.Fatalf("GetTree: %v", err)
	}
	log.Printf("Found %d tree entries", len(tree.Entries))

	var (
		linter = new(lint.Linter)

		wg       sync.WaitGroup
		problems struct {
			sync.Mutex
			list []string
		}
	)
	addProblem := func(s string) {
		problems.Lock()
		problems.list = append(problems.list, s)
		problems.Unlock()
	}

	nGo := 0
	for _, ent := range tree.Entries {
		if ent.SHA == nil || ent.Path == nil || ent.Size == nil {
			continue
		}
		sha1, path, size := *ent.SHA, *ent.Path, *ent.Size
		if !strings.HasSuffix(path, ".go") {
			continue
		}
		if strings.HasSuffix(path, ".pb.go") {
			continue
		}
		if size > sizeLimit {
			log.Printf("Skipping %s because it is too big: %d > %d", path, size, sizeLimit)
			continue
		}
		//log.Printf("+ %s (%d bytes)", path, size)

		wg.Add(1)
		nGo++
		go func() {
			defer wg.Done()

			src, err := client.GetBlob(sha1)
			if err != nil {
				log.Printf("Getting blob for %s: %v", path, err)
				return
			}
			ps, err := linter.Lint(path, src)
			if err != nil {
				log.Printf("Linting %s: %v", path, err)
				return
			}
			for _, p := range ps {
				if p.Confidence < 0.8 { // TODO: flag
					continue
				}
				addProblem(fmt.Sprintf("%s:%v: %s", path, p.Position, p.Text))
			}
		}()
	}
	wg.Wait()

	sort.Strings(problems.list)
	for _, p := range problems.list {
		fmt.Println(p)
	}
	log.Printf("wow, there were %d problems in %d Go source files!", len(problems.list), nGo)
}
<commit_msg>Improve cmd doc.<commit_after>/*
Fixhub tries to fix a GitHub repository containing Go source files.
*/
package main

import (
	"bytes"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"

	"github.com/dsymonds/fixhub/fixhub"
	"github.com/golang/lint"
)

var (
	personalAccessTokenFile = flag.String("personal_access_token_file", filepath.Join(os.Getenv("HOME"), ".fixhub-token"), "a file to load a GitHub personal access token from")
	rev                     = flag.String("rev", "master", "revision of the repo to check")
)

const (
	sizeLimit = 1 << 20 // 1 MB
)

func main() {
	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, "usage: fixhub [options] owner/repo")
		flag.PrintDefaults()
	}
	flag.Parse()

	if flag.NArg() != 1 {
		flag.Usage()
		os.Exit(1)
	}
	parts := strings.Split(flag.Arg(0), "/")
	if len(parts) != 2 {
		flag.Usage()
		os.Exit(1)
	}
	owner, repo := parts[0], parts[1]

	var accessToken string
	if pat, err := ioutil.ReadFile(*personalAccessTokenFile); err == nil {
		// security check
		fi, err := os.Stat(*personalAccessTokenFile)
		if err != nil {
			log.Fatalf("os.Stat(%q): %v", *personalAccessTokenFile, err)
		}
		if fi.Mode()&0077 != 0 { // check that no group/world perm bits are set
			log.Fatalf("%s is too accessible; run `chmod go= %s` to fix", *personalAccessTokenFile, *personalAccessTokenFile)
		}

		accessToken = string(bytes.TrimSpace(pat))
	}

	client, err := fixhub.NewClient(owner, repo, accessToken)
	if err != nil {
		log.Fatal(err)
	}

	ref, err := client.ResolveRef(*rev)
	if err != nil {
		log.Fatalf("GetCommit(%q): %v", *rev, err)
	}
	log.Printf("rev %q is %s", *rev, ref)

	tree, err := client.GetTree(ref)
	if err != nil {
		log.Fatalf("GetTree: %v", err)
	}
	log.Printf("Found %d tree entries", len(tree.Entries))

	var (
		linter = new(lint.Linter)

		wg       sync.WaitGroup
		problems struct {
			sync.Mutex
			list []string
		}
	)
	addProblem := func(s string) {
		problems.Lock()
		problems.list = append(problems.list, s)
		problems.Unlock()
	}

	nGo := 0
	for _, ent := range tree.Entries {
		if ent.SHA == nil || ent.Path == nil || ent.Size == nil {
			continue
		}
		sha1, path, size := *ent.SHA, *ent.Path, *ent.Size
		if !strings.HasSuffix(path, ".go") {
			continue
		}
		if strings.HasSuffix(path, ".pb.go") {
			continue
		}
		if size > sizeLimit {
			log.Printf("Skipping %s because it is too big: %d > %d", path, size, sizeLimit)
			continue
		}
		//log.Printf("+ %s (%d bytes)", path, size)

		wg.Add(1)
		nGo++
		go func() {
			defer wg.Done()

			src, err := client.GetBlob(sha1)
			if err != nil {
				log.Printf("Getting blob for %s: %v", path, err)
				return
			}
			ps, err := linter.Lint(path, src)
			if err != nil {
				log.Printf("Linting %s: %v", path, err)
				return
			}
			for _, p := range ps {
				if p.Confidence < 0.8 { // TODO: flag
					continue
				}
				addProblem(fmt.Sprintf("%s:%v: %s", path, p.Position, p.Text))
			}
		}()
	}
	wg.Wait()

	sort.Strings(problems.list)
	for _, p := range problems.list {
		fmt.Println(p)
	}
	log.Printf("wow, there were %d problems in %d Go source files!", len(problems.list), nGo)
}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

// TODO(crawshaw): build darwin/arm cross compiler on darwin/{386,amd64}
// TODO(crawshaw): android/{386,arm64}

import (
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
)

const ndkVersion = "ndk-r10d"

var cmdInit = &command{
	run:   runInit,
	Name:  "init",
	Short: "install android compiler toolchain",
	Long: `
Init downloads and installs the Android C++ compiler toolchain.

The toolchain is installed in $GOPATH/pkg/gomobile.
`,
}

func runInit(cmd *command) error {
	if err := checkGoVersion(); err != nil {
		return err
	}

	// Provide an early error message if Go was installed system-wide.
	goroot := goEnv("GOROOT")
	sentinel := filepath.Join(goroot, "gomobile-sentinel")
	if err := ioutil.WriteFile(sentinel, []byte("write test"), 0664); err != nil {
		if os.IsPermission(err) {
			return fmt.Errorf("GOROOT %q is not writable. Run:\n\tsudo gomobile init", goroot)
		}
		return fmt.Errorf("GOROOT not writable: %v", err)
	}
	os.Remove(sentinel)

	arch := runtime.GOARCH
	if arch == "amd64" {
		arch = "x86_64"
	}

	gopaths := filepath.SplitList(goEnv("GOPATH"))
	if len(gopaths) == 0 {
		return fmt.Errorf("GOPATH is not set")
	}
	ndkccpath = filepath.Join(gopaths[0], filepath.FromSlash("pkg/gomobile/android-"+ndkVersion))
	if buildX {
		fmt.Fprintln(os.Stderr, "NDKCCPATH="+ndkccpath)
	}

	if err := removeAll(ndkccpath); err != nil && !os.IsExist(err) {
		return err
	}
	dst := filepath.Join(ndkccpath, "arm")
	dstSysroot := filepath.Join(dst, "sysroot", "usr")
	if err := mkdir(dstSysroot); err != nil {
		return err
	}

	if buildN {
		tmpdir = filepath.Join(ndkccpath, "work")
	} else {
		var err error
		tmpdir, err = ioutil.TempDir(ndkccpath, "gomobile-init-")
		if err != nil {
			return err
		}
	}
	if buildX {
		fmt.Fprintln(os.Stderr, "WORK="+tmpdir)
	}
	defer removeAll(tmpdir)

	ndkName := "android-" + ndkVersion + "-" + runtime.GOOS + "-" + arch + "."
	if runtime.GOOS == "windows" {
		ndkName += "exe"
	} else {
		ndkName += "bin"
	}

	url := "http://dl.google.com/android/ndk/" + ndkName
	if err := fetch(filepath.Join(tmpdir, ndkName), url); err != nil {
		return err
	}

	inflate := exec.Command(filepath.Join(tmpdir, ndkName))
	inflate.Dir = tmpdir
	if buildX {
		printcmd("%s", inflate.Args[0])
	}
	if !buildN {
		out, err := inflate.CombinedOutput()
		if err != nil {
			if buildV {
				os.Stderr.Write(out)
			}
			return err
		}
	}
	srcSysroot := filepath.Join(tmpdir, "android-ndk-r10d", "platforms", "android-15", "arch-arm", "usr")
	if err := move(dstSysroot, srcSysroot, "include", "lib"); err != nil {
		return err
	}

	ndkpath := filepath.Join(tmpdir, "android-ndk-r10d", "toolchains", "arm-linux-androideabi-4.8", "prebuilt", runtime.GOOS+"-"+arch)
	if err := move(dst, ndkpath, "bin", "lib", "libexec"); err != nil {
		return err
	}

	linkpath := filepath.Join(dst, "arm-linux-androideabi", "bin")
	if err := mkdir(linkpath); err != nil {
		return err
	}
	for _, name := range []string{"ld", "ld.gold", "as", "gcc", "g++"} {
		if err := symlink(filepath.Join(dst, "bin", "arm-linux-androideabi-"+name), filepath.Join(linkpath, name)); err != nil {
			return err
		}
	}

	// TODO(crawshaw): make.bat on windows
	ccpath := filepath.Join(dst, "bin")
	make := exec.Command(filepath.Join(goroot, "src", "make.bash"), "--no-clean")
	make.Dir = filepath.Join(goroot, "src")
	make.Env = []string{
		`PATH=` + os.Getenv("PATH"),
		`TMPDIR=` + tmpdir,
		`HOME=` + os.Getenv("HOME"), // for default the go1.4 bootstrap
		`GOOS=android`,
		`GOARCH=arm`,
		`GOARM=7`,
		`CGO_ENABLED=1`,
		`CC_FOR_TARGET=` + filepath.Join(ccpath, "arm-linux-androideabi-gcc"),
		`CXX_FOR_TARGET=` + filepath.Join(ccpath, "arm-linux-androideabi-g++"),
		`GOBIN=` + tmpdir, // avoid overwriting current Go tool
	}
	if v := goEnv("GOROOT_BOOTSTRAP"); v != "" {
		make.Env = append(make.Env, `GOROOT_BOOTSTRAP=`+v)
	}
	if buildV {
		fmt.Fprintf(os.Stderr, "building android/arm cross compiler\n")
		make.Stdout = os.Stdout
		make.Stderr = os.Stderr
	}
	if buildX {
		printcmd("%s", strings.Join(make.Env, " ")+" "+strings.Join(make.Args, " "))
	}
	if buildN {
		return nil
	}
	if err := make.Run(); err != nil {
		return err
	}
	return nil
}

func move(dst, src string, names ...string) error {
	for _, name := range names {
		srcf := filepath.Join(src, name)
		dstf := filepath.Join(dst, name)
		if buildX {
			printcmd("mv %s %s", srcf, dstf)
		}
		if buildN {
			continue
		}
		if err := os.Rename(srcf, dstf); err != nil {
			return err
		}
	}
	return nil
}

func mkdir(dir string) error {
	if buildX {
		printcmd("mkdir -p %s", dir)
	}
	if buildN {
		return nil
	}
	return os.MkdirAll(dir, 0755)
}

func symlink(src, dst string) error {
	if buildX {
		printcmd("ln -s %s %s", src, dst)
	}
	if buildN {
		return nil
	}
	return os.Symlink(src, dst)
}

func checkGoVersion() error {
	if err := exec.Command("which", "go").Run(); err != nil {
		return fmt.Errorf(`no Go tool on $PATH`)
	}
	_, err := exec.Command("go", "version").Output()
	if err != nil {
		return fmt.Errorf("bad Go tool: %v", err)
	}
	return nil
}

func fetch(dst, url string) error {
	if buildV {
		fmt.Fprintf(os.Stderr, "fetching %s\n", url)
	}
	if buildX {
		printcmd("curl -o%s %s", dst, url)
	}
	if buildN {
		return nil
	}

	f, err := os.OpenFile(dst, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0755)
	if err != nil {
		return err
	}

	// TODO(crawshaw): The arm compiler toolchain compresses to 33 MB, less than a tenth of the NDK. Provide an alternative binary download.
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	_, err = io.Copy(f, resp.Body)
	err2 := resp.Body.Close()
	err3 := f.Close()
	if err != nil {
		return err
	}
	if err2 != nil {
		return err2
	}
	return err3
}

func removeAll(path string) error {
	if buildX {
		printcmd("rm -r -f %q", path)
	}
	if buildN {
		return nil
	}
	return os.RemoveAll(path)
}

func goEnv(name string) string {
	if val := os.Getenv(name); val != "" {
		return val
	}
	val, err := exec.Command("go", "env", name).Output()
	if err != nil {
		panic(err) // the Go tool was tested to work earlier
	}
	return strings.TrimSpace(string(val))
}
<commit_msg>cmd/gomobile: use stripped NDK tarballs<commit_after>// Copyright 2015 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

// TODO(crawshaw): build darwin/arm cross compiler on darwin/{386,amd64}
// TODO(crawshaw): android/{386,arm64}

import (
	"archive/tar"
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
)

// useStrippedNDK determines whether the init subcommand fetches the GCC
// toolchain from the original Android NDK, or from the stripped-down NDK
// hosted specifically for the gomobile tool.
//
// There is a significant size different (400MB compared to 30MB).
var useStrippedNDK = runtime.GOOS == "linux" || runtime.GOOS == "darwin"

const ndkVersion = "ndk-r10d"

var arch = runtime.GOARCH

var cmdInit = &command{
	run:   runInit,
	Name:  "init",
	Short: "install android compiler toolchain",
	Long: `
Init downloads and installs the Android C++ compiler toolchain.

The toolchain is installed in $GOPATH/pkg/gomobile.
`,
}

func runInit(cmd *command) error {
	if err := checkGoVersion(); err != nil {
		return err
	}

	// Provide an early error message if Go was installed system-wide.
	goroot := goEnv("GOROOT")
	sentinel := filepath.Join(goroot, "gomobile-sentinel")
	if err := ioutil.WriteFile(sentinel, []byte("write test"), 0664); err != nil {
		if os.IsPermission(err) {
			return fmt.Errorf("GOROOT %q is not writable. Run:\n\tsudo gomobile init", goroot)
		}
		return fmt.Errorf("GOROOT not writable: %v", err)
	}
	os.Remove(sentinel)

	if arch == "amd64" {
		arch = "x86_64"
	}

	gopaths := filepath.SplitList(goEnv("GOPATH"))
	if len(gopaths) == 0 {
		return fmt.Errorf("GOPATH is not set")
	}
	ndkccpath = filepath.Join(gopaths[0], filepath.FromSlash("pkg/gomobile/android-"+ndkVersion))
	if buildX {
		fmt.Fprintln(os.Stderr, "NDKCCPATH="+ndkccpath)
	}

	if err := removeAll(ndkccpath); err != nil && !os.IsExist(err) {
		return err
	}
	dst := filepath.Join(ndkccpath, "arm")
	dstSysroot := filepath.Join(dst, "sysroot", "usr")
	if err := mkdir(dstSysroot); err != nil {
		return err
	}

	if buildN {
		tmpdir = filepath.Join(ndkccpath, "work")
	} else {
		var err error
		tmpdir, err = ioutil.TempDir(ndkccpath, "gomobile-init-")
		if err != nil {
			return err
		}
	}
	if buildX {
		fmt.Fprintln(os.Stderr, "WORK="+tmpdir)
	}
	defer removeAll(tmpdir)

	if err := fetchNDK(); err != nil {
		return err
	}

	srcSysroot := filepath.Join(tmpdir, "android-ndk-r10d", "platforms", "android-15", "arch-arm", "usr")
	if err := move(dstSysroot, srcSysroot, "include", "lib"); err != nil {
		return err
	}

	ndkpath := filepath.Join(tmpdir, "android-ndk-r10d", "toolchains", "arm-linux-androideabi-4.8", "prebuilt", runtime.GOOS+"-"+arch)
	if err := move(dst, ndkpath, "bin", "lib", "libexec"); err != nil {
		return err
	}

	linkpath := filepath.Join(dst, "arm-linux-androideabi", "bin")
	if err := mkdir(linkpath); err != nil {
		return err
	}
	for _, name := range []string{"ld", "ld.gold", "as", "gcc", "g++"} {
		if err := symlink(filepath.Join(dst, "bin", "arm-linux-androideabi-"+name), filepath.Join(linkpath, name)); err != nil {
			return err
		}
	}

	// TODO(crawshaw): make.bat on windows
	ccpath := filepath.Join(dst, "bin")
	make := exec.Command(filepath.Join(goroot, "src", "make.bash"), "--no-clean")
	make.Dir = filepath.Join(goroot, "src")
	make.Env = []string{
		`PATH=` + os.Getenv("PATH"),
		`TMPDIR=` + tmpdir,
		`HOME=` + os.Getenv("HOME"), // for default the go1.4 bootstrap
		`GOOS=android`,
		`GOARCH=arm`,
		`GOARM=7`,
		`CGO_ENABLED=1`,
		`CC_FOR_TARGET=` + filepath.Join(ccpath, "arm-linux-androideabi-gcc"),
		`CXX_FOR_TARGET=` + filepath.Join(ccpath, "arm-linux-androideabi-g++"),
		`GOBIN=` + tmpdir, // avoid overwriting current Go tool
	}
	if v := goEnv("GOROOT_BOOTSTRAP"); v != "" {
		make.Env = append(make.Env, `GOROOT_BOOTSTRAP=`+v)
	}
	if buildV {
		fmt.Fprintf(os.Stderr, "building android/arm cross compiler\n")
		make.Stdout = os.Stdout
		make.Stderr = os.Stderr
	}
	if buildX {
		printcmd("%s", strings.Join(make.Env, " ")+" "+strings.Join(make.Args, " "))
	}
	if buildN {
		return nil
	}
	if err := make.Run(); err != nil {
		return err
	}
	return nil
}

func move(dst, src string, names ...string) error {
	for _, name := range names {
		srcf := filepath.Join(src, name)
		dstf := filepath.Join(dst, name)
		if buildX {
			printcmd("mv %s %s", srcf, dstf)
		}
		if buildN {
			continue
		}
		if err := os.Rename(srcf, dstf); err != nil {
			return err
		}
	}
	return nil
}

func mkdir(dir string) error {
	if buildX {
		printcmd("mkdir -p %s", dir)
	}
	if buildN {
		return nil
	}
	return os.MkdirAll(dir, 0755)
}

func symlink(src, dst string) error {
	if buildX {
		printcmd("ln -s %s %s", src, dst)
	}
	if buildN {
		return nil
	}
	return os.Symlink(src, dst)
}

func checkGoVersion() error {
	if err := exec.Command("which", "go").Run(); err != nil {
		return fmt.Errorf(`no Go tool on $PATH`)
	}
	_, err := exec.Command("go", "version").Output()
	if err != nil {
		return fmt.Errorf("bad Go tool: %v", err)
	}
	return nil
}

func fetchNDK() error {
	if useStrippedNDK {
		return fetchStrippedNDK()
	}

	ndkName := "android-" + ndkVersion + "-" + runtime.GOOS + "-" + arch + "."
	if runtime.GOOS == "windows" {
		ndkName += "exe"
	} else {
		ndkName += "bin"
	}
	url := "https://dl.google.com/android/ndk/" + ndkName
	if err := fetch(filepath.Join(tmpdir, ndkName), url); err != nil {
		return err
	}

	inflate := exec.Command(filepath.Join(tmpdir, ndkName))
	inflate.Dir = tmpdir
	if buildX {
		printcmd("%s", inflate.Args[0])
	}
	if buildN {
		return nil
	}
	out, err := inflate.CombinedOutput()
	if err != nil {
		if buildV {
			os.Stderr.Write(out)
		}
	}
	return err
}

func fetchStrippedNDK() error {
	name := "gomobile-ndk-r10d-" + runtime.GOOS + "-" + arch + ".tar.gz"
	url := "https://dl.google.com/go/mobile/" + name
	if err := fetch(filepath.Join(tmpdir, name), url); err != nil {
		return err
	}
	if buildX {
		printcmd("tar xfz %s", name)
	}
	if buildN {
		return nil
	}

	tf, err := os.Open(filepath.Join(tmpdir, name))
	if err != nil {
		return err
	}
	defer tf.Close()
	zr, err := gzip.NewReader(tf)
	if err != nil {
		return err
	}
	tr := tar.NewReader(zr)
	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}
		dst := filepath.Join(tmpdir, hdr.Name)
		if hdr.Typeflag == tar.TypeSymlink {
			if err := symlink(hdr.Linkname, dst); err != nil {
				return err
			}
			continue
		}
		if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
			return err
		}
		f, err := os.OpenFile(dst, os.O_CREATE|os.O_EXCL|os.O_WRONLY, os.FileMode(hdr.Mode)&0777)
		if err != nil {
			return err
		}
		if _, err := io.Copy(f, tr); err != nil {
			return err
		}
		if err := f.Close(); err != nil {
			return err
		}
	}
	return nil
}

func fetch(dst, url string) error {
	if buildV {
		fmt.Fprintf(os.Stderr, "fetching %s\n", url)
	}
	if buildX {
		printcmd("curl -o%s %s", dst, url)
	}
	if buildN {
		return nil
	}

	f, err := os.OpenFile(dst, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0755)
	if err != nil {
		return err
	}

	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	_, err = io.Copy(f, resp.Body)
	err2 := resp.Body.Close()
	err3 := f.Close()
	if err != nil {
		return err
	}
	if err2 != nil {
		return err2
	}
	return err3
}

func removeAll(path string) error {
	if buildX {
		printcmd("rm -r -f %q", path)
	}
	if buildN {
		return nil
	}
	return os.RemoveAll(path)
}

func goEnv(name string) string {
	if val := os.Getenv(name); val != "" {
		return val
	}
	val, err := exec.Command("go", "env", name).Output()
	if err != nil {
		panic(err) // the Go tool was tested to work earlier
	}
	return strings.TrimSpace(string(val))
}
<|endoftext|>


<commit_before>// groot-ls recursively dumps the hierarchy tree of a ROOT file
package main

import (
	"flag"
	"fmt"
	//"log"
	"os"
	//"runtime/pprof"
	"strings"

	"bitbucket.org/binet/go-root/pkg/groot"
)

var fname = flag.String("f", "ntuple.root", "ROOT file to inspect")
var detailed = flag.Bool("detailed", false, "enable detailed dump (of trees)")
//var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")

var s_tee = "|--"
var s_bot = "`--"

func normpath(path []string) string {
	name := strings.Join(path, "/")
	if len(name) > 2 && name[:2] == "//" {
		name = name[1:]
	}
	return name
}

func inspect(dir *groot.Directory, path []string, indent string) {
	name := normpath(path)
	if dir == nil {
		fmt.Printf("err: invalid directory [%s]\n", name)
		return
	}
	keys := dir.Keys()
	nkeys := len(keys)
	str := s_tee
	//fmt.Printf("%s%s -> #%d key(s)\n", indent, name, len(keys))
	for i, k := range keys {
		if i+1 >= nkeys {
			str = s_bot
		}
		switch v := k.Value().(type) {
		default:
			fmt.Printf("%s%s %s title='%s' type=%s\n",
				indent, str, k.Name(), k.Title(), k.Class())

		case *groot.Directory:
			fmt.Printf("%s%s %s title='%s' type=%s\n",
				indent, str,
				k.Name(), k.Title(), k.Class())
			path := append(path, k.Name())
			inspect(v, path, indent+"    ")

		case *groot.Tree:
			nbranches := len(v.Branches())
			fmt.Printf("%s%s %s title='%s' entries=%v nbranches=%v type=%s\n",
				indent, str,
				k.Name(), k.Title(), v.Entries(), nbranches, k.Class())
			if *detailed {
				strbr := s_tee
				for i,branch := range v.Branches() {
					if i+1 >= nbranches {
						strbr = s_bot
					}
					fmt.Printf(" %s%s%s %s type=%s\n",
						indent, "   ", strbr, branch.Name(), branch.Class())
				}
			}
		}
	}
}

func main() {
	fmt.Printf(":: groot-ls ::\n")
	flag.Parse()

	// if *cpuprofile != "" {
    //     f, err := os.Create(*cpuprofile)
    //     if err != nil {
    //         log.Fatal(err)
    //     }
    //     pprof.StartCPUProfile(f)
    //     defer pprof.StopCPUProfile()
    // }

	f, err := groot.NewFileReader(*fname)
	if err != nil {
		fmt.Printf("**error**: %v\n", err)
		os.Exit(1)
	}

	if f == nil {
		fmt.Printf("**error**: invalid file pointer\n")
		os.Exit(1)
	}

	fmt.Printf("file: '%s' (version=%v)\n", f.Name(), f.Version())

	dir := f.Dir()
	inspect(dir, []string{"/"}, "")

	fmt.Printf("::bye.\n")
}

// EOF
<commit_msg>cosmetics<commit_after>// groot-ls recursively dumps the hierarchy tree of a ROOT file
package main

import (
	"flag"
	"fmt"
	//"log"
	"os"
	//"runtime/pprof"
	"strings"

	"bitbucket.org/binet/go-root/pkg/groot"
)

var fname = flag.String("f", "", "ROOT file to inspect")
var detailed = flag.Bool("detailed", false, "enable detailed dump (of trees)")
//var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")

var s_tee = "|--"
var s_bot = "`--"

func normpath(path []string) string {
	name := strings.Join(path, "/")
	if len(name) > 2 && name[:2] == "//" {
		name = name[1:]
	}
	return name
}

func inspect(dir *groot.Directory, path []string, indent string) {
	name := normpath(path)
	if dir == nil {
		fmt.Printf("err: invalid directory [%s]\n", name)
		return
	}
	keys := dir.Keys()
	nkeys := len(keys)
	str := s_tee
	//fmt.Printf("%s%s -> #%d key(s)\n", indent, name, len(keys))
	for i, k := range keys {
		if i+1 >= nkeys {
			str = s_bot
		}
		switch v := k.Value().(type) {
		default:
			fmt.Printf("%s%s %s title='%s' type=%s\n",
				indent, str, k.Name(), k.Title(), k.Class())

		case *groot.Directory:
			fmt.Printf("%s%s %s title='%s' type=%s\n",
				indent, str,
				k.Name(), k.Title(), k.Class())
			path := append(path, k.Name())
			inspect(v, path, indent+"    ")

		case *groot.Tree:
			nbranches := len(v.Branches())
			fmt.Printf("%s%s %s title='%s' entries=%v nbranches=%v type=%s\n",
				indent, str,
				k.Name(), k.Title(), v.Entries(), nbranches, k.Class())
			if *detailed {
				strbr := s_tee
				for i,branch := range v.Branches() {
					if i+1 >= nbranches {
						strbr = s_bot
					}
					fmt.Printf(" %s%s%s %s type=%s\n",
						indent, "   ", strbr, branch.Name(), branch.Class())
				}
			}
		}
	}
}

func main() {
	fmt.Printf(":: groot-ls ::\n")
	flag.Parse()

	// if *cpuprofile != "" {
    //     f, err := os.Create(*cpuprofile)
    //     if err != nil {
    //         log.Fatal(err)
    //     }
    //     pprof.StartCPUProfile(f)
    //     defer pprof.StopCPUProfile()
    // }

	if *fname == "" {
		fmt.Printf("**error** you have to give a (valid) path to a ROOT file\n")
		os.Exit(1)
	}

	f, err := groot.NewFileReader(*fname)
	if err != nil {
		fmt.Printf("**error** %v\n", err)
		os.Exit(1)
	}

	if f == nil {
		fmt.Printf("**error** invalid file pointer\n")
		os.Exit(1)
	}

	fmt.Printf("file: '%s' (version=%v)\n", f.Name(), f.Version())

	dir := f.Dir()
	inspect(dir, []string{"/"}, "")

	fmt.Printf("::bye.\n")
}

// EOF
<|endoftext|>