<commit_before>/*
Package plugins provides support for creating plugin-based applications.

Following workflow is supposed:
	1) main application creates RPC Host service;
	2) plugin starts and connects to the RPC Host:
		2.1) plugin informs RPC Host about service, provided by that plugin and requests port to serve on;
  		2.2) RPC Host checks if such service is not provided by other connected plugin and generates port for plugin to serve on;
		2.3) plugin starts RPC server on port, provided by RPC Host, and informs RPC Host about it;
		2.4) RPC Host registers plugin service;
	3) when plugin terminates, it informs RPC Host; RPC Host unregisters plugin service;
	4) when RPC Host terminates, it informs all connected plugins; plugins terminates;
	5) when main application need to invoke some service method, RPC Host is used for dispatch it;

For handling remote calls net/rpc package usage is supposed.
A plugin must register object(s) that will be used for handling remote calls.
*/
package plugins

import (
	"fmt"
	"log"
	"time"
	"errors"
	"os"
	"os/signal"
	"syscall"
	"net"
	"net/rpc"
)

// Options for RPC client/server.
type Options struct {
	ipAddr	string
	Port	int				// network port for serving connection or connecting to the remote server
	Log		*log.Logger		// a object for logging operations
}

func (o *Options) listenAddr() string {
	return fmt.Sprintf(":%d", o.Port)
}

func (o *Options) dialAddr() string {
	return fmt.Sprintf("%s:%d", o.ipAddr, o.Port)
}

func (o *Options) applyDefaults(name string) {
	if len(o.ipAddr) == 0 {
		o.ipAddr = "127.0.0.1"
	}
	if o.Port <= 0 {
		o.Port = 4000
	}
	if o.Log == nil {
		o.Log = log.New(os.Stdout, fmt.Sprintf("[%s] ", name), 0)
	}
}

func prepareOptions(name string, options ...*Options) Options {
	var opt Options
	if len(options) > 0 {
		opt = *options[0]
	}
	opt.applyDefaults(name)
	return opt
}

// PluginInfo describes a plugin.
type PluginInfo struct {
	Name			string		// Human-readable name of plugin
	ServiceName		string		// Name of service provided by plugin
	Port			int			// Port plugin serves on
}

// rpcServer represents an RPC Server.
type rpcServer interface {
	Serve()
	onServe(func())
	onStop(func())
}

type rpcSrv struct {
	listener	net.Listener
	log			*log.Logger
	running		bool
	sChan		chan os.Signal
	serve		[]func()
	stop		[]func()
}

func (s *rpcSrv) Serve() {
	if !s.running {
		s.running = true
		go func() {
			time.Sleep(500 * time.Millisecond)
			for _, f := range s.serve {
				f()
			}
		}()

		for {
			if conn, err := s.listener.Accept(); err != nil {
				s.log.Fatal("accept error: " + err.Error())         //!!  panic???
			} else {
				go rpc.ServeConn(conn)
			}
		}
	}
};

func (s *rpcSrv) onServe(h func()) {
	s.serve = append(s.serve, h)
}

func (s *rpcSrv) onStop(h func()) {
	s.stop = append(s.stop, h)
}

func (s *rpcSrv) handleStop() {
	for _, f := range s.stop {
		f()
	}
	os.Exit(0);
}

// newRPCServer creates a new rpcServer instance.
func newRPCServer(opt *Options, rpcServices ...interface{}) (rpcServer, error) {
	err := registerRPCServices(rpcServices...)
	if err == nil {
		opt.applyDefaults("RPC Server")
		listener, err := net.Listen("tcp", opt.listenAddr())
		if err == nil {
			opt.Log.Printf("Listening on %s\n", opt.listenAddr())
			srv := &rpcSrv {
				listener: listener,
				log: opt.Log,
				sChan: make(chan os.Signal, 1),
				serve: make([]func(), 0),
				stop: make([]func(), 0)}
			signal.Notify(srv.sChan, os.Interrupt, os.Kill, syscall.SIGTERM)
			go func() {
				<- srv.sChan
				if srv.running {
					srv.handleStop()
				}
			}()
			return srv, nil
		}
	}
	return nil, err;
}

func registerRPCServices(rpcServices ...interface{}) error {
	for _, rpcService := range rpcServices {
		err := rpc.Register(rpcService)
		if err != nil {
			return err
		}
	}
	return nil
}

// rpcClient represents an RPC Client.
type rpcClient interface {
	Call(serviceMethod string, args interface{}, reply interface{}) error
	Close() error
}

type rpcClnt struct {
	clnt	*rpc.Client
	log		*log.Logger
}

func (c *rpcClnt) Call(serviceMethod string, args interface{}, reply interface{}) error {
	return c.clnt.Call(serviceMethod, args, reply)
}

func (c *rpcClnt) Close() error {
	return c.clnt.Close()
}

// newRPCClient creates a new rpcClient instance.
func newRPCClient(name string, options ...*Options) (rpcClient, error) {
	opt := prepareOptions(fmt.Sprintf("plugin %s", name), options...)
	clnt, err := rpc.Dial("tcp", opt.dialAddr())
	if err == nil {
		return &rpcClnt{clnt: clnt, log: opt.Log}, nil
	}
	return nil, err;

}

// A Host is a manager that plugins connects to.
// Host uses connected plugins for handling main application calls.
type Host interface {
	Serve()
	Call(serviceName string, serviceMethod string, args interface{}, reply interface{}) error
}

// ErrServiceNotRegistered returns when trying to access non-registered service
var ErrServiceNotRegistered = errors.New("service is not registered")

// ErrServiceAlreadyRegistered throws when trying to register already registered service
var ErrServiceAlreadyRegistered = errors.New("service is already registered")

type connectedPlugin struct {
	info	*PluginInfo
	clnt	rpcClient
}

func (c *connectedPlugin) Call(serviceMethod string, args interface{}, reply interface{}) error {
	return c.clnt.Call(serviceMethod, args, reply)
}

type host struct {
	srv			rpcServer
	log			*log.Logger
	nextPort	int					// free network port to connect
	availPorts	[]int				// network ports that are returned by disconnected plugins
	plugins		[]*connectedPlugin
}

func (h *host) Serve() {
	go h.srv.Serve()
}

func (h *host) Call(serviceName string, serviceMethod string, args interface{}, reply interface{}) error {
	var err error
	if i := h.indexOf(serviceName); i>=0 {
		if err = h.plugins[i].Call(serviceMethod, args, reply); err == rpc.ErrShutdown {
			h.removePlugin(serviceName)
			err = ErrServiceNotRegistered
		}
		return err
	}
	return ErrServiceNotRegistered
}

func (h *host) getNextPort() int {
	var rslt int
	if len(h.availPorts) > 0 {
		i := len(h.availPorts) - 1
		rslt, h.availPorts = h.availPorts[i], h.availPorts[:i]
	} else {
		rslt = h.nextPort
		h.nextPort++
	}
	return rslt
}

func (h *host) appendPlugin(cp *connectedPlugin) {
	if i := h.indexOf(cp.info.ServiceName); i<0 {
		h.plugins = append(h.plugins, cp)
	}
}

func (h *host) removePlugin(serviceName string) {
	if i := h.indexOf(serviceName); i>=0 {
		cp := h.plugins[i]
		var dummy int
		cp.Call("RPCPlugin.Terminate", dummy, &dummy)
		h.log.Printf("Plugin disconnected: \"%s\"", cp.info.Name)
		h.availPorts = append(h.availPorts, cp.info.Port)
		h.plugins = append(h.plugins[:i], h.plugins[i+1:]...)
	}
}

func (h *host) onStop() {
	for len(h.plugins)>0 {
		h.removePlugin(h.plugins[0].info.ServiceName)
	}
}

func (h *host) indexOf(serviceName string) int {
	for i, cp := range h.plugins {
		if cp.info.ServiceName == serviceName {
			return i
		}
	}
	return -1
}

// NewHost creates a new Host instance.
func NewHost(name string, options ...*Options) (Host, error) {
	opt := prepareOptions(name, options...)
	h := &host{
		log: opt.Log,
		nextPort: opt.Port+1,
		availPorts: make([]int, 0),
		plugins: make([]*connectedPlugin, 0)}
	srv, err := newRPCServer(&opt, &RPCHost{host: h})
	if err == nil {
		h.srv = srv
		srv.onStop(h.onStop)
		return h, nil
	}
	opt.Log.Println(err)
	return nil, err
}

// RPCHost implements Host RPC service. Handles plugins requests.
type RPCHost struct {
	host *host
}

// GetPort generates port for plugin to serve on.
func (rh *RPCHost) GetPort(info *PluginInfo, port *int) error {
	err := rh.mustBeDisconnected(info)
	if err == nil {
		*port = rh.host.getNextPort()
	}
	return err
}

// ConnectPlugin handles plugin connection to host.
func (rh *RPCHost) ConnectPlugin(info *PluginInfo, _ *int) error {
	err := rh.mustBeDisconnected(info)
	if err == nil {
		opt := prepareOptions(info.Name, &Options{Port: info.Port, Log: rh.host.log})
		if clnt, err := newRPCClient(info.Name, &opt); err == nil {
			cp := &connectedPlugin{info: info, clnt: clnt}
			rh.host.appendPlugin(cp)
			rh.host.log.Printf("Plugin connected: \"%s\", handling \"%s\", serves at %s", cp.info.Name, cp.info.ServiceName, opt.dialAddr())
			return nil
		}
	}
	return err
}

// DisconnectPlugin handles plugin disconnection from host.
func (rh *RPCHost) DisconnectPlugin(info *PluginInfo, _ *int) error {
	err := rh.mustBeConnected(info)
	if err == nil {
		rh.host.removePlugin(info.ServiceName)
	}
	return err
}

func (rh *RPCHost) mustBeDisconnected(info *PluginInfo) error {
	if rh.host.indexOf(info.ServiceName)>=0 {
		return ErrServiceAlreadyRegistered
	}
	return nil
}

func (rh *RPCHost) mustBeConnected(info *PluginInfo) error {
	if rh.host.indexOf(info.ServiceName)<0 {
		return ErrServiceNotRegistered
	}
	return nil
}

// Plugin is used for serving requests from main application.
type Plugin interface {
	Serve()
}

type plugin struct {
	info	*PluginInfo
	log		*log.Logger
	srv		rpcServer
	clnt	rpcClient
}

func (p *plugin) Serve() {
	p.srv.Serve()
}

func (p *plugin) onServe() {
	var dummy int
	if err := p.clnt.Call("RPCHost.ConnectPlugin", p.info, &dummy); err != nil {
		p.log.Fatal(err)
	}
}

func (p *plugin) onStop() {
	var dummy int
	if err := p.clnt.Call("RPCHost.DisconnectPlugin", p.info, &dummy); err != nil {
		p.log.Println(err)
	}
}

// NewPlugin creates a new Plugin instance.
func NewPlugin(pluginName string, serviceName string, options ...*Options) (Plugin, error) {
	opt := prepareOptions(pluginName, options...)
	clnt, err := newRPCClient(pluginName, &opt)
	if err == nil {
		opt.Log.Printf("Connected to RPC host: %s\n", opt.dialAddr())
		info := &PluginInfo{Name: pluginName, ServiceName: serviceName}
		if err = clnt.Call("RPCHost.GetPort", info, &info.Port); err == nil {
			p := &plugin{info: info, log: opt.Log, clnt: clnt}
			srvOpt := prepareOptions(pluginName, &Options{Port: info.Port, Log: opt.Log})
			if srv, err := newRPCServer(&srvOpt, &RPCPlugin{plugin: p}); err == nil {
				p.srv = srv
				srv.onServe(p.onServe)
				srv.onStop(p.onStop)
				return p, nil
			}
		}
		clnt.Close()
	}
	opt.Log.Println(err)
	return nil, err
}

// RPCPlugin implements Plugin RPC service. Handles  RPC Host requests.
type RPCPlugin struct {
	plugin	*plugin
}

// Terminate handles RPC Host termination.
func (rp *RPCPlugin) Terminate(_ int, _ *int) error {
	rp.plugin.log.Println("Terminating by RPC host request")
	os.Exit(0)
	return nil
}
<commit_msg>Added Host.Services() method<commit_after>/*
Package plugins provides support for creating plugin-based applications.

Following workflow is supposed:
	1) main application creates RPC Host service;
	2) plugin starts and connects to the RPC Host:
		2.1) plugin informs RPC Host about service, provided by that plugin and requests port to serve on;
  		2.2) RPC Host checks if such service is not provided by other connected plugin and generates port for plugin to serve on;
		2.3) plugin starts RPC server on port, provided by RPC Host, and informs RPC Host about it;
		2.4) RPC Host registers plugin service;
	3) when plugin terminates, it informs RPC Host; RPC Host unregisters plugin service;
	4) when RPC Host terminates, it informs all connected plugins; plugins terminates;
	5) when main application need to invoke some service method, RPC Host is used for dispatch it;

For handling remote calls net/rpc package usage is supposed.
A plugin must register object(s) that will be used for handling remote calls.
*/
package plugins

import (
	"fmt"
	"log"
	"time"
	"errors"
	"os"
	"os/signal"
	"syscall"
	"net"
	"net/rpc"
)

// Options for RPC client/server.
type Options struct {
	ipAddr	string
	Port	int				// network port for serving connection or connecting to the remote server
	Log		*log.Logger		// a object for logging operations
}

func (o *Options) listenAddr() string {
	return fmt.Sprintf(":%d", o.Port)
}

func (o *Options) dialAddr() string {
	return fmt.Sprintf("%s:%d", o.ipAddr, o.Port)
}

func (o *Options) applyDefaults(name string) {
	if len(o.ipAddr) == 0 {
		o.ipAddr = "127.0.0.1"
	}
	if o.Port <= 0 {
		o.Port = 4000
	}
	if o.Log == nil {
		o.Log = log.New(os.Stdout, fmt.Sprintf("[%s] ", name), 0)
	}
}

func prepareOptions(name string, options ...*Options) Options {
	var opt Options
	if len(options) > 0 {
		opt = *options[0]
	}
	opt.applyDefaults(name)
	return opt
}

// PluginInfo describes a plugin.
type PluginInfo struct {
	Name			string		// Human-readable name of plugin
	ServiceName		string		// Name of service provided by plugin
	Port			int			// Port plugin serves on
}

// rpcServer represents an RPC Server.
type rpcServer interface {
	Serve()
	onServe(func())
	onStop(func())
}

type rpcSrv struct {
	listener	net.Listener
	log			*log.Logger
	running		bool
	sChan		chan os.Signal
	serve		[]func()
	stop		[]func()
}

func (s *rpcSrv) Serve() {
	if !s.running {
		s.running = true
		go func() {
			time.Sleep(500 * time.Millisecond)
			for _, f := range s.serve {
				f()
			}
		}()

		for {
			if conn, err := s.listener.Accept(); err != nil {
				s.log.Fatal("accept error: " + err.Error())         //!!  panic???
			} else {
				go rpc.ServeConn(conn)
			}
		}
	}
};

func (s *rpcSrv) onServe(h func()) {
	s.serve = append(s.serve, h)
}

func (s *rpcSrv) onStop(h func()) {
	s.stop = append(s.stop, h)
}

func (s *rpcSrv) handleStop() {
	for _, f := range s.stop {
		f()
	}
	os.Exit(0);
}

// newRPCServer creates a new rpcServer instance.
func newRPCServer(opt *Options, rpcServices ...interface{}) (rpcServer, error) {
	err := registerRPCServices(rpcServices...)
	if err == nil {
		opt.applyDefaults("RPC Server")
		listener, err := net.Listen("tcp", opt.listenAddr())
		if err == nil {
			opt.Log.Printf("Listening on %s\n", opt.listenAddr())
			srv := &rpcSrv {
				listener: listener,
				log: opt.Log,
				sChan: make(chan os.Signal, 1),
				serve: make([]func(), 0),
				stop: make([]func(), 0)}
			signal.Notify(srv.sChan, os.Interrupt, os.Kill, syscall.SIGTERM)
			go func() {
				<- srv.sChan
				if srv.running {
					srv.handleStop()
				}
			}()
			return srv, nil
		}
	}
	return nil, err;
}

func registerRPCServices(rpcServices ...interface{}) error {
	for _, rpcService := range rpcServices {
		err := rpc.Register(rpcService)
		if err != nil {
			return err
		}
	}
	return nil
}

// rpcClient represents an RPC Client.
type rpcClient interface {
	Call(serviceMethod string, args interface{}, reply interface{}) error
	Close() error
}

type rpcClnt struct {
	clnt	*rpc.Client
	log		*log.Logger
}

func (c *rpcClnt) Call(serviceMethod string, args interface{}, reply interface{}) error {
	return c.clnt.Call(serviceMethod, args, reply)
}

func (c *rpcClnt) Close() error {
	return c.clnt.Close()
}

// newRPCClient creates a new rpcClient instance.
func newRPCClient(name string, options ...*Options) (rpcClient, error) {
	opt := prepareOptions(fmt.Sprintf("plugin %s", name), options...)
	clnt, err := rpc.Dial("tcp", opt.dialAddr())
	if err == nil {
		return &rpcClnt{clnt: clnt, log: opt.Log}, nil
	}
	return nil, err;

}

// A Host is a manager that plugins connects to.
// Host uses connected plugins for handling main application calls.
type Host interface {
	Serve()
	Services() []string
	Call(serviceName string, serviceMethod string, args interface{}, reply interface{}) error
}

// ErrServiceNotRegistered returns when trying to access non-registered service
var ErrServiceNotRegistered = errors.New("service is not registered")

// ErrServiceAlreadyRegistered throws when trying to register already registered service
var ErrServiceAlreadyRegistered = errors.New("service is already registered")

type connectedPlugin struct {
	info	*PluginInfo
	clnt	rpcClient
}

func (c *connectedPlugin) Call(serviceMethod string, args interface{}, reply interface{}) error {
	return c.clnt.Call(serviceMethod, args, reply)
}

type host struct {
	srv			rpcServer
	log			*log.Logger
	nextPort	int					// free network port to connect
	availPorts	[]int				// network ports that are returned by disconnected plugins
	plugins		[]*connectedPlugin
}

func (h *host) Serve() {
	go h.srv.Serve()
}

func (h *host) Services() []string {
	rslt := make([]string, len(h.plugins))
	for i, cp := range h.plugins {
		rslt[i] = cp.info.ServiceName
	}
	return rslt
}

func (h *host) Call(serviceName string, serviceMethod string, args interface{}, reply interface{}) error {
	var err error
	if i := h.indexOf(serviceName); i>=0 {
		if err = h.plugins[i].Call(serviceMethod, args, reply); err == rpc.ErrShutdown {
			h.removePlugin(serviceName)
			err = ErrServiceNotRegistered
		}
		return err
	}
	return ErrServiceNotRegistered
}

func (h *host) getNextPort() int {
	var rslt int
	if len(h.availPorts) > 0 {
		i := len(h.availPorts) - 1
		rslt, h.availPorts = h.availPorts[i], h.availPorts[:i]
	} else {
		rslt = h.nextPort
		h.nextPort++
	}
	return rslt
}

func (h *host) appendPlugin(cp *connectedPlugin) {
	if i := h.indexOf(cp.info.ServiceName); i<0 {
		h.plugins = append(h.plugins, cp)
	}
}

func (h *host) removePlugin(serviceName string) {
	if i := h.indexOf(serviceName); i>=0 {
		cp := h.plugins[i]
		var dummy int
		cp.Call("RPCPlugin.Terminate", dummy, &dummy)
		h.log.Printf("Plugin disconnected: \"%s\"", cp.info.Name)
		h.availPorts = append(h.availPorts, cp.info.Port)
		h.plugins = append(h.plugins[:i], h.plugins[i+1:]...)
	}
}

func (h *host) onStop() {
	for len(h.plugins)>0 {
		h.removePlugin(h.plugins[0].info.ServiceName)
	}
}

func (h *host) indexOf(serviceName string) int {
	for i, cp := range h.plugins {
		if cp.info.ServiceName == serviceName {
			return i
		}
	}
	return -1
}

// NewHost creates a new Host instance.
func NewHost(name string, options ...*Options) (Host, error) {
	opt := prepareOptions(name, options...)
	h := &host{
		log: opt.Log,
		nextPort: opt.Port+1,
		availPorts: make([]int, 0),
		plugins: make([]*connectedPlugin, 0)}
	srv, err := newRPCServer(&opt, &RPCHost{host: h})
	if err == nil {
		h.srv = srv
		srv.onStop(h.onStop)
		return h, nil
	}
	opt.Log.Println(err)
	return nil, err
}

// RPCHost implements Host RPC service. Handles plugins requests.
type RPCHost struct {
	host *host
}

// GetPort generates port for plugin to serve on.
func (rh *RPCHost) GetPort(info *PluginInfo, port *int) error {
	err := rh.mustBeDisconnected(info)
	if err == nil {
		*port = rh.host.getNextPort()
	}
	return err
}

// ConnectPlugin handles plugin connection to host.
func (rh *RPCHost) ConnectPlugin(info *PluginInfo, _ *int) error {
	err := rh.mustBeDisconnected(info)
	if err == nil {
		opt := prepareOptions(info.Name, &Options{Port: info.Port, Log: rh.host.log})
		if clnt, err := newRPCClient(info.Name, &opt); err == nil {
			cp := &connectedPlugin{info: info, clnt: clnt}
			rh.host.appendPlugin(cp)
			rh.host.log.Printf("Plugin connected: \"%s\", handling \"%s\", serves at %s", cp.info.Name, cp.info.ServiceName, opt.dialAddr())
			return nil
		}
	}
	return err
}

// DisconnectPlugin handles plugin disconnection from host.
func (rh *RPCHost) DisconnectPlugin(info *PluginInfo, _ *int) error {
	err := rh.mustBeConnected(info)
	if err == nil {
		rh.host.removePlugin(info.ServiceName)
	}
	return err
}

func (rh *RPCHost) mustBeDisconnected(info *PluginInfo) error {
	if rh.host.indexOf(info.ServiceName)>=0 {
		return ErrServiceAlreadyRegistered
	}
	return nil
}

func (rh *RPCHost) mustBeConnected(info *PluginInfo) error {
	if rh.host.indexOf(info.ServiceName)<0 {
		return ErrServiceNotRegistered
	}
	return nil
}

// Plugin is used for serving requests from main application.
type Plugin interface {
	Serve()
}

type plugin struct {
	info	*PluginInfo
	log		*log.Logger
	srv		rpcServer
	clnt	rpcClient
}

func (p *plugin) Serve() {
	p.srv.Serve()
}

func (p *plugin) onServe() {
	var dummy int
	if err := p.clnt.Call("RPCHost.ConnectPlugin", p.info, &dummy); err != nil {
		p.log.Fatal(err)
	}
}

func (p *plugin) onStop() {
	var dummy int
	if err := p.clnt.Call("RPCHost.DisconnectPlugin", p.info, &dummy); err != nil {
		p.log.Println(err)
	}
}

// NewPlugin creates a new Plugin instance.
func NewPlugin(pluginName string, serviceName string, options ...*Options) (Plugin, error) {
	opt := prepareOptions(pluginName, options...)
	clnt, err := newRPCClient(pluginName, &opt)
	if err == nil {
		opt.Log.Printf("Connected to RPC host: %s\n", opt.dialAddr())
		info := &PluginInfo{Name: pluginName, ServiceName: serviceName}
		if err = clnt.Call("RPCHost.GetPort", info, &info.Port); err == nil {
			p := &plugin{info: info, log: opt.Log, clnt: clnt}
			srvOpt := prepareOptions(pluginName, &Options{Port: info.Port, Log: opt.Log})
			if srv, err := newRPCServer(&srvOpt, &RPCPlugin{plugin: p}); err == nil {
				p.srv = srv
				srv.onServe(p.onServe)
				srv.onStop(p.onStop)
				return p, nil
			}
		}
		clnt.Close()
	}
	opt.Log.Println(err)
	return nil, err
}

// RPCPlugin implements Plugin RPC service. Handles  RPC Host requests.
type RPCPlugin struct {
	plugin	*plugin
}

// Terminate handles RPC Host termination.
func (rp *RPCPlugin) Terminate(_ int, _ *int) error {
	rp.plugin.log.Println("Terminating by RPC host request")
	os.Exit(0)
	return nil
}
<|endoftext|>


<commit_before>package guardiancmd

import (
	"context"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"syscall"
	"time"

	"code.cloudfoundry.org/commandrunner"
	"code.cloudfoundry.org/commandrunner/linux_command_runner"
	"code.cloudfoundry.org/garden-shed/distclient"
	quotaed_aufs "code.cloudfoundry.org/garden-shed/docker_drivers/aufs"
	"code.cloudfoundry.org/garden-shed/layercake"
	"code.cloudfoundry.org/garden-shed/layercake/cleaner"
	"code.cloudfoundry.org/garden-shed/quota_manager"
	"code.cloudfoundry.org/garden-shed/repository_fetcher"
	"code.cloudfoundry.org/garden-shed/rootfs_provider"
	"code.cloudfoundry.org/guardian/gardener"
	"code.cloudfoundry.org/guardian/kawasaki"
	"code.cloudfoundry.org/guardian/kawasaki/dns"
	"code.cloudfoundry.org/guardian/logging"
	"code.cloudfoundry.org/guardian/rundmc"
	"code.cloudfoundry.org/guardian/rundmc/bundlerules"
	"code.cloudfoundry.org/guardian/rundmc/cgroups"
	"code.cloudfoundry.org/guardian/rundmc/execrunner/dadoo"
	"code.cloudfoundry.org/guardian/rundmc/goci"
	"code.cloudfoundry.org/guardian/rundmc/preparerootfs"
	"code.cloudfoundry.org/guardian/rundmc/runcontainerd"
	"code.cloudfoundry.org/guardian/rundmc/runcontainerd/nerd"
	"code.cloudfoundry.org/guardian/rundmc/runrunc"
	"code.cloudfoundry.org/guardian/rundmc/signals"
	"code.cloudfoundry.org/idmapper"
	"code.cloudfoundry.org/lager"
	"github.com/containerd/containerd"
	"github.com/containerd/containerd/linux/proc"
	"github.com/containerd/containerd/namespaces"
	"github.com/docker/docker/daemon/graphdriver"
	"github.com/docker/docker/graph"
	"github.com/docker/docker/pkg/mount"
	"github.com/eapache/go-resiliency/retrier"
	specs "github.com/opencontainers/runtime-spec/specs-go"
)

type LinuxFactory struct {
	config           *ServerCommand
	commandRunner    commandrunner.CommandRunner
	signallerFactory *signals.SignallerFactory
	uidMappings      idmapper.MappingList
	gidMappings      idmapper.MappingList
}

func (cmd *ServerCommand) NewGardenFactory() GardenFactory {
	uidMappings, gidMappings := cmd.idMappings()
	return &LinuxFactory{
		config:           cmd,
		commandRunner:    linux_command_runner.New(),
		signallerFactory: &signals.SignallerFactory{PidGetter: wirePidfileReader()},
		uidMappings:      uidMappings,
		gidMappings:      gidMappings,
	}
}

func (f *LinuxFactory) CommandRunner() commandrunner.CommandRunner {
	return f.commandRunner
}

func (f *LinuxFactory) WireVolumizer(logger lager.Logger) gardener.Volumizer {
	if f.config.Image.Plugin.Path() != "" || f.config.Image.PrivilegedPlugin.Path() != "" {
		return f.config.wireImagePlugin(f.commandRunner, f.uidMappings.Map(0), f.gidMappings.Map(0))
	}

	if f.config.Graph.Dir == "" {
		noop := gardener.NoopVolumizer{}
		return gardener.NewVolumeProvider(noop, noop, gardener.CommandFactory(preparerootfs.Command), f.commandRunner, f.uidMappings.Map(0), f.gidMappings.Map(0))
	}

	shed := f.wireShed(logger)
	return gardener.NewVolumeProvider(shed, shed, gardener.CommandFactory(preparerootfs.Command), f.commandRunner, f.uidMappings.Map(0), f.gidMappings.Map(0))
}

func wireEnvFunc() runrunc.EnvFunc {
	return runrunc.EnvFunc(runrunc.UnixEnvFor)
}

func (f *LinuxFactory) WireMkdirer() runrunc.Mkdirer {
	if runningAsRoot() {
		return bundlerules.MkdirChowner{Command: preparerootfs.Command, CommandRunner: f.commandRunner}
	}

	return NoopMkdirer{}
}

type NoopMkdirer struct{}

func (NoopMkdirer) MkdirAs(rootFSPathFile string, uid, gid int, mode os.FileMode, recreate bool, path ...string) error {
	return nil
}

func (f *LinuxFactory) WireExecRunner(runMode, runcRoot string) runrunc.ExecRunner {
	return dadoo.NewExecRunner(
		f.config.Bin.Dadoo.Path(),
		f.config.Runtime.Plugin,
		runcRoot,
		f.signallerFactory,
		f.commandRunner,
		f.config.Containers.CleanupProcessDirsOnWait,
		runMode,
	)
}

func (f *LinuxFactory) WireCgroupsStarter(logger lager.Logger) gardener.Starter {
	return createCgroupsStarter(logger, f.config.Server.Tag, "", &cgroups.OSChowner{}, rundmc.IsMountPoint)
}

func (cmd *SetupCommand) WireCgroupsStarter(logger lager.Logger) gardener.Starter {
	return createCgroupsStarter(logger, cmd.Tag, cmd.CgroupRoot, &cgroups.OSChowner{UID: cmd.RootlessUID, GID: cmd.RootlessGID}, rundmc.IsMountPoint)
}

func createCgroupsStarter(logger lager.Logger, tag, cgroupRoot string, chowner cgroups.Chowner, mountPointChecker rundmc.MountPointChecker) gardener.Starter {
	cgroupsMountpoint := cgroups.CgroupRoot
	gardenCgroup := cgroups.GardenCgroup
	if tag != "" {
		gardenCgroup = fmt.Sprintf("%s-%s", gardenCgroup, tag)
	}
	if cgroupRoot != "" {
		cgroupsMountpoint = cgroupRoot
	}

	return cgroups.NewStarter(logger, mustOpen("/proc/cgroups"), mustOpen("/proc/self/cgroup"),
		cgroupsMountpoint, gardenCgroup, allowedDevices, linux_command_runner.New(), chowner, mountPointChecker)
}

func (f *LinuxFactory) WireResolvConfigurer() kawasaki.DnsResolvConfigurer {
	return &kawasaki.ResolvConfigurer{
		HostsFileCompiler: &dns.HostsFileCompiler{},
		ResolvCompiler:    &dns.ResolvCompiler{},
		ResolvFilePath:    "/etc/resolv.conf",
		DepotDir:          f.config.Containers.Dir,
	}
}

func (f *LinuxFactory) WireRootfsFileCreator() rundmc.RootfsFileCreator {
	return preparerootfs.SymlinkRefusingFileCreator{}
}

func initBindMountAndPath(initPathOnHost string) (specs.Mount, string) {
	initPathInContainer := filepath.Join("/tmp", "garden-init")
	return specs.Mount{
		Type:        "bind",
		Source:      initPathOnHost,
		Destination: initPathInContainer,
		Options:     []string{"bind"},
	}, initPathInContainer
}

func defaultBindMounts() []specs.Mount {
	devptsGid := 0
	if runningAsRoot() {
		devptsGid = 5
	}

	return []specs.Mount{
		{Type: "sysfs", Source: "sysfs", Destination: "/sys", Options: []string{"nosuid", "noexec", "nodev", "ro"}},
		{Type: "tmpfs", Source: "tmpfs", Destination: "/dev/shm", Options: []string{"rw", "nodev", "relatime"}},
		{Type: "devpts", Source: "devpts", Destination: "/dev/pts",
			Options: []string{"nosuid", "noexec", "newinstance", fmt.Sprintf("gid=%d", devptsGid), "ptmxmode=0666", "mode=0620"}},
	}
}

func privilegedMounts() []specs.Mount {
	return []specs.Mount{
		{Type: "proc", Source: "proc", Destination: "/proc", Options: []string{"nosuid", "noexec", "nodev"}},
	}
}

func unprivilegedMounts() []specs.Mount {
	return []specs.Mount{
		{Type: "proc", Source: "proc", Destination: "/proc", Options: []string{"nosuid", "noexec", "nodev"}},
		{Type: "cgroup", Source: "cgroup", Destination: "/sys/fs/cgroup", Options: []string{"ro", "nosuid", "noexec", "nodev"}},
	}
}

func getPrivilegedDevices() []specs.LinuxDevice {
	return []specs.LinuxDevice{fuseDevice}
}

func bindMountPoints() []string {
	return []string{"/etc/hosts", "/etc/resolv.conf"}
}

func mustGetMaxValidUID() int {
	return idmapper.MustGetMaxValidUID()
}

func ensureServerSocketDoesNotLeak(socketFD uintptr) error {
	_, _, errNo := syscall.Syscall(syscall.SYS_FCNTL, socketFD, syscall.F_SETFD, syscall.FD_CLOEXEC)
	if errNo != 0 {
		return fmt.Errorf("setting cloexec on server socket: %s", errNo)
	}
	return nil
}

func (f *LinuxFactory) wireShed(logger lager.Logger) *rootfs_provider.CakeOrdinator {
	graphRoot := f.config.Graph.Dir
	logger = logger.Session(gardener.VolumizerSession, lager.Data{"graphRoot": graphRoot})

	if err := exec.Command("modprobe", "aufs").Run(); err != nil {
		logger.Error("unable-to-load-aufs", err)
	}

	runner := &logging.Runner{CommandRunner: linux_command_runner.New(), Logger: logger}

	if err := os.MkdirAll(graphRoot, 0755); err != nil {
		logger.Fatal("failed-to-create-graph-directory", err)
	}

	dockerGraphDriver, err := graphdriver.New(graphRoot, nil)
	if err != nil {
		logger.Fatal("failed-to-construct-graph-driver", err)
	}

	backingStoresPath := filepath.Join(graphRoot, "backing_stores")
	if mkdirErr := os.MkdirAll(backingStoresPath, 0660); mkdirErr != nil {
		logger.Fatal("failed-to-mkdir-backing-stores", mkdirErr)
	}

	quotaedGraphDriver := &quotaed_aufs.QuotaedDriver{
		GraphDriver: dockerGraphDriver,
		Unmount:     quotaed_aufs.Unmount,
		BackingStoreMgr: &quotaed_aufs.BackingStore{
			RootPath: backingStoresPath,
			Logger:   logger.Session("backing-store-mgr"),
		},
		LoopMounter: &quotaed_aufs.Loop{
			Retrier: retrier.New(retrier.ConstantBackoff(200, 500*time.Millisecond), nil),
			Logger:  logger.Session("loop-mounter"),
		},
		Retrier:  retrier.New(retrier.ConstantBackoff(200, 500*time.Millisecond), nil),
		RootPath: graphRoot,
		Logger:   logger.Session("quotaed-driver"),
	}

	dockerGraph, err := graph.NewGraph(graphRoot, quotaedGraphDriver)
	if err != nil {
		logger.Fatal("failed-to-construct-graph", err)
	}

	var cake layercake.Cake = &layercake.Docker{
		Graph:  dockerGraph,
		Driver: quotaedGraphDriver,
	}

	if cake.DriverName() == "aufs" {
		cake = &layercake.AufsCake{
			Cake:      cake,
			Runner:    runner,
			GraphRoot: graphRoot,
		}
	}

	repoFetcher := repository_fetcher.Retryable{
		RepositoryFetcher: &repository_fetcher.CompositeFetcher{
			LocalFetcher: &repository_fetcher.Local{
				Cake:              cake,
				DefaultRootFSPath: f.config.Containers.DefaultRootFS,
				IDProvider:        repository_fetcher.LayerIDProvider{},
			},
			RemoteFetcher: repository_fetcher.NewRemote(
				f.config.Docker.Registry,
				cake,
				distclient.NewDialer(f.config.Docker.InsecureRegistries),
				repository_fetcher.VerifyFunc(repository_fetcher.Verify),
			),
		},
	}

	rootFSNamespacer := &rootfs_provider.UidNamespacer{
		Translator: rootfs_provider.NewUidTranslator(
			f.uidMappings,
			f.gidMappings,
		),
	}

	retainer := cleaner.NewRetainer()
	ovenCleaner := cleaner.NewOvenCleaner(retainer,
		cleaner.NewThreshold(int64(f.config.Graph.CleanupThresholdInMegabytes)*1024*1024),
	)

	imageRetainer := &repository_fetcher.ImageRetainer{
		GraphRetainer:             retainer,
		DirectoryRootfsIDProvider: repository_fetcher.LayerIDProvider{},
		DockerImageIDFetcher:      repoFetcher,

		NamespaceCacheKey: rootFSNamespacer.CacheKey(),
		Logger:            logger,
	}

	// spawn off in a go function to avoid blocking startup
	// worst case is if an image is immediately created and deleted faster than
	// we can retain it we'll garbage collect it when we shouldn't. This
	// is an OK trade-off for not having garden startup block on dockerhub.
	go imageRetainer.Retain(f.config.Graph.PersistentImages)

	layerCreator := rootfs_provider.NewLayerCreator(cake, rootfs_provider.SimpleVolumeCreator{}, rootFSNamespacer)

	quotaManager := &quota_manager.AUFSQuotaManager{
		BaseSizer: quota_manager.NewAUFSBaseSizer(cake),
		DiffSizer: &quota_manager.AUFSDiffSizer{
			AUFSDiffPathFinder: quotaedGraphDriver,
		},
	}

	return rootfs_provider.NewCakeOrdinator(cake,
		repoFetcher,
		layerCreator,
		rootfs_provider.NewMetricsAdapter(quotaManager.GetUsage, quotaedGraphDriver.GetMntPath),
		ovenCleaner)
}

func wireMounts() bundlerules.Mounts {
	return bundlerules.Mounts{
		MountOptionsGetter: rundmc.GetMountOptions,
		MountInfosProvider: func() ([]*mount.Info, error) {
			return mount.GetMounts()
		},
	}
}

func wireContainerd(socket string, bndlLoader *goci.BndlLoader, wireExecer func(pidGetter runrunc.PidGetter) *runrunc.Execer) (rundmc.OCIRuntime, error) {
	containerdClient, err := containerd.New(socket)
	if err != nil {
		return nil, err
	}
	ctx := namespaces.WithNamespace(context.Background(), containerdNamespace)
	nerd := nerd.New(containerdClient, ctx)
	return runcontainerd.New(nerd, bndlLoader, wireExecer(&runcontainerd.PidGetter{Nerd: nerd})), nil
}

func containerdRuncRoot() string {
	return proc.RuncRoot
}
<commit_msg>Move "wireShed" to shed<commit_after>package guardiancmd

import (
	"context"
	"fmt"
	"os"
	"path/filepath"

	"code.cloudfoundry.org/commandrunner"
	"code.cloudfoundry.org/commandrunner/linux_command_runner"
	"code.cloudfoundry.org/garden-shed/rootfs_provider"
	"code.cloudfoundry.org/guardian/gardener"
	"code.cloudfoundry.org/guardian/kawasaki"
	"code.cloudfoundry.org/guardian/kawasaki/dns"
	"code.cloudfoundry.org/guardian/logging"
	"code.cloudfoundry.org/guardian/rundmc"
	"code.cloudfoundry.org/guardian/rundmc/bundlerules"
	"code.cloudfoundry.org/guardian/rundmc/cgroups"
	"code.cloudfoundry.org/guardian/rundmc/execrunner/dadoo"
	"code.cloudfoundry.org/guardian/rundmc/goci"
	"code.cloudfoundry.org/guardian/rundmc/preparerootfs"
	"code.cloudfoundry.org/guardian/rundmc/runcontainerd"
	"code.cloudfoundry.org/guardian/rundmc/runcontainerd/nerd"
	"code.cloudfoundry.org/guardian/rundmc/runrunc"
	"code.cloudfoundry.org/guardian/rundmc/signals"
	"code.cloudfoundry.org/idmapper"
	"code.cloudfoundry.org/lager"
	"github.com/containerd/containerd"
	"github.com/containerd/containerd/linux/proc"
	"github.com/containerd/containerd/namespaces"
	"github.com/docker/docker/pkg/mount"
	specs "github.com/opencontainers/runtime-spec/specs-go"
	"golang.org/x/sys/unix"
)

type LinuxFactory struct {
	config           *ServerCommand
	commandRunner    commandrunner.CommandRunner
	signallerFactory *signals.SignallerFactory
	uidMappings      idmapper.MappingList
	gidMappings      idmapper.MappingList
}

func (cmd *ServerCommand) NewGardenFactory() GardenFactory {
	uidMappings, gidMappings := cmd.idMappings()
	return &LinuxFactory{
		config:           cmd,
		commandRunner:    linux_command_runner.New(),
		signallerFactory: &signals.SignallerFactory{PidGetter: wirePidfileReader()},
		uidMappings:      uidMappings,
		gidMappings:      gidMappings,
	}
}

func (f *LinuxFactory) CommandRunner() commandrunner.CommandRunner {
	return f.commandRunner
}

func (f *LinuxFactory) WireVolumizer(logger lager.Logger) gardener.Volumizer {
	if f.config.Image.Plugin.Path() != "" || f.config.Image.PrivilegedPlugin.Path() != "" {
		return f.config.wireImagePlugin(f.commandRunner, f.uidMappings.Map(0), f.gidMappings.Map(0))
	}

	if f.config.Graph.Dir == "" {
		noop := gardener.NoopVolumizer{}
		return gardener.NewVolumeProvider(noop, noop, gardener.CommandFactory(preparerootfs.Command), f.commandRunner, f.uidMappings.Map(0), f.gidMappings.Map(0))
	}

	runner := &logging.Runner{CommandRunner: linux_command_runner.New(), Logger: logger}
	shed := rootfs_provider.Wire(
		logger,
		runner,
		f.config.Graph.Dir,
		f.config.Containers.DefaultRootFS,
		f.config.Docker.Registry,
		f.config.Docker.InsecureRegistries,
		f.config.Graph.PersistentImages,
		f.config.Graph.CleanupThresholdInMegabytes,
		f.uidMappings,
		f.gidMappings,
	)
	return gardener.NewVolumeProvider(shed, shed, gardener.CommandFactory(preparerootfs.Command), f.commandRunner, f.uidMappings.Map(0), f.gidMappings.Map(0))
}

func wireEnvFunc() runrunc.EnvFunc {
	return runrunc.EnvFunc(runrunc.UnixEnvFor)
}

func (f *LinuxFactory) WireMkdirer() runrunc.Mkdirer {
	if runningAsRoot() {
		return bundlerules.MkdirChowner{Command: preparerootfs.Command, CommandRunner: f.commandRunner}
	}

	return NoopMkdirer{}
}

type NoopMkdirer struct{}

func (NoopMkdirer) MkdirAs(rootFSPathFile string, uid, gid int, mode os.FileMode, recreate bool, path ...string) error {
	return nil
}

func (f *LinuxFactory) WireExecRunner(runMode, runcRoot string) runrunc.ExecRunner {
	return dadoo.NewExecRunner(
		f.config.Bin.Dadoo.Path(),
		f.config.Runtime.Plugin,
		runcRoot,
		f.signallerFactory,
		f.commandRunner,
		f.config.Containers.CleanupProcessDirsOnWait,
		runMode,
	)
}

func (f *LinuxFactory) WireCgroupsStarter(logger lager.Logger) gardener.Starter {
	return createCgroupsStarter(logger, f.config.Server.Tag, "", &cgroups.OSChowner{}, rundmc.IsMountPoint)
}

func (cmd *SetupCommand) WireCgroupsStarter(logger lager.Logger) gardener.Starter {
	return createCgroupsStarter(logger, cmd.Tag, cmd.CgroupRoot, &cgroups.OSChowner{UID: cmd.RootlessUID, GID: cmd.RootlessGID}, rundmc.IsMountPoint)
}

func createCgroupsStarter(logger lager.Logger, tag, cgroupRoot string, chowner cgroups.Chowner, mountPointChecker rundmc.MountPointChecker) gardener.Starter {
	cgroupsMountpoint := cgroups.CgroupRoot
	gardenCgroup := cgroups.GardenCgroup
	if tag != "" {
		gardenCgroup = fmt.Sprintf("%s-%s", gardenCgroup, tag)
	}
	if cgroupRoot != "" {
		cgroupsMountpoint = cgroupRoot
	}

	return cgroups.NewStarter(logger, mustOpen("/proc/cgroups"), mustOpen("/proc/self/cgroup"),
		cgroupsMountpoint, gardenCgroup, allowedDevices, linux_command_runner.New(), chowner, mountPointChecker)
}

func (f *LinuxFactory) WireResolvConfigurer() kawasaki.DnsResolvConfigurer {
	return &kawasaki.ResolvConfigurer{
		HostsFileCompiler: &dns.HostsFileCompiler{},
		ResolvCompiler:    &dns.ResolvCompiler{},
		ResolvFilePath:    "/etc/resolv.conf",
		DepotDir:          f.config.Containers.Dir,
	}
}

func (f *LinuxFactory) WireRootfsFileCreator() rundmc.RootfsFileCreator {
	return preparerootfs.SymlinkRefusingFileCreator{}
}

func initBindMountAndPath(initPathOnHost string) (specs.Mount, string) {
	initPathInContainer := filepath.Join("/tmp", "garden-init")
	return specs.Mount{
		Type:        "bind",
		Source:      initPathOnHost,
		Destination: initPathInContainer,
		Options:     []string{"bind"},
	}, initPathInContainer
}

func defaultBindMounts() []specs.Mount {
	devptsGid := 0
	if runningAsRoot() {
		devptsGid = 5
	}

	return []specs.Mount{
		{Type: "sysfs", Source: "sysfs", Destination: "/sys", Options: []string{"nosuid", "noexec", "nodev", "ro"}},
		{Type: "tmpfs", Source: "tmpfs", Destination: "/dev/shm", Options: []string{"rw", "nodev", "relatime"}},
		{Type: "devpts", Source: "devpts", Destination: "/dev/pts",
			Options: []string{"nosuid", "noexec", "newinstance", fmt.Sprintf("gid=%d", devptsGid), "ptmxmode=0666", "mode=0620"}},
	}
}

func privilegedMounts() []specs.Mount {
	return []specs.Mount{
		{Type: "proc", Source: "proc", Destination: "/proc", Options: []string{"nosuid", "noexec", "nodev"}},
	}
}

func unprivilegedMounts() []specs.Mount {
	return []specs.Mount{
		{Type: "proc", Source: "proc", Destination: "/proc", Options: []string{"nosuid", "noexec", "nodev"}},
		{Type: "cgroup", Source: "cgroup", Destination: "/sys/fs/cgroup", Options: []string{"ro", "nosuid", "noexec", "nodev"}},
	}
}

func getPrivilegedDevices() []specs.LinuxDevice {
	return []specs.LinuxDevice{fuseDevice}
}

func bindMountPoints() []string {
	return []string{"/etc/hosts", "/etc/resolv.conf"}
}

func mustGetMaxValidUID() int {
	return idmapper.MustGetMaxValidUID()
}

func ensureServerSocketDoesNotLeak(socketFD uintptr) error {
	_, _, errNo := unix.Syscall(unix.SYS_FCNTL, socketFD, unix.F_SETFD, unix.FD_CLOEXEC)
	if errNo != 0 {
		return fmt.Errorf("setting cloexec on server socket: %s", errNo)
	}
	return nil
}

func wireMounts() bundlerules.Mounts {
	return bundlerules.Mounts{
		MountOptionsGetter: rundmc.GetMountOptions,
		MountInfosProvider: func() ([]*mount.Info, error) {
			return mount.GetMounts()
		},
	}
}

func wireContainerd(socket string, bndlLoader *goci.BndlLoader, wireExecer func(pidGetter runrunc.PidGetter) *runrunc.Execer) (rundmc.OCIRuntime, error) {
	containerdClient, err := containerd.New(socket)
	if err != nil {
		return nil, err
	}
	ctx := namespaces.WithNamespace(context.Background(), containerdNamespace)
	nerd := nerd.New(containerdClient, ctx)
	return runcontainerd.New(nerd, bndlLoader, wireExecer(&runcontainerd.PidGetter{Nerd: nerd})), nil
}

func containerdRuncRoot() string {
	return proc.RuncRoot
}
<|endoftext|>


<commit_before>// Copyright 2016-2017 Olivier Mengué. All rights reserved.
// Use of this source code is governed by the Apache 2.0 license that
// can be found in the LICENSE file.

package jsonptr

import (
	"bytes"
	"encoding/json"
	"strconv"
	"strings"
)

// Pointer represents a mutable parsed JSON Pointer.
//
// The Go representation is a array of non-encoded path elements. This
// allows to use type conversion from/to a []string.
type Pointer []string

// Parse parses a JSON pointer from its text representation.
func Parse(pointer string) (Pointer, error) {
	if pointer == "" {
		return nil, nil
	}
	if pointer[0] != '/' {
		return nil, ErrSyntax
	}
	ptr := strings.Split(pointer[1:], "/")
	// Optimize for the common case
	if strings.IndexByte(pointer, '~') == -1 {
		return ptr, nil
	}
	for i, part := range ptr {
		var err error
		if ptr[i], err = UnescapeString(part); err != nil {
			// TODO return the full prefix
			return nil, err
		}
	}
	return ptr, nil
}

// MarshalText() implements encoding.TextUnmarshaler
func (ptr *Pointer) UnmarshalText(text []byte) error {
	if len(text) == 0 {
		*ptr = nil
		return nil
	}
	if text[0] != '/' {
		return ErrSyntax
	}

	var p Pointer
	t := text[1:]
	for {
		i := bytes.IndexByte(t, '/')
		if i < 0 {
			break
		}
		part, err := Unescape(t[:i])
		if err != nil {
			return syntaxError(string(text[:len(text)-len(t)-i-1]))
		}
		p = append(p, string(part))
		t = t[i+1:]
	}
	part, err := Unescape(t)
	if err != nil {
		return syntaxError(string(text))
	}
	*ptr = append(p, string(part))
	return nil
}

// String returns a JSON Pointer string, escaping components when necessary:
// '~' is replaced by "~0", '/' by "~1"
func (ptr Pointer) String() string {
	if len(ptr) == 0 {
		return ""
	}

	dst := make([]byte, 0, 8*len(ptr))
	for _, part := range ptr {
		dst = AppendEscape(append(dst, '/'), part)
	}
	return string(dst)
}

// MarshalText() implements encoding.TextMarshaler
func (ptr Pointer) MarshalText() (text []byte, err error) {
	if len(ptr) == 0 {
		return nil, nil
	}
	dst := make([]byte, 0, 8*len(ptr))
	for _, part := range ptr {
		dst = AppendEscape(append(dst, '/'), part)
	}
	return dst, nil
}

// Copy returns a new, independant, copy of the pointer.
func (ptr Pointer) Copy() Pointer {
	return append(Pointer{}, ptr...)
}

// IsRoot returns true if the pointer is at root (empty).
func (ptr Pointer) IsRoot() bool {
	return len(ptr) == 0
}

// Up removes the last element of the pointer.
// The pointer is returned for chaining.
//
// Panics if already at root.
func (ptr *Pointer) Up() *Pointer {
	if ptr.IsRoot() {
		panic(ErrRoot)
	}
	*ptr = (*ptr)[:len(*ptr)-1]
	return ptr
}

// Pop removes the last element of the pointer and returns it.
//
// Panics if already at root.
func (ptr *Pointer) Pop() string {
	last := len(*ptr) - 1
	if last < 0 {
		panic(ErrRoot)
	}
	prop := (*ptr)[last]
	*ptr = (*ptr)[:last]
	return prop
}

// Property moves the pointer deeper, following a property name.
// The pointer is returned for chaining.
func (ptr *Pointer) Property(name string) *Pointer {
	*ptr = append(*ptr, name)
	return ptr
}

// Index moves the pointer deeper, following an array index.
// The pointer is returned for chaining.
func (ptr *Pointer) Index(index int) *Pointer {
	var prop string
	if index < 0 {
		prop = "-"
	} else {
		prop = strconv.Itoa(index)
	}
	return ptr.Property(prop)
}

// LeafName returns the name of the last part of the pointer.
func (ptr Pointer) LeafName() string {
	return ptr[len(ptr)-1]
}

// LeafIndex returns the last part of the pointer as an array index.
// -1 is returned for "-".
func (ptr Pointer) LeafIndex() (int, error) {
	return arrayIndex(ptr.LeafName())
}

// In returns the value from doc pointed by ptr.
//
// doc may be a deserialized document, or a json.RawMessage.
func (ptr Pointer) In(doc interface{}) (interface{}, error) {
	for i, key := range ptr {
		switch here := (doc).(type) {
		case map[string]interface{}:
			var ok bool
			if doc, ok = here[key]; !ok {
				return nil, propertyError(ptr[:i].String())
			}
		case []interface{}:
			n, err := arrayIndex(key)
			if err != nil {
				return nil, &PtrError{ptr[:i].String(), err}
			}
			if n < 0 || n >= len(here) {
				return nil, indexError(ptr[:i].String())
			}
			doc = here[n]
		case JSONDecoder:
			v, err := getJSON(here, ptr[i:].String())
			if err != nil {
				err.rebase(ptr[:i].String())
			}
			return v, err
		case json.RawMessage:
			v, err := getRaw(here, ptr[i:].String())
			if err != nil {
				err.rebase(ptr[:i].String())
			}
			return v, err
		default:
			// We report the error at the upper level
			return nil, docError(ptr[:i-1].String(), doc)
		}
	}

	doc, err := getLeaf(doc)
	if err != nil {
		err.rebase(ptr.String())
	}
	return doc, err
}

// Set changes a value in document pdoc at location pointed by ptr.
func (ptr Pointer) Set(pdoc *interface{}, value interface{}) error {
	// TODO Make an optimised implementation
	return Set(pdoc, ptr.String(), value)
}
<commit_msg>Pointer.Copy: start cloning with nil instead of a zero length array<commit_after>// Copyright 2016-2017 Olivier Mengué. All rights reserved.
// Use of this source code is governed by the Apache 2.0 license that
// can be found in the LICENSE file.

package jsonptr

import (
	"bytes"
	"encoding/json"
	"strconv"
	"strings"
)

// Pointer represents a mutable parsed JSON Pointer.
//
// The Go representation is a array of non-encoded path elements. This
// allows to use type conversion from/to a []string.
type Pointer []string

// Parse parses a JSON pointer from its text representation.
func Parse(pointer string) (Pointer, error) {
	if pointer == "" {
		return nil, nil
	}
	if pointer[0] != '/' {
		return nil, ErrSyntax
	}
	ptr := strings.Split(pointer[1:], "/")
	// Optimize for the common case
	if strings.IndexByte(pointer, '~') == -1 {
		return ptr, nil
	}
	for i, part := range ptr {
		var err error
		if ptr[i], err = UnescapeString(part); err != nil {
			// TODO return the full prefix
			return nil, err
		}
	}
	return ptr, nil
}

// MarshalText() implements encoding.TextUnmarshaler
func (ptr *Pointer) UnmarshalText(text []byte) error {
	if len(text) == 0 {
		*ptr = nil
		return nil
	}
	if text[0] != '/' {
		return ErrSyntax
	}

	var p Pointer
	t := text[1:]
	for {
		i := bytes.IndexByte(t, '/')
		if i < 0 {
			break
		}
		part, err := Unescape(t[:i])
		if err != nil {
			return syntaxError(string(text[:len(text)-len(t)-i-1]))
		}
		p = append(p, string(part))
		t = t[i+1:]
	}
	part, err := Unescape(t)
	if err != nil {
		return syntaxError(string(text))
	}
	*ptr = append(p, string(part))
	return nil
}

// String returns a JSON Pointer string, escaping components when necessary:
// '~' is replaced by "~0", '/' by "~1"
func (ptr Pointer) String() string {
	if len(ptr) == 0 {
		return ""
	}

	dst := make([]byte, 0, 8*len(ptr))
	for _, part := range ptr {
		dst = AppendEscape(append(dst, '/'), part)
	}
	return string(dst)
}

// MarshalText() implements encoding.TextMarshaler
func (ptr Pointer) MarshalText() (text []byte, err error) {
	if len(ptr) == 0 {
		return nil, nil
	}
	dst := make([]byte, 0, 8*len(ptr))
	for _, part := range ptr {
		dst = AppendEscape(append(dst, '/'), part)
	}
	return dst, nil
}

// Copy returns a new, independant, copy of the pointer.
func (ptr Pointer) Copy() Pointer {
	return append(Pointer(nil), ptr...)
}

// IsRoot returns true if the pointer is at root (empty).
func (ptr Pointer) IsRoot() bool {
	return len(ptr) == 0
}

// Up removes the last element of the pointer.
// The pointer is returned for chaining.
//
// Panics if already at root.
func (ptr *Pointer) Up() *Pointer {
	if ptr.IsRoot() {
		panic(ErrRoot)
	}
	*ptr = (*ptr)[:len(*ptr)-1]
	return ptr
}

// Pop removes the last element of the pointer and returns it.
//
// Panics if already at root.
func (ptr *Pointer) Pop() string {
	last := len(*ptr) - 1
	if last < 0 {
		panic(ErrRoot)
	}
	prop := (*ptr)[last]
	*ptr = (*ptr)[:last]
	return prop
}

// Property moves the pointer deeper, following a property name.
// The pointer is returned for chaining.
func (ptr *Pointer) Property(name string) *Pointer {
	*ptr = append(*ptr, name)
	return ptr
}

// Index moves the pointer deeper, following an array index.
// The pointer is returned for chaining.
func (ptr *Pointer) Index(index int) *Pointer {
	var prop string
	if index < 0 {
		prop = "-"
	} else {
		prop = strconv.Itoa(index)
	}
	return ptr.Property(prop)
}

// LeafName returns the name of the last part of the pointer.
func (ptr Pointer) LeafName() string {
	return ptr[len(ptr)-1]
}

// LeafIndex returns the last part of the pointer as an array index.
// -1 is returned for "-".
func (ptr Pointer) LeafIndex() (int, error) {
	return arrayIndex(ptr.LeafName())
}

// In returns the value from doc pointed by ptr.
//
// doc may be a deserialized document, or a json.RawMessage.
func (ptr Pointer) In(doc interface{}) (interface{}, error) {
	for i, key := range ptr {
		switch here := (doc).(type) {
		case map[string]interface{}:
			var ok bool
			if doc, ok = here[key]; !ok {
				return nil, propertyError(ptr[:i].String())
			}
		case []interface{}:
			n, err := arrayIndex(key)
			if err != nil {
				return nil, &PtrError{ptr[:i].String(), err}
			}
			if n < 0 || n >= len(here) {
				return nil, indexError(ptr[:i].String())
			}
			doc = here[n]
		case JSONDecoder:
			v, err := getJSON(here, ptr[i:].String())
			if err != nil {
				err.rebase(ptr[:i].String())
			}
			return v, err
		case json.RawMessage:
			v, err := getRaw(here, ptr[i:].String())
			if err != nil {
				err.rebase(ptr[:i].String())
			}
			return v, err
		default:
			// We report the error at the upper level
			return nil, docError(ptr[:i-1].String(), doc)
		}
	}

	doc, err := getLeaf(doc)
	if err != nil {
		err.rebase(ptr.String())
	}
	return doc, err
}

// Set changes a value in document pdoc at location pointed by ptr.
func (ptr Pointer) Set(pdoc *interface{}, value interface{}) error {
	// TODO Make an optimised implementation
	return Set(pdoc, ptr.String(), value)
}
<|endoftext|>


<commit_before>package commands

import (
	"log"
	"os"
	"os/exec"
	"path"
	"syscall"

	"github.com/khlieng/name_pending/Godeps/_workspace/src/github.com/spf13/cobra"

	"github.com/khlieng/name_pending/storage"
)

var (
	editors = []string{"nano", "notepad", "vi", "emacs"}

	configCmd = &cobra.Command{
		Use:   "config",
		Short: "Edit config file",
		Run: func(cmd *cobra.Command, _ []string) {
			if editor := findEditor(); editor != "" {
				args := []string{editor, path.Join(storage.AppDir, "config.toml")}
				syscall.Exec(editor, args, os.Environ())
			} else {
				log.Println("Unable to locate editor")
			}
		},
	}
)

func findEditor() string {
	if editor := os.Getenv("EDITOR"); editor != "" {
		editor, err := exec.LookPath(editor)
		if err == nil {
			return editor
		}
	}

	for _, editor := range editors {
		editor, err := exec.LookPath(editor)
		if err == nil {
			return editor
		}
	}

	return ""
}
<commit_msg>Use os/exec to run editor<commit_after>package commands

import (
	"log"
	"os"
	"os/exec"
	"path"

	"github.com/khlieng/name_pending/Godeps/_workspace/src/github.com/spf13/cobra"

	"github.com/khlieng/name_pending/storage"
)

var (
	editors = []string{"nano", "notepad", "vi", "emacs"}

	configCmd = &cobra.Command{
		Use:   "config",
		Short: "Edit config file",
		Run: func(cmd *cobra.Command, args []string) {
			if editor := findEditor(); editor != "" {
				process := exec.Command(editor, path.Join(storage.AppDir, "config.toml"))
				process.Stdin = os.Stdin
				process.Stdout = os.Stdout
				process.Stderr = os.Stderr
				process.Run()
			} else {
				log.Println("Unable to locate editor")
			}
		},
	}
)

func findEditor() string {
	if editor := os.Getenv("EDITOR"); editor != "" {
		editor, err := exec.LookPath(editor)
		if err == nil {
			return editor
		}
	}

	for _, editor := range editors {
		editor, err := exec.LookPath(editor)
		if err == nil {
			return editor
		}
	}

	return ""
}
<|endoftext|>


<commit_before>// Copyright 2016 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package libdokan

import (
	"sync/atomic"
	"time"

	"github.com/keybase/kbfs/dokan"
	"github.com/keybase/kbfs/libkbfs"
	"golang.org/x/net/context"
)

// FSO is a common type for file system objects, i.e. Dirs or Files.
type FSO struct {
	refcount refcount
	name     string
	folder   *Folder
	node     libkbfs.Node
	parent   libkbfs.Node
	emptyFile
}

// SetFileTime sets mtime for FSOs (File and Dir). TLFs have a separate SetFileTime.
func (f *FSO) SetFileTime(ctx context.Context, fi *dokan.FileInfo, creation time.Time, lastAccess time.Time, lastWrite time.Time) (err error) {
	f.folder.fs.logEnter(ctx, "FSO SetFileTime")
	defer func() { f.folder.reportErr(ctx, libkbfs.WriteMode, err) }()
	f.folder.fs.log.CDebugf(ctx, "FSO SetFileTime %v %v %v", creation, lastAccess, lastWrite)

	if !lastWrite.IsZero() {
		return f.folder.fs.config.KBFSOps().SetMtime(ctx, f.node, &lastWrite)
	}

	return dokan.ErrNotSupported
}

type refcount struct {
	x int32
}

func (rc *refcount) Increase() {
	atomic.AddInt32(&rc.x, 1)
}

func (rc *refcount) Decrease() bool {
	return atomic.AddInt32(&rc.x, -1) == 0
}
<commit_msg>libdokan: Not setting any times in SetFileTime is valid<commit_after>// Copyright 2016 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package libdokan

import (
	"sync/atomic"
	"time"

	"github.com/keybase/kbfs/dokan"
	"github.com/keybase/kbfs/libkbfs"
	"golang.org/x/net/context"
)

// FSO is a common type for file system objects, i.e. Dirs or Files.
type FSO struct {
	refcount refcount
	name     string
	folder   *Folder
	node     libkbfs.Node
	parent   libkbfs.Node
	emptyFile
}

// SetFileTime sets mtime for FSOs (File and Dir). TLFs have a separate SetFileTime.
func (f *FSO) SetFileTime(ctx context.Context, fi *dokan.FileInfo, creation time.Time, lastAccess time.Time, lastWrite time.Time) (err error) {
	f.folder.fs.logEnter(ctx, "FSO SetFileTime")
	defer func() { f.folder.reportErr(ctx, libkbfs.WriteMode, err) }()
	f.folder.fs.log.CDebugf(ctx, "FSO SetFileTime %v %v %v", creation, lastAccess, lastWrite)

	if !lastWrite.IsZero() {
		return f.folder.fs.config.KBFSOps().SetMtime(ctx, f.node, &lastWrite)
	}

	return nil
}

type refcount struct {
	x int32
}

func (rc *refcount) Increase() {
	atomic.AddInt32(&rc.x, 1)
}

func (rc *refcount) Decrease() bool {
	return atomic.AddInt32(&rc.x, -1) == 0
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"
	"os"

	"github.com/docker/machine/libmachine"
	"github.com/docker/machine/libmachine/check"
	"github.com/docker/machine/libmachine/log"
)

func cmdConfig(c CommandLine, api libmachine.API) error {
	// Ensure that log messages always go to stderr when this command is
	// being run (it is intended to be run in a subshell)
	log.SetOutWriter(os.Stderr)

	if len(c.Args()) != 1 {
		return ErrExpectedOneMachine
	}

	host, err := api.Load(c.Args().First())
	if err != nil {
		return err
	}

	dockerHost, authOptions, err := check.DefaultConnChecker.Check(host, c.Bool("swarm"))
	if err != nil {
		return fmt.Errorf("Error running connection boilerplate: %s", err)
	}

	log.Debug(dockerHost)

	fmt.Printf("--tlsverify --tlscacert=%q --tlscert=%q --tlskey=%q -H=%s",
		authOptions.CaCertPath, authOptions.ClientCertPath, authOptions.ClientKeyPath, dockerHost)

	return nil
}
<commit_msg>Fix for #2415<commit_after>package commands

import (
	"fmt"
	"os"

	"github.com/docker/machine/libmachine"
	"github.com/docker/machine/libmachine/check"
	"github.com/docker/machine/libmachine/log"
)

func cmdConfig(c CommandLine, api libmachine.API) error {
	// Ensure that log messages always go to stderr when this command is
	// being run (it is intended to be run in a subshell)
	log.SetOutWriter(os.Stderr)

	if len(c.Args()) != 1 {
		return ErrExpectedOneMachine
	}

	host, err := api.Load(c.Args().First())
	if err != nil {
		return err
	}

	dockerHost, authOptions, err := check.DefaultConnChecker.Check(host, c.Bool("swarm"))
	if err != nil {
		return fmt.Errorf("Error running connection boilerplate: %s", err)
	}

	log.Debug(dockerHost)

	fmt.Printf("--tlsverify\n--tlscacert=%q\n--tlscert=%q\n--tlskey=%q\n-H=%s\n",
		authOptions.CaCertPath, authOptions.ClientCertPath, authOptions.ClientKeyPath, dockerHost)

	return nil
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"
	"github.com/codegangsta/cli"
	"github.com/brooklyncentral/brooklyn-cli/api/application"
	"github.com/brooklyncentral/brooklyn-cli/command_metadata"
	"github.com/brooklyncentral/brooklyn-cli/net"
	"github.com/brooklyncentral/brooklyn-cli/scope"
)

type Deploy struct {
	network *net.Network
}

func NewDeploy(network *net.Network) (cmd *Deploy) {
	cmd = new(Deploy)
	cmd.network = network
	return
}

func (cmd *Deploy) Metadata() command_metadata.CommandMetadata {
	return command_metadata.CommandMetadata{
		Name:        "deploy",
		Description: "Create a new brooklyn application from the supplied YAML",
		Usage:       "BROOKLYN_NAME [ SCOPE ] deploy FILEPATH",
		Flags:       []cli.Flag{},
	}
}

func (cmd *Deploy) Run(scope scope.Scope, c *cli.Context) {
	create := application.Create(cmd.network, c.Args().First())
	fmt.Println(create)
}
<commit_msg>Fix description in help text<commit_after>package commands

import (
	"fmt"
	"github.com/codegangsta/cli"
	"github.com/brooklyncentral/brooklyn-cli/api/application"
	"github.com/brooklyncentral/brooklyn-cli/command_metadata"
	"github.com/brooklyncentral/brooklyn-cli/net"
	"github.com/brooklyncentral/brooklyn-cli/scope"
)

type Deploy struct {
	network *net.Network
}

func NewDeploy(network *net.Network) (cmd *Deploy) {
	cmd = new(Deploy)
	cmd.network = network
	return
}

func (cmd *Deploy) Metadata() command_metadata.CommandMetadata {
	return command_metadata.CommandMetadata{
		Name:        "deploy",
		Description: "Deploy a new brooklyn application from the supplied YAML",
		Usage:       "BROOKLYN_NAME [ SCOPE ] deploy FILEPATH",
		Flags:       []cli.Flag{},
	}
}

func (cmd *Deploy) Run(scope scope.Scope, c *cli.Context) {
	create := application.Create(cmd.network, c.Args().First())
	fmt.Println(create)
}
<|endoftext|>


<commit_before>// problem 16.3

package chapter16

func dfsHelper(adj [][]int) ([]bool, func(int) bool) {
	color := make([]bool, len(adj))

	state := make([]int, len(adj))

	var dfs func(int) bool
	dfs = func(pin int) bool {
		state[pin] = DISCOVERED
		for _, v := range adj[pin] {
			if state[v] == UNDISCOVERED {
				color[v] = !color[pin]
				if !dfs(v) {
					return false
				}
			} else if color[v] == color[pin] {
				return false
			}
		}
		return true
	}

	return color, dfs
}

func WiringCircuitBoard(board [][]int) bool {
	_, dfsFn := dfsHelper(board)
	return dfsFn(0)
}
<commit_msg>Refactor 16.3 - wiringcircuitboard.go<commit_after>// Problem 16.3

package chapter16

func WiringCircuitBoard(board [][]int) bool {
	dist := make([]int, len(board))

	for i := range dist {
		dist[i] = -1
	}

	bfs := func(v int) bool {
		q := []int{v}

		for len(q) > 0 {
			curr := q[0]

			for _, e := range board[curr] {
				if dist[e] == -1 {
					dist[e] = dist[curr] + 1
					q = append(q, e)
				} else if dist[e] == dist[curr] {
					return false
				}
			}

			q = q[1:]
		}

		return true
	}

	for v := range board {
		if dist[v] == -1 {
			dist[v] = 0
			if bfs(v) == false {
				return false
			}
		}
	}

	return true
}
<|endoftext|>


<commit_before>package x86

import (
	uc "github.com/lunixbochs/unicorn"

	"../../models"
	"../../syscalls"
)

var darwinSyscalls = map[int]string{
	1:   "exit",
	2:   "fork",
	3:   "read",
	4:   "write",
	5:   "open",
	6:   "close",
	7:   "wait4",
	9:   "link",
	10:  "unlink",
	73:  "munmap",
	197: "mmap",
	199: "lseek",
}

func DarwinSyscall(u models.Usercorn) {
	eax, _ := u.RegRead(uc.UC_X86_REG_EAX)
	getArgs := syscalls.StackArgs(u)
	ret, _ := u.Syscall(linuxSyscalls, int(eax), getArgs)
	u.RegWrite(uc.UC_X86_REG_EAX, ret)
}

func DarwinInterrupt(u models.Usercorn, intno uint32) {
	if intno == 0x80 {
		DarwinSyscall(u)
	}
}

func init() {
	Arch.RegisterOS(&models.OS{Name: "darwin", Syscall: DarwinSyscall, Interrupt: DarwinInterrupt})
}
<commit_msg>save and restore ESP for x86 darwin syscall<commit_after>package x86

import (
	uc "github.com/lunixbochs/unicorn"

	"../../models"
	"../../syscalls"
)

var darwinSyscalls = map[int]string{
	1:   "exit",
	2:   "fork",
	3:   "read",
	4:   "write",
	5:   "open",
	6:   "close",
	7:   "wait4",
	9:   "link",
	10:  "unlink",
	73:  "munmap",
	197: "mmap",
	199: "lseek",
}

func DarwinSyscall(u models.Usercorn) {
	esp, _ := u.RegRead(uc.UC_X86_REG_ESP)
	eax, _ := u.RegRead(uc.UC_X86_REG_EAX)
	getArgs := syscalls.StackArgs(u)
	ret, _ := u.Syscall(linuxSyscalls, int(eax), getArgs)
	u.RegWrite(uc.UC_X86_REG_EAX, ret)
	u.RegWrite(uc.UC_X86_REG_ESP, esp)
}

func DarwinInterrupt(u models.Usercorn, intno uint32) {
	if intno == 0x80 {
		DarwinSyscall(u)
	}
}

func init() {
	Arch.RegisterOS(&models.OS{Name: "darwin", Syscall: DarwinSyscall, Interrupt: DarwinInterrupt})
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// kube2consul is a bridge between Kubernetes and Consul.  It watches the
// Kubernetes master for changes in Services and creates new DNS records on the
// consul agent.

package main // import "github.com/jmccarty3/kube2consul"

import (
	"flag"
	"fmt"
	"strings"
	"net/url"
	"os"
	"time"
	"reflect"

	kapi "k8s.io/kubernetes/pkg/api"
	kclient "k8s.io/kubernetes/pkg/client/unversioned"
	kcache "k8s.io/kubernetes/pkg/client/cache"
	kclientcmd "k8s.io/kubernetes/pkg/client/unversioned/clientcmd"
	kframework "k8s.io/kubernetes/pkg/controller/framework"
	kcontrollerFramework "k8s.io/kubernetes/pkg/controller/framework"
	kSelector "k8s.io/kubernetes/pkg/fields"
	"k8s.io/kubernetes/pkg/util"
	"github.com/golang/glog"
	consulapi "github.com/hashicorp/consul/api"

  "k8s.io/kubernetes/pkg/api"
  "k8s.io/kubernetes/pkg/fields"
  "k8s.io/kubernetes/pkg/labels"
)

var (
	argConsulAgent         = flag.String("consul-agent", "http://127.0.0.1:8500", "URL to consul agent")
	argKubecfgFile         = flag.String("kubecfg_file", "", "Location of kubecfg file for access to kubernetes service")
	argKubeMasterUrl       = flag.String("kube_master_url", "https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}", "Url to reach kubernetes master. Env variables in this flag will be expanded.")
)

const (
	// Maximum number of attempts to connect to consul server.
	maxConnectAttempts = 12
	// Resync period for the kube controller loop.
	resyncPeriod = 5 * time.Second
)

type nodeInformation struct {
	name string

	address string

	ready bool
	// map[service] DNS IDs
	ids map[string][]string
}

type kube2consul struct {
	// Consul client.
	consulClient *consulapi.Client

	// DNS domain name.
	domain string

	//Nodes Name / valid
	nodes map[string]nodeInformation

	//All Services.
	services map[string]*kapi.Service
}

func Newkube2consul() *kube2consul {
	var k kube2consul
	k.nodes = make(map[string]nodeInformation)
	k.services = make(map[string]*kapi.Service)

	return &k
}

func NewnodeInformation() *nodeInformation {
	var n nodeInformation
	n.ready = false
	n.ids = make(map[string][]string)

	return &n
}

func Contains(s []string, e string) bool {
	for _,i := range s {
		if i == e {
			return true
		}
	}
	return false
}

func (ks *kube2consul) removeDNS(recordID string) error {
	glog.Infof("Removing %s from DNS", recordID)
	ks.consulClient.Agent().ServiceDeregister(recordID)
	return nil
}

func (ks *kube2consul) createDNS(record string, service *kapi.Service, node *nodeInformation) error {
	if strings.Contains(record, ".") {
		glog.Infof("Service names containing '.' are not supported: %s\n", service.Name)
		return nil
	}

	// if ClusterIP is not set, do not create a DNS records
	if !kapi.IsServiceIPSet(service) {
		glog.Infof("Skipping dns record for headless service: %s\n", service.Name)
		return nil
	}

	for i := range service.Spec.Ports {
			newId := node.name+record + service.Spec.Ports[i].Name
      var asrName string

			if len(service.Spec.Ports[i].Name) > 0 {
				asrName = record + "-" + service.Spec.Ports[i].Name
			} else {
				asrName = record
			}

			asr := &consulapi.AgentServiceRegistration{
				ID:			 newId,
				Name: 	 asrName,
				Address: node.address,
				Port:    service.Spec.Ports[i].NodePort,
				Tags: []string{"Kube"},
			}

			if Contains(node.ids[record], newId) == false {
				glog.Infof("Setting DNS record: %v -> %v:%d\n", asr.Name, asr.Address, asr.Port)
				if err := ks.consulClient.Agent().ServiceRegister(asr); err != nil {
					return err
				}

				node.ids[record] = append(node.ids[record], newId)
		}
	}
	return nil
}

func newConsulClient(consulAgent string) (*consulapi.Client, error) {
	var (
		client *consulapi.Client
		err    error
	)

	consulConfig := consulapi.DefaultConfig()
	consulAgentUrl, err := url.Parse(consulAgent)
	if err != nil {
			glog.Infof("Error parsing Consul url")
			return nil, err
	}

	if consulAgentUrl.Host != "" {
	  consulConfig.Address = consulAgentUrl.Host
	}

	if consulAgentUrl.Scheme != "" {
		consulConfig.Scheme = consulAgentUrl.Scheme
	}

	client, err = consulapi.NewClient(consulConfig)
	if err != nil {
			glog.Infof("Error creating Consul client")
			return nil, err
	}

	for attempt := 1; attempt <= maxConnectAttempts; attempt++ {
		if _, err = client.Agent().Self(); err == nil {
			break
		}

		if attempt == maxConnectAttempts {
			break
		}

		glog.Infof("[Attempt: %d] Attempting access to Consul after 5 second sleep", attempt)
		time.Sleep(5 * time.Second)
	}

	if err != nil {
		return nil, fmt.Errorf("failed to connect to Consul agent: %v, error: %v", consulAgent, err)
	}
	glog.Infof("Consul agent found: %v", consulAgent)

	return client, nil
}

func getKubeMasterUrl() (string, error) {
	if *argKubeMasterUrl == "" {
		return "", fmt.Errorf("no --kube_master_url specified")
	}
	parsedUrl, err := url.Parse(os.ExpandEnv(*argKubeMasterUrl))
	if err != nil {
		return "", fmt.Errorf("failed to parse --kube_master_url %s - %v", *argKubeMasterUrl, err)
	}
	if parsedUrl.Scheme == "" || parsedUrl.Host == "" || parsedUrl.Host == ":" {
		return "", fmt.Errorf("invalid --kube_master_url specified %s", *argKubeMasterUrl)
	}
	return parsedUrl.String(), nil
}

// TODO: evaluate using pkg/client/clientcmd
func newKubeClient() (*kclient.Client, error) {
	var config *kclient.Config
	masterUrl, err := getKubeMasterUrl()
	if err != nil {
		return nil, err
	}
	if *argKubecfgFile == "" {
		config = &kclient.Config{
			Host:    masterUrl,
			Version: "v1",
		}
	} else {
		var err error
		overrides := &kclientcmd.ConfigOverrides{}
		overrides.ClusterInfo.Server = masterUrl                                     // might be "", but that is OK
		rules := &kclientcmd.ClientConfigLoadingRules{ExplicitPath: *argKubecfgFile} // might be "", but that is OK
		if config, err = kclientcmd.NewNonInteractiveDeferredLoadingClientConfig(rules, overrides).ClientConfig(); err != nil {
			return nil, err
		}
	}
	glog.Infof("Using %s for kubernetes master", config.Host)
	glog.Infof("Using kubernetes API %s", config.Version)
	return kclient.New(config)
}

func buildNameString(service, namespace string) string {
	//glog.Infof("Name String: %s  %s", service, namespace)
	//return fmt.Sprintf("%s.%s", service, namespace)
	return fmt.Sprintf("%s", service)
}

// Returns a cache.ListWatch that gets all changes to services.
func createServiceLW(kubeClient *kclient.Client) *kcache.ListWatch {
	return kcache.NewListWatchFromClient(kubeClient, "services", kapi.NamespaceAll, kSelector.Everything())
}

// Returns a cache.ListWatch that gets all changes to services.
func createNodeLW(kubeClient *kclient.Client) *kcache.ListWatch {
	return kcache.NewListWatchFromClient(kubeClient, "nodes", kapi.NamespaceAll, kSelector.Everything())
}

func (ks *kube2consul) newService(obj interface{}) {
	if s, ok := obj.(*kapi.Service); ok {
		name := buildNameString(s.Name, s.Namespace)
		ks.services[name] = s

		glog.V(2).Info("Creating Service: ", name)
		//Add to all existing nodes
		for _,node := range ks.nodes {
			if node.ready {
				ks.createDNS(name, s, &node)
			}
		}
	}
}

func (ks *kube2consul) removeService(obj interface{}) {
	if s, ok := obj.(*kapi.Service); ok {
		name := buildNameString(s.Name, s.Namespace)
		//Remove service from node
		for _,node := range ks.nodes {
			for _,id := range node.ids[name] {
				ks.removeDNS(id)
			}
			delete(node.ids, name)
		}
		delete(ks.services,name)
	}
}

func (ks *kube2consul) updateService(oldObj, newObj interface{}) {
	if old, ok := oldObj.(*kapi.Service); ok {
		if new, ok := newObj.(*kapi.Service); ok {
			if reflect.DeepEqual(*old, *new) == false {
				ks.removeService(old)
				ks.newService(new)
			}
		}
	}
}

func (ks *kube2consul) updateNode(oldObj, newObj interface{}) {
	if n, ok := newObj.(*kapi.Node); ok {
		ready := n.Status.Conditions[0].Status == kapi.ConditionTrue
		nodeInfo := ks.nodes[n.Name]

		if nodeInfo.ready != ready {
			nodeInfo.ready = ready
			if ready {
				for serviceName, service := range ks.services {
					ks.createDNS(serviceName, service, &nodeInfo)
				}
			}	else {
				for _,serviceIDs := range nodeInfo.ids {
					for _,serviceID := range serviceIDs {
						ks.removeDNS(serviceID)
					}
				}
				nodeInfo.ids = make(map[string][]string) //Clear tha map
			}
		}
	}
}

func (ks *kube2consul) newNode(newObj interface{}) {
	if node, ok := newObj.(*kapi.Node); ok {
		if _, ok := ks.nodes[node.Name]; !ok {
			glog.Info("Adding Node: ", node.Name)
			var newNode = *NewnodeInformation()
			newNode.name = node.Name
			newNode.address = node.Status.Addresses[0].Address

			ks.nodes[node.Name] = newNode
		}
	}
}

func (ks *kube2consul) removeNode(oldObj interface{}) {
	if node, ok := oldObj.(*kapi.Node); ok {
		if info, ok := ks.nodes[node.Name]; ok {
			glog.Info("Removing Node: ", node.Name)
			for _,idSet := range info.ids {
				for _,id := range idSet {
					ks.removeDNS(id)
				}
			}

			delete(ks.nodes, node.Name)
		}
	}
}

func watchForServices(kubeClient *kclient.Client, ks *kube2consul) {
	var serviceController *kcontrollerFramework.Controller
	_, serviceController = kframework.NewInformer(
		createServiceLW(kubeClient),
		&kapi.Service{},
		resyncPeriod,
		kframework.ResourceEventHandlerFuncs{
			AddFunc:    ks.newService,
			DeleteFunc: ks.removeService,
			UpdateFunc: ks.updateService,
		},
	)
	go serviceController.Run(util.NeverStop)
}

func watchForNodes(kubeClient *kclient.Client, ks *kube2consul) kcache.Store {
	store, serviceController := kframework.NewInformer(
		createNodeLW(kubeClient),
		&kapi.Node{},
		resyncPeriod,
		kframework.ResourceEventHandlerFuncs{
			AddFunc: ks.newNode,
			DeleteFunc: ks.removeNode,
			UpdateFunc: ks.updateNode,
		},
	)
	glog.Info("About to call run!")
	go serviceController.Run(util.NeverStop)
	return store
}

func main() {
	flag.Parse()
	var err error
	// TODO: Validate input flags.
	ks := Newkube2consul()

	if ks.consulClient, err = newConsulClient(*argConsulAgent); err != nil {
		glog.Fatalf("Failed to create Consul client - %v", err)
	}

	kubeClient, err := newKubeClient()
	if err != nil {
		glog.Fatalf("Failed to create a kubernetes client: %v", err)
	}

	glog.Info(kubeClient.ServerVersion())
	glog.Info(kubeClient.Services(kapi.NamespaceAll).Get("sensu-core"))

	pods, err := kubeClient.Pods(api.NamespaceDefault).List(labels.Everything(), fields.Everything())
	if err != nil {
	  for pod := range pods.Items {
			glog.Info(pod)
		}
	}

	watchForServices(kubeClient, ks)
	watchForNodes(kubeClient, ks)
	glog.Info("Watchers running")
	select {}
}
<commit_msg>use CAP-ALL repo<commit_after>/*
Copyright 2014 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// kube2consul is a bridge between Kubernetes and Consul.  It watches the
// Kubernetes master for changes in Services and creates new DNS records on the
// consul agent.

package main // import "github.com/CAP-ALL/kube2consul"

import (
	"flag"
	"fmt"
	"strings"
	"net/url"
	"os"
	"time"
	"reflect"

	kapi "k8s.io/kubernetes/pkg/api"
	kclient "k8s.io/kubernetes/pkg/client/unversioned"
	kcache "k8s.io/kubernetes/pkg/client/cache"
	kclientcmd "k8s.io/kubernetes/pkg/client/unversioned/clientcmd"
	kframework "k8s.io/kubernetes/pkg/controller/framework"
	kcontrollerFramework "k8s.io/kubernetes/pkg/controller/framework"
	kSelector "k8s.io/kubernetes/pkg/fields"
	"k8s.io/kubernetes/pkg/util"
	"github.com/golang/glog"
	consulapi "github.com/hashicorp/consul/api"

  "k8s.io/kubernetes/pkg/api"
  "k8s.io/kubernetes/pkg/fields"
  "k8s.io/kubernetes/pkg/labels"
)

var (
	argConsulAgent         = flag.String("consul-agent", "http://127.0.0.1:8500", "URL to consul agent")
	argKubecfgFile         = flag.String("kubecfg_file", "", "Location of kubecfg file for access to kubernetes service")
	argKubeMasterUrl       = flag.String("kube_master_url", "https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}", "Url to reach kubernetes master. Env variables in this flag will be expanded.")
)

const (
	// Maximum number of attempts to connect to consul server.
	maxConnectAttempts = 12
	// Resync period for the kube controller loop.
	resyncPeriod = 5 * time.Second
)

type nodeInformation struct {
	name string

	address string

	ready bool
	// map[service] DNS IDs
	ids map[string][]string
}

type kube2consul struct {
	// Consul client.
	consulClient *consulapi.Client

	// DNS domain name.
	domain string

	//Nodes Name / valid
	nodes map[string]nodeInformation

	//All Services.
	services map[string]*kapi.Service
}

func Newkube2consul() *kube2consul {
	var k kube2consul
	k.nodes = make(map[string]nodeInformation)
	k.services = make(map[string]*kapi.Service)

	return &k
}

func NewnodeInformation() *nodeInformation {
	var n nodeInformation
	n.ready = false
	n.ids = make(map[string][]string)

	return &n
}

func Contains(s []string, e string) bool {
	for _,i := range s {
		if i == e {
			return true
		}
	}
	return false
}

func (ks *kube2consul) removeDNS(recordID string) error {
	glog.Infof("Removing %s from DNS", recordID)
	ks.consulClient.Agent().ServiceDeregister(recordID)
	return nil
}

func (ks *kube2consul) createDNS(record string, service *kapi.Service, node *nodeInformation) error {
	if strings.Contains(record, ".") {
		glog.Infof("Service names containing '.' are not supported: %s\n", service.Name)
		return nil
	}

	// if ClusterIP is not set, do not create a DNS records
	if !kapi.IsServiceIPSet(service) {
		glog.Infof("Skipping dns record for headless service: %s\n", service.Name)
		return nil
	}

	for i := range service.Spec.Ports {
			newId := node.name+record + service.Spec.Ports[i].Name
      var asrName string

			if len(service.Spec.Ports[i].Name) > 0 {
				asrName = record + "-" + service.Spec.Ports[i].Name
			} else {
				asrName = record
			}

			asr := &consulapi.AgentServiceRegistration{
				ID:			 newId,
				Name: 	 asrName,
				Address: node.address,
				Port:    service.Spec.Ports[i].NodePort,
				Tags: []string{"Kube"},
			}

			if Contains(node.ids[record], newId) == false {
				glog.Infof("Setting DNS record: %v -> %v:%d\n", asr.Name, asr.Address, asr.Port)
				if err := ks.consulClient.Agent().ServiceRegister(asr); err != nil {
					return err
				}

				node.ids[record] = append(node.ids[record], newId)
		}
	}
	return nil
}

func newConsulClient(consulAgent string) (*consulapi.Client, error) {
	var (
		client *consulapi.Client
		err    error
	)

	consulConfig := consulapi.DefaultConfig()
	consulAgentUrl, err := url.Parse(consulAgent)
	if err != nil {
			glog.Infof("Error parsing Consul url")
			return nil, err
	}

	if consulAgentUrl.Host != "" {
	  consulConfig.Address = consulAgentUrl.Host
	}

	if consulAgentUrl.Scheme != "" {
		consulConfig.Scheme = consulAgentUrl.Scheme
	}

	client, err = consulapi.NewClient(consulConfig)
	if err != nil {
			glog.Infof("Error creating Consul client")
			return nil, err
	}

	for attempt := 1; attempt <= maxConnectAttempts; attempt++ {
		if _, err = client.Agent().Self(); err == nil {
			break
		}

		if attempt == maxConnectAttempts {
			break
		}

		glog.Infof("[Attempt: %d] Attempting access to Consul after 5 second sleep", attempt)
		time.Sleep(5 * time.Second)
	}

	if err != nil {
		return nil, fmt.Errorf("failed to connect to Consul agent: %v, error: %v", consulAgent, err)
	}
	glog.Infof("Consul agent found: %v", consulAgent)

	return client, nil
}

func getKubeMasterUrl() (string, error) {
	if *argKubeMasterUrl == "" {
		return "", fmt.Errorf("no --kube_master_url specified")
	}
	parsedUrl, err := url.Parse(os.ExpandEnv(*argKubeMasterUrl))
	if err != nil {
		return "", fmt.Errorf("failed to parse --kube_master_url %s - %v", *argKubeMasterUrl, err)
	}
	if parsedUrl.Scheme == "" || parsedUrl.Host == "" || parsedUrl.Host == ":" {
		return "", fmt.Errorf("invalid --kube_master_url specified %s", *argKubeMasterUrl)
	}
	return parsedUrl.String(), nil
}

// TODO: evaluate using pkg/client/clientcmd
func newKubeClient() (*kclient.Client, error) {
	var config *kclient.Config
	masterUrl, err := getKubeMasterUrl()
	if err != nil {
		return nil, err
	}
	if *argKubecfgFile == "" {
		config = &kclient.Config{
			Host:    masterUrl,
			Version: "v1",
		}
	} else {
		var err error
		overrides := &kclientcmd.ConfigOverrides{}
		overrides.ClusterInfo.Server = masterUrl                                     // might be "", but that is OK
		rules := &kclientcmd.ClientConfigLoadingRules{ExplicitPath: *argKubecfgFile} // might be "", but that is OK
		if config, err = kclientcmd.NewNonInteractiveDeferredLoadingClientConfig(rules, overrides).ClientConfig(); err != nil {
			return nil, err
		}
	}
	glog.Infof("Using %s for kubernetes master", config.Host)
	glog.Infof("Using kubernetes API %s", config.Version)
	return kclient.New(config)
}

func buildNameString(service, namespace string) string {
	//glog.Infof("Name String: %s  %s", service, namespace)
	//return fmt.Sprintf("%s.%s", service, namespace)
	return fmt.Sprintf("%s", service)
}

// Returns a cache.ListWatch that gets all changes to services.
func createServiceLW(kubeClient *kclient.Client) *kcache.ListWatch {
	return kcache.NewListWatchFromClient(kubeClient, "services", kapi.NamespaceAll, kSelector.Everything())
}

// Returns a cache.ListWatch that gets all changes to services.
func createNodeLW(kubeClient *kclient.Client) *kcache.ListWatch {
	return kcache.NewListWatchFromClient(kubeClient, "nodes", kapi.NamespaceAll, kSelector.Everything())
}

func (ks *kube2consul) newService(obj interface{}) {
	if s, ok := obj.(*kapi.Service); ok {
		name := buildNameString(s.Name, s.Namespace)
		ks.services[name] = s

		glog.V(2).Info("Creating Service: ", name)
		//Add to all existing nodes
		for _,node := range ks.nodes {
			if node.ready {
				ks.createDNS(name, s, &node)
			}
		}
	}
}

func (ks *kube2consul) removeService(obj interface{}) {
	if s, ok := obj.(*kapi.Service); ok {
		name := buildNameString(s.Name, s.Namespace)
		//Remove service from node
		for _,node := range ks.nodes {
			for _,id := range node.ids[name] {
				ks.removeDNS(id)
			}
			delete(node.ids, name)
		}
		delete(ks.services,name)
	}
}

func (ks *kube2consul) updateService(oldObj, newObj interface{}) {
	if old, ok := oldObj.(*kapi.Service); ok {
		if new, ok := newObj.(*kapi.Service); ok {
			if reflect.DeepEqual(*old, *new) == false {
				ks.removeService(old)
				ks.newService(new)
			}
		}
	}
}

func (ks *kube2consul) updateNode(oldObj, newObj interface{}) {
	if n, ok := newObj.(*kapi.Node); ok {
		ready := n.Status.Conditions[0].Status == kapi.ConditionTrue
		nodeInfo := ks.nodes[n.Name]

		if nodeInfo.ready != ready {
			nodeInfo.ready = ready
			if ready {
				for serviceName, service := range ks.services {
					ks.createDNS(serviceName, service, &nodeInfo)
				}
			}	else {
				for _,serviceIDs := range nodeInfo.ids {
					for _,serviceID := range serviceIDs {
						ks.removeDNS(serviceID)
					}
				}
				nodeInfo.ids = make(map[string][]string) //Clear tha map
			}
		}
	}
}

func (ks *kube2consul) newNode(newObj interface{}) {
	if node, ok := newObj.(*kapi.Node); ok {
		if _, ok := ks.nodes[node.Name]; !ok {
			glog.Info("Adding Node: ", node.Name)
			var newNode = *NewnodeInformation()
			newNode.name = node.Name
			newNode.address = node.Status.Addresses[0].Address

			ks.nodes[node.Name] = newNode
		}
	}
}

func (ks *kube2consul) removeNode(oldObj interface{}) {
	if node, ok := oldObj.(*kapi.Node); ok {
		if info, ok := ks.nodes[node.Name]; ok {
			glog.Info("Removing Node: ", node.Name)
			for _,idSet := range info.ids {
				for _,id := range idSet {
					ks.removeDNS(id)
				}
			}

			delete(ks.nodes, node.Name)
		}
	}
}

func watchForServices(kubeClient *kclient.Client, ks *kube2consul) {
	var serviceController *kcontrollerFramework.Controller
	_, serviceController = kframework.NewInformer(
		createServiceLW(kubeClient),
		&kapi.Service{},
		resyncPeriod,
		kframework.ResourceEventHandlerFuncs{
			AddFunc:    ks.newService,
			DeleteFunc: ks.removeService,
			UpdateFunc: ks.updateService,
		},
	)
	go serviceController.Run(util.NeverStop)
}

func watchForNodes(kubeClient *kclient.Client, ks *kube2consul) kcache.Store {
	store, serviceController := kframework.NewInformer(
		createNodeLW(kubeClient),
		&kapi.Node{},
		resyncPeriod,
		kframework.ResourceEventHandlerFuncs{
			AddFunc: ks.newNode,
			DeleteFunc: ks.removeNode,
			UpdateFunc: ks.updateNode,
		},
	)
	glog.Info("About to call run!")
	go serviceController.Run(util.NeverStop)
	return store
}

func main() {
	flag.Parse()
	var err error
	// TODO: Validate input flags.
	ks := Newkube2consul()

	if ks.consulClient, err = newConsulClient(*argConsulAgent); err != nil {
		glog.Fatalf("Failed to create Consul client - %v", err)
	}

	kubeClient, err := newKubeClient()
	if err != nil {
		glog.Fatalf("Failed to create a kubernetes client: %v", err)
	}

	glog.Info(kubeClient.ServerVersion())
	glog.Info(kubeClient.Services(kapi.NamespaceAll).Get("sensu-core"))

	pods, err := kubeClient.Pods(api.NamespaceDefault).List(labels.Everything(), fields.Everything())
	if err != nil {
	  for pod := range pods.Items {
			glog.Info(pod)
		}
	}

	watchForServices(kubeClient, ks)
	watchForNodes(kubeClient, ks)
	glog.Info("Watchers running")
	select {}
}
<|endoftext|>


<commit_before>package kumo

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/sww/dumblog"
)

type fileTracker struct {
	expected int
	current  int
}

type Joiner struct {
	DownloadPath string
	Stop         chan bool
	Queue        chan *DecodedPart
	SegmentMap   map[string]int
	Map          map[string]*fileTracker
	Logger       *dumblog.DumbLog
	TempPath     string
	Wait         *sync.WaitGroup
	mu           sync.Mutex
}

func InitJoiner(w *sync.WaitGroup) *Joiner {
	return &Joiner{
		DownloadPath: "",
		SegmentMap:   make(map[string]int),
		Map:          make(map[string]*fileTracker),
		Queue:        make(chan *DecodedPart),
		Stop:         make(chan bool, 1),
		Wait:         w,
	}
}

func (j *Joiner) Run() {
	j.Logger.Printf("[JOINER] Joiner.Run()")
	for {
		select {
		case <-j.Stop:
			j.Logger.Printf("[JOINER] Joiner.Run() stopping")
			return
		case part := <-j.Queue:
			j.Wait.Add(1)
			tracker, exists := j.Map[part.Name]
			if !exists {
				j.Logger.Print("[JOINER] Part not in Map")

				tracker = new(fileTracker)
				tracker.current = 0
				tracker.expected = j.SegmentMap[part.SegmentName]
				j.mu.Lock()
				j.Map[part.Name] = tracker
				j.mu.Unlock()
			}

			j.mu.Lock()
			delete(j.SegmentMap, part.SegmentName)
			j.mu.Unlock()

			tracker.current++
			j.Logger.Print("[JOINER] tracker.current: ", tracker.current, ", tracker.expected: ", tracker.expected)

			if tracker.expected == tracker.current {
				j.Logger.Print("[JOINER] expected == current")
				go j.join(part.Name, tracker.expected)
			}
		default:
			time.Sleep(100 * time.Millisecond)
		}
	}
}

func (j *Joiner) JoinAll() {
	j.Logger.Print("[JOINER] JoinAll()")
	j.Logger.Printf("[JOINER] j.Map: %+v", j.Map)
	for k, tracker := range j.Map {
		if tracker.current != tracker.expected {
			// Since join() calls Done() `count` times, and if a segment
			// in a file is broken, the WaitGroup would be negative, so
			// we Add() the difference to prevent a negative WaitGroup.
			j.Wait.Add(tracker.expected - tracker.current)
			j.join(k, tracker.expected)
		}
	}
}

func (j *Joiner) join(filename string, count int) {
	fullFilename := filepath.Join(j.DownloadPath, filename)
	fullFile, err := os.Create(fullFilename)
	defer fullFile.Close()

	defer func() {
		j.Logger.Printf("[JOINER] Calling Done() %v times", count)
		for i := 0; i < count; i++ {
			j.Wait.Done()
		}
	}()
	defer func() {
		j.mu.Lock()
		delete(j.Map, filename)
		j.mu.Unlock()
	}()

	if err != nil {
		j.Logger.Print("[JOINER] Create fullFile err: ", err)
		return
	}

	// TODO: Fix broken file?

	j.Logger.Print("[JOINER] Joiner.Join(", filename, ", ", count, ")")

	if err != nil {
		j.Logger.Print("[JOINER] error opening ", filename, ": ", err)
		// TODO: Do something.
		return
	}

	bytesWritten := 0

	for i := 1; i < count+1; i++ {
		partFilename := filepath.Join(j.TempPath, fmt.Sprintf("%v.%v", filename, i))
		j.Logger.Print("[JOINER] Joining decoded part ", partFilename)
		file, err := os.Open(partFilename)
		if err != nil {
			// Probably a missing segment, but continue...
			j.Logger.Print("[JOINER] ", partFilename, " does not exist!")
			continue
		}

		defer os.Remove(partFilename)

		data, err := ioutil.ReadAll(file)
		if err != nil {
			j.Logger.Print("[JOINER] got err joining file: ", err)
			// Probably a broken file, but continue...
			continue
		}

		fullFile.Write(data)
		bytesWritten += len(data)
	}

	j.Logger.Print("[JOINER] Done joining file ", filename)
	j.Logger.Print("[JOINER] Wrote ", bytesWritten, " bytes")
}
<commit_msg>Made JoinAll return early if no files are left.<commit_after>package kumo

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/sww/dumblog"
)

type fileTracker struct {
	expected int
	current  int
}

type Joiner struct {
	DownloadPath string
	Stop         chan bool
	Queue        chan *DecodedPart
	SegmentMap   map[string]int
	Map          map[string]*fileTracker
	Logger       *dumblog.DumbLog
	TempPath     string
	Wait         *sync.WaitGroup
	mu           sync.Mutex
}

func InitJoiner(w *sync.WaitGroup) *Joiner {
	return &Joiner{
		DownloadPath: "",
		SegmentMap:   make(map[string]int),
		Map:          make(map[string]*fileTracker),
		Queue:        make(chan *DecodedPart),
		Stop:         make(chan bool, 1),
		Wait:         w,
	}
}

func (j *Joiner) Run() {
	j.Logger.Printf("[JOINER] Joiner.Run()")
	for {
		select {
		case <-j.Stop:
			j.Logger.Printf("[JOINER] Joiner.Run() stopping")
			return
		case part := <-j.Queue:
			j.Wait.Add(1)
			tracker, exists := j.Map[part.Name]
			if !exists {
				j.Logger.Print("[JOINER] Part not in Map")

				tracker = new(fileTracker)
				tracker.current = 0
				tracker.expected = j.SegmentMap[part.SegmentName]
				j.mu.Lock()
				j.Map[part.Name] = tracker
				j.mu.Unlock()
			}

			j.mu.Lock()
			delete(j.SegmentMap, part.SegmentName)
			j.mu.Unlock()

			tracker.current++
			j.Logger.Print("[JOINER] tracker.current: ", tracker.current, ", tracker.expected: ", tracker.expected)

			if tracker.expected == tracker.current {
				j.Logger.Print("[JOINER] expected == current")
				go j.join(part.Name, tracker.expected)
			}
		default:
			time.Sleep(100 * time.Millisecond)
		}
	}
}

func (j *Joiner) JoinAll() {
	if len(j.Map) == 0 {
		return
	}

	j.Logger.Print("[JOINER] JoinAll()")
	j.Logger.Printf("[JOINER] j.Map: %+v", j.Map)
	for k, tracker := range j.Map {
		if tracker.current != tracker.expected {
			// Since join() calls Done() `count` times, and if a segment
			// in a file is broken, the WaitGroup would be negative, so
			// we Add() the difference to prevent a negative WaitGroup.
			j.Wait.Add(tracker.expected - tracker.current)
			j.join(k, tracker.expected)
		}
	}
}

func (j *Joiner) join(filename string, count int) {
	fullFilename := filepath.Join(j.DownloadPath, filename)
	fullFile, err := os.Create(fullFilename)
	defer fullFile.Close()

	defer func() {
		j.Logger.Printf("[JOINER] Calling Done() %v times", count)
		for i := 0; i < count; i++ {
			j.Wait.Done()
		}
	}()
	defer func() {
		j.mu.Lock()
		delete(j.Map, filename)
		j.mu.Unlock()
	}()

	if err != nil {
		j.Logger.Print("[JOINER] Create fullFile err: ", err)
		return
	}

	// TODO: Fix broken file?

	j.Logger.Print("[JOINER] Joiner.Join(", filename, ", ", count, ")")

	if err != nil {
		j.Logger.Print("[JOINER] error opening ", filename, ": ", err)
		// TODO: Do something.
		return
	}

	bytesWritten := 0

	for i := 1; i < count+1; i++ {
		partFilename := filepath.Join(j.TempPath, fmt.Sprintf("%v.%v", filename, i))
		j.Logger.Print("[JOINER] Joining decoded part ", partFilename)
		file, err := os.Open(partFilename)
		if err != nil {
			// Probably a missing segment, but continue...
			j.Logger.Print("[JOINER] ", partFilename, " does not exist!")
			continue
		}

		defer os.Remove(partFilename)

		data, err := ioutil.ReadAll(file)
		if err != nil {
			j.Logger.Print("[JOINER] got err joining file: ", err)
			// Probably a broken file, but continue...
			continue
		}

		fullFile.Write(data)
		bytesWritten += len(data)
	}

	j.Logger.Print("[JOINER] Done joining file ", filename)
	j.Logger.Print("[JOINER] Wrote ", bytesWritten, " bytes")
}
<|endoftext|>


<commit_before>package arduino

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"sync"
	"time"

	"github.com/ninjasphere/go-ninja/logger"
	"github.com/ninjasphere/goserial"
)

var log = logger.GetLogger("arduino")

// Arduino provides two-way communication between go and the arduino on the
// Ninja Block shield and the Ninja Pi Crust
type Arduino struct {
	sync.Mutex
	Incoming chan Message
	port     io.ReadWriteCloser
	acks     chan []DeviceData
}

type Message struct {
	Device []DeviceData `json:"device,omitempty"`
	ACK    []DeviceData `json:"ACK,omitempty"`
	Error  struct {
		Code int
	} `json:"Errormomitempty"`
}

type DeviceData struct {
	G  string
	V  int
	D  int
	DA interface{}
}

func Connect(path string, baudRate int) (arduino *Arduino, err error) {

	config := &serial.Config{Name: path, Baud: baudRate}
	conn, err := serial.OpenPort(config)
	if err != nil {
		return
	}

	arduino = &Arduino{
		Incoming: make(chan Message, 10),
		port:     conn,
		acks:     make(chan []DeviceData),
	}

	reader := bufio.NewReader(conn)
	go func() {
		for {
			str, err := reader.ReadString('\n')
			if err != nil {
				log.Warningf("Failed to read message from serial port: %s", err)
				continue
			}

			log.Infof("Incoming: %s", str)
			var msg Message
			err = json.Unmarshal([]byte(str), &msg)

			if err != nil {
				log.Warningf("Error parsing json: %s", err)
			}

			if msg.ACK != nil {
				select {
				case arduino.acks <- msg.ACK:
				default:
					log.Warningf("Got ack we weren't listening for")
				}
			}

			select {
			case arduino.Incoming <- msg:
			default:
				log.Warningf("Incoming channel is full. Ignoring message: %s", str)
			}

		}
	}()

	return
}

func (a *Arduino) GetVersion() (string, error) {
	ack, err := a.Write(Message{
		Device: []DeviceData{
			DeviceData{
				G:  "0",
				V:  0,
				D:  1003,
				DA: "VNO",
			},
		},
	})

	if err != nil {
		return "", err
	}

	return ack[0].DA.(string), nil
}

func (a *Arduino) WriteDeviceData(data ...DeviceData) error {
	_, err := a.Write(Message{
		Device: data,
	})

	return err
}

func (a *Arduino) Write(message Message) ([]DeviceData, error) {
	a.Lock()
	defer a.Unlock()

	j, _ := json.Marshal(message)

	log.Debugf("Outgoing: %s", j)

	a.port.Write(j)
	a.port.Write([]byte("\n"))

	select {
	case ack := <-a.acks:
		return ack, nil
	case <-time.After(time.Second * 2):
		return nil, fmt.Errorf("Arduino write timed out after 2 seconds")
	}

}
<commit_msg>Fix error prop<commit_after>package arduino

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"sync"
	"time"

	"github.com/ninjasphere/go-ninja/logger"
	"github.com/ninjasphere/goserial"
)

var log = logger.GetLogger("arduino")

// Arduino provides two-way communication between go and the arduino on the
// Ninja Block shield and the Ninja Pi Crust
type Arduino struct {
	sync.Mutex
	Incoming chan Message
	port     io.ReadWriteCloser
	acks     chan []DeviceData
}

type Message struct {
	Device []DeviceData `json:"device,omitempty"`
	ACK    []DeviceData `json:"ACK,omitempty"`
	Error  *struct {
		Code int
	} `json:"Error,omitempty"`
}

type DeviceData struct {
	G  string
	V  int
	D  int
	DA interface{}
}

func Connect(path string, baudRate int) (arduino *Arduino, err error) {

	config := &serial.Config{Name: path, Baud: baudRate}
	conn, err := serial.OpenPort(config)
	if err != nil {
		return
	}

	arduino = &Arduino{
		Incoming: make(chan Message, 10),
		port:     conn,
		acks:     make(chan []DeviceData),
	}

	reader := bufio.NewReader(conn)
	go func() {
		for {
			str, err := reader.ReadString('\n')
			if err != nil {
				log.Warningf("Failed to read message from serial port: %s", err)
				continue
			}

			log.Infof("Incoming: %s", str)
			var msg Message
			err = json.Unmarshal([]byte(str), &msg)

			if err != nil {
				log.Warningf("Error parsing json: %s", err)
			}

			if msg.ACK != nil {
				select {
				case arduino.acks <- msg.ACK:
				default:
					log.Warningf("Got ack we weren't listening for")
				}
			}

			select {
			case arduino.Incoming <- msg:
			default:
				log.Warningf("Incoming channel is full. Ignoring message: %s", str)
			}

		}
	}()

	return
}

func (a *Arduino) GetVersion() (string, error) {
	ack, err := a.Write(Message{
		Device: []DeviceData{
			DeviceData{
				G:  "0",
				V:  0,
				D:  1003,
				DA: "VNO",
			},
		},
	})

	if err != nil {
		return "", err
	}

	return ack[0].DA.(string), nil
}

func (a *Arduino) WriteDeviceData(data ...DeviceData) error {
	_, err := a.Write(Message{
		Device: data,
	})

	return err
}

func (a *Arduino) Write(message Message) ([]DeviceData, error) {
	a.Lock()
	defer a.Unlock()

	j, _ := json.Marshal(message)

	log.Debugf("Outgoing: %s", j)

	a.port.Write(j)
	a.port.Write([]byte("\n"))

	select {
	case ack := <-a.acks:
		return ack, nil
	case <-time.After(time.Second * 2):
		return nil, fmt.Errorf("Arduino write timed out after 2 seconds")
	}

}
<|endoftext|>