<commit_before>package ezk

import (
	"github.com/betable/retry"
	"github.com/samuel/go-zookeeper/zk"
	"time"
)

// Client is a wrapper over github.com/samuel/go-zookeeper/zk that retries all but one of its operations according to the ClientConfig.Retry function. The one exception is for CreateProtectedEphemeralSequential(); it is not retried automatically.
type Client struct {

	// The configuration for the client.
	Cfg ClientConfig

	// The underlying github.com/samuel/go-zookeeper/zk connection.
	Conn *zk.Conn

	// WatchCh will be nil until Connect returns without error.
	// Watches that fire over the Zookeeper connection will be
	// received on WatchCh.
	WatchCh <-chan zk.Event
}

// ClientConfig is used to configure a Client; pass
// it to NewClient().
type ClientConfig struct {

	// The Chroot directory will be prepended to all paths
	Chroot string

	// The set of ACLs used by defeault when
	// calling Client.Create(), if the formal
	// parameter acl in Create() is length 0.
	Acl []zk.ACL

	// The URLs of the zookeepers to attempt to connect to.
	Servers []string

	// SessionTimeout defaults to 10 seconds if not
	// otherwise set.
	SessionTimeout time.Duration

	// The retry function determines how many times
	// and how often we retry our Zookeeper operations
	// before failing. See DefaultRetry() which is
	// used if this is not otherwise set.
	Retry Retry
}

// NewClient creates a new ezk.Client.
// If the cfg.SessionTimout is set to 0
// a default value of 10 seconds will be used.
// If cfg.Retry is nil then the zk.defaultRetry
// function will be used.
func NewClient(cfg ClientConfig) *Client {
	if cfg.Retry == nil {
		cfg.Retry = DefaultRetry
	}
	if cfg.SessionTimeout == 0 {
		cfg.SessionTimeout = 10 * time.Second
	}
	cli := &Client{
		Cfg: cfg,
	}
	return cli
}

// Retry defines the type of the retry method to use.
type Retry func(op, path string, f func() error)

// Connect connects to a Zookeeper server.
// Upon success it sets the z.WatchCh and returns nil.
func (z *Client) Connect() error {
	conn, ch, err := zk.Connect(z.Cfg.Servers, z.Cfg.SessionTimeout)
	if err != nil {
		return err
	}
	z.Conn = conn
	z.WatchCh = ch
	return nil
}

// Close closes the connection to the Zookeeper server.
func (z *Client) Close() {
	z.Conn.Close()
}

// Exists checks if a znode exists.
// z.Cfg.Chroot will be prepended to path. The call will be retried.
func (z *Client) Exists(path string) (ok bool, s *zk.Stat, err error) {
	path = z.fullpath(path)
	z.Cfg.Retry("exists", path, func() error {
		ok, s, err = z.Conn.Exists(path)
		return err
	})
	return ok, s, err
}

// ExistsW returns if a znode exists and sets a watch.
// z.Cfg.Chroot will be prepended to path. The call will be retried.
func (z *Client) ExistsW(path string) (ok bool, s *zk.Stat, ch <-chan zk.Event, err error) {
	path = z.fullpath(path)
	z.Cfg.Retry("existsw", path, func() error {
		ok, s, ch, err = z.Conn.ExistsW(path)
		return err
	})
	return ok, s, ch, err
}

// Create creates a znode with a content. If
// acl is nil then the z.Cfg.Acl set will be
// applied to the new znode.
// z.Cfg.Chroot will be prepended to path. The call will be retried.
func (z *Client) Create(path string, data []byte, flags int32, acl []zk.ACL) (s string, err error) {
	path = z.fullpath(path)
	if len(acl) == 0 && len(z.Cfg.Acl) != 0 {
		acl = z.Cfg.Acl
	}
	z.Cfg.Retry("create", path, func() error {
		s, err = z.Conn.Create(path, data, flags, acl)
		return err
	})
	return s, err
}

// Delete deletes a znode.
// z.Cfg.Chroot will be prepended to path. The call will be retried.
func (z *Client) Delete(path string, version int32) (err error) {
	path = z.fullpath(path)
	z.Cfg.Retry("delete", path, func() error {
		err = z.Conn.Delete(path, version)
		return err
	})
	return err
}

// Get returns the contents of a znode.
// z.Cfg.Chroot will be prepended to path. The call will be retried.
func (z *Client) Get(path string) (d []byte, s *zk.Stat, err error) {
	path = z.fullpath(path)
	z.Cfg.Retry("get", path, func() error {
		d, s, err = z.Conn.Get(path)
		return err
	})
	return d, s, err
}

// GetW returns the contents of a znode and sets a watch.
// z.Cfg.Chroot will be prepended to path. The call will be retried.
func (z *Client) GetW(path string) (d []byte, s *zk.Stat, ch <-chan zk.Event, err error) {
	path = z.fullpath(path)
	z.Cfg.Retry("getw", path, func() error {
		d, s, ch, err = z.Conn.GetW(path)
		return err
	})
	return d, s, ch, err
}

// Set writes content in an existent znode.
// z.Cfg.Chroot will be prepended to path. The call will be retried.
func (z *Client) Set(path string, data []byte, version int32) (s *zk.Stat, err error) {
	path = z.fullpath(path)
	z.Cfg.Retry("set", path, func() error {
		s, err = z.Conn.Set(path, data, version)
		return err
	})
	return s, err
}

// Children returns the children of a znode.
// z.Cfg.Chroot will be prepended to path. The call will be retried.
func (z *Client) Children(path string) (c []string, s *zk.Stat, err error) {
	path = z.fullpath(path)
	z.Cfg.Retry("children", path, func() error {
		c, s, err = z.Conn.Children(path)
		return err
	})
	return c, s, err
}

// ChildrenW returns the children of a znode and sets a watch.
// z.Cfg.Chroot will be prepended to path. The call will be retried.
func (z *Client) ChildrenW(path string) (c []string, s *zk.Stat, ch <-chan zk.Event, err error) {
	path = z.fullpath(path)
	z.Cfg.Retry("childrenw", path, func() error {
		c, s, ch, err = z.Conn.ChildrenW(path)
		return err
	})
	return c, s, ch, err
}

// Sync performs a sync from the master in the Zookeeper server.
// z.Cfg.Chroot will be prepended to path. The call will be retried.
func (z *Client) Sync(path string) (s string, err error) {
	path = z.fullpath(path)
	z.Cfg.Retry("sync", path, func() error {
		s, err = z.Conn.Sync(path)
		return err
	})
	return s, err
}

// GetACL returns the ACL for a znode.
// z.Cfg.Chroot will be prepended to path. The call will be retried.
func (z *Client) GetACL(path string) (a []zk.ACL, s *zk.Stat, err error) {
	path = z.fullpath(path)
	z.Cfg.Retry("getacl", path, func() error {
		a, s, err = z.Conn.GetACL(path)
		return err
	})
	return a, s, err
}

// SetACL sets a ACL to a znode.
// z.Cfg.Chroot will be prepended to path. The call will be retried.
func (z *Client) SetACL(path string, acl []zk.ACL, version int32) (s *zk.Stat, err error) {
	path = z.fullpath(path)
	z.Cfg.Retry("setacl", path, func() error {
		s, err = z.Conn.SetACL(path, acl, version)
		return err
	})
	return s, err
}

// CreateProtectedEphemeralSequential creates a sequential ephemeral znode.
// z.Cfg.Chroot will be prepended to path. The call will be NOT be retried.
func (z *Client) CreateProtectedEphemeralSequential(path string, data []byte, acl []zk.ACL) (string, error) {
	path = z.fullpath(path)
	return z.Conn.CreateProtectedEphemeralSequential(path, data, acl)
}

// CreateDir is a helper method that creates and empty znode if it does not exists.
// z.Cfg.Chroot will be prepended to path. The call will be retried.
func (z *Client) CreateDir(path string, acl []zk.ACL) error {
	path = z.fullpath(path)
	ok, _, err := z.Exists(path)
	if err != nil {
		return err
	}

	if !ok {
		_, err = z.Create(path, []byte{}, 0, acl)
		if err == zk.ErrNodeExists {
			return nil
		}
	}

	return err
}

// SafeSet is a helper method that writes a znode creating it first if it does not exists.
// z.Cfg.Chroot will be prepended to path. The call will be retried.
func (z *Client) SafeSet(path string, data []byte, version int32, acl []zk.ACL) (*zk.Stat, error) {
	path = z.fullpath(path)
	_, err := z.Sync(path)
	if err != nil {
		return nil, err
	}

	ok, _, err := z.Exists(path)
	if err != nil {
		return nil, err
	}

	if !ok {
		_, err := z.Create(path, data, 0, acl)
		if err != nil {
			return nil, err
		}
		_, s, err := z.Exists(path)
		return s, err
	}

	return z.Set(path, data, version)
}

// SafeGet is a helper method that syncs Zookeeper and return the content of a znode.
// z.Cfg.Chroot will be prepended to path. The call will be retried.
func (z *Client) SafeGet(path string) ([]byte, *zk.Stat, error) {
	path = z.fullpath(path)
	_, err := z.Sync(path)
	if err != nil {
		return nil, nil, err
	}

	return z.Get(path)
}

// fullpath returns the path with the chroot prepended.
func (z *Client) fullpath(path string) string {
	//	if z.Cfg.Chroot != "" && !strings.HasPrefix(path, z.Cfg.Chroot) {
	//    return z.Cfg.Chroot + path
	//	}
	//	return path

	return z.Cfg.Chroot + path
}

// The DefaultRetry function will retry four times if the
// first Zookeeper call fails, after sleeping in turn: 0ms, 100ms, 500ms and 1500ms.
func DefaultRetry(op, path string, f func() error) {
	retry.NewExecutor().
		WithRetries(4).
		WithBackoff(retry.ExponentialDelayBackoff(100*time.Millisecond, 5)).
		WithErrorComparator(func(err error) bool {
		return err == zk.ErrConnectionClosed || err == zk.ErrSessionExpired || err == zk.ErrSessionMoved
	}).Execute(f)
}
<commit_msg>atg. improve docs<commit_after>package ezk

import (
	"github.com/betable/retry"
	"github.com/samuel/go-zookeeper/zk"
	"time"
)

// Client is a wrapper over github.com/samuel/go-zookeeper/zk that retries all but one of its operations according to the ClientConfig.Retry function. The one exception is for CreateProtectedEphemeralSequential(); it is not retried automatically.
type Client struct {

	// The configuration for the client.
	Cfg ClientConfig

	// The underlying github.com/samuel/go-zookeeper/zk connection.
	Conn *zk.Conn

	// WatchCh will be nil until Connect returns without error.
	// Watches that fire over the Zookeeper connection will be
	// received on WatchCh.
	WatchCh <-chan zk.Event
}

// ClientConfig is used to configure a Client; pass
// it to NewClient().
type ClientConfig struct {

	// The Chroot directory will be prepended to all paths
	Chroot string

	// The set of ACLs used by defeault when
	// calling Client.Create(), if the formal
	// parameter acl in Create() is length 0.
	Acl []zk.ACL

	// The URLs of the zookeepers to attempt to connect to.
	Servers []string

	// SessionTimeout defaults to 10 seconds if not
	// otherwise set.
	SessionTimeout time.Duration

	// The retry function determines how many times
	// and how often we retry our Zookeeper operations
	// before failing. See DefaultRetry() which is
	// used if this is not otherwise set.
	Retry Retry
}

// NewClient creates a new ezk.Client.
// If the cfg.SessionTimout is set to 0
// a default value of 10 seconds will be used.
// If cfg.Retry is nil then the DefaultRetry
// function will be used.
func NewClient(cfg ClientConfig) *Client {
	if cfg.Retry == nil {
		cfg.Retry = DefaultRetry
	}
	if cfg.SessionTimeout == 0 {
		cfg.SessionTimeout = 10 * time.Second
	}
	cli := &Client{
		Cfg: cfg,
	}
	return cli
}

// Retry defines the type of the retry method to use.
type Retry func(op, path string, f func() error)

// Connect connects to a Zookeeper server.
// Upon success it sets the z.WatchCh and returns nil.
func (z *Client) Connect() error {
	conn, ch, err := zk.Connect(z.Cfg.Servers, z.Cfg.SessionTimeout)
	if err != nil {
		return err
	}
	z.Conn = conn
	z.WatchCh = ch
	return nil
}

// Close closes the connection to the Zookeeper server.
func (z *Client) Close() {
	z.Conn.Close()
}

// Exists checks if a znode exists.
// z.Cfg.Chroot will be prepended to path. The call will be retried.
func (z *Client) Exists(path string) (ok bool, s *zk.Stat, err error) {
	path = z.fullpath(path)
	z.Cfg.Retry("exists", path, func() error {
		ok, s, err = z.Conn.Exists(path)
		return err
	})
	return ok, s, err
}

// ExistsW returns if a znode exists and sets a watch.
// z.Cfg.Chroot will be prepended to path. The call will be retried.
func (z *Client) ExistsW(path string) (ok bool, s *zk.Stat, ch <-chan zk.Event, err error) {
	path = z.fullpath(path)
	z.Cfg.Retry("existsw", path, func() error {
		ok, s, ch, err = z.Conn.ExistsW(path)
		return err
	})
	return ok, s, ch, err
}

// Create creates a znode with a content. If
// acl is nil then the z.Cfg.Acl set will be
// applied to the new znode.
// z.Cfg.Chroot will be prepended to path. The call will be retried.
func (z *Client) Create(path string, data []byte, flags int32, acl []zk.ACL) (s string, err error) {
	path = z.fullpath(path)
	if len(acl) == 0 && len(z.Cfg.Acl) != 0 {
		acl = z.Cfg.Acl
	}
	z.Cfg.Retry("create", path, func() error {
		s, err = z.Conn.Create(path, data, flags, acl)
		return err
	})
	return s, err
}

// Delete deletes a znode.
// z.Cfg.Chroot will be prepended to path. The call will be retried.
func (z *Client) Delete(path string, version int32) (err error) {
	path = z.fullpath(path)
	z.Cfg.Retry("delete", path, func() error {
		err = z.Conn.Delete(path, version)
		return err
	})
	return err
}

// Get returns the contents of a znode.
// z.Cfg.Chroot will be prepended to path. The call will be retried.
func (z *Client) Get(path string) (d []byte, s *zk.Stat, err error) {
	path = z.fullpath(path)
	z.Cfg.Retry("get", path, func() error {
		d, s, err = z.Conn.Get(path)
		return err
	})
	return d, s, err
}

// GetW returns the contents of a znode and sets a watch.
// z.Cfg.Chroot will be prepended to path. The call will be retried.
func (z *Client) GetW(path string) (d []byte, s *zk.Stat, ch <-chan zk.Event, err error) {
	path = z.fullpath(path)
	z.Cfg.Retry("getw", path, func() error {
		d, s, ch, err = z.Conn.GetW(path)
		return err
	})
	return d, s, ch, err
}

// Set writes content in an existent znode.
// z.Cfg.Chroot will be prepended to path. The call will be retried.
func (z *Client) Set(path string, data []byte, version int32) (s *zk.Stat, err error) {
	path = z.fullpath(path)
	z.Cfg.Retry("set", path, func() error {
		s, err = z.Conn.Set(path, data, version)
		return err
	})
	return s, err
}

// Children returns the children of a znode.
// z.Cfg.Chroot will be prepended to path. The call will be retried.
func (z *Client) Children(path string) (c []string, s *zk.Stat, err error) {
	path = z.fullpath(path)
	z.Cfg.Retry("children", path, func() error {
		c, s, err = z.Conn.Children(path)
		return err
	})
	return c, s, err
}

// ChildrenW returns the children of a znode and sets a watch.
// z.Cfg.Chroot will be prepended to path. The call will be retried.
func (z *Client) ChildrenW(path string) (c []string, s *zk.Stat, ch <-chan zk.Event, err error) {
	path = z.fullpath(path)
	z.Cfg.Retry("childrenw", path, func() error {
		c, s, ch, err = z.Conn.ChildrenW(path)
		return err
	})
	return c, s, ch, err
}

// Sync performs a sync from the master in the Zookeeper server.
// z.Cfg.Chroot will be prepended to path. The call will be retried.
func (z *Client) Sync(path string) (s string, err error) {
	path = z.fullpath(path)
	z.Cfg.Retry("sync", path, func() error {
		s, err = z.Conn.Sync(path)
		return err
	})
	return s, err
}

// GetACL returns the ACL for a znode.
// z.Cfg.Chroot will be prepended to path. The call will be retried.
func (z *Client) GetACL(path string) (a []zk.ACL, s *zk.Stat, err error) {
	path = z.fullpath(path)
	z.Cfg.Retry("getacl", path, func() error {
		a, s, err = z.Conn.GetACL(path)
		return err
	})
	return a, s, err
}

// SetACL sets a ACL to a znode.
// z.Cfg.Chroot will be prepended to path. The call will be retried.
func (z *Client) SetACL(path string, acl []zk.ACL, version int32) (s *zk.Stat, err error) {
	path = z.fullpath(path)
	z.Cfg.Retry("setacl", path, func() error {
		s, err = z.Conn.SetACL(path, acl, version)
		return err
	})
	return s, err
}

// CreateProtectedEphemeralSequential creates a sequential ephemeral znode.
// z.Cfg.Chroot will be prepended to path. The call will be NOT be retried.
func (z *Client) CreateProtectedEphemeralSequential(path string, data []byte, acl []zk.ACL) (string, error) {
	path = z.fullpath(path)
	return z.Conn.CreateProtectedEphemeralSequential(path, data, acl)
}

// CreateDir is a helper method that creates and empty znode if it does not exists.
// z.Cfg.Chroot will be prepended to path. The call will be retried.
func (z *Client) CreateDir(path string, acl []zk.ACL) error {
	path = z.fullpath(path)
	ok, _, err := z.Exists(path)
	if err != nil {
		return err
	}

	if !ok {
		_, err = z.Create(path, []byte{}, 0, acl)
		if err == zk.ErrNodeExists {
			return nil
		}
	}

	return err
}

// SafeSet is a helper method that writes a znode creating it first if it does not exists.
// z.Cfg.Chroot will be prepended to path. The call will be retried.
func (z *Client) SafeSet(path string, data []byte, version int32, acl []zk.ACL) (*zk.Stat, error) {
	path = z.fullpath(path)
	_, err := z.Sync(path)
	if err != nil {
		return nil, err
	}

	ok, _, err := z.Exists(path)
	if err != nil {
		return nil, err
	}

	if !ok {
		_, err := z.Create(path, data, 0, acl)
		if err != nil {
			return nil, err
		}
		_, s, err := z.Exists(path)
		return s, err
	}

	return z.Set(path, data, version)
}

// SafeGet is a helper method that syncs Zookeeper and return the content of a znode.
// z.Cfg.Chroot will be prepended to path. The call will be retried.
func (z *Client) SafeGet(path string) ([]byte, *zk.Stat, error) {
	path = z.fullpath(path)
	_, err := z.Sync(path)
	if err != nil {
		return nil, nil, err
	}

	return z.Get(path)
}

// fullpath returns the path with the chroot prepended.
func (z *Client) fullpath(path string) string {
	//	if z.Cfg.Chroot != "" && !strings.HasPrefix(path, z.Cfg.Chroot) {
	//    return z.Cfg.Chroot + path
	//	}
	//	return path

	return z.Cfg.Chroot + path
}

// The DefaultRetry function will retry four times if the
// first Zookeeper call fails, after sleeping in turn: 0ms, 100ms, 500ms and 1500ms.
func DefaultRetry(op, path string, f func() error) {
	retry.NewExecutor().
		WithRetries(4).
		WithBackoff(retry.ExponentialDelayBackoff(100*time.Millisecond, 5)).
		WithErrorComparator(func(err error) bool {
		return err == zk.ErrConnectionClosed || err == zk.ErrSessionExpired || err == zk.ErrSessionMoved
	}).Execute(f)
}
<|endoftext|>


<commit_before>// +build ignore

package main

import (
	"."
	"flag"
	"github.com/bmizerany/pat"
	"github.com/timeredbull/tsuru/api/app"
	"github.com/timeredbull/tsuru/api/auth"
	"github.com/timeredbull/tsuru/api/service"
	"github.com/timeredbull/tsuru/config"
	"github.com/timeredbull/tsuru/db"
	"github.com/timeredbull/tsuru/log"
	"github.com/timeredbull/tsuru/repository"
	stdlog "log"
	"log/syslog"
	"net/http"
)

func main() {
	var err error
	log.Target, err = syslog.NewLogger(syslog.LOG_INFO, stdlog.LstdFlags)
	if err != nil {
		panic(err)
	}
	configFile := flag.String("config", "/etc/tsuru/tsuru.conf", "tsuru config file")
	dry := flag.Bool("dry", false, "dry-run: does not start the server (for testing purpose)")
	flag.Parse()
	err = config.ReadConfigFile(*configFile)
	if err != nil {
		log.Panic(err.Error())
	}
	connString, err := config.GetString("database:url")
	if err != nil {
		panic(err)
	}
	dbName, err := config.GetString("database:name")
	if err != nil {
		panic(err)
	}
	db.Session, err = db.Open(connString, dbName)
	if err != nil {
		log.Panic(err.Error())
	}
	defer db.Session.Close()

	repository.RunAgent()
	m := pat.New()

	m.Post("/services", webserver.AuthorizationRequiredHandler(service.CreateHandler))
	m.Get("/services", webserver.AuthorizationRequiredHandler(service.ServicesHandler))
	m.Get("/services/types", webserver.Handler(service.ServiceTypesHandler))
	m.Del("/services/:name", webserver.AuthorizationRequiredHandler(service.DeleteHandler))
	m.Post("/services/bind", webserver.AuthorizationRequiredHandler(service.BindHandler))
	m.Post("/services/unbind", webserver.AuthorizationRequiredHandler(service.UnbindHandler))
	m.Put("/services/:service/:team", webserver.AuthorizationRequiredHandler(service.GrantAccessToTeamHandler))
	m.Del("/services/:service/:team", webserver.AuthorizationRequiredHandler(service.RevokeAccessFromTeamHandler))

	m.Del("/apps/:name", webserver.AuthorizationRequiredHandler(app.AppDelete))
	m.Get("/apps/:name/repository/clone", webserver.Handler(app.CloneRepositoryHandler))
	m.Get("/apps/:name", webserver.AuthorizationRequiredHandler(app.AppInfo))
	m.Post("/apps/:name/run", webserver.AuthorizationRequiredHandler(app.RunCommand))
	m.Get("/apps/:name/env", webserver.AuthorizationRequiredHandler(app.GetEnv))
	m.Post("/apps/:name/env", webserver.AuthorizationRequiredHandler(app.SetEnv))
	m.Del("/apps/:name/env", webserver.AuthorizationRequiredHandler(app.UnsetEnv))
	m.Get("/apps", webserver.AuthorizationRequiredHandler(app.AppList))
	m.Post("/apps", webserver.AuthorizationRequiredHandler(app.CreateAppHandler))
	m.Put("/apps/:app/:team", webserver.AuthorizationRequiredHandler(app.GrantAccessToTeamHandler))
	m.Del("/apps/:app/:team", webserver.AuthorizationRequiredHandler(app.RevokeAccessFromTeamHandler))

	m.Post("/users", webserver.Handler(auth.CreateUser))
	m.Post("/users/:email/tokens", webserver.Handler(auth.Login))
	m.Post("/users/keys", webserver.AuthorizationRequiredHandler(auth.AddKeyToUser))
	m.Del("/users/keys", webserver.AuthorizationRequiredHandler(auth.RemoveKeyFromUser))

	m.Post("/teams", webserver.AuthorizationRequiredHandler(auth.CreateTeam))
	m.Put("/teams/:team/:user", webserver.AuthorizationRequiredHandler(auth.AddUserToTeam))
	m.Del("/teams/:team/:user", webserver.AuthorizationRequiredHandler(auth.RemoveUserFromTeam))

	listen, err := config.GetString("listen")
	if err != nil {
		panic(err)
	}
	if !*dry {
		log.Fatal(http.ListenAndServe(listen, m))
	}
}
<commit_msg>mapping url for app log handler<commit_after>// +build ignore

package main

import (
	"."
	"flag"
	"github.com/bmizerany/pat"
	"github.com/timeredbull/tsuru/api/app"
	"github.com/timeredbull/tsuru/api/auth"
	"github.com/timeredbull/tsuru/api/service"
	"github.com/timeredbull/tsuru/config"
	"github.com/timeredbull/tsuru/db"
	"github.com/timeredbull/tsuru/log"
	"github.com/timeredbull/tsuru/repository"
	stdlog "log"
	"log/syslog"
	"net/http"
)

func main() {
	var err error
	log.Target, err = syslog.NewLogger(syslog.LOG_INFO, stdlog.LstdFlags)
	if err != nil {
		panic(err)
	}
	configFile := flag.String("config", "/etc/tsuru/tsuru.conf", "tsuru config file")
	dry := flag.Bool("dry", false, "dry-run: does not start the server (for testing purpose)")
	flag.Parse()
	err = config.ReadConfigFile(*configFile)
	if err != nil {
		log.Panic(err.Error())
	}
	connString, err := config.GetString("database:url")
	if err != nil {
		panic(err)
	}
	dbName, err := config.GetString("database:name")
	if err != nil {
		panic(err)
	}
	db.Session, err = db.Open(connString, dbName)
	if err != nil {
		log.Panic(err.Error())
	}
	defer db.Session.Close()

	repository.RunAgent()
	m := pat.New()

	m.Post("/services", webserver.AuthorizationRequiredHandler(service.CreateHandler))
	m.Get("/services", webserver.AuthorizationRequiredHandler(service.ServicesHandler))
	m.Get("/services/types", webserver.Handler(service.ServiceTypesHandler))
	m.Del("/services/:name", webserver.AuthorizationRequiredHandler(service.DeleteHandler))
	m.Post("/services/bind", webserver.AuthorizationRequiredHandler(service.BindHandler))
	m.Post("/services/unbind", webserver.AuthorizationRequiredHandler(service.UnbindHandler))
	m.Put("/services/:service/:team", webserver.AuthorizationRequiredHandler(service.GrantAccessToTeamHandler))
	m.Del("/services/:service/:team", webserver.AuthorizationRequiredHandler(service.RevokeAccessFromTeamHandler))

	m.Del("/apps/:name", webserver.AuthorizationRequiredHandler(app.AppDelete))
	m.Get("/apps/:name/repository/clone", webserver.Handler(app.CloneRepositoryHandler))
	m.Get("/apps/:name", webserver.AuthorizationRequiredHandler(app.AppInfo))
	m.Post("/apps/:name/run", webserver.AuthorizationRequiredHandler(app.RunCommand))
	m.Get("/apps/:name/env", webserver.AuthorizationRequiredHandler(app.GetEnv))
	m.Post("/apps/:name/env", webserver.AuthorizationRequiredHandler(app.SetEnv))
	m.Del("/apps/:name/env", webserver.AuthorizationRequiredHandler(app.UnsetEnv))
	m.Get("/apps", webserver.AuthorizationRequiredHandler(app.AppList))
	m.Post("/apps", webserver.AuthorizationRequiredHandler(app.CreateAppHandler))
	m.Put("/apps/:app/:team", webserver.AuthorizationRequiredHandler(app.GrantAccessToTeamHandler))
	m.Del("/apps/:app/:team", webserver.AuthorizationRequiredHandler(app.RevokeAccessFromTeamHandler))
	m.Del("/apps/:app/log", webserver.AuthorizationRequiredHandler(app.AppLog))

	m.Post("/users", webserver.Handler(auth.CreateUser))
	m.Post("/users/:email/tokens", webserver.Handler(auth.Login))
	m.Post("/users/keys", webserver.AuthorizationRequiredHandler(auth.AddKeyToUser))
	m.Del("/users/keys", webserver.AuthorizationRequiredHandler(auth.RemoveKeyFromUser))

	m.Post("/teams", webserver.AuthorizationRequiredHandler(auth.CreateTeam))
	m.Put("/teams/:team/:user", webserver.AuthorizationRequiredHandler(auth.AddUserToTeam))
	m.Del("/teams/:team/:user", webserver.AuthorizationRequiredHandler(auth.RemoveUserFromTeam))

	listen, err := config.GetString("listen")
	if err != nil {
		panic(err)
	}
	if !*dry {
		log.Fatal(http.ListenAndServe(listen, m))
	}
}
<|endoftext|>


<commit_before>package mongo

import (
	"fmt"
	"os"
	"os/exec"
	"path"
	"path/filepath"

	"github.com/juju/loggo"

	"launchpad.net/juju-core/upstart"
	"launchpad.net/juju-core/utils"
)

const (
	maxFiles = 65000
	maxProcs = 20000
)

var (
	logger = loggo.GetLogger("juju.agent.mongo")

	oldMongoServiceName = "juju-db"

	// JujuMongodPath holds the default path to the juju-specific mongod.
	JujuMongodPath = "/usr/lib/juju/bin/mongod"
	// MongodbServerPath holds the default path to the generic mongod.
	MongodbServerPath = "/usr/bin/mongod"
)

// MongoPackageForSeries returns the name of the mongo package for the series
// of the machine that it is going to be running on.
func MongoPackageForSeries(series string) string {
	switch series {
	case "precise", "raring", "saucy":
		return "mongodb-server"
	default:
		// trusty and onwards
		return "juju-mongodb"
	}
}

// MongodPathForSeries returns the path to the mongod executable for the
// series of the machine that it is going to be running on.
func MongodPathForSeries(series string) string {
	if series == "trusty" {
		return JujuMongodPath
	}
	return MongodbServerPath
}

// MongoPath returns the executable path to be used to run mongod on this
// machine. If the juju-bundled version of mongo exists, it will return that
// path, otherwise it will return the command to run mongod from the path.
func MongodPath() (string, error) {
	if _, err := os.Stat(JujuMongodPath); err == nil {
		return JujuMongodPath, nil
	}

	path, err := exec.LookPath("mongod")
	if err != nil {
		return "", err
	}
	return path, nil
}

// EnsureMongoServer ensures that the correct mongo upstart script is installed
// and running.
//
// This method will remove old versions of the mongo upstart script as necessary
// before installing the new version.
func EnsureMongoServer(dir string, port int) error {
	// NOTE: ensure that the right package is installed?
	name := makeServiceName(mongoScriptVersion)
	// TODO: get the series from somewhere, non trusty values return
	// the existing default path.
	mongodPath := MongodPathForSeries("some-series")
	service, err := MongoUpstartService(name, mongodPath, dir, port)
	if err != nil {
		return err
	}
	if service.Installed() {
		return nil
	}

	if err := removeOldMongoServices(mongoScriptVersion); err != nil {
		return err
	}

	if err := makeJournalDirs(dir); err != nil {
		return err
	}

	if err := service.Install(); err != nil {
		return fmt.Errorf("failed to install mongo service %q: %v", service.Name, err)
	}
	return service.Start()
}

func makeJournalDirs(dir string) error {
	journalDir := path.Join(dir, "journal")

	if err := os.MkdirAll(journalDir, 0700); err != nil {
		logger.Errorf("failed to make mongo journal dir %s: %v", journalDir, err)
		return err
	}

	// manually create the prealloc files, since otherwise they get created as 100M files.
	zeroes := make([]byte, 64*1024) // should be enough for anyone
	for x := 0; x < 3; x++ {
		name := fmt.Sprintf("prealloc.%d", x)
		filename := filepath.Join(journalDir, name)
		f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0700)
		if err != nil {
			return fmt.Errorf("failed to open mongo prealloc file %q: %v", filename, err)
		}
		defer f.Close()
		for total := 0; total < 1024*1024; {
			n, err := f.Write(zeroes)
			if err != nil {
				return fmt.Errorf("failed to write to mongo prealloc file %q: %v", filename, err)
			}
			total += n
		}
	}
	return nil
}

// removeOldMongoServices looks for any old juju mongo upstart scripts and
// removes them.
func removeOldMongoServices(curVersion int) error {
	old := upstart.NewService(oldMongoServiceName)
	if err := old.StopAndRemove(); err != nil {
		logger.Errorf("Failed to remove old mongo upstart service %q: %v", old.Name, err)
		return err
	}

	// the new formatting for the script name started at version 2
	for x := 2; x < curVersion; x++ {
		old := upstart.NewService(makeServiceName(x))
		if err := old.StopAndRemove(); err != nil {
			logger.Errorf("Failed to remove old mongo upstart service %q: %v", old.Name, err)
			return err
		}
	}
	return nil
}

func makeServiceName(version int) string {
	return fmt.Sprintf("juju-db-v%d", version)
}

// mongoScriptVersion keeps track of changes to the mongo upstart script.
// Update this version when you update the script that gets installed from
// MongoUpstartService.
const mongoScriptVersion = 2

// MongoUpstartService returns the upstart config for the mongo state service.
//
// This method assumes there is a server.pem keyfile in dataDir.
func MongoUpstartService(name, mongodExec, dataDir string, port int) (*upstart.Conf, error) {

	keyFile := path.Join(dataDir, "server.pem")
	svc := upstart.NewService(name)

	dbDir := path.Join(dataDir, "db")

	conf := &upstart.Conf{
		Service: *svc,
		Desc:    "juju state database",
		Limit: map[string]string{
			"nofile": fmt.Sprintf("%d %d", maxFiles, maxFiles),
			"nproc":  fmt.Sprintf("%d %d", maxProcs, maxProcs),
		},
		Cmd: mongodExec +
			" --auth" +
			" --dbpath=" + dbDir +
			" --sslOnNormalPorts" +
			" --sslPEMKeyFile " + utils.ShQuote(keyFile) +
			" --sslPEMKeyPassword ignored" +
			" --bind_ip 0.0.0.0" +
			" --port " + fmt.Sprint(port) +
			" --noprealloc" +
			" --syslog" +
			" --smallfiles",
		// TODO(Nate): uncomment when we commit HA stuff
		// +
		//	" --replSet juju",
	}
	return conf, nil
}
<commit_msg>Some tests explicitly testing quantal.<commit_after>package mongo

import (
	"fmt"
	"os"
	"os/exec"
	"path"
	"path/filepath"

	"github.com/juju/loggo"

	"launchpad.net/juju-core/upstart"
	"launchpad.net/juju-core/utils"
)

const (
	maxFiles = 65000
	maxProcs = 20000
)

var (
	logger = loggo.GetLogger("juju.agent.mongo")

	oldMongoServiceName = "juju-db"

	// JujuMongodPath holds the default path to the juju-specific mongod.
	JujuMongodPath = "/usr/lib/juju/bin/mongod"
	// MongodbServerPath holds the default path to the generic mongod.
	MongodbServerPath = "/usr/bin/mongod"
)

// MongoPackageForSeries returns the name of the mongo package for the series
// of the machine that it is going to be running on.
func MongoPackageForSeries(series string) string {
	switch series {
	case "precise", "quantal", "raring", "saucy":
		return "mongodb-server"
	default:
		// trusty and onwards
		return "juju-mongodb"
	}
}

// MongodPathForSeries returns the path to the mongod executable for the
// series of the machine that it is going to be running on.
func MongodPathForSeries(series string) string {
	if series == "trusty" {
		return JujuMongodPath
	}
	return MongodbServerPath
}

// MongoPath returns the executable path to be used to run mongod on this
// machine. If the juju-bundled version of mongo exists, it will return that
// path, otherwise it will return the command to run mongod from the path.
func MongodPath() (string, error) {
	if _, err := os.Stat(JujuMongodPath); err == nil {
		return JujuMongodPath, nil
	}

	path, err := exec.LookPath("mongod")
	if err != nil {
		return "", err
	}
	return path, nil
}

// EnsureMongoServer ensures that the correct mongo upstart script is installed
// and running.
//
// This method will remove old versions of the mongo upstart script as necessary
// before installing the new version.
func EnsureMongoServer(dir string, port int) error {
	// NOTE: ensure that the right package is installed?
	name := makeServiceName(mongoScriptVersion)
	// TODO: get the series from somewhere, non trusty values return
	// the existing default path.
	mongodPath := MongodPathForSeries("some-series")
	service, err := MongoUpstartService(name, mongodPath, dir, port)
	if err != nil {
		return err
	}
	if service.Installed() {
		return nil
	}

	if err := removeOldMongoServices(mongoScriptVersion); err != nil {
		return err
	}

	if err := makeJournalDirs(dir); err != nil {
		return err
	}

	if err := service.Install(); err != nil {
		return fmt.Errorf("failed to install mongo service %q: %v", service.Name, err)
	}
	return service.Start()
}

func makeJournalDirs(dir string) error {
	journalDir := path.Join(dir, "journal")

	if err := os.MkdirAll(journalDir, 0700); err != nil {
		logger.Errorf("failed to make mongo journal dir %s: %v", journalDir, err)
		return err
	}

	// manually create the prealloc files, since otherwise they get created as 100M files.
	zeroes := make([]byte, 64*1024) // should be enough for anyone
	for x := 0; x < 3; x++ {
		name := fmt.Sprintf("prealloc.%d", x)
		filename := filepath.Join(journalDir, name)
		f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0700)
		if err != nil {
			return fmt.Errorf("failed to open mongo prealloc file %q: %v", filename, err)
		}
		defer f.Close()
		for total := 0; total < 1024*1024; {
			n, err := f.Write(zeroes)
			if err != nil {
				return fmt.Errorf("failed to write to mongo prealloc file %q: %v", filename, err)
			}
			total += n
		}
	}
	return nil
}

// removeOldMongoServices looks for any old juju mongo upstart scripts and
// removes them.
func removeOldMongoServices(curVersion int) error {
	old := upstart.NewService(oldMongoServiceName)
	if err := old.StopAndRemove(); err != nil {
		logger.Errorf("Failed to remove old mongo upstart service %q: %v", old.Name, err)
		return err
	}

	// the new formatting for the script name started at version 2
	for x := 2; x < curVersion; x++ {
		old := upstart.NewService(makeServiceName(x))
		if err := old.StopAndRemove(); err != nil {
			logger.Errorf("Failed to remove old mongo upstart service %q: %v", old.Name, err)
			return err
		}
	}
	return nil
}

func makeServiceName(version int) string {
	return fmt.Sprintf("juju-db-v%d", version)
}

// mongoScriptVersion keeps track of changes to the mongo upstart script.
// Update this version when you update the script that gets installed from
// MongoUpstartService.
const mongoScriptVersion = 2

// MongoUpstartService returns the upstart config for the mongo state service.
//
// This method assumes there is a server.pem keyfile in dataDir.
func MongoUpstartService(name, mongodExec, dataDir string, port int) (*upstart.Conf, error) {

	keyFile := path.Join(dataDir, "server.pem")
	svc := upstart.NewService(name)

	dbDir := path.Join(dataDir, "db")

	conf := &upstart.Conf{
		Service: *svc,
		Desc:    "juju state database",
		Limit: map[string]string{
			"nofile": fmt.Sprintf("%d %d", maxFiles, maxFiles),
			"nproc":  fmt.Sprintf("%d %d", maxProcs, maxProcs),
		},
		Cmd: mongodExec +
			" --auth" +
			" --dbpath=" + dbDir +
			" --sslOnNormalPorts" +
			" --sslPEMKeyFile " + utils.ShQuote(keyFile) +
			" --sslPEMKeyPassword ignored" +
			" --bind_ip 0.0.0.0" +
			" --port " + fmt.Sprint(port) +
			" --noprealloc" +
			" --syslog" +
			" --smallfiles",
		// TODO(Nate): uncomment when we commit HA stuff
		// +
		//	" --replSet juju",
	}
	return conf, nil
}
<|endoftext|>


<commit_before>//
// Copyright (c) 2015 The heketi Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package rest

import (
	"github.com/gorilla/mux"
	"github.com/heketi/utils"
	"github.com/lpabon/godbc"
	"net/http"
	"sync"
	"time"
)

var (
	logger = utils.NewLogger("[asynchttp]", utils.LEVEL_INFO)
)

// Contains information about the asynchronous operation
type AsyncHttpHandler struct {
	err          error
	completed    bool
	manager      *AsyncHttpManager
	location, id string
}

// Manager of asynchronous operations
type AsyncHttpManager struct {
	lock     sync.RWMutex
	route    string
	handlers map[string]*AsyncHttpHandler
}

// Creates a new manager
func NewAsyncHttpManager(route string) *AsyncHttpManager {
	return &AsyncHttpManager{
		route:    route,
		handlers: make(map[string]*AsyncHttpHandler),
	}
}

// Use to create a new asynchronous operation handler.
// Only use this function if you need to do every step by hand.
// It is recommended to use AsyncHttpRedirectFunc() instead
func (a *AsyncHttpManager) NewHandler() *AsyncHttpHandler {
	handler := &AsyncHttpHandler{
		manager: a,
		id:      utils.GenUUID(),
	}

	a.lock.Lock()
	defer a.lock.Unlock()

	a.handlers[handler.id] = handler

	return handler
}

// Create an asynchronous operation handler and return the appropiate
// information the caller.
// This function will call handlerfunc() in a new go routine, then
// return to the caller a HTTP status 202 setting up the `Location` header
// to point to the new asynchronous handler.
//
// If handlerfunc() returns failure, the asynchronous handler will return
// an http status of 500 and save the error string in the body.
// If handlerfunc() is successful and returns a location url path in "string",
// the asynchronous handler will return 303 (See Other) with the Location
// header set to the value returned in the string.
// If handlerfunc() is successful and returns an empty string, then the
// asynchronous handler will return 204 to the caller.
//
// Example:
//      package rest
//		import (
//			"github.com/gorilla/mux"
//          "github.com/heketi/rest"
//			"net/http"
//			"net/http/httptest"
//			"time"
//		)
//
//		// Setup asynchronous manager
//		route := "/x"
//		manager := rest.NewAsyncHttpManager(route)
//
//		// Setup the route
//		router := mux.NewRouter()
//	 	router.HandleFunc(route+"/{id}", manager.HandlerStatus).Methods("GET")
//		router.HandleFunc("/result", func(w http.ResponseWriter, r *http.Request) {
//			w.Header().Set("Content-Type", "text/plain; charset=UTF-8")
//			w.WriteHeader(http.StatusOK)
//			fmt.Fprint(w, "HelloWorld")
//		}).Methods("GET")
//
//		router.HandleFunc("/app", func(w http.ResponseWriter, r *http.Request) {
//			manager.AsyncHttpRedirectFunc(w, r, func() (string, error) {
//				time.Sleep(100 * time.Millisecond)
//				return "/result", nil
//			})
//		}).Methods("GET")
//
//		// Setup the server
//		ts := httptest.NewServer(router)
//		defer ts.Close()
//
func (a *AsyncHttpManager) AsyncHttpRedirectFunc(w http.ResponseWriter,
	r *http.Request,
	handlerfunc func() (string, error)) {

	handler := a.NewHandler()
	go func() {
		logger.Info("Started job %v", handler.id)

		ts := time.Now()
		url, err := handlerfunc()
		logger.Info("Completed job %v in %v", handler.id, time.Since(ts))

		if err != nil {
			handler.CompletedWithError(err)
		} else if url != "" {
			handler.CompletedWithLocation(url)
		} else {
			handler.Completed()
		}
	}()
	http.Redirect(w, r, handler.Url(), http.StatusAccepted)
}

// Handler for asynchronous operation status
// Register this handler with a router like Gorilla Mux
//
// Returns the following HTTP status codes
// 		200 Operation is still pending
//		404 Id requested does not exist
//		500 Operation finished and has failed.  Body will be filled in with the
//			error in plain text.
//		303 Operation finished and has setup a new location to retreive data.
//		204 Operation finished and has no data to return
//
// Example:
//      package rest
//		import (
//			"github.com/gorilla/mux"
//          "github.com/heketi/rest"
//			"net/http"
//			"net/http/httptest"
//			"time"
//		)
//
//		// Setup asynchronous manager
//		route := "/x"
//		manager := rest.NewAsyncHttpManager(route)
//
//		// Setup the route
//		router := mux.NewRouter()
//	 	router.HandleFunc(route+"/{id:[A-Fa-f0-9]+}", manager.HandlerStatus).Methods("GET")
//
//		// Setup the server
//		ts := httptest.NewServer(router)
//		defer ts.Close()
//
func (a *AsyncHttpManager) HandlerStatus(w http.ResponseWriter, r *http.Request) {
	// Get the id from the URL
	vars := mux.Vars(r)
	id := vars["id"]

	a.lock.Lock()
	defer a.lock.Unlock()

	// Check the id is in the map
	if handler, ok := a.handlers[id]; ok {

		if handler.completed {
			if handler.err != nil {

				// Return 500 status
				http.Error(w, handler.err.Error(), http.StatusInternalServerError)
			} else {
				if handler.location != "" {

					// Redirect to new location
					http.Redirect(w, r, handler.location, http.StatusSeeOther)
				} else {

					// Return 204 status
					w.WriteHeader(http.StatusNoContent)
				}
			}

			// It has been completed, we can now remove it from the map
			delete(a.handlers, id)
		} else {
			// Still pending
			// Could add a JSON body here later
			w.Header().Add("X-Pending", "true")
			w.WriteHeader(http.StatusOK)
		}

	} else {
		http.Error(w, "Id not found", http.StatusNotFound)
	}
}

// Returns the url for the specified asynchronous handler
func (h *AsyncHttpHandler) Url() string {
	h.manager.lock.RLock()
	defer h.manager.lock.RUnlock()

	return h.manager.route + "/" + h.id
}

// Registers that the handler has completed with an error
func (h *AsyncHttpHandler) CompletedWithError(err error) {

	h.manager.lock.RLock()
	defer h.manager.lock.RUnlock()

	godbc.Require(h.completed == false)

	h.err = err
	h.completed = true

	godbc.Ensure(h.completed == true)
}

// Registers that the handler has completed and has provided a location
// where information can be retreived
func (h *AsyncHttpHandler) CompletedWithLocation(location string) {

	h.manager.lock.RLock()
	defer h.manager.lock.RUnlock()

	godbc.Require(h.completed == false)

	h.location = location
	h.completed = true

	godbc.Ensure(h.completed == true)
	godbc.Ensure(h.location == location)
	godbc.Ensure(h.err == nil)
}

// Registers that the handler has completed and no data needs to be returned
func (h *AsyncHttpHandler) Completed() {

	h.manager.lock.RLock()
	defer h.manager.lock.RUnlock()

	godbc.Require(h.completed == false)

	h.completed = true

	godbc.Ensure(h.completed == true)
	godbc.Ensure(h.location == "")
	godbc.Ensure(h.err == nil)
}
<commit_msg>asynchttp: tweak formatting of import list<commit_after>//
// Copyright (c) 2015 The heketi Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package rest

import (
	"net/http"
	"sync"
	"time"

	"github.com/gorilla/mux"
	"github.com/heketi/utils"
	"github.com/lpabon/godbc"
)

var (
	logger = utils.NewLogger("[asynchttp]", utils.LEVEL_INFO)
)

// Contains information about the asynchronous operation
type AsyncHttpHandler struct {
	err          error
	completed    bool
	manager      *AsyncHttpManager
	location, id string
}

// Manager of asynchronous operations
type AsyncHttpManager struct {
	lock     sync.RWMutex
	route    string
	handlers map[string]*AsyncHttpHandler
}

// Creates a new manager
func NewAsyncHttpManager(route string) *AsyncHttpManager {
	return &AsyncHttpManager{
		route:    route,
		handlers: make(map[string]*AsyncHttpHandler),
	}
}

// Use to create a new asynchronous operation handler.
// Only use this function if you need to do every step by hand.
// It is recommended to use AsyncHttpRedirectFunc() instead
func (a *AsyncHttpManager) NewHandler() *AsyncHttpHandler {
	handler := &AsyncHttpHandler{
		manager: a,
		id:      utils.GenUUID(),
	}

	a.lock.Lock()
	defer a.lock.Unlock()

	a.handlers[handler.id] = handler

	return handler
}

// Create an asynchronous operation handler and return the appropiate
// information the caller.
// This function will call handlerfunc() in a new go routine, then
// return to the caller a HTTP status 202 setting up the `Location` header
// to point to the new asynchronous handler.
//
// If handlerfunc() returns failure, the asynchronous handler will return
// an http status of 500 and save the error string in the body.
// If handlerfunc() is successful and returns a location url path in "string",
// the asynchronous handler will return 303 (See Other) with the Location
// header set to the value returned in the string.
// If handlerfunc() is successful and returns an empty string, then the
// asynchronous handler will return 204 to the caller.
//
// Example:
//      package rest
//		import (
//			"github.com/gorilla/mux"
//          "github.com/heketi/rest"
//			"net/http"
//			"net/http/httptest"
//			"time"
//		)
//
//		// Setup asynchronous manager
//		route := "/x"
//		manager := rest.NewAsyncHttpManager(route)
//
//		// Setup the route
//		router := mux.NewRouter()
//	 	router.HandleFunc(route+"/{id}", manager.HandlerStatus).Methods("GET")
//		router.HandleFunc("/result", func(w http.ResponseWriter, r *http.Request) {
//			w.Header().Set("Content-Type", "text/plain; charset=UTF-8")
//			w.WriteHeader(http.StatusOK)
//			fmt.Fprint(w, "HelloWorld")
//		}).Methods("GET")
//
//		router.HandleFunc("/app", func(w http.ResponseWriter, r *http.Request) {
//			manager.AsyncHttpRedirectFunc(w, r, func() (string, error) {
//				time.Sleep(100 * time.Millisecond)
//				return "/result", nil
//			})
//		}).Methods("GET")
//
//		// Setup the server
//		ts := httptest.NewServer(router)
//		defer ts.Close()
//
func (a *AsyncHttpManager) AsyncHttpRedirectFunc(w http.ResponseWriter,
	r *http.Request,
	handlerfunc func() (string, error)) {

	handler := a.NewHandler()
	go func() {
		logger.Info("Started job %v", handler.id)

		ts := time.Now()
		url, err := handlerfunc()
		logger.Info("Completed job %v in %v", handler.id, time.Since(ts))

		if err != nil {
			handler.CompletedWithError(err)
		} else if url != "" {
			handler.CompletedWithLocation(url)
		} else {
			handler.Completed()
		}
	}()
	http.Redirect(w, r, handler.Url(), http.StatusAccepted)
}

// Handler for asynchronous operation status
// Register this handler with a router like Gorilla Mux
//
// Returns the following HTTP status codes
// 		200 Operation is still pending
//		404 Id requested does not exist
//		500 Operation finished and has failed.  Body will be filled in with the
//			error in plain text.
//		303 Operation finished and has setup a new location to retreive data.
//		204 Operation finished and has no data to return
//
// Example:
//      package rest
//		import (
//			"github.com/gorilla/mux"
//          "github.com/heketi/rest"
//			"net/http"
//			"net/http/httptest"
//			"time"
//		)
//
//		// Setup asynchronous manager
//		route := "/x"
//		manager := rest.NewAsyncHttpManager(route)
//
//		// Setup the route
//		router := mux.NewRouter()
//	 	router.HandleFunc(route+"/{id:[A-Fa-f0-9]+}", manager.HandlerStatus).Methods("GET")
//
//		// Setup the server
//		ts := httptest.NewServer(router)
//		defer ts.Close()
//
func (a *AsyncHttpManager) HandlerStatus(w http.ResponseWriter, r *http.Request) {
	// Get the id from the URL
	vars := mux.Vars(r)
	id := vars["id"]

	a.lock.Lock()
	defer a.lock.Unlock()

	// Check the id is in the map
	if handler, ok := a.handlers[id]; ok {

		if handler.completed {
			if handler.err != nil {

				// Return 500 status
				http.Error(w, handler.err.Error(), http.StatusInternalServerError)
			} else {
				if handler.location != "" {

					// Redirect to new location
					http.Redirect(w, r, handler.location, http.StatusSeeOther)
				} else {

					// Return 204 status
					w.WriteHeader(http.StatusNoContent)
				}
			}

			// It has been completed, we can now remove it from the map
			delete(a.handlers, id)
		} else {
			// Still pending
			// Could add a JSON body here later
			w.Header().Add("X-Pending", "true")
			w.WriteHeader(http.StatusOK)
		}

	} else {
		http.Error(w, "Id not found", http.StatusNotFound)
	}
}

// Returns the url for the specified asynchronous handler
func (h *AsyncHttpHandler) Url() string {
	h.manager.lock.RLock()
	defer h.manager.lock.RUnlock()

	return h.manager.route + "/" + h.id
}

// Registers that the handler has completed with an error
func (h *AsyncHttpHandler) CompletedWithError(err error) {

	h.manager.lock.RLock()
	defer h.manager.lock.RUnlock()

	godbc.Require(h.completed == false)

	h.err = err
	h.completed = true

	godbc.Ensure(h.completed == true)
}

// Registers that the handler has completed and has provided a location
// where information can be retreived
func (h *AsyncHttpHandler) CompletedWithLocation(location string) {

	h.manager.lock.RLock()
	defer h.manager.lock.RUnlock()

	godbc.Require(h.completed == false)

	h.location = location
	h.completed = true

	godbc.Ensure(h.completed == true)
	godbc.Ensure(h.location == location)
	godbc.Ensure(h.err == nil)
}

// Registers that the handler has completed and no data needs to be returned
func (h *AsyncHttpHandler) Completed() {

	h.manager.lock.RLock()
	defer h.manager.lock.RUnlock()

	godbc.Require(h.completed == false)

	h.completed = true

	godbc.Ensure(h.completed == true)
	godbc.Ensure(h.location == "")
	godbc.Ensure(h.err == nil)
}
<|endoftext|>


<commit_before>package terminal

import (
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/cloudfoundry/cli/cf"
	"github.com/cloudfoundry/cli/cf/configuration/core_config"
	term "github.com/cloudfoundry/cli/cf/terminal"
)

const QuietPanic = "I should not print anything"

type FakeUI struct {
	Outputs                    []string
	UncapturedOutput           []string
	WarnOutputs                []string
	Prompts                    []string
	PasswordPrompts            []string
	Inputs                     []string
	FailedWithUsage            bool
	FailedWithUsageCommandName string
	PanickedQuietly            bool
	ShowConfigurationCalled    bool

	sayMutex sync.Mutex
}

func (ui *FakeUI) PrintPaginator(rows []string, err error) {
	if err != nil {
		ui.Failed(err.Error())
		return
	}

	for _, row := range rows {
		ui.Say(row)
	}
}

func (ui *FakeUI) PrintCapturingNoOutput(message string, args ...interface{}) {
	ui.sayMutex.Lock()
	defer ui.sayMutex.Unlock()

	message = fmt.Sprintf(message, args...)
	ui.UncapturedOutput = append(ui.UncapturedOutput, strings.Split(message, "\n")...)
	return
}

func (ui *FakeUI) Say(message string, args ...interface{}) {
	ui.sayMutex.Lock()
	defer ui.sayMutex.Unlock()

	message = fmt.Sprintf(message, args...)
	ui.Outputs = append(ui.Outputs, strings.Split(message, "\n")...)
	return
}

func (ui *FakeUI) Warn(message string, args ...interface{}) {
	message = fmt.Sprintf(message, args...)
	ui.WarnOutputs = append(ui.WarnOutputs, strings.Split(message, "\n")...)
	ui.Say(message, args...)
	return
}

func (ui *FakeUI) Ask(prompt string, args ...interface{}) (answer string) {
	ui.Prompts = append(ui.Prompts, fmt.Sprintf(prompt, args...))
	if len(ui.Inputs) == 0 {
		panic("No input provided to Fake UI for prompt: " + fmt.Sprintf(prompt, args...))
	}

	answer = ui.Inputs[0]
	ui.Inputs = ui.Inputs[1:]
	return
}

func (ui *FakeUI) ConfirmDelete(modelType, modelName string) bool {
	return ui.Confirm(
		"Really delete the %s %s?%s",
		modelType,
		term.EntityNameColor(modelName),
		term.PromptColor(">"))
}

func (ui *FakeUI) ConfirmDeleteWithAssociations(modelType, modelName string) bool {
	return ui.ConfirmDelete(modelType, modelName)
}

func (ui *FakeUI) Confirm(prompt string, args ...interface{}) bool {
	response := ui.Ask(prompt, args...)
	switch strings.ToLower(response) {
	case "y", "yes":
		return true
	}
	return false
}

func (ui *FakeUI) AskForPassword(prompt string, args ...interface{}) (answer string) {
	ui.PasswordPrompts = append(ui.PasswordPrompts, fmt.Sprintf(prompt, args...))
	if len(ui.Inputs) == 0 {
		println("__________________PANIC__________________")
		println(ui.DumpOutputs())
		println(ui.DumpPrompts())
		println("_________________________________________")
		panic("No input provided to Fake UI for prompt: " + fmt.Sprintf(prompt, args...))
	}

	answer = ui.Inputs[0]
	ui.Inputs = ui.Inputs[1:]
	return
}

func (ui *FakeUI) Ok() {
	ui.Say("OK")
}

func (ui *FakeUI) Failed(message string, args ...interface{}) {
	ui.Say("FAILED")
	ui.Say(message, args...)
	panic(QuietPanic)
}

func (ui *FakeUI) PanicQuietly() {
	ui.PanickedQuietly = true
}

func (ui *FakeUI) DumpWarnOutputs() string {
	return "****************************\n" + strings.Join(ui.WarnOutputs, "\n")
}

func (ui *FakeUI) DumpOutputs() string {
	return "****************************\n" + strings.Join(ui.Outputs, "\n")
}

func (ui *FakeUI) DumpPrompts() string {
	return "****************************\n" + strings.Join(ui.Prompts, "\n")
}

func (ui *FakeUI) ClearOutputs() {
	ui.Outputs = []string{}
}

func (ui *FakeUI) ShowConfiguration(config core_config.Reader) {
	ui.ShowConfigurationCalled = true
}

func (ui *FakeUI) LoadingIndication() {
}

func (ui *FakeUI) Wait(duration time.Duration) {
	time.Sleep(duration)
}

func (ui *FakeUI) Table(headers []string) term.Table {
	return term.NewTable(ui, headers)
}

func (ui *FakeUI) NotifyUpdateIfNeeded(config core_config.Reader) {
	if !config.IsMinCliVersion(cf.Version) {
		ui.Say("Cloud Foundry API version {{.ApiVer}} requires CLI version " + config.MinCliVersion() + "  You are currently on version {{.CliVer}}. To upgrade your CLI, please visit: https://github.com/cloudfoundry/cli#downloads")
	}
}
<commit_msg>FakeUI accepts a channel for inputs<commit_after>package terminal

import (
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/cloudfoundry/cli/cf"
	"github.com/cloudfoundry/cli/cf/configuration/core_config"
	term "github.com/cloudfoundry/cli/cf/terminal"
)

const QuietPanic = "I should not print anything"

type FakeUI struct {
	Outputs                    []string
	UncapturedOutput           []string
	WarnOutputs                []string
	Prompts                    []string
	PasswordPrompts            []string
	InputsChan                 chan string
	Inputs                     []string
	FailedWithUsage            bool
	FailedWithUsageCommandName string
	PanickedQuietly            bool
	ShowConfigurationCalled    bool

	sayMutex sync.Mutex
}

func (ui *FakeUI) PrintPaginator(rows []string, err error) {
	if err != nil {
		ui.Failed(err.Error())
		return
	}

	for _, row := range rows {
		ui.Say(row)
	}
}

func (ui *FakeUI) PrintCapturingNoOutput(message string, args ...interface{}) {
	ui.sayMutex.Lock()
	defer ui.sayMutex.Unlock()

	message = fmt.Sprintf(message, args...)
	ui.UncapturedOutput = append(ui.UncapturedOutput, strings.Split(message, "\n")...)
	return
}

func (ui *FakeUI) Say(message string, args ...interface{}) {
	ui.sayMutex.Lock()
	defer ui.sayMutex.Unlock()

	message = fmt.Sprintf(message, args...)
	ui.Outputs = append(ui.Outputs, strings.Split(message, "\n")...)
	return
}

func (ui *FakeUI) Warn(message string, args ...interface{}) {
	message = fmt.Sprintf(message, args...)
	ui.WarnOutputs = append(ui.WarnOutputs, strings.Split(message, "\n")...)
	ui.Say(message, args...)
	return
}

func (ui *FakeUI) Ask(prompt string, args ...interface{}) string {
	ui.Prompts = append(ui.Prompts, fmt.Sprintf(prompt, args...))

	if ui.InputsChan == nil {
		if len(ui.Inputs) == 0 {
			panic("No input provided to Fake UI for prompt: " + fmt.Sprintf(prompt, args...))
		}

		answer := ui.Inputs[0]
		ui.Inputs = ui.Inputs[1:]
		return answer
	}

	return <-ui.InputsChan
}

func (ui *FakeUI) ConfirmDelete(modelType, modelName string) bool {
	return ui.Confirm(
		"Really delete the %s %s?%s",
		modelType,
		term.EntityNameColor(modelName),
		term.PromptColor(">"))
}

func (ui *FakeUI) ConfirmDeleteWithAssociations(modelType, modelName string) bool {
	return ui.ConfirmDelete(modelType, modelName)
}

func (ui *FakeUI) Confirm(prompt string, args ...interface{}) bool {
	response := ui.Ask(prompt, args...)
	switch strings.ToLower(response) {
	case "y", "yes":
		return true
	}
	return false
}

func (ui *FakeUI) AskForPassword(prompt string, args ...interface{}) string {
	ui.PasswordPrompts = append(ui.PasswordPrompts, fmt.Sprintf(prompt, args...))
	if ui.InputsChan == nil {
		if len(ui.Inputs) == 0 {
			panic("No input provided to Fake UI for prompt: " + fmt.Sprintf(prompt, args...))
		}

		answer := ui.Inputs[0]
		ui.Inputs = ui.Inputs[1:]
		return answer
	}

	return <-ui.InputsChan
}

func (ui *FakeUI) Ok() {
	ui.Say("OK")
}

func (ui *FakeUI) Failed(message string, args ...interface{}) {
	ui.Say("FAILED")
	ui.Say(message, args...)
	panic(QuietPanic)
}

func (ui *FakeUI) PanicQuietly() {
	ui.PanickedQuietly = true
}

func (ui *FakeUI) DumpWarnOutputs() string {
	return "****************************\n" + strings.Join(ui.WarnOutputs, "\n")
}

func (ui *FakeUI) DumpOutputs() string {
	return "****************************\n" + strings.Join(ui.Outputs, "\n")
}

func (ui *FakeUI) DumpPrompts() string {
	return "****************************\n" + strings.Join(ui.Prompts, "\n")
}

func (ui *FakeUI) ClearOutputs() {
	ui.Outputs = []string{}
}

func (ui *FakeUI) ShowConfiguration(config core_config.Reader) {
	ui.ShowConfigurationCalled = true
}

func (ui *FakeUI) LoadingIndication() {
}

func (ui *FakeUI) Wait(duration time.Duration) {
	time.Sleep(duration)
}

func (ui *FakeUI) Table(headers []string) term.Table {
	return term.NewTable(ui, headers)
}

func (ui *FakeUI) NotifyUpdateIfNeeded(config core_config.Reader) {
	if !config.IsMinCliVersion(cf.Version) {
		ui.Say("Cloud Foundry API version {{.ApiVer}} requires CLI version " + config.MinCliVersion() + "  You are currently on version {{.CliVer}}. To upgrade your CLI, please visit: https://github.com/cloudfoundry/cli#downloads")
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

package watcher

import (
	"context"
	"expvar"
	"fmt"
	"os"
	"path"
	"path/filepath"
	"sync"
	"time"

	"github.com/fsnotify/fsnotify"
	"github.com/golang/glog"
	"github.com/pkg/errors"
)

var (
	errorCount = expvar.NewInt("log_watcher_errors_total")
)

type watch struct {
	ps []Processor
	fi os.FileInfo
}

// hasChanged indicates that a FileInfo has changed.
// http://apenwarr.ca/log/20181113 suggests that comparing mtime is
// insufficient for sub-second resolution on many platforms, and we can do
// better by comparing a few fields in the FileInfo.  This set of tests is less
// than the ones suggested in the blog post, but seem sufficient for making
// tests (notably, sub-millisecond accuracy) pass quickly.  mtime-only diff has
// caused race conditions in test and likely caused strange behaviour in
// production environments.
func hasChanged(a, b os.FileInfo) bool {
	if a == nil || b == nil {
		glog.V(2).Info("One or both FileInfos are nil")
		return true
	}
	if a.ModTime() != b.ModTime() {
		glog.V(2).Info("modtimes differ")
		return true
	}
	if a.Size() != b.Size() {
		glog.V(2).Info("sizes differ")
		return true
	}
	if a.Mode() != b.Mode() {
		glog.V(2).Info("modes differ")
		return true
	}
	return false
}

// LogWatcher implements a Watcher for watching real filesystems.
type LogWatcher struct {
	watcher    *fsnotify.Watcher
	pollTicker *time.Ticker

	watchedMu sync.RWMutex // protects `watched'
	watched   map[string]*watch

	stopTicks chan struct{} // Channel to notify ticker to stop.

	ticksDone  chan struct{} // Channel to notify when the ticks handler is done.
	eventsDone chan struct{} // Channel to notify when the events handler is done.

	pollMu sync.Mutex // protects `Poll()`

	closeOnce sync.Once
}

// NewLogWatcher returns a new LogWatcher, or returns an error.
func NewLogWatcher(pollInterval time.Duration, enableFsnotify bool) (*LogWatcher, error) {
	var f *fsnotify.Watcher
	if enableFsnotify {
		var err error
		f, err = fsnotify.NewWatcher()
		if err != nil {
			glog.Warning(err)
		}
	}
	w := &LogWatcher{
		watcher: f,
		watched: make(map[string]*watch),
	}
	if pollInterval > 0 {
		w.pollTicker = time.NewTicker(pollInterval)
		w.stopTicks = make(chan struct{})
		w.ticksDone = make(chan struct{})
		go w.runTicks()
		glog.V(2).Infof("started ticker with %s interval", pollInterval)
	}
	if f != nil {
		w.eventsDone = make(chan struct{})
		go w.runEvents()
	}
	return w, nil
}

func (w *LogWatcher) sendEvent(e Event) {
	w.watchedMu.RLock()
	watch, ok := w.watched[e.Pathname]
	w.watchedMu.RUnlock()
	if !ok {
		d := filepath.Dir(e.Pathname)
		w.watchedMu.RLock()
		watch, ok = w.watched[d]
		w.watchedMu.RUnlock()
		if !ok {
			glog.V(2).Infof("No watch for path %q", e.Pathname)
			return
		}
	}
	w.sendWatchedEvent(watch, e)
}

// Send an event to a watch; all locks assumed to be held.
func (w *LogWatcher) sendWatchedEvent(watch *watch, e Event) {
	for _, p := range watch.ps {
		p.ProcessFileEvent(context.TODO(), e)
	}
}

func (w *LogWatcher) runTicks() {
	defer close(w.ticksDone)

	if w.pollTicker == nil {
		return
	}

	for {
		select {
		case <-w.pollTicker.C:
			w.Poll()
		case <-w.stopTicks:
			w.pollTicker.Stop()
			return
		}
	}
}

// Poll all watched objects for updates, dispatching events if required.
func (w *LogWatcher) Poll() {
	w.pollMu.Lock()
	defer w.pollMu.Unlock()
	glog.V(2).Info("Polling watched files.")
	w.watchedMu.RLock()
	for n, watch := range w.watched {
		w.watchedMu.RUnlock()
		w.pollWatchedPath(n, watch)
		w.watchedMu.RLock()
	}
	w.watchedMu.RUnlock()
}

// pollWatchedPathLocked polls an already-watched path for updates.
func (w *LogWatcher) pollWatchedPath(pathname string, watched *watch) {
	glog.V(2).Infof("Stat %q", pathname)
	fi, err := os.Stat(pathname)
	if err != nil {
		if os.IsNotExist(err) {
			glog.V(2).Infof("sending delete for %s", pathname)
			w.sendWatchedEvent(watched, Event{Delete, pathname})
			// Need to remove the watch for any subsequent create to be sent.
			w.watchedMu.Lock()
			delete(w.watched, pathname)
			w.watchedMu.Unlock()
		} else {
			glog.V(1).Info(err)
		}
		return
	}

	// fsnotify does not send update events for the directory itself.
	if fi.IsDir() {
		w.pollDirectory(watched, pathname)
	} else if hasChanged(fi, watched.fi) {
		glog.V(2).Infof("sending update for %s", pathname)
		w.sendWatchedEvent(watched, Event{Update, pathname})
	}

	w.watchedMu.Lock()
	if _, ok := w.watched[pathname]; ok {
		w.watched[pathname].fi = fi
	}
	w.watchedMu.Unlock()
}

func (w *LogWatcher) pollDirectory(parentWatch *watch, pathname string) {
	matches, err := filepath.Glob(path.Join(pathname, "*"))
	if err != nil {
		glog.V(1).Info(err)
		return
	}
	for _, match := range matches {
		w.watchedMu.RLock()
		_, ok := w.watched[match]
		w.watchedMu.RUnlock()
		if !ok {
			// The object has no watch object so it must be new, but we can't
			// decide that -- wait for the Tailer to match pattern and instruct
			// us to Observe it directly.
			glog.V(2).Infof("sending create for %s", match)
			w.sendWatchedEvent(parentWatch, Event{Create, match})
		}
		fi, err := os.Stat(match)
		if err != nil {
			glog.V(1).Info(err)
			continue
		}
		if fi.IsDir() {
			w.pollDirectory(parentWatch, match)
		}
	}
}

// runEvents assumes that w.watcher is not nil
func (w *LogWatcher) runEvents() {
	defer close(w.eventsDone)

	// Suck out errors and dump them to the error log.
	go func() {
		for err := range w.watcher.Errors {
			errorCount.Add(1)
			glog.Errorf("fsnotify error: %s\n", err)
		}
	}()

	for e := range w.watcher.Events {
		glog.V(2).Infof("fsnotify watcher event %v", e)
		switch {
		case e.Op&fsnotify.Create == fsnotify.Create:
			w.sendEvent(Event{Create, e.Name})
		case e.Op&fsnotify.Write == fsnotify.Write,
			e.Op&fsnotify.Chmod == fsnotify.Chmod:
			w.sendEvent(Event{Update, e.Name})
		case e.Op&fsnotify.Remove == fsnotify.Remove:
			w.sendEvent(Event{Delete, e.Name})
		case e.Op&fsnotify.Rename == fsnotify.Rename:
			// Rename is only issued on the original file path; the new name receives a Create event
			w.sendEvent(Event{Delete, e.Name})
		default:
			panic(fmt.Sprintf("unknown op type %v", e.Op))
		}
	}
	glog.Infof("Shutting down log watcher.")
}

// Close shuts down the LogWatcher.  It is safe to call this from multiple clients.
func (w *LogWatcher) Close() (err error) {
	w.closeOnce.Do(func() {
		if w.watcher != nil {
			err = w.watcher.Close()
			<-w.eventsDone
		}
		if w.pollTicker != nil {
			close(w.stopTicks)
			<-w.ticksDone
		}
	})
	return nil
}

// Observe adds a path to the list of watched items.
// If this path has a new event, then the processor being registered will be sent the event.
func (w *LogWatcher) Observe(path string, processor Processor) error {
	absPath, err := w.addWatch(path)
	if err != nil {
		return err
	}
	w.watchedMu.Lock()
	defer w.watchedMu.Unlock()
	watched, ok := w.watched[absPath]
	if !ok {
		fi, err := os.Stat(absPath)
		if err != nil {
			glog.V(1).Info(err)
		}
		w.watched[absPath] = &watch{ps: []Processor{processor}, fi: fi}
		glog.Infof("No abspath in watched list, added new one for %s", absPath)
		return nil
	}
	for _, p := range watched.ps {
		if p == processor {
			glog.Infof("Found this processor in watched list")
			return nil
		}
	}
	watched.ps = append(watched.ps, processor)
	glog.Infof("appended this processor")
	return nil
}

func (w *LogWatcher) addWatch(path string) (string, error) {
	absPath, err := filepath.Abs(path)
	if err != nil {
		return "", errors.Wrapf(err, "Failed to lookup absolutepath of %q", path)
	}
	glog.V(2).Infof("Adding a watch on resolved path %q", absPath)
	if w.watcher != nil {
		err = w.watcher.Add(absPath)
		if err != nil {
			if os.IsPermission(err) {
				glog.V(2).Infof("Skipping permission denied error on adding a watch.")
			} else {
				return "", errors.Wrapf(err, "Failed to create a new watch on %q", absPath)
			}
		}
	}
	return absPath, nil
}

// IsWatching indicates if the path is being watched. It includes both
// filenames and directories.
func (w *LogWatcher) IsWatching(path string) bool {
	absPath, err := filepath.Abs(path)
	if err != nil {
		glog.V(2).Infof("Couldn't resolve path %q: %s", absPath, err)
		return false
	}
	glog.V(2).Infof("Resolved path for lookup %q", absPath)
	w.watchedMu.RLock()
	_, ok := w.watched[absPath]
	w.watchedMu.RUnlock()
	return ok
}

func (w *LogWatcher) Unobserve(path string, processor Processor) error {
	w.watchedMu.Lock()
	defer w.watchedMu.Unlock()
	_, ok := w.watched[path]
	if !ok {
		return nil
	}

	for i, p := range w.watched[path].ps {
		if p == processor {
			w.watched[path].ps = append(w.watched[path].ps[0:i], w.watched[path].ps[i+1:]...)
			break
		}
	}
	if len(w.watched[path].ps) == 0 {
		delete(w.watched, path)
	}
	if w.watcher != nil {
		return w.watcher.Remove(path)
	}
	return nil
}
<commit_msg>Fix up a comment.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

package watcher

import (
	"context"
	"expvar"
	"fmt"
	"os"
	"path"
	"path/filepath"
	"sync"
	"time"

	"github.com/fsnotify/fsnotify"
	"github.com/golang/glog"
	"github.com/pkg/errors"
)

var (
	errorCount = expvar.NewInt("log_watcher_errors_total")
)

type watch struct {
	ps []Processor
	fi os.FileInfo
}

// hasChanged indicates that a FileInfo has changed.
// http://apenwarr.ca/log/20181113 suggests that comparing mtime is
// insufficient for sub-second resolution on many platforms, and we can do
// better by comparing a few fields in the FileInfo.  This set of tests is less
// than the ones suggested in the blog post, but seem sufficient for making
// tests (notably, sub-millisecond accuracy) pass quickly.  mtime-only diff has
// caused race conditions in test and likely caused strange behaviour in
// production environments.
func hasChanged(a, b os.FileInfo) bool {
	if a == nil || b == nil {
		glog.V(2).Info("One or both FileInfos are nil")
		return true
	}
	if a.ModTime() != b.ModTime() {
		glog.V(2).Info("modtimes differ")
		return true
	}
	if a.Size() != b.Size() {
		glog.V(2).Info("sizes differ")
		return true
	}
	if a.Mode() != b.Mode() {
		glog.V(2).Info("modes differ")
		return true
	}
	return false
}

// LogWatcher implements a Watcher for watching real filesystems.
type LogWatcher struct {
	watcher    *fsnotify.Watcher
	pollTicker *time.Ticker

	watchedMu sync.RWMutex // protects `watched'
	watched   map[string]*watch

	stopTicks chan struct{} // Channel to notify ticker to stop.

	ticksDone  chan struct{} // Channel to notify when the ticks handler is done.
	eventsDone chan struct{} // Channel to notify when the events handler is done.

	pollMu sync.Mutex // protects `Poll()`

	closeOnce sync.Once
}

// NewLogWatcher returns a new LogWatcher, or returns an error.
func NewLogWatcher(pollInterval time.Duration, enableFsnotify bool) (*LogWatcher, error) {
	var f *fsnotify.Watcher
	if enableFsnotify {
		var err error
		f, err = fsnotify.NewWatcher()
		if err != nil {
			glog.Warning(err)
		}
	}
	w := &LogWatcher{
		watcher: f,
		watched: make(map[string]*watch),
	}
	if pollInterval > 0 {
		w.pollTicker = time.NewTicker(pollInterval)
		w.stopTicks = make(chan struct{})
		w.ticksDone = make(chan struct{})
		go w.runTicks()
		glog.V(2).Infof("started ticker with %s interval", pollInterval)
	}
	if f != nil {
		w.eventsDone = make(chan struct{})
		go w.runEvents()
	}
	return w, nil
}

func (w *LogWatcher) sendEvent(e Event) {
	w.watchedMu.RLock()
	watch, ok := w.watched[e.Pathname]
	w.watchedMu.RUnlock()
	if !ok {
		d := filepath.Dir(e.Pathname)
		w.watchedMu.RLock()
		watch, ok = w.watched[d]
		w.watchedMu.RUnlock()
		if !ok {
			glog.V(2).Infof("No watch for path %q", e.Pathname)
			return
		}
	}
	w.sendWatchedEvent(watch, e)
}

// Send an event to a watch; all locks assumed to be held.
func (w *LogWatcher) sendWatchedEvent(watch *watch, e Event) {
	for _, p := range watch.ps {
		p.ProcessFileEvent(context.TODO(), e)
	}
}

func (w *LogWatcher) runTicks() {
	defer close(w.ticksDone)

	if w.pollTicker == nil {
		return
	}

	for {
		select {
		case <-w.pollTicker.C:
			w.Poll()
		case <-w.stopTicks:
			w.pollTicker.Stop()
			return
		}
	}
}

// Poll all watched objects for updates, dispatching events if required.
func (w *LogWatcher) Poll() {
	w.pollMu.Lock()
	defer w.pollMu.Unlock()
	glog.V(2).Info("Polling watched files.")
	w.watchedMu.RLock()
	for n, watch := range w.watched {
		w.watchedMu.RUnlock()
		w.pollWatchedPath(n, watch)
		w.watchedMu.RLock()
	}
	w.watchedMu.RUnlock()
}

// pollWatchedPathLocked polls an already-watched path for updates.
func (w *LogWatcher) pollWatchedPath(pathname string, watched *watch) {
	glog.V(2).Infof("Stat %q", pathname)
	fi, err := os.Stat(pathname)
	if err != nil {
		if os.IsNotExist(err) {
			glog.V(2).Infof("sending delete for %s", pathname)
			w.sendWatchedEvent(watched, Event{Delete, pathname})
			// Need to remove the watch for any subsequent create to be sent.
			w.watchedMu.Lock()
			delete(w.watched, pathname)
			w.watchedMu.Unlock()
		} else {
			glog.V(1).Info(err)
		}
		return
	}

	// fsnotify does not send update events for the directory itself.
	if fi.IsDir() {
		w.pollDirectory(watched, pathname)
	} else if hasChanged(fi, watched.fi) {
		glog.V(2).Infof("sending update for %s", pathname)
		w.sendWatchedEvent(watched, Event{Update, pathname})
	}

	w.watchedMu.Lock()
	if _, ok := w.watched[pathname]; ok {
		w.watched[pathname].fi = fi
	}
	w.watchedMu.Unlock()
}

// pollDirectory walks the directory tree for a parent watch, and notifies of any new files.
func (w *LogWatcher) pollDirectory(parentWatch *watch, pathname string) {
	matches, err := filepath.Glob(path.Join(pathname, "*"))
	if err != nil {
		glog.V(1).Info(err)
		return
	}
	for _, match := range matches {
		w.watchedMu.RLock()
		_, ok := w.watched[match]
		w.watchedMu.RUnlock()
		if !ok {
			// The object has no watch object so it must be new, but we can't
			// decide to watch it yet -- wait for the Tailer to match pattern
			// and instruct us to Observe it directly.  Technically not
			// everything is created here, it's literally everything in a path
			// that we aren't watching, so we make a lot of stats below, but we
			// need to find which ones are directories so we can traverse them.
			// TODO(jaq): teach log watcher about the TailPatterns from tailer.
			glog.V(2).Infof("sending create for %s", match)
			w.sendWatchedEvent(parentWatch, Event{Create, match})
		}
		fi, err := os.Stat(match)
		if err != nil {
			glog.V(1).Info(err)
			continue
		}
		if fi.IsDir() {
			w.pollDirectory(parentWatch, match)
		}
	}
}

// runEvents assumes that w.watcher is not nil
func (w *LogWatcher) runEvents() {
	defer close(w.eventsDone)

	// Suck out errors and dump them to the error log.
	go func() {
		for err := range w.watcher.Errors {
			errorCount.Add(1)
			glog.Errorf("fsnotify error: %s\n", err)
		}
	}()

	for e := range w.watcher.Events {
		glog.V(2).Infof("fsnotify watcher event %v", e)
		switch {
		case e.Op&fsnotify.Create == fsnotify.Create:
			w.sendEvent(Event{Create, e.Name})
		case e.Op&fsnotify.Write == fsnotify.Write,
			e.Op&fsnotify.Chmod == fsnotify.Chmod:
			w.sendEvent(Event{Update, e.Name})
		case e.Op&fsnotify.Remove == fsnotify.Remove:
			w.sendEvent(Event{Delete, e.Name})
		case e.Op&fsnotify.Rename == fsnotify.Rename:
			// Rename is only issued on the original file path; the new name receives a Create event
			w.sendEvent(Event{Delete, e.Name})
		default:
			panic(fmt.Sprintf("unknown op type %v", e.Op))
		}
	}
	glog.Infof("Shutting down log watcher.")
}

// Close shuts down the LogWatcher.  It is safe to call this from multiple clients.
func (w *LogWatcher) Close() (err error) {
	w.closeOnce.Do(func() {
		if w.watcher != nil {
			err = w.watcher.Close()
			<-w.eventsDone
		}
		if w.pollTicker != nil {
			close(w.stopTicks)
			<-w.ticksDone
		}
	})
	return nil
}

// Observe adds a path to the list of watched items.
// If this path has a new event, then the processor being registered will be sent the event.
func (w *LogWatcher) Observe(path string, processor Processor) error {
	absPath, err := w.addWatch(path)
	if err != nil {
		return err
	}
	w.watchedMu.Lock()
	defer w.watchedMu.Unlock()
	watched, ok := w.watched[absPath]
	if !ok {
		fi, err := os.Stat(absPath)
		if err != nil {
			glog.V(1).Info(err)
		}
		w.watched[absPath] = &watch{ps: []Processor{processor}, fi: fi}
		glog.Infof("No abspath in watched list, added new one for %s", absPath)
		return nil
	}
	for _, p := range watched.ps {
		if p == processor {
			glog.Infof("Found this processor in watched list")
			return nil
		}
	}
	watched.ps = append(watched.ps, processor)
	glog.Infof("appended this processor")
	return nil
}

func (w *LogWatcher) addWatch(path string) (string, error) {
	absPath, err := filepath.Abs(path)
	if err != nil {
		return "", errors.Wrapf(err, "Failed to lookup absolutepath of %q", path)
	}
	glog.V(2).Infof("Adding a watch on resolved path %q", absPath)
	if w.watcher != nil {
		err = w.watcher.Add(absPath)
		if err != nil {
			if os.IsPermission(err) {
				glog.V(2).Infof("Skipping permission denied error on adding a watch.")
			} else {
				return "", errors.Wrapf(err, "Failed to create a new watch on %q", absPath)
			}
		}
	}
	return absPath, nil
}

// IsWatching indicates if the path is being watched. It includes both
// filenames and directories.
func (w *LogWatcher) IsWatching(path string) bool {
	absPath, err := filepath.Abs(path)
	if err != nil {
		glog.V(2).Infof("Couldn't resolve path %q: %s", absPath, err)
		return false
	}
	glog.V(2).Infof("Resolved path for lookup %q", absPath)
	w.watchedMu.RLock()
	_, ok := w.watched[absPath]
	w.watchedMu.RUnlock()
	return ok
}

func (w *LogWatcher) Unobserve(path string, processor Processor) error {
	w.watchedMu.Lock()
	defer w.watchedMu.Unlock()
	_, ok := w.watched[path]
	if !ok {
		return nil
	}

	for i, p := range w.watched[path].ps {
		if p == processor {
			w.watched[path].ps = append(w.watched[path].ps[0:i], w.watched[path].ps[i+1:]...)
			break
		}
	}
	if len(w.watched[path].ps) == 0 {
		delete(w.watched, path)
	}
	if w.watcher != nil {
		return w.watcher.Remove(path)
	}
	return nil
}
<|endoftext|>


<commit_before>// Package implements creation of XLSX simple spreadsheet files

package xlsx

import (
	"archive/zip"
	"bufio"
	"bytes"
	"fmt"
	"html"
	"io"
	"os"
	"strconv"
	"time"
)

type CellType uint

// Basic spreadsheet cell types
const (
	CellTypeNumber CellType = iota
	CellTypeString
	CellTypeDatetime
)

// XLSX Spreadsheet Cell
type Cell struct {
	Type  CellType
	Value string
}

// XLSX Spreadsheet Row
type Row struct {
	Cells []Cell
}

// XLSX Spreadsheet Column
type Column struct {
	Name  string
	Width uint64
}

// XLSX Spreadsheet Document Properties
type DocumentInfo struct {
	CreatedBy  string
	ModifiedBy string
	CreatedAt  time.Time
	ModifiedAt time.Time
}

// XLSX Spreadsheet
type Sheet struct {
	Title           string
	columns         []Column
	rows            []Row
	sharedStringMap map[string]int
	sharedStrings   []string
	DocumentInfo    DocumentInfo
}

// Create a sheet with no dimensions
func NewSheet(title string) Sheet {
	c := make([]Column, 0)
	r := make([]Row, 0)
	ssm := make(map[string]int)
	sst := make([]string, 0)

	s := Sheet{
		Title:           title,
		columns:         c,
		rows:            r,
		sharedStringMap: ssm,
		sharedStrings:   sst,
	}

	return s
}

// Create a sheet with dimensions derived from the given columns
func NewSheetWithColumns(c []Column, title string) Sheet {
	r := make([]Row, 0)
	ssm := make(map[string]int)
	sst := make([]string, 0)

	s := Sheet{
		Title:           title,
		columns:         c,
		rows:            r,
		sharedStringMap: ssm,
		sharedStrings:   sst,
	}

	s.DocumentInfo.CreatedBy = "xlsx.go"
	s.DocumentInfo.CreatedAt = time.Now()

	s.DocumentInfo.ModifiedBy = s.DocumentInfo.CreatedBy
	s.DocumentInfo.ModifiedAt = s.DocumentInfo.CreatedAt

	return s
}

// Create a new row with a length caculated by the sheets known column count
func (s *Sheet) NewRow() Row {
	c := make([]Cell, len(s.columns))
	r := Row{
		Cells: c,
	}
	return r
}

// Append a row to the sheet
func (s *Sheet) AppendRow(r Row) error {
	if len(r.Cells) != len(s.columns) {
		return fmt.Errorf("the given row has %d cells and %d were expected", len(r.Cells), len(s.columns))
	}

	cells := make([]Cell, len(s.columns))

	for n, c := range r.Cells {
		cells[n].Type = c.Type
		cells[n].Value = c.Value

		if cells[n].Type == CellTypeString {
			// calculate string reference
			cells[n].Value = html.EscapeString(cells[n].Value)
			i, exists := s.sharedStringMap[cells[n].Value]
			if !exists {
				i = len(s.sharedStrings)
				s.sharedStringMap[cells[n].Value] = i
				s.sharedStrings = append(s.sharedStrings, cells[n].Value)
			}
			cells[n].Value = strconv.Itoa(i)
		} else if cells[n].Type == CellTypeDatetime {
			d, err := time.Parse(time.RFC3339, cells[n].Value)
			if err == nil {
				cells[n].Value = OADate(d)
			}
		}
	}

	row := s.NewRow()
	row.Cells = cells

	s.rows = append(s.rows, row)

	return nil
}

// Get the Shared Strings in the order they were added to the map
func (s *Sheet) SharedStrings() []string {
	return s.sharedStrings
}

// Given zero-based array indices output the Excel cell reference. For
// example (0,0) => "A1"; (2,2) => "C3"; (26,45) => "AA46"
func CellIndex(x, y uint64) string {
	return fmt.Sprintf("%s%d", colName(x), y+1)
}

// From a zero-based column number return the Excel column name.
// For example: 0 => "A"; 2 => "C"; 26 => "AA"
func colName(n uint64) string {
	var s string
	n += 1

	for n > 0 {
		n -= 1
		s = fmt.Sprintf("%s%s", string(65+(n%26)), s)
		n /= 26
	}

	return s
}

// Convert time to the OLE Automation format.
func OADate(d time.Time) string {
	epoch := time.Date(1899, 12, 30, 0, 0, 0, 0, time.UTC)
	nsPerDay := 24 * time.Hour

	v := -1 * float64(epoch.Sub(d)) / float64(nsPerDay)

	// TODO: deal with dates before epoch
	// e.g. http://stackoverflow.com/questions/15549823/oadate-to-milliseconds-timestamp-in-javascript/15550284#15550284

	if d.Hour() == 0 && d.Minute() == 0 && d.Second() == 0 {
		return fmt.Sprintf("%d", int64(v))
	} else {
		return fmt.Sprintf("%f", v)
	}
}

// Create filename and save the XLSX file
func (s *Sheet) SaveToFile(filename string) error {
	outputfile, err := os.Create(filename)
	if err != nil {
		return err
	}
	w := bufio.NewWriter(outputfile)
	err = s.SaveToWriter(w)
	defer w.Flush()
	return err
}

func (sw *SheetWriter) WriteRows(rows []Row) error {

	var err error

	for i, r := range rows {
		rb := &bytes.Buffer{}

		if sw.maxNCols < uint64(len(r.Cells)) {
			sw.maxNCols = uint64(len(r.Cells))
		}

		for j, c := range r.Cells {

			cell := struct {
				CellIndex string
				Value     string
			}{
				CellIndex: CellIndex(uint64(j), uint64(i)+sw.currentIndex),
				Value:     c.Value,
			}

			switch c.Type {
			case CellTypeString:
				err = TemplateCellString.Execute(rb, cell)
			case CellTypeNumber:
				err = TemplateCellNumber.Execute(rb, cell)
			case CellTypeDatetime:
				err = TemplateCellDateTime.Execute(rb, cell)
			}

			if err != nil {
				return err
			}
		}

		rowString := fmt.Sprintf(`<row r="%d">%s</row>`, uint64(i)+sw.currentIndex+1, rb.String())

		_, err = io.WriteString(sw.f, rowString)
		if err != nil {
			return err
		}

	}

	sw.currentIndex += uint64(len(rows))

	return nil
}

// Save the XLSX file to the given writer
func (s *Sheet) SaveToWriter(w io.Writer) error {

	ww := NewWorkbookWriter(w)

	err := ww.WriteHeader(s)
	if err != nil {
		return err
	}

	sw := ww.NewSheetWriter(s.Title)

	sw.Write(s)
	sw.WriteRows(s.rows)
	sw.Close()

	err = ww.Close()
	if err != nil {
		return err
	}

	return nil
}

func (ww *WorkbookWriter) WriteHeader(s *Sheet) error {

	z := ww.zipWriter

	f, err := z.Create("[Content_Types].xml")
	err = TemplateContentTypes.Execute(f, nil)
	if err != nil {
		return err
	}

	f, err = z.Create("docProps/app.xml")
	err = TemplateApp.Execute(f, s)
	if err != nil {
		return err
	}

	f, err = z.Create("docProps/core.xml")
	err = TemplateCore.Execute(f, s.DocumentInfo)
	if err != nil {
		return err
	}

	f, err = z.Create("_rels/.rels")
	err = TemplateRelationships.Execute(f, nil)
	if err != nil {
		return err
	}

	f, err = z.Create("xl/workbook.xml")
	err = TemplateWorkbook.Execute(f, s)
	if err != nil {
		return err
	}

	f, err = z.Create("xl/_rels/workbook.xml.rels")
	err = TemplateWorkbookRelationships.Execute(f, nil)
	if err != nil {
		return err
	}

	f, err = z.Create("xl/styles.xml")
	err = TemplateStyles.Execute(f, nil)
	if err != nil {
		return err
	}

	f, err = z.Create("xl/sharedStrings.xml")
	err = TemplateStringLookups.Execute(f, s.SharedStrings())
	if err != nil {
		return err
	}

	return nil
}

type WorkbookWriter struct {
	zipWriter *zip.Writer
}

func NewWorkbookWriter(w io.Writer) *WorkbookWriter {
	return &WorkbookWriter{zip.NewWriter(w)}
}

func (ww *WorkbookWriter) Close() error {
	return ww.zipWriter.Close()
}

func (ww *WorkbookWriter) NewSheetWriter(title string) *SheetWriter {
	f, err := ww.zipWriter.Create("xl/worksheets/" + "sheet1" + ".xml")
	return &SheetWriter{f, err, 0, 0}
}

type SheetWriter struct {
	f            io.Writer
	err          error
	currentIndex uint64
	maxNCols     uint64
}

func (sw *SheetWriter) Close() error {
	sheet := struct {
		Start string
		End   string
	}{
		Start: "A1",
		End:   CellIndex(sw.maxNCols-1, sw.currentIndex-1),
	}

	err := TemplateSheetEnd.Execute(sw.f, sheet)
	return err
}

func (sw *SheetWriter) Write(s *Sheet) error {
	sheet := struct {
		Cols []Column
	}{
		Cols: s.columns,
	}

	err := TemplateSheetStart.Execute(sw.f, sheet)
	return err
}
<commit_msg>Made it so that the API isn't broken<commit_after>// Package implements creation of XLSX simple spreadsheet files

package xlsx

import (
	"archive/zip"
	"bufio"
	"bytes"
	"fmt"
	"html"
	"io"
	"os"
	"strconv"
	"time"
)

type CellType uint

// Basic spreadsheet cell types
const (
	CellTypeNumber CellType = iota
	CellTypeString
	CellTypeDatetime
)

// XLSX Spreadsheet Cell
type Cell struct {
	Type  CellType
	Value string
}

// XLSX Spreadsheet Row
type Row struct {
	Cells []Cell
}

// XLSX Spreadsheet Column
type Column struct {
	Name  string
	Width uint64
}

// XLSX Spreadsheet Document Properties
type DocumentInfo struct {
	CreatedBy  string
	ModifiedBy string
	CreatedAt  time.Time
	ModifiedAt time.Time
}

// XLSX Spreadsheet
type Sheet struct {
	Title           string
	columns         []Column
	rows            []Row
	sharedStringMap map[string]int
	sharedStrings   []string
	DocumentInfo    DocumentInfo
}

// Create a sheet with no dimensions
func NewSheet() Sheet {
	c := make([]Column, 0)
	r := make([]Row, 0)
	ssm := make(map[string]int)
	sst := make([]string, 0)

	s := Sheet{
		Title:           "Data",
		columns:         c,
		rows:            r,
		sharedStringMap: ssm,
		sharedStrings:   sst,
	}

	return s
}

// Create a sheet with dimensions derived from the given columns
func NewSheetWithColumns(c []Column) Sheet {
	r := make([]Row, 0)
	ssm := make(map[string]int)
	sst := make([]string, 0)

	s := Sheet{
		Title:           "Data",
		columns:         c,
		rows:            r,
		sharedStringMap: ssm,
		sharedStrings:   sst,
	}

	s.DocumentInfo.CreatedBy = "xlsx.go"
	s.DocumentInfo.CreatedAt = time.Now()

	s.DocumentInfo.ModifiedBy = s.DocumentInfo.CreatedBy
	s.DocumentInfo.ModifiedAt = s.DocumentInfo.CreatedAt

	return s
}

// Create a new row with a length caculated by the sheets known column count
func (s *Sheet) NewRow() Row {
	c := make([]Cell, len(s.columns))
	r := Row{
		Cells: c,
	}
	return r
}

// Append a row to the sheet
func (s *Sheet) AppendRow(r Row) error {
	if len(r.Cells) != len(s.columns) {
		return fmt.Errorf("the given row has %d cells and %d were expected", len(r.Cells), len(s.columns))
	}

	cells := make([]Cell, len(s.columns))

	for n, c := range r.Cells {
		cells[n].Type = c.Type
		cells[n].Value = c.Value

		if cells[n].Type == CellTypeString {
			// calculate string reference
			cells[n].Value = html.EscapeString(cells[n].Value)
			i, exists := s.sharedStringMap[cells[n].Value]
			if !exists {
				i = len(s.sharedStrings)
				s.sharedStringMap[cells[n].Value] = i
				s.sharedStrings = append(s.sharedStrings, cells[n].Value)
			}
			cells[n].Value = strconv.Itoa(i)
		} else if cells[n].Type == CellTypeDatetime {
			d, err := time.Parse(time.RFC3339, cells[n].Value)
			if err == nil {
				cells[n].Value = OADate(d)
			}
		}
	}

	row := s.NewRow()
	row.Cells = cells

	s.rows = append(s.rows, row)

	return nil
}

// Get the Shared Strings in the order they were added to the map
func (s *Sheet) SharedStrings() []string {
	return s.sharedStrings
}

// Given zero-based array indices output the Excel cell reference. For
// example (0,0) => "A1"; (2,2) => "C3"; (26,45) => "AA46"
func CellIndex(x, y uint64) string {
	return fmt.Sprintf("%s%d", colName(x), y+1)
}

// From a zero-based column number return the Excel column name.
// For example: 0 => "A"; 2 => "C"; 26 => "AA"
func colName(n uint64) string {
	var s string
	n += 1

	for n > 0 {
		n -= 1
		s = fmt.Sprintf("%s%s", string(65+(n%26)), s)
		n /= 26
	}

	return s
}

// Convert time to the OLE Automation format.
func OADate(d time.Time) string {
	epoch := time.Date(1899, 12, 30, 0, 0, 0, 0, time.UTC)
	nsPerDay := 24 * time.Hour

	v := -1 * float64(epoch.Sub(d)) / float64(nsPerDay)

	// TODO: deal with dates before epoch
	// e.g. http://stackoverflow.com/questions/15549823/oadate-to-milliseconds-timestamp-in-javascript/15550284#15550284

	if d.Hour() == 0 && d.Minute() == 0 && d.Second() == 0 {
		return fmt.Sprintf("%d", int64(v))
	} else {
		return fmt.Sprintf("%f", v)
	}
}

// Create filename and save the XLSX file
func (s *Sheet) SaveToFile(filename string) error {
	outputfile, err := os.Create(filename)
	if err != nil {
		return err
	}
	w := bufio.NewWriter(outputfile)
	err = s.SaveToWriter(w)
	defer w.Flush()
	return err
}

func (sw *SheetWriter) WriteRows(rows []Row) error {

	var err error

	for i, r := range rows {
		rb := &bytes.Buffer{}

		if sw.maxNCols < uint64(len(r.Cells)) {
			sw.maxNCols = uint64(len(r.Cells))
		}

		for j, c := range r.Cells {

			cell := struct {
				CellIndex string
				Value     string
			}{
				CellIndex: CellIndex(uint64(j), uint64(i)+sw.currentIndex),
				Value:     c.Value,
			}

			switch c.Type {
			case CellTypeString:
				err = TemplateCellString.Execute(rb, cell)
			case CellTypeNumber:
				err = TemplateCellNumber.Execute(rb, cell)
			case CellTypeDatetime:
				err = TemplateCellDateTime.Execute(rb, cell)
			}

			if err != nil {
				return err
			}
		}

		rowString := fmt.Sprintf(`<row r="%d">%s</row>`, uint64(i)+sw.currentIndex+1, rb.String())

		_, err = io.WriteString(sw.f, rowString)
		if err != nil {
			return err
		}

	}

	sw.currentIndex += uint64(len(rows))

	return nil
}

// Save the XLSX file to the given writer
func (s *Sheet) SaveToWriter(w io.Writer) error {

	ww := NewWorkbookWriter(w)

	err := ww.WriteHeader(s)
	if err != nil {
		return err
	}

	sw := ww.NewSheetWriter(s.Title)

	sw.Write(s)
	sw.WriteRows(s.rows)
	sw.Close()

	err = ww.Close()
	if err != nil {
		return err
	}

	return nil
}

func (ww *WorkbookWriter) WriteHeader(s *Sheet) error {

	z := ww.zipWriter

	f, err := z.Create("[Content_Types].xml")
	err = TemplateContentTypes.Execute(f, nil)
	if err != nil {
		return err
	}

	f, err = z.Create("docProps/app.xml")
	err = TemplateApp.Execute(f, s)
	if err != nil {
		return err
	}

	f, err = z.Create("docProps/core.xml")
	err = TemplateCore.Execute(f, s.DocumentInfo)
	if err != nil {
		return err
	}

	f, err = z.Create("_rels/.rels")
	err = TemplateRelationships.Execute(f, nil)
	if err != nil {
		return err
	}

	f, err = z.Create("xl/workbook.xml")
	err = TemplateWorkbook.Execute(f, s)
	if err != nil {
		return err
	}

	f, err = z.Create("xl/_rels/workbook.xml.rels")
	err = TemplateWorkbookRelationships.Execute(f, nil)
	if err != nil {
		return err
	}

	f, err = z.Create("xl/styles.xml")
	err = TemplateStyles.Execute(f, nil)
	if err != nil {
		return err
	}

	f, err = z.Create("xl/sharedStrings.xml")
	err = TemplateStringLookups.Execute(f, s.SharedStrings())
	if err != nil {
		return err
	}

	return nil
}

type WorkbookWriter struct {
	zipWriter *zip.Writer
}

func NewWorkbookWriter(w io.Writer) *WorkbookWriter {
	return &WorkbookWriter{zip.NewWriter(w)}
}

func (ww *WorkbookWriter) Close() error {
	return ww.zipWriter.Close()
}

func (ww *WorkbookWriter) NewSheetWriter(title string) *SheetWriter {
	f, err := ww.zipWriter.Create("xl/worksheets/" + "sheet1" + ".xml")
	return &SheetWriter{f, err, 0, 0}
}

type SheetWriter struct {
	f            io.Writer
	err          error
	currentIndex uint64
	maxNCols     uint64
}

func (sw *SheetWriter) Close() error {
	sheet := struct {
		Start string
		End   string
	}{
		Start: "A1",
		End:   CellIndex(sw.maxNCols-1, sw.currentIndex-1),
	}

	err := TemplateSheetEnd.Execute(sw.f, sheet)
	return err
}

func (sw *SheetWriter) Write(s *Sheet) error {
	sheet := struct {
		Cols []Column
	}{
		Cols: s.columns,
	}

	err := TemplateSheetStart.Execute(sw.f, sheet)
	return err
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"net/http"
	"os"

	"github.com/orange-cloudfoundry/custom_exporter/collector"
	"github.com/orange-cloudfoundry/custom_exporter/config"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/common/log"
	"github.com/prometheus/common/version"
)

/*
Copyright 2017 Orange

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

var ArgsRequire []string
var ArgsSeen map[string]bool

var showVersion = flag.Bool(
	"version",
	false,
	"Print version information.",
)

var listenAddress = flag.String(
	"web.listen-address",
	":9213",
	"Address to listen on for web interface and telemetry.",
)

var metricPath = flag.String(
	"web.telemetry-path",
	"/metrics",
	"Path under which to expose metrics.",
)

var configFile = flag.String(
	"collector.config",
	"",
	"Path to config.yml file to read custom exporter definition.",
)

func init() {
	ArgsRequire = []string{
		"collector.config",
	}

	ArgsSeen = make(map[string]bool, 0)

	prometheus.MustRegister(version.NewCollector(config.Namespace + "_" + config.Exporter))
}

func main() {
	fmt.Fprintln(os.Stdout, version.Info())
	fmt.Fprintln(os.Stdout, version.BuildContext())

	if *showVersion {
		os.Exit(0)
	}

	if ok := checkRequireArgs(); !ok {
		os.Exit(2)
	}

	if _, err := os.Stat(*configFile); err != nil {
		log.Errorln("Error:", err.Error())
		os.Exit(2)
	}

	var myConfig *config.Config

	if cnf, err := config.NewConfig(*configFile); err != nil {
		log.Fatalf("FATAL: %s", err.Error())
	} else {
		myConfig = cnf
	}

	prometheus.MustRegister(createListCollectors(myConfig)...)

	http.Handle(*metricPath, prometheus.Handler())
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`<html><head><title>Custom exporter</title></head><body><h1>Custom exporter</h1><p><a href='` + *metricPath + `'>Metrics</a></p></body></html>`))
	})

	log.Infoln("Listening on", *listenAddress)
	log.Fatal(http.ListenAndServe(*listenAddress, nil))
}

func checkRequireArgs() bool {
	var res bool

	res = true

	flag.Parse()
	flag.Visit(func(f *flag.Flag) { ArgsSeen[f.Name] = true })

	for _, req := range ArgsRequire {
		if !ArgsSeen[req] {
			fmt.Fprintf(os.Stderr, "missing required -%s argument/flag\n", req)
			res = false
		}
	}

	if !res {
		fmt.Fprintf(os.Stdout, "")
		fmt.Fprintf(os.Stdout, "")
		flag.Usage()
	}

	return res
}

func createListCollectors(c *config.Config) []prometheus.Collector {
	var result []prometheus.Collector

	for _, cnf := range c.Metrics {
		if col := createNewCollector(&cnf); col != nil {
			result = append(result, col)
		}
	}

	if len(result) < 1 {
		log.Fatalf("Error : the metrics list is empty !!")
	}

	return result
}

func createNewCollector(m *config.MetricsItem) prometheus.Collector {
	var col prometheus.Collector
	var err error

	switch m.Credential.Collector {
	case "bash":
		col, err = collector.NewPrometheusBashCollector(*m)
	case "mysql":
		col, err = collector.NewPrometheusMysqlCollector(*m)
	case "redis":
		col, err = collector.NewPrometheusRedisCollector(*m)
	default:
		return nil
	}

	if err != nil {
		log.Errorf("Error:", err.Error())
		return nil
	}

	return col
}
<commit_msg>fix error with mandatory flag/args<commit_after>package main

import (
	"flag"
	"fmt"
	"net/http"
	"os"

	"github.com/orange-cloudfoundry/custom_exporter/collector"
	"github.com/orange-cloudfoundry/custom_exporter/config"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/common/log"
	"github.com/prometheus/common/version"
)

/*
Copyright 2017 Orange

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

var ArgsRequire []string
var ArgsSeen map[string]bool

var showVersion = flag.Bool(
	"version",
	false,
	"Print version information.",
)

var listenAddress = flag.String(
	"web.listen-address",
	":9213",
	"Address to listen on for web interface and telemetry.",
)

var metricPath = flag.String(
	"web.telemetry-path",
	"/metrics",
	"Path under which to expose metrics.",
)

var configFile = flag.String(
	"collector.config",
	"",
	"Path to config.yml file to read custom exporter definition.",
)

func init() {
	ArgsRequire = []string{
		"collector.config",
	}

	ArgsSeen = make(map[string]bool, 0)

	prometheus.MustRegister(version.NewCollector(config.Namespace + "_" + config.Exporter))
}

func main() {
	fmt.Fprintln(os.Stdout, version.Info())
	fmt.Fprintln(os.Stdout, version.BuildContext())

	flag.Parse()

	if *showVersion {
		os.Exit(0)
	}

	if ok := checkRequireArgs(); !ok {
		os.Exit(2)
	}

	if _, err := os.Stat(*configFile); err != nil {
		log.Errorln("Error:", err.Error())
		os.Exit(2)
	}

	var myConfig *config.Config

	if cnf, err := config.NewConfig(*configFile); err != nil {
		log.Fatalf("FATAL: %s", err.Error())
	} else {
		myConfig = cnf
	}

	prometheus.MustRegister(createListCollectors(myConfig)...)

	http.Handle(*metricPath, prometheus.Handler())
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`<html><head><title>Custom exporter</title></head><body><h1>Custom exporter</h1><p><a href='` + *metricPath + `'>Metrics</a></p></body></html>`))
	})

	log.Infoln("Listening on", *listenAddress)
	log.Fatal(http.ListenAndServe(*listenAddress, nil))
}

func checkRequireArgs() bool {
	var res bool

	res = true

	flag.Visit(func(f *flag.Flag) { ArgsSeen[f.Name] = true })

	for _, req := range ArgsRequire {
		if !ArgsSeen[req] {
			fmt.Fprintf(os.Stderr, "missing required -%s argument/flag\n", req)
			res = false
		}
	}

	if !res {
		fmt.Fprintf(os.Stdout, "")
		fmt.Fprintf(os.Stdout, "")
		flag.Usage()
	}

	return res
}

func createListCollectors(c *config.Config) []prometheus.Collector {
	var result []prometheus.Collector

	for _, cnf := range c.Metrics {
		if col := createNewCollector(&cnf); col != nil {
			result = append(result, col)
		}
	}

	if len(result) < 1 {
		log.Fatalf("Error : the metrics list is empty !!")
	}

	return result
}

func createNewCollector(m *config.MetricsItem) prometheus.Collector {
	var col prometheus.Collector
	var err error

	switch m.Credential.Collector {
	case "bash":
		col, err = collector.NewPrometheusBashCollector(*m)
	case "mysql":
		col, err = collector.NewPrometheusMysqlCollector(*m)
	case "redis":
		col, err = collector.NewPrometheusRedisCollector(*m)
	default:
		return nil
	}

	if err != nil {
		log.Errorf("Error:", err.Error())
		return nil
	}

	return col
}
<|endoftext|>