<commit_before>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package testing

import (
	"bufio"
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	stdtesting "testing"
	"time"

	"github.com/juju/loggo"
	"labix.org/v2/mgo"
	gc "launchpad.net/gocheck"

	"launchpad.net/juju-core/cert"
	"launchpad.net/juju-core/log"
	"launchpad.net/juju-core/utils"
)

var (
	// MgoServer is a shared mongo server used by tests.
	MgoServer = &MgoInstance{ssl: true}
	logger = loggo.GetLogger("juju.testing")
)

type MgoInstance struct {
	// addr holds the address of the MongoDB server
	addr string

	// MgoPort holds the port of the MongoDB server.
	port int

	// server holds the running MongoDB command.
	server *exec.Cmd

	// exited receives a value when the mongodb server exits.
	exited <-chan struct{}

	// dir holds the directory that MongoDB is running in.
	dir string

	// ssl determines whether the MongoDB server will use TLS
	ssl bool

	// Params is a list of additional parameters that will be passed to
	// the mongod application
	Params []string
}

// Addr returns the address of the MongoDB server.
func (m *MgoInstance) Addr() string {
	return m.addr
}

// Port returns the port of the MongoDB server.
func (m *MgoInstance) Port() int {
	return m.port
}

// We specify a timeout to mgo.Dial, to prevent
// mongod failures hanging the tests.
const mgoDialTimeout = 15 * time.Second

// MgoSuite is a suite that deletes all content from the shared MongoDB
// server at the end of every test and supplies a connection to the shared
// MongoDB server.
type MgoSuite struct {
	Session *mgo.Session
}

// Start starts a MongoDB server in a temporary directory.
func (inst *MgoInstance) Start(ssl bool) error {
	dbdir, err := ioutil.TempDir("", "test-mgo")
	if err != nil {
		return err
	}

	// give them all the same keyfile so they can talk appropriately
	keyFilePath := filepath.Join(dbdir, "keyfile")
	err = ioutil.WriteFile(keyFilePath, []byte("not very secret"), 0600)
	if err != nil {
		return fmt.Errorf("cannot write key file: %v", err)
	}

	pemPath := filepath.Join(dbdir, "server.pem")
	err = ioutil.WriteFile(pemPath, []byte(ServerCert+ServerKey), 0600)
	if err != nil {
		return fmt.Errorf("cannot write cert/key PEM: %v", err)
	}
	inst.port = FindTCPPort()
	inst.addr = fmt.Sprintf("localhost:%d", inst.port)
	inst.dir = dbdir
	inst.ssl = ssl
	if err := inst.run(); err != nil {
		inst.addr = ""
		inst.port = 0
		os.RemoveAll(inst.dir)
		inst.dir = ""
	}
	return err
}

// run runs the MongoDB server at the
// address and directory already configured.
func (inst *MgoInstance) run() error {
	if inst.server != nil {
		panic("mongo server is already running")
	}
	logPath := filepath.Join(dbdir, "server.log")

	mgoport := strconv.Itoa(inst.port)
	mgoargs := []string{
		"--auth",
		"--dbpath", inst.dir,
		"--logpath", logPath,
		"--port", mgoport,
		"--nssize", "1",
		"--noprealloc",
		"--smallfiles",
		"--nojournal",
		"--nounixsocket",
		"--oplogSize", "10",
		"--keyFile", filepath.Join(inst.dir, "keyfile"),
	}
	if inst.ssl {
		mgoargs = append(mgoargs,
			"--sslOnNormalPorts",
			"--sslPEMKeyFile", filepath.Join(inst.dir, "server.pem"),
			"--sslPEMKeyPassword", "ignored")
	}
	if inst.Params != nil {
		mgoargs = append(mgoargs, inst.Params...)
	}
	server := exec.Command("mongod", mgoargs...)
	out, err := server.StdoutPipe()
	if err != nil {
		return err
	}
	server.Stderr = server.Stdout
	exited := make(chan struct{})
	started := make(chan struct{})
	go func() {
		<-started
		lines := readLines(out, 20)
		err := server.Wait()
		exitErr, _ := err.(*exec.ExitError)
		if err == nil || exitErr != nil && exitErr.Exited() {
			// mongodb has exited without being killed, so print the
			// last few lines of its log output.
			for _, line := range lines {
				log.Infof("mongod: %s", line)
			}
		}
		close(exited)
	}()
	inst.exited = exited
	err = server.Start()
	close(started)
	if err != nil {
		return err
	}
	inst.server = server

	return nil
}

func (inst *MgoInstance) kill() {
	inst.server.Process.Kill()
	<-inst.exited
	inst.server = nil
	inst.exited = nil
}

func (inst *MgoInstance) Destroy() {
	if inst.server != nil {
		inst.kill()
		os.RemoveAll(inst.dir)
		inst.addr, inst.dir = "", ""
	}
}

// Restart restarts the mongo server, useful for
// testing what happens when a state server goes down.
func (inst *MgoInstance) Restart() {
	inst.kill()
	if err := inst.Start(inst.ssl); err != nil {
		panic(err)
	}
}

// MgoTestPackage should be called to register the tests for any package that
// requires a MongoDB server.
func MgoTestPackage(t *stdtesting.T) {
	MgoTestPackageSsl(t, true)
}

func MgoTestPackageSsl(t *stdtesting.T, ssl bool) {
	if err := MgoServer.Start(ssl); err != nil {
		t.Fatal(err)
	}
	defer MgoServer.Destroy()
	gc.TestingT(t)
}

func (s *MgoSuite) SetUpSuite(c *gc.C) {
	if MgoServer.addr == "" {
		panic("MgoSuite tests must be run with MgoTestPackage")
	}
	mgo.SetStats(true)
	// Make tests that use password authentication faster.
	utils.FastInsecureHash = true
}

// readLines reads lines from the given reader and returns
// the last n non-empty lines, ignoring empty lines.
func readLines(r io.Reader, n int) []string {
	br := bufio.NewReader(r)
	lines := make([]string, n)
	i := 0
	for {
		line, err := br.ReadString('\n')
		if line = strings.TrimRight(line, "\n"); line != "" {
			lines[i%n] = line
			i++
		}
		if err != nil {
			break
		}
	}
	final := make([]string, 0, n+1)
	if i > n {
		final = append(final, fmt.Sprintf("[%d lines omitted]", i-n))
	}
	for j := 0; j < n; j++ {
		if line := lines[(j+i)%n]; line != "" {
			final = append(final, line)
		}
	}
	return final
}

func (s *MgoSuite) TearDownSuite(c *gc.C) {
	utils.FastInsecureHash = false
}

// MustDial returns a new connection to the MongoDB server, and panics on
// errors.
func (inst *MgoInstance) MustDial() *mgo.Session {
	s, err := inst.dial(false)
	if err != nil {
		panic(err)
	}
	return s
}

// Dial returns a new connection to the MongoDB server.
func (inst *MgoInstance) Dial() (*mgo.Session, error) {
	return inst.dial(false)
}

// DialDirect returns a new direct connection to the shared MongoDB server. This
// must be used if you're connecting to a replicaset that hasn't been initiated
// yet.
func (inst *MgoInstance) DialDirect() (*mgo.Session, error) {
	return inst.dial(true)
}

// MustDialDirect works like DialDirect, but panics on errors.
func (inst *MgoInstance) MustDialDirect() *mgo.Session {
	session, err := inst.dial(true)
	if err != nil {
		panic(err)
	}
	return session
}

func (inst *MgoInstance) dial(direct bool) (*mgo.Session, error) {
	pool := x509.NewCertPool()
	xcert, err := cert.ParseCert([]byte(CACert))
	if err != nil {
		return nil, err
	}
	pool.AddCert(xcert)
	tlsConfig := &tls.Config{
		RootCAs:    pool,
		ServerName: "anything",
	}
	session, err := mgo.DialWithInfo(&mgo.DialInfo{
		Direct: direct,
		Addrs:  []string{inst.addr},
		Dial: func(addr net.Addr) (net.Conn, error) {
			conn, err := tls.Dial("tcp", addr.String(), tlsConfig)
			if err != nil {
				logger.Infof("tls.Dial(%s) failed with %v", addr, err)
				return nil, err
			}
			return conn, nil
		},
		Timeout: mgoDialTimeout,
	})
	return session, err
}

func (s *MgoSuite) SetUpTest(c *gc.C) {
	mgo.ResetStats()
	s.Session = MgoServer.MustDial()
}

// Reset deletes all content from the MongoDB server and panics if it encounters
// errors.
func (inst *MgoInstance) Reset() {
	session := inst.MustDial()
	defer session.Close()

	dbnames, ok := resetAdminPasswordAndFetchDBNames(session)
	if ok {
		log.Infof("Reset successfully reset admin password")
	} else {
		// We restart it to regain access.  This should only
		// happen when tests fail.
		log.Noticef("testing: restarting MongoDB server after unauthorized access")
		inst.Destroy()
		if err := inst.Start(inst.ssl); err != nil {
			panic(err)
		}
		return
	}
	for _, name := range dbnames {
		switch name {
		case "admin", "local", "config":
		default:
			if err := session.DB(name).DropDatabase(); err != nil {
				panic(fmt.Errorf("Cannot drop MongoDB database %v: %v", name, err))
			}
		}
	}
}

// resetAdminPasswordAndFetchDBNames logs into the database with a
// plausible password and returns all the database's db names. We need
// to try several passwords because we don't know what state the mongo
// server is in when Reset is called. If the test has set a custom
// password, we're out of luck, but if they are using
// DefaultStatePassword, we can succeed.
func resetAdminPasswordAndFetchDBNames(session *mgo.Session) ([]string, bool) {
	// First try with no password
	dbnames, err := session.DatabaseNames()
	if err == nil {
		return dbnames, true
	}
	if !isUnauthorized(err) {
		panic(err)
	}
	// Then try the two most likely passwords in turn.
	for _, password := range []string{
		DefaultMongoPassword,
		utils.UserPasswordHash(DefaultMongoPassword, utils.CompatSalt),
	} {
		admin := session.DB("admin")
		if err := admin.Login("admin", password); err != nil {
			log.Infof("failed to log in with password %q", password)
			continue
		}
		dbnames, err := session.DatabaseNames()
		if err == nil {
			if err := admin.RemoveUser("admin"); err != nil {
				panic(err)
			}
			return dbnames, true
		}
		if !isUnauthorized(err) {
			panic(err)
		}
		log.Infof("unauthorized access when getting database names; password %q", password)
	}
	return nil, false
}

// isUnauthorized is a copy of the same function in state/open.go.
func isUnauthorized(err error) bool {
	if err == nil {
		return false
	}
	// Some unauthorized access errors have no error code,
	// just a simple error string.
	if err.Error() == "auth fails" {
		return true
	}
	if err, ok := err.(*mgo.QueryError); ok {
		return err.Code == 10057 ||
			err.Message == "need to login" ||
			err.Message == "unauthorized"
	}
	return false
}

func (s *MgoSuite) TearDownTest(c *gc.C) {
	MgoServer.Reset()
	s.Session.Close()
	for i := 0; ; i++ {
		stats := mgo.GetStats()
		if stats.SocketsInUse == 0 && stats.SocketsAlive == 0 {
			break
		}
		if i == 20 {
			c.Fatal("Test left sockets in a dirty state")
		}
		c.Logf("Waiting for sockets to die: %d in use, %d alive", stats.SocketsInUse, stats.SocketsAlive)
		time.Sleep(500 * time.Millisecond)
	}
}

// FindTCPPort finds an unused TCP port and returns it.
// Use of this function has an inherent race condition - another
// process may claim the port before we try to use it.
// We hope that the probability is small enough during
// testing to be negligible.
func FindTCPPort() int {
	l, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		panic(err)
	}
	l.Close()
	return l.Addr().(*net.TCPAddr).Port
}
<commit_msg>logging of mongod lifetime, start/stop/etc.<commit_after>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package testing

import (
	"bufio"
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	stdtesting "testing"
	"time"

	"github.com/juju/loggo"
	"labix.org/v2/mgo"
	gc "launchpad.net/gocheck"

	"launchpad.net/juju-core/cert"
	"launchpad.net/juju-core/log"
	"launchpad.net/juju-core/utils"
)

var (
	// MgoServer is a shared mongo server used by tests.
	MgoServer = &MgoInstance{ssl: true}
	logger = loggo.GetLogger("juju.testing")
)

type MgoInstance struct {
	// addr holds the address of the MongoDB server
	addr string

	// MgoPort holds the port of the MongoDB server.
	port int

	// server holds the running MongoDB command.
	server *exec.Cmd

	// exited receives a value when the mongodb server exits.
	exited <-chan struct{}

	// dir holds the directory that MongoDB is running in.
	dir string

	// ssl determines whether the MongoDB server will use TLS
	ssl bool

	// Params is a list of additional parameters that will be passed to
	// the mongod application
	Params []string
}

// Addr returns the address of the MongoDB server.
func (m *MgoInstance) Addr() string {
	return m.addr
}

// Port returns the port of the MongoDB server.
func (m *MgoInstance) Port() int {
	return m.port
}

// We specify a timeout to mgo.Dial, to prevent
// mongod failures hanging the tests.
const mgoDialTimeout = 15 * time.Second

// MgoSuite is a suite that deletes all content from the shared MongoDB
// server at the end of every test and supplies a connection to the shared
// MongoDB server.
type MgoSuite struct {
	Session *mgo.Session
}

// Start starts a MongoDB server in a temporary directory.
func (inst *MgoInstance) Start(ssl bool) error {
	dbdir, err := ioutil.TempDir("", "test-mgo")
	if err != nil {
		return err
	}
	logger.Debugf("starting mongo in %s", dbdir)

	// give them all the same keyfile so they can talk appropriately
	keyFilePath := filepath.Join(dbdir, "keyfile")
	err = ioutil.WriteFile(keyFilePath, []byte("not very secret"), 0600)
	if err != nil {
		return fmt.Errorf("cannot write key file: %v", err)
	}

	pemPath := filepath.Join(dbdir, "server.pem")
	err = ioutil.WriteFile(pemPath, []byte(ServerCert+ServerKey), 0600)
	if err != nil {
		return fmt.Errorf("cannot write cert/key PEM: %v", err)
	}
	inst.port = FindTCPPort()
	inst.addr = fmt.Sprintf("localhost:%d", inst.port)
	inst.dir = dbdir
	inst.ssl = ssl
	if err := inst.run(); err != nil {
		inst.addr = ""
		inst.port = 0
		os.RemoveAll(inst.dir)
		inst.dir = ""
	}
	logger.Debugf("started mongod pid %d in %s on port %d", inst.server.Process.Pid, dbdir, inst.port)
	return err
}

// run runs the MongoDB server at the
// address and directory already configured.
func (inst *MgoInstance) run() error {
	if inst.server != nil {
		panic("mongo server is already running")
	}
	logPath := filepath.Join(dbdir, "server.log")

	mgoport := strconv.Itoa(inst.port)
	mgoargs := []string{
		"--auth",
		"--dbpath", inst.dir,
		"--logpath", logPath,
		"--port", mgoport,
		"--nssize", "1",
		"--noprealloc",
		"--smallfiles",
		"--nojournal",
		"--nounixsocket",
		"--oplogSize", "10",
		"--keyFile", filepath.Join(inst.dir, "keyfile"),
	}
	if inst.ssl {
		mgoargs = append(mgoargs,
			"--sslOnNormalPorts",
			"--sslPEMKeyFile", filepath.Join(inst.dir, "server.pem"),
			"--sslPEMKeyPassword", "ignored")
	}
	if inst.Params != nil {
		mgoargs = append(mgoargs, inst.Params...)
	}
	server := exec.Command("mongod", mgoargs...)
	out, err := server.StdoutPipe()
	if err != nil {
		return err
	}
	server.Stderr = server.Stdout
	exited := make(chan struct{})
	started := make(chan struct{})
	go func() {
		<-started
		lines := readLines(out, 20)
		err := server.Wait()
		exitErr, _ := err.(*exec.ExitError)
		if err == nil || exitErr != nil && exitErr.Exited() {
			// mongodb has exited without being killed, so print the
			// last few lines of its log output.
			for _, line := range lines {
				log.Infof("mongod: %s", line)
			}
		}
		close(exited)
	}()
	inst.exited = exited
	err = server.Start()
	close(started)
	if err != nil {
		return err
	}
	inst.server = server

	return nil
}

func (inst *MgoInstance) kill() {
	inst.server.Process.Kill()
	<-inst.exited
	inst.server = nil
	inst.exited = nil
}

func (inst *MgoInstance) Destroy() {
	if inst.server != nil {
		logger.Debugf("killing mongod pid %d in %s on port %d", inst.server.Process.Pid, inst.dir, inst.port)
		inst.kill()
		os.RemoveAll(inst.dir)
		inst.addr, inst.dir = "", ""
	}
}

// Restart restarts the mongo server, useful for
// testing what happens when a state server goes down.
func (inst *MgoInstance) Restart() {
	logger.Debugf("restarting mongod pid %d in %s on port %d", inst.server.Process.Pid, inst.dir, inst.port)
	inst.kill()
	if err := inst.Start(inst.ssl); err != nil {
		panic(err)
	}
}

// MgoTestPackage should be called to register the tests for any package that
// requires a MongoDB server.
func MgoTestPackage(t *stdtesting.T) {
	MgoTestPackageSsl(t, true)
}

func MgoTestPackageSsl(t *stdtesting.T, ssl bool) {
	if err := MgoServer.Start(ssl); err != nil {
		t.Fatal(err)
	}
	defer MgoServer.Destroy()
	gc.TestingT(t)
}

func (s *MgoSuite) SetUpSuite(c *gc.C) {
	if MgoServer.addr == "" {
		panic("MgoSuite tests must be run with MgoTestPackage")
	}
	mgo.SetStats(true)
	// Make tests that use password authentication faster.
	utils.FastInsecureHash = true
}

// readLines reads lines from the given reader and returns
// the last n non-empty lines, ignoring empty lines.
func readLines(r io.Reader, n int) []string {
	br := bufio.NewReader(r)
	lines := make([]string, n)
	i := 0
	for {
		line, err := br.ReadString('\n')
		if line = strings.TrimRight(line, "\n"); line != "" {
			lines[i%n] = line
			i++
		}
		if err != nil {
			break
		}
	}
	final := make([]string, 0, n+1)
	if i > n {
		final = append(final, fmt.Sprintf("[%d lines omitted]", i-n))
	}
	for j := 0; j < n; j++ {
		if line := lines[(j+i)%n]; line != "" {
			final = append(final, line)
		}
	}
	return final
}

func (s *MgoSuite) TearDownSuite(c *gc.C) {
	utils.FastInsecureHash = false
}

// MustDial returns a new connection to the MongoDB server, and panics on
// errors.
func (inst *MgoInstance) MustDial() *mgo.Session {
	s, err := inst.dial(false)
	if err != nil {
		panic(err)
	}
	return s
}

// Dial returns a new connection to the MongoDB server.
func (inst *MgoInstance) Dial() (*mgo.Session, error) {
	return inst.dial(false)
}

// DialDirect returns a new direct connection to the shared MongoDB server. This
// must be used if you're connecting to a replicaset that hasn't been initiated
// yet.
func (inst *MgoInstance) DialDirect() (*mgo.Session, error) {
	return inst.dial(true)
}

// MustDialDirect works like DialDirect, but panics on errors.
func (inst *MgoInstance) MustDialDirect() *mgo.Session {
	session, err := inst.dial(true)
	if err != nil {
		panic(err)
	}
	return session
}

func (inst *MgoInstance) dial(direct bool) (*mgo.Session, error) {
	pool := x509.NewCertPool()
	xcert, err := cert.ParseCert([]byte(CACert))
	if err != nil {
		return nil, err
	}
	pool.AddCert(xcert)
	tlsConfig := &tls.Config{
		RootCAs:    pool,
		ServerName: "anything",
	}
	session, err := mgo.DialWithInfo(&mgo.DialInfo{
		Direct: direct,
		Addrs:  []string{inst.addr},
		Dial: func(addr net.Addr) (net.Conn, error) {
			conn, err := tls.Dial("tcp", addr.String(), tlsConfig)
			if err != nil {
				logger.Infof("tls.Dial(%s) failed with %v", addr, err)
				return nil, err
			}
			return conn, nil
		},
		Timeout: mgoDialTimeout,
	})
	return session, err
}

func (s *MgoSuite) SetUpTest(c *gc.C) {
	mgo.ResetStats()
	s.Session = MgoServer.MustDial()
}

// Reset deletes all content from the MongoDB server and panics if it encounters
// errors.
func (inst *MgoInstance) Reset() {
	session := inst.MustDial()
	defer session.Close()

	dbnames, ok := resetAdminPasswordAndFetchDBNames(session)
	if ok {
		log.Infof("Reset successfully reset admin password")
	} else {
		// We restart it to regain access.  This should only
		// happen when tests fail.
		log.Noticef("testing: restarting MongoDB server after unauthorized access")
		inst.Destroy()
		if err := inst.Start(inst.ssl); err != nil {
			panic(err)
		}
		return
	}
	for _, name := range dbnames {
		switch name {
		case "admin", "local", "config":
		default:
			if err := session.DB(name).DropDatabase(); err != nil {
				panic(fmt.Errorf("Cannot drop MongoDB database %v: %v", name, err))
			}
		}
	}
}

// resetAdminPasswordAndFetchDBNames logs into the database with a
// plausible password and returns all the database's db names. We need
// to try several passwords because we don't know what state the mongo
// server is in when Reset is called. If the test has set a custom
// password, we're out of luck, but if they are using
// DefaultStatePassword, we can succeed.
func resetAdminPasswordAndFetchDBNames(session *mgo.Session) ([]string, bool) {
	// First try with no password
	dbnames, err := session.DatabaseNames()
	if err == nil {
		return dbnames, true
	}
	if !isUnauthorized(err) {
		panic(err)
	}
	// Then try the two most likely passwords in turn.
	for _, password := range []string{
		DefaultMongoPassword,
		utils.UserPasswordHash(DefaultMongoPassword, utils.CompatSalt),
	} {
		admin := session.DB("admin")
		if err := admin.Login("admin", password); err != nil {
			log.Infof("failed to log in with password %q", password)
			continue
		}
		dbnames, err := session.DatabaseNames()
		if err == nil {
			if err := admin.RemoveUser("admin"); err != nil {
				panic(err)
			}
			return dbnames, true
		}
		if !isUnauthorized(err) {
			panic(err)
		}
		log.Infof("unauthorized access when getting database names; password %q", password)
	}
	return nil, false
}

// isUnauthorized is a copy of the same function in state/open.go.
func isUnauthorized(err error) bool {
	if err == nil {
		return false
	}
	// Some unauthorized access errors have no error code,
	// just a simple error string.
	if err.Error() == "auth fails" {
		return true
	}
	if err, ok := err.(*mgo.QueryError); ok {
		return err.Code == 10057 ||
			err.Message == "need to login" ||
			err.Message == "unauthorized"
	}
	return false
}

func (s *MgoSuite) TearDownTest(c *gc.C) {
	MgoServer.Reset()
	s.Session.Close()
	for i := 0; ; i++ {
		stats := mgo.GetStats()
		if stats.SocketsInUse == 0 && stats.SocketsAlive == 0 {
			break
		}
		if i == 20 {
			c.Fatal("Test left sockets in a dirty state")
		}
		c.Logf("Waiting for sockets to die: %d in use, %d alive", stats.SocketsInUse, stats.SocketsAlive)
		time.Sleep(500 * time.Millisecond)
	}
}

// FindTCPPort finds an unused TCP port and returns it.
// Use of this function has an inherent race condition - another
// process may claim the port before we try to use it.
// We hope that the probability is small enough during
// testing to be negligible.
func FindTCPPort() int {
	l, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		panic(err)
	}
	l.Close()
	return l.Addr().(*net.TCPAddr).Port
}
<|endoftext|>


<commit_before>// Swift interface
package swift

import (
	"errors"
	"fmt"
	"io"
	"path"
	"regexp"
	"strings"
	"time"

	"github.com/ncw/rclone/fs"
	"github.com/ncw/swift"
)

// Register with Fs
func init() {
	fs.Register(&fs.FsInfo{
		Name:  "swift",
		NewFs: NewFs,
		Options: []fs.Option{{
			Name: "user",
			Help: "User name to log in.",
		}, {
			Name: "key",
			Help: "API key or password.",
		}, {
			Name: "auth",
			Help: "Authentication URL for server.",
			Examples: []fs.OptionExample{{
				Help:  "Rackspace US",
				Value: "https://auth.api.rackspacecloud.com/v1.0",
			}, {
				Help:  "Rackspace UK",
				Value: "https://lon.auth.api.rackspacecloud.com/v1.0",
			}, {
				Help:  "Rackspace v2",
				Value: "https://identity.api.rackspacecloud.com/v2.0",
			}, {
				Help:  "Memset Memstore UK",
				Value: "https://auth.storage.memset.com/v1.0",
			}, {
				Help:  "Memset Memstore UK v2",
				Value: "https://auth.storage.memset.com/v2.0",
			}},
		},
		// snet     = flag.Bool("swift-snet", false, "Use internal service network") // FIXME not implemented
		},
	})
}

// FsSwift represents a remote swift server
type FsSwift struct {
	c         swift.Connection // the connection to the swift server
	container string           // the container we are working on
	root      string           // the path we are working on if any
}

// FsObjectSwift describes a swift object
//
// Will definitely have info but maybe not meta
type FsObjectSwift struct {
	swift  *FsSwift        // what this object is part of
	remote string          // The remote path
	info   swift.Object    // Info from the swift object if known
	meta   *swift.Metadata // The object metadata if known
}

// ------------------------------------------------------------

// String converts this FsSwift to a string
func (f *FsSwift) String() string {
	if f.root == "" {
		return fmt.Sprintf("Swift container %s", f.container)
	}
	return fmt.Sprintf("Swift container %s path %s", f.container, f.root)
}

// Pattern to match a swift path
var matcher = regexp.MustCompile(`^([^/]*)(.*)$`)

// parseParse parses a swift 'url'
func parsePath(path string) (container, directory string, err error) {
	parts := matcher.FindStringSubmatch(path)
	if parts == nil {
		err = fmt.Errorf("Couldn't find container in swift path %q", path)
	} else {
		container, directory = parts[1], parts[2]
		directory = strings.Trim(directory, "/")
	}
	return
}

// swiftConnection makes a connection to swift
func swiftConnection(name string) (*swift.Connection, error) {
	userName := fs.ConfigFile.MustValue(name, "user")
	if userName == "" {
		return nil, errors.New("user not found")
	}
	apiKey := fs.ConfigFile.MustValue(name, "key")
	if apiKey == "" {
		return nil, errors.New("key not found")
	}
	authUrl := fs.ConfigFile.MustValue(name, "auth")
	if authUrl == "" {
		return nil, errors.New("auth not found")
	}
	c := &swift.Connection{
		UserName:  userName,
		ApiKey:    apiKey,
		AuthUrl:   authUrl,
		UserAgent: fs.UserAgent,
	}
	err := c.Authenticate()
	if err != nil {
		return nil, err
	}
	return c, nil
}

// NewFs contstructs an FsSwift from the path, container:path
func NewFs(name, root string) (fs.Fs, error) {
	container, directory, err := parsePath(root)
	if err != nil {
		return nil, err
	}
	c, err := swiftConnection(name)
	if err != nil {
		return nil, err
	}
	f := &FsSwift{
		c:         *c,
		container: container,
		root:      directory,
	}
	if f.root != "" {
		f.root += "/"
		// Check to see if the object exists
		_, _, err = f.c.Object(container, directory)
		if err == nil {
			remote := path.Base(directory)
			f.root = path.Dir(directory)
			if f.root == "." {
				f.root = ""
			} else {
				f.root += "/"
			}
			obj := f.NewFsObject(remote)
			// return a Fs Limited to this object
			return fs.NewLimited(f, obj), nil
		}
	}
	return f, nil
}

// Return an FsObject from a path
//
// May return nil if an error occurred
func (f *FsSwift) NewFsObjectWithInfo(remote string, info *swift.Object) fs.Object {
	fs := &FsObjectSwift{
		swift:  f,
		remote: remote,
	}
	if info != nil {
		// Set info but not meta
		fs.info = *info
	} else {
		err := fs.readMetaData() // reads info and meta, returning an error
		if err != nil {
			// logged already FsDebug("Failed to read info: %s", err)
			return nil
		}
	}
	return fs
}

// Return an FsObject from a path
//
// May return nil if an error occurred
func (f *FsSwift) NewFsObject(remote string) fs.Object {
	return f.NewFsObjectWithInfo(remote, nil)
}

// list the objects into the function supplied
//
// If directories is set it only sends directories
func (f *FsSwift) list(directories bool, fn func(string, *swift.Object)) {
	// Options for ObjectsWalk
	opts := swift.ObjectsOpts{
		Prefix: f.root,
		Limit:  256,
	}
	if directories {
		opts.Delimiter = '/'
	}
	rootLength := len(f.root)
	err := f.c.ObjectsWalk(f.container, &opts, func(opts *swift.ObjectsOpts) (interface{}, error) {
		objects, err := f.c.Objects(f.container, opts)
		if err == nil {
			for i := range objects {
				object := &objects[i]
				// FIXME if there are no directories, swift gives back the files for some reason!
				if directories && !strings.HasSuffix(object.Name, "/") {
					continue
				}
				if !strings.HasPrefix(object.Name, f.root) {
					fs.Log(f, "Odd name received %q", object.Name)
					continue
				}
				remote := object.Name[rootLength:]
				fn(remote, object)
			}
		}
		return objects, err
	})
	if err != nil {
		fs.Stats.Error()
		fs.Log(f, "Couldn't read container %q: %s", f.container, err)
	}
}

// Walk the path returning a channel of FsObjects
func (f *FsSwift) List() fs.ObjectsChan {
	out := make(fs.ObjectsChan, fs.Config.Checkers)
	if f.container == "" {
		// Return no objects at top level list
		close(out)
		fs.Stats.Error()
		fs.Log(f, "Can't list objects at root - choose a container using lsd")
	} else {
		// List the objects
		go func() {
			defer close(out)
			f.list(false, func(remote string, object *swift.Object) {
				if fs := f.NewFsObjectWithInfo(remote, object); fs != nil {
					out <- fs
				}
			})
		}()
	}
	return out
}

// Lists the containers
func (f *FsSwift) ListDir() fs.DirChan {
	out := make(fs.DirChan, fs.Config.Checkers)
	if f.container == "" {
		// List the containers
		go func() {
			defer close(out)
			containers, err := f.c.ContainersAll(nil)
			if err != nil {
				fs.Stats.Error()
				fs.Log(f, "Couldn't list containers: %v", err)
			} else {
				for _, container := range containers {
					out <- &fs.Dir{
						Name:  container.Name,
						Bytes: container.Bytes,
						Count: container.Count,
					}
				}
			}
		}()
	} else {
		// List the directories in the path in the container
		go func() {
			defer close(out)
			f.list(true, func(remote string, object *swift.Object) {
				out <- &fs.Dir{
					Name:  remote,
					Bytes: object.Bytes,
					Count: 0,
				}
			})
		}()
	}
	return out
}

// Put the object into the container
//
// Copy the reader in to the new object which is returned
//
// The new object may have been created if an error is returned
func (f *FsSwift) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
	// Temporary FsObject under construction
	fs := &FsObjectSwift{swift: f, remote: remote}
	return fs, fs.Update(in, modTime, size)
}

// Mkdir creates the container if it doesn't exist
func (f *FsSwift) Mkdir() error {
	return f.c.ContainerCreate(f.container, nil)
}

// Rmdir deletes the container
//
// Returns an error if it isn't empty
func (f *FsSwift) Rmdir() error {
	return f.c.ContainerDelete(f.container)
}

// Return the precision
func (fs *FsSwift) Precision() time.Duration {
	return time.Nanosecond
}

// ------------------------------------------------------------

// Return the parent Fs
func (o *FsObjectSwift) Fs() fs.Fs {
	return o.swift
}

// Return a string version
func (o *FsObjectSwift) String() string {
	if o == nil {
		return "<nil>"
	}
	return o.remote
}

// Return the remote path
func (o *FsObjectSwift) Remote() string {
	return o.remote
}

// Md5sum returns the Md5sum of an object returning a lowercase hex string
func (o *FsObjectSwift) Md5sum() (string, error) {
	return strings.ToLower(o.info.Hash), nil
}

// Size returns the size of an object in bytes
func (o *FsObjectSwift) Size() int64 {
	return o.info.Bytes
}

// readMetaData gets the metadata if it hasn't already been fetched
//
// it also sets the info
func (o *FsObjectSwift) readMetaData() (err error) {
	if o.meta != nil {
		return nil
	}
	info, h, err := o.swift.c.Object(o.swift.container, o.swift.root+o.remote)
	if err != nil {
		fs.Debug(o, "Failed to read info: %s", err)
		return err
	}
	meta := h.ObjectMetadata()
	o.info = info
	o.meta = &meta
	return nil
}

// ModTime returns the modification time of the object
//
//
// It attempts to read the objects mtime and if that isn't present the
// LastModified returned in the http headers
func (o *FsObjectSwift) ModTime() time.Time {
	err := o.readMetaData()
	if err != nil {
		// fs.Log(o, "Failed to read metadata: %s", err)
		return o.info.LastModified
	}
	modTime, err := o.meta.GetModTime()
	if err != nil {
		// fs.Log(o, "Failed to read mtime from object: %s", err)
		return o.info.LastModified
	}
	return modTime
}

// Sets the modification time of the local fs object
func (o *FsObjectSwift) SetModTime(modTime time.Time) {
	err := o.readMetaData()
	if err != nil {
		fs.Stats.Error()
		fs.Log(o, "Failed to read metadata: %s", err)
		return
	}
	o.meta.SetModTime(modTime)
	err = o.swift.c.ObjectUpdate(o.swift.container, o.swift.root+o.remote, o.meta.ObjectHeaders())
	if err != nil {
		fs.Stats.Error()
		fs.Log(o, "Failed to update remote mtime: %s", err)
	}
}

// Is this object storable
func (o *FsObjectSwift) Storable() bool {
	return true
}

// Open an object for read
func (o *FsObjectSwift) Open() (in io.ReadCloser, err error) {
	in, _, err = o.swift.c.ObjectOpen(o.swift.container, o.swift.root+o.remote, true, nil)
	return
}

// Update the object with the contents of the io.Reader, modTime and size
//
// The new object may have been created if an error is returned
func (o *FsObjectSwift) Update(in io.Reader, modTime time.Time, size int64) error {
	// Set the mtime
	m := swift.Metadata{}
	m.SetModTime(modTime)
	_, err := o.swift.c.ObjectPut(o.swift.container, o.swift.root+o.remote, in, true, "", "", m.ObjectHeaders())
	return err
}

// Remove an object
func (o *FsObjectSwift) Remove() error {
	return o.swift.c.ObjectDelete(o.swift.container, o.swift.root+o.remote)
}

// Check the interfaces are satisfied
var _ fs.Fs = &FsSwift{}
var _ fs.Object = &FsObjectSwift{}
<commit_msg>swift: Read metadata after Update or Put<commit_after>// Swift interface
package swift

import (
	"errors"
	"fmt"
	"io"
	"path"
	"regexp"
	"strings"
	"time"

	"github.com/ncw/rclone/fs"
	"github.com/ncw/swift"
)

// Register with Fs
func init() {
	fs.Register(&fs.FsInfo{
		Name:  "swift",
		NewFs: NewFs,
		Options: []fs.Option{{
			Name: "user",
			Help: "User name to log in.",
		}, {
			Name: "key",
			Help: "API key or password.",
		}, {
			Name: "auth",
			Help: "Authentication URL for server.",
			Examples: []fs.OptionExample{{
				Help:  "Rackspace US",
				Value: "https://auth.api.rackspacecloud.com/v1.0",
			}, {
				Help:  "Rackspace UK",
				Value: "https://lon.auth.api.rackspacecloud.com/v1.0",
			}, {
				Help:  "Rackspace v2",
				Value: "https://identity.api.rackspacecloud.com/v2.0",
			}, {
				Help:  "Memset Memstore UK",
				Value: "https://auth.storage.memset.com/v1.0",
			}, {
				Help:  "Memset Memstore UK v2",
				Value: "https://auth.storage.memset.com/v2.0",
			}},
		},
		// snet     = flag.Bool("swift-snet", false, "Use internal service network") // FIXME not implemented
		},
	})
}

// FsSwift represents a remote swift server
type FsSwift struct {
	c         swift.Connection // the connection to the swift server
	container string           // the container we are working on
	root      string           // the path we are working on if any
}

// FsObjectSwift describes a swift object
//
// Will definitely have info but maybe not meta
type FsObjectSwift struct {
	swift  *FsSwift        // what this object is part of
	remote string          // The remote path
	info   swift.Object    // Info from the swift object if known
	meta   *swift.Metadata // The object metadata if known
}

// ------------------------------------------------------------

// String converts this FsSwift to a string
func (f *FsSwift) String() string {
	if f.root == "" {
		return fmt.Sprintf("Swift container %s", f.container)
	}
	return fmt.Sprintf("Swift container %s path %s", f.container, f.root)
}

// Pattern to match a swift path
var matcher = regexp.MustCompile(`^([^/]*)(.*)$`)

// parseParse parses a swift 'url'
func parsePath(path string) (container, directory string, err error) {
	parts := matcher.FindStringSubmatch(path)
	if parts == nil {
		err = fmt.Errorf("Couldn't find container in swift path %q", path)
	} else {
		container, directory = parts[1], parts[2]
		directory = strings.Trim(directory, "/")
	}
	return
}

// swiftConnection makes a connection to swift
func swiftConnection(name string) (*swift.Connection, error) {
	userName := fs.ConfigFile.MustValue(name, "user")
	if userName == "" {
		return nil, errors.New("user not found")
	}
	apiKey := fs.ConfigFile.MustValue(name, "key")
	if apiKey == "" {
		return nil, errors.New("key not found")
	}
	authUrl := fs.ConfigFile.MustValue(name, "auth")
	if authUrl == "" {
		return nil, errors.New("auth not found")
	}
	c := &swift.Connection{
		UserName:  userName,
		ApiKey:    apiKey,
		AuthUrl:   authUrl,
		UserAgent: fs.UserAgent,
	}
	err := c.Authenticate()
	if err != nil {
		return nil, err
	}
	return c, nil
}

// NewFs contstructs an FsSwift from the path, container:path
func NewFs(name, root string) (fs.Fs, error) {
	container, directory, err := parsePath(root)
	if err != nil {
		return nil, err
	}
	c, err := swiftConnection(name)
	if err != nil {
		return nil, err
	}
	f := &FsSwift{
		c:         *c,
		container: container,
		root:      directory,
	}
	if f.root != "" {
		f.root += "/"
		// Check to see if the object exists
		_, _, err = f.c.Object(container, directory)
		if err == nil {
			remote := path.Base(directory)
			f.root = path.Dir(directory)
			if f.root == "." {
				f.root = ""
			} else {
				f.root += "/"
			}
			obj := f.NewFsObject(remote)
			// return a Fs Limited to this object
			return fs.NewLimited(f, obj), nil
		}
	}
	return f, nil
}

// Return an FsObject from a path
//
// May return nil if an error occurred
func (f *FsSwift) NewFsObjectWithInfo(remote string, info *swift.Object) fs.Object {
	fs := &FsObjectSwift{
		swift:  f,
		remote: remote,
	}
	if info != nil {
		// Set info but not meta
		fs.info = *info
	} else {
		err := fs.readMetaData() // reads info and meta, returning an error
		if err != nil {
			// logged already FsDebug("Failed to read info: %s", err)
			return nil
		}
	}
	return fs
}

// Return an FsObject from a path
//
// May return nil if an error occurred
func (f *FsSwift) NewFsObject(remote string) fs.Object {
	return f.NewFsObjectWithInfo(remote, nil)
}

// list the objects into the function supplied
//
// If directories is set it only sends directories
func (f *FsSwift) list(directories bool, fn func(string, *swift.Object)) {
	// Options for ObjectsWalk
	opts := swift.ObjectsOpts{
		Prefix: f.root,
		Limit:  256,
	}
	if directories {
		opts.Delimiter = '/'
	}
	rootLength := len(f.root)
	err := f.c.ObjectsWalk(f.container, &opts, func(opts *swift.ObjectsOpts) (interface{}, error) {
		objects, err := f.c.Objects(f.container, opts)
		if err == nil {
			for i := range objects {
				object := &objects[i]
				// FIXME if there are no directories, swift gives back the files for some reason!
				if directories && !strings.HasSuffix(object.Name, "/") {
					continue
				}
				if !strings.HasPrefix(object.Name, f.root) {
					fs.Log(f, "Odd name received %q", object.Name)
					continue
				}
				remote := object.Name[rootLength:]
				fn(remote, object)
			}
		}
		return objects, err
	})
	if err != nil {
		fs.Stats.Error()
		fs.Log(f, "Couldn't read container %q: %s", f.container, err)
	}
}

// Walk the path returning a channel of FsObjects
func (f *FsSwift) List() fs.ObjectsChan {
	out := make(fs.ObjectsChan, fs.Config.Checkers)
	if f.container == "" {
		// Return no objects at top level list
		close(out)
		fs.Stats.Error()
		fs.Log(f, "Can't list objects at root - choose a container using lsd")
	} else {
		// List the objects
		go func() {
			defer close(out)
			f.list(false, func(remote string, object *swift.Object) {
				if fs := f.NewFsObjectWithInfo(remote, object); fs != nil {
					out <- fs
				}
			})
		}()
	}
	return out
}

// Lists the containers
func (f *FsSwift) ListDir() fs.DirChan {
	out := make(fs.DirChan, fs.Config.Checkers)
	if f.container == "" {
		// List the containers
		go func() {
			defer close(out)
			containers, err := f.c.ContainersAll(nil)
			if err != nil {
				fs.Stats.Error()
				fs.Log(f, "Couldn't list containers: %v", err)
			} else {
				for _, container := range containers {
					out <- &fs.Dir{
						Name:  container.Name,
						Bytes: container.Bytes,
						Count: container.Count,
					}
				}
			}
		}()
	} else {
		// List the directories in the path in the container
		go func() {
			defer close(out)
			f.list(true, func(remote string, object *swift.Object) {
				out <- &fs.Dir{
					Name:  remote,
					Bytes: object.Bytes,
					Count: 0,
				}
			})
		}()
	}
	return out
}

// Put the object into the container
//
// Copy the reader in to the new object which is returned
//
// The new object may have been created if an error is returned
func (f *FsSwift) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
	// Temporary FsObject under construction
	fs := &FsObjectSwift{swift: f, remote: remote}
	return fs, fs.Update(in, modTime, size)
}

// Mkdir creates the container if it doesn't exist
func (f *FsSwift) Mkdir() error {
	return f.c.ContainerCreate(f.container, nil)
}

// Rmdir deletes the container
//
// Returns an error if it isn't empty
func (f *FsSwift) Rmdir() error {
	return f.c.ContainerDelete(f.container)
}

// Return the precision
func (fs *FsSwift) Precision() time.Duration {
	return time.Nanosecond
}

// ------------------------------------------------------------

// Return the parent Fs
func (o *FsObjectSwift) Fs() fs.Fs {
	return o.swift
}

// Return a string version
func (o *FsObjectSwift) String() string {
	if o == nil {
		return "<nil>"
	}
	return o.remote
}

// Return the remote path
func (o *FsObjectSwift) Remote() string {
	return o.remote
}

// Md5sum returns the Md5sum of an object returning a lowercase hex string
func (o *FsObjectSwift) Md5sum() (string, error) {
	return strings.ToLower(o.info.Hash), nil
}

// Size returns the size of an object in bytes
func (o *FsObjectSwift) Size() int64 {
	return o.info.Bytes
}

// readMetaData gets the metadata if it hasn't already been fetched
//
// it also sets the info
func (o *FsObjectSwift) readMetaData() (err error) {
	if o.meta != nil {
		return nil
	}
	info, h, err := o.swift.c.Object(o.swift.container, o.swift.root+o.remote)
	if err != nil {
		fs.Debug(o, "Failed to read info: %s", err)
		return err
	}
	meta := h.ObjectMetadata()
	o.info = info
	o.meta = &meta
	return nil
}

// ModTime returns the modification time of the object
//
//
// It attempts to read the objects mtime and if that isn't present the
// LastModified returned in the http headers
func (o *FsObjectSwift) ModTime() time.Time {
	err := o.readMetaData()
	if err != nil {
		// fs.Log(o, "Failed to read metadata: %s", err)
		return o.info.LastModified
	}
	modTime, err := o.meta.GetModTime()
	if err != nil {
		// fs.Log(o, "Failed to read mtime from object: %s", err)
		return o.info.LastModified
	}
	return modTime
}

// Sets the modification time of the local fs object
func (o *FsObjectSwift) SetModTime(modTime time.Time) {
	err := o.readMetaData()
	if err != nil {
		fs.Stats.Error()
		fs.Log(o, "Failed to read metadata: %s", err)
		return
	}
	o.meta.SetModTime(modTime)
	err = o.swift.c.ObjectUpdate(o.swift.container, o.swift.root+o.remote, o.meta.ObjectHeaders())
	if err != nil {
		fs.Stats.Error()
		fs.Log(o, "Failed to update remote mtime: %s", err)
	}
}

// Is this object storable
func (o *FsObjectSwift) Storable() bool {
	return true
}

// Open an object for read
func (o *FsObjectSwift) Open() (in io.ReadCloser, err error) {
	in, _, err = o.swift.c.ObjectOpen(o.swift.container, o.swift.root+o.remote, true, nil)
	return
}

// Update the object with the contents of the io.Reader, modTime and size
//
// The new object may have been created if an error is returned
func (o *FsObjectSwift) Update(in io.Reader, modTime time.Time, size int64) error {
	// Set the mtime
	m := swift.Metadata{}
	m.SetModTime(modTime)
	_, err := o.swift.c.ObjectPut(o.swift.container, o.swift.root+o.remote, in, true, "", "", m.ObjectHeaders())
	if err != nil {
		return err
	}
	// Read the metadata from the newly created object
	err = o.readMetaData()
	return err
}

// Remove an object
func (o *FsObjectSwift) Remove() error {
	return o.swift.c.ObjectDelete(o.swift.container, o.swift.root+o.remote)
}

// Check the interfaces are satisfied
var _ fs.Fs = &FsSwift{}
var _ fs.Object = &FsObjectSwift{}
<|endoftext|>


<commit_before>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

// Package tailer provides a class that is responsible for tailing a log file
// and extracting new log lines to be passed into the virtual machines.
package tailer

// mtail gets notified on modifications (i.e. appends) to log files that are
// being watched, in order to read the new lines. Log files can also be
// rotated, so mtail is also notified of creates in the log file directory.

import (
	"errors"
	"expvar"
	"fmt"
	"io"
	"os"
	"path"
	"path/filepath"
	"sync"
	"syscall"
	"unicode/utf8"

	"github.com/golang/glog"

	"github.com/google/mtail/watcher"

	"github.com/spf13/afero"
)

var (
	logCount     = expvar.NewInt("log_count")
	logErrors    = expvar.NewMap("log_errors_total")
	logRotations = expvar.NewMap("log_rotations_total")
)

// Tailer receives notification of changes from a Watcher and extracts new log
// lines from files. It also handles new log file creation events and log
// rotations.
type Tailer struct {
	w watcher.Watcher

	watched     map[string]struct{}   // Names of logs being watched.
	watchedLock sync.RWMutex          // protects `watched'
	lines       chan<- string         // Logfile lines being emitted.
	files       map[string]afero.File // File handles for each pathname.
	filesLock   sync.Mutex            // protects `files'
	partials    map[string]string     // Accumulator for the currently read line for each pathname.

	fs afero.Fs // mockable filesystem interface
}

// Options configures a Tailer
type Options struct {
	Lines chan<- string
	W     watcher.Watcher // Not required, will use watcher.LogWatcher if it is zero.
	FS    afero.Fs        // Not required, will use afero.OsFs if it is zero.
}

// New returns a new Tailer, configured with the supplied Options
func New(o Options) (*Tailer, error) {
	if o.Lines == nil {
		return nil, errors.New("tailer needs lines")
	}
	fs := o.FS
	if fs == nil {
		fs = &afero.OsFs{}
	}
	w := o.W
	if w == nil {
		var err error
		w, err = watcher.NewLogWatcher()
		if err != nil {
			return nil, fmt.Errorf("Couldn't create a watcher for tailer: %s", err)
		}
	}
	t := &Tailer{
		w:        w,
		watched:  make(map[string]struct{}),
		lines:    o.Lines,
		files:    make(map[string]afero.File),
		partials: make(map[string]string),
		fs:       fs,
	}
	go t.run()
	return t, nil
}

func (t *Tailer) addWatched(path string) {
	t.watchedLock.Lock()
	defer t.watchedLock.Unlock()
	t.watched[path] = struct{}{}
}

func (t *Tailer) isWatching(path string) bool {
	t.watchedLock.RLock()
	defer t.watchedLock.RUnlock()
	_, ok := t.watched[path]
	return ok
}

// Tail registers a file to be tailed.
func (t *Tailer) Tail(pathname string) {
	fullpath, err := filepath.Abs(pathname)
	if err != nil {
		glog.Infof("Failed to find absolute path for %q: %s\n", pathname, err)
		return
	}
	if !t.isWatching(fullpath) {
		t.addWatched(fullpath)
		logCount.Add(1)
		t.openLogFile(fullpath, false)
	}
}

// handleLogUpdate reads all available bytes from an already opened file
// identified by pathname, and sends them to be processed on the lines channel.
func (t *Tailer) handleLogUpdate(pathname string) {
	for {
		f, ok := t.files[pathname]
		if !ok {
			glog.Infof("No file found for %q", pathname)
			return
		}
		partial, err := t.readPartial(f, t.partials[pathname])
		t.partials[pathname] = partial
		if err != nil {
			if err == io.EOF {
				// end of file for now, return
				return
			}
			glog.Infof("Failed to read updates from %q: %s", pathname, err)
			return
		}
	}
}

// readPartial reads blocks of 4096 bytes from the File, sending lines to the
// channel as it encounters newlines.  If EOF is encountered, the partial line
// is returned to be concatenated with on the next call.
func (t *Tailer) readPartial(f afero.File, partial string) (string, error) {
	b := make([]byte, 4096)
	n, err := f.Read(b)
	if err != nil {
		return partial, err // TODO(jaq): handle this path better
	}
	for i, width := 0, 0; i < len(b) && i < n; i += width {
		var rune rune
		rune, width = utf8.DecodeRune(b[i:])
		switch {
		case rune != '\n':
			partial += string(rune)
		default:
			// send off line for processing
			t.lines <- partial
			// reset accumulator
			partial = ""
		}
	}
	return partial, nil
}

// inode returns the inode number of a file, or 0 if the file has no underlying Sys implementation.
func inode(f os.FileInfo) uint64 {
	s := f.Sys()
	if s == nil {
		return 0
	}
	switch s := s.(type) {
	case *syscall.Stat_t:
		return s.Ino
	default:
		return 0
	}
}

// handleLogCreate handles both new and rotated log files.
func (t *Tailer) handleLogCreate(pathname string) {
	if !t.isWatching(pathname) {
		glog.V(1).Info("Not watching path %q, ignoring.\n", pathname)
		return
	}

	t.filesLock.Lock()
	fd, ok := t.files[pathname]
	t.filesLock.Unlock()
	if ok {
		s1, err := fd.Stat()
		if err != nil {
			glog.Infof("Stat failed on %q: %s", t.files[pathname].Name(), err)
			return
		}
		s2, err := t.fs.Stat(pathname)
		if err != nil {
			glog.Infof("Stat failed on %q: %s", pathname, err)
			return
		}
		if inode(s1) != inode(s2) {
			logRotations.Add(pathname, 1)
			// flush the old log, pathname is still an index into t.files with the old inode.
			t.handleLogUpdate(pathname)
			fd.Close()
			err := t.w.Remove(pathname)
			if err != nil {
				glog.Info("Failed removing watches on", pathname)
			}
			// Always seek to start on log rotation.
			glog.Infof("Seek to start on %s", pathname)
			t.openLogFile(pathname, true)
		} else {
			glog.Infof("Path %s already being watched, and inode not changed.",
				pathname)
		}
	} else {
		// Freshly opened log file, never seen before.
		t.openLogFile(pathname, true)
	}
}

// openLogFile opens a new log file at pathname, and optionally seeks to the
// start or end of the file. Rotated logs should start at the start, but logs
// opened for the first time start at the end.
func (t *Tailer) openLogFile(pathname string, seekStart bool) {
	if !t.isWatching(pathname) {
		glog.Infof("Not watching %q, ignoring.", pathname)
		return
	}

	d := path.Dir(pathname)
	if !t.isWatching(d) {
		err := t.w.Add(d)
		if err != nil {
			glog.Infof("Adding a create watch failed on %q: %s", d, err)
		}
		t.addWatched(d)
	}

	f, err := t.fs.Open(pathname)
	if err != nil {
		// Doesn't exist yet. We're watching the directory, so we'll pick it up
		// again on create; return successfully.
		if os.IsNotExist(err) {
			return
		}
		glog.Infof("Failed to open %q for reading: %s", pathname, err)
		logErrors.Add(pathname, 1)
		return
	}
	t.filesLock.Lock()
	t.files[pathname] = f

	if seekStart {
		t.files[pathname].Seek(0, os.SEEK_SET)
	} else {
		t.files[pathname].Seek(0, os.SEEK_END)
	}

	t.filesLock.Unlock()

	err = t.w.Add(pathname)
	if err != nil {
		glog.Infof("Adding a change watch failed on %q: %s", pathname, err)
	}

	glog.Infof("Tailing %s", pathname)

	// In case the new log has been written to already, attempt to read the first lines.
	t.handleLogUpdate(pathname)
}

// start is the main event loop for the Tailer.
// It receives notification of log file changes from the watcher channel, and
// handles them.
func (t *Tailer) run() {
	for e := range t.w.Events() {
		switch e := e.(type) {
		case watcher.UpdateEvent:
			t.handleLogUpdate(e.Pathname)
		case watcher.CreateEvent:
			t.handleLogCreate(e.Pathname)
		default:
			glog.Infof("Unexpected event %q", e)
		}
	}
	glog.Infof("Shutting down tailer.")
	close(t.lines)
}

// Close signals termination to the watcher.
func (t *Tailer) Close() {
	t.w.Close()
}
<commit_msg>Move the filename lookup outside of the loop in handleLogUpdate.<commit_after>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

// Package tailer provides a class that is responsible for tailing a log file
// and extracting new log lines to be passed into the virtual machines.
package tailer

// mtail gets notified on modifications (i.e. appends) to log files that are
// being watched, in order to read the new lines. Log files can also be
// rotated, so mtail is also notified of creates in the log file directory.

import (
	"errors"
	"expvar"
	"fmt"
	"io"
	"os"
	"path"
	"path/filepath"
	"sync"
	"syscall"
	"unicode/utf8"

	"github.com/golang/glog"

	"github.com/google/mtail/watcher"

	"github.com/spf13/afero"
)

var (
	logCount     = expvar.NewInt("log_count")
	logErrors    = expvar.NewMap("log_errors_total")
	logRotations = expvar.NewMap("log_rotations_total")
)

// Tailer receives notification of changes from a Watcher and extracts new log
// lines from files. It also handles new log file creation events and log
// rotations.
type Tailer struct {
	w watcher.Watcher

	watched     map[string]struct{}   // Names of logs being watched.
	watchedLock sync.RWMutex          // protects `watched'
	lines       chan<- string         // Logfile lines being emitted.
	files       map[string]afero.File // File handles for each pathname.
	filesLock   sync.Mutex            // protects `files'
	partials    map[string]string     // Accumulator for the currently read line for each pathname.

	fs afero.Fs // mockable filesystem interface
}

// Options configures a Tailer
type Options struct {
	Lines chan<- string
	W     watcher.Watcher // Not required, will use watcher.LogWatcher if it is zero.
	FS    afero.Fs        // Not required, will use afero.OsFs if it is zero.
}

// New returns a new Tailer, configured with the supplied Options
func New(o Options) (*Tailer, error) {
	if o.Lines == nil {
		return nil, errors.New("tailer needs lines")
	}
	fs := o.FS
	if fs == nil {
		fs = &afero.OsFs{}
	}
	w := o.W
	if w == nil {
		var err error
		w, err = watcher.NewLogWatcher()
		if err != nil {
			return nil, fmt.Errorf("Couldn't create a watcher for tailer: %s", err)
		}
	}
	t := &Tailer{
		w:        w,
		watched:  make(map[string]struct{}),
		lines:    o.Lines,
		files:    make(map[string]afero.File),
		partials: make(map[string]string),
		fs:       fs,
	}
	go t.run()
	return t, nil
}

func (t *Tailer) addWatched(path string) {
	t.watchedLock.Lock()
	defer t.watchedLock.Unlock()
	t.watched[path] = struct{}{}
}

func (t *Tailer) isWatching(path string) bool {
	t.watchedLock.RLock()
	defer t.watchedLock.RUnlock()
	_, ok := t.watched[path]
	return ok
}

// Tail registers a file to be tailed.
func (t *Tailer) Tail(pathname string) {
	fullpath, err := filepath.Abs(pathname)
	if err != nil {
		glog.Infof("Failed to find absolute path for %q: %s\n", pathname, err)
		return
	}
	if !t.isWatching(fullpath) {
		t.addWatched(fullpath)
		logCount.Add(1)
		t.openLogFile(fullpath, false)
	}
}

// handleLogUpdate reads all available bytes from an already opened file
// identified by pathname, and sends them to be processed on the lines channel.
func (t *Tailer) handleLogUpdate(pathname string) {
	f, ok := t.files[pathname]
	if !ok {
		glog.Infof("No file found for %q", pathname)
		return
	}
	for {
		partial, err := t.readPartial(f, t.partials[pathname])
		t.partials[pathname] = partial
		if err != nil {
			if err == io.EOF {
				// end of file for now, return
				return
			}
			glog.Infof("Failed to read updates from %q: %s", pathname, err)
			return
		}
	}
}

// readPartial reads blocks of 4096 bytes from the File, sending lines to the
// channel as it encounters newlines.  If EOF is encountered, the partial line
// is returned to be concatenated with on the next call.
func (t *Tailer) readPartial(f afero.File, partial string) (string, error) {
	b := make([]byte, 4096)
	n, err := f.Read(b)
	if err != nil {
		return partial, err // TODO(jaq): handle this path better
	}
	for i, width := 0, 0; i < len(b) && i < n; i += width {
		var rune rune
		rune, width = utf8.DecodeRune(b[i:])
		switch {
		case rune != '\n':
			partial += string(rune)
		default:
			// send off line for processing
			t.lines <- partial
			// reset accumulator
			partial = ""
		}
	}
	return partial, nil
}

// inode returns the inode number of a file, or 0 if the file has no underlying Sys implementation.
func inode(f os.FileInfo) uint64 {
	s := f.Sys()
	if s == nil {
		return 0
	}
	switch s := s.(type) {
	case *syscall.Stat_t:
		return s.Ino
	default:
		return 0
	}
}

// handleLogCreate handles both new and rotated log files.
func (t *Tailer) handleLogCreate(pathname string) {
	if !t.isWatching(pathname) {
		glog.V(1).Info("Not watching path %q, ignoring.\n", pathname)
		return
	}

	t.filesLock.Lock()
	fd, ok := t.files[pathname]
	t.filesLock.Unlock()
	if ok {
		s1, err := fd.Stat()
		if err != nil {
			glog.Infof("Stat failed on %q: %s", t.files[pathname].Name(), err)
			return
		}
		s2, err := t.fs.Stat(pathname)
		if err != nil {
			glog.Infof("Stat failed on %q: %s", pathname, err)
			return
		}
		if inode(s1) != inode(s2) {
			logRotations.Add(pathname, 1)
			// flush the old log, pathname is still an index into t.files with the old inode.
			t.handleLogUpdate(pathname)
			fd.Close()
			err := t.w.Remove(pathname)
			if err != nil {
				glog.Info("Failed removing watches on", pathname)
			}
			// Always seek to start on log rotation.
			glog.Infof("Seek to start on %s", pathname)
			t.openLogFile(pathname, true)
		} else {
			glog.Infof("Path %s already being watched, and inode not changed.",
				pathname)
		}
	} else {
		// Freshly opened log file, never seen before.
		t.openLogFile(pathname, true)
	}
}

// openLogFile opens a new log file at pathname, and optionally seeks to the
// start or end of the file. Rotated logs should start at the start, but logs
// opened for the first time start at the end.
func (t *Tailer) openLogFile(pathname string, seekStart bool) {
	if !t.isWatching(pathname) {
		glog.Infof("Not watching %q, ignoring.", pathname)
		return
	}

	d := path.Dir(pathname)
	if !t.isWatching(d) {
		err := t.w.Add(d)
		if err != nil {
			glog.Infof("Adding a create watch failed on %q: %s", d, err)
		}
		t.addWatched(d)
	}

	f, err := t.fs.Open(pathname)
	if err != nil {
		// Doesn't exist yet. We're watching the directory, so we'll pick it up
		// again on create; return successfully.
		if os.IsNotExist(err) {
			return
		}
		glog.Infof("Failed to open %q for reading: %s", pathname, err)
		logErrors.Add(pathname, 1)
		return
	}
	t.filesLock.Lock()
	t.files[pathname] = f

	if seekStart {
		t.files[pathname].Seek(0, os.SEEK_SET)
	} else {
		t.files[pathname].Seek(0, os.SEEK_END)
	}

	t.filesLock.Unlock()

	err = t.w.Add(pathname)
	if err != nil {
		glog.Infof("Adding a change watch failed on %q: %s", pathname, err)
	}

	glog.Infof("Tailing %s", pathname)

	// In case the new log has been written to already, attempt to read the first lines.
	t.handleLogUpdate(pathname)
}

// start is the main event loop for the Tailer.
// It receives notification of log file changes from the watcher channel, and
// handles them.
func (t *Tailer) run() {
	for e := range t.w.Events() {
		switch e := e.(type) {
		case watcher.UpdateEvent:
			t.handleLogUpdate(e.Pathname)
		case watcher.CreateEvent:
			t.handleLogCreate(e.Pathname)
		default:
			glog.Infof("Unexpected event %q", e)
		}
	}
	glog.Infof("Shutting down tailer.")
	close(t.lines)
}

// Close signals termination to the watcher.
func (t *Tailer) Close() {
	t.w.Close()
}
<|endoftext|>


<commit_before>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package azure

import (
	"fmt"
	"io/ioutil"
	. "launchpad.net/gocheck"
	"launchpad.net/juju-core/environs/config"
	"launchpad.net/juju-core/instance"
)

type EnvironProviderSuite struct {
	ProviderSuite
}

var _ = Suite(new(EnvironProviderSuite))

func (*EnvironProviderSuite) TestOpen(c *C) {
	prov := azureEnvironProvider{}
	attrs := makeAzureConfigMap(c)
	attrs["name"] = "my-shiny-new-env"
	cfg, err := config.New(attrs)
	c.Assert(err, IsNil)

	env, err := prov.Open(cfg)
	c.Assert(err, IsNil)

	c.Check(env.Name(), Equals, attrs["name"])
}

func (EnvironProviderSuite) TestOpenReturnsNilInterfaceUponFailure(c *C) {
	prov := azureEnvironProvider{}
	attrs := makeAzureConfigMap(c)
	// Make the config invalid.
	attrs["public-storage-account-name"] = ""
	cfg, err := config.New(attrs)
	c.Assert(err, IsNil)

	env, err := prov.Open(cfg)
	// When Open() fails (i.e. returns a non-nil error), it returns an
	// environs.Environ interface object with a nil value and a nil
	// type.
	c.Check(env, Equals, nil)
	c.Check(err, ErrorMatches, ".*must be specified both or none of them.*")
}

// writeWALASharedConfig creates a temporary file with a valid WALinux config
// built using the given parameters. The file will be cleaned up at the end
// of the test calling this method.
func writeWALASharedConfig(c *C, deploymentId string, deploymentName string, internalAddress string) string {
	configTemplateXML := `
	<SharedConfig version="1.0.0.0" goalStateIncarnation="1">
	  <Deployment name="%s" guid="{495985a8-8e5a-49aa-826f-d1f7f51045b6}" incarnation="0">
	    <Service name="%s" guid="{00000000-0000-0000-0000-000000000000}" />
	    <ServiceInstance name="%s" guid="{9806cac7-e566-42b8-9ecb-de8da8f69893}" />
	  </Deployment>
	  <Instances>
            <Instance id="gwaclroleldc1o5p" address="%s">
            </Instance>
          </Instances>
        </SharedConfig>`
	config := fmt.Sprintf(configTemplateXML, deploymentId, deploymentName, deploymentId, internalAddress)
	file, err := ioutil.TempFile(c.MkDir(), "")
	c.Assert(err, IsNil)
	filename := file.Name()
	err = ioutil.WriteFile(filename, []byte(config), 0644)
	c.Assert(err, IsNil)
	return filename
}

// overrideWALASharedConfig:
// - creates a temporary file with a valid WALinux config built using the
// given parameters.  The file will be cleaned up at the end of the test
// calling this method.
// - monkey patches the value of '_WALAConfigPath' (the path to the WALA
// configuration file) so that it contains the path to the temporary file.
// overrideWALASharedConfig returns a cleanup method that the caller *must*
// call in order to restore the original value of '_WALAConfigPath'
func overrideWALASharedConfig(c *C, deploymentId, deploymentName, internalAddress string) func() {
	filename := writeWALASharedConfig(c, deploymentId, deploymentName,
		internalAddress)
	oldConfigPath := _WALAConfigPath
	_WALAConfigPath = filename
	// Return cleanup method to restore the original value of
	// '_WALAConfigPath'.
	return func() {
		_WALAConfigPath = oldConfigPath
	}
}

func (*EnvironProviderSuite) TestParseWALASharedConfig(c *C) {
	deploymentId := "b6de4c4c7d4a49c39270e0c57481fd9b"
	deploymentName := "gwaclmachineex95rsek"
	internalAddress := "10.76.200.59"

	cleanup := overrideWALASharedConfig(c, deploymentId, deploymentName, internalAddress)
	defer cleanup()

	config, err := parseWALAConfig()
	c.Assert(err, IsNil)
	c.Check(config.Deployment.Name, Equals, deploymentId)
	c.Check(config.Deployment.Service.Name, Equals, deploymentName)
	c.Check(config.Instances[0].Address, Equals, internalAddress)
}

func (*EnvironProviderSuite) TestConfigGetDeploymentFQDN(c *C) {
	deploymentId := "b6de4c4c7d4a49c39270e0c57481fd9b"
	serviceName := "r12slechtstschrijvende5"
	config := WALASharedConfig{
		Deployment: WALADeployment{
			Name:    deploymentId,
			Service: WALADeploymentService{Name: serviceName},
		},
	}

	c.Check(config.getDeploymentFQDN(), Equals, serviceName+".cloudapp.net")
}

func (*EnvironProviderSuite) TestConfigGetDeploymentHostname(c *C) {
	deploymentName := "gwaclmachineex95rsek"
	config := WALASharedConfig{Deployment: WALADeployment{Name: "id", Service: WALADeploymentService{Name: deploymentName}}}

	c.Check(config.getDeploymentName(), Equals, deploymentName)
}

func (*EnvironProviderSuite) TestConfigGetInternalIP(c *C) {
	internalAddress := "10.76.200.59"
	config := WALASharedConfig{Instances: []WALAInstance{{Address: internalAddress}}}

	c.Check(config.getInternalIP(), Equals, internalAddress)
}

func (*EnvironProviderSuite) TestPublicAddress(c *C) {
	deploymentName := "b6de4c4c7d4a49c39270e0c57481fd9b"
	cleanup := overrideWALASharedConfig(c, "deploymentid", deploymentName, "10.76.200.59")
	defer cleanup()

	expectedAddress := deploymentName + ".cloudapp.net"
	prov := azureEnvironProvider{}
	pubAddress, err := prov.PublicAddress()
	c.Assert(err, IsNil)
	c.Check(pubAddress, Equals, expectedAddress)
}

// azureEnvironProvider.PrivateAddress() currently returns the public address
// of the instance.  We need to figure out how to do instance-to-instance
// communication using the private IPs before we can use the Azure private
// address.
func (*EnvironProviderSuite) TestPrivateAddressReturnsPublicAddress(c *C) {
	deploymentName := "b6de4c4c7d4a49c39270e0c57481fd9b"
	cleanup := overrideWALASharedConfig(c, "deploymentid", deploymentName, "10.76.200.59")
	defer cleanup()

	expectedAddress := deploymentName + ".cloudapp.net"
	prov := azureEnvironProvider{}
	pubAddress, err := prov.PrivateAddress()
	c.Assert(err, IsNil)
	c.Check(pubAddress, Equals, expectedAddress)
}

/*
func (*EnvironProviderSuite) TestPrivateAddress(c *C) {
	internalAddress := "10.76.200.59"
	cleanup := overrideWALASharedConfig(c, "deploy-id", "name", internalAddress)
	defer cleanup()

	prov := azureEnvironProvider{}
	privAddress, err := prov.PrivateAddress()
	c.Assert(err, IsNil)
	c.Check(privAddress, Equals, internalAddress)
}
*/

func (*EnvironProviderSuite) TestInstanceId(c *C) {
	deploymentName := "deploymentname"
	cleanup := overrideWALASharedConfig(c, "deploy-id", deploymentName, "10.76.200.59")
	defer cleanup()

	prov := azureEnvironProvider{}
	instanceId, err := prov.InstanceId()
	c.Assert(err, IsNil)
	c.Check(instanceId, Equals, instance.Id(deploymentName))
}
<commit_msg>Resolved conflict.<commit_after>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package azure

import (
	"fmt"
	"io/ioutil"
	. "launchpad.net/gocheck"
	"launchpad.net/juju-core/environs/config"
	"launchpad.net/juju-core/instance"
)

type EnvironProviderSuite struct {
	ProviderSuite
}

var _ = Suite(new(EnvironProviderSuite))

func (*EnvironProviderSuite) TestOpen(c *C) {
	prov := azureEnvironProvider{}
	attrs := makeAzureConfigMap(c)
	attrs["name"] = "my-shiny-new-env"
	cfg, err := config.New(attrs)
	c.Assert(err, IsNil)

	env, err := prov.Open(cfg)
	c.Assert(err, IsNil)

	c.Check(env.Name(), Equals, attrs["name"])
}

func (EnvironProviderSuite) TestOpenReturnsNilInterfaceUponFailure(c *C) {
	prov := azureEnvironProvider{}
	attrs := makeAzureConfigMap(c)
	// Make the config invalid.
	attrs["public-storage-account-name"] = ""
	cfg, err := config.New(attrs)
	c.Assert(err, IsNil)

	env, err := prov.Open(cfg)
	// When Open() fails (i.e. returns a non-nil error), it returns an
	// environs.Environ interface object with a nil value and a nil
	// type.
	c.Check(env, Equals, nil)
	c.Check(err, ErrorMatches, ".*must be specified both or none of them.*")
}

// writeWALASharedConfig creates a temporary file with a valid WALinux config
// built using the given parameters. The file will be cleaned up at the end
// of the test calling this method.
func writeWALASharedConfig(c *C, deploymentId string, deploymentName string, internalAddress string) string {
	configTemplateXML := `
	<SharedConfig version="1.0.0.0" goalStateIncarnation="1">
	  <Deployment name="%s" guid="{495985a8-8e5a-49aa-826f-d1f7f51045b6}" incarnation="0">
	    <Service name="%s" guid="{00000000-0000-0000-0000-000000000000}" />
	    <ServiceInstance name="%s" guid="{9806cac7-e566-42b8-9ecb-de8da8f69893}" />
	  </Deployment>
	  <Instances>
            <Instance id="gwaclroleldc1o5p" address="%s">
            </Instance>
          </Instances>
        </SharedConfig>`
	config := fmt.Sprintf(configTemplateXML, deploymentId, deploymentName, deploymentId, internalAddress)
	file, err := ioutil.TempFile(c.MkDir(), "")
	c.Assert(err, IsNil)
	filename := file.Name()
	err = ioutil.WriteFile(filename, []byte(config), 0644)
	c.Assert(err, IsNil)
	return filename
}

// overrideWALASharedConfig:
// - creates a temporary file with a valid WALinux config built using the
// given parameters.  The file will be cleaned up at the end of the test
// calling this method.
// - monkey patches the value of '_WALAConfigPath' (the path to the WALA
// configuration file) so that it contains the path to the temporary file.
// overrideWALASharedConfig returns a cleanup method that the caller *must*
// call in order to restore the original value of '_WALAConfigPath'
func overrideWALASharedConfig(c *C, deploymentId, deploymentName, internalAddress string) func() {
	filename := writeWALASharedConfig(c, deploymentId, deploymentName,
		internalAddress)
	oldConfigPath := _WALAConfigPath
	_WALAConfigPath = filename
	// Return cleanup method to restore the original value of
	// '_WALAConfigPath'.
	return func() {
		_WALAConfigPath = oldConfigPath
	}
}

func (*EnvironProviderSuite) TestParseWALASharedConfig(c *C) {
	deploymentId := "b6de4c4c7d4a49c39270e0c57481fd9b"
	deploymentName := "gwaclmachineex95rsek"
	internalAddress := "10.76.200.59"

	cleanup := overrideWALASharedConfig(c, deploymentId, deploymentName, internalAddress)
	defer cleanup()

	config, err := parseWALAConfig()
	c.Assert(err, IsNil)
	c.Check(config.Deployment.Name, Equals, deploymentId)
	c.Check(config.Deployment.Service.Name, Equals, deploymentName)
	c.Check(config.Instances[0].Address, Equals, internalAddress)
}

func (*EnvironProviderSuite) TestConfigGetDeploymentFQDN(c *C) {
	deploymentId := "b6de4c4c7d4a49c39270e0c57481fd9b"
	serviceName := "gwaclr12slechtstschrijvende5"
	config := WALASharedConfig{
		Deployment: WALADeployment{
			Name:    deploymentId,
			Service: WALADeploymentService{Name: serviceName},
		},
	}

	c.Check(config.getDeploymentFQDN(), Equals, serviceName+".cloudapp.net")
}

func (*EnvironProviderSuite) TestConfigGetDeploymentHostname(c *C) {
	deploymentName := "gwaclmachineex95rsek"
	config := WALASharedConfig{Deployment: WALADeployment{Name: "id", Service: WALADeploymentService{Name: deploymentName}}}

	c.Check(config.getDeploymentName(), Equals, deploymentName)
}

func (*EnvironProviderSuite) TestConfigGetInternalIP(c *C) {
	internalAddress := "10.76.200.59"
	config := WALASharedConfig{Instances: []WALAInstance{{Address: internalAddress}}}

	c.Check(config.getInternalIP(), Equals, internalAddress)
}

func (*EnvironProviderSuite) TestPublicAddress(c *C) {
	deploymentName := "b6de4c4c7d4a49c39270e0c57481fd9b"
	cleanup := overrideWALASharedConfig(c, "deploymentid", deploymentName, "10.76.200.59")
	defer cleanup()

	expectedAddress := deploymentName + ".cloudapp.net"
	prov := azureEnvironProvider{}
	pubAddress, err := prov.PublicAddress()
	c.Assert(err, IsNil)
	c.Check(pubAddress, Equals, expectedAddress)
}

// azureEnvironProvider.PrivateAddress() currently returns the public address
// of the instance.  We need to figure out how to do instance-to-instance
// communication using the private IPs before we can use the Azure private
// address.
func (*EnvironProviderSuite) TestPrivateAddressReturnsPublicAddress(c *C) {
	deploymentName := "b6de4c4c7d4a49c39270e0c57481fd9b"
	cleanup := overrideWALASharedConfig(c, "deploymentid", deploymentName, "10.76.200.59")
	defer cleanup()

	expectedAddress := deploymentName + ".cloudapp.net"
	prov := azureEnvironProvider{}
	pubAddress, err := prov.PrivateAddress()
	c.Assert(err, IsNil)
	c.Check(pubAddress, Equals, expectedAddress)
}

/*
func (*EnvironProviderSuite) TestPrivateAddress(c *C) {
	internalAddress := "10.76.200.59"
	cleanup := overrideWALASharedConfig(c, "deploy-id", "name", internalAddress)
	defer cleanup()

	prov := azureEnvironProvider{}
	privAddress, err := prov.PrivateAddress()
	c.Assert(err, IsNil)
	c.Check(privAddress, Equals, internalAddress)
}
*/

func (*EnvironProviderSuite) TestInstanceId(c *C) {
	deploymentName := "deploymentname"
	cleanup := overrideWALASharedConfig(c, "deploy-id", deploymentName, "10.76.200.59")
	defer cleanup()

	prov := azureEnvironProvider{}
	instanceId, err := prov.InstanceId()
	c.Assert(err, IsNil)
	c.Check(instanceId, Equals, instance.Id(deploymentName))
}
<|endoftext|>


<commit_before>// Package sanitize provides functions for sanitizing text.
package sanitize

import (
	"bytes"
	"html"
	"html/template"
	"io"
	"path"
	"regexp"
	"strings"

	parser "golang.org/x/net/html"
)

var (
	ignoreTags = []string{"title", "script", "style", "iframe", "frame", "frameset", "noframes", "noembed", "embed", "applet", "object", "base"}

	defaultTags = []string{"h1", "h2", "h3", "h4", "h5", "h6", "div", "span", "hr", "p", "br", "b", "i", "strong", "em", "ol", "ul", "li", "a", "img"}

	defaultAttributes = []string{"id", "class", "src", "href", "title", "alt", "name", "rel"}
)

// HTMLAllowing sanitizes html, allowing some tags.
// Arrays of allowed tags and allowed attributes may optionally be passed as the second and third arguments.
func HTMLAllowing(s string, args ...[]string) (string, error) {

	allowedTags := defaultTags
	if len(args) > 0 {
		allowedTags = args[0]
	}
	allowedAttributes := defaultAttributes
	if len(args) > 1 {
		allowedAttributes = args[1]
	}

	// Parse the html
	tokenizer := parser.NewTokenizer(strings.NewReader(s))

	buffer := bytes.NewBufferString("")
	ignore := ""

	for {
		tokenType := tokenizer.Next()
		token := tokenizer.Token()

		switch tokenType {

		case parser.ErrorToken:
			err := tokenizer.Err()
			if err == io.EOF {
				return buffer.String(), nil
			}
			return "", err

		case parser.StartTagToken:

			if len(ignore) == 0 && includes(allowedTags, token.Data) {
				token.Attr = cleanAttributes(token.Attr, allowedAttributes)
				buffer.WriteString(token.String())
			} else if includes(ignoreTags, token.Data) {
				ignore = token.Data
			}

		case parser.SelfClosingTagToken:

			if len(ignore) == 0 && includes(allowedTags, token.Data) {
				token.Attr = cleanAttributes(token.Attr, allowedAttributes)
				buffer.WriteString(token.String())
			} else if token.Data == ignore {
				ignore = ""
			}

		case parser.EndTagToken:
			if len(ignore) == 0 && includes(allowedTags, token.Data) {
				token.Attr = []parser.Attribute{}
				buffer.WriteString(token.String())
			} else if token.Data == ignore {
				ignore = ""
			}

		case parser.TextToken:
			// We allow text content through, unless ignoring this entire tag and its contents (including other tags)
			if ignore == "" {
				buffer.WriteString(token.String())
			}
		case parser.CommentToken:
			// We ignore comments by default
		case parser.DoctypeToken:
			// We ignore doctypes by default - html5 does not require them and this is intended for sanitizing snippets of text
		default:
			// We ignore unknown token types by default

		}

	}

}

// HTML strips html tags, replace common entities, and escapes <>&;'" in the result.
// Note the returned text may contain entities as it is escaped by HTMLEscapeString, and most entities are not translated.
func HTML(s string) string {

	output := ""

	// Shortcut strings with no tags in them
	if !strings.ContainsAny(s, "<>") {
		output = s
	} else {

		rp, _ := regexp.Compile("(


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net/url"
	"os"
	"path"
	"strings"
	"time"

	"github.com/leesper/couchdb-golang"
)

type tuple struct {
	left, right string
}

func init() {
	log.SetFlags(log.Lshortfile)
}

func findPath(s string) (string, string) {
	if s == "." {
		return couchdb.DefaultBaseURL, ""
	}

	if !strings.HasPrefix(s, "http") {
		return couchdb.DefaultBaseURL, s
	}

	u, err := url.Parse(s)
	if err != nil {
		log.Fatal(err)
	}

	var base string
	if u.User != nil {
		base = fmt.Sprintf("%s://%s@%s", u.Scheme, u.User.String(), u.Host)
	} else {
		base = fmt.Sprintf("%s://%s", u.Scheme, u.Host)
	}
	res, err := couchdb.NewResource(base, nil)
	if err != nil {
		log.Fatal(err)
	}
	_, data, err := res.GetJSON("", nil, nil)
	if err != nil {
		log.Fatal(err)
	}

	info := map[string]interface{}{}
	err = json.Unmarshal(data, &info)
	if err != nil {
		log.Fatal(err)
	}

	if _, ok := info["couchdb"]; !ok {
		log.Fatal(fmt.Sprintf("%s doe not appear to be a CouchDB", s))
	}

	return base, strings.TrimLeft(u.Path, "/")
}

func main() {
	isContinous := flag.Bool("continous", false, "trigger continous replication in couchdb")
	isCompact := flag.Bool("compact", false, "compact target database after replication")
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "usage: %s [options] <source> <target>\n", os.Args[0])
		flag.PrintDefaults()
	}
	flag.Parse()
	if len(os.Args) != 3 {
		log.Fatal("need source and target arguments")
	}

	src, tgt := os.Args[1], os.Args[2]

	sbase, spath := findPath(src)
	source, err := couchdb.NewServer(sbase)
	if err != nil {
		log.Fatal(err)
	}

	tbase, tpath := findPath(tgt)
	target, err := couchdb.NewServer(tbase)
	if err != nil {
		log.Fatal(err)
	}

	// check dtabase name specs
	if strings.Contains(tpath, "*") {
		log.Fatal("invalid target path: must be single db or empty")
	}

	all, err := source.DBs()
	if err != nil {
		log.Fatal(err)
	}

	if spath == "" {
		log.Fatal("source database must be specified")
	}

	sources := []string{}
	for _, db := range all {
		if !strings.HasPrefix(db, "_") { // skip reserved names
			var ok bool
			ok, err = path.Match(spath, db)
			if err != nil {
				log.Fatal(err)
			}
			if ok {
				sources = append(sources, db)
			}
		}
	}

	if len(sources) == 0 {
		log.Fatalf("no source databases match glob %s", spath)
	}

	databases := []tuple{}
	if len(sources) > 1 && tpath != "" {
		log.Fatal("target path must be empty with multiple sources")
	} else if len(sources) == 1 {
		databases = append(databases, tuple{sources[0], tpath})
	} else {
		for _, source := range sources {
			databases = append(databases, tuple{source, source})
		}
	}

	// actual replication
	for _, t := range databases {
		sdb, tdb := t.left, t.right
		start := time.Now()
		fmt.Printf("%s -> %s\n", sdb, tdb)

		if !target.Contains(tdb) {
			_, err = target.Create(tdb)
			if err != nil {
				log.Println(err)
				continue
			}
		}

		sdb = fmt.Sprintf("%s/%s", sbase, sdb)
		var opts map[string]interface{}
		if *isContinous {
			opts = map[string]interface{}{"continuous": true}
		}
		_, err = target.Replicate(sdb, tdb, opts)
		if err != nil {
			log.Println(err)
		}
		fmt.Printf("%.1fs\n", time.Now().Sub(start).Seconds())
	}

	if *isCompact {
		for _, t := range databases {
			tdb := t.right
			fmt.Println("compact", tdb)
			targetDB, _ := target.Get(tdb)
			targetDB.Compact()
		}
	}
}
<commit_msg>spelling: doe -> does<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net/url"
	"os"
	"path"
	"strings"
	"time"

	"github.com/leesper/couchdb-golang"
)

type tuple struct {
	left, right string
}

func init() {
	log.SetFlags(log.Lshortfile)
}

func findPath(s string) (string, string) {
	if s == "." {
		return couchdb.DefaultBaseURL, ""
	}

	if !strings.HasPrefix(s, "http") {
		return couchdb.DefaultBaseURL, s
	}

	u, err := url.Parse(s)
	if err != nil {
		log.Fatal(err)
	}

	var base string
	if u.User != nil {
		base = fmt.Sprintf("%s://%s@%s", u.Scheme, u.User.String(), u.Host)
	} else {
		base = fmt.Sprintf("%s://%s", u.Scheme, u.Host)
	}
	res, err := couchdb.NewResource(base, nil)
	if err != nil {
		log.Fatal(err)
	}
	_, data, err := res.GetJSON("", nil, nil)
	if err != nil {
		log.Fatal(err)
	}

	info := map[string]interface{}{}
	err = json.Unmarshal(data, &info)
	if err != nil {
		log.Fatal(err)
	}

	if _, ok := info["couchdb"]; !ok {
		log.Fatal(fmt.Sprintf("%s does not appear to be a CouchDB", s))
	}

	return base, strings.TrimLeft(u.Path, "/")
}

func main() {
	isContinous := flag.Bool("continous", false, "trigger continous replication in couchdb")
	isCompact := flag.Bool("compact", false, "compact target database after replication")
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "usage: %s [options] <source> <target>\n", os.Args[0])
		flag.PrintDefaults()
	}
	flag.Parse()
	if len(os.Args) != 3 {
		log.Fatal("need source and target arguments")
	}

	src, tgt := os.Args[1], os.Args[2]

	sbase, spath := findPath(src)
	source, err := couchdb.NewServer(sbase)
	if err != nil {
		log.Fatal(err)
	}

	tbase, tpath := findPath(tgt)
	target, err := couchdb.NewServer(tbase)
	if err != nil {
		log.Fatal(err)
	}

	// check dtabase name specs
	if strings.Contains(tpath, "*") {
		log.Fatal("invalid target path: must be single db or empty")
	}

	all, err := source.DBs()
	if err != nil {
		log.Fatal(err)
	}

	if spath == "" {
		log.Fatal("source database must be specified")
	}

	sources := []string{}
	for _, db := range all {
		if !strings.HasPrefix(db, "_") { // skip reserved names
			var ok bool
			ok, err = path.Match(spath, db)
			if err != nil {
				log.Fatal(err)
			}
			if ok {
				sources = append(sources, db)
			}
		}
	}

	if len(sources) == 0 {
		log.Fatalf("no source databases match glob %s", spath)
	}

	databases := []tuple{}
	if len(sources) > 1 && tpath != "" {
		log.Fatal("target path must be empty with multiple sources")
	} else if len(sources) == 1 {
		databases = append(databases, tuple{sources[0], tpath})
	} else {
		for _, source := range sources {
			databases = append(databases, tuple{source, source})
		}
	}

	// actual replication
	for _, t := range databases {
		sdb, tdb := t.left, t.right
		start := time.Now()
		fmt.Printf("%s -> %s\n", sdb, tdb)

		if !target.Contains(tdb) {
			_, err = target.Create(tdb)
			if err != nil {
				log.Println(err)
				continue
			}
		}

		sdb = fmt.Sprintf("%s/%s", sbase, sdb)
		var opts map[string]interface{}
		if *isContinous {
			opts = map[string]interface{}{"continuous": true}
		}
		_, err = target.Replicate(sdb, tdb, opts)
		if err != nil {
			log.Println(err)
		}
		fmt.Printf("%.1fs\n", time.Now().Sub(start).Seconds())
	}

	if *isCompact {
		for _, t := range databases {
			tdb := t.right
			fmt.Println("compact", tdb)
			targetDB, _ := target.Get(tdb)
			targetDB.Compact()
		}
	}
}
<|endoftext|>


<commit_before>package docker

import (
	"fmt"
	"log"
	"net/http"

	"github.com/cpg1111/maestrod/manager"

	dockerEngine "github.com/docker/engine-api/client"
	dockerTypes "github.com/docker/engine-api/types"
	dockerContainer "github.com/docker/engine-api/types/container"
	dockerNetwork "github.com/docker/engine-api/types/network"
	"golang.org/x/net/context"
)

type Driver struct {
	manager.Driver
	client      *dockerEngine.Client
	containerID string
	image       string
	cmd         []string
}

func New(host, apiVersion, maestroVersion, name string) (*Driver, error) {
	hClient := &http.Client{}
	dClient, dockerErr := dockerEngine.NewClient(host, apiVersion, hClient, make(map[string]string))
	if dockerErr != nil {
		return nil, dockerErr
	}
	return &Driver{
		client:      dClient,
		containerID: name,
		image:       fmt.Sprintf("cpg1111/maestro:%s", maestroVersion),
	}, nil
}

func (d *Driver) needToPull(ctx context.Context) (bool, error) {
	listOptions := &dockerTypes.ImageListOptions{
		MatchName: "cpg1111/maestro",
		All:       true,
	}
	images, listErr := d.client.ImageList(ctx, *listOptions)
	if listErr != nil {
		return false, listErr
	}
	return len(images) > 0, nil
}

func (d *Driver) pull(ctx context.Context) error {
	pullOptions := dockerTypes.ImagePullOptions{}
	res, resErr := d.client.ImagePull(ctx, d.image, pullOptions)
	defer res.Close()
	if resErr != nil {
		return resErr
	}
	resp := make([]byte, 4096)
	_, readErr := res.Read(resp)
	if readErr != nil {
		return readErr
	}
	log.Println((string)(resp))
	return nil
}

func (d *Driver) getContainerConfig() *dockerContainer.Config {
	labels := make(map[string]string)
	labels["NAME"] = d.containerID
	timeout := 5
	return &dockerContainer.Config{
		User:         "maestro",
		AttachStdin:  false,
		AttachStdout: true,
		AttachStderr: true,
		Tty:          false,
		OpenStdin:    false,
		StdinOnce:    false,
		Cmd:          d.cmd,
		Healthcheck: &dockerContainer.HealthConfig{
			Test: []string{""},
		},
		ArgsEscaped:     true,
		Image:           d.image,
		Volumes:         make(map[string]struct{}),
		WorkingDir:      "/",
		NetworkDisabled: false,
		Labels:          labels,
		StopSignal:      "SIGTERM",
		StopTimeout:     &timeout,
	}
}

func (d *Driver) getHostConfig() *dockerContainer.HostConfig {
	logConf := make(map[string]string)
	return &dockerContainer.HostConfig{
		Binds:           []string{},
		ContainerIDFile: "/tmp/containers",
		LogConfig: dockerContainer.LogConfig{
			Type:   "json-file",
			Config: logConf,
		},
		NetworkMode: "host",
		RestartPolicy: dockerContainer.RestartPolicy{
			Name:              "never",
			MaximumRetryCount: 0,
		},
		AutoRemove:      true,
		VolumeDriver:    "local",
		Privileged:      false,
		PublishAllPorts: false,
		ReadonlyRootfs:  false,
	}
}

func (d *Driver) getNetworkConfig() *dockerNetwork.NetworkingConfig {
	return &dockerNetwork.NetworkingConfig{}
}

func (d *Driver) create(ctx context.Context) error {
	createResp, err := d.client.ContainerCreate(ctx, d.getContainerConfig(), d.getHostConfig(), d.getNetworkConfig(), d.containerID)
	if err != nil {
		return err
	}
	log.Println(createResp)
	return nil
}

func (d Driver) Run(args []string) error {
	ctx := context.Background()
	needToPull, checkErr := d.needToPull(ctx)
	if checkErr != nil {
		return checkErr
	}
	if needToPull {
		pullErr := d.pull(ctx)
		if pullErr != nil {
			return pullErr
		}
	}
	createErr := d.create(ctx)
	if createErr != nil {
		return createErr
	}
	return nil
}
<commit_msg>Update docker manager driver<commit_after>package docker

import (
	"fmt"
	"log"
	"net/http"

	"github.com/cpg1111/maestrod/manager"

	dockerEngine "github.com/docker/engine-api/client"
	dockerTypes "github.com/docker/engine-api/types"
	dockerContainer "github.com/docker/engine-api/types/container"
	dockerNetwork "github.com/docker/engine-api/types/network"
	"golang.org/x/net/context"
)

type Driver struct {
	manager.Driver
	client      *dockerEngine.Client
	containerID string
	image       string
	cmd         []string
}

func New(host, apiVersion, maestroVersion, name string) (*Driver, error) {
	hClient := &http.Client{}
	dClient, dockerErr := dockerEngine.NewClient(host, apiVersion, hClient, make(map[string]string))
	if dockerErr != nil {
		return nil, dockerErr
	}
	return &Driver{
		client:      dClient,
		containerID: name,
		image:       fmt.Sprintf("cpg1111/maestro:%s", maestroVersion),
	}, nil
}

func (d *Driver) needToPull(ctx context.Context) (bool, error) {
	listOptions := &dockerTypes.ImageListOptions{
		MatchName: "cpg1111/maestro",
		All:       true,
	}
	images, listErr := d.client.ImageList(ctx, *listOptions)
	if listErr != nil {
		return false, listErr
	}
	return len(images) > 0, nil
}

func (d *Driver) pull(ctx context.Context) error {
	pullOptions := dockerTypes.ImagePullOptions{}
	res, resErr := d.client.ImagePull(ctx, d.image, pullOptions)
	defer res.Close()
	if resErr != nil {
		return resErr
	}
	resp := make([]byte, 4096)
	_, readErr := res.Read(resp)
	if readErr != nil {
		return readErr
	}
	log.Println((string)(resp))
	return nil
}

func (d *Driver) getContainerConfig() *dockerContainer.Config {
	labels := make(map[string]string)
	labels["NAME"] = d.containerID
	timeout := 5
	return &dockerContainer.Config{
		User:         "maestro",
		AttachStdin:  false,
		AttachStdout: true,
		AttachStderr: true,
		Tty:          false,
		OpenStdin:    false,
		StdinOnce:    false,
		Cmd:          d.cmd,
		Healthcheck: &dockerContainer.HealthConfig{
			Test: []string{""},
		},
		ArgsEscaped:     true,
		Image:           d.image,
		Volumes:         make(map[string]struct{}),
		WorkingDir:      "/",
		NetworkDisabled: false,
		Labels:          labels,
		StopSignal:      "SIGTERM",
		StopTimeout:     &timeout,
	}
}

func (d *Driver) getHostConfig() *dockerContainer.HostConfig {
	logConf := make(map[string]string)
	return &dockerContainer.HostConfig{
		Binds:           []string{},
		ContainerIDFile: "/tmp/containers",
		LogConfig: dockerContainer.LogConfig{
			Type:   "json-file",
			Config: logConf,
		},
		NetworkMode: "host",
		RestartPolicy: dockerContainer.RestartPolicy{
			Name:              "never",
			MaximumRetryCount: 0,
		},
		AutoRemove:      true,
		VolumeDriver:    "local",
		Privileged:      false,
		PublishAllPorts: false,
		ReadonlyRootfs:  false,
	}
}

func (d *Driver) getNetworkConfig() *dockerNetwork.NetworkingConfig {
	return &dockerNetwork.NetworkingConfig{}
}

func (d *Driver) create(ctx context.Context) error {
	createResp, err := d.client.ContainerCreate(ctx, d.getContainerConfig(), d.getHostConfig(), d.getNetworkConfig(), d.containerID)
	if err != nil {
		return err
	}
	log.Println(createResp)
	return nil
}

func (d Driver) Run(args []string) error {
	d.cmd = args
	ctx := context.Background()
	needToPull, checkErr := d.needToPull(ctx)
	if checkErr != nil {
		return checkErr
	}
	if needToPull {
		pullErr := d.pull(ctx)
		if pullErr != nil {
			return pullErr
		}
	}
	createErr := d.create(ctx)
	if createErr != nil {
		return createErr
	}
	return nil
}
<|endoftext|>


<commit_before>package scan

import (
	"bytes"
	"crypto/x509"
	"fmt"
	"time"

	"github.com/cloudflare/cf-tls/tls"
	"github.com/cloudflare/cfssl/helpers"
)

// PKI contains scanners for the Public Key Infrastructure.
var PKI = &Family{
	Description: "Scans for the Public Key Infrastructure",
	Scanners: map[string]*Scanner{
		"ChainExpiration": {
			"Host's chain hasn't expired and won't expire in the next 30 days",
			chainExpiration,
		},
		"ChainValidation": {
			"All certificates in host's chain are valid",
			chainValidation,
		},
		"MultipleCerts": {
			"Host serves same certificate chain across all IPs",
			multipleCerts,
		},
	},
}

// getChain is a helper function that retreives the host's certificate chain.
func getChain(host string, config *tls.Config) (chain []*x509.Certificate, err error) {
	var conn *tls.Conn
	conn, err = tls.DialWithDialer(Dialer, Network, host, config)
	if err != nil {
		return
	}

	err = conn.Close()
	if err != nil {
		return
	}

	chain = conn.ConnectionState().PeerCertificates
	if len(chain) == 0 {
		err = fmt.Errorf("%s returned empty certificate chain", host)
	}
	return
}

type expiration time.Time

func (e expiration) String() string {
	return time.Time(e).Format("Jan 2 15:04:05 2006 MST")
}

func chainExpiration(host string) (grade Grade, output Output, err error) {
	chain, err := getChain(host, defaultTLSConfig(host))
	if err != nil {
		return
	}

	e := helpers.ExpiryTime(chain)
	if e == nil {
		return
	}
	expirationTime := *e
	output = expirationTime

	if time.Now().After(expirationTime) {
		return
	}

	// Warn if cert will expire in the next 30 days
	if time.Now().Add(time.Hour * 24 * 30).After(expirationTime) {
		grade = Warning
		return
	}

	grade = Good
	return
}

func chainValidation(host string) (grade Grade, output Output, err error) {
	chain, err := getChain(host, defaultTLSConfig(host))
	if err != nil {
		return
	}

	var warnings []string

	for i := 0; i < len(chain)-1; i++ {
		cert, parent := chain[i], chain[i+1]

		if !parent.IsCA {
			err = fmt.Errorf("%s is not a CA", parent.Subject.CommonName)
			return
		}

		if !bytes.Equal(cert.AuthorityKeyId, parent.SubjectKeyId) {
			err = fmt.Errorf("%s AuthorityKeyId differs from %s SubjectKeyId", cert.Subject.CommonName, parent.Subject.CommonName)
			return
		}

		if err = cert.CheckSignatureFrom(parent); err != nil {
			return
		}

		// TODO: Check CRL and OCSP revocation.

		switch cert.SignatureAlgorithm {
		case x509.ECDSAWithSHA1:
			warnings = append(warnings, fmt.Sprintf("%s is signed by ECDSAWithSHA1", cert.Subject.CommonName))
		case x509.SHA1WithRSA:
			warnings = append(warnings, fmt.Sprintf("%s is signed by RSAWithSHA1", cert.Subject.CommonName))
		}
	}

	if len(warnings) == 0 {
		grade = Good
	} else {
		grade = Warning
		output = warnings
	}
	return
}

func multipleCerts(host string) (grade Grade, output Output, err error) {
	config := defaultTLSConfig(host)

	firstChain, err := getChain(host, config)
	if err != nil {
		return
	}

	grade, _, err = multiscan(host, func(addrport string) (g Grade, o Output, e error) {
		g = Good
		chain, e1 := getChain(addrport, config)
		if e1 != nil {
			return
		}

		if !chain[0].Equal(firstChain[0]) {
			e = fmt.Errorf("%s not equal to %s", chain[0].Subject.CommonName, firstChain[0].Subject.CommonName)
			g = Bad
			return
		}
		return
	})
	return
}
<commit_msg>scan: add check ChainValidation for to verify cert against system root store<commit_after>package scan

import (
	"bytes"
	"crypto/x509"
	"fmt"
	"net"
	"time"

	"github.com/cloudflare/cf-tls/tls"
	"github.com/cloudflare/cfssl/helpers"
)

// PKI contains scanners for the Public Key Infrastructure.
var PKI = &Family{
	Description: "Scans for the Public Key Infrastructure",
	Scanners: map[string]*Scanner{
		"ChainExpiration": {
			"Host's chain hasn't expired and won't expire in the next 30 days",
			chainExpiration,
		},
		"ChainValidation": {
			"All certificates in host's chain are valid",
			chainValidation,
		},
		"MultipleCerts": {
			"Host serves same certificate chain across all IPs",
			multipleCerts,
		},
	},
}

// getChain is a helper function that retreives the host's certificate chain.
func getChain(host string, config *tls.Config) (chain []*x509.Certificate, err error) {
	var conn *tls.Conn
	conn, err = tls.DialWithDialer(Dialer, Network, host, config)
	if err != nil {
		return
	}

	err = conn.Close()
	if err != nil {
		return
	}

	chain = conn.ConnectionState().PeerCertificates
	if len(chain) == 0 {
		err = fmt.Errorf("%s returned empty certificate chain", host)
	}
	return
}

type expiration time.Time

func (e expiration) String() string {
	return time.Time(e).Format("Jan 2 15:04:05 2006 MST")
}

func chainExpiration(host string) (grade Grade, output Output, err error) {
	chain, err := getChain(host, defaultTLSConfig(host))
	if err != nil {
		return
	}

	e := helpers.ExpiryTime(chain)
	if e == nil {
		return
	}
	expirationTime := *e
	output = expirationTime

	if time.Now().After(expirationTime) {
		return
	}

	// Warn if cert will expire in the next 30 days
	if time.Now().Add(time.Hour * 24 * 30).After(expirationTime) {
		grade = Warning
		return
	}

	grade = Good
	return
}

func chainValidation(host string) (grade Grade, output Output, err error) {
	chain, err := getChain(host, defaultTLSConfig(host))
	if err != nil {
		return
	}

	h, _, err := net.SplitHostPort(host)
	if err != nil {
		return
	}

	if err = chain[0].VerifyHostname(h); err != nil {
		return
	}

	var warnings []string

	for i := 0; i < len(chain)-1; i++ {
		cert, parent := chain[i], chain[i+1]

		if !parent.IsCA {
			err = fmt.Errorf("%s is not a CA", parent.Subject.CommonName)
			return
		}

		if !bytes.Equal(cert.AuthorityKeyId, parent.SubjectKeyId) {
			err = fmt.Errorf("%s AuthorityKeyId differs from %s SubjectKeyId", cert.Subject.CommonName, parent.Subject.CommonName)
			return
		}

		if err = cert.CheckSignatureFrom(parent); err != nil {
			return
		}

		// TODO: Check CRL and OCSP revocation.

		switch cert.SignatureAlgorithm {
		case x509.ECDSAWithSHA1:
			warnings = append(warnings, fmt.Sprintf("%s is signed by ECDSAWithSHA1", cert.Subject.CommonName))
		case x509.SHA1WithRSA:
			warnings = append(warnings, fmt.Sprintf("%s is signed by RSAWithSHA1", cert.Subject.CommonName))
		}
	}

	if len(warnings) == 0 {
		grade = Good
	} else {
		grade = Warning
		output = warnings
	}
	return
}

func multipleCerts(host string) (grade Grade, output Output, err error) {
	config := defaultTLSConfig(host)

	firstChain, err := getChain(host, config)
	if err != nil {
		return
	}

	grade, _, err = multiscan(host, func(addrport string) (g Grade, o Output, e error) {
		g = Good
		chain, e1 := getChain(addrport, config)
		if e1 != nil {
			return
		}

		if !chain[0].Equal(firstChain[0]) {
			e = fmt.Errorf("%s not equal to %s", chain[0].Subject.CommonName, firstChain[0].Subject.CommonName)
			g = Bad
			return
		}
		return
	})
	return
}
<|endoftext|>