<commit_before>package workers

import (
	"fmt"
	"github.com/APTrust/exchange/constants"
	"github.com/APTrust/exchange/context"
	"github.com/APTrust/exchange/models"
	"github.com/APTrust/exchange/network"
	"github.com/nsqio/go-nsq"
	"strings"
	"time"
)

// APTFixityChecker performs ongoing fixity checks on files stored in S3.
type APTFixityChecker struct {
	Context            *context.Context
	FixityChannel      chan *models.FixityResult
	RecordChannel      chan *models.FixityResult
	PostProcessChannel chan *models.FixityResult
	ItemsInProcess     *models.SynchronizedMap
}

func NewAPTFixityChecker(_context *context.Context) *APTFixityChecker {
	checker := &APTFixityChecker{
		Context:        _context,
		ItemsInProcess: models.NewSynchronizedMap(),
	}
	workerBufferSize := _context.Config.FixityWorker.Workers * 10
	checker.FixityChannel = make(chan *models.FixityResult, workerBufferSize)
	checker.RecordChannel = make(chan *models.FixityResult, workerBufferSize)
	checker.PostProcessChannel = make(chan *models.FixityResult, workerBufferSize)
	for i := 0; i < _context.Config.StoreWorker.Workers; i++ {
		go checker.checkFixity()
		go checker.record()
		go checker.postProcess()
	}
	return checker
}

func (checker *APTFixityChecker) HandleMessage(message *nsq.Message) error {
	fixityResult := checker.buildFixityResult(message)
	if fixityResult.Error != nil {
		checker.Context.MessageLog.Error("Cannot process %s: %v",
			string(message.Body), fixityResult.Error.Error())
		message.Finish()
		return nil // Should we return an error to NSQ?
	}
	// Check syncmap to see if this item is already in process.
	startedAt := checker.ItemsInProcess.Get(fixityResult.GenericFile.Identifier)
	if startedAt != "" {
		checker.Context.MessageLog.Info("Skipping %s: already in process as of %s.",
			fixityResult.GenericFile.Identifier, startedAt)
		message.Finish()
		return nil
	}

	// Note that we're working on this.
	checker.ItemsInProcess.Add(fixityResult.GenericFile.Identifier, time.Now().UTC().Format(time.RFC3339))
	checker.Context.MessageLog.Info("Added %s to items in process", fixityResult.GenericFile.Identifier)

	// We'll ping NSQ manually when we need to.
	message.DisableAutoResponse()
	checker.Context.MessageLog.Info("Putting %s into fixity channel",
		fixityResult.GenericFile.Identifier)

	checker.FixityChannel <- fixityResult
	return nil
}

func (checker *APTFixityChecker) checkFixity() {
	for fixityResult := range checker.FixityChannel {
		// Here's where we do the actual digest calculation.
		checker.getFixityValueOfS3File(fixityResult)
		if fixityResult.Error != nil {
			checker.PostProcessChannel <- fixityResult
		} else {
			checker.RecordChannel <- fixityResult
		}
	}
}

func (checker *APTFixityChecker) record() {
	for fixityResult := range checker.RecordChannel {
		// Create PREMIS event saying whether fixity event
		// succeeded or failed.
		event, err := models.NewEventGenericFileFixityCheck(
			time.Now().UTC(),
			constants.AlgSha256,
			fixityResult.Sha256,
			fixityResult.Sha256 == fixityResult.PharosSha256())
		if err != nil {
			fixityResult.Error = fmt.Errorf("Could not create Premis Event for %s: %v",
				fixityResult.GenericFile.Identifier, err)
		} else {
			resp := checker.Context.PharosClient.PremisEventSave(event)
			if resp.Error != nil {
				fixityResult.Error = fmt.Errorf("After completing fixity check for %s, "+
					"could not save PremisEvent to Pharos: %v. Event data: %v",
					fixityResult.GenericFile.Identifier, resp.Error, event)
			} else {
				checker.Context.MessageLog.Info("Completing fixity check for %s, "+
					"and saved PremisEvent %s to Pharos",
					fixityResult.GenericFile.Identifier, event.Identifier)
			}
		}
		checker.PostProcessChannel <- fixityResult
	}
}

func (checker *APTFixityChecker) postProcess() {
	for fixityResult := range checker.PostProcessChannel {
		// Finish or requeue NSQ
		if fixityResult.Error != nil {
			if fixityResult.ErrorIsFatal {
				checker.Context.MessageLog.Error("%s (FATAL)", fixityResult.Error.Error())
				fixityResult.NSQMessage.Finish()
			} else {
				checker.Context.MessageLog.Error("%s (transient)", fixityResult.Error.Error())
				fixityResult.NSQMessage.Requeue(1 * time.Minute)
			}
		} else {
			if fixityResult.PharosSha256() == fixityResult.Sha256 {
				checker.Context.MessageLog.Info("Fixity check complete for %s. Fixity %s matches.",
					fixityResult.GenericFile.Identifier, fixityResult.Sha256)
			} else {
				checker.Context.MessageLog.Warning("Fixity check complete for %s. S3 fixity %s "+
					"DOES NOT MATCH PHAROS FIXITY %s",
					fixityResult.GenericFile.Identifier, fixityResult.Sha256,
					fixityResult.PharosSha256())
			}
			fixityResult.NSQMessage.Finish()
		}
		checker.ItemsInProcess.Delete(fixityResult.GenericFile.Identifier)
		checker.Context.MessageLog.Info("Removed %s from items in process", fixityResult.GenericFile.Identifier)
	}
}

// getFixityValueOfS3File calculates the sha256 digest of an S3 file.
// The downloader streams the file from S3 to /dev/null, because
// we don't need to have the file on disk. We can calculate the
// digest from the stream. We get the file from S3/Virginia, not
// Glacier/Oregon! When this is done, the fixity value will be in
// fixityResult.Sha256.
func (checker *APTFixityChecker) getFixityValueOfS3File(fixityResult *models.FixityResult) {
	bucket, key, err := fixityResult.BucketAndKey()
	if err != nil {
		fixityResult.Error = fmt.Errorf("Can't get S3 bucket and key names for %s: %v",
			fixityResult.GenericFile.Identifier, err)
		fixityResult.ErrorIsFatal = true
		return
	}
	downloader := network.NewS3Download(
		constants.AWSVirginia,
		bucket,      // should be S3 preservation bucket
		key,         // s3 key to fetch
		"/dev/null", // local path at which to save the s3 file
		false,       // don't calculate md5 digest
		true)        // do calculate sha256 digest
	downloader.Fetch()
	if downloader.ErrorMessage != "" {
		fixityResult.Error = fmt.Errorf("Error fetching file %s (%s/%s) from S3: %s",
			fixityResult.GenericFile.Identifier, bucket, key,
			downloader.ErrorMessage)
		if strings.Contains(downloader.ErrorMessage, "NoSuchKey") {
			fixityResult.ErrorIsFatal = true
		}
		return
	}
	fixityResult.S3FileExists = true
	fixityResult.Sha256 = downloader.Sha256Digest
	return
}

func (checker *APTFixityChecker) buildFixityResult(message *nsq.Message) *models.FixityResult {
	fixityResult := models.NewFixityResult(message)
	gfIdentifier := strings.TrimSpace(string(message.Body))
	// Get GenericFile with checksums (param includeRelations = true)
	resp := checker.Context.PharosClient.GenericFileGet(gfIdentifier, true)
	if resp.Error != nil {
		fixityResult.Error = fmt.Errorf("Can't get generic file %s from Pharos: %v", resp.Error.Error())
		if resp.Response.StatusCode == 404 {
			fixityResult.ErrorIsFatal = true
		}
		return fixityResult
	}
	fixityResult.GenericFile = resp.GenericFile()
	if fixityResult.GenericFile.URI == "" {
		fixityResult.Error = fmt.Errorf("GenericFile %s has no S3 URI.", fixityResult.GenericFile.Identifier)
		fixityResult.ErrorIsFatal = true
	}
	return fixityResult
}
<commit_msg>Check for recent fixity before doing work<commit_after>package workers

import (
	"fmt"
	"github.com/APTrust/exchange/constants"
	"github.com/APTrust/exchange/context"
	"github.com/APTrust/exchange/models"
	"github.com/APTrust/exchange/network"
	"github.com/nsqio/go-nsq"
	"strings"
	"time"
)

// APTFixityChecker performs ongoing fixity checks on files stored in S3.
type APTFixityChecker struct {
	// Context contains contextual info, such as a reference to the logger,
	// a Pharos client, basic configuration data and other things the fixity
	// checker needs to do its work.
	Context *context.Context
	// FixityChannel is where we calculate Sha256 digests of files stored in S3.
	FixityChannel chan *models.FixityResult
	// RecordChannel is where record PremisEvents in Pharos.
	RecordChannel chan *models.FixityResult
	// PostProcessChannel is where we dispose of NSQ messages and log outcomes.
	PostProcessChannel chan *models.FixityResult
	// ItemsInProcess contains a map of items we're currently processing.
	// When there's a backlog, it's common for items to be queued more than
	// once for fixity checking. We don't want to perform the fixity check
	// if it's already underway.
	ItemsInProcess *models.SynchronizedMap
}

func NewAPTFixityChecker(_context *context.Context) *APTFixityChecker {
	checker := &APTFixityChecker{
		Context:        _context,
		ItemsInProcess: models.NewSynchronizedMap(),
	}
	workerBufferSize := _context.Config.FixityWorker.Workers * 10
	checker.FixityChannel = make(chan *models.FixityResult, workerBufferSize)
	checker.RecordChannel = make(chan *models.FixityResult, workerBufferSize)
	checker.PostProcessChannel = make(chan *models.FixityResult, workerBufferSize)
	for i := 0; i < _context.Config.StoreWorker.Workers; i++ {
		go checker.checkFixity()
		go checker.record()
		go checker.postProcess()
	}
	return checker
}

// HandleMessage handles a new message from NSQ. Unlike most other NSQ messages,
// where the message.Body is a WorkItem.Id (int as string), messages in the
// apt_fixity queue contain a GenericFile.Identifier. So the entire message body
// will be something like "georgetown.edu/georgetown.edu.10822_707412".
func (checker *APTFixityChecker) HandleMessage(message *nsq.Message) error {
	fixityResult := checker.buildFixityResult(message)
	if fixityResult.Error != nil {
		checker.Context.MessageLog.Error("Cannot process %s: %v",
			string(message.Body), fixityResult.Error.Error())
		message.Finish()
		return nil // Should we return an error to NSQ?
	}

	// Item may have been queued multiple times and then checked a few hours ago.
	if !checker.stillNeedsFixityCheck(fixityResult.GenericFile) {
		checker.Context.MessageLog.Info("Skipping %s because it had a fixity check at %s.",
			fixityResult.GenericFile.Identifier,
			fixityResult.GenericFile.LastFixityCheck.Format(time.RFC3339))
		message.Finish()
		return nil
	}

	// Check syncmap to see if this item is already in process.
	startedAt := checker.ItemsInProcess.Get(fixityResult.GenericFile.Identifier)
	if startedAt != "" {
		checker.Context.MessageLog.Info("Skipping %s: already in process as of %s.",
			fixityResult.GenericFile.Identifier, startedAt)
		message.Finish()
		return nil
	}

	// Note that we're working on this.
	checker.ItemsInProcess.Add(fixityResult.GenericFile.Identifier, time.Now().UTC().Format(time.RFC3339))
	checker.Context.MessageLog.Info("Added %s to items in process", fixityResult.GenericFile.Identifier)

	checker.Context.MessageLog.Info("Putting %s into fixity channel",
		fixityResult.GenericFile.Identifier)

	checker.FixityChannel <- fixityResult
	return nil
}

// checkFixity calls the downloader to calculate the sha256 digest of the
// file in S3.
func (checker *APTFixityChecker) checkFixity() {
	for fixityResult := range checker.FixityChannel {
		// Here's where we do the actual digest calculation.
		checker.getFixityValueOfS3File(fixityResult)
		if fixityResult.Error != nil {
			checker.PostProcessChannel <- fixityResult
		} else {
			checker.RecordChannel <- fixityResult
		}
	}
}

// record records a PremisEvent in Pharos saying when this fixity check
// was performed and whether it succeeded.
func (checker *APTFixityChecker) record() {
	for fixityResult := range checker.RecordChannel {
		// Create PREMIS event saying whether fixity event
		// succeeded or failed.
		event, err := models.NewEventGenericFileFixityCheck(
			time.Now().UTC(),
			constants.AlgSha256,
			fixityResult.Sha256,
			fixityResult.Sha256 == fixityResult.PharosSha256())
		if err != nil {
			fixityResult.Error = fmt.Errorf("Could not create Premis Event for %s: %v",
				fixityResult.GenericFile.Identifier, err)
		} else {
			resp := checker.Context.PharosClient.PremisEventSave(event)
			if resp.Error != nil {
				fixityResult.Error = fmt.Errorf("After completing fixity check for %s, "+
					"could not save PremisEvent to Pharos: %v. Event data: %v",
					fixityResult.GenericFile.Identifier, resp.Error, event)
			} else {
				checker.Context.MessageLog.Info("Completing fixity check for %s, "+
					"and saved PremisEvent %s to Pharos",
					fixityResult.GenericFile.Identifier, event.Identifier)
			}
		}
		checker.PostProcessChannel <- fixityResult
	}
}

// postProcess does some logging and tells NSQ to either finish
// the message or requeue it.
func (checker *APTFixityChecker) postProcess() {
	for fixityResult := range checker.PostProcessChannel {
		// Finish or requeue NSQ
		if fixityResult.Error != nil {
			if fixityResult.ErrorIsFatal {
				checker.Context.MessageLog.Error("%s (FATAL)", fixityResult.Error.Error())
				fixityResult.NSQMessage.Finish()
			} else {
				checker.Context.MessageLog.Error("%s (transient)", fixityResult.Error.Error())
				fixityResult.NSQMessage.Requeue(1 * time.Minute)
			}
		} else {
			if fixityResult.PharosSha256() == fixityResult.Sha256 {
				checker.Context.MessageLog.Info("Fixity check complete for %s. Fixity %s matches.",
					fixityResult.GenericFile.Identifier, fixityResult.Sha256)
			} else {
				checker.Context.MessageLog.Warning("Fixity check complete for %s. S3 fixity %s "+
					"DOES NOT MATCH PHAROS FIXITY %s",
					fixityResult.GenericFile.Identifier, fixityResult.Sha256,
					fixityResult.PharosSha256())
			}
			fixityResult.NSQMessage.Finish()
		}
		checker.ItemsInProcess.Delete(fixityResult.GenericFile.Identifier)
		checker.Context.MessageLog.Info("Removed %s from items in process", fixityResult.GenericFile.Identifier)
	}
}

// getFixityValueOfS3File calculates the sha256 digest of an S3 file.
// The downloader streams the file from S3 to /dev/null, because
// we don't need to have the file on disk. We can calculate the
// digest from the stream. We get the file from S3/Virginia, not
// Glacier/Oregon! When this is done, the fixity value will be in
// fixityResult.Sha256.
func (checker *APTFixityChecker) getFixityValueOfS3File(fixityResult *models.FixityResult) {
	bucket, key, err := fixityResult.BucketAndKey()
	if err != nil {
		fixityResult.Error = fmt.Errorf("Can't get S3 bucket and key names for %s: %v",
			fixityResult.GenericFile.Identifier, err)
		fixityResult.ErrorIsFatal = true
		return
	}
	downloader := network.NewS3Download(
		constants.AWSVirginia,
		bucket,      // should be S3 preservation bucket
		key,         // s3 key to fetch
		"/dev/null", // local path at which to save the s3 file
		false,       // don't calculate md5 digest
		true)        // do calculate sha256 digest
	downloader.Fetch()
	if downloader.ErrorMessage != "" {
		fixityResult.Error = fmt.Errorf("Error fetching file %s (%s/%s) from S3: %s",
			fixityResult.GenericFile.Identifier, bucket, key,
			downloader.ErrorMessage)
		if strings.Contains(downloader.ErrorMessage, "NoSuchKey") {
			fixityResult.ErrorIsFatal = true
		}
		return
	}
	fixityResult.S3FileExists = true
	fixityResult.Sha256 = downloader.Sha256Digest
	return
}

// buildFixityResult builds the manifest that we'll need to record
// the fixity check process and its outcome.
func (checker *APTFixityChecker) buildFixityResult(message *nsq.Message) *models.FixityResult {
	fixityResult := models.NewFixityResult(message)
	gfIdentifier := strings.TrimSpace(string(message.Body))
	// Get GenericFile with checksums (param includeRelations = true)
	resp := checker.Context.PharosClient.GenericFileGet(gfIdentifier, true)
	if resp.Error != nil {
		fixityResult.Error = fmt.Errorf("Can't get generic file %s from Pharos: %v", resp.Error.Error())
		if resp.Response.StatusCode == 404 {
			fixityResult.ErrorIsFatal = true
		}
		return fixityResult
	}
	fixityResult.GenericFile = resp.GenericFile()
	if fixityResult.GenericFile.URI == "" {
		fixityResult.Error = fmt.Errorf("GenericFile %s has no S3 URI.", fixityResult.GenericFile.Identifier)
		fixityResult.ErrorIsFatal = true
	}
	return fixityResult
}

// stillNeedsFixityCheck returns true if the GenericFile still needs a fixity check.
// When we have a backlog of items to check, we may run into the following scenario:
//
// - apt_queue finds a GenericFile that hasn't been checked in 90 days,
//   so it adds it to the NSQ fixity check topic
// - there are already 10,000 items in that topic
// - apt_queue runs again an hour later and adds the same file to the queue
// - the fixity checker finally gets around to checking that file
// - hours later, the fixity checker encounters that file once again in the queue
//
// In this common case, the fixity checker should just mark the item as finished
// and move on if it sees that the item has been checked recently.
func (checker *APTFixityChecker) stillNeedsFixityCheck(gf *models.GenericFile) bool {
	hoursSinceLastFixityCheck := time.Since(gf.LastFixityCheck).Hours()
	maxHoursAllowed := float64(checker.Context.Config.MaxDaysSinceFixityCheck * 24)
	return hoursSinceLastFixityCheck >= maxHoursAllowed
}
<|endoftext|>


<commit_before>package huton

import (
	"fmt"
	"github.com/boltdb/bolt"
	"github.com/hashicorp/raft"
	"github.com/hashicorp/raft-boltdb"
	"github.com/hashicorp/serf/serf"
	"github.com/juju/errors"
	"google.golang.org/grpc"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"path/filepath"
	"sync"
	"time"
)

var (
	ErrNoName error = errors.New("No instance name provided.")
)

// Config provides configuration to a Huton instance. It is composed of Serf and Raft configs, as well as some
// huton-specific configurations. A few of the 3rd party configurations are ignored, such as Serf.EventCh, but most
// all of it is used.
type Config struct {
	BindAddr                string
	BindPort                int
	Peers                   []string
	BaseDir                 string
	CacheDBTimeout          string
	RaftApplicationRetries  int
	RaftApplicationTimeout  string
	RaftRetainSnapshotCount int
	RaftMaxPool             int
	RaftTransportTimeout    string
	LogOutput               io.Writer
	SerfEventChannel        chan serf.Event
}

// DefaultConfig creates a Config instance initialized with default values.
func DefaultConfig() *Config {
	c := &Config{
		BindAddr:                "127.0.0.1",
		BindPort:                8100,
		RaftRetainSnapshotCount: 2,
		RaftMaxPool:             3,
		RaftTransportTimeout:    "10s",
		RaftApplicationTimeout:  "10s",
		Peers:            make([]string, 0),
		CacheDBTimeout:   "10s",
		SerfEventChannel: make(chan serf.Event),
		LogOutput:        os.Stdout,
	}
	return c
}

// Instance is an interface for the Huton instance.
type Instance interface {
	// Bucket returns the bucket in the off-heap database with the given name.
	// If the bucket doesn't exist, it is automatically created.
	Bucket(name string) (Bucket, error)
	// Peers returns the current list of cluster peers. The list includes the local peer.
	Peers() []*Peer
	// Local returns the local peer.
	Local() *Peer
	// Shutdown gracefully shuts down the instance.
	Shutdown() error
}

type instance struct {
	name             string
	serf             *serf.Serf
	raft             *raft.Raft
	raftJSONPeers    *raft.JSONPeers
	raftBoltStore    *raftboltdb.BoltStore
	raftTransport    *raft.NetworkTransport
	dbFilePath       string
	db               *bolt.DB
	rpcListener      net.Listener
	rpc              *grpc.Server
	serfEventChannel chan serf.Event
	shutdownCh       chan struct{}
	peersMu          sync.Mutex
	peers            map[string]*Peer
	dbMu             sync.Mutex
	config           *Config
	buckets          map[string]*bucket
	logger           *log.Logger
}

func (i *instance) Bucket(name string) (Bucket, error) {
	i.dbMu.Lock()
	defer i.dbMu.Unlock()
	if c, ok := i.buckets[name]; ok {
		return c, nil
	}
	return newBucket(i.db, name, i)
}

func (i *instance) Shutdown() error {
	i.logger.Println("Shutting down instance...")
	if i.serf != nil {
		if err := i.serf.Leave(); err != nil {
			return fmt.Errorf("Failed to leave Serf cluster: %s", err)
		}
	}
	if i.raftBoltStore != nil {
		if err := i.raftBoltStore.Close(); err != nil {
			return fmt.Errorf("Failed to close Raft store: %s", err)
		}
	}
	if i.raftTransport != nil {
		if err := i.raftTransport.Close(); err != nil {
			return fmt.Errorf("Failed to close Raft transport: %s", err)
		}
	}
	if i.raft != nil {
		if err := i.raft.Shutdown().Error(); err != nil {
			return fmt.Errorf("Failed to shut down Raft instance: %s", err)
		}
	}
	if i.rpcListener != nil {
		if err := i.rpcListener.Close(); err != nil {
			return fmt.Errorf("Failed to close RPC Listener: %s", err)
		}
	}
	if i.db != nil {
		if err := i.db.Close(); err != nil {
			return fmt.Errorf("Failed to close datastore: %s", err)
		}
	}
	close(i.shutdownCh)
	return nil
}

// NewInstance creates a new Huton instance and initializes it and all of its sub-components, such as Serf, Raft, and
// GRPC server, with the provided configuration.
//
// If this function returns successfully, the instance should be considered started and ready for use.
func NewInstance(name string, config *Config) (Instance, error) {
	if name == "" {
		return nil, ErrNoName
	}
	if config.LogOutput != nil {
		config.LogOutput = ioutil.Discard
	}
	i := &instance{
		name:       name,
		shutdownCh: make(chan struct{}),
		peers:      make(map[string]*Peer),
		config:     config,
		buckets:    make(map[string]*bucket),
		dbFilePath: filepath.Join(config.BaseDir, name, "store.db"),
		logger:     log.New(config.LogOutput, "huton", log.LstdFlags),
	}
	i.logger.Println("Initializing datastore...")
	if err := i.setupDB(); err != nil {
		return i, err
	}
	ip := net.ParseIP(config.BindAddr)
	raftAddr := &net.TCPAddr{
		IP:   ip,
		Port: config.BindPort + 1,
	}
	rpcAddr := &net.TCPAddr{
		IP:   ip,
		Port: config.BindPort + 2,
	}
	i.logger.Println("Initializing Serf cluster...")
	if err := i.setupSerf(raftAddr, rpcAddr); err != nil {
		i.Shutdown()
		return i, err
	}
	i.logger.Println("Configuring peers...")
	i.peersMu.Lock()
	members := i.serf.Members()
	for _, member := range members {
		p, err := newPeer(member)
		if err != nil {
			i.Shutdown()
			return i, err
		}
		i.peers[p.RaftAddr.String()] = p
	}
	i.peersMu.Unlock()
	i.logger.Println("Initializing Raft cluster...")
	if err := i.setupRaft(); err != nil {
		i.Shutdown()
		return i, err
	}
	i.logger.Println("Initializing RPC server...")
	if err := i.setupRPC(); err != nil {
		i.Shutdown()
		return i, err
	}
	// Wait for initial leader state
	i.logger.Println("Waiting for initial leader state...")
	for {
		if i.raft.Leader() != "" {
			break
		}
	}
	go i.handleEvents()
	return i, nil
}

func (i *instance) setupDB() error {
	timeout, err := time.ParseDuration(i.config.CacheDBTimeout)
	if err != nil {
		return fmt.Errorf("Failed to parse duration: %s", err)
	}
	cachesDB, err := bolt.Open(i.dbFilePath, 0644, &bolt.Options{
		Timeout: timeout,
	})
	if err != nil {
		return fmt.Errorf("Failed to open DB file %s: %s", i.dbFilePath, err)
	}
	i.db = cachesDB
	return nil
}

func (i *instance) handleEvents() {
	for {
		select {
		case e := <-i.config.SerfEventChannel:
			i.handleSerfEvent(e)
		case <-i.serf.ShutdownCh():
			i.Shutdown()
		case <-i.shutdownCh:
			return
		}
	}
}

func (i *instance) isLeader() bool {
	if i.raft == nil {
		return false
	}
	return i.raft.State() == raft.Leader
}
<commit_msg>Fixed issue where config.LogOutput was being incorrectly overridden to io.Discard.<commit_after>package huton

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/boltdb/bolt"
	"github.com/hashicorp/raft"
	"github.com/hashicorp/raft-boltdb"
	"github.com/hashicorp/serf/serf"
	"github.com/juju/errors"
	"google.golang.org/grpc"
)

var (
	ErrNoName error = errors.New("No instance name provided.")
)

// Config provides configuration to a Huton instance. It is composed of Serf and Raft configs, as well as some
// huton-specific configurations. A few of the 3rd party configurations are ignored, such as Serf.EventCh, but most
// all of it is used.
type Config struct {
	BindAddr                string
	BindPort                int
	Peers                   []string
	BaseDir                 string
	CacheDBTimeout          string
	RaftApplicationRetries  int
	RaftApplicationTimeout  string
	RaftRetainSnapshotCount int
	RaftMaxPool             int
	RaftTransportTimeout    string
	LogOutput               io.Writer
	SerfEventChannel        chan serf.Event
}

// DefaultConfig creates a Config instance initialized with default values.
func DefaultConfig() *Config {
	c := &Config{
		BindAddr:                "127.0.0.1",
		BindPort:                8100,
		RaftRetainSnapshotCount: 2,
		RaftMaxPool:             3,
		RaftTransportTimeout:    "10s",
		RaftApplicationTimeout:  "10s",
		Peers:            make([]string, 0),
		CacheDBTimeout:   "10s",
		SerfEventChannel: make(chan serf.Event),
		LogOutput:        os.Stdout,
	}
	return c
}

// Instance is an interface for the Huton instance.
type Instance interface {
	// Bucket returns the bucket in the off-heap database with the given name.
	// If the bucket doesn't exist, it is automatically created.
	Bucket(name string) (Bucket, error)
	// Peers returns the current list of cluster peers. The list includes the local peer.
	Peers() []*Peer
	// Local returns the local peer.
	Local() *Peer
	// Shutdown gracefully shuts down the instance.
	Shutdown() error
}

type instance struct {
	name             string
	serf             *serf.Serf
	raft             *raft.Raft
	raftJSONPeers    *raft.JSONPeers
	raftBoltStore    *raftboltdb.BoltStore
	raftTransport    *raft.NetworkTransport
	dbFilePath       string
	db               *bolt.DB
	rpcListener      net.Listener
	rpc              *grpc.Server
	serfEventChannel chan serf.Event
	shutdownCh       chan struct{}
	peersMu          sync.Mutex
	peers            map[string]*Peer
	dbMu             sync.Mutex
	config           *Config
	buckets          map[string]*bucket
	logger           *log.Logger
}

func (i *instance) Bucket(name string) (Bucket, error) {
	i.dbMu.Lock()
	defer i.dbMu.Unlock()
	if c, ok := i.buckets[name]; ok {
		return c, nil
	}
	return newBucket(i.db, name, i)
}

func (i *instance) Shutdown() error {
	i.logger.Println("Shutting down instance...")
	if i.serf != nil {
		if err := i.serf.Leave(); err != nil {
			return fmt.Errorf("Failed to leave Serf cluster: %s", err)
		}
	}
	if i.raftBoltStore != nil {
		if err := i.raftBoltStore.Close(); err != nil {
			return fmt.Errorf("Failed to close Raft store: %s", err)
		}
	}
	if i.raftTransport != nil {
		if err := i.raftTransport.Close(); err != nil {
			return fmt.Errorf("Failed to close Raft transport: %s", err)
		}
	}
	if i.raft != nil {
		if err := i.raft.Shutdown().Error(); err != nil {
			return fmt.Errorf("Failed to shut down Raft instance: %s", err)
		}
	}
	if i.rpcListener != nil {
		if err := i.rpcListener.Close(); err != nil {
			return fmt.Errorf("Failed to close RPC Listener: %s", err)
		}
	}
	if i.db != nil {
		if err := i.db.Close(); err != nil {
			return fmt.Errorf("Failed to close datastore: %s", err)
		}
	}
	close(i.shutdownCh)
	return nil
}

// NewInstance creates a new Huton instance and initializes it and all of its sub-components, such as Serf, Raft, and
// GRPC server, with the provided configuration.
//
// If this function returns successfully, the instance should be considered started and ready for use.
func NewInstance(name string, config *Config) (Instance, error) {
	if name == "" {
		return nil, ErrNoName
	}
	if config.LogOutput == nil {
		config.LogOutput = ioutil.Discard
	}
	i := &instance{
		name:       name,
		shutdownCh: make(chan struct{}),
		peers:      make(map[string]*Peer),
		config:     config,
		buckets:    make(map[string]*bucket),
		dbFilePath: filepath.Join(config.BaseDir, name, "store.db"),
		logger:     log.New(config.LogOutput, "huton", log.LstdFlags),
	}
	i.logger.Println("Initializing datastore...")
	if err := i.setupDB(); err != nil {
		return i, err
	}
	ip := net.ParseIP(config.BindAddr)
	raftAddr := &net.TCPAddr{
		IP:   ip,
		Port: config.BindPort + 1,
	}
	rpcAddr := &net.TCPAddr{
		IP:   ip,
		Port: config.BindPort + 2,
	}
	i.logger.Println("Initializing Serf cluster...")
	if err := i.setupSerf(raftAddr, rpcAddr); err != nil {
		i.Shutdown()
		return i, err
	}
	i.logger.Println("Configuring peers...")
	i.peersMu.Lock()
	members := i.serf.Members()
	for _, member := range members {
		p, err := newPeer(member)
		if err != nil {
			i.Shutdown()
			return i, err
		}
		i.peers[p.RaftAddr.String()] = p
	}
	i.peersMu.Unlock()
	i.logger.Println("Initializing Raft cluster...")
	if err := i.setupRaft(); err != nil {
		i.Shutdown()
		return i, err
	}
	i.logger.Println("Initializing RPC server...")
	if err := i.setupRPC(); err != nil {
		i.Shutdown()
		return i, err
	}
	// Wait for initial leader state
	i.logger.Println("Waiting for initial leader state...")
	for {
		if i.raft.Leader() != "" {
			break
		}
	}
	go i.handleEvents()
	return i, nil
}

func (i *instance) setupDB() error {
	timeout, err := time.ParseDuration(i.config.CacheDBTimeout)
	if err != nil {
		return fmt.Errorf("Failed to parse duration: %s", err)
	}
	cachesDB, err := bolt.Open(i.dbFilePath, 0644, &bolt.Options{
		Timeout: timeout,
	})
	if err != nil {
		return fmt.Errorf("Failed to open DB file %s: %s", i.dbFilePath, err)
	}
	i.db = cachesDB
	return nil
}

func (i *instance) handleEvents() {
	for {
		select {
		case e := <-i.config.SerfEventChannel:
			i.handleSerfEvent(e)
		case <-i.serf.ShutdownCh():
			i.Shutdown()
		case <-i.shutdownCh:
			return
		}
	}
}

func (i *instance) isLeader() bool {
	if i.raft == nil {
		return false
	}
	return i.raft.State() == raft.Leader
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"

	"github.com/lxc/lxd/shared"
)

func help(me string, status int) {
	fmt.Printf("Usage: %s directory [-t] <range1> [<range2> ...]\n", me)
	fmt.Printf("  -t implies test mode.  No file ownerships will be changed.\n")
	fmt.Printf("  A range is [u|b|g]:<first_container_id:first_host_id:range>.\n")
	fmt.Printf("  where u means shift uids, g means shift gids, b means shift both.\n")
	fmt.Printf("  For example: %s directory b:0:100000:65536 u:10000:1000:1\n", me)
	os.Exit(status)
}

func main() {
	if err := run(); err != nil {
		fmt.Printf("Error: %q\n", err)
		help(os.Args[0], 1)
	}
}

func run() error {
	if len(os.Args) < 4 {
		if len(os.Args) > 1 && (os.Args[1] == "-h" || os.Args[1] == "--help" || os.Args[1] == "help") {
			help(os.Args[0], 0)
		} else {
			help(os.Args[0], 1)
		}
	}

	directory := os.Args[1]
	idmap := shared.IdmapSet{}
	testmode := false

	for pos := 2; pos < len(os.Args); pos++ {

		switch os.Args[pos] {
		case "t", "-t", "--test", "test":
			testmode = true
		default:
			var err error
			idmap, err = idmap.Append(os.Args[pos])
			if err != nil {
				return err
			}
		}
	}

	if idmap.Len() == 0 {
		fmt.Printf("No idmaps given\n")
		help(os.Args[0], 1)
	}

	if !testmode && os.Geteuid() != 0 {
		fmt.Printf("This must be run as root\n")
		os.Exit(1)
	}

	return shared.Uidshift(directory, idmap, testmode)
}
<commit_msg>fuidshift: fix args len checking<commit_after>package main

import (
	"fmt"
	"os"

	"github.com/lxc/lxd/shared"
)

func help(me string, status int) {
	fmt.Printf("Usage: %s directory [-t] <range1> [<range2> ...]\n", me)
	fmt.Printf("  -t implies test mode.  No file ownerships will be changed.\n")
	fmt.Printf("  A range is [u|b|g]:<first_container_id:first_host_id:range>.\n")
	fmt.Printf("  where u means shift uids, g means shift gids, b means shift both.\n")
	fmt.Printf("  For example: %s directory b:0:100000:65536 u:10000:1000:1\n", me)
	os.Exit(status)
}

func main() {
	if err := run(); err != nil {
		fmt.Printf("Error: %q\n", err)
		help(os.Args[0], 1)
	}
}

func run() error {
	if len(os.Args) < 3 {
		if len(os.Args) > 1 && (os.Args[1] == "-h" || os.Args[1] == "--help" || os.Args[1] == "help") {
			help(os.Args[0], 0)
		} else {
			help(os.Args[0], 1)
		}
	}

	directory := os.Args[1]
	idmap := shared.IdmapSet{}
	testmode := false

	for pos := 2; pos < len(os.Args); pos++ {

		switch os.Args[pos] {
		case "t", "-t", "--test", "test":
			testmode = true
		default:
			var err error
			idmap, err = idmap.Append(os.Args[pos])
			if err != nil {
				return err
			}
		}
	}

	if idmap.Len() == 0 {
		fmt.Printf("No idmaps given\n")
		help(os.Args[0], 1)
	}

	if !testmode && os.Geteuid() != 0 {
		fmt.Printf("This must be run as root\n")
		os.Exit(1)
	}

	return shared.Uidshift(directory, idmap, testmode)
}
<|endoftext|>


<commit_before>package fuse

import (
	"fmt"
	"os"
	//	"sort"
	"strings"
	"syscall"
)

var openFlagNames map[int]string
var initFlagNames map[int]string
var fuseOpenFlagNames map[int]string
var writeFlagNames map[int]string
var readFlagNames map[int]string
var releaseFlagNames map[int]string
var accessFlagName map[int]string

func init() {
	releaseFlagNames = map[int]string{
		RELEASE_FLUSH: "FLUSH",
	}
	openFlagNames = map[int]string{
		os.O_WRONLY:        "WRONLY",
		os.O_RDWR:          "RDWR",
		os.O_APPEND:        "APPEND",
		syscall.O_ASYNC:    "ASYNC",
		os.O_CREATE:        "CREAT",
		os.O_EXCL:          "EXCL",
		syscall.O_NOCTTY:   "NOCTTY",
		syscall.O_NONBLOCK: "NONBLOCK",
		os.O_SYNC:          "SYNC",
		os.O_TRUNC:         "TRUNC",

		syscall.O_CLOEXEC:   "CLOEXEC",
		syscall.O_DIRECT:    "DIRECT",
		syscall.O_DIRECTORY: "DIRECTORY",
		syscall.O_LARGEFILE: "LARGEFILE",
		syscall.O_NOATIME:   "NOATIME",
	}
	initFlagNames = map[int]string{
		CAP_ASYNC_READ:     "ASYNC_READ",
		CAP_POSIX_LOCKS:    "POSIX_LOCKS",
		CAP_FILE_OPS:       "FILE_OPS",
		CAP_ATOMIC_O_TRUNC: "ATOMIC_O_TRUNC",
		CAP_EXPORT_SUPPORT: "EXPORT_SUPPORT",
		CAP_BIG_WRITES:     "BIG_WRITES",
		CAP_DONT_MASK:      "DONT_MASK",
		CAP_SPLICE_WRITE:   "SPLICE_WRITE",
		CAP_SPLICE_MOVE:    "SPLICE_MOVE",
		CAP_SPLICE_READ:    "SPLICE_READ",
	}
	fuseOpenFlagNames = map[int]string{
		FOPEN_DIRECT_IO:   "DIRECT",
		FOPEN_KEEP_CACHE:  "CACHE",
		FOPEN_NONSEEKABLE: "NONSEEK",
	}
	writeFlagNames = map[int]string{
		WRITE_CACHE:     "CACHE",
		WRITE_LOCKOWNER: "LOCKOWNER",
	}
	readFlagNames = map[int]string{
		READ_LOCKOWNER: "LOCKOWNER",
	}
	accessFlagName = map[int]string{
		X_OK: "x",
		W_OK: "w",
		R_OK: "r",
	}
}

func flagString(names map[int]string, fl int, def string) string {
	s := []string{}
	for k, v := range names {
		if fl&k != 0 {
			s = append(s, v)
			fl ^= k
		}
	}
	if len(s) == 0 && def != "" {
		s = []string{def}
	}
	if fl != 0 {
		s = append(s, fmt.Sprintf("0x%x", fl))
	}

	return strings.Join(s, ",")
}

func (me *OpenIn) String() string {
	return fmt.Sprintf("{%s}", flagString(openFlagNames, int(me.Flags), "O_RDONLY"))
}

func (me *SetAttrIn) String() string {
	s := []string{}
	if me.Valid&FATTR_MODE != 0 {
		s = append(s, fmt.Sprintf("mode 0%o", me.Mode))
	}
	if me.Valid&FATTR_UID != 0 {
		s = append(s, fmt.Sprintf("uid %d", me.Uid))
	}
	if me.Valid&FATTR_GID != 0 {
		s = append(s, fmt.Sprintf("uid %d", me.Gid))
	}
	if me.Valid&FATTR_SIZE != 0 {
		s = append(s, fmt.Sprintf("size %d", me.Size))
	}
	if me.Valid&FATTR_ATIME != 0 {
		s = append(s, fmt.Sprintf("atime %d.%09d", me.Atime, me.Atimensec))
	}
	if me.Valid&FATTR_MTIME != 0 {
		s = append(s, fmt.Sprintf("mtime %d.%09d", me.Mtime, me.Mtimensec))
	}
	if me.Valid&FATTR_MTIME != 0 {
		s = append(s, fmt.Sprintf("fh %d", me.Fh))
	}
	// TODO - FATTR_ATIME_NOW = (1 << 7), FATTR_MTIME_NOW = (1 << 8), FATTR_LOCKOWNER = (1 << 9)
	return fmt.Sprintf("{%s}", strings.Join(s, ", "))
}

func (me *Attr) String() string {
	return fmt.Sprintf(
		"{M0%o S=%d L=%d "+
			"%d:%d "+
			"%d*%d %d:%d "+
			"A %d.%09d "+
			"M %d.%09d "+
			"C %d.%09d}",
		me.Mode, me.Size, me.Nlink,
		me.Uid, me.Gid,
		me.Blocks, me.Blksize,
		me.Rdev, me.Ino, me.Atime, me.Atimensec, me.Mtime, me.Mtimensec,
		me.Ctime, me.Ctimensec)
}

func (me *AttrOut) String() string {
	return fmt.Sprintf(
		"{A%d.%09d %v}",
		me.AttrValid, me.AttrValidNsec, &me.Attr)
}

func (me *CreateIn) String() string {
	return fmt.Sprintf(
		"{0%o [%s] (0%o)}", me.Mode,
		flagString(openFlagNames, int(me.Flags), "O_RDONLY"), me.Umask)
}

func (me *EntryOut) String() string {
	return fmt.Sprintf("{%d E%d.%09d A%d.%09d %v}",
		me.NodeId, me.EntryValid, me.EntryValidNsec,
		me.AttrValid, me.AttrValidNsec, &me.Attr)
}

func (me *CreateOut) String() string {
	return fmt.Sprintf("{%v %v}", &me.EntryOut, &me.OpenOut)
}

func (me *OpenOut) String() string {
	return fmt.Sprintf("{Fh %d %s}", me.Fh,
		flagString(fuseOpenFlagNames, int(me.OpenFlags), ""))
}

func (me *GetAttrIn) String() string {
	return fmt.Sprintf("{Fh %d}", me.Fh)
}

func (me *InitIn) String() string {
	return fmt.Sprintf("{%d.%d Ra 0x%x %s}",
		me.Major, me.Minor, me.MaxReadAhead,
		flagString(initFlagNames, int(me.Flags), ""))
}
func (me *InitOut) String() string {
	return fmt.Sprintf("{%d.%d Ra 0x%x %s %d/%d Wr 0x%x}",
		me.Major, me.Minor, me.MaxReadAhead,
		flagString(initFlagNames, int(me.Flags), ""),
		me.CongestionThreshold, me.MaxBackground, me.MaxWrite)
}

func (me *ReadIn) String() string {
	return fmt.Sprintf("{Fh %d off %d sz %d %s L %d %s}",
		me.Fh, me.Offset, me.Size,
		flagString(readFlagNames, int(me.ReadFlags), ""),
		me.LockOwner,
		flagString(openFlagNames, int(me.Flags), "RDONLY"))
}

func (me *MkdirIn) String() string {
	return fmt.Sprintf("{0%o (0%o)}", me.Mode, me.Umask)
}

func (me *MknodIn) String() string {
	return fmt.Sprintf("{0%o (0%o), %d}", me.Mode, me.Umask, me.Rdev)
}

func (me *ReleaseIn) String() string {
	return fmt.Sprintf("{Fh %d %s %s L%d}",
		me.Fh, flagString(openFlagNames, int(me.Flags), ""),
		flagString(releaseFlagNames, int(me.ReleaseFlags), ""),
		me.LockOwner)
}

func (me *FlushIn) String() string {
	return fmt.Sprintf("{Fh %d}", me.Fh)
}

func (me *AccessIn) String() string {
	return fmt.Sprintf("{%s}", flagString(accessFlagName, int(me.Mask), ""))
}

func (me *Kstatfs) String() string {
	return fmt.Sprintf(
		"{b%d f%d fs%d ff%d bs%d nl%d frs%d}",
		me.Blocks, me.Bfree, me.Bavail, me.Files, me.Ffree,
		me.Bsize, me.NameLen, me.Frsize)
}

func (me *WithFlags) String() string {
	return fmt.Sprintf("File %s (%s) %s %s",
		me.File, me.Description, flagString(openFlagNames, int(me.OpenFlags), "O_RDONLY"),
		flagString(fuseOpenFlagNames, int(me.FuseFlags), ""))
}
<commit_msg>Print methods for XAttr types.<commit_after>package fuse

import (
	"fmt"
	"os"
	//	"sort"
	"strings"
	"syscall"
)

var openFlagNames map[int]string
var initFlagNames map[int]string
var fuseOpenFlagNames map[int]string
var writeFlagNames map[int]string
var readFlagNames map[int]string
var releaseFlagNames map[int]string
var accessFlagName map[int]string

func init() {
	releaseFlagNames = map[int]string{
		RELEASE_FLUSH: "FLUSH",
	}
	openFlagNames = map[int]string{
		os.O_WRONLY:        "WRONLY",
		os.O_RDWR:          "RDWR",
		os.O_APPEND:        "APPEND",
		syscall.O_ASYNC:    "ASYNC",
		os.O_CREATE:        "CREAT",
		os.O_EXCL:          "EXCL",
		syscall.O_NOCTTY:   "NOCTTY",
		syscall.O_NONBLOCK: "NONBLOCK",
		os.O_SYNC:          "SYNC",
		os.O_TRUNC:         "TRUNC",

		syscall.O_CLOEXEC:   "CLOEXEC",
		syscall.O_DIRECT:    "DIRECT",
		syscall.O_DIRECTORY: "DIRECTORY",
		syscall.O_LARGEFILE: "LARGEFILE",
		syscall.O_NOATIME:   "NOATIME",
	}
	initFlagNames = map[int]string{
		CAP_ASYNC_READ:     "ASYNC_READ",
		CAP_POSIX_LOCKS:    "POSIX_LOCKS",
		CAP_FILE_OPS:       "FILE_OPS",
		CAP_ATOMIC_O_TRUNC: "ATOMIC_O_TRUNC",
		CAP_EXPORT_SUPPORT: "EXPORT_SUPPORT",
		CAP_BIG_WRITES:     "BIG_WRITES",
		CAP_DONT_MASK:      "DONT_MASK",
		CAP_SPLICE_WRITE:   "SPLICE_WRITE",
		CAP_SPLICE_MOVE:    "SPLICE_MOVE",
		CAP_SPLICE_READ:    "SPLICE_READ",
	}
	fuseOpenFlagNames = map[int]string{
		FOPEN_DIRECT_IO:   "DIRECT",
		FOPEN_KEEP_CACHE:  "CACHE",
		FOPEN_NONSEEKABLE: "NONSEEK",
	}
	writeFlagNames = map[int]string{
		WRITE_CACHE:     "CACHE",
		WRITE_LOCKOWNER: "LOCKOWNER",
	}
	readFlagNames = map[int]string{
		READ_LOCKOWNER: "LOCKOWNER",
	}
	accessFlagName = map[int]string{
		X_OK: "x",
		W_OK: "w",
		R_OK: "r",
	}
}

func flagString(names map[int]string, fl int, def string) string {
	s := []string{}
	for k, v := range names {
		if fl&k != 0 {
			s = append(s, v)
			fl ^= k
		}
	}
	if len(s) == 0 && def != "" {
		s = []string{def}
	}
	if fl != 0 {
		s = append(s, fmt.Sprintf("0x%x", fl))
	}

	return strings.Join(s, ",")
}

func (me *OpenIn) String() string {
	return fmt.Sprintf("{%s}", flagString(openFlagNames, int(me.Flags), "O_RDONLY"))
}

func (me *SetAttrIn) String() string {
	s := []string{}
	if me.Valid&FATTR_MODE != 0 {
		s = append(s, fmt.Sprintf("mode 0%o", me.Mode))
	}
	if me.Valid&FATTR_UID != 0 {
		s = append(s, fmt.Sprintf("uid %d", me.Uid))
	}
	if me.Valid&FATTR_GID != 0 {
		s = append(s, fmt.Sprintf("uid %d", me.Gid))
	}
	if me.Valid&FATTR_SIZE != 0 {
		s = append(s, fmt.Sprintf("size %d", me.Size))
	}
	if me.Valid&FATTR_ATIME != 0 {
		s = append(s, fmt.Sprintf("atime %d.%09d", me.Atime, me.Atimensec))
	}
	if me.Valid&FATTR_MTIME != 0 {
		s = append(s, fmt.Sprintf("mtime %d.%09d", me.Mtime, me.Mtimensec))
	}
	if me.Valid&FATTR_MTIME != 0 {
		s = append(s, fmt.Sprintf("fh %d", me.Fh))
	}
	// TODO - FATTR_ATIME_NOW = (1 << 7), FATTR_MTIME_NOW = (1 << 8), FATTR_LOCKOWNER = (1 << 9)
	return fmt.Sprintf("{%s}", strings.Join(s, ", "))
}

func (me *Attr) String() string {
	return fmt.Sprintf(
		"{M0%o S=%d L=%d "+
			"%d:%d "+
			"%d*%d %d:%d "+
			"A %d.%09d "+
			"M %d.%09d "+
			"C %d.%09d}",
		me.Mode, me.Size, me.Nlink,
		me.Uid, me.Gid,
		me.Blocks, me.Blksize,
		me.Rdev, me.Ino, me.Atime, me.Atimensec, me.Mtime, me.Mtimensec,
		me.Ctime, me.Ctimensec)
}

func (me *AttrOut) String() string {
	return fmt.Sprintf(
		"{A%d.%09d %v}",
		me.AttrValid, me.AttrValidNsec, &me.Attr)
}

func (me *CreateIn) String() string {
	return fmt.Sprintf(
		"{0%o [%s] (0%o)}", me.Mode,
		flagString(openFlagNames, int(me.Flags), "O_RDONLY"), me.Umask)
}

func (me *EntryOut) String() string {
	return fmt.Sprintf("{%d E%d.%09d A%d.%09d %v}",
		me.NodeId, me.EntryValid, me.EntryValidNsec,
		me.AttrValid, me.AttrValidNsec, &me.Attr)
}

func (me *CreateOut) String() string {
	return fmt.Sprintf("{%v %v}", &me.EntryOut, &me.OpenOut)
}

func (me *OpenOut) String() string {
	return fmt.Sprintf("{Fh %d %s}", me.Fh,
		flagString(fuseOpenFlagNames, int(me.OpenFlags), ""))
}

func (me *GetAttrIn) String() string {
	return fmt.Sprintf("{Fh %d}", me.Fh)
}

func (me *InitIn) String() string {
	return fmt.Sprintf("{%d.%d Ra 0x%x %s}",
		me.Major, me.Minor, me.MaxReadAhead,
		flagString(initFlagNames, int(me.Flags), ""))
}
func (me *InitOut) String() string {
	return fmt.Sprintf("{%d.%d Ra 0x%x %s %d/%d Wr 0x%x}",
		me.Major, me.Minor, me.MaxReadAhead,
		flagString(initFlagNames, int(me.Flags), ""),
		me.CongestionThreshold, me.MaxBackground, me.MaxWrite)
}

func (me *ReadIn) String() string {
	return fmt.Sprintf("{Fh %d off %d sz %d %s L %d %s}",
		me.Fh, me.Offset, me.Size,
		flagString(readFlagNames, int(me.ReadFlags), ""),
		me.LockOwner,
		flagString(openFlagNames, int(me.Flags), "RDONLY"))
}

func (me *MkdirIn) String() string {
	return fmt.Sprintf("{0%o (0%o)}", me.Mode, me.Umask)
}

func (me *MknodIn) String() string {
	return fmt.Sprintf("{0%o (0%o), %d}", me.Mode, me.Umask, me.Rdev)
}

func (me *ReleaseIn) String() string {
	return fmt.Sprintf("{Fh %d %s %s L%d}",
		me.Fh, flagString(openFlagNames, int(me.Flags), ""),
		flagString(releaseFlagNames, int(me.ReleaseFlags), ""),
		me.LockOwner)
}

func (me *FlushIn) String() string {
	return fmt.Sprintf("{Fh %d}", me.Fh)
}

func (me *AccessIn) String() string {
	return fmt.Sprintf("{%s}", flagString(accessFlagName, int(me.Mask), ""))
}

func (me *SetXAttrIn) String() string {
	return fmt.Sprintf("{sz %d f%o}", me.Size, me.Flags)
}

func (me *GetXAttrIn) String() string {
	return fmt.Sprintf("{sz %d}", me.Size)
}

func (me *GetXAttrOut) String() string {
	return fmt.Sprintf("{sz %d}", me.Size)
}

func (me *Kstatfs) String() string {
	return fmt.Sprintf(
		"{b%d f%d fs%d ff%d bs%d nl%d frs%d}",
		me.Blocks, me.Bfree, me.Bavail, me.Files, me.Ffree,
		me.Bsize, me.NameLen, me.Frsize)
}

func (me *WithFlags) String() string {
	return fmt.Sprintf("File %s (%s) %s %s",
		me.File, me.Description, flagString(openFlagNames, int(me.OpenFlags), "O_RDONLY"),
		flagString(fuseOpenFlagNames, int(me.FuseFlags), ""))
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"github.com/RackSec/srslog"
	"github.com/buger/jsonparser"
	"github.com/urfave/cli"
	"log"
	"strconv"
	"strings"
	"time"
)

var papertrailprefix string

type PapertrailLog struct {
	Message       string  `json:"message,omitempty"`
	Component     string  `json:"component,omitempty"`
	Severity      string  `json:"severity,omitempty"`
	Action        string  `json:"action,omitempty"`
	Subject       string  `json:"subject,omitempty"`
	CorrelationId string  `json:"correlation_id,omitempty"`
	MessageId     string  `json:"message_id,omitempty"`
	Name          string  `json:"name,omitempty"`
	Error         string  `json:"error,omitempty"`
	Stack         string  `json:"stack,omitempty"`
	State         string  `json:"state,omitempty"`
	Time          int64   `json:"time,omitempty"`
	DurationS     float64 `json:"duration_s,omitempty"`
}

func init() {
	app.Commands = append(app.Commands,
		cli.Command{
			Name:  "papertrail",
			Usage: "forward logs to papertrail",
			Action: func(c *cli.Context) {

				papertrailprefix = c.String("prefix")
				consumer := NewConsumer(globalFlags.Brokers, globalFlags.Groupid)
				consumer.Init(globalFlags.Topic)

				log.Println(c.String("papertrail"))

				go listenforchecks()
				papertrail := make(chan *data, 20)
				defer close(papertrail)
				go func(messages <-chan *Message, papertrail chan<- *data) {
					for msg := range messages {
						jsondata := &data{start: time.Now(), offset: msg.Offset}
						jsondata.Ts, _ = jsonparser.GetUnsafeString(msg.Data, "time")
						jsondata.Msg, _ = jsonparser.GetUnsafeString(msg.Data, "log")
						jsondata.Level, _ = jsonparser.GetUnsafeString(msg.Data, "level")
						jsondata.Service, _ = jsonparser.GetUnsafeString(msg.Data, "kubernetes", "labels", "app")
						jsondata.Environment, _ = jsonparser.GetUnsafeString(msg.Data, "kubernetes", "labels", "environment")
						jsondata.Host, _ = jsonparser.GetUnsafeString(msg.Data, "kubernetes", "host")
						jsondata.IPAddress, _ = jsonparser.GetUnsafeString(msg.Data, "ip_address")
						jsondata.ServerIP, _ = jsonparser.GetUnsafeString(msg.Data, "server_ip")
						jsondata.DockerImage, _ = jsonparser.GetUnsafeString(msg.Data, "docker_image")
						jsondata.ContainerName, _ = jsonparser.GetUnsafeString(msg.Data, "kubernetes", "pod_name")
						jsondata.ContainerID, _ = jsonparser.GetUnsafeString(msg.Data, "docker", "container_id")

						// Just send the original data as the message if it for some reason doesn't have anything in the log field.
						if jsondata.Msg == "" {
							var papertrailLog PapertrailLog
							papertrailLog.Message, _ = jsonparser.GetUnsafeString(msg.Data, "message")
							papertrailLog.Component, _ = jsonparser.GetUnsafeString(msg.Data, "component")
							papertrailLog.Severity, _ = jsonparser.GetUnsafeString(msg.Data, "severity")
							papertrailLog.Action, _ = jsonparser.GetUnsafeString(msg.Data, "action")
							papertrailLog.Subject, _ = jsonparser.GetUnsafeString(msg.Data, "subject")
							papertrailLog.CorrelationId, _ = jsonparser.GetUnsafeString(msg.Data, "correlation_id")
							papertrailLog.MessageId, _ = jsonparser.GetUnsafeString(msg.Data, "message_id")
							papertrailLog.Name, _ = jsonparser.GetUnsafeString(msg.Data, "name")
							papertrailLog.Error, _ = jsonparser.GetUnsafeString(msg.Data, "error")
							papertrailLog.Stack, _ = jsonparser.GetUnsafeString(msg.Data, "stack")
							papertrailLog.DurationS, _ = jsonparser.GetFloat(msg.Data, "duration_s")
							papertrailLog.Time, _ = jsonparser.GetInt(msg.Data, "time")
							papertraiLogJson, _ := json.Marshal(papertrailLog)
							jsondata.Msg = string(papertraiLogJson)
						}

						papertrail <- jsondata
					}
				}(consumer.Chan, papertrail)
				go Sender(papertrail, c.String("papertrail"), c.String("cert"))

				consumer.StartConsumingTopic()

				consumer.Wait()
			},
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:   "papertrail",
					Usage:  "The papertrail address where the logs should be forwarded",
					EnvVar: "PAPERTRAIL_ADDRESS",
				},
				cli.StringFlag{
					Name:   "cert",
					Usage:  "Papertrail root certificate path",
					Value:  "./papertrail-bundle.pem",
					EnvVar: "PAPERTRAIL_CERT",
				},
				cli.StringFlag{
					Name:   "prefix",
					Usage:  "String that is prepended to the system name",
					EnvVar: "PAPERTRAIL_PREFIX",
				},
			},
		})
}

type data struct {
	Ts            string `json:"ts"`
	Msg           string `json:"msg"`
	Level         string `json:"level"`
	Stream        string `json:"stream"`
	Service       string `json:"service"`
	Environment   string `json:"environment"`
	Host          string `json:"host"`
	IPAddress     string `json:"ip_address"` // legacy
	ServerIP      string `json:"server_ip"`
	DockerImage   string `json:"docker_image"`
	ContainerName string `json:"container_name"`
	ContainerID   string `json:"container_id"`

	start  time.Time
	offset int64
}

// Sender receives valid entries from chan 'c' and uploads them into papertrail over an encrypted TCP connection.
func Sender(c <-chan *data, address, cert string) {
	w, err := srslog.DialWithTLSCertPath("tcp+tls", address, srslog.LOG_INFO, "kafkatopapertrail", cert)
	if err != nil {
		log.Fatal(err)
	}

	var timechan chan time.Duration
	if globalFlags.Verbose {
		timechan = make(chan time.Duration, 10)
		defer close(timechan)
		go func(timechan chan time.Duration) {
			var (
				eventcount int
				counter    time.Duration
				ticker     = time.NewTicker(time.Second * 30)
			)

		loop:
			for {
				select {
				case duration, ok := <-timechan:
					if ok {
						counter += duration
						eventcount++
					} else {
						break loop
					}
				case <-ticker.C:
					if eventcount != 0 {
						counter = counter / time.Duration(eventcount)
					}
					log.Printf("Sent %d messages during the last 30 seconds, averaging %s per message\n", eventcount, counter)
					eventcount = 0
					counter = 0
				}
			}
		}(timechan)
	}

	w.SetFormatter(LogFormatter)
	for msg := range c {
		switch msg.Level {
		case "INFO":
			err = w.Info(msg.String())
		case "ALERT":
			err = w.Alert(msg.String())
		case "CRIT", "CRITICAL":
			err = w.Crit(msg.String())
		case "DEBUG":
			err = w.Debug(msg.String())
		case "EMERG":
			err = w.Emerg(msg.String())
		case "ERR", "ERROR":
			err = w.Err(msg.String())
		case "NOTICE":
			err = w.Notice(msg.String())
		case "WARNING", "WARN":
			err = w.Warning(msg.String())
		default:
			err = w.Info(msg.String())
			//      log.Printf("Unknown log level: %s\n", msg.Level)
		}
		if err != nil {
			log.Println(err)
		}
		if globalFlags.Verbose {
			timechan <- time.Since(msg.start)
		}
	}
}

func (d *data) String() string {
	prefix := papertrailprefix
	ts, _ := strconv.ParseInt(d.Ts, 10, 64)
	timestamp := time.Unix(ts, 0).Format(time.RFC3339)
	if d.Environment == "stg" {
		// A bit conflicting with the whole point of shared prefix but mostly a easy solution until some things are sorted.
		prefix = "staging-"
	}
	return fmt.Sprintf("%s|%s%s|%s|%s %s", timestamp, prefix, d.Service, d.ContainerName, timestamp, d.Msg)
}

func parseImage(image string) (repository, tag string) {
	// Docker image format is:
	// {registry}/(_|/r/{user_or_org})/{repository}:{tag}
	// Here we get the repository and tag from that format
	var (
		parts []string
	)
	parts = strings.Split(image, "/")
	image = parts[len(parts)-1]
	parts = strings.Split(image, ":")
	repository = parts[0]
	if len(parts) > 1 {
		tag = parts[1]
		if len(tag) > 8 {
			tag = tag[:8]
		}
	}
	return
}

// LogFormatter is a custom syslog formatter that uses log data to fill timestamp, hostname and tag instead of using local system information.
func LogFormatter(p srslog.Priority, hostname, tag, content string) (msg string) {
	parts := strings.SplitN(content, "|", 4)
	if len(parts) == 4 {
		msg = fmt.Sprintf("<%d>1 %s %s %s - - - %s", p, parts[0], parts[1], parts[2], parts[3])
	} else {
		msg = srslog.DefaultFormatter(p, hostname, tag, content)
	}
	return
}
<commit_msg>Add errors field<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"github.com/RackSec/srslog"
	"github.com/buger/jsonparser"
	"github.com/urfave/cli"
	"log"
	"strconv"
	"strings"
	"time"
)

var papertrailprefix string

type PapertrailLog struct {
	Message       string  `json:"message,omitempty"`
	Component     string  `json:"component,omitempty"`
	Severity      string  `json:"severity,omitempty"`
	Action        string  `json:"action,omitempty"`
	Subject       string  `json:"subject,omitempty"`
	CorrelationId string  `json:"correlation_id,omitempty"`
	MessageId     string  `json:"message_id,omitempty"`
	Name          string  `json:"name,omitempty"`
	Error         string  `json:"error,omitempty"`
	Errors        string  `json:"errors,omitempty"`
	Stack         string  `json:"stack,omitempty"`
	State         string  `json:"state,omitempty"`
	Time          int64   `json:"time,omitempty"`
	DurationS     float64 `json:"duration_s,omitempty"`
}

func init() {
	app.Commands = append(app.Commands,
		cli.Command{
			Name:  "papertrail",
			Usage: "forward logs to papertrail",
			Action: func(c *cli.Context) {

				papertrailprefix = c.String("prefix")
				consumer := NewConsumer(globalFlags.Brokers, globalFlags.Groupid)
				consumer.Init(globalFlags.Topic)

				log.Println(c.String("papertrail"))

				go listenforchecks()
				papertrail := make(chan *data, 20)
				defer close(papertrail)
				go func(messages <-chan *Message, papertrail chan<- *data) {
					for msg := range messages {
						jsondata := &data{start: time.Now(), offset: msg.Offset}
						jsondata.Ts, _ = jsonparser.GetUnsafeString(msg.Data, "time")
						jsondata.Msg, _ = jsonparser.GetUnsafeString(msg.Data, "log")
						jsondata.Level, _ = jsonparser.GetUnsafeString(msg.Data, "level")
						jsondata.Service, _ = jsonparser.GetUnsafeString(msg.Data, "kubernetes", "labels", "app")
						jsondata.Environment, _ = jsonparser.GetUnsafeString(msg.Data, "kubernetes", "labels", "environment")
						jsondata.Host, _ = jsonparser.GetUnsafeString(msg.Data, "kubernetes", "host")
						jsondata.IPAddress, _ = jsonparser.GetUnsafeString(msg.Data, "ip_address")
						jsondata.ServerIP, _ = jsonparser.GetUnsafeString(msg.Data, "server_ip")
						jsondata.DockerImage, _ = jsonparser.GetUnsafeString(msg.Data, "docker_image")
						jsondata.ContainerName, _ = jsonparser.GetUnsafeString(msg.Data, "kubernetes", "pod_name")
						jsondata.ContainerID, _ = jsonparser.GetUnsafeString(msg.Data, "docker", "container_id")

						// Just send the original data as the message if it for some reason doesn't have anything in the log field.
						if jsondata.Msg == "" {
							var papertrailLog PapertrailLog
							papertrailLog.Message, _ = jsonparser.GetUnsafeString(msg.Data, "message")
							papertrailLog.Component, _ = jsonparser.GetUnsafeString(msg.Data, "component")
							papertrailLog.Severity, _ = jsonparser.GetUnsafeString(msg.Data, "severity")
							papertrailLog.Action, _ = jsonparser.GetUnsafeString(msg.Data, "action")
							papertrailLog.Subject, _ = jsonparser.GetUnsafeString(msg.Data, "subject")
							papertrailLog.CorrelationId, _ = jsonparser.GetUnsafeString(msg.Data, "correlation_id")
							papertrailLog.MessageId, _ = jsonparser.GetUnsafeString(msg.Data, "message_id")
							papertrailLog.Name, _ = jsonparser.GetUnsafeString(msg.Data, "name")
							papertrailLog.Error, _ = jsonparser.GetUnsafeString(msg.Data, "error")
							papertrailLog.Errors, _ = jsonparser.GetUnsafeString(msg.Data, "errors")
							papertrailLog.Stack, _ = jsonparser.GetUnsafeString(msg.Data, "stack")
							papertrailLog.DurationS, _ = jsonparser.GetFloat(msg.Data, "duration_s")
							papertrailLog.Time, _ = jsonparser.GetInt(msg.Data, "time")
							papertraiLogJson, _ := json.Marshal(papertrailLog)
							jsondata.Msg = string(papertraiLogJson)
						}

						papertrail <- jsondata
					}
				}(consumer.Chan, papertrail)
				go Sender(papertrail, c.String("papertrail"), c.String("cert"))

				consumer.StartConsumingTopic()

				consumer.Wait()
			},
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:   "papertrail",
					Usage:  "The papertrail address where the logs should be forwarded",
					EnvVar: "PAPERTRAIL_ADDRESS",
				},
				cli.StringFlag{
					Name:   "cert",
					Usage:  "Papertrail root certificate path",
					Value:  "./papertrail-bundle.pem",
					EnvVar: "PAPERTRAIL_CERT",
				},
				cli.StringFlag{
					Name:   "prefix",
					Usage:  "String that is prepended to the system name",
					EnvVar: "PAPERTRAIL_PREFIX",
				},
			},
		})
}

type data struct {
	Ts            string `json:"ts"`
	Msg           string `json:"msg"`
	Level         string `json:"level"`
	Stream        string `json:"stream"`
	Service       string `json:"service"`
	Environment   string `json:"environment"`
	Host          string `json:"host"`
	IPAddress     string `json:"ip_address"` // legacy
	ServerIP      string `json:"server_ip"`
	DockerImage   string `json:"docker_image"`
	ContainerName string `json:"container_name"`
	ContainerID   string `json:"container_id"`

	start  time.Time
	offset int64
}

// Sender receives valid entries from chan 'c' and uploads them into papertrail over an encrypted TCP connection.
func Sender(c <-chan *data, address, cert string) {
	w, err := srslog.DialWithTLSCertPath("tcp+tls", address, srslog.LOG_INFO, "kafkatopapertrail", cert)
	if err != nil {
		log.Fatal(err)
	}

	var timechan chan time.Duration
	if globalFlags.Verbose {
		timechan = make(chan time.Duration, 10)
		defer close(timechan)
		go func(timechan chan time.Duration) {
			var (
				eventcount int
				counter    time.Duration
				ticker     = time.NewTicker(time.Second * 30)
			)

		loop:
			for {
				select {
				case duration, ok := <-timechan:
					if ok {
						counter += duration
						eventcount++
					} else {
						break loop
					}
				case <-ticker.C:
					if eventcount != 0 {
						counter = counter / time.Duration(eventcount)
					}
					log.Printf("Sent %d messages during the last 30 seconds, averaging %s per message\n", eventcount, counter)
					eventcount = 0
					counter = 0
				}
			}
		}(timechan)
	}

	w.SetFormatter(LogFormatter)
	for msg := range c {
		switch msg.Level {
		case "INFO":
			err = w.Info(msg.String())
		case "ALERT":
			err = w.Alert(msg.String())
		case "CRIT", "CRITICAL":
			err = w.Crit(msg.String())
		case "DEBUG":
			err = w.Debug(msg.String())
		case "EMERG":
			err = w.Emerg(msg.String())
		case "ERR", "ERROR":
			err = w.Err(msg.String())
		case "NOTICE":
			err = w.Notice(msg.String())
		case "WARNING", "WARN":
			err = w.Warning(msg.String())
		default:
			err = w.Info(msg.String())
			//      log.Printf("Unknown log level: %s\n", msg.Level)
		}
		if err != nil {
			log.Println(err)
		}
		if globalFlags.Verbose {
			timechan <- time.Since(msg.start)
		}
	}
}

func (d *data) String() string {
	prefix := papertrailprefix
	ts, _ := strconv.ParseInt(d.Ts, 10, 64)
	timestamp := time.Unix(ts, 0).Format(time.RFC3339)
	if d.Environment == "stg" {
		// A bit conflicting with the whole point of shared prefix but mostly a easy solution until some things are sorted.
		prefix = "staging-"
	}
	return fmt.Sprintf("%s|%s%s|%s|%s %s", timestamp, prefix, d.Service, d.ContainerName, timestamp, d.Msg)
}

func parseImage(image string) (repository, tag string) {
	// Docker image format is:
	// {registry}/(_|/r/{user_or_org})/{repository}:{tag}
	// Here we get the repository and tag from that format
	var (
		parts []string
	)
	parts = strings.Split(image, "/")
	image = parts[len(parts)-1]
	parts = strings.Split(image, ":")
	repository = parts[0]
	if len(parts) > 1 {
		tag = parts[1]
		if len(tag) > 8 {
			tag = tag[:8]
		}
	}
	return
}

// LogFormatter is a custom syslog formatter that uses log data to fill timestamp, hostname and tag instead of using local system information.
func LogFormatter(p srslog.Priority, hostname, tag, content string) (msg string) {
	parts := strings.SplitN(content, "|", 4)
	if len(parts) == 4 {
		msg = fmt.Sprintf("<%d>1 %s %s %s - - - %s", p, parts[0], parts[1], parts[2], parts[3])
	} else {
		msg = srslog.DefaultFormatter(p, hostname, tag, content)
	}
	return
}
<|endoftext|>


<commit_before>// Copyright 2017 gf Author(https://gitee.com/johng/gf). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://gitee.com/johng/gf.

// 配置管理.
// 配置文件格式支持：json, xml, toml, yaml/yml
package gcfg

import (
    "sync"
    "strings"
    "gitee.com/johng/gf/g/os/gfile"
    "gitee.com/johng/gf/g/container/gmap"
    "gitee.com/johng/gf/g/encoding/gjson"
    "gitee.com/johng/gf/g/container/gtype"
    "fmt"
)

const (
    gDEFAULT_CONFIG_FILE = "config.yml" // 默认的配置管理文件名称
)

// 配置管理对象
type Config struct {
    mu    sync.RWMutex             // 并发互斥锁
    path  *gtype.String            // 配置文件存放目录，绝对路径
    jsons *gmap.StringInterfaceMap // 配置文件对象
}

// 生成一个配置管理对象
func New(path string) *Config {
    return &Config {
        path  : gtype.NewString(path),
        jsons : gmap.NewStringInterfaceMap(),
    }
}

// 判断从哪个配置文件中获取内容
func (c *Config) filePath(file []string) string {
    path := gDEFAULT_CONFIG_FILE
    if len(file) > 0 {
        path = file[0]
    }
    fpath := c.path.Val() + gfile.Separator + path
    return fpath
}

// 设置配置管理器的配置文件存放目录绝对路径
func (c *Config) SetPath(path string) {
    if strings.Compare(c.path.Val(), path) != 0 {
        c.path.Set(path)
        c.mu.Lock()
        c.jsons = gmap.NewStringInterfaceMap()
        c.mu.Unlock()
    }
}

// 设置配置管理器的配置文件存放目录绝对路径
func (c *Config) GetPath() string {
    return c.path.Val()
}

// 添加配置文件到配置管理器中，第二个参数为非必须，如果不输入表示添加进入默认的配置名称中
func (c *Config) getJson(file []string) *gjson.Json {
    fpath := c.filePath(file)
    fmt.Println(fpath)
    if r := c.jsons.Get(fpath); r != nil {
        return r.(*gjson.Json)
    }
    if j, err := gjson.Load(fpath); err == nil {
        c.jsons.Set(fpath, j)
        return j
    }
    return nil
}

// 获取配置项，当不存在时返回nil
func (c *Config) Get(pattern string, file...string) interface{} {
    if j := c.getJson(file); j != nil {
        return j.Get(pattern)
    }
    return nil
}

// 获得一个键值对关联数组/哈希表，方便操作，不需要自己做类型转换
// 注意，如果获取的值不存在，或者类型与json类型不匹配，那么将会返回nil
func (c *Config) GetMap(pattern string, file...string)  map[string]interface{} {
    if j := c.getJson(file); j != nil {
        return j.GetMap(pattern)
    }
    return nil
}

// 获得一个数组[]interface{}，方便操作，不需要自己做类型转换
// 注意，如果获取的值不存在，或者类型与json类型不匹配，那么将会返回nil
func (c *Config) GetArray(pattern string, file...string)  []interface{} {
    if j := c.getJson(file); j != nil {
        return j.GetArray(pattern)
    }
    return nil
}

// 返回指定json中的string
func (c *Config) GetString(pattern string, file...string) string {
    if j := c.getJson(file); j != nil {
        return j.GetString(pattern)
    }
    return ""
}

// 返回指定json中的bool
func (c *Config) GetBool(pattern string, file...string) bool {
    if j := c.getJson(file); j != nil {
        return j.GetBool(pattern)
    }
    return false
}

// 返回指定json中的float32
func (c *Config) GetFloat32(pattern string, file...string) float32 {
    if j := c.getJson(file); j != nil {
        return j.GetFloat32(pattern)
    }
    return 0
}

// 返回指定json中的float64
func (c *Config) GetFloat64(pattern string, file...string) float64 {
    if j := c.getJson(file); j != nil {
        return j.GetFloat64(pattern)
    }
    return 0
}

// 返回指定json中的float64->int
func (c *Config) GetInt(pattern string, file...string)  int {
    if j := c.getJson(file); j != nil {
        return j.GetInt(pattern)
    }
    return 0
}

// 返回指定json中的float64->uint
func (c *Config) GetUint(pattern string, file...string)  uint {
    if j := c.getJson(file); j != nil {
        return j.GetUint(pattern)
    }
    return 0
}<commit_msg>去掉gcfg包中的debug<commit_after>// Copyright 2017 gf Author(https://gitee.com/johng/gf). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://gitee.com/johng/gf.

// 配置管理.
// 配置文件格式支持：json, xml, toml, yaml/yml
package gcfg

import (
    "sync"
    "strings"
    "gitee.com/johng/gf/g/os/gfile"
    "gitee.com/johng/gf/g/container/gmap"
    "gitee.com/johng/gf/g/encoding/gjson"
    "gitee.com/johng/gf/g/container/gtype"
)

const (
    gDEFAULT_CONFIG_FILE = "config.yml" // 默认的配置管理文件名称
)

// 配置管理对象
type Config struct {
    mu    sync.RWMutex             // 并发互斥锁
    path  *gtype.String            // 配置文件存放目录，绝对路径
    jsons *gmap.StringInterfaceMap // 配置文件对象
}

// 生成一个配置管理对象
func New(path string) *Config {
    return &Config {
        path  : gtype.NewString(path),
        jsons : gmap.NewStringInterfaceMap(),
    }
}

// 判断从哪个配置文件中获取内容
func (c *Config) filePath(file []string) string {
    path := gDEFAULT_CONFIG_FILE
    if len(file) > 0 {
        path = file[0]
    }
    fpath := c.path.Val() + gfile.Separator + path
    return fpath
}

// 设置配置管理器的配置文件存放目录绝对路径
func (c *Config) SetPath(path string) {
    if strings.Compare(c.path.Val(), path) != 0 {
        c.path.Set(path)
        c.mu.Lock()
        c.jsons = gmap.NewStringInterfaceMap()
        c.mu.Unlock()
    }
}

// 设置配置管理器的配置文件存放目录绝对路径
func (c *Config) GetPath() string {
    return c.path.Val()
}

// 添加配置文件到配置管理器中，第二个参数为非必须，如果不输入表示添加进入默认的配置名称中
func (c *Config) getJson(file []string) *gjson.Json {
    fpath := c.filePath(file)
    if r := c.jsons.Get(fpath); r != nil {
        return r.(*gjson.Json)
    }
    if j, err := gjson.Load(fpath); err == nil {
        c.jsons.Set(fpath, j)
        return j
    }
    return nil
}

// 获取配置项，当不存在时返回nil
func (c *Config) Get(pattern string, file...string) interface{} {
    if j := c.getJson(file); j != nil {
        return j.Get(pattern)
    }
    return nil
}

// 获得一个键值对关联数组/哈希表，方便操作，不需要自己做类型转换
// 注意，如果获取的值不存在，或者类型与json类型不匹配，那么将会返回nil
func (c *Config) GetMap(pattern string, file...string)  map[string]interface{} {
    if j := c.getJson(file); j != nil {
        return j.GetMap(pattern)
    }
    return nil
}

// 获得一个数组[]interface{}，方便操作，不需要自己做类型转换
// 注意，如果获取的值不存在，或者类型与json类型不匹配，那么将会返回nil
func (c *Config) GetArray(pattern string, file...string)  []interface{} {
    if j := c.getJson(file); j != nil {
        return j.GetArray(pattern)
    }
    return nil
}

// 返回指定json中的string
func (c *Config) GetString(pattern string, file...string) string {
    if j := c.getJson(file); j != nil {
        return j.GetString(pattern)
    }
    return ""
}

// 返回指定json中的bool
func (c *Config) GetBool(pattern string, file...string) bool {
    if j := c.getJson(file); j != nil {
        return j.GetBool(pattern)
    }
    return false
}

// 返回指定json中的float32
func (c *Config) GetFloat32(pattern string, file...string) float32 {
    if j := c.getJson(file); j != nil {
        return j.GetFloat32(pattern)
    }
    return 0
}

// 返回指定json中的float64
func (c *Config) GetFloat64(pattern string, file...string) float64 {
    if j := c.getJson(file); j != nil {
        return j.GetFloat64(pattern)
    }
    return 0
}

// 返回指定json中的float64->int
func (c *Config) GetInt(pattern string, file...string)  int {
    if j := c.getJson(file); j != nil {
        return j.GetInt(pattern)
    }
    return 0
}

// 返回指定json中的float64->uint
func (c *Config) GetUint(pattern string, file...string)  uint {
    if j := c.getJson(file); j != nil {
        return j.GetUint(pattern)
    }
    return 0
}<|endoftext|>


<commit_before>package ec2

import (
	"fmt"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

func Ls(*cobra.Command, []string) {
	svc := ec2.New(
		session.New(),
		&aws.Config{
			Credentials: getCredentials(viper.GetString("profile")),
			Region:      aws.String(viper.GetString("region")),
		},
	)

	var stateFilter *ec2.Filter
	if viper.GetBool("ec2.ls.all") {
		stateFilter = &ec2.Filter{}
	} else {
		stateFilter = &ec2.Filter{
			Name: aws.String("instance-state-name"),
			Values: []*string{
				aws.String("running"),
			},
		}
	}

	var tagFilter *ec2.Filter
	filterTag := viper.GetString("ec2.ls.filter-tag")
	if filterTag == "" {
	} else {
		tagParts := strings.Split(filterTag, ":")
		tagFilter = &ec2.Filter{
			Name: aws.String("tag:" + tagParts[0]),
			Values: []*string{
				aws.String("*" + tagParts[1] + "*"),
			},
		}
	}

	params := &ec2.DescribeInstancesInput{
		Filters: []*ec2.Filter{
			stateFilter,
			tagFilter,
		},
	}

	resp, err := svc.DescribeInstances(params)
	if err != nil {
		panic(err)
	}

	for _, res := range resp.Reservations {
		for _, inst := range res.Instances {
			fmt.Println(formatInstance(inst))
		}
	}
}

func getCredentials(profile string) *credentials.Credentials {
	var cred *credentials.Credentials
	if profile != "" {
		cred = credentials.NewSharedCredentials("", profile)
	}
	return cred
}

func formatInstance(inst *ec2.Instance) string {
	output := []string{
		*inst.InstanceId,
		*inst.InstanceType,
		publicIpAddress(inst),
		*inst.PrivateIpAddress,
		*inst.State.Name,
		(*inst.LaunchTime).Format("2006-01-02 15:04:05"),
	}
	tags := lookupTags(inst, viper.GetString("ec2.ls.output-tags"))
	output = append(output, tags...)
	return strings.Join(output[:], "\t")
}

func publicIpAddress(inst *ec2.Instance) string {
	ip := "___.___.___.___"
	if *inst.State.Name == "running" {
		ip = *inst.PublicIpAddress
	}
	return ip
}

func lookupTags(inst *ec2.Instance, keys string) []string {
	tags := strings.Split(keys, ",")
	var values []string

	for _, tag := range tags {
		values = append(values, lookupTag(inst, tag))
	}
	return values
}

func lookupTag(inst *ec2.Instance, key string) string {
	var value string
	for _, t := range inst.Tags {
		if *t.Key == key {
			value = *t.Value
			break
		}
	}
	return value
}
<commit_msg>Refactor ec2 ls --output-tags<commit_after>package ec2

import (
	"fmt"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

func Ls(*cobra.Command, []string) {
	svc := ec2.New(
		session.New(),
		&aws.Config{
			Credentials: getCredentials(viper.GetString("profile")),
			Region:      aws.String(viper.GetString("region")),
		},
	)

	var stateFilter *ec2.Filter
	if viper.GetBool("ec2.ls.all") {
		stateFilter = &ec2.Filter{}
	} else {
		stateFilter = &ec2.Filter{
			Name: aws.String("instance-state-name"),
			Values: []*string{
				aws.String("running"),
			},
		}
	}

	var tagFilter *ec2.Filter
	filterTag := viper.GetString("ec2.ls.filter-tag")
	if filterTag == "" {
	} else {
		tagParts := strings.Split(filterTag, ":")
		tagFilter = &ec2.Filter{
			Name: aws.String("tag:" + tagParts[0]),
			Values: []*string{
				aws.String("*" + tagParts[1] + "*"),
			},
		}
	}

	params := &ec2.DescribeInstancesInput{
		Filters: []*ec2.Filter{
			stateFilter,
			tagFilter,
		},
	}

	resp, err := svc.DescribeInstances(params)
	if err != nil {
		panic(err)
	}

	for _, res := range resp.Reservations {
		for _, inst := range res.Instances {
			fmt.Println(formatInstance(inst, viper.GetString("ec2.ls.output-tags")))
		}
	}
}

func getCredentials(profile string) *credentials.Credentials {
	var cred *credentials.Credentials
	if profile != "" {
		cred = credentials.NewSharedCredentials("", profile)
	}
	return cred
}

func formatInstance(inst *ec2.Instance, outputTags string) string {
	output := []string{
		*inst.InstanceId,
		*inst.InstanceType,
		publicIpAddress(inst),
		*inst.PrivateIpAddress,
		*inst.State.Name,
		(*inst.LaunchTime).Format("2006-01-02 15:04:05"),
	}
	tags := lookupTags(inst, outputTags)
	output = append(output, tags...)
	return strings.Join(output[:], "\t")
}

func publicIpAddress(inst *ec2.Instance) string {
	ip := "___.___.___.___"
	if *inst.State.Name == "running" {
		ip = *inst.PublicIpAddress
	}
	return ip
}

func lookupTags(inst *ec2.Instance, keys string) []string {
	tags := strings.Split(keys, ",")
	var values []string

	for _, tag := range tags {
		values = append(values, lookupTag(inst, tag))
	}
	return values
}

func lookupTag(inst *ec2.Instance, key string) string {
	var value string
	for _, t := range inst.Tags {
		if *t.Key == key {
			value = *t.Value
			break
		}
	}
	return value
}
<|endoftext|>


<commit_before>package gateway

import (
	"fmt"
	"sync"
	"time"

	"github.com/42wim/matterbridge/bridge"
	"github.com/42wim/matterbridge/bridge/config"
	samechannelgateway "github.com/42wim/matterbridge/gateway/samechannel"
)

type Router struct {
	config.Config

	BridgeMap        map[string]bridge.Factory
	Gateways         map[string]*Gateway
	Message          chan config.Message
	MattermostPlugin chan config.Message
	sync.RWMutex
}

func NewRouter(cfg config.Config, bridgeMap map[string]bridge.Factory) (*Router, error) {
	r := &Router{
		Config:           cfg,
		BridgeMap:        bridgeMap,
		Message:          make(chan config.Message),
		MattermostPlugin: make(chan config.Message),
		Gateways:         make(map[string]*Gateway),
	}
	sgw := samechannelgateway.New(cfg)
	gwconfigs := sgw.GetConfig()

	for _, entry := range append(gwconfigs, cfg.BridgeValues().Gateway...) {
		if !entry.Enable {
			continue
		}
		if entry.Name == "" {
			return nil, fmt.Errorf("%s", "Gateway without name found")
		}
		if _, ok := r.Gateways[entry.Name]; ok {
			return nil, fmt.Errorf("Gateway with name %s already exists", entry.Name)
		}
		r.Gateways[entry.Name] = New(entry, r)
	}
	return r, nil
}

func (r *Router) Start() error {
	m := make(map[string]*bridge.Bridge)
	for _, gw := range r.Gateways {
		flog.Infof("Parsing gateway %s", gw.Name)
		for _, br := range gw.Bridges {
			m[br.Account] = br
		}
	}
	for _, br := range m {
		flog.Infof("Starting bridge: %s ", br.Account)
		err := br.Connect()
		if err != nil {
			e := fmt.Errorf("Bridge %s failed to start: %v", br.Account, err)
			if r.disableBridge(br, e) {
				continue
			}
			return e
		}
		err = br.JoinChannels()
		if err != nil {
			e := fmt.Errorf("Bridge %s failed to join channel: %v", br.Account, err)
			if r.disableBridge(br, e) {
				continue
			}
			return e
		}
	}
	// remove unused bridges
	for _, gw := range r.Gateways {
		for i, br := range gw.Bridges {
			if br.Bridger == nil {
				flog.Errorf("removing failed bridge %s", i)
				delete(gw.Bridges, i)
			}
		}
	}
	go r.handleReceive()
	go r.updateChannelMembers()
	return nil
}

// disableBridge returns true and empties a bridge if we have IgnoreFailureOnStart configured
// otherwise returns false
func (r *Router) disableBridge(br *bridge.Bridge, err error) bool {
	if r.BridgeValues().General.IgnoreFailureOnStart {
		flog.Error(err)
		// setting this bridge empty
		*br = bridge.Bridge{}
		return true
	}
	return false
}

func (r *Router) getBridge(account string) *bridge.Bridge {
	for _, gw := range r.Gateways {
		if br, ok := gw.Bridges[account]; ok {
			return br
		}
	}
	return nil
}

func (r *Router) handleReceive() {
	for msg := range r.Message {
		msg := msg // scopelint
		r.handleEventGetChannelMembers(&msg)
		r.handleEventFailure(&msg)
		r.handleEventRejoinChannels(&msg)
		for _, gw := range r.Gateways {
			// record all the message ID's of the different bridges
			var msgIDs []*BrMsgID
			if gw.ignoreMessage(&msg) {
				continue
			}
			msg.Timestamp = time.Now()
			gw.modifyMessage(&msg)
			gw.handleFiles(&msg)
			for _, br := range gw.Bridges {
				msgIDs = append(msgIDs, gw.handleMessage(msg, br)...)
			}
			// only add the message ID if it doesn't already exists
			if _, ok := gw.Messages.Get(msg.Protocol + " " + msg.ID); !ok && msg.ID != "" {
				gw.Messages.Add(msg.Protocol+" "+msg.ID, msgIDs)
			}
		}
	}
}

// updateChannelMembers sends every minute an GetChannelMembers event to all bridges.
func (r *Router) updateChannelMembers() {
	// TODO sleep a minute because slack can take a while
	// fix this by having actually connectionDone events send to the router
	time.Sleep(time.Minute)
	for {
		for _, gw := range r.Gateways {
			for _, br := range gw.Bridges {
				// only for slack now
				if br.Protocol != "slack" {
					continue
				}
				flog.Debugf("sending %s to %s", config.EventGetChannelMembers, br.Account)
				if _, err := br.Send(config.Message{Event: config.EventGetChannelMembers}); err != nil {
					flog.Errorf("updateChannelMembers: %s", err)
				}
			}
		}
		time.Sleep(time.Minute)
	}
}
<commit_msg>Disable updateChannelMembers for now<commit_after>package gateway

import (
	"fmt"
	"sync"
	"time"

	"github.com/42wim/matterbridge/bridge"
	"github.com/42wim/matterbridge/bridge/config"
	samechannelgateway "github.com/42wim/matterbridge/gateway/samechannel"
)

type Router struct {
	config.Config

	BridgeMap        map[string]bridge.Factory
	Gateways         map[string]*Gateway
	Message          chan config.Message
	MattermostPlugin chan config.Message
	sync.RWMutex
}

func NewRouter(cfg config.Config, bridgeMap map[string]bridge.Factory) (*Router, error) {
	r := &Router{
		Config:           cfg,
		BridgeMap:        bridgeMap,
		Message:          make(chan config.Message),
		MattermostPlugin: make(chan config.Message),
		Gateways:         make(map[string]*Gateway),
	}
	sgw := samechannelgateway.New(cfg)
	gwconfigs := sgw.GetConfig()

	for _, entry := range append(gwconfigs, cfg.BridgeValues().Gateway...) {
		if !entry.Enable {
			continue
		}
		if entry.Name == "" {
			return nil, fmt.Errorf("%s", "Gateway without name found")
		}
		if _, ok := r.Gateways[entry.Name]; ok {
			return nil, fmt.Errorf("Gateway with name %s already exists", entry.Name)
		}
		r.Gateways[entry.Name] = New(entry, r)
	}
	return r, nil
}

func (r *Router) Start() error {
	m := make(map[string]*bridge.Bridge)
	for _, gw := range r.Gateways {
		flog.Infof("Parsing gateway %s", gw.Name)
		for _, br := range gw.Bridges {
			m[br.Account] = br
		}
	}
	for _, br := range m {
		flog.Infof("Starting bridge: %s ", br.Account)
		err := br.Connect()
		if err != nil {
			e := fmt.Errorf("Bridge %s failed to start: %v", br.Account, err)
			if r.disableBridge(br, e) {
				continue
			}
			return e
		}
		err = br.JoinChannels()
		if err != nil {
			e := fmt.Errorf("Bridge %s failed to join channel: %v", br.Account, err)
			if r.disableBridge(br, e) {
				continue
			}
			return e
		}
	}
	// remove unused bridges
	for _, gw := range r.Gateways {
		for i, br := range gw.Bridges {
			if br.Bridger == nil {
				flog.Errorf("removing failed bridge %s", i)
				delete(gw.Bridges, i)
			}
		}
	}
	go r.handleReceive()
	//go r.updateChannelMembers()
	return nil
}

// disableBridge returns true and empties a bridge if we have IgnoreFailureOnStart configured
// otherwise returns false
func (r *Router) disableBridge(br *bridge.Bridge, err error) bool {
	if r.BridgeValues().General.IgnoreFailureOnStart {
		flog.Error(err)
		// setting this bridge empty
		*br = bridge.Bridge{}
		return true
	}
	return false
}

func (r *Router) getBridge(account string) *bridge.Bridge {
	for _, gw := range r.Gateways {
		if br, ok := gw.Bridges[account]; ok {
			return br
		}
	}
	return nil
}

func (r *Router) handleReceive() {
	for msg := range r.Message {
		msg := msg // scopelint
		r.handleEventGetChannelMembers(&msg)
		r.handleEventFailure(&msg)
		r.handleEventRejoinChannels(&msg)
		for _, gw := range r.Gateways {
			// record all the message ID's of the different bridges
			var msgIDs []*BrMsgID
			if gw.ignoreMessage(&msg) {
				continue
			}
			msg.Timestamp = time.Now()
			gw.modifyMessage(&msg)
			gw.handleFiles(&msg)
			for _, br := range gw.Bridges {
				msgIDs = append(msgIDs, gw.handleMessage(msg, br)...)
			}
			// only add the message ID if it doesn't already exists
			if _, ok := gw.Messages.Get(msg.Protocol + " " + msg.ID); !ok && msg.ID != "" {
				gw.Messages.Add(msg.Protocol+" "+msg.ID, msgIDs)
			}
		}
	}
}

// updateChannelMembers sends every minute an GetChannelMembers event to all bridges.
func (r *Router) updateChannelMembers() {
	// TODO sleep a minute because slack can take a while
	// fix this by having actually connectionDone events send to the router
	time.Sleep(time.Minute)
	for {
		for _, gw := range r.Gateways {
			for _, br := range gw.Bridges {
				// only for slack now
				if br.Protocol != "slack" {
					continue
				}
				flog.Debugf("sending %s to %s", config.EventGetChannelMembers, br.Account)
				if _, err := br.Send(config.Message{Event: config.EventGetChannelMembers}); err != nil {
					flog.Errorf("updateChannelMembers: %s", err)
				}
			}
		}
		time.Sleep(time.Minute)
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 Spinpunch, Inc. All Rights Reserved.
// See License.txt for license information.

package store

import (
	"github.com/mattermost/platform/model"
	"testing"
)

func TestTeamStoreSave(t *testing.T) {
	Setup()

	o1 := model.Team{}
	o1.Name = "Name"
	o1.Domain = "a" + model.NewId() + "b"
	o1.Email = model.NewId() + "@nowhere.com"
	o1.Type = model.TEAM_OPEN

	if err := (<-store.Team().Save(&o1)).Err; err != nil {
		t.Fatal("couldn't save item", err)
	}

	if err := (<-store.Team().Save(&o1)).Err; err == nil {
		t.Fatal("shouldn't be able to update from save")
	}

	o1.Id = ""
	if err := (<-store.Team().Save(&o1)).Err; err == nil {
		t.Fatal("should be unique domain")
	}
}

func TestTeamStoreUpdate(t *testing.T) {
	Setup()

	o1 := model.Team{}
	o1.Name = "Name"
	o1.Domain = "a" + model.NewId() + "b"
	o1.Email = model.NewId() + "@nowhere.com"
	o1.Type = model.TEAM_OPEN
	<-store.Team().Save(&o1)

	if err := (<-store.Team().Update(&o1)).Err; err != nil {
		t.Fatal(err)
	}

	o1.Id = "missing"
	if err := (<-store.Team().Update(&o1)).Err; err == nil {
		t.Fatal("Update should have failed because of missing key")
	}

	o1.Id = model.NewId()
	if err := (<-store.Team().Update(&o1)).Err; err == nil {
		t.Fatal("Update should have faile because id change")
	}
}

func TestTeamStoreUpdateName(t *testing.T) {
	Setup()

	o1 := &model.Team{}
	o1.Name = "Name"
	o1.Domain = "a" + model.NewId() + "b"
	o1.Email = model.NewId() + "@nowhere.com"
	o1.Type = model.TEAM_OPEN
	o1 = (<-store.Team().Save(o1)).Data.(*model.Team)

	newName := "NewName"

	if err := (<-store.Team().UpdateName(newName, o1.Id)).Err; err != nil {
		t.Fatal(err)
	}

	ro1 := (<-store.Team().Get(o1.Id)).Data.(*model.Team)
	if ro1.Name != newName {
		t.Fatal("Name not updated")
	}
}

func TestTeamStoreGet(t *testing.T) {
	Setup()

	o1 := model.Team{}
	o1.Name = "Name"
	o1.Domain = "a" + model.NewId() + "b"
	o1.Email = model.NewId() + "@nowhere.com"
	o1.Type = model.TEAM_OPEN
	<-store.Team().Save(&o1)

	if r1 := <-store.Team().Get(o1.Id); r1.Err != nil {
		t.Fatal(r1.Err)
	} else {
		if r1.Data.(*model.Team).ToJson() != o1.ToJson() {
			t.Fatal("invalid returned team")
		}
	}

	if err := (<-store.Team().Get("")).Err; err == nil {
		t.Fatal("Missing id should have failed")
	}
}

func TestTeamStoreGetByDomain(t *testing.T) {
	Setup()

	o1 := model.Team{}
	o1.Name = "Name"
	o1.Domain = "a" + model.NewId() + "b"
	o1.Email = model.NewId() + "@nowhere.com"
	o1.Type = model.TEAM_OPEN

	if err := (<-store.Team().Save(&o1)).Err; err != nil {
		t.Fatal(rrr)
	}

	if r1 := <-store.Team().GetByDomain(o1.Domain); r1.Err != nil {
		t.Fatal(r1.Err)
	} else {
		if r1.Data.(*model.Team).ToJson() != o1.ToJson() {
			t.Fatal("invalid returned team")
		}
	}

	if err := (<-store.Team().GetByDomain("")).Err; err == nil {
		t.Fatal("Missing id should have failed")
	}
}

func TestTeamStoreGetForEmail(t *testing.T) {
	Setup()

	o1 := model.Team{}
	o1.Name = "Name"
	o1.Domain = "a" + model.NewId() + "b"
	o1.Email = model.NewId() + "@nowhere.com"
	o1.Type = model.TEAM_OPEN
	<-store.Team().Save(&o1)

	u1 := model.User{}
	u1.TeamId = o1.Id
	u1.Email = model.NewId()
	<-store.User().Save(&u1)

	if r1 := <-store.Team().GetTeamsForEmail(u1.Email); r1.Err != nil {
		t.Fatal(r1.Err)
	} else {
		teams := r1.Data.([]*model.Team)

		if teams[0].Id != o1.Id {
			t.Fatal("failed to lookup by email")
		}
	}

	if r1 := <-store.Team().GetTeamsForEmail("missing"); r1.Err != nil {
		t.Fatal(r1.Err)
	}
}
<commit_msg>fixing unit test<commit_after>// Copyright (c) 2015 Spinpunch, Inc. All Rights Reserved.
// See License.txt for license information.

package store

import (
	"github.com/mattermost/platform/model"
	"testing"
)

func TestTeamStoreSave(t *testing.T) {
	Setup()

	o1 := model.Team{}
	o1.Name = "Name"
	o1.Domain = "a" + model.NewId() + "b"
	o1.Email = model.NewId() + "@nowhere.com"
	o1.Type = model.TEAM_OPEN

	if err := (<-store.Team().Save(&o1)).Err; err != nil {
		t.Fatal("couldn't save item", err)
	}

	if err := (<-store.Team().Save(&o1)).Err; err == nil {
		t.Fatal("shouldn't be able to update from save")
	}

	o1.Id = ""
	if err := (<-store.Team().Save(&o1)).Err; err == nil {
		t.Fatal("should be unique domain")
	}
}

func TestTeamStoreUpdate(t *testing.T) {
	Setup()

	o1 := model.Team{}
	o1.Name = "Name"
	o1.Domain = "a" + model.NewId() + "b"
	o1.Email = model.NewId() + "@nowhere.com"
	o1.Type = model.TEAM_OPEN
	<-store.Team().Save(&o1)

	if err := (<-store.Team().Update(&o1)).Err; err != nil {
		t.Fatal(err)
	}

	o1.Id = "missing"
	if err := (<-store.Team().Update(&o1)).Err; err == nil {
		t.Fatal("Update should have failed because of missing key")
	}

	o1.Id = model.NewId()
	if err := (<-store.Team().Update(&o1)).Err; err == nil {
		t.Fatal("Update should have faile because id change")
	}
}

func TestTeamStoreUpdateName(t *testing.T) {
	Setup()

	o1 := &model.Team{}
	o1.Name = "Name"
	o1.Domain = "a" + model.NewId() + "b"
	o1.Email = model.NewId() + "@nowhere.com"
	o1.Type = model.TEAM_OPEN
	o1 = (<-store.Team().Save(o1)).Data.(*model.Team)

	newName := "NewName"

	if err := (<-store.Team().UpdateName(newName, o1.Id)).Err; err != nil {
		t.Fatal(err)
	}

	ro1 := (<-store.Team().Get(o1.Id)).Data.(*model.Team)
	if ro1.Name != newName {
		t.Fatal("Name not updated")
	}
}

func TestTeamStoreGet(t *testing.T) {
	Setup()

	o1 := model.Team{}
	o1.Name = "Name"
	o1.Domain = "a" + model.NewId() + "b"
	o1.Email = model.NewId() + "@nowhere.com"
	o1.Type = model.TEAM_OPEN
	<-store.Team().Save(&o1)

	if r1 := <-store.Team().Get(o1.Id); r1.Err != nil {
		t.Fatal(r1.Err)
	} else {
		if r1.Data.(*model.Team).ToJson() != o1.ToJson() {
			t.Fatal("invalid returned team")
		}
	}

	if err := (<-store.Team().Get("")).Err; err == nil {
		t.Fatal("Missing id should have failed")
	}
}

func TestTeamStoreGetByDomain(t *testing.T) {
	Setup()

	o1 := model.Team{}
	o1.Name = "Name"
	o1.Domain = "a" + model.NewId() + "b"
	o1.Email = model.NewId() + "@nowhere.com"
	o1.Type = model.TEAM_OPEN

	if err := (<-store.Team().Save(&o1)).Err; err != nil {
		t.Fatal(err)
	}

	if r1 := <-store.Team().GetByDomain(o1.Domain); r1.Err != nil {
		t.Fatal(r1.Err)
	} else {
		if r1.Data.(*model.Team).ToJson() != o1.ToJson() {
			t.Fatal("invalid returned team")
		}
	}

	if err := (<-store.Team().GetByDomain("")).Err; err == nil {
		t.Fatal("Missing id should have failed")
	}
}

func TestTeamStoreGetForEmail(t *testing.T) {
	Setup()

	o1 := model.Team{}
	o1.Name = "Name"
	o1.Domain = "a" + model.NewId() + "b"
	o1.Email = model.NewId() + "@nowhere.com"
	o1.Type = model.TEAM_OPEN
	<-store.Team().Save(&o1)

	u1 := model.User{}
	u1.TeamId = o1.Id
	u1.Email = model.NewId()
	<-store.User().Save(&u1)

	if r1 := <-store.Team().GetTeamsForEmail(u1.Email); r1.Err != nil {
		t.Fatal(r1.Err)
	} else {
		teams := r1.Data.([]*model.Team)

		if teams[0].Id != o1.Id {
			t.Fatal("failed to lookup by email")
		}
	}

	if r1 := <-store.Team().GetTeamsForEmail("missing"); r1.Err != nil {
		t.Fatal(r1.Err)
	}
}
<|endoftext|>


<commit_before><commit_msg>xbee test spif-up<commit_after><|endoftext|>


<commit_before><commit_msg>Add test case for new API<commit_after><|endoftext|>