<commit_before>package tsdb

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"sync"

	"github.com/influxdb/influxdb/influxql"
)

func NewStore(path string) *Store {
	return &Store{
		path:   path,
		Logger: log.New(os.Stderr, "[store] ", log.LstdFlags),
	}
}

var (
	ErrShardNotFound = fmt.Errorf("shard not found")
)

type Store struct {
	mu   sync.RWMutex
	path string

	databaseIndexes map[string]*DatabaseIndex
	shards          map[uint64]*Shard

	Logger *log.Logger
}

func (s *Store) CreateShard(database, retentionPolicy string, shardID uint64) error {
	s.mu.Lock()
	defer s.mu.Unlock()

	// shard already exists
	if _, ok := s.shards[shardID]; ok {
		return nil
	}

	// created the db and retention policy dirs if they don't exist
	if err := os.MkdirAll(filepath.Join(s.path, database, retentionPolicy), 0700); err != nil {
		return err
	}

	// create the database index if it does not exist
	db, ok := s.databaseIndexes[database]
	if !ok {
		db = NewDatabaseIndex()
		s.databaseIndexes[database] = db
	}

	shardPath := filepath.Join(s.path, database, retentionPolicy, strconv.FormatUint(shardID, 10))
	shard := NewShard(db, shardPath)
	if err := shard.Open(); err != nil {
		return err
	}

	s.shards[shardID] = shard

	return nil
}

// DeleteShard removes a shard from disk.
func (s *Store) DeleteShard(shardID uint64) error {
	s.mu.Lock()
	defer s.mu.Unlock()

	// ensure shard exists
	sh, ok := s.shards[shardID]
	if !ok {
		return nil
	}

	if err := sh.Close(); err != nil {
		return err
	}

	if err := os.Remove(sh.path); err != nil {
		return err
	}

	delete(s.shards, shardID)

	return nil
}

// DeleteDatabase will close all shards associated with a database and remove the directory and files from disk.
func (s *Store) DeleteDatabase(name string, shardIDs []uint64) error {
	s.mu.Lock()
	defer s.mu.Unlock()
	for _, id := range shardIDs {
		shard := s.shards[id]
		if shard != nil {
			shard.Close()
		}
	}
	return os.RemoveAll(s.path)
}

func (s *Store) Shard(shardID uint64) *Shard {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.shards[shardID]
}

// ShardIDs returns a slice of all ShardIDs under management.
func (s *Store) ShardIDs() []uint64 {
	ids := make([]uint64, 0, len(s.shards))
	for i, _ := range s.shards {
		ids = append(ids, i)
	}
	return ids
}

func (s *Store) ValidateAggregateFieldsInStatement(shardID uint64, measurementName string, stmt *influxql.SelectStatement) error {
	s.mu.RLock()
	shard := s.shards[shardID]
	s.mu.RUnlock()
	if shard == nil {
		return ErrShardNotFound
	}
	return shard.ValidateAggregateFieldsInStatement(measurementName, stmt)
}

func (s *Store) DatabaseIndex(name string) *DatabaseIndex {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.databaseIndexes[name]
}

func (s *Store) Measurement(database, name string) *Measurement {
	s.mu.RLock()
	db := s.databaseIndexes[database]
	s.mu.RUnlock()
	if db == nil {
		return nil
	}
	return db.Measurement(name)
}

// deleteSeries lopos through the local shards and deletes the series data and metadata for the passed in series keys
func (s *Store) deleteSeries(keys []string) error {
	s.mu.RLock()
	defer s.mu.RUnlock()
	for _, sh := range s.shards {
		if err := sh.deleteSeries(keys); err != nil {
			return err
		}
	}
	return nil
}

// deleteMeasurement loops through the local shards and removes the measurement field encodings from each shard
func (s *Store) deleteMeasurement(name string, seriesKeys []string) error {
	s.mu.RLock()
	defer s.mu.RUnlock()
	for _, sh := range s.shards {
		if err := sh.deleteMeasurement(name, seriesKeys); err != nil {
			return err
		}
	}
	return nil
}

func (s *Store) loadIndexes() error {
	dbs, err := ioutil.ReadDir(s.path)
	if err != nil {
		return err
	}
	for _, db := range dbs {
		if !db.IsDir() {
			s.Logger.Printf("Skipping database dir: %s. Not a directory", db.Name())
			continue
		}
		s.databaseIndexes[db.Name()] = NewDatabaseIndex()
	}
	return nil
}

func (s *Store) loadShards() error {
	// loop through the current database indexes
	for db := range s.databaseIndexes {
		rps, err := ioutil.ReadDir(filepath.Join(s.path, db))
		if err != nil {
			return err
		}

		for _, rp := range rps {
			// retention policies should be directories.  Skip anything that is not a dir.
			if !rp.IsDir() {
				s.Logger.Printf("Skipping retention policy dir: %s. Not a directory", rp.Name())
				continue
			}

			shards, err := ioutil.ReadDir(filepath.Join(s.path, db, rp.Name()))
			if err != nil {
				return err
			}
			for _, sh := range shards {
				path := filepath.Join(s.path, db, rp.Name(), sh.Name())

				// Shard file names are numeric shardIDs
				shardID, err := strconv.ParseUint(sh.Name(), 10, 64)
				if err != nil {
					s.Logger.Printf("Skipping shard: %s. Not a valid path", rp.Name())
					continue
				}

				shard := NewShard(s.databaseIndexes[db], path)
				shard.Open()
				s.shards[shardID] = shard
			}
		}
	}
	return nil

}

func (s *Store) Open() error {
	s.mu.Lock()
	defer s.mu.Unlock()

	s.shards = map[uint64]*Shard{}
	s.databaseIndexes = map[string]*DatabaseIndex{}

	// Create directory.
	if err := os.MkdirAll(s.path, 0777); err != nil {
		return err
	}

	// TODO: Start AE for Node
	if err := s.loadIndexes(); err != nil {
		return err
	}

	if err := s.loadShards(); err != nil {
		return err
	}

	return nil
}

func (s *Store) WriteToShard(shardID uint64, points []Point) error {
	s.mu.RLock()
	defer s.mu.RUnlock()
	sh, ok := s.shards[shardID]
	if !ok {
		return ErrShardNotFound
	}

	return sh.WritePoints(points)
}

func (s *Store) Close() error {
	s.mu.Lock()
	defer s.mu.Unlock()

	for _, sh := range s.shards {
		if err := sh.Close(); err != nil {
			return err
		}
	}
	s.shards = nil
	s.databaseIndexes = nil

	return nil
}
<commit_msg>Fix minor spelling mistake<commit_after>package tsdb

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"sync"

	"github.com/influxdb/influxdb/influxql"
)

func NewStore(path string) *Store {
	return &Store{
		path:   path,
		Logger: log.New(os.Stderr, "[store] ", log.LstdFlags),
	}
}

var (
	ErrShardNotFound = fmt.Errorf("shard not found")
)

type Store struct {
	mu   sync.RWMutex
	path string

	databaseIndexes map[string]*DatabaseIndex
	shards          map[uint64]*Shard

	Logger *log.Logger
}

func (s *Store) CreateShard(database, retentionPolicy string, shardID uint64) error {
	s.mu.Lock()
	defer s.mu.Unlock()

	// shard already exists
	if _, ok := s.shards[shardID]; ok {
		return nil
	}

	// created the db and retention policy dirs if they don't exist
	if err := os.MkdirAll(filepath.Join(s.path, database, retentionPolicy), 0700); err != nil {
		return err
	}

	// create the database index if it does not exist
	db, ok := s.databaseIndexes[database]
	if !ok {
		db = NewDatabaseIndex()
		s.databaseIndexes[database] = db
	}

	shardPath := filepath.Join(s.path, database, retentionPolicy, strconv.FormatUint(shardID, 10))
	shard := NewShard(db, shardPath)
	if err := shard.Open(); err != nil {
		return err
	}

	s.shards[shardID] = shard

	return nil
}

// DeleteShard removes a shard from disk.
func (s *Store) DeleteShard(shardID uint64) error {
	s.mu.Lock()
	defer s.mu.Unlock()

	// ensure shard exists
	sh, ok := s.shards[shardID]
	if !ok {
		return nil
	}

	if err := sh.Close(); err != nil {
		return err
	}

	if err := os.Remove(sh.path); err != nil {
		return err
	}

	delete(s.shards, shardID)

	return nil
}

// DeleteDatabase will close all shards associated with a database and remove the directory and files from disk.
func (s *Store) DeleteDatabase(name string, shardIDs []uint64) error {
	s.mu.Lock()
	defer s.mu.Unlock()
	for _, id := range shardIDs {
		shard := s.shards[id]
		if shard != nil {
			shard.Close()
		}
	}
	return os.RemoveAll(s.path)
}

func (s *Store) Shard(shardID uint64) *Shard {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.shards[shardID]
}

// ShardIDs returns a slice of all ShardIDs under management.
func (s *Store) ShardIDs() []uint64 {
	ids := make([]uint64, 0, len(s.shards))
	for i, _ := range s.shards {
		ids = append(ids, i)
	}
	return ids
}

func (s *Store) ValidateAggregateFieldsInStatement(shardID uint64, measurementName string, stmt *influxql.SelectStatement) error {
	s.mu.RLock()
	shard := s.shards[shardID]
	s.mu.RUnlock()
	if shard == nil {
		return ErrShardNotFound
	}
	return shard.ValidateAggregateFieldsInStatement(measurementName, stmt)
}

func (s *Store) DatabaseIndex(name string) *DatabaseIndex {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.databaseIndexes[name]
}

func (s *Store) Measurement(database, name string) *Measurement {
	s.mu.RLock()
	db := s.databaseIndexes[database]
	s.mu.RUnlock()
	if db == nil {
		return nil
	}
	return db.Measurement(name)
}

// deleteSeries loops through the local shards and deletes the series data and metadata for the passed in series keys
func (s *Store) deleteSeries(keys []string) error {
	s.mu.RLock()
	defer s.mu.RUnlock()
	for _, sh := range s.shards {
		if err := sh.deleteSeries(keys); err != nil {
			return err
		}
	}
	return nil
}

// deleteMeasurement loops through the local shards and removes the measurement field encodings from each shard
func (s *Store) deleteMeasurement(name string, seriesKeys []string) error {
	s.mu.RLock()
	defer s.mu.RUnlock()
	for _, sh := range s.shards {
		if err := sh.deleteMeasurement(name, seriesKeys); err != nil {
			return err
		}
	}
	return nil
}

func (s *Store) loadIndexes() error {
	dbs, err := ioutil.ReadDir(s.path)
	if err != nil {
		return err
	}
	for _, db := range dbs {
		if !db.IsDir() {
			s.Logger.Printf("Skipping database dir: %s. Not a directory", db.Name())
			continue
		}
		s.databaseIndexes[db.Name()] = NewDatabaseIndex()
	}
	return nil
}

func (s *Store) loadShards() error {
	// loop through the current database indexes
	for db := range s.databaseIndexes {
		rps, err := ioutil.ReadDir(filepath.Join(s.path, db))
		if err != nil {
			return err
		}

		for _, rp := range rps {
			// retention policies should be directories.  Skip anything that is not a dir.
			if !rp.IsDir() {
				s.Logger.Printf("Skipping retention policy dir: %s. Not a directory", rp.Name())
				continue
			}

			shards, err := ioutil.ReadDir(filepath.Join(s.path, db, rp.Name()))
			if err != nil {
				return err
			}
			for _, sh := range shards {
				path := filepath.Join(s.path, db, rp.Name(), sh.Name())

				// Shard file names are numeric shardIDs
				shardID, err := strconv.ParseUint(sh.Name(), 10, 64)
				if err != nil {
					s.Logger.Printf("Skipping shard: %s. Not a valid path", rp.Name())
					continue
				}

				shard := NewShard(s.databaseIndexes[db], path)
				shard.Open()
				s.shards[shardID] = shard
			}
		}
	}
	return nil

}

func (s *Store) Open() error {
	s.mu.Lock()
	defer s.mu.Unlock()

	s.shards = map[uint64]*Shard{}
	s.databaseIndexes = map[string]*DatabaseIndex{}

	// Create directory.
	if err := os.MkdirAll(s.path, 0777); err != nil {
		return err
	}

	// TODO: Start AE for Node
	if err := s.loadIndexes(); err != nil {
		return err
	}

	if err := s.loadShards(); err != nil {
		return err
	}

	return nil
}

func (s *Store) WriteToShard(shardID uint64, points []Point) error {
	s.mu.RLock()
	defer s.mu.RUnlock()
	sh, ok := s.shards[shardID]
	if !ok {
		return ErrShardNotFound
	}

	return sh.WritePoints(points)
}

func (s *Store) Close() error {
	s.mu.Lock()
	defer s.mu.Unlock()

	for _, sh := range s.shards {
		if err := sh.Close(); err != nil {
			return err
		}
	}
	s.shards = nil
	s.databaseIndexes = nil

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"github.com/grsakea/kappastat/common"
	"github.com/mrshankly/go-twitch/twitch"
	"gopkg.in/mgo.v2"
	"gopkg.in/redis.v3"
	"log"
	"net/http"
	"os"
	"time"
)

type Controller struct {
	config      Config
	infosChat   chan kappastat.ChatEntry
	infosViewer chan kappastat.ViewerCount
	cViewer     chan Message
	cChat       chan Message
	cStat       chan Message
	tracked     map[string]bool
	storage     StorageController
	comm        *redis.Client
	twitchAPI   *twitch.Client
}

type StorageController struct {
	db     *mgo.Database
	views  *mgo.Collection
	chat   *mgo.Collection
	follow *mgo.Collection
}

type Signal int

const (
	AddStream Signal = iota
	RemoveStream
	Stop
	Restart
)

type Message struct {
	s Signal
	v string
}

func (c *Controller) Loop() {
	log.Print("Start Loop")

	go loopViewers(c.twitchAPI, c.cViewer, c.infosViewer)
	go loopChat(c.cChat, c.infosChat)
	go loopStat(c.cStat, c.storage.db)

	t := time.NewTicker(time.Minute).C

	for {
		select {
		case temp, ok := <-c.infosViewer:
			if !ok {
				log.Println("InfosViewer failed")
				return
			}
			storeViewerCount(c.storage.views, temp)

		case temp, ok := <-c.infosChat:
			if !ok {
				log.Println("InfosChat failed")
				return
			}
			storeChatEntry(c.storage.chat, temp)
		case <-t:
			for c.comm.LLen("add").Val() != 0 {
				val, _ := c.comm.LPop("add").Result()
				c.AddStream(val)
			}
			for c.comm.LLen("del").Val() != 0 {
				c.AddStream(c.comm.LPop("del").String())
			}
		}
	}
}

func SetupController(dbName string) (contr *Controller) {
	store := StorageController{
		db: setupStorage(dbName),
	}
	store.views = store.db.C("viewer_count")
	store.chat = store.db.C("chat_entries")
	store.follow = store.db.C("follow")

	client := redis.NewClient(&redis.Options{
		Addr:     "localhost:6379",
		Password: "", // no password set
		DB:       0,  // use default DB
	})

	contr = &Controller{
		config:      LoadConfig("config.json"),
		infosChat:   make(chan kappastat.ChatEntry),
		infosViewer: make(chan kappastat.ViewerCount),
		cViewer:     make(chan Message),
		cChat:       make(chan Message),
		cStat:       make(chan Message),
		tracked:     make(map[string]bool),
		comm:        client,
		storage:     store,
		twitchAPI:   twitch.NewClient(&http.Client{}),
	}

	contr.loadFollowed()

	os.Setenv("GO-TWITCH_CLIENTID", contr.config.ClientID)
	return
}

func (c *Controller) AddStream(name string) error {
	_, present := c.tracked[name]
	if present {
		log.Println("Already Following")
		return errors.New("Already Following")
	}
	log.Println("Adding", name)
	user, err := c.twitchAPI.Users.User(name)
	if err != nil {
		log.Print(err)
		return err
	}
	c.storage.follow.Insert(user)

	c.tracked[name] = true

	go func(name string) {
		c.cChat <- Message{AddStream, name}
		c.cViewer <- Message{AddStream, name}
		c.cStat <- Message{AddStream, name}
	}(name)
	log.Println("Finished adding", name)
	return nil
}

func (c *Controller) RemoveStream(name string) {
	_, present := c.tracked[name]
	if !present {
		log.Println("Not Following")
		return
	}
	log.Println("Removing ", name)

	go func(name string) {
		c.cChat <- Message{RemoveStream, name}
		c.cViewer <- Message{RemoveStream, name}
		c.cStat <- Message{RemoveStream, name}
	}(name)
	delete(c.tracked, name)
}

func (c *Controller) ListStreams() []string {
	keys := make([]string, 0, len(c.tracked))
	for k := range c.tracked {
		keys = append(keys, k)
	}
	return keys
}

func (c *Controller) loadFollowed() {
	var f []twitch.UserS
	c.storage.follow.Find(nil).All(&f)

	for _, v := range f {
		c.tracked[v.Name] = true
		go func(name string) {
			c.cChat <- Message{AddStream, name}
			c.cViewer <- Message{AddStream, name}
			c.cStat <- Message{AddStream, name}
		}(v.Name)
	}
}
<commit_msg>Bugfix<commit_after>package main

import (
	"errors"
	"github.com/grsakea/kappastat/common"
	"github.com/mrshankly/go-twitch/twitch"
	"gopkg.in/mgo.v2"
	"gopkg.in/redis.v3"
	"log"
	"net/http"
	"os"
	"time"
)

type Controller struct {
	config      Config
	infosChat   chan kappastat.ChatEntry
	infosViewer chan kappastat.ViewerCount
	cViewer     chan Message
	cChat       chan Message
	cStat       chan Message
	tracked     map[string]bool
	storage     StorageController
	comm        *redis.Client
	twitchAPI   *twitch.Client
}

type StorageController struct {
	db     *mgo.Database
	views  *mgo.Collection
	chat   *mgo.Collection
	follow *mgo.Collection
}

type Signal int

const (
	AddStream Signal = iota
	RemoveStream
	Stop
	Restart
)

type Message struct {
	s Signal
	v string
}

func (c *Controller) Loop() {
	log.Print("Start Loop")

	go loopViewers(c.twitchAPI, c.cViewer, c.infosViewer)
	go loopChat(c.cChat, c.infosChat)
	go loopStat(c.cStat, c.storage.db)

	t := time.NewTicker(time.Minute).C

	for {
		select {
		case temp, ok := <-c.infosViewer:
			if !ok {
				log.Println("InfosViewer failed")
				return
			}
			storeViewerCount(c.storage.views, temp)

		case temp, ok := <-c.infosChat:
			if !ok {
				log.Println("InfosChat failed")
				return
			}
			storeChatEntry(c.storage.chat, temp)
		case <-t:
			for c.comm.LLen("add").Val() != 0 {
				val, _ := c.comm.LPop("add").Result()
				c.AddStream(val)
			}
			for c.comm.LLen("del").Val() != 0 {
				c.RemoveStream(c.comm.LPop("del").String())
			}
		}
	}
}

func SetupController(dbName string) (contr *Controller) {
	store := StorageController{
		db: setupStorage(dbName),
	}
	store.views = store.db.C("viewer_count")
	store.chat = store.db.C("chat_entries")
	store.follow = store.db.C("follow")

	client := redis.NewClient(&redis.Options{
		Addr:     "localhost:6379",
		Password: "", // no password set
		DB:       0,  // use default DB
	})

	contr = &Controller{
		config:      LoadConfig("config.json"),
		infosChat:   make(chan kappastat.ChatEntry),
		infosViewer: make(chan kappastat.ViewerCount),
		cViewer:     make(chan Message),
		cChat:       make(chan Message),
		cStat:       make(chan Message),
		tracked:     make(map[string]bool),
		comm:        client,
		storage:     store,
		twitchAPI:   twitch.NewClient(&http.Client{}),
	}

	contr.loadFollowed()

	os.Setenv("GO-TWITCH_CLIENTID", contr.config.ClientID)
	return
}

func (c *Controller) AddStream(name string) error {
	_, present := c.tracked[name]
	if present {
		log.Println("Already Following")
		return errors.New("Already Following")
	}
	log.Println("Adding", name)
	user, err := c.twitchAPI.Users.User(name)
	if err != nil {
		log.Print(err)
		return err
	}
	c.storage.follow.Insert(user)

	c.tracked[name] = true

	go func(name string) {
		c.cChat <- Message{AddStream, name}
		c.cViewer <- Message{AddStream, name}
		c.cStat <- Message{AddStream, name}
	}(name)
	log.Println("Finished adding", name)
	return nil
}

func (c *Controller) RemoveStream(name string) {
	_, present := c.tracked[name]
	if !present {
		log.Println("Not Following")
		return
	}
	log.Println("Removing ", name)

	go func(name string) {
		c.cChat <- Message{RemoveStream, name}
		c.cViewer <- Message{RemoveStream, name}
		c.cStat <- Message{RemoveStream, name}
	}(name)
	delete(c.tracked, name)
}

func (c *Controller) ListStreams() []string {
	keys := make([]string, 0, len(c.tracked))
	for k := range c.tracked {
		keys = append(keys, k)
	}
	return keys
}

func (c *Controller) loadFollowed() {
	var f []twitch.UserS
	c.storage.follow.Find(nil).All(&f)

	for _, v := range f {
		c.tracked[v.Name] = true
		go func(name string) {
			c.cChat <- Message{AddStream, name}
			c.cViewer <- Message{AddStream, name}
			c.cStat <- Message{AddStream, name}
		}(v.Name)
	}
}
<|endoftext|>


<commit_before>package agent

import (
	"context"
	"flag"
	"fmt"
	"os"
	"os/signal"
	"path/filepath"
	"strings"
	"syscall"
	"time"

	"github.com/hashicorp/go-checkpoint"
	"github.com/hashicorp/go-hclog"
	mcli "github.com/mitchellh/cli"

	"github.com/hashicorp/consul/agent"
	"github.com/hashicorp/consul/agent/config"
	"github.com/hashicorp/consul/command/cli"
	"github.com/hashicorp/consul/command/flags"
	"github.com/hashicorp/consul/lib"
	"github.com/hashicorp/consul/logging"
	"github.com/hashicorp/consul/service_os"
	consulversion "github.com/hashicorp/consul/version"
)

func New(ui cli.Ui) *cmd {
	c := &cmd{
		ui:                ui,
		revision:          consulversion.GitCommit,
		version:           consulversion.Version,
		versionPrerelease: consulversion.VersionPrerelease,
		versionHuman:      consulversion.GetHumanVersion(),
		flags:             flag.NewFlagSet("", flag.ContinueOnError),
	}
	config.AddFlags(c.flags, &c.configLoadOpts)
	c.help = flags.Usage(help, c.flags)
	return c
}

// AgentCommand is a Command implementation that runs a Consul agent.
// The command will not end unless a shutdown message is sent on the
// ShutdownCh. If two messages are sent on the ShutdownCh it will forcibly
// exit.
type cmd struct {
	ui                cli.Ui
	flags             *flag.FlagSet
	http              *flags.HTTPFlags
	help              string
	revision          string
	version           string
	versionPrerelease string
	versionHuman      string
	configLoadOpts    config.LoadOpts
	logger            hclog.InterceptLogger
}

func (c *cmd) Run(args []string) int {
	code := c.run(args)
	if c.logger != nil {
		c.logger.Info("Exit code", "code", code)
	}
	return code
}

// checkpointResults is used to handler periodic results from our update checker
func (c *cmd) checkpointResults(results *checkpoint.CheckResponse, err error) {
	if err != nil {
		c.logger.Error("Failed to check for updates", "error", err)
		return
	}
	if results.Outdated {
		c.logger.Info("Newer Consul version available", "new_version", results.CurrentVersion, "current_version", c.version)
	}
	for _, alert := range results.Alerts {
		switch alert.Level {
		case "info":
			c.logger.Info("Bulletin", "alert_level", alert.Level, "alert_message", alert.Message, "alert_URL", alert.URL)
		default:
			c.logger.Error("Bulletin", "alert_level", alert.Level, "alert_message", alert.Message, "alert_URL", alert.URL)
		}
	}
}

func (c *cmd) startupUpdateCheck(config *config.RuntimeConfig) {
	version := config.Version
	if config.VersionPrerelease != "" {
		version += fmt.Sprintf("-%s", config.VersionPrerelease)
	}
	updateParams := &checkpoint.CheckParams{
		Product: "consul",
		Version: version,
	}
	if !config.DisableAnonymousSignature {
		updateParams.SignatureFile = filepath.Join(config.DataDir, "checkpoint-signature")
	}

	// Schedule a periodic check with expected interval of 24 hours
	checkpoint.CheckInterval(updateParams, 24*time.Hour, c.checkpointResults)

	// Do an immediate check within the next 30 seconds
	go func() {
		time.Sleep(lib.RandomStagger(30 * time.Second))
		c.checkpointResults(checkpoint.Check(updateParams))
	}()
}

// startupJoin is invoked to handle any joins specified to take place at start time
func (c *cmd) startupJoin(agent *agent.Agent, cfg *config.RuntimeConfig) error {
	if len(cfg.StartJoinAddrsLAN) == 0 {
		return nil
	}

	c.logger.Info("Joining cluster")
	n, err := agent.JoinLAN(cfg.StartJoinAddrsLAN)
	if err != nil {
		return err
	}

	c.logger.Info("Join completed. Initial agents synced with", "agent_count", n)
	return nil
}

// startupJoinWan is invoked to handle any joins -wan specified to take place at start time
func (c *cmd) startupJoinWan(agent *agent.Agent, cfg *config.RuntimeConfig) error {
	if len(cfg.StartJoinAddrsWAN) == 0 {
		return nil
	}

	c.logger.Info("Joining wan cluster")
	n, err := agent.JoinWAN(cfg.StartJoinAddrsWAN)
	if err != nil {
		return err
	}

	c.logger.Info("Join wan completed. Initial agents synced with", "agent_count", n)
	return nil
}

func (c *cmd) run(args []string) int {
	ui := &mcli.PrefixedUi{
		OutputPrefix: "==> ",
		InfoPrefix:   "    ",
		ErrorPrefix:  "==> ",
		Ui:           c.ui,
	}

	if err := c.flags.Parse(args); err != nil {
		if !strings.Contains(err.Error(), "help requested") {
			ui.Error(fmt.Sprintf("error parsing flags: %v", err))
		}
		return 1
	}
	if len(c.flags.Args()) > 0 {
		ui.Error(fmt.Sprintf("Unexpected extra arguments: %v", c.flags.Args()))
		return 1
	}

	// FIXME: logs should always go to stderr, but previously they were sent to
	// stdout, so continue to use Stdout for now, and fix this in a future release.
	logGate := &logging.GatedWriter{Writer: c.ui.Stdout()}
	loader := func(source config.Source) (config.LoadResult, error) {
		c.configLoadOpts.DefaultConfig = source
		return config.Load(c.configLoadOpts)
	}
	bd, err := agent.NewBaseDeps(loader, logGate)
	if err != nil {
		ui.Error(err.Error())
		return 1
	}

	c.logger = bd.Logger
	agent, err := agent.New(bd)
	if err != nil {
		ui.Error(err.Error())
		return 1
	}

	config := bd.RuntimeConfig

	// Setup gate to check if we should output CLI information
	cli := GatedUi{
		JSONoutput: config.Logging.LogJSON,
		ui:         ui,
	}

	// Create the agent
	cli.output("Starting Consul agent...")

	segment := config.SegmentName
	if config.ServerMode {
		segment = "<all>"
	}
	cli.info(fmt.Sprintf("       Version: '%s'", c.versionHuman))
	cli.info(fmt.Sprintf("       Node ID: '%s'", config.NodeID))
	cli.info(fmt.Sprintf("     Node name: '%s'", config.NodeName))
	cli.info(fmt.Sprintf("    Datacenter: '%s' (Segment: '%s')", config.Datacenter, segment))
	cli.info(fmt.Sprintf("        Server: %v (Bootstrap: %v)", config.ServerMode, config.Bootstrap))
	cli.info(fmt.Sprintf("   Client Addr: %v (HTTP: %d, HTTPS: %d, gRPC: %d, DNS: %d)", config.ClientAddrs,
		config.HTTPPort, config.HTTPSPort, config.GRPCPort, config.DNSPort))
	cli.info(fmt.Sprintf("  Cluster Addr: %v (LAN: %d, WAN: %d)", config.AdvertiseAddrLAN,
		config.SerfPortLAN, config.SerfPortWAN))
	cli.info(fmt.Sprintf("       Encrypt: Gossip: %v, TLS-Outgoing: %v, TLS-Incoming: %v, Auto-Encrypt-TLS: %t",
		config.EncryptKey != "", config.VerifyOutgoing, config.VerifyIncoming, config.AutoEncryptTLS || config.AutoEncryptAllowTLS))
	// Enable log streaming
	cli.output("")
	cli.output("Log data will now stream in as it occurs:\n")
	logGate.Flush()

	// wait for signal
	signalCh := make(chan os.Signal, 10)
	signal.Notify(signalCh, os.Interrupt, syscall.SIGTERM, syscall.SIGHUP, syscall.SIGPIPE)

	ctx, cancel := context.WithCancel(context.Background())

	go func() {
		for {
			var sig os.Signal
			select {
			case s := <-signalCh:
				sig = s
			case <-ctx.Done():
				return
			}

			switch sig {
			case syscall.SIGPIPE:
				continue

			case syscall.SIGHUP:
				err := fmt.Errorf("cannot reload before agent started")
				c.logger.Error("Caught", "signal", sig, "error", err)

			default:
				c.logger.Info("Caught", "signal", sig)
				cancel()
				return
			}
		}
	}()

	err = agent.Start(ctx)
	signal.Stop(signalCh)
	cancel()

	if err != nil {
		c.logger.Error("Error starting agent", "error", err)
		return 1
	}

	// shutdown agent before endpoints
	defer agent.ShutdownEndpoints()
	defer agent.ShutdownAgent()

	if !config.DisableUpdateCheck && !config.DevMode {
		c.startupUpdateCheck(config)
	}

	if err := c.startupJoin(agent, config); err != nil {
		c.logger.Error(err.Error())
		return 1
	}

	if err := c.startupJoinWan(agent, config); err != nil {
		c.logger.Error(err.Error())
		return 1
	}

	// Let the agent know we've finished registration
	agent.StartSync()

	cli.output("Consul agent running!")

	// wait for signal
	signalCh = make(chan os.Signal, 10)
	signal.Notify(signalCh, os.Interrupt, syscall.SIGTERM, syscall.SIGHUP, syscall.SIGPIPE)

	for {
		var sig os.Signal
		select {
		case s := <-signalCh:
			sig = s
		case <-service_os.Shutdown_Channel():
			sig = os.Interrupt
		case err := <-agent.RetryJoinCh():
			c.logger.Error("Retry join failed", "error", err)
			return 1
		case <-agent.Failed():
			// The deferred Shutdown method will log the appropriate error
			return 1
		case <-agent.ShutdownCh():
			// agent is already down!
			return 0
		}

		switch sig {
		case syscall.SIGPIPE:
			continue

		case syscall.SIGHUP:
			c.logger.Info("Caught", "signal", sig)

			err := agent.ReloadConfig()
			if err != nil {
				c.logger.Error("Reload config failed", "error", err)
			}
			config = agent.GetConfig()
		default:
			c.logger.Info("Caught", "signal", sig)

			graceful := (sig == os.Interrupt && !(config.SkipLeaveOnInt)) || (sig == syscall.SIGTERM && (config.LeaveOnTerm))
			if !graceful {
				c.logger.Info("Graceful shutdown disabled. Exiting")
				return 1
			}

			c.logger.Info("Gracefully shutting down agent...")
			gracefulCh := make(chan struct{})
			go func() {
				if err := agent.Leave(); err != nil {
					c.logger.Error("Error on leave", "error", err)
					return
				}
				close(gracefulCh)
			}()

			gracefulTimeout := 15 * time.Second
			select {
			case <-signalCh:
				c.logger.Info("Caught second signal, Exiting", "signal", sig)
				return 1
			case <-time.After(gracefulTimeout):
				c.logger.Info("Timeout on graceful leave. Exiting")
				return 1
			case <-gracefulCh:
				c.logger.Info("Graceful exit completed")
				return 0
			}
		}
	}
}

type GatedUi struct {
	JSONoutput bool
	ui         mcli.Ui
}

func (g *GatedUi) output(s string) {
	if !g.JSONoutput {
		g.ui.Output(s)
	}
}

func (g *GatedUi) info(s string) {
	if !g.JSONoutput {
		g.ui.Info(s)
	}
}

func (c *cmd) Synopsis() string {
	return synopsis
}

func (c *cmd) Help() string {
	return c.help
}

const synopsis = "Runs a Consul agent"
const help = `
Usage: consul agent [options]

  Starts the Consul agent and runs until an interrupt is received. The
  agent represents a single node in a cluster.
`
<commit_msg>cmd: move agent running message to logs<commit_after>package agent

import (
	"context"
	"flag"
	"fmt"
	"os"
	"os/signal"
	"path/filepath"
	"strings"
	"syscall"
	"time"

	"github.com/hashicorp/go-checkpoint"
	"github.com/hashicorp/go-hclog"
	mcli "github.com/mitchellh/cli"

	"github.com/hashicorp/consul/agent"
	"github.com/hashicorp/consul/agent/config"
	"github.com/hashicorp/consul/command/cli"
	"github.com/hashicorp/consul/command/flags"
	"github.com/hashicorp/consul/lib"
	"github.com/hashicorp/consul/logging"
	"github.com/hashicorp/consul/service_os"
	consulversion "github.com/hashicorp/consul/version"
)

func New(ui cli.Ui) *cmd {
	c := &cmd{
		ui:                ui,
		revision:          consulversion.GitCommit,
		version:           consulversion.Version,
		versionPrerelease: consulversion.VersionPrerelease,
		versionHuman:      consulversion.GetHumanVersion(),
		flags:             flag.NewFlagSet("", flag.ContinueOnError),
	}
	config.AddFlags(c.flags, &c.configLoadOpts)
	c.help = flags.Usage(help, c.flags)
	return c
}

// AgentCommand is a Command implementation that runs a Consul agent.
// The command will not end unless a shutdown message is sent on the
// ShutdownCh. If two messages are sent on the ShutdownCh it will forcibly
// exit.
type cmd struct {
	ui                cli.Ui
	flags             *flag.FlagSet
	http              *flags.HTTPFlags
	help              string
	revision          string
	version           string
	versionPrerelease string
	versionHuman      string
	configLoadOpts    config.LoadOpts
	logger            hclog.InterceptLogger
}

func (c *cmd) Run(args []string) int {
	code := c.run(args)
	if c.logger != nil {
		c.logger.Info("Exit code", "code", code)
	}
	return code
}

// checkpointResults is used to handler periodic results from our update checker
func (c *cmd) checkpointResults(results *checkpoint.CheckResponse, err error) {
	if err != nil {
		c.logger.Error("Failed to check for updates", "error", err)
		return
	}
	if results.Outdated {
		c.logger.Info("Newer Consul version available", "new_version", results.CurrentVersion, "current_version", c.version)
	}
	for _, alert := range results.Alerts {
		switch alert.Level {
		case "info":
			c.logger.Info("Bulletin", "alert_level", alert.Level, "alert_message", alert.Message, "alert_URL", alert.URL)
		default:
			c.logger.Error("Bulletin", "alert_level", alert.Level, "alert_message", alert.Message, "alert_URL", alert.URL)
		}
	}
}

func (c *cmd) startupUpdateCheck(config *config.RuntimeConfig) {
	version := config.Version
	if config.VersionPrerelease != "" {
		version += fmt.Sprintf("-%s", config.VersionPrerelease)
	}
	updateParams := &checkpoint.CheckParams{
		Product: "consul",
		Version: version,
	}
	if !config.DisableAnonymousSignature {
		updateParams.SignatureFile = filepath.Join(config.DataDir, "checkpoint-signature")
	}

	// Schedule a periodic check with expected interval of 24 hours
	checkpoint.CheckInterval(updateParams, 24*time.Hour, c.checkpointResults)

	// Do an immediate check within the next 30 seconds
	go func() {
		time.Sleep(lib.RandomStagger(30 * time.Second))
		c.checkpointResults(checkpoint.Check(updateParams))
	}()
}

// startupJoin is invoked to handle any joins specified to take place at start time
func (c *cmd) startupJoin(agent *agent.Agent, cfg *config.RuntimeConfig) error {
	if len(cfg.StartJoinAddrsLAN) == 0 {
		return nil
	}

	c.logger.Info("Joining cluster")
	n, err := agent.JoinLAN(cfg.StartJoinAddrsLAN)
	if err != nil {
		return err
	}

	c.logger.Info("Join completed. Initial agents synced with", "agent_count", n)
	return nil
}

// startupJoinWan is invoked to handle any joins -wan specified to take place at start time
func (c *cmd) startupJoinWan(agent *agent.Agent, cfg *config.RuntimeConfig) error {
	if len(cfg.StartJoinAddrsWAN) == 0 {
		return nil
	}

	c.logger.Info("Joining wan cluster")
	n, err := agent.JoinWAN(cfg.StartJoinAddrsWAN)
	if err != nil {
		return err
	}

	c.logger.Info("Join wan completed. Initial agents synced with", "agent_count", n)
	return nil
}

func (c *cmd) run(args []string) int {
	ui := &mcli.PrefixedUi{
		OutputPrefix: "==> ",
		InfoPrefix:   "    ",
		ErrorPrefix:  "==> ",
		Ui:           c.ui,
	}

	if err := c.flags.Parse(args); err != nil {
		if !strings.Contains(err.Error(), "help requested") {
			ui.Error(fmt.Sprintf("error parsing flags: %v", err))
		}
		return 1
	}
	if len(c.flags.Args()) > 0 {
		ui.Error(fmt.Sprintf("Unexpected extra arguments: %v", c.flags.Args()))
		return 1
	}

	// FIXME: logs should always go to stderr, but previously they were sent to
	// stdout, so continue to use Stdout for now, and fix this in a future release.
	logGate := &logging.GatedWriter{Writer: c.ui.Stdout()}
	loader := func(source config.Source) (config.LoadResult, error) {
		c.configLoadOpts.DefaultConfig = source
		return config.Load(c.configLoadOpts)
	}
	bd, err := agent.NewBaseDeps(loader, logGate)
	if err != nil {
		ui.Error(err.Error())
		return 1
	}

	c.logger = bd.Logger
	agent, err := agent.New(bd)
	if err != nil {
		ui.Error(err.Error())
		return 1
	}

	config := bd.RuntimeConfig

	// Setup gate to check if we should output CLI information
	cli := GatedUi{
		JSONoutput: config.Logging.LogJSON,
		ui:         ui,
	}

	// Create the agent
	cli.output("Starting Consul agent...")

	segment := config.SegmentName
	if config.ServerMode {
		segment = "<all>"
	}
	cli.info(fmt.Sprintf("       Version: '%s'", c.versionHuman))
	cli.info(fmt.Sprintf("       Node ID: '%s'", config.NodeID))
	cli.info(fmt.Sprintf("     Node name: '%s'", config.NodeName))
	cli.info(fmt.Sprintf("    Datacenter: '%s' (Segment: '%s')", config.Datacenter, segment))
	cli.info(fmt.Sprintf("        Server: %v (Bootstrap: %v)", config.ServerMode, config.Bootstrap))
	cli.info(fmt.Sprintf("   Client Addr: %v (HTTP: %d, HTTPS: %d, gRPC: %d, DNS: %d)", config.ClientAddrs,
		config.HTTPPort, config.HTTPSPort, config.GRPCPort, config.DNSPort))
	cli.info(fmt.Sprintf("  Cluster Addr: %v (LAN: %d, WAN: %d)", config.AdvertiseAddrLAN,
		config.SerfPortLAN, config.SerfPortWAN))
	cli.info(fmt.Sprintf("       Encrypt: Gossip: %v, TLS-Outgoing: %v, TLS-Incoming: %v, Auto-Encrypt-TLS: %t",
		config.EncryptKey != "", config.VerifyOutgoing, config.VerifyIncoming, config.AutoEncryptTLS || config.AutoEncryptAllowTLS))
	// Enable log streaming
	cli.output("")
	cli.output("Log data will now stream in as it occurs:\n")
	logGate.Flush()

	// wait for signal
	signalCh := make(chan os.Signal, 10)
	signal.Notify(signalCh, os.Interrupt, syscall.SIGTERM, syscall.SIGHUP, syscall.SIGPIPE)

	ctx, cancel := context.WithCancel(context.Background())

	go func() {
		for {
			var sig os.Signal
			select {
			case s := <-signalCh:
				sig = s
			case <-ctx.Done():
				return
			}

			switch sig {
			case syscall.SIGPIPE:
				continue

			case syscall.SIGHUP:
				err := fmt.Errorf("cannot reload before agent started")
				c.logger.Error("Caught", "signal", sig, "error", err)

			default:
				c.logger.Info("Caught", "signal", sig)
				cancel()
				return
			}
		}
	}()

	err = agent.Start(ctx)
	signal.Stop(signalCh)
	cancel()

	if err != nil {
		c.logger.Error("Error starting agent", "error", err)
		return 1
	}

	// shutdown agent before endpoints
	defer agent.ShutdownEndpoints()
	defer agent.ShutdownAgent()

	if !config.DisableUpdateCheck && !config.DevMode {
		c.startupUpdateCheck(config)
	}

	if err := c.startupJoin(agent, config); err != nil {
		c.logger.Error(err.Error())
		return 1
	}

	if err := c.startupJoinWan(agent, config); err != nil {
		c.logger.Error(err.Error())
		return 1
	}

	// Let the agent know we've finished registration
	agent.StartSync()

	c.logger.Info("Consul agent running!")

	// wait for signal
	signalCh = make(chan os.Signal, 10)
	signal.Notify(signalCh, os.Interrupt, syscall.SIGTERM, syscall.SIGHUP, syscall.SIGPIPE)

	for {
		var sig os.Signal
		select {
		case s := <-signalCh:
			sig = s
		case <-service_os.Shutdown_Channel():
			sig = os.Interrupt
		case err := <-agent.RetryJoinCh():
			c.logger.Error("Retry join failed", "error", err)
			return 1
		case <-agent.Failed():
			// The deferred Shutdown method will log the appropriate error
			return 1
		case <-agent.ShutdownCh():
			// agent is already down!
			return 0
		}

		switch sig {
		case syscall.SIGPIPE:
			continue

		case syscall.SIGHUP:
			c.logger.Info("Caught", "signal", sig)

			err := agent.ReloadConfig()
			if err != nil {
				c.logger.Error("Reload config failed", "error", err)
			}
			config = agent.GetConfig()
		default:
			c.logger.Info("Caught", "signal", sig)

			graceful := (sig == os.Interrupt && !(config.SkipLeaveOnInt)) || (sig == syscall.SIGTERM && (config.LeaveOnTerm))
			if !graceful {
				c.logger.Info("Graceful shutdown disabled. Exiting")
				return 1
			}

			c.logger.Info("Gracefully shutting down agent...")
			gracefulCh := make(chan struct{})
			go func() {
				if err := agent.Leave(); err != nil {
					c.logger.Error("Error on leave", "error", err)
					return
				}
				close(gracefulCh)
			}()

			gracefulTimeout := 15 * time.Second
			select {
			case <-signalCh:
				c.logger.Info("Caught second signal, Exiting", "signal", sig)
				return 1
			case <-time.After(gracefulTimeout):
				c.logger.Info("Timeout on graceful leave. Exiting")
				return 1
			case <-gracefulCh:
				c.logger.Info("Graceful exit completed")
				return 0
			}
		}
	}
}

type GatedUi struct {
	JSONoutput bool
	ui         mcli.Ui
}

func (g *GatedUi) output(s string) {
	if !g.JSONoutput {
		g.ui.Output(s)
	}
}

func (g *GatedUi) info(s string) {
	if !g.JSONoutput {
		g.ui.Info(s)
	}
}

func (c *cmd) Synopsis() string {
	return synopsis
}

func (c *cmd) Help() string {
	return c.help
}

const synopsis = "Runs a Consul agent"
const help = `
Usage: consul agent [options]

  Starts the Consul agent and runs until an interrupt is received. The
  agent represents a single node in a cluster.
`
<|endoftext|>


<commit_before>package command

import (
	"fmt"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/dustin/go-humanize"
	"github.com/mitchellh/colorstring"

	"github.com/hashicorp/nomad/api"
)

type NodeStatusCommand struct {
	Meta
	color *colorstring.Colorize
}

func (c *NodeStatusCommand) Help() string {
	helpText := `
Usage: nomad node-status [options] <node>

  Display status information about a given node. The list of nodes
  returned includes only nodes which jobs may be scheduled to, and
  includes status and other high-level information.

  If a node ID is passed, information for that specific node will
  be displayed. If no node ID's are passed, then a short-hand
  list of all nodes will be displayed. The -self flag is useful to
  quickly access the status of the local node.

General Options:

  ` + generalOptionsUsage() + `

Node Status Options:

  -short
    Display short output. Used only when a single node is being
    queried, and drops verbose output about node allocations.

  -verbose
    Display full information.

  -stats
    Display detailed resource usage statistics

  -self
    Query the status of the local node.

  -allocs
    Display a count of running allocations for each node.
`
	return strings.TrimSpace(helpText)
}

func (c *NodeStatusCommand) Synopsis() string {
	return "Display status information about nodes"
}

func (c *NodeStatusCommand) Run(args []string) int {
	var short, verbose, list_allocs, self, stats bool
	var hostStats *api.HostStats

	flags := c.Meta.FlagSet("node-status", FlagSetClient)
	flags.Usage = func() { c.Ui.Output(c.Help()) }
	flags.BoolVar(&short, "short", false, "")
	flags.BoolVar(&verbose, "verbose", false, "")
	flags.BoolVar(&list_allocs, "allocs", false, "")
	flags.BoolVar(&self, "self", false, "")
	flags.BoolVar(&stats, "stats", false, "")

	if err := flags.Parse(args); err != nil {
		return 1
	}

	// Check that we got either a single node or none
	args = flags.Args()
	if len(args) > 1 {
		c.Ui.Error(c.Help())
		return 1
	}

	// Truncate the id unless full length is requested
	length := shortId
	if verbose {
		length = fullId
	}

	// Get the HTTP client
	client, err := c.Meta.Client()
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err))
		return 1
	}

	// Use list mode if no node name was provided
	if len(args) == 0 && !self {
		// Query the node info
		nodes, _, err := client.Nodes().List(nil)
		if err != nil {
			c.Ui.Error(fmt.Sprintf("Error querying node status: %s", err))
			return 1
		}

		// Return nothing if no nodes found
		if len(nodes) == 0 {
			return 0
		}

		// Format the nodes list
		out := make([]string, len(nodes)+1)
		if list_allocs {
			out[0] = "ID|DC|Name|Class|Drain|Status|Running Allocs"
		} else {
			out[0] = "ID|DC|Name|Class|Drain|Status"
		}
		for i, node := range nodes {
			if list_allocs {
				numAllocs, err := getRunningAllocs(client, node.ID)
				if err != nil {
					c.Ui.Error(fmt.Sprintf("Error querying node allocations: %s", err))
					return 1
				}
				out[i+1] = fmt.Sprintf("%s|%s|%s|%s|%v|%s|%v",
					limit(node.ID, length),
					node.Datacenter,
					node.Name,
					node.NodeClass,
					node.Drain,
					node.Status,
					len(numAllocs))
			} else {
				out[i+1] = fmt.Sprintf("%s|%s|%s|%s|%v|%s",
					limit(node.ID, length),
					node.Datacenter,
					node.Name,
					node.NodeClass,
					node.Drain,
					node.Status)
			}
		}

		// Dump the output
		c.Ui.Output(formatList(out))
		return 0
	}

	// Query the specific node
	nodeID := ""
	if !self {
		nodeID = args[0]
	} else {
		var err error
		if nodeID, err = getLocalNodeID(client); err != nil {
			c.Ui.Error(err.Error())
			return 1
		}
	}
	if len(nodeID) == 1 {
		c.Ui.Error(fmt.Sprintf("Identifier must contain at least two characters."))
		return 1
	}
	if len(nodeID)%2 == 1 {
		// Identifiers must be of even length, so we strip off the last byte
		// to provide a consistent user experience.
		nodeID = nodeID[:len(nodeID)-1]
	}

	nodes, _, err := client.Nodes().PrefixList(nodeID)
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error querying node info: %s", err))
		return 1
	}
	// Return error if no nodes are found
	if len(nodes) == 0 {
		c.Ui.Error(fmt.Sprintf("No node(s) with prefix %q found", nodeID))
		return 1
	}
	if len(nodes) > 1 {
		// Format the nodes list that matches the prefix so that the user
		// can create a more specific request
		out := make([]string, len(nodes)+1)
		out[0] = "ID|DC|Name|Class|Drain|Status"
		for i, node := range nodes {
			out[i+1] = fmt.Sprintf("%s|%s|%s|%s|%v|%s",
				limit(node.ID, length),
				node.Datacenter,
				node.Name,
				node.NodeClass,
				node.Drain,
				node.Status)
		}
		// Dump the output
		c.Ui.Output(fmt.Sprintf("Prefix matched multiple nodes\n\n%s", formatList(out)))
		return 0
	}
	// Prefix lookup matched a single node
	node, _, err := client.Nodes().Info(nodes[0].ID, nil)
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error querying node info: %s", err))
		return 1
	}

	if hostStats, err = client.Nodes().Stats(node.ID, nil); err != nil {
		c.Ui.Error(fmt.Sprintf("error fetching node resource utilization stats: %#v", err))
	}

	// Format the output
	basic := []string{
		fmt.Sprintf("[bold]Node ID[reset]|%s", limit(node.ID, length)),
		fmt.Sprintf("Name|%s", node.Name),
		fmt.Sprintf("Class|%s", node.NodeClass),
		fmt.Sprintf("DC|%s", node.Datacenter),
		fmt.Sprintf("Drain|%v", node.Drain),
		fmt.Sprintf("Status|%s", node.Status),
	}
	if hostStats != nil {
		uptime := time.Duration(hostStats.Uptime * uint64(time.Second))
		basic = append(basic, fmt.Sprintf("Uptime|%s", uptime.String()))
	}
	c.Ui.Output(c.Colorize().Color(formatKV(basic)))

	if !short {
		resources, err := getResources(client, node)
		if err != nil {
			c.Ui.Error(fmt.Sprintf("Error querying node resources: %s", err))
			return 1
		}
		c.Ui.Output(c.Colorize().Color("\n[bold]==> Resource Utilization (Actual)[reset]"))
		c.Ui.Output(formatList(resources))
		if hostStats != nil && stats {
			c.Ui.Output(c.Colorize().Color("\n===> [bold]Detailed CPU Stats[reset]"))
			c.printCpuStats(hostStats)
			c.Ui.Output(c.Colorize().Color("\n===> [bold]Detailed Memory Stats[reset]"))
			c.printMemoryStats(hostStats)
			c.Ui.Output(c.Colorize().Color("\n===> [bold]Detailed Disk Stats[reset]"))
			c.printDiskStats(hostStats)
		}

		allocs, err := getAllocs(client, node, length)
		if err != nil {
			c.Ui.Error(fmt.Sprintf("Error querying node allocations: %s", err))
			return 1
		}

		if len(allocs) > 1 {
			c.Ui.Output("\n==> Allocations")
			c.Ui.Output(formatList(allocs))
		}
	}

	if verbose {
		// Print the attributes
		keys := make([]string, len(node.Attributes))
		for k := range node.Attributes {
			keys = append(keys, k)
		}
		sort.Strings(keys)

		var attributes []string
		for _, k := range keys {
			if k != "" {
				attributes = append(attributes, fmt.Sprintf("%s|%s", k, node.Attributes[k]))
			}
		}
		c.Ui.Output("\n==> Attributes")
		c.Ui.Output(formatKV(attributes))
	}

	return 0
}

func (c *NodeStatusCommand) printCpuStats(hostStats *api.HostStats) {
	for _, cpuStat := range hostStats.CPU {
		cpuStatsAttr := make([]string, 4)
		cpuStatsAttr[0] = fmt.Sprintf("CPU|%v", cpuStat.CPU)
		cpuStatsAttr[1] = fmt.Sprintf("User|%v", formatFloat64(cpuStat.User))
		cpuStatsAttr[2] = fmt.Sprintf("System|%v", formatFloat64(cpuStat.System))
		cpuStatsAttr[3] = fmt.Sprintf("Idle|%v", formatFloat64(cpuStat.Idle))
		c.Ui.Output(formatKV(cpuStatsAttr))
		c.Ui.Output("")
	}
}

func (c *NodeStatusCommand) printMemoryStats(hostStats *api.HostStats) {
	memoryStat := hostStats.Memory
	memStatsAttr := make([]string, 4)
	memStatsAttr[0] = fmt.Sprintf("Total|%v", humanize.Bytes(memoryStat.Total))
	memStatsAttr[1] = fmt.Sprintf("Available|%v", humanize.Bytes(memoryStat.Available))
	memStatsAttr[2] = fmt.Sprintf("Used|%v", humanize.Bytes(memoryStat.Used))
	memStatsAttr[3] = fmt.Sprintf("Free|%v", humanize.Bytes(memoryStat.Free))
	c.Ui.Output(formatKV(memStatsAttr))
}

func (c *NodeStatusCommand) printDiskStats(hostStats *api.HostStats) {
	for _, diskStat := range hostStats.DiskStats {
		diskStatsAttr := make([]string, 6)
		diskStatsAttr[0] = fmt.Sprintf("Device|%s", diskStat.Device)
		diskStatsAttr[1] = fmt.Sprintf("MountPoint|%s", diskStat.Mountpoint)
		diskStatsAttr[2] = fmt.Sprintf("Size|%s", humanize.Bytes(diskStat.Size))
		diskStatsAttr[3] = fmt.Sprintf("Used|%s", humanize.Bytes(diskStat.Used))
		diskStatsAttr[4] = fmt.Sprintf("Available|%s", humanize.Bytes(diskStat.Available))
		diskStatsAttr[5] = fmt.Sprintf("Used Percent|%s", formatFloat64(diskStat.UsedPercent))
		c.Ui.Output(formatKV(diskStatsAttr))
		c.Ui.Output("")
	}
}

// getRunningAllocs returns a slice of allocation id's running on the node
func getRunningAllocs(client *api.Client, nodeID string) ([]*api.Allocation, error) {
	var allocs []*api.Allocation

	// Query the node allocations
	nodeAllocs, _, err := client.Nodes().Allocations(nodeID, nil)
	// Filter list to only running allocations
	for _, alloc := range nodeAllocs {
		if alloc.ClientStatus == "running" {
			allocs = append(allocs, alloc)
		}
	}
	return allocs, err
}

// getAllocs returns information about every running allocation on the node
func getAllocs(client *api.Client, node *api.Node, length int) ([]string, error) {
	var allocs []string
	// Query the node allocations
	nodeAllocs, _, err := client.Nodes().Allocations(node.ID, nil)
	// Format the allocations
	allocs = make([]string, len(nodeAllocs)+1)
	allocs[0] = "ID|Eval ID|Job ID|Task Group|Desired Status|Client Status"
	for i, alloc := range nodeAllocs {
		allocs[i+1] = fmt.Sprintf("%s|%s|%s|%s|%s|%s",
			limit(alloc.ID, length),
			limit(alloc.EvalID, length),
			alloc.JobID,
			alloc.TaskGroup,
			alloc.DesiredStatus,
			alloc.ClientStatus)
	}
	return allocs, err
}

// getResources returns the resource usage of the node.
func getResources(client *api.Client, node *api.Node) ([]string, error) {
	var resources []string
	var cpu, mem, disk, iops int
	var totalCpu, totalMem, totalDisk, totalIops int

	// Compute the total
	r := node.Resources
	res := node.Reserved
	if res == nil {
		res = &api.Resources{}
	}
	totalCpu = r.CPU - res.CPU
	totalMem = r.MemoryMB - res.MemoryMB
	totalDisk = r.DiskMB - res.DiskMB
	totalIops = r.IOPS - res.IOPS

	// Get list of running allocations on the node
	runningAllocs, err := getRunningAllocs(client, node.ID)

	// Get Resources
	for _, alloc := range runningAllocs {
		cpu += alloc.Resources.CPU
		mem += alloc.Resources.MemoryMB
		disk += alloc.Resources.DiskMB
		iops += alloc.Resources.IOPS
	}

	resources = make([]string, 2)
	resources[0] = "CPU|Memory MB|Disk MB|IOPS"
	resources[1] = fmt.Sprintf("%v/%v|%v/%v|%v/%v|%v/%v",
		cpu,
		totalCpu,
		mem,
		totalMem,
		disk,
		totalDisk,
		iops,
		totalIops)

	return resources, err
}

func formatFloat64(val float64) string {
	return strconv.FormatFloat(val, 'f', 2, 64)
}
<commit_msg>Showing actual resource usage<commit_after>package command

import (
	"fmt"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/dustin/go-humanize"
	"github.com/mitchellh/colorstring"

	"github.com/hashicorp/nomad/api"
)

type NodeStatusCommand struct {
	Meta
	color *colorstring.Colorize
}

func (c *NodeStatusCommand) Help() string {
	helpText := `
Usage: nomad node-status [options] <node>

  Display status information about a given node. The list of nodes
  returned includes only nodes which jobs may be scheduled to, and
  includes status and other high-level information.

  If a node ID is passed, information for that specific node will
  be displayed. If no node ID's are passed, then a short-hand
  list of all nodes will be displayed. The -self flag is useful to
  quickly access the status of the local node.

General Options:

  ` + generalOptionsUsage() + `

Node Status Options:

  -short
    Display short output. Used only when a single node is being
    queried, and drops verbose output about node allocations.

  -verbose
    Display full information.

  -stats
    Display detailed resource usage statistics

  -self
    Query the status of the local node.

  -allocs
    Display a count of running allocations for each node.
`
	return strings.TrimSpace(helpText)
}

func (c *NodeStatusCommand) Synopsis() string {
	return "Display status information about nodes"
}

func (c *NodeStatusCommand) Run(args []string) int {
	var short, verbose, list_allocs, self, stats bool
	var hostStats *api.HostStats

	flags := c.Meta.FlagSet("node-status", FlagSetClient)
	flags.Usage = func() { c.Ui.Output(c.Help()) }
	flags.BoolVar(&short, "short", false, "")
	flags.BoolVar(&verbose, "verbose", false, "")
	flags.BoolVar(&list_allocs, "allocs", false, "")
	flags.BoolVar(&self, "self", false, "")
	flags.BoolVar(&stats, "stats", false, "")

	if err := flags.Parse(args); err != nil {
		return 1
	}

	// Check that we got either a single node or none
	args = flags.Args()
	if len(args) > 1 {
		c.Ui.Error(c.Help())
		return 1
	}

	// Truncate the id unless full length is requested
	length := shortId
	if verbose {
		length = fullId
	}

	// Get the HTTP client
	client, err := c.Meta.Client()
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err))
		return 1
	}

	// Use list mode if no node name was provided
	if len(args) == 0 && !self {
		// Query the node info
		nodes, _, err := client.Nodes().List(nil)
		if err != nil {
			c.Ui.Error(fmt.Sprintf("Error querying node status: %s", err))
			return 1
		}

		// Return nothing if no nodes found
		if len(nodes) == 0 {
			return 0
		}

		// Format the nodes list
		out := make([]string, len(nodes)+1)
		if list_allocs {
			out[0] = "ID|DC|Name|Class|Drain|Status|Running Allocs"
		} else {
			out[0] = "ID|DC|Name|Class|Drain|Status"
		}
		for i, node := range nodes {
			if list_allocs {
				numAllocs, err := getRunningAllocs(client, node.ID)
				if err != nil {
					c.Ui.Error(fmt.Sprintf("Error querying node allocations: %s", err))
					return 1
				}
				out[i+1] = fmt.Sprintf("%s|%s|%s|%s|%v|%s|%v",
					limit(node.ID, length),
					node.Datacenter,
					node.Name,
					node.NodeClass,
					node.Drain,
					node.Status,
					len(numAllocs))
			} else {
				out[i+1] = fmt.Sprintf("%s|%s|%s|%s|%v|%s",
					limit(node.ID, length),
					node.Datacenter,
					node.Name,
					node.NodeClass,
					node.Drain,
					node.Status)
			}
		}

		// Dump the output
		c.Ui.Output(formatList(out))
		return 0
	}

	// Query the specific node
	nodeID := ""
	if !self {
		nodeID = args[0]
	} else {
		var err error
		if nodeID, err = getLocalNodeID(client); err != nil {
			c.Ui.Error(err.Error())
			return 1
		}
	}
	if len(nodeID) == 1 {
		c.Ui.Error(fmt.Sprintf("Identifier must contain at least two characters."))
		return 1
	}
	if len(nodeID)%2 == 1 {
		// Identifiers must be of even length, so we strip off the last byte
		// to provide a consistent user experience.
		nodeID = nodeID[:len(nodeID)-1]
	}

	nodes, _, err := client.Nodes().PrefixList(nodeID)
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error querying node info: %s", err))
		return 1
	}
	// Return error if no nodes are found
	if len(nodes) == 0 {
		c.Ui.Error(fmt.Sprintf("No node(s) with prefix %q found", nodeID))
		return 1
	}
	if len(nodes) > 1 {
		// Format the nodes list that matches the prefix so that the user
		// can create a more specific request
		out := make([]string, len(nodes)+1)
		out[0] = "ID|DC|Name|Class|Drain|Status"
		for i, node := range nodes {
			out[i+1] = fmt.Sprintf("%s|%s|%s|%s|%v|%s",
				limit(node.ID, length),
				node.Datacenter,
				node.Name,
				node.NodeClass,
				node.Drain,
				node.Status)
		}
		// Dump the output
		c.Ui.Output(fmt.Sprintf("Prefix matched multiple nodes\n\n%s", formatList(out)))
		return 0
	}
	// Prefix lookup matched a single node
	node, _, err := client.Nodes().Info(nodes[0].ID, nil)
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error querying node info: %s", err))
		return 1
	}

	if hostStats, err = client.Nodes().Stats(node.ID, nil); err != nil {
		c.Ui.Error(fmt.Sprintf("error fetching node resource utilization stats: %#v", err))
	}

	// Format the output
	basic := []string{
		fmt.Sprintf("[bold]Node ID[reset]|%s", limit(node.ID, length)),
		fmt.Sprintf("Name|%s", node.Name),
		fmt.Sprintf("Class|%s", node.NodeClass),
		fmt.Sprintf("DC|%s", node.Datacenter),
		fmt.Sprintf("Drain|%v", node.Drain),
		fmt.Sprintf("Status|%s", node.Status),
	}
	if hostStats != nil {
		uptime := time.Duration(hostStats.Uptime * uint64(time.Second))
		basic = append(basic, fmt.Sprintf("Uptime|%s", uptime.String()))
	}
	c.Ui.Output(c.Colorize().Color(formatKV(basic)))

	if !short {
		allocatedResources, err := getAllocatedResources(client, node)
		if err != nil {
			c.Ui.Error(fmt.Sprintf("Error querying node resources: %s", err))
			return 1
		}
		c.Ui.Output(c.Colorize().Color("\n[bold]==> Resource Utilization (Allocated)[reset]"))
		c.Ui.Output(formatList(allocatedResources))

		actualResources, err := getActualResources(hostStats, node)
		if err == nil {
			c.Ui.Output(c.Colorize().Color("\n[bold]==> Resource Utilization (Actual)[reset]"))
			c.Ui.Output(formatList(actualResources))
		}

		if hostStats != nil && stats {
			c.Ui.Output(c.Colorize().Color("\n===> [bold]Detailed CPU Stats[reset]"))
			c.printCpuStats(hostStats)
			c.Ui.Output(c.Colorize().Color("\n===> [bold]Detailed Memory Stats[reset]"))
			c.printMemoryStats(hostStats)
			c.Ui.Output(c.Colorize().Color("\n===> [bold]Detailed Disk Stats[reset]"))
			c.printDiskStats(hostStats)
		}

		allocs, err := getAllocs(client, node, length)
		if err != nil {
			c.Ui.Error(fmt.Sprintf("Error querying node allocations: %s", err))
			return 1
		}

		if len(allocs) > 1 {
			c.Ui.Output("\n==> Allocations")
			c.Ui.Output(formatList(allocs))
		}
	}

	if verbose {
		// Print the attributes
		keys := make([]string, len(node.Attributes))
		for k := range node.Attributes {
			keys = append(keys, k)
		}
		sort.Strings(keys)

		var attributes []string
		for _, k := range keys {
			if k != "" {
				attributes = append(attributes, fmt.Sprintf("%s|%s", k, node.Attributes[k]))
			}
		}
		c.Ui.Output("\n==> Attributes")
		c.Ui.Output(formatKV(attributes))
	}

	return 0
}

func (c *NodeStatusCommand) printCpuStats(hostStats *api.HostStats) {
	for _, cpuStat := range hostStats.CPU {
		cpuStatsAttr := make([]string, 4)
		cpuStatsAttr[0] = fmt.Sprintf("CPU|%v", cpuStat.CPU)
		cpuStatsAttr[1] = fmt.Sprintf("User|%v", formatFloat64(cpuStat.User))
		cpuStatsAttr[2] = fmt.Sprintf("System|%v", formatFloat64(cpuStat.System))
		cpuStatsAttr[3] = fmt.Sprintf("Idle|%v", formatFloat64(cpuStat.Idle))
		c.Ui.Output(formatKV(cpuStatsAttr))
		c.Ui.Output("")
	}
}

func (c *NodeStatusCommand) printMemoryStats(hostStats *api.HostStats) {
	memoryStat := hostStats.Memory
	memStatsAttr := make([]string, 4)
	memStatsAttr[0] = fmt.Sprintf("Total|%v", humanize.Bytes(memoryStat.Total))
	memStatsAttr[1] = fmt.Sprintf("Available|%v", humanize.Bytes(memoryStat.Available))
	memStatsAttr[2] = fmt.Sprintf("Used|%v", humanize.Bytes(memoryStat.Used))
	memStatsAttr[3] = fmt.Sprintf("Free|%v", humanize.Bytes(memoryStat.Free))
	c.Ui.Output(formatKV(memStatsAttr))
}

func (c *NodeStatusCommand) printDiskStats(hostStats *api.HostStats) {
	for _, diskStat := range hostStats.DiskStats {
		diskStatsAttr := make([]string, 6)
		diskStatsAttr[0] = fmt.Sprintf("Device|%s", diskStat.Device)
		diskStatsAttr[1] = fmt.Sprintf("MountPoint|%s", diskStat.Mountpoint)
		diskStatsAttr[2] = fmt.Sprintf("Size|%s", humanize.Bytes(diskStat.Size))
		diskStatsAttr[3] = fmt.Sprintf("Used|%s", humanize.Bytes(diskStat.Used))
		diskStatsAttr[4] = fmt.Sprintf("Available|%s", humanize.Bytes(diskStat.Available))
		diskStatsAttr[5] = fmt.Sprintf("Used Percent|%s", formatFloat64(diskStat.UsedPercent))
		c.Ui.Output(formatKV(diskStatsAttr))
		c.Ui.Output("")
	}
}

// getRunningAllocs returns a slice of allocation id's running on the node
func getRunningAllocs(client *api.Client, nodeID string) ([]*api.Allocation, error) {
	var allocs []*api.Allocation

	// Query the node allocations
	nodeAllocs, _, err := client.Nodes().Allocations(nodeID, nil)
	// Filter list to only running allocations
	for _, alloc := range nodeAllocs {
		if alloc.ClientStatus == "running" {
			allocs = append(allocs, alloc)
		}
	}
	return allocs, err
}

// getAllocs returns information about every running allocation on the node
func getAllocs(client *api.Client, node *api.Node, length int) ([]string, error) {
	var allocs []string
	// Query the node allocations
	nodeAllocs, _, err := client.Nodes().Allocations(node.ID, nil)
	// Format the allocations
	allocs = make([]string, len(nodeAllocs)+1)
	allocs[0] = "ID|Eval ID|Job ID|Task Group|Desired Status|Client Status"
	for i, alloc := range nodeAllocs {
		allocs[i+1] = fmt.Sprintf("%s|%s|%s|%s|%s|%s",
			limit(alloc.ID, length),
			limit(alloc.EvalID, length),
			alloc.JobID,
			alloc.TaskGroup,
			alloc.DesiredStatus,
			alloc.ClientStatus)
	}
	return allocs, err
}

// getAllocatedResources returns the resource usage of the node.
func getAllocatedResources(client *api.Client, node *api.Node) ([]string, error) {
	var resources []string
	var cpu, mem, disk, iops int
	var totalCpu, totalMem, totalDisk, totalIops int

	// Compute the total
	r := node.Resources
	res := node.Reserved
	if res == nil {
		res = &api.Resources{}
	}
	totalCpu = r.CPU - res.CPU
	totalMem = r.MemoryMB - res.MemoryMB
	totalDisk = r.DiskMB - res.DiskMB
	totalIops = r.IOPS - res.IOPS

	// Get list of running allocations on the node
	runningAllocs, err := getRunningAllocs(client, node.ID)

	// Get Resources
	for _, alloc := range runningAllocs {
		cpu += alloc.Resources.CPU
		mem += alloc.Resources.MemoryMB
		disk += alloc.Resources.DiskMB
		iops += alloc.Resources.IOPS
	}

	resources = make([]string, 2)
	resources[0] = "CPU|Memory MB|Disk MB|IOPS"
	resources[1] = fmt.Sprintf("%v/%v|%v/%v|%v/%v|%v/%v",
		cpu,
		totalCpu,
		mem,
		totalMem,
		disk,
		totalDisk,
		iops,
		totalIops)

	return resources, err
}

// getActualResources returns the actual resource usage of the node.
func getActualResources(hostStats *api.HostStats, node *api.Node) ([]string, error) {
	if hostStats == nil {
		return nil, fmt.Errorf("actual resource usage not present")
	}
	var resources []string

	usedCPUPercent := 0.0
	for _, cpu := range hostStats.CPU {
		usedCPUPercent += (cpu.User + cpu.System)
	}
	usedCPUTicks := (usedCPUPercent / 100) * float64(node.Resources.CPU)

	storageDevice := node.Attributes["unique.storage.volume"]
	var diskUsed, diskSize uint64
	for _, disk := range hostStats.DiskStats {
		if disk.Device == storageDevice {
			diskUsed = disk.Used
			diskSize = disk.Size
		}
	}

	resources = make([]string, 2)
	resources[0] = "CPU|Memory MB|Disk MB"
	resources[1] = fmt.Sprintf("%v/%v|%v/%v|%v/%v",
		int64(usedCPUTicks),
		node.Resources.CPU,
		humanize.Bytes(hostStats.Memory.Used),
		humanize.Bytes(hostStats.Memory.Total),
		humanize.Bytes(diskUsed),
		humanize.Bytes(diskSize),
	)
	return resources, nil
}

func formatFloat64(val float64) string {
	return strconv.FormatFloat(val, 'f', 2, 64)
}
<|endoftext|>


<commit_before>// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package commands

import (
	"bytes"
	"errors"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"sync"
	"time"

	hconfig "github.com/gohugoio/hugo/config"

	"golang.org/x/sync/semaphore"

	"github.com/gohugoio/hugo/common/herrors"
	"github.com/gohugoio/hugo/common/hugo"

	jww "github.com/spf13/jwalterweatherman"

	"github.com/gohugoio/hugo/common/loggers"
	"github.com/gohugoio/hugo/config"

	"github.com/spf13/cobra"

	"github.com/gohugoio/hugo/hugolib"
	"github.com/spf13/afero"

	"github.com/bep/debounce"
	"github.com/gohugoio/hugo/common/types"
	"github.com/gohugoio/hugo/deps"
	"github.com/gohugoio/hugo/helpers"
	"github.com/gohugoio/hugo/hugofs"
	"github.com/gohugoio/hugo/langs"
)

type commandeerHugoState struct {
	*deps.DepsCfg
	hugoSites *hugolib.HugoSites
	fsCreate  sync.Once
	created   chan struct{}
}

type commandeer struct {
	*commandeerHugoState

	logger       loggers.Logger
	serverConfig *config.Server

	// Currently only set when in "fast render mode". But it seems to
	// be fast enough that we could maybe just add it for all server modes.
	changeDetector *fileChangeDetector

	// We need to reuse this on server rebuilds.
	destinationFs afero.Fs

	h    *hugoBuilderCommon
	ftch flagsToConfigHandler

	visitedURLs *types.EvictingStringQueue

	cfgInit func(c *commandeer) error

	// We watch these for changes.
	configFiles []string

	// Used in cases where we get flooded with events in server mode.
	debounce func(f func())

	serverPorts         []int
	languagesConfigured bool
	languages           langs.Languages
	doLiveReload        bool
	fastRenderMode      bool
	showErrorInBrowser  bool
	wasError            bool

	configured bool
	paused     bool

	fullRebuildSem *semaphore.Weighted

	// Any error from the last build.
	buildErr error
}

func newCommandeerHugoState() *commandeerHugoState {
	return &commandeerHugoState{
		created: make(chan struct{}),
	}
}

func (c *commandeerHugoState) hugo() *hugolib.HugoSites {
	<-c.created
	return c.hugoSites
}

func (c *commandeer) errCount() int {
	return int(c.logger.LogCounters().ErrorCounter.Count())
}

func (c *commandeer) getErrorWithContext() interface{} {
	errCount := c.errCount()

	if errCount == 0 {
		return nil
	}

	m := make(map[string]interface{})

	m["Error"] = errors.New(removeErrorPrefixFromLog(c.logger.Errors()))
	m["Version"] = hugo.BuildVersionString()

	fe := herrors.UnwrapErrorWithFileContext(c.buildErr)
	if fe != nil {
		m["File"] = fe
	}

	if c.h.verbose {
		var b bytes.Buffer
		herrors.FprintStackTraceFromErr(&b, c.buildErr)
		m["StackTrace"] = b.String()
	}

	return m
}

func (c *commandeer) Set(key string, value interface{}) {
	if c.configured {
		panic("commandeer cannot be changed")
	}
	c.Cfg.Set(key, value)
}

func (c *commandeer) initFs(fs *hugofs.Fs) error {
	c.destinationFs = fs.Destination
	c.DepsCfg.Fs = fs

	return nil
}

func newCommandeer(mustHaveConfigFile, running bool, h *hugoBuilderCommon, f flagsToConfigHandler, cfgInit func(c *commandeer) error, subCmdVs ...*cobra.Command) (*commandeer, error) {
	var rebuildDebouncer func(f func())
	if running {
		// The time value used is tested with mass content replacements in a fairly big Hugo site.
		// It is better to wait for some seconds in those cases rather than get flooded
		// with rebuilds.
		rebuildDebouncer = debounce.New(4 * time.Second)
	}

	out := ioutil.Discard
	if !h.quiet {
		out = os.Stdout
	}

	c := &commandeer{
		h:                   h,
		ftch:                f,
		commandeerHugoState: newCommandeerHugoState(),
		cfgInit:             cfgInit,
		visitedURLs:         types.NewEvictingStringQueue(10),
		debounce:            rebuildDebouncer,
		fullRebuildSem:      semaphore.NewWeighted(1),
		// This will be replaced later, but we need something to log to before the configuration is read.
		logger: loggers.NewLogger(jww.LevelWarn, jww.LevelError, out, ioutil.Discard, running),
	}

	return c, c.loadConfig(mustHaveConfigFile, running)
}

type fileChangeDetector struct {
	sync.Mutex
	current map[string]string
	prev    map[string]string

	irrelevantRe *regexp.Regexp
}

func (f *fileChangeDetector) OnFileClose(name, md5sum string) {
	f.Lock()
	defer f.Unlock()
	f.current[name] = md5sum
}

func (f *fileChangeDetector) changed() []string {
	if f == nil {
		return nil
	}
	f.Lock()
	defer f.Unlock()
	var c []string
	for k, v := range f.current {
		vv, found := f.prev[k]
		if !found || v != vv {
			c = append(c, k)
		}
	}

	return f.filterIrrelevant(c)
}

func (f *fileChangeDetector) filterIrrelevant(in []string) []string {
	var filtered []string
	for _, v := range in {
		if !f.irrelevantRe.MatchString(v) {
			filtered = append(filtered, v)
		}
	}
	return filtered
}

func (f *fileChangeDetector) PrepareNew() {
	if f == nil {
		return
	}

	f.Lock()
	defer f.Unlock()

	if f.current == nil {
		f.current = make(map[string]string)
		f.prev = make(map[string]string)
		return
	}

	f.prev = make(map[string]string)
	for k, v := range f.current {
		f.prev[k] = v
	}
	f.current = make(map[string]string)
}

func (c *commandeer) loadConfig(mustHaveConfigFile, running bool) error {
	if c.DepsCfg == nil {
		c.DepsCfg = &deps.DepsCfg{}
	}

	if c.logger != nil {
		// Truncate the error log if this is a reload.
		c.logger.Reset()
	}

	cfg := c.DepsCfg
	c.configured = false
	cfg.Running = running

	var dir string
	if c.h.source != "" {
		dir, _ = filepath.Abs(c.h.source)
	} else {
		dir, _ = os.Getwd()
	}

	var sourceFs afero.Fs = hugofs.Os
	if c.DepsCfg.Fs != nil {
		sourceFs = c.DepsCfg.Fs.Source
	}

	environment := c.h.getEnvironment(running)

	doWithConfig := func(cfg config.Provider) error {
		if c.ftch != nil {
			c.ftch.flagsToConfig(cfg)
		}

		cfg.Set("workingDir", dir)
		cfg.Set("environment", environment)
		return nil
	}

	cfgSetAndInit := func(cfg config.Provider) error {
		c.Cfg = cfg
		if c.cfgInit == nil {
			return nil
		}
		err := c.cfgInit(c)
		return err
	}

	configPath := c.h.source
	if configPath == "" {
		configPath = dir
	}
	config, configFiles, err := hugolib.LoadConfig(
		hugolib.ConfigSourceDescriptor{
			Fs:           sourceFs,
			Logger:       c.logger,
			Path:         configPath,
			WorkingDir:   dir,
			Filename:     c.h.cfgFile,
			AbsConfigDir: c.h.getConfigDir(dir),
			Environment:  environment,
		},
		cfgSetAndInit,
		doWithConfig)

	if err != nil && mustHaveConfigFile {
		return err
	} else if mustHaveConfigFile && len(configFiles) == 0 {
		return hugolib.ErrNoConfigFile
	}

	c.configFiles = configFiles

	if l, ok := c.Cfg.Get("languagesSorted").(langs.Languages); ok {
		c.languagesConfigured = true
		c.languages = l
	}

	// Set some commonly used flags
	c.doLiveReload = running && !c.Cfg.GetBool("disableLiveReload")
	c.fastRenderMode = c.doLiveReload && !c.Cfg.GetBool("disableFastRender")
	c.showErrorInBrowser = c.doLiveReload && !c.Cfg.GetBool("disableBrowserError")

	// This is potentially double work, but we need to do this one more time now
	// that all the languages have been configured.
	if c.cfgInit != nil {
		if err := c.cfgInit(c); err != nil {
			return err
		}
	}

	logger, err := c.createLogger(config, running)
	if err != nil {
		return err
	}

	cfg.Logger = logger
	c.logger = logger
	c.serverConfig, err = hconfig.DecodeServer(cfg.Cfg)
	if err != nil {
		return err
	}

	createMemFs := config.GetBool("renderToMemory")

	if createMemFs {
		// Rendering to memoryFS, publish to Root regardless of publishDir.
		config.Set("publishDir", "/")
	}

	c.fsCreate.Do(func() {
		fs := hugofs.NewFrom(sourceFs, config)

		if c.destinationFs != nil {
			// Need to reuse the destination on server rebuilds.
			fs.Destination = c.destinationFs
		} else if createMemFs {
			// Hugo writes the output to memory instead of the disk.
			fs.Destination = new(afero.MemMapFs)
		}

		if c.fastRenderMode {
			// For now, fast render mode only. It should, however, be fast enough
			// for the full variant, too.
			changeDetector := &fileChangeDetector{
				// We use this detector to decide to do a Hot reload of a single path or not.
				// We need to filter out source maps and possibly some other to be able
				// to make that decision.
				irrelevantRe: regexp.MustCompile(`\.map$`),
			}

			changeDetector.PrepareNew()
			fs.Destination = hugofs.NewHashingFs(fs.Destination, changeDetector)
			c.changeDetector = changeDetector
		}

		if c.Cfg.GetBool("logPathWarnings") {
			fs.Destination = hugofs.NewCreateCountingFs(fs.Destination)
		}

		// To debug hard-to-find path issues.
		// fs.Destination = hugofs.NewStacktracerFs(fs.Destination, `fr/fr`)

		err = c.initFs(fs)
		if err != nil {
			close(c.created)
			return
		}

		var h *hugolib.HugoSites

		h, err = hugolib.NewHugoSites(*c.DepsCfg)
		c.hugoSites = h
		close(c.created)
	})

	if err != nil {
		return err
	}

	cacheDir, err := helpers.GetCacheDir(sourceFs, config)
	if err != nil {
		return err
	}
	config.Set("cacheDir", cacheDir)

	return nil
}
<commit_msg>commands: Fix panic on invalid config in "hugo mod get" and similar<commit_after>// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package commands

import (
	"bytes"
	"errors"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"sync"
	"time"

	hconfig "github.com/gohugoio/hugo/config"

	"golang.org/x/sync/semaphore"

	"github.com/gohugoio/hugo/common/herrors"
	"github.com/gohugoio/hugo/common/hugo"

	jww "github.com/spf13/jwalterweatherman"

	"github.com/gohugoio/hugo/common/loggers"
	"github.com/gohugoio/hugo/config"

	"github.com/spf13/cobra"

	"github.com/gohugoio/hugo/hugolib"
	"github.com/spf13/afero"

	"github.com/bep/debounce"
	"github.com/gohugoio/hugo/common/types"
	"github.com/gohugoio/hugo/deps"
	"github.com/gohugoio/hugo/helpers"
	"github.com/gohugoio/hugo/hugofs"
	"github.com/gohugoio/hugo/langs"
)

type commandeerHugoState struct {
	*deps.DepsCfg
	hugoSites *hugolib.HugoSites
	fsCreate  sync.Once
	created   chan struct{}
}

type commandeer struct {
	*commandeerHugoState

	logger       loggers.Logger
	serverConfig *config.Server

	// Currently only set when in "fast render mode". But it seems to
	// be fast enough that we could maybe just add it for all server modes.
	changeDetector *fileChangeDetector

	// We need to reuse this on server rebuilds.
	destinationFs afero.Fs

	h    *hugoBuilderCommon
	ftch flagsToConfigHandler

	visitedURLs *types.EvictingStringQueue

	cfgInit func(c *commandeer) error

	// We watch these for changes.
	configFiles []string

	// Used in cases where we get flooded with events in server mode.
	debounce func(f func())

	serverPorts         []int
	languagesConfigured bool
	languages           langs.Languages
	doLiveReload        bool
	fastRenderMode      bool
	showErrorInBrowser  bool
	wasError            bool

	configured bool
	paused     bool

	fullRebuildSem *semaphore.Weighted

	// Any error from the last build.
	buildErr error
}

func newCommandeerHugoState() *commandeerHugoState {
	return &commandeerHugoState{
		created: make(chan struct{}),
	}
}

func (c *commandeerHugoState) hugo() *hugolib.HugoSites {
	<-c.created
	return c.hugoSites
}

func (c *commandeer) errCount() int {
	return int(c.logger.LogCounters().ErrorCounter.Count())
}

func (c *commandeer) getErrorWithContext() interface{} {
	errCount := c.errCount()

	if errCount == 0 {
		return nil
	}

	m := make(map[string]interface{})

	m["Error"] = errors.New(removeErrorPrefixFromLog(c.logger.Errors()))
	m["Version"] = hugo.BuildVersionString()

	fe := herrors.UnwrapErrorWithFileContext(c.buildErr)
	if fe != nil {
		m["File"] = fe
	}

	if c.h.verbose {
		var b bytes.Buffer
		herrors.FprintStackTraceFromErr(&b, c.buildErr)
		m["StackTrace"] = b.String()
	}

	return m
}

func (c *commandeer) Set(key string, value interface{}) {
	if c.configured {
		panic("commandeer cannot be changed")
	}
	c.Cfg.Set(key, value)
}

func (c *commandeer) initFs(fs *hugofs.Fs) error {
	c.destinationFs = fs.Destination
	c.DepsCfg.Fs = fs

	return nil
}

func newCommandeer(mustHaveConfigFile, running bool, h *hugoBuilderCommon, f flagsToConfigHandler, cfgInit func(c *commandeer) error, subCmdVs ...*cobra.Command) (*commandeer, error) {
	var rebuildDebouncer func(f func())
	if running {
		// The time value used is tested with mass content replacements in a fairly big Hugo site.
		// It is better to wait for some seconds in those cases rather than get flooded
		// with rebuilds.
		rebuildDebouncer = debounce.New(4 * time.Second)
	}

	out := ioutil.Discard
	if !h.quiet {
		out = os.Stdout
	}

	c := &commandeer{
		h:                   h,
		ftch:                f,
		commandeerHugoState: newCommandeerHugoState(),
		cfgInit:             cfgInit,
		visitedURLs:         types.NewEvictingStringQueue(10),
		debounce:            rebuildDebouncer,
		fullRebuildSem:      semaphore.NewWeighted(1),
		// This will be replaced later, but we need something to log to before the configuration is read.
		logger: loggers.NewLogger(jww.LevelWarn, jww.LevelError, out, ioutil.Discard, running),
	}

	return c, c.loadConfig(mustHaveConfigFile, running)
}

type fileChangeDetector struct {
	sync.Mutex
	current map[string]string
	prev    map[string]string

	irrelevantRe *regexp.Regexp
}

func (f *fileChangeDetector) OnFileClose(name, md5sum string) {
	f.Lock()
	defer f.Unlock()
	f.current[name] = md5sum
}

func (f *fileChangeDetector) changed() []string {
	if f == nil {
		return nil
	}
	f.Lock()
	defer f.Unlock()
	var c []string
	for k, v := range f.current {
		vv, found := f.prev[k]
		if !found || v != vv {
			c = append(c, k)
		}
	}

	return f.filterIrrelevant(c)
}

func (f *fileChangeDetector) filterIrrelevant(in []string) []string {
	var filtered []string
	for _, v := range in {
		if !f.irrelevantRe.MatchString(v) {
			filtered = append(filtered, v)
		}
	}
	return filtered
}

func (f *fileChangeDetector) PrepareNew() {
	if f == nil {
		return
	}

	f.Lock()
	defer f.Unlock()

	if f.current == nil {
		f.current = make(map[string]string)
		f.prev = make(map[string]string)
		return
	}

	f.prev = make(map[string]string)
	for k, v := range f.current {
		f.prev[k] = v
	}
	f.current = make(map[string]string)
}

func (c *commandeer) loadConfig(mustHaveConfigFile, running bool) error {
	if c.DepsCfg == nil {
		c.DepsCfg = &deps.DepsCfg{}
	}

	if c.logger != nil {
		// Truncate the error log if this is a reload.
		c.logger.Reset()
	}

	cfg := c.DepsCfg
	c.configured = false
	cfg.Running = running

	var dir string
	if c.h.source != "" {
		dir, _ = filepath.Abs(c.h.source)
	} else {
		dir, _ = os.Getwd()
	}

	var sourceFs afero.Fs = hugofs.Os
	if c.DepsCfg.Fs != nil {
		sourceFs = c.DepsCfg.Fs.Source
	}

	environment := c.h.getEnvironment(running)

	doWithConfig := func(cfg config.Provider) error {
		if c.ftch != nil {
			c.ftch.flagsToConfig(cfg)
		}

		cfg.Set("workingDir", dir)
		cfg.Set("environment", environment)
		return nil
	}

	cfgSetAndInit := func(cfg config.Provider) error {
		c.Cfg = cfg
		if c.cfgInit == nil {
			return nil
		}
		err := c.cfgInit(c)
		return err
	}

	configPath := c.h.source
	if configPath == "" {
		configPath = dir
	}
	config, configFiles, err := hugolib.LoadConfig(
		hugolib.ConfigSourceDescriptor{
			Fs:           sourceFs,
			Logger:       c.logger,
			Path:         configPath,
			WorkingDir:   dir,
			Filename:     c.h.cfgFile,
			AbsConfigDir: c.h.getConfigDir(dir),
			Environment:  environment,
		},
		cfgSetAndInit,
		doWithConfig)

	if err != nil {
		return err
	} else if mustHaveConfigFile && len(configFiles) == 0 {
		return hugolib.ErrNoConfigFile
	}

	c.configFiles = configFiles

	if l, ok := c.Cfg.Get("languagesSorted").(langs.Languages); ok {
		c.languagesConfigured = true
		c.languages = l
	}

	// Set some commonly used flags
	c.doLiveReload = running && !c.Cfg.GetBool("disableLiveReload")
	c.fastRenderMode = c.doLiveReload && !c.Cfg.GetBool("disableFastRender")
	c.showErrorInBrowser = c.doLiveReload && !c.Cfg.GetBool("disableBrowserError")

	// This is potentially double work, but we need to do this one more time now
	// that all the languages have been configured.
	if c.cfgInit != nil {
		if err := c.cfgInit(c); err != nil {
			return err
		}
	}

	logger, err := c.createLogger(config, running)
	if err != nil {
		return err
	}

	cfg.Logger = logger
	c.logger = logger
	c.serverConfig, err = hconfig.DecodeServer(cfg.Cfg)
	if err != nil {
		return err
	}

	createMemFs := config.GetBool("renderToMemory")

	if createMemFs {
		// Rendering to memoryFS, publish to Root regardless of publishDir.
		config.Set("publishDir", "/")
	}

	c.fsCreate.Do(func() {
		fs := hugofs.NewFrom(sourceFs, config)

		if c.destinationFs != nil {
			// Need to reuse the destination on server rebuilds.
			fs.Destination = c.destinationFs
		} else if createMemFs {
			// Hugo writes the output to memory instead of the disk.
			fs.Destination = new(afero.MemMapFs)
		}

		if c.fastRenderMode {
			// For now, fast render mode only. It should, however, be fast enough
			// for the full variant, too.
			changeDetector := &fileChangeDetector{
				// We use this detector to decide to do a Hot reload of a single path or not.
				// We need to filter out source maps and possibly some other to be able
				// to make that decision.
				irrelevantRe: regexp.MustCompile(`\.map$`),
			}

			changeDetector.PrepareNew()
			fs.Destination = hugofs.NewHashingFs(fs.Destination, changeDetector)
			c.changeDetector = changeDetector
		}

		if c.Cfg.GetBool("logPathWarnings") {
			fs.Destination = hugofs.NewCreateCountingFs(fs.Destination)
		}

		// To debug hard-to-find path issues.
		// fs.Destination = hugofs.NewStacktracerFs(fs.Destination, `fr/fr`)

		err = c.initFs(fs)
		if err != nil {
			close(c.created)
			return
		}

		var h *hugolib.HugoSites

		h, err = hugolib.NewHugoSites(*c.DepsCfg)
		c.hugoSites = h
		close(c.created)
	})

	if err != nil {
		return err
	}

	cacheDir, err := helpers.GetCacheDir(sourceFs, config)
	if err != nil {
		return err
	}
	config.Set("cacheDir", cacheDir)

	return nil
}
<|endoftext|>


<commit_before>/*
 * Spreed Speak Freely.
 * Copyright (C) 2013-2014 struktur AG
 *
 * This file is part of Spreed Speak Freely.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

package main

import (
	"bytes"
	"crypto/rand"
	"encoding/base64"
	"encoding/binary"
	"log"
	"strings"
	"sync"
	"time"
)

var imageFilenames map[string]string

type Image struct {
	updateIdx    int
	lastChange   time.Time
	lastChangeId string
	sessionid    string
	mimetype     string
	data         []byte
}

type ImageCache interface {
	Update(sessionId string, image string) string

	Get(imageId string) *Image

	Delete(sessionId string)
}

type imageCache struct {
	images        map[string]*Image
	sessionImages map[string]string
	mutex         sync.RWMutex
}

func NewImageCache() ImageCache {
	result := &imageCache{}
	result.images = make(map[string]*Image)
	result.sessionImages = make(map[string]string)
	if imageFilenames == nil {
		imageFilenames = map[string]string{
			"image/png":  "picture.png",
			"image/jpeg": "picture.jpg",
			"image/gif":  "picture.gif",
		}
	}
	return result
}

func (self *imageCache) Update(sessionId string, image string) string {
	mimetype := "image/x-unknown"
	pos := strings.Index(image, ";")
	if pos != -1 {
		mimetype = image[:pos]
		image = image[pos+1:]
	}
	pos = strings.Index(image, ",")
	var decoded []byte
	var err error
	if pos != -1 {
		encoding := image[:pos]
		switch encoding {
		case "base64":
			decoded, err = base64.StdEncoding.DecodeString(image[pos+1:])
			if err != nil {
				return ""
			}
		default:
			log.Println("Unknown encoding", encoding)
			return ""
		}
	} else {
		decoded = []byte(image[pos+1:])
	}
	var img *Image
	self.mutex.RLock()
	result, ok := self.sessionImages[sessionId]
	if !ok {
		self.mutex.RUnlock()
		imageId := make([]byte, 15, 15)
		if _, err = rand.Read(imageId); err != nil {
			return ""
		}
		result = base64.URLEncoding.EncodeToString(imageId)
		img = &Image{sessionid: sessionId}
		self.mutex.Lock()
		resultTmp, ok := self.sessionImages[sessionId]
		if !ok {
			self.sessionImages[sessionId] = result
			self.images[result] = img
		} else {
			result = resultTmp
			img = self.images[result]
		}
		self.mutex.Unlock()
	} else {
		img = self.images[result]
		self.mutex.RUnlock()
	}
	if mimetype != img.mimetype || !bytes.Equal(img.data, decoded) {
		img.updateIdx++
		img.lastChange = time.Now()
		tmp := make([]byte, binary.MaxVarintLen64)
		count := binary.PutUvarint(tmp, uint64(img.lastChange.UnixNano()))
		img.lastChangeId = base64.URLEncoding.EncodeToString(tmp[:count])
		img.mimetype = mimetype
		img.data = decoded
	}
	result += "/" + img.lastChangeId
	filename, ok := imageFilenames[mimetype]
	if ok {
		result += "/" + filename
	}
	return result
}

func (self *imageCache) Get(imageId string) *Image {
	self.mutex.RLock()
	image := self.images[imageId]
	self.mutex.RUnlock()
	return image
}

func (self *imageCache) Delete(sessionId string) {
	self.mutex.Lock()
	imageId, ok := self.sessionImages[sessionId]
	if ok {
		delete(self.sessionImages, sessionId)
		delete(self.images, imageId)
	}
	self.mutex.Unlock()
}
<commit_msg>Include filename for WebP images.<commit_after>/*
 * Spreed Speak Freely.
 * Copyright (C) 2013-2014 struktur AG
 *
 * This file is part of Spreed Speak Freely.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

package main

import (
	"bytes"
	"crypto/rand"
	"encoding/base64"
	"encoding/binary"
	"log"
	"strings"
	"sync"
	"time"
)

var imageFilenames map[string]string

type Image struct {
	updateIdx    int
	lastChange   time.Time
	lastChangeId string
	sessionid    string
	mimetype     string
	data         []byte
}

type ImageCache interface {
	Update(sessionId string, image string) string

	Get(imageId string) *Image

	Delete(sessionId string)
}

type imageCache struct {
	images        map[string]*Image
	sessionImages map[string]string
	mutex         sync.RWMutex
}

func NewImageCache() ImageCache {
	result := &imageCache{}
	result.images = make(map[string]*Image)
	result.sessionImages = make(map[string]string)
	if imageFilenames == nil {
		imageFilenames = map[string]string{
			"image/png":  "picture.png",
			"image/jpeg": "picture.jpg",
			"image/gif":  "picture.gif",
			"image/webp": "picture.webp",
		}
	}
	return result
}

func (self *imageCache) Update(sessionId string, image string) string {
	mimetype := "image/x-unknown"
	pos := strings.Index(image, ";")
	if pos != -1 {
		mimetype = image[:pos]
		image = image[pos+1:]
	}
	pos = strings.Index(image, ",")
	var decoded []byte
	var err error
	if pos != -1 {
		encoding := image[:pos]
		switch encoding {
		case "base64":
			decoded, err = base64.StdEncoding.DecodeString(image[pos+1:])
			if err != nil {
				return ""
			}
		default:
			log.Println("Unknown encoding", encoding)
			return ""
		}
	} else {
		decoded = []byte(image[pos+1:])
	}
	var img *Image
	self.mutex.RLock()
	result, ok := self.sessionImages[sessionId]
	if !ok {
		self.mutex.RUnlock()
		imageId := make([]byte, 15, 15)
		if _, err = rand.Read(imageId); err != nil {
			return ""
		}
		result = base64.URLEncoding.EncodeToString(imageId)
		img = &Image{sessionid: sessionId}
		self.mutex.Lock()
		resultTmp, ok := self.sessionImages[sessionId]
		if !ok {
			self.sessionImages[sessionId] = result
			self.images[result] = img
		} else {
			result = resultTmp
			img = self.images[result]
		}
		self.mutex.Unlock()
	} else {
		img = self.images[result]
		self.mutex.RUnlock()
	}
	if mimetype != img.mimetype || !bytes.Equal(img.data, decoded) {
		img.updateIdx++
		img.lastChange = time.Now()
		tmp := make([]byte, binary.MaxVarintLen64)
		count := binary.PutUvarint(tmp, uint64(img.lastChange.UnixNano()))
		img.lastChangeId = base64.URLEncoding.EncodeToString(tmp[:count])
		img.mimetype = mimetype
		img.data = decoded
	}
	result += "/" + img.lastChangeId
	filename, ok := imageFilenames[mimetype]
	if ok {
		result += "/" + filename
	}
	return result
}

func (self *imageCache) Get(imageId string) *Image {
	self.mutex.RLock()
	image := self.images[imageId]
	self.mutex.RUnlock()
	return image
}

func (self *imageCache) Delete(sessionId string) {
	self.mutex.Lock()
	imageId, ok := self.sessionImages[sessionId]
	if ok {
		delete(self.sessionImages, sessionId)
		delete(self.images, imageId)
	}
	self.mutex.Unlock()
}
<|endoftext|>


<commit_before>package gorethink

import (
	test "launchpad.net/gocheck"
	"time"
)

func (s *RethinkSuite) TestTimeTime(c *test.C) {
	var response time.Time
	err := Time(1986, 11, 3, 12, 30, 15, "Z").RunRow(sess).Scan(&response)
	c.Assert(err, test.IsNil)
	c.Assert(response.Equal(time.Date(1986, 11, 3, 12, 30, 15, 0, time.Local)), test.Equals, true)
}

func (s *RethinkSuite) TestTimeEpochTime(c *test.C) {
	var response time.Time
	err := EpochTime(531360000).RunRow(sess).Scan(&response)
	c.Assert(err, test.IsNil)
	c.Assert(response.Equal(time.Date(1986, 11, 3, 0, 0, 0, 0, time.Local)), test.Equals, true)
}

func (s *RethinkSuite) TestTimeISO8601(c *test.C) {
	var t1, t2 time.Time
	t2, _ = time.Parse("2006-01-02T15:04:05-07:00", "1986-11-03T08:30:00-07:00")
	err := ISO8601("1986-11-03T08:30:00-07:00").RunRow(sess).Scan(&t1)
	c.Assert(err, test.IsNil)
	c.Assert(t1.Equal(t2), test.Equals, true)
}

func (s *RethinkSuite) TestTimeInTimezone(c *test.C) {
	loc, err := time.LoadLocation("MST")
	c.Assert(err, test.IsNil)
	var response []time.Time
	err = Expr([]interface{}{Now(), Now().InTimezone("-07:00")}).RunRow(sess).Scan(&response)
	c.Assert(err, test.IsNil)
	c.Assert(response[1].Equal(response[0].In(loc)), test.Equals, true)
}

func (s *RethinkSuite) TestTimeBetween(c *test.C) {
	var response interface{}

	times := Expr([]interface{}{
		Time(1986, 9, 3, 12, 30, 15, "Z"),
		Time(1986, 10, 3, 12, 30, 15, "Z"),
		Time(1986, 11, 3, 12, 30, 15, "Z"),
		Time(1986, 12, 3, 12, 30, 15, "Z"),
	})
	err := times.Filter(func(row RqlTerm) RqlTerm {
		return row.During(Time(1986, 9, 3, 12, 30, 15, "Z"), Time(1986, 11, 3, 12, 30, 15, "Z"))
	}).Count().RunRow(sess).Scan(&response)

	c.Assert(err, test.IsNil)
	c.Assert(int(response.(float64)), test.Equals, 2)
}

func (s *RethinkSuite) TestTimeYear(c *test.C) {
	var response interface{}

	err := Time(1986, 12, 3, 12, 30, 15, "Z").Year().RunRow(sess).Scan(&response)

	c.Assert(err, test.IsNil)
	c.Assert(int(response.(float64)), test.Equals, 1986)
}

func (s *RethinkSuite) TestTimeMonth(c *test.C) {
	var response interface{}

	err := Time(1986, 12, 3, 12, 30, 15, "Z").Month().Eq(December()).RunRow(sess).Scan(&response)

	c.Assert(err, test.IsNil)
	c.Assert(response.(bool), test.Equals, true)
}

func (s *RethinkSuite) TestTimeDay(c *test.C) {
	var response interface{}

	err := Time(1986, 12, 3, 12, 30, 15, "Z").Day().Eq(Wednesday()).RunRow(sess).Scan(&response)

	c.Assert(err, test.IsNil)
	c.Assert(response.(bool), test.Equals, true)
}
<commit_msg>Fixed time tests not using time.UTC. Fixes #12"<commit_after>package gorethink

import (
	test "launchpad.net/gocheck"
	"time"
)

func (s *RethinkSuite) TestTimeTime(c *test.C) {
	var response time.Time
	err := Time(1986, 11, 3, 12, 30, 15, "Z").RunRow(sess).Scan(&response)
	c.Assert(err, test.IsNil)
	c.Assert(response.Equal(time.Date(1986, 11, 3, 12, 30, 15, 0, time.UTC)), test.Equals, true)
}

func (s *RethinkSuite) TestTimeEpochTime(c *test.C) {
	var response time.Time
	err := EpochTime(531360000).RunRow(sess).Scan(&response)
	c.Assert(err, test.IsNil)
	c.Assert(response.Equal(time.Date(1986, 11, 3, 0, 0, 0, 0, time.UTC)), test.Equals, true)
}

func (s *RethinkSuite) TestTimeISO8601(c *test.C) {
	var t1, t2 time.Time
	t2, _ = time.Parse("2006-01-02T15:04:05-07:00", "1986-11-03T08:30:00-07:00")
	err := ISO8601("1986-11-03T08:30:00-07:00").RunRow(sess).Scan(&t1)
	c.Assert(err, test.IsNil)
	c.Assert(t1.Equal(t2), test.Equals, true)
}

func (s *RethinkSuite) TestTimeInTimezone(c *test.C) {
	loc, err := time.LoadLocation("MST")
	c.Assert(err, test.IsNil)
	var response []time.Time
	err = Expr([]interface{}{Now(), Now().InTimezone("-07:00")}).RunRow(sess).Scan(&response)
	c.Assert(err, test.IsNil)
	c.Assert(response[1].Equal(response[0].In(loc)), test.Equals, true)
}

func (s *RethinkSuite) TestTimeBetween(c *test.C) {
	var response interface{}

	times := Expr([]interface{}{
		Time(1986, 9, 3, 12, 30, 15, "Z"),
		Time(1986, 10, 3, 12, 30, 15, "Z"),
		Time(1986, 11, 3, 12, 30, 15, "Z"),
		Time(1986, 12, 3, 12, 30, 15, "Z"),
	})
	err := times.Filter(func(row RqlTerm) RqlTerm {
		return row.During(Time(1986, 9, 3, 12, 30, 15, "Z"), Time(1986, 11, 3, 12, 30, 15, "Z"))
	}).Count().RunRow(sess).Scan(&response)

	c.Assert(err, test.IsNil)
	c.Assert(int(response.(float64)), test.Equals, 2)
}

func (s *RethinkSuite) TestTimeYear(c *test.C) {
	var response interface{}

	err := Time(1986, 12, 3, 12, 30, 15, "Z").Year().RunRow(sess).Scan(&response)

	c.Assert(err, test.IsNil)
	c.Assert(int(response.(float64)), test.Equals, 1986)
}

func (s *RethinkSuite) TestTimeMonth(c *test.C) {
	var response interface{}

	err := Time(1986, 12, 3, 12, 30, 15, "Z").Month().Eq(December()).RunRow(sess).Scan(&response)

	c.Assert(err, test.IsNil)
	c.Assert(response.(bool), test.Equals, true)
}

func (s *RethinkSuite) TestTimeDay(c *test.C) {
	var response interface{}

	err := Time(1986, 12, 3, 12, 30, 15, "Z").Day().Eq(Wednesday()).RunRow(sess).Scan(&response)

	c.Assert(err, test.IsNil)
	c.Assert(response.(bool), test.Equals, true)
}
<|endoftext|>