<commit_before>package main

import (
	"bufio"
	"fmt"
	"os"
	"os/exec"
	"strings"
	"sync"
	"time"
)

type rCommand struct {
	command   *exec.Cmd
	output    string
	startTime int64
	endTime   int64
	err       error
	mainOut   string
}

//Scheduler the main task scheduler
type Scheduler struct {
	commands []*rCommand
	cMutex   *sync.Mutex
	rMutex   *sync.Mutex
	rMap     map[string]*rCommand
	Log      func(string)
}

func (s *Scheduler) getState(key string) string {
	s.rMutex.Lock()
	defer s.rMutex.Unlock()
	return fmt.Sprintf("%v -> %v", s.rMap[key].endTime, s.rMap[key].output)
}

func (s *Scheduler) markComplete(key string) {
	s.rMutex.Lock()
	if val, ok := s.rMap[key]; ok {
		val.endTime = time.Now().Unix()
		val.output = key
	} else {
		s.rMap[key] = &rCommand{endTime: time.Now().Unix(), output: key}
	}
	s.rMutex.Unlock()
}

// Schedule schedules a task
func (s *Scheduler) Schedule(c *rCommand) string {
	key := fmt.Sprintf("%v-%v", time.Now().Nanosecond(), c.command.Path)
	s.commands = append(s.commands, c)
	s.rMutex.Lock()
	s.rMap[key] = c
	s.rMutex.Unlock()
	s.processCommands()
	return key
}

func (s *Scheduler) getOutput(key string) string {
	s.rMutex.Lock()
	if val, ok := s.rMap[key]; ok {
		s.rMutex.Unlock()
		return val.output
	}

	s.rMutex.Unlock()
	return fmt.Sprintf("KEY NOT_IN_MAP: %v", key)
}

func (s *Scheduler) killJob(key string) {
	s.rMutex.Lock()
	if val, ok := s.rMap[key]; ok {
		val.command.Process.Kill()
		val.command.Process.Wait()
	}
	s.rMutex.Unlock()
}

func (s *Scheduler) removeJob(key string) {
	delete(s.rMap, key)
}

func (s *Scheduler) schedulerComplete(key string) bool {
	s.rMutex.Lock()
	defer s.rMutex.Unlock()
	if val, ok := s.rMap[key]; ok {
		return val.endTime > 0
	}

	return false
}

func (s *Scheduler) processCommands() {
	s.cMutex.Lock()
	if len(s.commands) > 0 {
		c := s.commands[0]
		s.commands = s.commands[1:]
		run(c)
	}
	s.cMutex.Unlock()
}

func run(c *rCommand) error {
	env := os.Environ()
	home := ""
	for _, s := range env {
		if strings.HasPrefix(s, "HOME=") {
			home = s[5:]
		}
	}

	gpath := home + "/gobuild"
	c.command.Path = strings.Replace(c.command.Path, "$GOPATH", gpath, -1)
	for i := range c.command.Args {
		c.command.Args[i] = strings.Replace(c.command.Args[i], "$GOPATH", gpath, -1)
	}
	path := fmt.Sprintf("GOPATH=" + home + "/gobuild")
	pathbin := fmt.Sprintf("GOBIN=" + home + "/gobuild/bin")
	found := false
	for i, blah := range env {
		if strings.HasPrefix(blah, "GOPATH") {
			env[i] = path
			found = true
		}
		if strings.HasPrefix(blah, "GOBIN") {
			env[i] = pathbin
			found = true
		}
	}
	if !found {
		env = append(env, path)
	}
	c.command.Env = env

	out, err := c.command.StderrPipe()
	outr, err := c.command.StdoutPipe()

	if out != nil {
		scanner := bufio.NewScanner(out)
		go func() {
			for scanner != nil && scanner.Scan() {
				c.output += scanner.Text()
			}
			out.Close()
		}()
	}

	if outr != nil {
		scanner2 := bufio.NewScanner(outr)
		go func() {
			for scanner2 != nil && scanner2.Scan() {
				c.mainOut += scanner2.Text()
			}
			outr.Close()
		}()
	}

	err = c.command.Start()
	if err != nil {
		return err
	}
	c.startTime = time.Now().Unix()

	// Monitor the job and report completion
	go func() {
		err := c.command.Wait()
		if err != nil {
			c.err = err
		}
		c.endTime = time.Now().Unix()
	}()

	return nil
}
<commit_msg>Improve map locking. This closes #273<commit_after>package main

import (
	"bufio"
	"fmt"
	"os"
	"os/exec"
	"strings"
	"sync"
	"time"
)

type rCommand struct {
	command   *exec.Cmd
	output    string
	startTime int64
	endTime   int64
	err       error
	mainOut   string
}

//Scheduler the main task scheduler
type Scheduler struct {
	commands []*rCommand
	cMutex   *sync.Mutex
	rMutex   *sync.Mutex
	rMap     map[string]*rCommand
	Log      func(string)
}

func (s *Scheduler) getState(key string) string {
	s.rMutex.Lock()
	defer s.rMutex.Unlock()
	return fmt.Sprintf("%v -> %v", s.rMap[key].endTime, s.rMap[key].output)
}

func (s *Scheduler) markComplete(key string) {
	s.rMutex.Lock()
	defer s.rMutex.Unlock()
	if val, ok := s.rMap[key]; ok {
		val.endTime = time.Now().Unix()
		val.output = key
	} else {
		s.rMap[key] = &rCommand{endTime: time.Now().Unix(), output: key}
	}
}

// Schedule schedules a task
func (s *Scheduler) Schedule(c *rCommand) string {
	key := fmt.Sprintf("%v-%v", time.Now().Nanosecond(), c.command.Path)
	s.commands = append(s.commands, c)
	s.rMutex.Lock()
	s.rMap[key] = c
	s.rMutex.Unlock()
	s.processCommands()
	return key
}

func (s *Scheduler) getOutput(key string) string {
	s.rMutex.Lock()
	defer s.rMutex.Unlock()
	if val, ok := s.rMap[key]; ok {
		return val.output
	}

	return fmt.Sprintf("KEY NOT_IN_MAP: %v", key)
}

func (s *Scheduler) killJob(key string) {
	s.rMutex.Lock()
	if val, ok := s.rMap[key]; ok {
		val.command.Process.Kill()
		val.command.Process.Wait()
	}
	s.rMutex.Unlock()
}

func (s *Scheduler) removeJob(key string) {
	s.rMutex.Lock()
	defer s.rMutex.Unlock()
	delete(s.rMap, key)
}

func (s *Scheduler) schedulerComplete(key string) bool {
	s.rMutex.Lock()
	defer s.rMutex.Unlock()
	if val, ok := s.rMap[key]; ok {
		return val.endTime > 0
	}

	return false
}

func (s *Scheduler) processCommands() {
	s.cMutex.Lock()
	if len(s.commands) > 0 {
		c := s.commands[0]
		s.commands = s.commands[1:]
		run(c)
	}
	s.cMutex.Unlock()
}

func run(c *rCommand) error {
	env := os.Environ()
	home := ""
	for _, s := range env {
		if strings.HasPrefix(s, "HOME=") {
			home = s[5:]
		}
	}

	gpath := home + "/gobuild"
	c.command.Path = strings.Replace(c.command.Path, "$GOPATH", gpath, -1)
	for i := range c.command.Args {
		c.command.Args[i] = strings.Replace(c.command.Args[i], "$GOPATH", gpath, -1)
	}
	path := fmt.Sprintf("GOPATH=" + home + "/gobuild")
	pathbin := fmt.Sprintf("GOBIN=" + home + "/gobuild/bin")
	found := false
	for i, blah := range env {
		if strings.HasPrefix(blah, "GOPATH") {
			env[i] = path
			found = true
		}
		if strings.HasPrefix(blah, "GOBIN") {
			env[i] = pathbin
			found = true
		}
	}
	if !found {
		env = append(env, path)
	}
	c.command.Env = env

	out, err := c.command.StderrPipe()
	outr, err := c.command.StdoutPipe()

	if out != nil {
		scanner := bufio.NewScanner(out)
		go func() {
			for scanner != nil && scanner.Scan() {
				c.output += scanner.Text()
			}
			out.Close()
		}()
	}

	if outr != nil {
		scanner2 := bufio.NewScanner(outr)
		go func() {
			for scanner2 != nil && scanner2.Scan() {
				c.mainOut += scanner2.Text()
			}
			outr.Close()
		}()
	}

	err = c.command.Start()
	if err != nil {
		return err
	}
	c.startTime = time.Now().Unix()

	// Monitor the job and report completion
	go func() {
		err := c.command.Wait()
		if err != nil {
			c.err = err
		}
		c.endTime = time.Now().Unix()
	}()

	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2014-2017 Ludovic Fauvet
// Licensed under the MIT license

package scan

import (
	"errors"
	"fmt"
	. "github.com/etix/mirrorbits/config"
	"github.com/etix/mirrorbits/database"
	"github.com/etix/mirrorbits/filesystem"
	"github.com/etix/mirrorbits/network"
	"github.com/etix/mirrorbits/utils"
	"github.com/garyburd/redigo/redis"
	"github.com/op/go-logging"
	"os"
	"path/filepath"
	"strconv"
	"time"
)

var (
	ScanAborted    = errors.New("scan aborted")
	ScanInProgress = errors.New("scan already in progress")

	log = logging.MustGetLogger("main")
)

type ScannerType int8

const (
	RSYNC ScannerType = iota
	FTP
)

type Scanner interface {
	Scan(url, identifier string, conn redis.Conn, stop chan bool) error
}

type filedata struct {
	path    string
	sha1    string
	sha256  string
	md5     string
	size    int64
	modTime time.Time
}

type scan struct {
	redis *database.Redis

	conn        redis.Conn
	identifier  string
	filesKey    string
	filesTmpKey string
	count       uint
}

func IsScanning(conn redis.Conn, identifier string) (bool, error) {
	return redis.Bool(conn.Do("EXISTS", fmt.Sprintf("SCANNING_%s", identifier)))
}

func Scan(typ ScannerType, r *database.Redis, url, identifier string, stop chan bool) error {
	s := &scan{
		redis:      r,
		identifier: identifier,
	}

	var scanner Scanner
	switch typ {
	case RSYNC:
		scanner = &RsyncScanner{
			scan: s,
		}
	case FTP:
		scanner = &FTPScanner{
			scan: s,
		}
	default:
		panic(fmt.Sprintf("Unknown scanner"))
	}

	// Connect to the database
	conn := s.redis.Get()
	defer conn.Close()

	s.conn = conn

	// Try to aquire a lock so we don't have a scanning race
	// from different nodes.
	// Also make the key expire automatically in case our process
	// gets killed.
	lock := network.NewClusterLock(s.redis, fmt.Sprintf("SCANNING_%s", identifier), identifier)

	done, err := lock.Get()
	if err != nil {
		return err
	} else if done == nil {
		return ScanInProgress
	}

	defer lock.Release()

	s.setLastSync(conn, identifier, false)

	conn.Send("MULTI")

	s.filesKey = fmt.Sprintf("MIRROR_%s_FILES", identifier)
	s.filesTmpKey = fmt.Sprintf("MIRROR_%s_FILES_TMP", identifier)

	// Remove any left over
	conn.Send("DEL", s.filesTmpKey)

	err = scanner.Scan(url, identifier, conn, stop)
	if err != nil {
		// Discard MULTI
		s.ScannerDiscard()

		// Remove the temporary key
		conn.Do("DEL", s.filesTmpKey)

		log.Errorf("[%s] %s", identifier, err.Error())
		return err
	}

	// Exec multi
	s.ScannerCommit()

	// Get the list of files no more present on this mirror
	toremove, err := redis.Values(conn.Do("SDIFF", s.filesKey, s.filesTmpKey))
	if err != nil {
		return err
	}

	// Remove this mirror from the given file SET
	if len(toremove) > 0 {
		conn.Send("MULTI")
		for _, e := range toremove {
			log.Debugf("[%s] Removing %s from mirror", identifier, e)
			conn.Send("SREM", fmt.Sprintf("FILEMIRRORS_%s", e), identifier)
			conn.Send("DEL", fmt.Sprintf("FILEINFO_%s_%s", identifier, e))
			// Publish update
			database.SendPublish(conn, database.MIRROR_FILE_UPDATE, fmt.Sprintf("%s %s", identifier, e))

		}
		_, err = conn.Do("EXEC")
		if err != nil {
			return err
		}
	}

	// Finally rename the temporary sets containing the list
	// of files for this mirror to the production key
	_, err = conn.Do("RENAME", s.filesTmpKey, s.filesKey)
	if err != nil {
		return err
	}

	sinterKey := fmt.Sprintf("HANDLEDFILES_%s", identifier)

	// Count the number of files known on the remote end
	common, _ := redis.Int64(conn.Do("SINTERSTORE", sinterKey, "FILES", s.filesKey))

	if err != nil {
		return err
	}

	s.setLastSync(conn, identifier, true)
	log.Infof("[%s] Indexed %d files (%d known), %d removed", identifier, s.count, common, len(toremove))
	return nil
}

func (s *scan) ScannerAddFile(f filedata) {
	s.count++

	// Add all the files to a temporary key
	s.conn.Send("SADD", s.filesTmpKey, f.path)

	// Mark the file as being supported by this mirror
	rk := fmt.Sprintf("FILEMIRRORS_%s", f.path)
	s.conn.Send("SADD", rk, s.identifier)

	// Save the size of the current file found on this mirror
	ik := fmt.Sprintf("FILEINFO_%s_%s", s.identifier, f.path)
	s.conn.Send("HSET", ik, "size", f.size)

	// Publish update
	database.SendPublish(s.conn, database.MIRROR_FILE_UPDATE, fmt.Sprintf("%s %s", s.identifier, f.path))
}

func (s *scan) ScannerDiscard() {
	s.conn.Do("DISCARD")
}

func (s *scan) ScannerCommit() error {
	_, err := s.conn.Do("EXEC")
	return err
}

func (s *scan) setLastSync(conn redis.Conn, identifier string, successful bool) error {
	now := time.Now().UTC().Unix()

	conn.Send("MULTI")

	// Set the last sync time
	conn.Send("HSET", fmt.Sprintf("MIRROR_%s", identifier), "lastSync", now)

	// Set the last successful sync time
	if successful {
		conn.Send("HSET", fmt.Sprintf("MIRROR_%s", identifier), "lastSuccessfulSync", now)
	}

	_, err := conn.Do("EXEC")

	// Publish an update on redis
	database.Publish(conn, database.MIRROR_UPDATE, identifier)

	return err
}

// Walk inside the source/reference repository
func (s *scan) walkSource(conn redis.Conn, path string, f os.FileInfo, err error) (*filedata, error) {
	if f == nil || f.IsDir() || f.Mode()&os.ModeSymlink != 0 {
		return nil, nil
	}

	d := new(filedata)
	d.path = path[len(GetConfig().Repository):]
	d.size = f.Size()
	d.modTime = f.ModTime()

	// Get the previous file properties
	properties, err := redis.Strings(conn.Do("HMGET", fmt.Sprintf("FILE_%s", d.path), "size", "modTime", "sha1", "sha256", "md5"))
	if err != nil && err != redis.ErrNil {
		return nil, err
	} else if len(properties) < 5 {
		// This will force a rehash
		properties = make([]string, 5)
	}

	size, _ := strconv.ParseInt(properties[0], 10, 64)
	modTime, _ := time.Parse("2006-01-02 15:04:05.999999999 -0700 MST", properties[1])
	sha1 := properties[2]
	sha256 := properties[3]
	md5 := properties[4]

	rehash := (GetConfig().Hashes.SHA1 && len(sha1) == 0) ||
		(GetConfig().Hashes.SHA256 && len(sha256) == 0) ||
		(GetConfig().Hashes.MD5 && len(md5) == 0)

	if rehash || size != d.size || !modTime.Equal(d.modTime) {
		h, err := filesystem.HashFile(GetConfig().Repository + d.path)
		if err != nil {
			log.Warningf("%s: hashing failed: %s", d.path, err.Error())
		} else {
			d.sha1 = h.Sha1
			d.sha256 = h.Sha256
			d.md5 = h.Md5
			if len(d.sha1) > 0 {
				log.Infof("%s: SHA1 %s", d.path, d.sha1)
			}
			if len(d.sha256) > 0 {
				log.Infof("%s: SHA256 %s", d.path, d.sha256)
			}
			if len(d.md5) > 0 {
				log.Infof("%s: MD5 %s", d.path, d.md5)
			}
		}
	} else {
		d.sha1 = sha1
		d.sha256 = sha256
		d.md5 = md5
	}

	return d, nil
}

func ScanSource(r *database.Redis, stop chan bool) (err error) {
	s := &scan{
		redis: r,
	}

	conn := s.redis.Get()
	defer conn.Close()

	if conn.Err() != nil {
		return conn.Err()
	}

	sourceFiles := make([]*filedata, 0, 1000)

	//TODO lock atomically inside redis to avoid two simultanous scan

	if _, err := os.Stat(GetConfig().Repository); os.IsNotExist(err) {
		return fmt.Errorf("%s: No such file or directory", GetConfig().Repository)
	}

	log.Info("[source] Scanning the filesystem...")
	err = filepath.Walk(GetConfig().Repository, func(path string, f os.FileInfo, err error) error {
		fd, err := s.walkSource(conn, path, f, err)
		if err != nil {
			return err
		}
		if fd != nil {
			sourceFiles = append(sourceFiles, fd)
		}
		return nil
	})

	if utils.IsStopped(stop) {
		return ScanAborted
	}
	if err != nil {
		return err
	}
	log.Info("[source] Indexing the files...")

	conn.Send("MULTI")

	// Remove any left over
	conn.Send("DEL", "FILES_TMP")

	// Add all the files to a temporary key
	count := 0
	for _, e := range sourceFiles {
		conn.Send("SADD", "FILES_TMP", e.path)
		count++
	}

	_, err = conn.Do("EXEC")
	if err != nil {
		return err
	}

	// Do a diff between the sets to get the removed files
	toremove, err := redis.Values(conn.Do("SDIFF", "FILES", "FILES_TMP"))

	// Create/Update the files' hash keys with the fresh infos
	conn.Send("MULTI")
	for _, e := range sourceFiles {
		conn.Send("HMSET", fmt.Sprintf("FILE_%s", e.path),
			"size", e.size,
			"modTime", e.modTime,
			"sha1", e.sha1,
			"sha256", e.sha256,
			"md5", e.md5)

		// Publish update
		database.SendPublish(conn, database.FILE_UPDATE, e.path)
	}

	// Remove old keys
	if len(toremove) > 0 {
		for _, e := range toremove {
			conn.Send("DEL", fmt.Sprintf("FILE_%s", e))

			// Publish update
			database.SendPublish(conn, database.FILE_UPDATE, fmt.Sprintf("%s", e))
		}
	}

	// Finally rename the temporary sets containing the list
	// of files to the production key
	conn.Send("RENAME", "FILES_TMP", "FILES")

	_, err = conn.Do("EXEC")
	if err != nil {
		return err
	}

	log.Infof("[source] Scanned %d files", count)

	return nil
}
<commit_msg>scan: lock before pushing repository updates<commit_after>// Copyright (c) 2014-2017 Ludovic Fauvet
// Licensed under the MIT license

package scan

import (
	"errors"
	"fmt"
	. "github.com/etix/mirrorbits/config"
	"github.com/etix/mirrorbits/database"
	"github.com/etix/mirrorbits/filesystem"
	"github.com/etix/mirrorbits/network"
	"github.com/etix/mirrorbits/utils"
	"github.com/garyburd/redigo/redis"
	"github.com/op/go-logging"
	"os"
	"path/filepath"
	"strconv"
	"time"
)

var (
	ScanAborted    = errors.New("scan aborted")
	ScanInProgress = errors.New("scan already in progress")

	log = logging.MustGetLogger("main")
)

type ScannerType int8

const (
	RSYNC ScannerType = iota
	FTP
)

type Scanner interface {
	Scan(url, identifier string, conn redis.Conn, stop chan bool) error
}

type filedata struct {
	path    string
	sha1    string
	sha256  string
	md5     string
	size    int64
	modTime time.Time
}

type scan struct {
	redis *database.Redis

	conn        redis.Conn
	identifier  string
	filesKey    string
	filesTmpKey string
	count       uint
}

func IsScanning(conn redis.Conn, identifier string) (bool, error) {
	return redis.Bool(conn.Do("EXISTS", fmt.Sprintf("SCANNING_%s", identifier)))
}

func Scan(typ ScannerType, r *database.Redis, url, identifier string, stop chan bool) error {
	s := &scan{
		redis:      r,
		identifier: identifier,
	}

	var scanner Scanner
	switch typ {
	case RSYNC:
		scanner = &RsyncScanner{
			scan: s,
		}
	case FTP:
		scanner = &FTPScanner{
			scan: s,
		}
	default:
		panic(fmt.Sprintf("Unknown scanner"))
	}

	// Connect to the database
	conn := s.redis.Get()
	defer conn.Close()

	s.conn = conn

	// Try to aquire a lock so we don't have a scanning race
	// from different nodes.
	// Also make the key expire automatically in case our process
	// gets killed.
	lock := network.NewClusterLock(s.redis, fmt.Sprintf("SCANNING_%s", identifier), identifier)

	done, err := lock.Get()
	if err != nil {
		return err
	} else if done == nil {
		return ScanInProgress
	}

	defer lock.Release()

	s.setLastSync(conn, identifier, false)

	conn.Send("MULTI")

	s.filesKey = fmt.Sprintf("MIRROR_%s_FILES", identifier)
	s.filesTmpKey = fmt.Sprintf("MIRROR_%s_FILES_TMP", identifier)

	// Remove any left over
	conn.Send("DEL", s.filesTmpKey)

	err = scanner.Scan(url, identifier, conn, stop)
	if err != nil {
		// Discard MULTI
		s.ScannerDiscard()

		// Remove the temporary key
		conn.Do("DEL", s.filesTmpKey)

		log.Errorf("[%s] %s", identifier, err.Error())
		return err
	}

	// Exec multi
	s.ScannerCommit()

	// Get the list of files no more present on this mirror
	toremove, err := redis.Values(conn.Do("SDIFF", s.filesKey, s.filesTmpKey))
	if err != nil {
		return err
	}

	// Remove this mirror from the given file SET
	if len(toremove) > 0 {
		conn.Send("MULTI")
		for _, e := range toremove {
			log.Debugf("[%s] Removing %s from mirror", identifier, e)
			conn.Send("SREM", fmt.Sprintf("FILEMIRRORS_%s", e), identifier)
			conn.Send("DEL", fmt.Sprintf("FILEINFO_%s_%s", identifier, e))
			// Publish update
			database.SendPublish(conn, database.MIRROR_FILE_UPDATE, fmt.Sprintf("%s %s", identifier, e))

		}
		_, err = conn.Do("EXEC")
		if err != nil {
			return err
		}
	}

	// Finally rename the temporary sets containing the list
	// of files for this mirror to the production key
	_, err = conn.Do("RENAME", s.filesTmpKey, s.filesKey)
	if err != nil {
		return err
	}

	sinterKey := fmt.Sprintf("HANDLEDFILES_%s", identifier)

	// Count the number of files known on the remote end
	common, _ := redis.Int64(conn.Do("SINTERSTORE", sinterKey, "FILES", s.filesKey))

	if err != nil {
		return err
	}

	s.setLastSync(conn, identifier, true)
	log.Infof("[%s] Indexed %d files (%d known), %d removed", identifier, s.count, common, len(toremove))
	return nil
}

func (s *scan) ScannerAddFile(f filedata) {
	s.count++

	// Add all the files to a temporary key
	s.conn.Send("SADD", s.filesTmpKey, f.path)

	// Mark the file as being supported by this mirror
	rk := fmt.Sprintf("FILEMIRRORS_%s", f.path)
	s.conn.Send("SADD", rk, s.identifier)

	// Save the size of the current file found on this mirror
	ik := fmt.Sprintf("FILEINFO_%s_%s", s.identifier, f.path)
	s.conn.Send("HSET", ik, "size", f.size)

	// Publish update
	database.SendPublish(s.conn, database.MIRROR_FILE_UPDATE, fmt.Sprintf("%s %s", s.identifier, f.path))
}

func (s *scan) ScannerDiscard() {
	s.conn.Do("DISCARD")
}

func (s *scan) ScannerCommit() error {
	_, err := s.conn.Do("EXEC")
	return err
}

func (s *scan) setLastSync(conn redis.Conn, identifier string, successful bool) error {
	now := time.Now().UTC().Unix()

	conn.Send("MULTI")

	// Set the last sync time
	conn.Send("HSET", fmt.Sprintf("MIRROR_%s", identifier), "lastSync", now)

	// Set the last successful sync time
	if successful {
		conn.Send("HSET", fmt.Sprintf("MIRROR_%s", identifier), "lastSuccessfulSync", now)
	}

	_, err := conn.Do("EXEC")

	// Publish an update on redis
	database.Publish(conn, database.MIRROR_UPDATE, identifier)

	return err
}

// Walk inside the source/reference repository
func (s *scan) walkSource(conn redis.Conn, path string, f os.FileInfo, err error) (*filedata, error) {
	if f == nil || f.IsDir() || f.Mode()&os.ModeSymlink != 0 {
		return nil, nil
	}

	d := new(filedata)
	d.path = path[len(GetConfig().Repository):]
	d.size = f.Size()
	d.modTime = f.ModTime()

	// Get the previous file properties
	properties, err := redis.Strings(conn.Do("HMGET", fmt.Sprintf("FILE_%s", d.path), "size", "modTime", "sha1", "sha256", "md5"))
	if err != nil && err != redis.ErrNil {
		return nil, err
	} else if len(properties) < 5 {
		// This will force a rehash
		properties = make([]string, 5)
	}

	size, _ := strconv.ParseInt(properties[0], 10, 64)
	modTime, _ := time.Parse("2006-01-02 15:04:05.999999999 -0700 MST", properties[1])
	sha1 := properties[2]
	sha256 := properties[3]
	md5 := properties[4]

	rehash := (GetConfig().Hashes.SHA1 && len(sha1) == 0) ||
		(GetConfig().Hashes.SHA256 && len(sha256) == 0) ||
		(GetConfig().Hashes.MD5 && len(md5) == 0)

	if rehash || size != d.size || !modTime.Equal(d.modTime) {
		h, err := filesystem.HashFile(GetConfig().Repository + d.path)
		if err != nil {
			log.Warningf("%s: hashing failed: %s", d.path, err.Error())
		} else {
			d.sha1 = h.Sha1
			d.sha256 = h.Sha256
			d.md5 = h.Md5
			if len(d.sha1) > 0 {
				log.Infof("%s: SHA1 %s", d.path, d.sha1)
			}
			if len(d.sha256) > 0 {
				log.Infof("%s: SHA256 %s", d.path, d.sha256)
			}
			if len(d.md5) > 0 {
				log.Infof("%s: MD5 %s", d.path, d.md5)
			}
		}
	} else {
		d.sha1 = sha1
		d.sha256 = sha256
		d.md5 = md5
	}

	return d, nil
}

func ScanSource(r *database.Redis, stop chan bool) (err error) {
	s := &scan{
		redis: r,
	}

	conn := s.redis.Get()
	defer conn.Close()

	if conn.Err() != nil {
		return conn.Err()
	}

	sourceFiles := make([]*filedata, 0, 1000)

	//TODO lock atomically inside redis to avoid two simultanous scan

	if _, err := os.Stat(GetConfig().Repository); os.IsNotExist(err) {
		return fmt.Errorf("%s: No such file or directory", GetConfig().Repository)
	}

	log.Info("[source] Scanning the filesystem...")
	err = filepath.Walk(GetConfig().Repository, func(path string, f os.FileInfo, err error) error {
		fd, err := s.walkSource(conn, path, f, err)
		if err != nil {
			return err
		}
		if fd != nil {
			sourceFiles = append(sourceFiles, fd)
		}
		return nil
	})

	if utils.IsStopped(stop) {
		return ScanAborted
	}
	if err != nil {
		return err
	}
	log.Info("[source] Indexing the files...")

	lock := network.NewClusterLock(s.redis, "SOURCE_REPO_SYNC", "source repository")

	retry := 10
	for {
		if retry == 0 {
			return ScanInProgress
		}
		done, err := lock.Get()
		if err != nil {
			return err
		} else if done != nil {
			break
		}
		time.Sleep(1 * time.Second)
		retry--
	}

	defer lock.Release()

	conn.Send("MULTI")

	// Remove any left over
	conn.Send("DEL", "FILES_TMP")

	// Add all the files to a temporary key
	count := 0
	for _, e := range sourceFiles {
		conn.Send("SADD", "FILES_TMP", e.path)
		count++
	}

	_, err = conn.Do("EXEC")
	if err != nil {
		return err
	}

	// Do a diff between the sets to get the removed files
	toremove, err := redis.Values(conn.Do("SDIFF", "FILES", "FILES_TMP"))

	// Create/Update the files' hash keys with the fresh infos
	conn.Send("MULTI")
	for _, e := range sourceFiles {
		conn.Send("HMSET", fmt.Sprintf("FILE_%s", e.path),
			"size", e.size,
			"modTime", e.modTime,
			"sha1", e.sha1,
			"sha256", e.sha256,
			"md5", e.md5)

		// Publish update
		database.SendPublish(conn, database.FILE_UPDATE, e.path)
	}

	// Remove old keys
	if len(toremove) > 0 {
		for _, e := range toremove {
			conn.Send("DEL", fmt.Sprintf("FILE_%s", e))

			// Publish update
			database.SendPublish(conn, database.FILE_UPDATE, fmt.Sprintf("%s", e))
		}
	}

	// Finally rename the temporary sets containing the list
	// of files to the production key
	conn.Send("RENAME", "FILES_TMP", "FILES")

	_, err = conn.Do("EXEC")
	if err != nil {
		return err
	}

	log.Infof("[source] Scanned %d files", count)

	return nil
}
<|endoftext|>


<commit_before>package chevalier

import (
	"crypto/sha1"
	"encoding/base64"
	"fmt"
	"github.com/mattbaird/elastigo/api"
	es "github.com/mattbaird/elastigo/core"
	"strings"
)

type ElasticsearchSource struct {
	Origin string
	Source map[string]string `json:"source"`
}

// GetID returns a (probably) unique ID for an ElasticsearchSource, in
// the form of a sha1 hash of underscore-separated field-value pairs
// separated by newlines.
func (s *ElasticsearchSource) GetID() string {
	tagKeys := make([]string, len(s.Source))
	idx := 0
	for field, value := range s.Source {
		tagKeys[idx] = fmt.Sprintf("%s_%s", field, value)
		idx++
	}
	key := []byte(strings.Join(tagKeys, "\n"))
	hash := sha1.Sum(key)
	id := base64.StdEncoding.EncodeToString(hash[:sha1.Size])
	return id
}

func NewElasticsearchSource(origin string, source *DataSource) *ElasticsearchSource {
	esSource := new(ElasticsearchSource)
	esSource.Origin = origin
	esSource.Source = make(map[string]string, 0)
	for _, tagPtr := range source.Source {
		esSource.Source[*tagPtr.Field] = *tagPtr.Value
	}
	return esSource
}

// Unmarshal turns an ElasticsearchSource (presumably itself unmarshaled
// from a JSON object stored in Elasticsearch) into the equivalent
// DataSource.
func (s *ElasticsearchSource) Unmarshal() *DataSource {
	tags := make([]*DataSource_Tag, len(s.Source))
	idx := 0
	for field, value := range s.Source {
		tags[idx] = NewDataSourceTag(field, value)
		idx++
	}
	pb := NewDataSource(tags)
	return pb
}

func MarshalElasticsearchSources(origin string, b *DataSourceBurst) []*ElasticsearchSource {
	sources := make([]*ElasticsearchSource, len(b.Sources))
	for i, s := range b.Sources {
		esSource := NewElasticsearchSource(origin, s)
		sources[i] = esSource
	}
	return sources
}

// ElasticsearchWriter maintains context for writes to the index.
type ElasticsearchWriter struct {
	indexer   *es.BulkIndexer
	indexName string
	dataType  string
	done      chan bool
}

// NewElasticsearchWriter builds a new Writer. retrySeconds is for the
// bulk indexer. index and dataType can be anything as long as they're
// consistent.
func NewElasticsearchWriter(host string, maxConns int, retrySeconds int, index, dataType string) *ElasticsearchWriter {
	writer := new(ElasticsearchWriter)
	api.Domain = host
	writer.indexer = es.NewBulkIndexerErrors(maxConns, retrySeconds)
	writer.indexName = index
	writer.dataType = dataType
	writer.done = make(chan bool)
	writer.indexer.Run(writer.done)
	return writer
}

// Write queues a DataSource for writing by the bulk indexer.
// Non-blocking.
func (w *ElasticsearchWriter) Write(origin string, source *DataSource) error {
	esSource := NewElasticsearchSource(origin, source)
	update := map[string]interface{} {
		"doc" : esSource,
		"doc_as_upsert" : true,
	}
	err := w.indexer.Update(w.indexName, w.dataType, esSource.GetID(), "", nil, update)
	return err
}

// Shutdown signals the bulk indexer to flush all pending writes.
func (w *ElasticsearchWriter) Shutdown() {
	w.done <- true
}

// GetErrorChan returns the channel the bulk indexer writes errors to.
func (w *ElasticsearchWriter) GetErrorChan() chan *es.ErrorBuffer {
	return w.indexer.ErrorChannel
}
<commit_msg>Include origin in ES ID<commit_after>package chevalier

import (
	"crypto/sha1"
	"encoding/base64"
	"fmt"
	"github.com/mattbaird/elastigo/api"
	es "github.com/mattbaird/elastigo/core"
	"strings"
)

type ElasticsearchSource struct {
	Origin string
	Source map[string]string `json:"source"`
}

// GetID returns a (probably) unique ID for an ElasticsearchSource, in
// the form of a sha1 hash of underscore-separated field-value pairs
// separated by newlines.
func (s *ElasticsearchSource) GetID() string {
	tagKeys := make([]string, len(s.Source) + 1)
	idx := 0
	for field, value := range s.Source {
		tagKeys[idx] = fmt.Sprintf("%s_%s", field, value)
		idx++
	}
	tagKeys[idx] = fmt.Sprintf("Origin", s.Origin)
	key := []byte(strings.Join(tagKeys, "\n"))
	hash := sha1.Sum(key)
	id := base64.StdEncoding.EncodeToString(hash[:sha1.Size])
	return id
}

func NewElasticsearchSource(origin string, source *DataSource) *ElasticsearchSource {
	esSource := new(ElasticsearchSource)
	esSource.Origin = origin
	esSource.Source = make(map[string]string, 0)
	for _, tagPtr := range source.Source {
		esSource.Source[*tagPtr.Field] = *tagPtr.Value
	}
	return esSource
}

// Unmarshal turns an ElasticsearchSource (presumably itself unmarshaled
// from a JSON object stored in Elasticsearch) into the equivalent
// DataSource.
func (s *ElasticsearchSource) Unmarshal() *DataSource {
	tags := make([]*DataSource_Tag, len(s.Source))
	idx := 0
	for field, value := range s.Source {
		tags[idx] = NewDataSourceTag(field, value)
		idx++
	}
	pb := NewDataSource(tags)
	return pb
}

func MarshalElasticsearchSources(origin string, b *DataSourceBurst) []*ElasticsearchSource {
	sources := make([]*ElasticsearchSource, len(b.Sources))
	for i, s := range b.Sources {
		esSource := NewElasticsearchSource(origin, s)
		sources[i] = esSource
	}
	return sources
}

// ElasticsearchWriter maintains context for writes to the index.
type ElasticsearchWriter struct {
	indexer   *es.BulkIndexer
	indexName string
	dataType  string
	done      chan bool
}

// NewElasticsearchWriter builds a new Writer. retrySeconds is for the
// bulk indexer. index and dataType can be anything as long as they're
// consistent.
func NewElasticsearchWriter(host string, maxConns int, retrySeconds int, index, dataType string) *ElasticsearchWriter {
	writer := new(ElasticsearchWriter)
	api.Domain = host
	writer.indexer = es.NewBulkIndexerErrors(maxConns, retrySeconds)
	writer.indexName = index
	writer.dataType = dataType
	writer.done = make(chan bool)
	writer.indexer.Run(writer.done)
	return writer
}

// Write queues a DataSource for writing by the bulk indexer.
// Non-blocking.
func (w *ElasticsearchWriter) Write(origin string, source *DataSource) error {
	esSource := NewElasticsearchSource(origin, source)
	update := map[string]interface{} {
		"doc" : esSource,
		"doc_as_upsert" : true,
	}
	err := w.indexer.Update(w.indexName, w.dataType, esSource.GetID(), "", nil, update)
	return err
}

// Shutdown signals the bulk indexer to flush all pending writes.
func (w *ElasticsearchWriter) Shutdown() {
	w.done <- true
}

// GetErrorChan returns the channel the bulk indexer writes errors to.
func (w *ElasticsearchWriter) GetErrorChan() chan *es.ErrorBuffer {
	return w.indexer.ErrorChannel
}
<|endoftext|>


<commit_before>package whois

import (
	"net/url"
	"strings"
)

var az = &Server{
	Resolve: func(req *Request) error {
		labels := strings.SplitN(req.Query, ".", 2)
		values := url.Values{}
		values.Set("lang", "en")
		values.Set("domain", labels[0])
		values.Set("dom", labels[1])
		req.URL = "http://www.whois.az/cgi-bin/whois.cgi"
		req.Body = values.Encode()
		req.Referer = "http://www.whois.az/"
		return nil
	},
}

func init() {
	register(
		az,
		"www.whois.az",
		"www.nic.az",
	)
}
<commit_msg>Whoops.<commit_after>package whois

import (
	"net/url"
	"strings"
)

var az = &Server{
	Resolve: func(req *Request) error {
		labels := strings.SplitN(req.Query, ".", 2)
		values := url.Values{}
		values.Set("lang", "en")
		values.Set("domain", labels[0])
		values.Set("dom", labels[1])
		req.URL = "http://www.whois.az/cgi-bin/whois.cgi"
		req.Body = values.Encode()
		return nil
	},
}

func init() {
	register(
		az,
		"www.whois.az",
		"www.nic.az",
	)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"log"
	"os"
	"os/signal"
	"strings"
	"syscall"

	"github.com/fatih/color"
	pb "github.com/taylorflatt/go-chat"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
)

// ControlExit handles any interrupts during program execution.
// Note: The routine control is dictated by the existence of a stream. If one is present, the user is in a group and needs
// to be removed. Otherwise, the user is still in the menu system.
// It doesn't return anything.
func ControlExit(w chan os.Signal, c pb.ChatClient, q chan bool, stream pb.Chat_RouteChatClient, u string, g string) {

	signal.Notify(w, syscall.SIGINT, syscall.SIGTERM)

	for {
		select {
		case sig := <-w:
			if sig == os.Interrupt {
				if stream != nil {
					stream.Send(&pb.ChatMessage{Sender: u, Receiver: g, Message: u + " left chat!\n"})
					ExitChat(c, stream, u, g)
					return
				}

				c.UnRegister(context.Background(), &pb.ClientInfo{Sender: u})
				os.Exit(1)
				return
			}
		case stop := <-q:
			if stop { // Stop thread.
				return
			}
		}
	}
}

// ExitChat handles removing the client from the server and exiting the program.
// It doesn't return anything.
func ExitChat(c pb.ChatClient, stream pb.Chat_RouteChatClient, u string, g string) {

	c.UnRegister(context.Background(), &pb.ClientInfo{Sender: u})
	os.Exit(1)
}

// ListenToClient listens to the client for input and adds that input to the sQueue with
// the username of the sender, group name, and the message.
// It doesn't return anything.
func ListenToClient(sQueue chan pb.ChatMessage, reader *bufio.Reader, uName string, gName string) {

	for {
		// TODO: When a user joins the channel, it effectively appends that join message to the You>.
		// 		 Need to find a way to get around this issue.
		//fmt.Print("You> ")
		msg, _ := reader.ReadString('\n')
		if strings.TrimSpace(msg) == "!leave" {
			sQueue <- pb.ChatMessage{Sender: uName, Message: msg, Receiver: gName}
			return
		}
		sQueue <- pb.ChatMessage{Sender: uName, Message: msg, Receiver: gName}
	}
}

// ReceiveMessages listens on the client's (NOT the client's group) stream and adds any incoming
// message to the client's inbox.
// It doesn't return anything.
func ReceiveMessages(stream pb.Chat_RouteChatClient, inbox chan pb.ChatMessage, u string) {

	var test bool
	for {
		if test {
			return
		}

		msg, _ := stream.Recv()
		if strings.TrimSpace(msg.Message) == u+" left chat!\n" {
			test = true
		}
		inbox <- *msg

	}
}

// DisplayCurrentMembers displays the members who are currently in the group chat.
// It doesn't return anything.
func DisplayCurrentMembers(c pb.ChatClient, g string) {

	m, _ := c.GetGroupClientList(context.Background(), &pb.GroupInfo{GroupName: g})
	if len(m.Clients) > 0 {
		fmt.Print("Current Members: ")
		for i := 0; i < len(m.Clients); i++ {
			if i == len(m.Clients)-1 {
				fmt.Print(m.Clients[i])
			} else {
				fmt.Print(m.Clients[i] + ", ")
			}
		}
		AddSpacing(2)
	}
}

func main() {

	r := bufio.NewReader(os.Stdin)

	var uName string // Client username
	var gName string // Client's chat group

	a := SetServer(r)

	// Set up a connection to the server.
	conn, err := grpc.Dial(a, grpc.WithInsecure())

	if err != nil {
		log.Fatalf("Could not connect: %v", err)
	} else {
		fmt.Printf("\nYou have successfully connected to %s! To disconnect, hit ctrl+c or type !exit.\n\n", a)
	}

	// Close the connection after main returns.
	defer conn.Close()

	// Create the client
	c := pb.NewChatClient(conn)

	uName = SetName(c, r)
	inMenu := true               // Control whether the user sees the menu or exits.
	w := make(chan os.Signal, 1) // Watch for ctrl+c
	q := make(chan bool)         // Watch for which exit routine to run.

	go ControlExit(w, c, q, nil, uName, gName)

	for inMenu {
		gName, err = TopMenu(c, r, uName)

		if err != nil {
			fmt.Print(err)
			os.Exit(1)
		}

		inMenu = false
		AddSpacing(1)
		fmt.Println("You are now chatting in " + gName + ".")
		Frame()

		stream, serr := c.RouteChat(context.Background())
		q <- true // Stop the old thread and start a new one with the stream passed in.
		go ControlExit(w, c, q, stream, uName, gName)

		// TODO: Find out why the first message is always dropped so an empty message needn't be sent.
		stream.Send(&pb.ChatMessage{Sender: uName, Receiver: gName, Message: ""})
		stream.Send(&pb.ChatMessage{Sender: uName, Receiver: gName, Message: "joined chat!\n"})

		DisplayCurrentMembers(c, gName)

		if serr != nil {
			fmt.Print(serr)
		} else {

			sQueue := make(chan pb.ChatMessage, 100)
			go ListenToClient(sQueue, r, uName, gName)

			inbox := make(chan pb.ChatMessage, 100)
			go ReceiveMessages(stream, inbox, uName)

			for !inMenu {
				select {
				case toSend := <-sQueue:
					switch msg := strings.TrimSpace(toSend.Message); msg {
					case "!members":
						DisplayCurrentMembers(c, gName)
					case "!leave":
						stream.Send(&pb.ChatMessage{Sender: uName, Receiver: gName, Message: uName + " left chat!\n"})
						c.LeaveRoom(context.Background(), &pb.GroupInfo{Client: uName, GroupName: gName})
						stream.CloseSend()
						inMenu = true
					case "!exit":
						stream.Send(&pb.ChatMessage{Sender: uName, Receiver: gName, Message: uName + " left chat!\n"})
						ExitChat(c, stream, uName, gName)
						stream.CloseSend()
						conn.Close()
					case "!help":
						AddSpacing(1)
						fmt.Println("The following commands are available to you: ")
						color.New(color.FgHiYellow).Print("   !members")
						fmt.Print(": Lists the current members in the group.")

						AddSpacing(1)
						color.New(color.FgHiYellow).Print("   !exit")
						fmt.Println(": Leaves the chat server.")
						AddSpacing(1)

					default:
						stream.Send(&toSend)
					}
				case received := <-inbox:
					fmt.Printf("%s> %s", received.Sender, received.Message)
				}
			}
		}
	}
}
<commit_msg>Added debug comments.<commit_after>package main

import (
	"bufio"
	"fmt"
	"log"
	"os"
	"os/signal"
	"strings"
	"syscall"

	"github.com/fatih/color"
	pb "github.com/taylorflatt/go-chat"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
)

// ControlExit handles any interrupts during program execution.
// Note: The routine control is dictated by the existence of a stream. If one is present, the user is in a group and needs
// to be removed. Otherwise, the user is still in the menu system.
// It doesn't return anything.
func ControlExit(w chan os.Signal, c pb.ChatClient, q chan bool, stream pb.Chat_RouteChatClient, u string, g string) {

	signal.Notify(w, syscall.SIGINT, syscall.SIGTERM)

	for {
		select {
		case sig := <-w:
			if sig == os.Interrupt {
				if stream != nil {
					stream.Send(&pb.ChatMessage{Sender: u, Receiver: g, Message: u + " left chat!\n"})
					ExitChat(c, stream, u, g)
					return
				}

				c.UnRegister(context.Background(), &pb.ClientInfo{Sender: u})
				os.Exit(1)
				return
			}
		case stop := <-q:
			if stop { // Stop thread.
				return
			}
		}
	}
}

// ExitChat handles removing the client from the server and exiting the program.
// It doesn't return anything.
func ExitChat(c pb.ChatClient, stream pb.Chat_RouteChatClient, u string, g string) {

	c.UnRegister(context.Background(), &pb.ClientInfo{Sender: u})
	os.Exit(1)
}

// ListenToClient listens to the client for input and adds that input to the sQueue with
// the username of the sender, group name, and the message.
// It doesn't return anything.
func ListenToClient(sQueue chan pb.ChatMessage, reader *bufio.Reader, uName string, gName string) {

	log.Println("[ListenToClient]: Starting.")

	for {
		// TODO: When a user joins the channel, it effectively appends that join message to the You>.
		// 		 Need to find a way to get around this issue.
		//fmt.Print("You> ")
		msg, _ := reader.ReadString('\n')
		if strings.TrimSpace(msg) == "!leave" {
			log.Println("[ListenToClient]: Stopping.")
			sQueue <- pb.ChatMessage{Sender: uName, Message: msg, Receiver: gName}
			return
		}
		log.Println("[ListenToClient]: Adding message to send queue.")

		sQueue <- pb.ChatMessage{Sender: uName, Message: msg, Receiver: gName}
	}
}

// ReceiveMessages listens on the client's (NOT the client's group) stream and adds any incoming
// message to the client's inbox.
// It doesn't return anything.
func ReceiveMessages(stream pb.Chat_RouteChatClient, inbox chan pb.ChatMessage, u string) {

	log.Println("[ReceiveMessages]: Starting.")

	var test bool
	for {
		if test {
			log.Println("[ReceiveMessages]: Stopping.")
			return
		}

		log.Println("[ReceiveMessages]: Listening for incoming messages.")
		msg, _ := stream.Recv()
		log.Println("[ReceiveMessages]: I see " + msg.Message)
		if msg.Message == "!leave" {
			log.Println("[ReceiveMessages]: Found special signal!")
			test = true
		}
		inbox <- *msg
	}

	//for {
	//	log.Println("[ReceiveMessages]: Listening for incoming messages.")
	//	msg, _ := stream.Recv()
	//	log.Println("[ReceiveMessages]: Found " + msg.Message + " from " + msg.Sender)
	//	inbox <- *msg
	//}
}

// DisplayCurrentMembers displays the members who are currently in the group chat.
// It doesn't return anything.
func DisplayCurrentMembers(c pb.ChatClient, g string) {

	m, _ := c.GetGroupClientList(context.Background(), &pb.GroupInfo{GroupName: g})
	if len(m.Clients) > 0 {
		fmt.Print("Current Members: ")
		for i := 0; i < len(m.Clients); i++ {
			if i == len(m.Clients)-1 {
				fmt.Print(m.Clients[i])
			} else {
				fmt.Print(m.Clients[i] + ", ")
			}
		}
		AddSpacing(2)
	}
}

func main() {

	r := bufio.NewReader(os.Stdin)

	var uName string // Client username
	var gName string // Client's chat group

	a := SetServer(r)

	// Set up a connection to the server.
	conn, err := grpc.Dial(a, grpc.WithInsecure())

	if err != nil {
		log.Fatalf("Could not connect: %v", err)
	} else {
		fmt.Printf("\nYou have successfully connected to %s! To disconnect, hit ctrl+c or type !exit.\n\n", a)
	}

	// Close the connection after main returns.
	defer conn.Close()

	// Create the client
	c := pb.NewChatClient(conn)

	uName = SetName(c, r)
	inMenu := true               // Control whether the user sees the menu or exits.
	w := make(chan os.Signal, 1) // Watch for ctrl+c
	q := make(chan bool)         // Watch for which exit routine to run.
	f := true                    // Signals for space allocation for the channels.
	var sQueue chan pb.ChatMessage
	var inbox chan pb.ChatMessage

	go ControlExit(w, c, q, nil, uName, gName)

	for inMenu {
		gName, err = TopMenu(c, r, uName)

		if err != nil {
			fmt.Print(err)
			os.Exit(1)
		}

		inMenu = false
		AddSpacing(1)
		fmt.Println("You are now chatting in " + gName + ".")
		Frame()

		stream, serr := c.RouteChat(context.Background())
		q <- true // Stop the old thread and start a new one with the stream passed in.
		go ControlExit(w, c, q, stream, uName, gName)

		// TODO: Find out why the first message is always dropped so an empty message needn't be sent.
		stream.Send(&pb.ChatMessage{Sender: uName, Receiver: gName, Message: ""})
		stream.Send(&pb.ChatMessage{Sender: uName, Receiver: gName, Message: "joined chat!\n"})

		DisplayCurrentMembers(c, gName)

		if serr != nil {
			fmt.Print(serr)
		} else {
			if f {
				sQueue = make(chan pb.ChatMessage, 100)
				inbox = make(chan pb.ChatMessage, 100)
			}

			go ListenToClient(sQueue, r, uName, gName)
			go ReceiveMessages(stream, inbox, uName)

			for !inMenu {
				log.Println("[Main]: Begin for-loop.")
				select {
				case toSend := <-sQueue:
					switch msg := strings.TrimSpace(toSend.Message); msg {
					case "!members":
						log.Println("[Main]: Stuck in !members.")
						DisplayCurrentMembers(c, gName)
					//case "!leave":
					//	log.Println("[Main]: Stuck in !leave.")
					//	stream.Send(&pb.ChatMessage{Sender: uName, Receiver: gName, Message: uName + " left chat!\n"})
					//	c.LeaveRoom(context.Background(), &pb.GroupInfo{Client: uName, GroupName: gName})
					//	stream.CloseSend()
					//	inMenu = true
					//	f = false
					case "!exit":
						log.Println("[Main]: Stuck in !exit.")
						stream.Send(&pb.ChatMessage{Sender: uName, Receiver: gName, Message: uName + " left chat!\n"})
						ExitChat(c, stream, uName, gName)
						stream.CloseSend()
						conn.Close()
					case "!help":
						log.Println("[Main]: Stuck in !help.")
						AddSpacing(1)
						fmt.Println("The following commands are available to you: ")
						color.New(color.FgHiYellow).Print("   !members")
						fmt.Print(": Lists the current members in the group.")

						AddSpacing(1)
						color.New(color.FgHiYellow).Print("   !exit")
						fmt.Println(": Leaves the chat server.")
						AddSpacing(1)

					default:
						log.Println("[Main]: Sending the message.")
						stream.Send(&toSend)
					}
				case received := <-inbox:
					log.Println("[Main]: Receiving the message.")
					if received.Message != "!leave" {
						fmt.Printf("%s> %s", received.Sender, received.Message)
					}
				}
				log.Println("[Main]: End for-loop.")
			}
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 The goyy Authors.  All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package env

import (
	"gopkg.in/goyy/goyy.v0/comm/i18n"
)

var i18N = i18n.New(locales, i18n.Locale_en_US)

var locales = map[string]map[string]string{
	i18n.Locale_en_US: en_US,
}

var en_US = map[string]string{
	"empty.environments.default":               "environments.default: can not be empty!",
	"empty.environments.environment.database":  "environments.environment.database[@name='%s']: can not be empty!",
	"empty.environments.environment.mail":      "environments.environment.mail[@name='%s']: can not be empty!",
	"empty.environments.environment.session":   "environments.environment.session[@name='%s']: can not be empty!",
	"empty.environments.environment.api":       "environments.environment.api[@name='%s']: can not be empty!",
	"empty.environments.environment.asset":     "environments.environment.asset[@name='%s']: can not be empty!",
	"empty.environments.environment.static":    "environments.environment.static[@name='%s']: can not be empty!",
	"empty.environments.environment.developer": "environments.environment.developer[@name='%s']: can not be empty!",
	"empty.environments.environment.operation": "environments.environment.operation[@name='%s']: can not be empty!",
	"empty.environments.environment.upload":    "environments.environment.upload[@name='%s']: can not be empty!",
}
<commit_msg>Add support for SetLocale<commit_after>// Copyright 2014 The goyy Authors.  All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package env

import (
	"gopkg.in/goyy/goyy.v0/comm/i18n"
)

func SetLocale(locale string) {
	i18N.SetLocale(locale)
}

var i18N = i18n.New(locales, i18n.Locale_en_US)

var locales = map[string]map[string]string{
	i18n.Locale_en_US: en_US,
}

var en_US = map[string]string{
	"empty.environments.default":               "environments.default: can not be empty!",
	"empty.environments.environment.database":  "environments.environment.database[@name='%s']: can not be empty!",
	"empty.environments.environment.mail":      "environments.environment.mail[@name='%s']: can not be empty!",
	"empty.environments.environment.session":   "environments.environment.session[@name='%s']: can not be empty!",
	"empty.environments.environment.api":       "environments.environment.api[@name='%s']: can not be empty!",
	"empty.environments.environment.asset":     "environments.environment.asset[@name='%s']: can not be empty!",
	"empty.environments.environment.static":    "environments.environment.static[@name='%s']: can not be empty!",
	"empty.environments.environment.developer": "environments.environment.developer[@name='%s']: can not be empty!",
	"empty.environments.environment.operation": "environments.environment.operation[@name='%s']: can not be empty!",
	"empty.environments.environment.upload":    "environments.environment.upload[@name='%s']: can not be empty!",
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"
	"github.com/spf13/cobra"
	"github.com/zorkian/go-datadog-api"
	"os"
	"time"
)

var copyCmd = &cobra.Command{
	Use:   "copy",
	Short: "Copy a Consul key to another location.",
	Long:  `copy is for copying already existing keys.`,
	PreRun: func(cmd *cobra.Command, args []string) {
		checkCopyFlags()
		AutoEnable()
	},
	Run: copyRun,
}

func copyRun(cmd *cobra.Command, args []string) {
	start := time.Now()
	var dog = new(datadog.Client)

	// Set the source key locations.
	KeyData := KeyPath(KeyFrom, "data")
	KeyChecksum := KeyPath(KeyFrom, "checksum")

	c, _ := Connect(ConsulServer, Token)

	if DatadogAPIKey != "" && DatadogAPPKey != "" {
		dog = DDAPIConnect(DatadogAPIKey, DatadogAPPKey)
	}

	// Get the KV data out of Consul.
	KVData := Get(c, KeyData)

	// Decompress here if necessary.
	if Compress {
		KVData = DecompressData(KVData)
	}

	// Get the Checksum data out of Consul.
	Checksum := Get(c, KeyChecksum)

	// Is the data long enough?
	longEnough := LengthCheck(KVData, MinFileLength)
	Log(fmt.Sprintf("longEnough='%t'", longEnough), "debug")

	// Does the checksum match?
	checksumMatch := ChecksumCompare(KVData, Checksum)
	Log(fmt.Sprintf("checksumMatch='%t'", checksumMatch), "debug")

	// If the data is long enough and the checksum matches, save to the new key location.
	if longEnough && checksumMatch {
		Log(fmt.Sprintf("copy='true' keyFrom='%s' keyTo='%s'", KeyFrom, KeyTo), "info")
		if Compress {
			KVData = CompressData(KVData)
		}
		// New destination key Locations
		KeyData = KeyPath(KeyTo, "data")
		KeyChecksum = KeyPath(KeyTo, "checksum")
		// Save it.
		saved := Set(c, KeyData, KVData)
		if saved {
			KVDataBytes := len(KVData)
			Log(fmt.Sprintf("consul KeyData='%s' saved='true' size='%d'", KeyData, KVDataBytes), "info")
			Set(c, KeyChecksum, Checksum)
			if DatadogAPIKey != "" && DatadogAPPKey != "" {
				DDCopyDataEvent(dog, KeyFrom, KeyTo)
			}
			StatsdIn(KeyTo, KVDataBytes, KVData)
		} else {
			Log(fmt.Sprintf("consul KeyData='%s' saved='false'", KeyData), "info")
			RunTime(start, KeyTo, "consul_checksums_match")
			os.Exit(0)
		}
	} else {
		Log(fmt.Sprintf("longEnough='%t' checksumMatch='%t'", longEnough, checksumMatch), "info")
		os.Exit(0)
	}

	// Run this command after the file is written.
	if PostExec != "" {
		Log(fmt.Sprintf("exec='%s'", PostExec), "debug")
		RunCommand(PostExec)
	}
	RunTime(start, RawKeyOutLocation, "complete")
}

func checkCopyFlags() {
	Log("Checking cli flags.", "debug")
	if KeyFrom == "" {
		fmt.Println("Need a key location in --keyfrom")
		os.Exit(1)
	}
	if KeyTo == "" {
		fmt.Println("Need a key destination in --keyto")
		os.Exit(1)
	}
	Log("Required cli flags present.", "debug")
}

var (
	// KeyFrom is the key in the Consul KV data store where we want to pull data.
	// This configuration variable will pull data from:
	//  /PrefixLocation/KeyFrom/data
	//  /PrefixLocation/KeyFrom/checksum
	KeyFrom string

	// KeyTo is the key in the Consul KV data store where we want to send data to.
	// This configuration variable will save data into:
	//  /PrefixLocation/KeyTo/data
	//  /PrefixLocation/KeyTo/checksum
	KeyTo string
)

func init() {
	RootCmd.AddCommand(copyCmd)
	copyCmd.Flags().StringVarP(&KeyFrom, "keyfrom", "", "", "key to pull data from")
	copyCmd.Flags().StringVarP(&KeyTo, "keyto", "", "", "key to write the data to")
}
<commit_msg>This is the proper key value to use for stats.<commit_after>package commands

import (
	"fmt"
	"github.com/spf13/cobra"
	"github.com/zorkian/go-datadog-api"
	"os"
	"time"
)

var copyCmd = &cobra.Command{
	Use:   "copy",
	Short: "Copy a Consul key to another location.",
	Long:  `copy is for copying already existing keys.`,
	PreRun: func(cmd *cobra.Command, args []string) {
		checkCopyFlags()
		AutoEnable()
	},
	Run: copyRun,
}

func copyRun(cmd *cobra.Command, args []string) {
	start := time.Now()
	var dog = new(datadog.Client)

	// Set the source key locations.
	KeyData := KeyPath(KeyFrom, "data")
	KeyChecksum := KeyPath(KeyFrom, "checksum")

	c, _ := Connect(ConsulServer, Token)

	if DatadogAPIKey != "" && DatadogAPPKey != "" {
		dog = DDAPIConnect(DatadogAPIKey, DatadogAPPKey)
	}

	// Get the KV data out of Consul.
	KVData := Get(c, KeyData)

	// Decompress here if necessary.
	if Compress {
		KVData = DecompressData(KVData)
	}

	// Get the Checksum data out of Consul.
	Checksum := Get(c, KeyChecksum)

	// Is the data long enough?
	longEnough := LengthCheck(KVData, MinFileLength)
	Log(fmt.Sprintf("longEnough='%t'", longEnough), "debug")

	// Does the checksum match?
	checksumMatch := ChecksumCompare(KVData, Checksum)
	Log(fmt.Sprintf("checksumMatch='%t'", checksumMatch), "debug")

	// If the data is long enough and the checksum matches, save to the new key location.
	if longEnough && checksumMatch {
		Log(fmt.Sprintf("copy='true' keyFrom='%s' keyTo='%s'", KeyFrom, KeyTo), "info")
		if Compress {
			KVData = CompressData(KVData)
		}
		// New destination key Locations
		KeyData = KeyPath(KeyTo, "data")
		KeyChecksum = KeyPath(KeyTo, "checksum")
		// Save it.
		saved := Set(c, KeyData, KVData)
		if saved {
			KVDataBytes := len(KVData)
			Log(fmt.Sprintf("consul KeyData='%s' saved='true' size='%d'", KeyData, KVDataBytes), "info")
			Set(c, KeyChecksum, Checksum)
			if DatadogAPIKey != "" && DatadogAPPKey != "" {
				DDCopyDataEvent(dog, KeyFrom, KeyTo)
			}
			StatsdIn(KeyTo, KVDataBytes, KVData)
		} else {
			Log(fmt.Sprintf("consul KeyData='%s' saved='false'", KeyData), "info")
			RunTime(start, KeyTo, "consul_checksums_match")
			os.Exit(0)
		}
	} else {
		Log(fmt.Sprintf("longEnough='%t' checksumMatch='%t'", longEnough, checksumMatch), "info")
		os.Exit(0)
	}

	// Run this command after the file is written.
	if PostExec != "" {
		Log(fmt.Sprintf("exec='%s'", PostExec), "debug")
		RunCommand(PostExec)
	}
	RunTime(start, KeyTo, "complete")
}

func checkCopyFlags() {
	Log("Checking cli flags.", "debug")
	if KeyFrom == "" {
		fmt.Println("Need a key location in --keyfrom")
		os.Exit(1)
	}
	if KeyTo == "" {
		fmt.Println("Need a key destination in --keyto")
		os.Exit(1)
	}
	Log("Required cli flags present.", "debug")
}

var (
	// KeyFrom is the key in the Consul KV data store where we want to pull data.
	// This configuration variable will pull data from:
	//  /PrefixLocation/KeyFrom/data
	//  /PrefixLocation/KeyFrom/checksum
	KeyFrom string

	// KeyTo is the key in the Consul KV data store where we want to send data to.
	// This configuration variable will save data into:
	//  /PrefixLocation/KeyTo/data
	//  /PrefixLocation/KeyTo/checksum
	KeyTo string
)

func init() {
	RootCmd.AddCommand(copyCmd)
	copyCmd.Flags().StringVarP(&KeyFrom, "keyfrom", "", "", "key to pull data from")
	copyCmd.Flags().StringVarP(&KeyTo, "keyto", "", "", "key to write the data to")
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"net/url"
	"os"
	"os/exec"

	"github.com/concourse/archive-resource/models"
)

func main() {
	if len(os.Args) < 2 {
		println("usage: " + os.Args[0] + " <destination>")
		os.Exit(1)
	}

	destination := os.Args[1]

	err := os.MkdirAll(destination, 0755)
	if err != nil {
		fatal("creating destination", err)
	}

	var request models.InRequest

	err = json.NewDecoder(os.Stdin).Decode(&request)
	if err != nil {
		fatal("reading request", err)
	}

	sourceURL, err := url.Parse(request.Source.URI)
	if err != nil {
		fatal("parsing uri", err)
	}

	// busybox's wget does not support basic auth in the uri,
	// so convert it to the header format ourselves
	authHeader := "Authorization: "
	if sourceURL.User != nil {
		buf := new(bytes.Buffer)

		encoder := base64.NewEncoder(base64.StdEncoding, buf)

		_, err := encoder.Write([]byte(sourceURL.User.Username()))
		if err != nil {
			fatal("encoding username", err)
		}

		if password, hasPassword := sourceURL.User.Password(); hasPassword {
			_, err := fmt.Fprintf(encoder, ":%s", password)
			if err != nil {
				fatal("encoding password", err)
			}
		}

		authHeader += "Basic " + buf.String()

		sourceURL.User = nil
	}

	wgetPipe := exec.Command(
		"sh",
		"-c",
		"wget --header $3 -O - $1 | gunzip | tar -C $2 -xf -",
		"sh", sourceURL.String(), destination, authHeader,
	)

	wgetPipe.Stdout = os.Stderr
	wgetPipe.Stderr = os.Stderr

	err = wgetPipe.Run()
	if err != nil {
		println(err.Error())
		os.Exit(1)
	}

	json.NewEncoder(os.Stdout).Encode(models.InResponse{})
}

func fatal(doing string, err error) {
	println("error " + doing + ": " + err.Error())
	os.Exit(1)
}
<commit_msg>quote arguments<commit_after>package main

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"net/url"
	"os"
	"os/exec"

	"github.com/concourse/archive-resource/models"
)

func main() {
	if len(os.Args) < 2 {
		println("usage: " + os.Args[0] + " <destination>")
		os.Exit(1)
	}

	destination := os.Args[1]

	err := os.MkdirAll(destination, 0755)
	if err != nil {
		fatal("creating destination", err)
	}

	var request models.InRequest

	err = json.NewDecoder(os.Stdin).Decode(&request)
	if err != nil {
		fatal("reading request", err)
	}

	sourceURL, err := url.Parse(request.Source.URI)
	if err != nil {
		fatal("parsing uri", err)
	}

	// busybox's wget does not support basic auth in the uri,
	// so convert it to the header format ourselves
	authHeader := "Authorization: "
	if sourceURL.User != nil {
		buf := new(bytes.Buffer)

		encoder := base64.NewEncoder(base64.StdEncoding, buf)

		_, err := encoder.Write([]byte(sourceURL.User.Username()))
		if err != nil {
			fatal("encoding username", err)
		}

		if password, hasPassword := sourceURL.User.Password(); hasPassword {
			_, err := fmt.Fprintf(encoder, ":%s", password)
			if err != nil {
				fatal("encoding password", err)
			}
		}

		authHeader += "Basic " + buf.String()

		sourceURL.User = nil
	}

	wgetPipe := exec.Command(
		"sh",
		"-c",
		"wget --header \"$3\" -O - \"$1\" | gunzip | tar -C \"$2\" -xf -",
		"sh", sourceURL.String(), destination, authHeader,
	)

	wgetPipe.Stdout = os.Stderr
	wgetPipe.Stderr = os.Stderr

	err = wgetPipe.Run()
	if err != nil {
		println(err.Error())
		os.Exit(1)
	}

	json.NewEncoder(os.Stdout).Encode(models.InResponse{})
}

func fatal(doing string, err error) {
	println("error " + doing + ": " + err.Error())
	os.Exit(1)
}
<|endoftext|>


<commit_before>// Package commands utilities and response structs, constants, etc.
package commands

import (
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"regexp"
	//"strconv"
	//"github.com/pquerna/ffjson/ffjson"
	ct "github.com/daviddengcn/go-colortext"
	"github.com/tmaiaroto/discfg/config"
	"io/ioutil"
	"time"
)

// TODO: Refactor: Change the following ...Msg constants and use config/status.go instead to centralize the error codes and messages.

// NotEnoughArgsMsg defines a message for input validation
const NotEnoughArgsMsg = "Not enough arguments passed. Run 'discfg help' for usage."

// ValueRequired defines a message for input validation
const ValueRequiredMsg = "A value is required. Run 'discfg help' for usage."

// DiscfgFileName defines the temporary filename used to hold the current working config name
const DiscfgFileName = ".discfg"

// NoCurrentWorkingCfgMsg defines a message for an error when a config name can not be found in a .discfg file
const NoCurrentWorkingCfgMsg = "No current working configuration has been set at this path."

// MissingKeyNameMsg defines a message for input validation when a key name was not passed
const MissingKeyNameMsg = "Missing key name"

// InvalidKeyNameMsg defines a message for input validation
const InvalidKeyNameMsg = "Invalid key name"

// MissingCfgNameMsg defines a message for input validation
const MissingCfgNameMsg = "Missing configuration name"

// Out formats a config.ResponseObject for suitable output
func Out(opts config.Options, resp config.ResponseObject) config.ResponseObject {
	// We've stored everything as binary data. But that can be many things.
	// A string, a number, or even JSON. We can check to see if it's something we can marshal to JSON.
	// If that fails, then we'll just return it as a string in the JSON response under the "value" key.
	//
	// If it isn't JSON, then return a base64 string.
	// TODO: Add Content-Type field of some sort so there's some context?
	//
	// TODO: Stuff like this will now be handled by an output interface.
	// ...and will also handle the content-type situation.
	// Output JSON, output Msgpack, output Protobuf? output whatever Content-Type.
	//
	// if resp.Item.Value != nil {
	// 	if !isJSON(string(resp.Item.Value)) {
	// 		// Return base64 when not JSON?
	// 		// b64Str := base64.StdEncoding.EncodeToString(resp.Item.Value)
	// 		//resp.Item.Value = []byte(strconv.Quote(b64Str))
	// 		resp.Item.Value = []byte(strconv.Quote(string(resp.Item.Value)))
	// 	}
	// 	// The output value is always raw JSON. It is not stored in the data store.
	// 	// It's simply for display.
	// 	resp.Item.OutputValue = json.RawMessage(resp.Item.Value)
	// }

	// // Same for the PrevItem if set
	// if resp.PrevItem.Value != nil {
	// 	if !isJSON(string(resp.PrevItem.Value)) {
	// 		resp.PrevItem.Value = []byte(strconv.Quote(string(resp.PrevItem.Value)))
	// 	}
	// 	resp.PrevItem.OutputValue = json.RawMessage(resp.PrevItem.Value)
	// }

	// Format the expiration time (if applicable). This prevents output like "0001-01-01T00:00:00Z" when empty
	// and allows for the time.RFC3339Nano format to be used whereas time.Time normally marshals to a different format.
	if resp.Item.TTL > 0 {
		resp.Item.OutputExpiration = resp.Item.Expiration.Format(time.RFC3339Nano)
	}

	switch opts.OutputFormat {
	case "json":
		o, _ := json.Marshal(&resp)
		// TODO: Benchmark this - is it faster?
		// o, _ := ffjson.Marshal(&resp)
		//
		// TODO: verbose mode here too? Shouldn't be in a situation where it can't be marshaled but who knows.
		// Always best to handle errors.
		// if(oErr) {
		// 	errorLabel("Error")
		// 	fmt.Print(oErr)
		// }
		fmt.Print(string(o))
	case "human":
		if resp.Error != "" {
			errorLabel(resp.Error)
		}
		if resp.Item.Value != nil {
			// The value should be a byte array, for th CLI we want a string.
			fmt.Println(string(resp.Item.Value.([]byte)))
		} else {
			if resp.Message != "" {
				fmt.Println(resp.Message)
			}
		}
	}
	return resp
}

// Changes the color for error messages. Good for one line heading. Any lengthy response should probably not be colored with a red background.
func errorLabel(message string) {
	ct.ChangeColor(ct.White, true, ct.Red, false)
	fmt.Print(message)
	ct.ResetColor()
	fmt.Println("")
}

// Changes the color for the messages to green for success.
func successLabel(message string) {
	ct.Foreground(ct.Green, true)
	fmt.Print(message)
	ct.ResetColor()
	fmt.Println("")
}

// GetDiscfgNameFromFile simply returns the name of the set discfg name (TODO: will need to change as .discfg gets more complex).
func GetDiscfgNameFromFile() string {
	name := ""
	currentCfg, err := ioutil.ReadFile(DiscfgFileName)
	if err == nil {
		name = string(currentCfg)
	}
	return name
}

// Simple substring function
func substr(s string, pos, length int) string {
	runes := []rune(s)
	l := pos + length
	if l > len(runes) {
		l = len(runes)
	}
	return string(runes[pos:l])
}

// Checks and formats the key name.
func formatKeyName(key string) (string, error) {
	var err error
	k := ""
	if len(key) > 0 {
		k = key
	} else {
		return "", errors.New(MissingKeyNameMsg)
	}

	// Ensure valid characters
	r, _ := regexp.Compile(`[\w\/\-]+$`)
	if !r.MatchString(k) {
		return "", errors.New(InvalidKeyNameMsg)
	}

	// Remove any trailing slashes (unless there's only one, the root).
	// NOTE: A tree structure is not yet supported. The user can define one, but there are no recursive features when getting/deleting.
	// This may come in a future version, for now the structure is flat. However, convention set by other tools (along with REST API endpoints)
	// makes using slashes a natural fit and discfg will assume they are being used. It could be thought of as a namespace.
	if len(k) > 1 {
		for k[len(k)-1:] == "/" {
			k = k[:len(k)-1]
		}
	}

	return k, err
}

func isJSONString(s string) bool {
	var js string
	err := json.Unmarshal([]byte(s), &js)
	return err == nil
}
func isJSON(s string) bool {
	var js map[string]interface{}
	return json.Unmarshal([]byte(s), &js) == nil

}

// FormatJSONValue sets the Item Value (an interface{}) as a map[string]interface{} (from []byte, which is how it's stored)
// so that it can be converted to JSON in an HTTP response. If it can't be represented in a map, then it'll be set as a string.
// For example, a string was set as the value, we can still represent that in JSON. However, if an image was stored...Then it's
// going to look ugly. It won't be a base64 string, it'll be the string representation of the binary data. Which apparently Chrome
// will render if given...But still. Not so hot. The user should know what they are setting and getting though and this should
// still technically return JSON with a usable value. Valid JSON at that. Just with some funny looking characters =)
func FormatJSONValue(resp config.ResponseObject) config.ResponseObject {
	// Don't attempt to Unmarshal or anything if the Value is empty. We wouldn't want to create a panic now.
	if resp.Item.Value == nil {
		return resp
	}
	resp.Item.Value = string(resp.Item.Value.([]byte))

	// The value could be base64 encoded, but it need not be.
	val, err := base64.StdEncoding.DecodeString(resp.Item.Value.(string)) //`eyJ1cGRhdGVkIjogImZyaWRheSJ9`)
	if err == nil && val != nil {
		resp.Item.Value = string(val)
	}

	var jsonData map[string]interface{}
	// Back to byte (because it could have potentially been base64 encoded)
	err = json.Unmarshal([]byte(resp.Item.Value.(string)), &jsonData)
	if err == nil {
		resp.Item.Value = jsonData
	}

	return resp
}
<commit_msg>fix #19 - could be better though<commit_after>// Package commands utilities and response structs, constants, etc.
package commands

import (
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"regexp"
	//"strconv"
	//"github.com/pquerna/ffjson/ffjson"
	ct "github.com/daviddengcn/go-colortext"
	"github.com/tmaiaroto/discfg/config"
	"io/ioutil"
	"time"
)

// TODO: Refactor: Change the following ...Msg constants and use config/status.go instead to centralize the error codes and messages.

// NotEnoughArgsMsg defines a message for input validation
const NotEnoughArgsMsg = "Not enough arguments passed. Run 'discfg help' for usage."

// ValueRequired defines a message for input validation
const ValueRequiredMsg = "A value is required. Run 'discfg help' for usage."

// DiscfgFileName defines the temporary filename used to hold the current working config name
const DiscfgFileName = ".discfg"

// NoCurrentWorkingCfgMsg defines a message for an error when a config name can not be found in a .discfg file
const NoCurrentWorkingCfgMsg = "No current working configuration has been set at this path."

// MissingKeyNameMsg defines a message for input validation when a key name was not passed
const MissingKeyNameMsg = "Missing key name"

// InvalidKeyNameMsg defines a message for input validation
const InvalidKeyNameMsg = "Invalid key name"

// MissingCfgNameMsg defines a message for input validation
const MissingCfgNameMsg = "Missing configuration name"

// Out formats a config.ResponseObject for suitable output
func Out(opts config.Options, resp config.ResponseObject) config.ResponseObject {
	// We've stored everything as binary data. But that can be many things.
	// A string, a number, or even JSON. We can check to see if it's something we can marshal to JSON.
	// If that fails, then we'll just return it as a string in the JSON response under the "value" key.
	//
	// If it isn't JSON, then return a base64 string.
	// TODO: Add Content-Type field of some sort so there's some context?
	//
	// TODO: Stuff like this will now be handled by an output interface.
	// ...and will also handle the content-type situation.
	// Output JSON, output Msgpack, output Protobuf? output whatever Content-Type.
	//
	// if resp.Item.Value != nil {
	// 	if !isJSON(string(resp.Item.Value)) {
	// 		// Return base64 when not JSON?
	// 		// b64Str := base64.StdEncoding.EncodeToString(resp.Item.Value)
	// 		//resp.Item.Value = []byte(strconv.Quote(b64Str))
	// 		resp.Item.Value = []byte(strconv.Quote(string(resp.Item.Value)))
	// 	}
	// 	// The output value is always raw JSON. It is not stored in the data store.
	// 	// It's simply for display.
	// 	resp.Item.OutputValue = json.RawMessage(resp.Item.Value)
	// }

	// // Same for the PrevItem if set
	// if resp.PrevItem.Value != nil {
	// 	if !isJSON(string(resp.PrevItem.Value)) {
	// 		resp.PrevItem.Value = []byte(strconv.Quote(string(resp.PrevItem.Value)))
	// 	}
	// 	resp.PrevItem.OutputValue = json.RawMessage(resp.PrevItem.Value)
	// }

	// Format the expiration time (if applicable). This prevents output like "0001-01-01T00:00:00Z" when empty
	// and allows for the time.RFC3339Nano format to be used whereas time.Time normally marshals to a different format.
	if resp.Item.TTL > 0 {
		resp.Item.OutputExpiration = resp.Item.Expiration.Format(time.RFC3339Nano)
	}

	switch opts.OutputFormat {
	case "json":
		o, _ := json.Marshal(&resp)
		// TODO: Benchmark this - is it faster?
		// o, _ := ffjson.Marshal(&resp)
		//
		// TODO: verbose mode here too? Shouldn't be in a situation where it can't be marshaled but who knows.
		// Always best to handle errors.
		// if(oErr) {
		// 	errorLabel("Error")
		// 	fmt.Print(oErr)
		// }
		fmt.Print(string(o))
	case "human":
		if resp.Error != "" {
			errorLabel(resp.Error)
		}
		if resp.Item.Value != nil {
			// The value should be a byte array, for th CLI we want a string.
			fmt.Println(string(resp.Item.Value.([]byte)))
		} else {
			if resp.Message != "" {
				fmt.Println(resp.Message)
			}
		}
	}
	return resp
}

// Changes the color for error messages. Good for one line heading. Any lengthy response should probably not be colored with a red background.
func errorLabel(message string) {
	ct.ChangeColor(ct.White, true, ct.Red, false)
	fmt.Print(message)
	ct.ResetColor()
	fmt.Println("")
}

// Changes the color for the messages to green for success.
func successLabel(message string) {
	ct.Foreground(ct.Green, true)
	fmt.Print(message)
	ct.ResetColor()
	fmt.Println("")
}

// GetDiscfgNameFromFile simply returns the name of the set discfg name (TODO: will need to change as .discfg gets more complex).
func GetDiscfgNameFromFile() string {
	name := ""
	currentCfg, err := ioutil.ReadFile(DiscfgFileName)
	if err == nil {
		name = string(currentCfg)
	}
	return name
}

// Simple substring function
func substr(s string, pos, length int) string {
	runes := []rune(s)
	l := pos + length
	if l > len(runes) {
		l = len(runes)
	}
	return string(runes[pos:l])
}

// Checks and formats the key name.
func formatKeyName(key string) (string, error) {
	var err error
	k := ""
	if len(key) > 0 {
		k = key
	} else {
		return "", errors.New(MissingKeyNameMsg)
	}

	// Ensure valid characters
	r, _ := regexp.Compile(`[\w\/\-]+$`)
	if !r.MatchString(k) {
		return "", errors.New(InvalidKeyNameMsg)
	}

	// Remove any trailing slashes (unless there's only one, the root).
	// NOTE: A tree structure is not yet supported. The user can define one, but there are no recursive features when getting/deleting.
	// This may come in a future version, for now the structure is flat. However, convention set by other tools (along with REST API endpoints)
	// makes using slashes a natural fit and discfg will assume they are being used. It could be thought of as a namespace.
	if len(k) > 1 {
		for k[len(k)-1:] == "/" {
			k = k[:len(k)-1]
		}
	}

	return k, err
}

func isJSONString(s string) bool {
	var js string
	err := json.Unmarshal([]byte(s), &js)
	return err == nil
}
func isJSON(s string) bool {
	var js map[string]interface{}
	return json.Unmarshal([]byte(s), &js) == nil

}

// FormatJSONValue sets the Item Value (an interface{}) as a map[string]interface{} (from []byte, which is how it's stored)
// so that it can be converted to JSON in an HTTP response. If it can't be represented in a map, then it'll be set as a string.
// For example, a string was set as the value, we can still represent that in JSON. However, if an image was stored...Then it's
// going to look ugly. It won't be a base64 string, it'll be the string representation of the binary data. Which apparently Chrome
// will render if given...But still. Not so hot. The user should know what they are setting and getting though and this should
// still technically return JSON with a usable value. Valid JSON at that. Just with some funny looking characters =)
func FormatJSONValue(resp config.ResponseObject) config.ResponseObject {
	// Don't attempt to Unmarshal or anything if the Value is empty. We wouldn't want to create a panic now.
	if resp.Item.Value != nil {
		resp.Item.Value = string(resp.Item.Value.([]byte))

		// The value could be base64 encoded, but it need not be.
		val, err := base64.StdEncoding.DecodeString(resp.Item.Value.(string)) //`eyJ1cGRhdGVkIjogImZyaWRheSJ9`)
		if err == nil && val != nil {
			resp.Item.Value = string(val)
		}

		var jsonData map[string]interface{}
		// Back to byte (because it could have potentially been base64 encoded)
		err = json.Unmarshal([]byte(resp.Item.Value.(string)), &jsonData)
		if err == nil {
			resp.Item.Value = jsonData
		}
	}

	// The previous value as well
	if resp.PrevItem.Value != nil {
		resp.PrevItem.Value = string(resp.PrevItem.Value.([]byte))

		// The value could be base64 encoded, but it need not be.
		val, err := base64.StdEncoding.DecodeString(resp.PrevItem.Value.(string)) //`eyJ1cGRhdGVkIjogImZyaWRheSJ9`)
		if err == nil && val != nil {
			resp.PrevItem.Value = string(val)
		}

		var jsonData map[string]interface{}
		err = json.Unmarshal([]byte(resp.PrevItem.Value.(string)), &jsonData)
		if err == nil {
			resp.PrevItem.Value = jsonData
		}
	}

	return resp
}
<|endoftext|>


<commit_before>package common

import (
	"log"
	"net"
	"path"
	"strconv"
	"strings"
	"time"
)

const (
	//HeaderToken security token
	HeaderToken = "jrdwptoken"
	//HeaderPort jdwp port
	HeaderPort = "jdwpport"
	//PublicKeyDir default client key path
	PublicKeyDir = "."
	//PublicKeyFile filename
	PublicKeyFile = ".jrdwp_key"
)

var DeadlineDuration = time.Second * 60 * 30

//PublicKeyPath concat file path of client key
func PublicKeyPath() string {
	return path.Join(PublicKeyDir, PublicKeyFile)
}

//SplitToInt split comma delimited string to int array
func SplitToInt(commaDelimitedString string) []int {
	ports := []int{}

	for _, token := range strings.Split(commaDelimitedString, ",") {
		port, err := strconv.Atoi(string(strings.TrimSpace(token)))
		if err != nil {
			log.Fatalln("bad integer", token, err.Error)
		} else {
			ports = append(ports, port)
		}
	}

	return ports
}

//InitTCPConn initialize tcp connection to keep alive
func InitTCPConn(conn *net.TCPConn) {
	conn.SetKeepAlive(true)
	conn.SetNoDelay(true)
	conn.SetLinger(3)
	conn.SetDeadline(time.Now().Add(DeadlineDuration))
}
<commit_msg>Updated for Private Key<commit_after>package common

import (
	"log"
	"net"
	"path"
	"strconv"
	"strings"
	"time"
)

const (
	//HeaderToken security token
	HeaderToken = "jrdwptoken"
	//HeaderPort jdwp port
	HeaderPort = "jdwpport"
	//PublicKeyDir default client key path
	PublicKeyDir = "."
	//PublicKeyFile filename
	PublicKeyFile = ".jrdwp_key"
	//PrivateKeyDir default client key path
	PrivateKeyDir = "."
	//PrivateKeyFile filename
	PrivateKeyFile = ".jrdwp_privatekey"
)

var DeadlineDuration = time.Second * 60 * 30

//PublicKeyPath concat file path of client key
func PublicKeyPath() string {
	return path.Join(PublicKeyDir, PublicKeyFile)
}

//PrivateKeyPath concat file path of private key
func PrivateKeyPath() string {
	return path.Join(PrivateKeyDir, PrivateKeyFile)
}

//SplitToInt split comma delimited string to int array
func SplitToInt(commaDelimitedString string) []int {
	ports := []int{}

	for _, token := range strings.Split(commaDelimitedString, ",") {
		port, err := strconv.Atoi(string(strings.TrimSpace(token)))
		if err != nil {
			log.Fatalln("bad integer", token, err.Error)
		} else {
			ports = append(ports, port)
		}
	}

	return ports
}

//InitTCPConn initialize tcp connection to keep alive
func InitTCPConn(conn *net.TCPConn) {
	conn.SetKeepAlive(true)
	conn.SetNoDelay(true)
	conn.SetLinger(3)
	conn.SetDeadline(time.Now().Add(DeadlineDuration))
}
<|endoftext|>


<commit_before>package common

import (
	"fmt"
	"net/url"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"time"
)

// PackerKeyEnv is used to specify the key interval (delay) between keystrokes
// sent to the VM, typically in boot commands. This is to prevent host CPU
// utilization from causing key presses to be skipped or repeated incorrectly.
const PackerKeyEnv = "PACKER_KEY_INTERVAL"

// PackerKeyDefault 100ms is appropriate for shared build infrastructure while a
// shorter delay (e.g. 10ms) can be used on a workstation. See PackerKeyEnv.
const PackerKeyDefault = 100 * time.Millisecond

// ScrubConfig is a helper that returns a string representation of
// any struct with the given values stripped out.
func ScrubConfig(target interface{}, values ...string) string {
	conf := fmt.Sprintf("Config: %+v", target)
	for _, value := range values {
		if value == "" {
			continue
		}
		conf = strings.Replace(conf, value, "<Filtered>", -1)
	}
	return conf
}

// ChooseString returns the first non-empty value.
func ChooseString(vals ...string) string {
	for _, el := range vals {
		if el != "" {
			return el
		}
	}

	return ""
}

// DownloadableURL processes a URL that may also be a file path and returns
// a completely valid URL. For example, the original URL might be "local/file.iso"
// which isn't a valid URL. DownloadableURL will return "file:///local/file.iso"
func DownloadableURL(original string) (string, error) {
	fmt.Printf("Swampy: user input was %s\n", original)
	if runtime.GOOS == "windows" {
		// If the distance to the first ":" is just one character, assume
		// we're dealing with a drive letter and thus a file path.
		idx := strings.Index(original, ":")
		if idx == 1 {
			original = "file:///" + original
		}
	}

	url, err := url.Parse(original)
	if err != nil {
		return "", err
	}

	if url.Scheme == "" {
		url.Scheme = "file"
	}

	if url.Scheme == "file" {
		// Windows file handling is all sorts of tricky...
		if runtime.GOOS == "windows" {
			// If the path is using Windows-style slashes, URL parses
			// it into the host field.
			if url.Path == "" && strings.Contains(url.Host, `\`) {
				url.Path = url.Host
				url.Host = ""
			}
		}

		// Only do the filepath transformations if the file appears
		// to actually exist.
		if _, err := os.Stat(url.Path); err == nil {
			url.Path, err = filepath.Abs(url.Path)
			if err != nil {
				return "", err
			}

			url.Path, err = filepath.EvalSymlinks(url.Path)
			if err != nil {
				return "", err
			}

			// url.Path = filepath.Clean(url.Path)
		}

		if runtime.GOOS == "windows" {
			// Also replace all backslashes with forwardslashes since Windows
			// users are likely to do this but the URL should actually only
			// contain forward slashes.
			url.Path = strings.Replace(url.Path, `\`, `/`, -1)
		}
	}

	// Make sure it is lowercased
	url.Scheme = strings.ToLower(url.Scheme)

	// Verify that the scheme is something we support in our common downloader.
	supported := []string{"file", "http", "https"}
	found := false
	for _, s := range supported {
		if url.Scheme == s {
			found = true
			break
		}
	}

	if !found {
		return "", fmt.Errorf("Unsupported URL scheme: %s", url.Scheme)
	}
	fmt.Printf("Swampy: parsed string after DownloadableURL is %s\n", url.String())
	return url.String(), nil
}

// FileExistsLocally takes the URL output from DownloadableURL, and determines
// whether it is present on the file system.
// example usage:
//
// myFile, err = common.DownloadableURL(c.SourcePath)
// ...
// fileExists, err := common.StatURL(myFile)
// possible output:
// true, nil -- should occur if the file is present
// false, nil -- should occur if the file is not present, but is not supposed to
// be (e.g. the schema is http://, not file://)
// true, error -- shouldn't occur ever
// false, error -- should occur if there was an error stating the file, so the
// file is not present when it should be.

func FileExistsLocally(original string) (bool, error) {
	// original should be something like file://C:/my/path.iso
	// on windows, c drive will be parsed as host if it's file://c instead of file:///c
	prefix = "file://"
	filePath = strings.Replace(original, prefix, "", 1)
	fmt.Printf("Swampy: original is %s\n", original)
	fmt.Printf("Swampy: filePath is %#v\n", filePath)

	if fileURL.Scheme == "file" {
		_, err := os.Stat(filePath)
		if err != nil {
			err = fmt.Errorf("could not stat file: %s\n", err)
			return fileExists, err
		} else {
			fileExists = true
		}
	}
	return fileExists, nil
}
<commit_msg>disambiguate url variable from url library<commit_after>package common

import (
	"fmt"
	"net/url"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"time"
)

// PackerKeyEnv is used to specify the key interval (delay) between keystrokes
// sent to the VM, typically in boot commands. This is to prevent host CPU
// utilization from causing key presses to be skipped or repeated incorrectly.
const PackerKeyEnv = "PACKER_KEY_INTERVAL"

// PackerKeyDefault 100ms is appropriate for shared build infrastructure while a
// shorter delay (e.g. 10ms) can be used on a workstation. See PackerKeyEnv.
const PackerKeyDefault = 100 * time.Millisecond

// ScrubConfig is a helper that returns a string representation of
// any struct with the given values stripped out.
func ScrubConfig(target interface{}, values ...string) string {
	conf := fmt.Sprintf("Config: %+v", target)
	for _, value := range values {
		if value == "" {
			continue
		}
		conf = strings.Replace(conf, value, "<Filtered>", -1)
	}
	return conf
}

// ChooseString returns the first non-empty value.
func ChooseString(vals ...string) string {
	for _, el := range vals {
		if el != "" {
			return el
		}
	}

	return ""
}

// DownloadableURL processes a URL that may also be a file path and returns
// a completely valid URL. For example, the original URL might be "local/file.iso"
// which isn't a valid URL. DownloadableURL will return "file:///local/file.iso"
func DownloadableURL(original string) (string, error) {
	if runtime.GOOS == "windows" {
		// If the distance to the first ":" is just one character, assume
		// we're dealing with a drive letter and thus a file path.
		// prepend with "file:///"" now so that url.Parse won't accidentally
		// parse the drive letter into the url scheme.
		// See https://blogs.msdn.microsoft.com/ie/2006/12/06/file-uris-in-windows/
		// for more info about valid windows URIs
		idx := strings.Index(original, ":")
		if idx == 1 {
			original = "file:///" + original
		}
	}
	u, err := url.Parse(original)
	if err != nil {
		return "", err
	}

	if u.Scheme == "" {
		u.Scheme = "file"
	}

	if u.Scheme == "file" {
		// Windows file handling is all sorts of tricky...
		if runtime.GOOS == "windows" {
			// If the path is using Windows-style slashes, URL parses
			// it into the host field.
			if u.Path == "" && strings.Contains(u.Host, `\`) {
				u.Path = u.Host
				u.Host = ""
			}
		}

		// Only do the filepath transformations if the file appears
		// to actually exist.
		if _, err := os.Stat(u.Path); err == nil {
			u.Path, err = filepath.Abs(u.Path)
			if err != nil {
				return "", err
			}

			u.Path, err = filepath.EvalSymlinks(u.Path)
			if err != nil {
				return "", err
			}

			u.Path = filepath.Clean(u.Path)
		}

		if runtime.GOOS == "windows" {
			// Also replace all backslashes with forwardslashes since Windows
			// users are likely to do this but the URL should actually only
			// contain forward slashes.
			u.Path = strings.Replace(u.Path, `\`, `/`, -1)
			// prepend absolute windows paths with "/" so that when we
			// compose u.String() below the outcome will be correct
			// file:///c/blah syntax; otherwise u.String() will only add
			// file:// which is not technically a correct windows URI
			if filepath.IsAbs(u.Path) && !strings.HasPrefix(u.Path, "/") {
				u.Path = "/" + u.Path
			}

		}
	}

	// Make sure it is lowercased
	u.Scheme = strings.ToLower(u.Scheme)

	// Verify that the scheme is something we support in our common downloader.
	supported := []string{"file", "http", "https"}
	found := false
	for _, s := range supported {
		if u.Scheme == s {
			found = true
			break
		}
	}

	if !found {
		return "", fmt.Errorf("Unsupported URL scheme: %s", u.Scheme)
	}
	return u.String(), nil
}

// FileExistsLocally takes the URL output from DownloadableURL, and determines
// whether it is present on the file system.
// example usage:
//
// myFile, err = common.DownloadableURL(c.SourcePath)
// ...
// fileExists, err := common.StatURL(myFile)
// possible output:
// true, nil -- should occur if the file is present
// false, nil -- should occur if the file is not present, but is not supposed to
// be (e.g. the schema is http://, not file://)
// true, error -- shouldn't occur ever
// false, error -- should occur if there was an error stating the file, so the
// file is not present when it should be.

func FileExistsLocally(original string) (bool, error) {
	// original should be something like file://C:/my/path.iso

	fileURL, _ := url.Parse(original)
	fileExists := false

	if fileURL.Scheme == "file" {
		// on windows, correct URI is file:///c:/blah/blah.iso.
		// url.Parse will pull out the scheme "file://" and leave the path as
		// "/c:/blah/blah/iso".  Here we remove this forward slash on absolute
		// Windows file URLs before processing
		// see https://blogs.msdn.microsoft.com/ie/2006/12/06/file-uris-in-windows/
		// for more info about valid windows URIs
		filePath := fileURL.Path
		if runtime.GOOS == "windows" && len(filePath) > 0 && filePath[0] == '/' {
			filePath = filePath[1:]
		}
		_, err := os.Stat(filePath)
		if err != nil {
			err = fmt.Errorf("could not stat file: %s\n", err)
			return fileExists, err
		} else {
			fileExists = true
		}
	}
	return fileExists, nil
}
<|endoftext|>