<commit_before>package library

import (
	"fmt"
	"os"
	"path"
	"syscall"
	"time"

	"github.com/facette/facette/pkg/logger"
	"github.com/facette/facette/pkg/utils"
	"github.com/facette/facette/thirdparty/github.com/fatih/set"
	uuid "github.com/facette/facette/thirdparty/github.com/nu7hatch/gouuid"
)

// Item represents the base structure of a library item.
type Item struct {
	path        string
	ID          string    `json:"id"`
	Name        string    `json:"name"`
	Description string    `json:"description"`
	Modified    time.Time `json:"-"`
}

// GetItem returns the base structure of a library item.
func (item *Item) GetItem() *Item {
	return item
}

// DeleteItem removes an existing item from the library.
func (library *Library) DeleteItem(id string, itemType int) error {
	if !library.ItemExists(id, itemType) {
		return os.ErrNotExist
	}

	// Delete sub-collections
	if itemType == LibraryItemCollection {
		for _, child := range library.Collections[id].Children {
			library.DeleteItem(child.ID, LibraryItemCollection)
		}
	}

	// Remove stored JSON
	if err := syscall.Unlink(library.getFilePath(id, itemType)); err != nil {
		return err
	}

	// Delete item from library
	switch itemType {
	case LibraryItemSourceGroup, LibraryItemMetricGroup:
		delete(library.Groups, id)

	case LibraryItemScale:
		delete(library.Scales, id)

	case LibraryItemGraph:
		delete(library.Graphs, id)

	case LibraryItemCollection:
		delete(library.Collections, id)
	}

	return nil
}

// GetItem gets an item from the library by its identifier.
func (library *Library) GetItem(id string, itemType int) (interface{}, error) {
	if !library.ItemExists(id, itemType) {
		return nil, os.ErrNotExist
	}

	switch itemType {
	case LibraryItemSourceGroup, LibraryItemMetricGroup:
		return library.Groups[id], nil

	case LibraryItemScale:
		return library.Scales[id], nil

	case LibraryItemGraph:
		return library.Graphs[id], nil

	case LibraryItemCollection:
		return library.Collections[id], nil
	}

	return nil, fmt.Errorf("no item found")
}

// GetItemByName gets an item from the library by its name.
func (library *Library) GetItemByName(name string, itemType int) (interface{}, error) {
	switch itemType {
	case LibraryItemSourceGroup, LibraryItemMetricGroup:
		for _, item := range library.Groups {
			if item.Type != itemType || item.Name != name {
				continue
			}

			return item, nil
		}

	case LibraryItemScale:
		for _, item := range library.Scales {
			if item.Name != name {
				continue
			}

			return item, nil
		}

	case LibraryItemGraph:
		for _, item := range library.Graphs {
			if item.Name != name {
				continue
			}

			return item, nil
		}

	case LibraryItemCollection:
		for _, item := range library.Collections {
			if item.Name != name {
				continue
			}

			return item, nil
		}
	}

	return nil, os.ErrNotExist
}

// ItemExists returns whether an item exists the library or not.
func (library *Library) ItemExists(id string, itemType int) bool {
	exists := false

	switch itemType {
	case LibraryItemSourceGroup, LibraryItemMetricGroup:
		if _, ok := library.Groups[id]; ok && library.Groups[id].Type == itemType {
			exists = true
		}

	case LibraryItemScale:
		_, exists = library.Scales[id]

	case LibraryItemGraph:
		_, exists = library.Graphs[id]

	case LibraryItemCollection:
		_, exists = library.Collections[id]
	}

	return exists
}

// LoadItem loads an item by its identifier.
func (library *Library) LoadItem(id string, itemType int) error {
	// Load item from file
	switch itemType {
	case LibraryItemSourceGroup, LibraryItemMetricGroup:
		tmpGroup := &Group{}

		filePath := library.getFilePath(id, itemType)

		fileInfo, err := utils.JSONLoad(filePath, &tmpGroup)
		if err != nil {
			return fmt.Errorf("in %s, %s", filePath, err)
		}

		library.Groups[id] = tmpGroup
		library.Groups[id].Type = itemType
		library.Groups[id].Modified = fileInfo.ModTime()

	case LibraryItemScale:
		tmpScale := &Scale{}

		filePath := library.getFilePath(id, itemType)

		fileInfo, err := utils.JSONLoad(filePath, &tmpScale)
		if err != nil {
			return fmt.Errorf("in %s, %s", filePath, err)
		}

		library.Scales[id] = tmpScale
		library.Scales[id].Modified = fileInfo.ModTime()

	case LibraryItemGraph:
		tmpGraph := &Graph{}

		filePath := library.getFilePath(id, itemType)

		fileInfo, err := utils.JSONLoad(filePath, &tmpGraph)
		if err != nil {
			return fmt.Errorf("in %s, %s", filePath, err)
		}

		library.Graphs[id] = tmpGraph
		library.Graphs[id].Modified = fileInfo.ModTime()

	case LibraryItemCollection:
		var tmpCollection *struct {
			*Collection
			Parent string `json:"parent"`
		}

		filePath := library.getFilePath(id, LibraryItemCollection)

		fileInfo, err := utils.JSONLoad(filePath, &tmpCollection)
		if err != nil {
			return fmt.Errorf("in %s, %s", filePath, err)
		}

		if !library.ItemExists(id, LibraryItemCollection) {
			library.Collections[id] = &Collection{}
		}

		*library.Collections[id] = *tmpCollection.Collection

		if tmpCollection.Parent != "" {
			library.Collections[id].ParentID = tmpCollection.Parent
		}

		library.Collections[id].Modified = fileInfo.ModTime()
	}

	return nil
}

// StoreItem stores an item into the library.
func (library *Library) StoreItem(item interface{}, itemType int) error {
	var itemStruct *Item

	switch itemType {
	case LibraryItemSourceGroup, LibraryItemMetricGroup:
		itemStruct = item.(*Group).GetItem()

	case LibraryItemScale:
		itemStruct = item.(*Scale).GetItem()

	case LibraryItemGraph:
		itemStruct = item.(*Graph).GetItem()

	case LibraryItemCollection:
		itemStruct = item.(*Collection).GetItem()
	}

	if itemStruct.ID == "" {
		uuidTemp, err := uuid.NewV4()
		if err != nil {
			return err
		}

		itemStruct.ID = uuidTemp.String()
	} else if !library.ItemExists(itemStruct.ID, itemType) {
		return os.ErrNotExist
	}

	// Check for name field presence/duplicates
	if itemStruct.Name == "" {
		return os.ErrInvalid
	}

	itemTemp, err := library.GetItemByName(itemStruct.Name, itemType)
	if err == nil {
		switch itemType {
		case LibraryItemSourceGroup, LibraryItemMetricGroup:
			if itemTemp.(*Group).ID != itemStruct.ID {
				logger.Log(logger.LevelError, "library", "duplicate group identifier `%s'", itemStruct.ID)
				return os.ErrExist
			}

		case LibraryItemScale:
			if itemTemp.(*Scale).ID != itemStruct.ID {
				logger.Log(logger.LevelError, "library", "duplicate scale identifier `%s'", itemStruct.ID)
				return os.ErrExist
			}

		case LibraryItemGraph:
			if itemTemp.(*Graph).ID != itemStruct.ID {
				logger.Log(logger.LevelError, "library", "duplicate graph identifier `%s'", itemStruct.ID)
				return os.ErrExist
			}

		case LibraryItemCollection:
			if itemTemp.(*Collection).ID != itemStruct.ID {
				logger.Log(logger.LevelError, "library", "duplicate collection identifier `%s'", itemStruct.ID)
				return os.ErrExist
			}
		}
	}

	// Store item into library
	switch itemType {
	case LibraryItemSourceGroup, LibraryItemMetricGroup:
		library.Groups[itemStruct.ID] = item.(*Group)
		library.Groups[itemStruct.ID].ID = itemStruct.ID

	case LibraryItemScale:
		library.Scales[itemStruct.ID] = item.(*Scale)
		library.Scales[itemStruct.ID].ID = itemStruct.ID

	case LibraryItemGraph:
		// Check for definition names duplicates
		groupSet := set.New(set.ThreadSafe)
		serieSet := set.New(set.ThreadSafe)

		for _, group := range item.(*Graph).Groups {
			if group == nil {
				logger.Log(logger.LevelError, "library", "found null group")
				return os.ErrInvalid
			} else if groupSet.Has(group.Name) {
				logger.Log(logger.LevelError, "library", "duplicate group name `%s'", group.Name)
				return os.ErrExist
			}

			groupSet.Add(group.Name)

			for _, serie := range group.Series {
				if serie == nil {
					logger.Log(logger.LevelError, "library", "found null serie in group `%s'", group.Name)
					return os.ErrInvalid
				} else if serieSet.Has(serie.Name) {
					logger.Log(logger.LevelError, "library", "duplicate serie name `%s'", serie.Name)
					return os.ErrExist
				}

				serieSet.Add(serie.Name)
			}
		}

		library.Graphs[itemStruct.ID] = item.(*Graph)
		library.Graphs[itemStruct.ID].ID = itemStruct.ID

	case LibraryItemCollection:
		library.Collections[itemStruct.ID] = item.(*Collection)
		library.Collections[itemStruct.ID].ID = itemStruct.ID
	}

	// Store JSON data
	if err := utils.JSONDump(library.getFilePath(itemStruct.ID, itemType), item, itemStruct.Modified); err != nil {
		return err
	}

	return nil
}

func (library *Library) getDirPath(itemType int) string {
	var dirName string

	switch itemType {
	case LibraryItemSourceGroup:
		dirName = "sourcegroups"

	case LibraryItemMetricGroup:
		dirName = "metricgroups"

	case LibraryItemScale:
		dirName = "scales"

	case LibraryItemGraph:
		dirName = "graphs"

	case LibraryItemCollection:
		dirName = "collections"
	}

	return path.Join(library.Config.DataDir, dirName)
}

func (library *Library) getFilePath(id string, itemType int) string {
	return path.Join(library.getDirPath(itemType), id[0:2], id[2:4], id+".json")
}
<commit_msg>Handle invalid library item types<commit_after>package library

import (
	"fmt"
	"os"
	"path"
	"syscall"
	"time"

	"github.com/facette/facette/pkg/logger"
	"github.com/facette/facette/pkg/utils"
	"github.com/facette/facette/thirdparty/github.com/fatih/set"
	uuid "github.com/facette/facette/thirdparty/github.com/nu7hatch/gouuid"
)

// Item represents the base structure of a library item.
type Item struct {
	path        string
	ID          string    `json:"id"`
	Name        string    `json:"name"`
	Description string    `json:"description"`
	Modified    time.Time `json:"-"`
}

// GetItem returns the base structure of a library item.
func (item *Item) GetItem() *Item {
	return item
}

// DeleteItem removes an existing item from the library.
func (library *Library) DeleteItem(id string, itemType int) error {
	if !library.ItemExists(id, itemType) {
		return os.ErrNotExist
	}

	// Delete sub-collections
	if itemType == LibraryItemCollection {
		for _, child := range library.Collections[id].Children {
			library.DeleteItem(child.ID, LibraryItemCollection)
		}
	}

	// Remove stored JSON
	if err := syscall.Unlink(library.getFilePath(id, itemType)); err != nil {
		return err
	}

	// Delete item from library
	switch itemType {
	case LibraryItemSourceGroup, LibraryItemMetricGroup:
		delete(library.Groups, id)

	case LibraryItemScale:
		delete(library.Scales, id)

	case LibraryItemGraph:
		delete(library.Graphs, id)

	case LibraryItemCollection:
		delete(library.Collections, id)
	}

	return nil
}

// GetItem gets an item from the library by its identifier.
func (library *Library) GetItem(id string, itemType int) (interface{}, error) {
	if !library.ItemExists(id, itemType) {
		return nil, os.ErrNotExist
	}

	switch itemType {
	case LibraryItemSourceGroup, LibraryItemMetricGroup:
		return library.Groups[id], nil

	case LibraryItemScale:
		return library.Scales[id], nil

	case LibraryItemGraph:
		return library.Graphs[id], nil

	case LibraryItemCollection:
		return library.Collections[id], nil
	}

	return nil, fmt.Errorf("no item found")
}

// GetItemByName gets an item from the library by its name.
func (library *Library) GetItemByName(name string, itemType int) (interface{}, error) {
	switch itemType {
	case LibraryItemSourceGroup, LibraryItemMetricGroup:
		for _, item := range library.Groups {
			if item.Type != itemType || item.Name != name {
				continue
			}

			return item, nil
		}

	case LibraryItemScale:
		for _, item := range library.Scales {
			if item.Name != name {
				continue
			}

			return item, nil
		}

	case LibraryItemGraph:
		for _, item := range library.Graphs {
			if item.Name != name {
				continue
			}

			return item, nil
		}

	case LibraryItemCollection:
		for _, item := range library.Collections {
			if item.Name != name {
				continue
			}

			return item, nil
		}
	}

	return nil, os.ErrNotExist
}

// ItemExists returns whether an item exists the library or not.
func (library *Library) ItemExists(id string, itemType int) bool {
	exists := false

	switch itemType {
	case LibraryItemSourceGroup, LibraryItemMetricGroup:
		if _, ok := library.Groups[id]; ok && library.Groups[id].Type == itemType {
			exists = true
		}

	case LibraryItemScale:
		_, exists = library.Scales[id]

	case LibraryItemGraph:
		_, exists = library.Graphs[id]

	case LibraryItemCollection:
		_, exists = library.Collections[id]
	}

	return exists
}

// LoadItem loads an item by its identifier.
func (library *Library) LoadItem(id string, itemType int) error {
	// Load item from file
	switch itemType {
	case LibraryItemSourceGroup, LibraryItemMetricGroup:
		tmpGroup := &Group{}

		filePath := library.getFilePath(id, itemType)

		fileInfo, err := utils.JSONLoad(filePath, &tmpGroup)
		if err != nil {
			return fmt.Errorf("in %s, %s", filePath, err)
		}

		library.Groups[id] = tmpGroup
		library.Groups[id].Type = itemType
		library.Groups[id].Modified = fileInfo.ModTime()

	case LibraryItemScale:
		tmpScale := &Scale{}

		filePath := library.getFilePath(id, itemType)

		fileInfo, err := utils.JSONLoad(filePath, &tmpScale)
		if err != nil {
			return fmt.Errorf("in %s, %s", filePath, err)
		}

		library.Scales[id] = tmpScale
		library.Scales[id].Modified = fileInfo.ModTime()

	case LibraryItemGraph:
		tmpGraph := &Graph{}

		filePath := library.getFilePath(id, itemType)

		fileInfo, err := utils.JSONLoad(filePath, &tmpGraph)
		if err != nil {
			return fmt.Errorf("in %s, %s", filePath, err)
		}

		library.Graphs[id] = tmpGraph
		library.Graphs[id].Modified = fileInfo.ModTime()

	case LibraryItemCollection:
		var tmpCollection *struct {
			*Collection
			Parent string `json:"parent"`
		}

		filePath := library.getFilePath(id, LibraryItemCollection)

		fileInfo, err := utils.JSONLoad(filePath, &tmpCollection)
		if err != nil {
			return fmt.Errorf("in %s, %s", filePath, err)
		}

		if !library.ItemExists(id, LibraryItemCollection) {
			library.Collections[id] = &Collection{}
		}

		*library.Collections[id] = *tmpCollection.Collection

		if tmpCollection.Parent != "" {
			library.Collections[id].ParentID = tmpCollection.Parent
		}

		library.Collections[id].Modified = fileInfo.ModTime()
	}

	return nil
}

// StoreItem stores an item into the library.
func (library *Library) StoreItem(item interface{}, itemType int) error {
	var itemStruct *Item

	switch itemType {
	case LibraryItemSourceGroup, LibraryItemMetricGroup:
		itemStruct = item.(*Group).GetItem()

	case LibraryItemScale:
		itemStruct = item.(*Scale).GetItem()

	case LibraryItemGraph:
		itemStruct = item.(*Graph).GetItem()

	case LibraryItemCollection:
		itemStruct = item.(*Collection).GetItem()

	default:
		return os.ErrInvalid
	}

	if itemStruct.ID == "" {
		uuidTemp, err := uuid.NewV4()
		if err != nil {
			return err
		}

		itemStruct.ID = uuidTemp.String()
	} else if !library.ItemExists(itemStruct.ID, itemType) {
		return os.ErrNotExist
	}

	// Check for name field presence/duplicates
	if itemStruct.Name == "" {
		return os.ErrInvalid
	}

	itemTemp, err := library.GetItemByName(itemStruct.Name, itemType)
	if err == nil {
		switch itemType {
		case LibraryItemSourceGroup, LibraryItemMetricGroup:
			if itemTemp.(*Group).ID != itemStruct.ID {
				logger.Log(logger.LevelError, "library", "duplicate group identifier `%s'", itemStruct.ID)
				return os.ErrExist
			}

		case LibraryItemScale:
			if itemTemp.(*Scale).ID != itemStruct.ID {
				logger.Log(logger.LevelError, "library", "duplicate scale identifier `%s'", itemStruct.ID)
				return os.ErrExist
			}

		case LibraryItemGraph:
			if itemTemp.(*Graph).ID != itemStruct.ID {
				logger.Log(logger.LevelError, "library", "duplicate graph identifier `%s'", itemStruct.ID)
				return os.ErrExist
			}

		case LibraryItemCollection:
			if itemTemp.(*Collection).ID != itemStruct.ID {
				logger.Log(logger.LevelError, "library", "duplicate collection identifier `%s'", itemStruct.ID)
				return os.ErrExist
			}
		}
	}

	// Store item into library
	switch itemType {
	case LibraryItemSourceGroup, LibraryItemMetricGroup:
		library.Groups[itemStruct.ID] = item.(*Group)
		library.Groups[itemStruct.ID].ID = itemStruct.ID

	case LibraryItemScale:
		library.Scales[itemStruct.ID] = item.(*Scale)
		library.Scales[itemStruct.ID].ID = itemStruct.ID

	case LibraryItemGraph:
		// Check for definition names duplicates
		groupSet := set.New(set.ThreadSafe)
		serieSet := set.New(set.ThreadSafe)

		for _, group := range item.(*Graph).Groups {
			if group == nil {
				logger.Log(logger.LevelError, "library", "found null group")
				return os.ErrInvalid
			} else if groupSet.Has(group.Name) {
				logger.Log(logger.LevelError, "library", "duplicate group name `%s'", group.Name)
				return os.ErrExist
			}

			groupSet.Add(group.Name)

			for _, serie := range group.Series {
				if serie == nil {
					logger.Log(logger.LevelError, "library", "found null serie in group `%s'", group.Name)
					return os.ErrInvalid
				} else if serieSet.Has(serie.Name) {
					logger.Log(logger.LevelError, "library", "duplicate serie name `%s'", serie.Name)
					return os.ErrExist
				}

				serieSet.Add(serie.Name)
			}
		}

		library.Graphs[itemStruct.ID] = item.(*Graph)
		library.Graphs[itemStruct.ID].ID = itemStruct.ID

	case LibraryItemCollection:
		library.Collections[itemStruct.ID] = item.(*Collection)
		library.Collections[itemStruct.ID].ID = itemStruct.ID
	}

	// Store JSON data
	if err := utils.JSONDump(library.getFilePath(itemStruct.ID, itemType), item, itemStruct.Modified); err != nil {
		return err
	}

	return nil
}

func (library *Library) getDirPath(itemType int) string {
	var dirName string

	switch itemType {
	case LibraryItemSourceGroup:
		dirName = "sourcegroups"

	case LibraryItemMetricGroup:
		dirName = "metricgroups"

	case LibraryItemScale:
		dirName = "scales"

	case LibraryItemGraph:
		dirName = "graphs"

	case LibraryItemCollection:
		dirName = "collections"
	}

	return path.Join(library.Config.DataDir, dirName)
}

func (library *Library) getFilePath(id string, itemType int) string {
	return path.Join(library.getDirPath(itemType), id[0:2], id[2:4], id+".json")
}
<|endoftext|>


<commit_before>/*
Package present has moved to code.google.com/p/go.tools/present.
*/
package present
<commit_msg>go.talks: remove pkg/present placeholder<commit_after><|endoftext|>


<commit_before>// Copyright 2016 CodisLabs. All Rights Reserved.
// Licensed under the MIT (MIT-LICENSE.txt) license.

package proxy

import (
	"bytes"
	"hash/crc32"
	"strings"

	"github.com/CodisLabs/codis/pkg/proxy/redis"
	"github.com/CodisLabs/codis/pkg/utils/errors"
)

var charmap [256]byte

func init() {
	for i := range charmap {
		c := byte(i)
		switch {
		case c >= 'A' && c <= 'Z':
			charmap[i] = c
		case c >= 'a' && c <= 'z':
			charmap[i] = c - 'a' + 'A'
		}
	}
}

type OpFlag uint32

func (f OpFlag) IsNotAllowed() bool {
	return (f & FlagNotAllow) != 0
}

func (f OpFlag) IsReadOnly() bool {
	const mask = FlagWrite | FlagMayWrite
	return (f & mask) == 0
}

type OpInfo struct {
	Name string
	Flag OpFlag
}

const (
	FlagWrite = 1 << iota
	FlagMayWrite
	FlagNotAllow
)

var opTable = make(map[string]OpInfo, 256)

func init() {
	for _, i := range []OpInfo{
		{"APPEND", FlagWrite},
		{"AUTH", 0},
		{"BGREWRITEAOF", FlagNotAllow},
		{"BGSAVE", FlagNotAllow},
		{"BITCOUNT", 0},
		{"BITOP", FlagWrite | FlagNotAllow},
		{"BITPOS", 0},
		{"BLPOP", FlagWrite | FlagNotAllow},
		{"BRPOP", FlagWrite | FlagNotAllow},
		{"BRPOPLPUSH", FlagWrite | FlagNotAllow},
		{"CLIENT", FlagNotAllow},
		{"COMMAND", 0},
		{"CONFIG", FlagNotAllow},
		{"DBSIZE", FlagNotAllow},
		{"DEBUG", FlagNotAllow},
		{"DECR", FlagWrite},
		{"DECRBY", FlagWrite},
		{"DEL", FlagWrite},
		{"DISCARD", FlagNotAllow},
		{"DUMP", 0},
		{"ECHO", 0},
		{"EVAL", FlagWrite},
		{"EVALSHA", FlagWrite},
		{"EXEC", FlagNotAllow},
		{"EXISTS", 0},
		{"EXPIRE", FlagWrite},
		{"EXPIREAT", FlagWrite},
		{"FLUSHALL", FlagWrite | FlagNotAllow},
		{"FLUSHDB", FlagWrite | FlagNotAllow},
		{"GET", 0},
		{"GETBIT", 0},
		{"GETRANGE", 0},
		{"GETSET", FlagWrite},
		{"HDEL", FlagWrite},
		{"HEXISTS", 0},
		{"HGET", 0},
		{"HGETALL", 0},
		{"HINCRBY", FlagWrite},
		{"HINCRBYFLOAT", FlagWrite},
		{"HKEYS", 0},
		{"HLEN", 0},
		{"HMGET", 0},
		{"HMSET", FlagWrite},
		{"HSCAN", 0},
		{"HSET", FlagWrite},
		{"HSETNX", FlagWrite},
		{"HVALS", 0},
		{"INCR", FlagWrite},
		{"INCRBY", FlagWrite},
		{"INCRBYFLOAT", FlagWrite},
		{"INFO", 0},
		{"KEYS", FlagNotAllow},
		{"LASTSAVE", FlagNotAllow},
		{"LATENCY", FlagNotAllow},
		{"LINDEX", 0},
		{"LINSERT", FlagWrite},
		{"LLEN", 0},
		{"LPOP", FlagWrite},
		{"LPUSH", FlagWrite},
		{"LPUSHX", FlagWrite},
		{"LRANGE", 0},
		{"LREM", FlagWrite},
		{"LSET", FlagWrite},
		{"LTRIM", FlagWrite},
		{"MGET", 0},
		{"MIGRATE", FlagWrite | FlagNotAllow},
		{"MONITOR", FlagNotAllow},
		{"MOVE", FlagWrite | FlagNotAllow},
		{"MSET", FlagWrite},
		{"MSETNX", FlagWrite | FlagNotAllow},
		{"MULTI", FlagNotAllow},
		{"OBJECT", FlagNotAllow},
		{"PERSIST", FlagWrite},
		{"PEXPIRE", FlagWrite},
		{"PEXPIREAT", FlagWrite},
		{"PFADD", FlagWrite},
		{"PFCOUNT", 0},
		{"PFDEBUG", FlagWrite},
		{"PFMERGE", FlagWrite},
		{"PFSELFTEST", 0},
		{"PING", 0},
		{"PSETEX", FlagWrite},
		{"PSUBSCRIBE", FlagNotAllow},
		{"PSYNC", FlagNotAllow},
		{"PTTL", 0},
		{"PUBLISH", FlagNotAllow},
		{"PUBSUB", 0},
		{"PUNSUBSCRIBE", FlagNotAllow},
		{"RANDOMKEY", FlagNotAllow},
		{"RENAME", FlagWrite | FlagNotAllow},
		{"RENAMENX", FlagWrite | FlagNotAllow},
		{"REPLCONF", FlagNotAllow},
		{"RESTORE", FlagWrite | FlagNotAllow},
		{"ROLE", 0},
		{"RPOP", FlagWrite},
		{"RPOPLPUSH", FlagWrite},
		{"RPUSH", FlagWrite},
		{"RPUSHX", FlagWrite},
		{"SADD", FlagWrite},
		{"SAVE", FlagNotAllow},
		{"SCAN", FlagNotAllow},
		{"SCARD", 0},
		{"SCRIPT", FlagNotAllow},
		{"SDIFF", 0},
		{"SDIFFSTORE", FlagWrite},
		{"SELECT", 0},
		{"SET", FlagWrite},
		{"SETBIT", FlagWrite},
		{"SETEX", FlagWrite},
		{"SETNX", FlagWrite},
		{"SETRANGE", FlagWrite},
		{"SHUTDOWN", FlagNotAllow},
		{"SINTER", 0},
		{"SINTERSTORE", FlagWrite},
		{"SISMEMBER", 0},
		{"SLAVEOF", FlagNotAllow},
		{"SLOTSCHECK", FlagNotAllow},
		{"SLOTSDEL", FlagWrite | FlagNotAllow},
		{"SLOTSHASHKEY", 0},
		{"SLOTSINFO", 0},
		{"SLOTSMAPPING", 0},
		{"SLOTSMGRTONE", FlagWrite | FlagNotAllow},
		{"SLOTSMGRTSLOT", FlagWrite | FlagNotAllow},
		{"SLOTSMGRTTAGONE", FlagWrite | FlagNotAllow},
		{"SLOTSMGRTTAGSLOT", FlagWrite | FlagNotAllow},
		{"SLOTSRESTORE", FlagWrite},
		{"SLOTSSCAN", 0},
		{"SLOWLOG", FlagNotAllow},
		{"SMEMBERS", 0},
		{"SMOVE", FlagWrite},
		{"SORT", FlagWrite},
		{"SPOP", FlagWrite},
		{"SRANDMEMBER", 0},
		{"SREM", FlagWrite},
		{"SSCAN", 0},
		{"STRLEN", 0},
		{"SUBSCRIBE", FlagNotAllow},
		{"SUBSTR", 0},
		{"SUNION", 0},
		{"SUNIONSTORE", FlagWrite},
		{"SYNC", FlagNotAllow},
		{"TIME", FlagNotAllow},
		{"TTL", 0},
		{"TYPE", 0},
		{"UNSUBSCRIBE", FlagNotAllow},
		{"UNWATCH", FlagNotAllow},
		{"WATCH", FlagNotAllow},
		{"ZADD", FlagWrite},
		{"ZCARD", 0},
		{"ZCOUNT", 0},
		{"ZINCRBY", FlagWrite},
		{"ZINTERSTORE", FlagWrite},
		{"ZLEXCOUNT", 0},
		{"ZRANGE", 0},
		{"ZRANGEBYLEX", 0},
		{"ZRANGEBYSCORE", 0},
		{"ZRANK", 0},
		{"ZREM", FlagWrite},
		{"ZREMRANGEBYLEX", FlagWrite},
		{"ZREMRANGEBYRANK", FlagWrite},
		{"ZREMRANGEBYSCORE", FlagWrite},
		{"ZREVRANGE", 0},
		{"ZREVRANGEBYLEX", 0},
		{"ZREVRANGEBYSCORE", 0},
		{"ZREVRANK", 0},
		{"ZSCAN", 0},
		{"ZSCORE", 0},
		{"ZUNIONSTORE", FlagWrite},
	} {
		opTable[i.Name] = i
	}
}

var (
	ErrBadMultiBulk = errors.New("bad multi-bulk for command")
	ErrBadOpStrLen  = errors.New("bad command length, too short or too long")
)

const MaxOpStrLen = 64

func getOpInfo(multi []*redis.Resp) (string, OpFlag, error) {
	if len(multi) < 1 {
		return "", 0, errors.Trace(ErrBadMultiBulk)
	}

	var upper [MaxOpStrLen]byte

	var op = multi[0].Value
	if len(op) == 0 || len(op) > len(upper) {
		return "", 0, errors.Trace(ErrBadOpStrLen)
	}
	for i := range op {
		if c := charmap[op[i]]; c != 0 {
			upper[i] = c
		} else {
			return strings.ToUpper(string(op)), FlagMayWrite, nil
		}
	}
	op = upper[:len(op)]
	if r, ok := opTable[string(op)]; ok {
		return r.Name, r.Flag, nil
	}
	return string(op), FlagMayWrite, nil
}

func Hash(key []byte) uint32 {
	const (
		TagBeg = '{'
		TagEnd = '}'
	)
	if beg := bytes.IndexByte(key, TagBeg); beg >= 0 {
		if end := bytes.IndexByte(key[beg+1:], TagEnd); end >= 0 {
			key = key[beg+1 : beg+1+end]
		}
	}
	return crc32.ChecksumIEEE(key)
}

func getHashKey(multi []*redis.Resp, opstr string) []byte {
	var index = 1
	switch opstr {
	case "ZINTERSTORE", "ZUNIONSTORE", "EVAL", "EVALSHA":
		index = 3
	}
	if index < len(multi) {
		return multi[index].Value
	}
	return nil
}
<commit_msg>proxy: update mapper for new commands<commit_after>// Copyright 2016 CodisLabs. All Rights Reserved.
// Licensed under the MIT (MIT-LICENSE.txt) license.

package proxy

import (
	"bytes"
	"hash/crc32"
	"strings"

	"github.com/CodisLabs/codis/pkg/proxy/redis"
	"github.com/CodisLabs/codis/pkg/utils/errors"
)

var charmap [256]byte

func init() {
	for i := range charmap {
		c := byte(i)
		switch {
		case c >= 'A' && c <= 'Z':
			charmap[i] = c
		case c >= 'a' && c <= 'z':
			charmap[i] = c - 'a' + 'A'
		}
	}
}

type OpFlag uint32

func (f OpFlag) IsNotAllowed() bool {
	return (f & FlagNotAllow) != 0
}

func (f OpFlag) IsReadOnly() bool {
	const mask = FlagWrite | FlagMayWrite
	return (f & mask) == 0
}

type OpInfo struct {
	Name string
	Flag OpFlag
}

const (
	FlagWrite = 1 << iota
	FlagMayWrite
	FlagNotAllow
)

var opTable = make(map[string]OpInfo, 256)

func init() {
	for _, i := range []OpInfo{
		{"APPEND", FlagWrite},
		{"ASKING", FlagNotAllow},
		{"AUTH", 0},
		{"BGREWRITEAOF", FlagNotAllow},
		{"BGSAVE", FlagNotAllow},
		{"BITCOUNT", 0},
		{"BITFIELD", FlagWrite},
		{"BITOP", FlagWrite | FlagNotAllow},
		{"BITPOS", 0},
		{"BLPOP", FlagWrite | FlagNotAllow},
		{"BRPOP", FlagWrite | FlagNotAllow},
		{"BRPOPLPUSH", FlagWrite | FlagNotAllow},
		{"CLIENT", FlagNotAllow},
		{"CLUSTER", FlagNotAllow},
		{"COMMAND", 0},
		{"CONFIG", FlagNotAllow},
		{"DBSIZE", FlagNotAllow},
		{"DEBUG", FlagNotAllow},
		{"DECR", FlagWrite},
		{"DECRBY", FlagWrite},
		{"DEL", FlagWrite},
		{"DISCARD", FlagNotAllow},
		{"DUMP", 0},
		{"ECHO", 0},
		{"EVAL", FlagWrite},
		{"EVALSHA", FlagWrite},
		{"EXEC", FlagNotAllow},
		{"EXISTS", 0},
		{"EXPIRE", FlagWrite},
		{"EXPIREAT", FlagWrite},
		{"FLUSHALL", FlagWrite | FlagNotAllow},
		{"FLUSHDB", FlagWrite | FlagNotAllow},
		{"GEOADD", FlagWrite},
		{"GEODIST", 0},
		{"GEOHASH", 0},
		{"GEOPOS", 0},
		{"GEORADIUS", FlagWrite},
		{"GEORADIUSBYMEMBER", FlagWrite},
		{"GET", 0},
		{"GETBIT", 0},
		{"GETRANGE", 0},
		{"GETSET", FlagWrite},
		{"HDEL", FlagWrite},
		{"HEXISTS", 0},
		{"HGET", 0},
		{"HGETALL", 0},
		{"HINCRBY", FlagWrite},
		{"HINCRBYFLOAT", FlagWrite},
		{"HKEYS", 0},
		{"HLEN", 0},
		{"HMGET", 0},
		{"HMSET", FlagWrite},
		{"HSCAN", 0},
		{"HSET", FlagWrite},
		{"HSETNX", FlagWrite},
		{"HSTRLEN", 0},
		{"HVALS", 0},
		{"INCR", FlagWrite},
		{"INCRBY", FlagWrite},
		{"INCRBYFLOAT", FlagWrite},
		{"INFO", 0},
		{"KEYS", FlagNotAllow},
		{"LASTSAVE", FlagNotAllow},
		{"LATENCY", FlagNotAllow},
		{"LINDEX", 0},
		{"LINSERT", FlagWrite},
		{"LLEN", 0},
		{"LPOP", FlagWrite},
		{"LPUSH", FlagWrite},
		{"LPUSHX", FlagWrite},
		{"LRANGE", 0},
		{"LREM", FlagWrite},
		{"LSET", FlagWrite},
		{"LTRIM", FlagWrite},
		{"MGET", 0},
		{"MIGRATE", FlagWrite | FlagNotAllow},
		{"MONITOR", FlagNotAllow},
		{"MOVE", FlagWrite | FlagNotAllow},
		{"MSET", FlagWrite},
		{"MSETNX", FlagWrite | FlagNotAllow},
		{"MULTI", FlagNotAllow},
		{"OBJECT", FlagNotAllow},
		{"PERSIST", FlagWrite},
		{"PEXPIRE", FlagWrite},
		{"PEXPIREAT", FlagWrite},
		{"PFADD", FlagWrite},
		{"PFCOUNT", 0},
		{"PFDEBUG", FlagWrite},
		{"PFMERGE", FlagWrite},
		{"PFSELFTEST", 0},
		{"PING", 0},
		{"PSETEX", FlagWrite},
		{"PSUBSCRIBE", FlagNotAllow},
		{"PSYNC", FlagNotAllow},
		{"PTTL", 0},
		{"PUBLISH", FlagNotAllow},
		{"PUBSUB", 0},
		{"PUNSUBSCRIBE", FlagNotAllow},
		{"RANDOMKEY", FlagNotAllow},
		{"READONLY", FlagNotAllow},
		{"READWRITE", FlagNotAllow},
		{"RENAME", FlagWrite | FlagNotAllow},
		{"RENAMENX", FlagWrite | FlagNotAllow},
		{"REPLCONF", FlagNotAllow},
		{"RESTORE", FlagWrite | FlagNotAllow},
		{"RESTORE-ASKING", FlagWrite | FlagNotAllow},
		{"ROLE", 0},
		{"RPOP", FlagWrite},
		{"RPOPLPUSH", FlagWrite},
		{"RPUSH", FlagWrite},
		{"RPUSHX", FlagWrite},
		{"SADD", FlagWrite},
		{"SAVE", FlagNotAllow},
		{"SCAN", FlagNotAllow},
		{"SCARD", 0},
		{"SCRIPT", FlagNotAllow},
		{"SDIFF", 0},
		{"SDIFFSTORE", FlagWrite},
		{"SELECT", 0},
		{"SET", FlagWrite},
		{"SETBIT", FlagWrite},
		{"SETEX", FlagWrite},
		{"SETNX", FlagWrite},
		{"SETRANGE", FlagWrite},
		{"SHUTDOWN", FlagNotAllow},
		{"SINTER", 0},
		{"SINTERSTORE", FlagWrite},
		{"SISMEMBER", 0},
		{"SLAVEOF", FlagNotAllow},
		{"SLOTSCHECK", FlagNotAllow},
		{"SLOTSDEL", FlagWrite | FlagNotAllow},
		{"SLOTSHASHKEY", 0},
		{"SLOTSINFO", 0},
		{"SLOTSMAPPING", 0},
		{"SLOTSMGRTONE", FlagWrite | FlagNotAllow},
		{"SLOTSMGRTSLOT", FlagWrite | FlagNotAllow},
		{"SLOTSMGRTTAGONE", FlagWrite | FlagNotAllow},
		{"SLOTSMGRTTAGSLOT", FlagWrite | FlagNotAllow},
		{"SLOTSRESTORE", FlagWrite},
		{"SLOTSSCAN", 0},
		{"SLOWLOG", FlagNotAllow},
		{"SMEMBERS", 0},
		{"SMOVE", FlagWrite},
		{"SORT", FlagWrite},
		{"SPOP", FlagWrite},
		{"SRANDMEMBER", 0},
		{"SREM", FlagWrite},
		{"SSCAN", 0},
		{"STRLEN", 0},
		{"SUBSCRIBE", FlagNotAllow},
		{"SUBSTR", 0},
		{"SUNION", 0},
		{"SUNIONSTORE", FlagWrite},
		{"SYNC", FlagNotAllow},
		{"TIME", FlagNotAllow},
		{"TOUCH", FlagWrite},
		{"TTL", 0},
		{"TYPE", 0},
		{"UNSUBSCRIBE", FlagNotAllow},
		{"UNWATCH", FlagNotAllow},
		{"WAIT", FlagNotAllow},
		{"WATCH", FlagNotAllow},
		{"ZADD", FlagWrite},
		{"ZCARD", 0},
		{"ZCOUNT", 0},
		{"ZINCRBY", FlagWrite},
		{"ZINTERSTORE", FlagWrite},
		{"ZLEXCOUNT", 0},
		{"ZRANGE", 0},
		{"ZRANGEBYLEX", 0},
		{"ZRANGEBYSCORE", 0},
		{"ZRANK", 0},
		{"ZREM", FlagWrite},
		{"ZREMRANGEBYLEX", FlagWrite},
		{"ZREMRANGEBYRANK", FlagWrite},
		{"ZREMRANGEBYSCORE", FlagWrite},
		{"ZREVRANGE", 0},
		{"ZREVRANGEBYLEX", 0},
		{"ZREVRANGEBYSCORE", 0},
		{"ZREVRANK", 0},
		{"ZSCAN", 0},
		{"ZSCORE", 0},
		{"ZUNIONSTORE", FlagWrite},
	} {
		opTable[i.Name] = i
	}
}

var (
	ErrBadMultiBulk = errors.New("bad multi-bulk for command")
	ErrBadOpStrLen  = errors.New("bad command length, too short or too long")
)

const MaxOpStrLen = 64

func getOpInfo(multi []*redis.Resp) (string, OpFlag, error) {
	if len(multi) < 1 {
		return "", 0, errors.Trace(ErrBadMultiBulk)
	}

	var upper [MaxOpStrLen]byte

	var op = multi[0].Value
	if len(op) == 0 || len(op) > len(upper) {
		return "", 0, errors.Trace(ErrBadOpStrLen)
	}
	for i := range op {
		if c := charmap[op[i]]; c != 0 {
			upper[i] = c
		} else {
			return strings.ToUpper(string(op)), FlagMayWrite, nil
		}
	}
	op = upper[:len(op)]
	if r, ok := opTable[string(op)]; ok {
		return r.Name, r.Flag, nil
	}
	return string(op), FlagMayWrite, nil
}

func Hash(key []byte) uint32 {
	const (
		TagBeg = '{'
		TagEnd = '}'
	)
	if beg := bytes.IndexByte(key, TagBeg); beg >= 0 {
		if end := bytes.IndexByte(key[beg+1:], TagEnd); end >= 0 {
			key = key[beg+1 : beg+1+end]
		}
	}
	return crc32.ChecksumIEEE(key)
}

func getHashKey(multi []*redis.Resp, opstr string) []byte {
	var index = 1
	switch opstr {
	case "ZINTERSTORE", "ZUNIONSTORE", "EVAL", "EVALSHA":
		index = 3
	}
	if index < len(multi) {
		return multi[index].Value
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"net/smtp"
	"fmt"
)

func SendEmail(to string, subject string, body string) {
	config, _ := Configuration()
  from := config.Email
	pass := config.Password
	cc := config.Email
	fmt.Println("Report User or Comment to " + from)
	msg := "From: " + from + "\n" +
		"To: " + from + "\n" +
    "Cc: " + cc + "\n" +
		"Subject: " + subject + "\n\n" +
		body

	smtp.SendMail("smtp.gmail.com:587",
		smtp.PlainAuth("", from, pass, "smtp.gmail.com"),
		from, []string{to}, []byte(msg))
}
<commit_msg>fix bug reporting<commit_after>package main

import (
	"net/smtp"
)

func SendEmail(to string, subject string, body string) {
	config, _ := Configuration()
  from := config.Email
	pass := config.Password
	cc := config.Email

	msg := "From: " + from + "\n" +
		"To: " + from + "\n" +
    "Cc: " + cc + "\n" +
		"Subject: " + subject + "\n\n" +
		body

	smtp.SendMail("smtp.gmail.com:587",
		smtp.PlainAuth("", from, pass, "smtp.gmail.com"),
		from, []string{from}, []byte(msg))
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/tls"
	"encoding/base64"
	"flag"
	"fmt"
	"github.com/zerklabs/auburn"
	"io/ioutil"
	"log"
	"net/smtp"
	"path/filepath"
	"strings"
	"time"
)

var (
	attachment = flag.String("attachment", "", "Include the attachment with the message")
	body       = flag.String("body", "This is a test message", "Set the body of the message")
	from       = flag.String("from", "", "Set the mail sender")
	port       = flag.Int("port", 25, "Set the SMTP server port. Default is 25")
	server     = flag.String("server", "", "Set the SMTP server")
	subject    = flag.String("subject", fmt.Sprintf("smtpsend test - %s", time.Now()), "Set the mail subject")
	useTls     = flag.Bool("tls", false, "If given, will try and send the message with STARTTLS")
	to         = flag.String("to", "", "Set the mail recipient(s). Separate multiple entries with commas")
)

func main() {
	flag.Parse()

	if len(*server) == 0 {
		log.Fatal("SMTP server required")
	}

	if len(*from) == 0 {
		log.Fatal("Mail sender required")
	}

	if len(*to) == 0 {
		log.Fatal("Mail recipient(s) required")
	}

	buildMailMessage()
}

func buildMailMessage() {
	boundary := auburn.RandomBase36()
	buf := bytes.NewBuffer(nil)

	buf.WriteString(fmt.Sprintf("From: %s\n", *from))
	buf.WriteString(fmt.Sprintf("To: %s\n", *to))
	buf.WriteString(fmt.Sprintf("Subject: %s\n", *subject))
	buf.WriteString("MIME-version: 1.0;\n")

	if len(*attachment) > 0 {
		buf.WriteString(fmt.Sprintf("Content-Type: multipart/mixed; boundary=\"%s\"\n", boundary))
		buf.WriteString(fmt.Sprintf("--%s\n", boundary))
	}

	buf.WriteString("Content-Type: text/plain; charset=\"UTF-8\";\n\n")
	buf.WriteString(*body)

	if len(*attachment) > 0 {
		attachmentName := filepath.Base(*attachment)
		b, err := ioutil.ReadFile(*attachment)

		if err != nil {
			log.Fatalf("Problem reading the given attachment:\n\t%s", err)
		}

		encodedLen := base64.StdEncoding.EncodedLen(len(b))
		encodedAttachment := make([]byte, encodedLen)
		base64.StdEncoding.Encode(encodedAttachment, b)

		buf.WriteString(fmt.Sprintf("\n\n--%s\n", boundary))
		buf.WriteString(fmt.Sprintf("Content-Type: application/octet-stream; name=\"%s\"\n", attachmentName))
		buf.WriteString(fmt.Sprintf("Content-Description: %s\n", attachmentName))
		buf.WriteString(fmt.Sprintf("Content-Disposition: attachment; filename=\"%s\"; size=%d\n", attachmentName, encodedLen))
		buf.WriteString("Content-Transfer-Encoding: base64\n\n")

		buf.Write(encodedAttachment)
		buf.WriteString(fmt.Sprintf("\n--%s--", boundary))
	}

	smtpUri := fmt.Sprintf("%s:%d", *server, *port)

	c, err := smtp.Dial(smtpUri)

	if err != nil {
		log.Fatalf("Error creating SMTP connection: %s", err)
	}

	if *useTls {
		// check if TLS is supported
		if ok, _ := c.Extension("STARTTLS"); ok {
			if err = c.StartTLS(&tls.Config{InsecureSkipVerify: true, ServerName: *server}); err != nil {
				c.Reset()
				c.Quit()

				log.Fatalf("Failed to establish TLS session: %s", err)
			}

			log.Println("TLS negotiated, sending over an encrypted channel")
		} else {
			log.Println("Server doesn't support TLS.. Sending over an unencrypted channel")
		}
	}

	// set the from addr
	if err = c.Mail(*from); err != nil {
		c.Reset()
		c.Quit()

		log.Fatalf("Failed to set the From address: %s", err)
	}

	// add the recipients
	for _, v := range strings.Split(*to, ",") {
		if err = c.Rcpt(v); err != nil {
			c.Reset()
			c.Quit()

			log.Fatalf("Failed to set a recipient: %s", err)
		}
	}

	w, err := c.Data()

	if err != nil {
		c.Reset()
		c.Quit()

		log.Fatalf("Failed to issue DATA command: %s", err)
	}

	_, err = w.Write(buf.Bytes())

	if err != nil {
		c.Reset()
		c.Quit()

		log.Fatalf("Failed to write DATA: %s", err)
	}

	if err = w.Close(); err != nil {
		c.Reset()
		c.Quit()

		log.Fatalf("Failed to close the DATA stream: %s", err)
	}

	c.Quit()

	log.Println("Message Sent")
}
<commit_msg>Removed functionality that was replaced by github.com/zerklabs/libsmtp<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/zerklabs/libsmtp"
	"log"
	"strings"
	"time"
)

var (
	attachment = flag.String("attachment", "", "Include the attachment with the message")
	body       = flag.String("body", "This is a test message", "Set the body of the message")
	from       = flag.String("from", "", "Set the mail sender")
	port       = flag.Int("port", 25, "Set the SMTP server port. Default is 25")
	server     = flag.String("server", "", "Set the SMTP server")
	subject    = flag.String("subject", fmt.Sprintf("smtpsend test - %s", time.Now()), "Set the mail subject")
	useTls     = flag.Bool("tls", false, "If given, will try and send the message with STARTTLS")
	to         = flag.String("to", "", "Set the mail recipient(s). Separate multiple entries with commas")
)

func main() {
	flag.Parse()

	client, err := libsmtp.New(*server, *port, *from, strings.Split(*to, ","), *useTls)

	if err != nil {
		log.Fatal(err)
	}

	client.Subject(*subject)
	client.Body(*body)

	if len(*attachment) > 0 {
		if err = client.AddAttachment(*attachment); err != nil {
			log.Fatal(err)
		}
	}

	if err = client.Send(); err != nil {
		log.Fatal(err)
	}

	log.Println("Message Sent")
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"os"
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
)

const (
	FAILURE_PACKAGE_YML_BLANK      = "./test/blank.yml"
	FAILURE_PACKAGE_YML_REPO_BLANK = "./test/repo-blank.yml"
)

func before() (*CLI, *bytes.Buffer, *bytes.Buffer) {
	outStream, errStream := new(bytes.Buffer), new(bytes.Buffer)
	cli := &CLI{logger: NewLogger(outStream, errStream)}
	return cli, outStream, errStream
}

func TestInstallSuccess(t *testing.T) {
	cli, outStream, _ := before()
	args := strings.Split(fmt.Sprintf("spm install %s -u %s -p %s", os.Getenv("REPOSITORY"), os.Getenv("USERNAME"), os.Getenv("PASSWORD")), " ")
	cli.Run(args)
	outString := outStream.String()
	assert.Contains(t, outString, fmt.Sprintf("Clone repository from https://github.com/%s (branch: %s)", os.Getenv("REPOSITORY"), "master"))
	assert.Contains(t, outString, "Check Deploy Result...")
	assert.Contains(t, outString, "Deploy is successful")
}

func TestInstallFailureNoUsername(t *testing.T) {
	cli, outStream, _ := before()
	args := strings.Split(fmt.Sprintf("spm install %s -p %s", os.Getenv("REPOSITORY"), os.Getenv("PASSWORD")), " ")
	_ = cli.Run(args)
	outString := outStream.String()
	assert.Contains(t, outString, "Username is required")
}

func TestInstallFailureNoPassword(t *testing.T) {
	cli, outStream, _ := before()
	args := strings.Split(fmt.Sprintf("spm install %s -u %s", os.Getenv("REPOSITORY"), os.Getenv("USERNAME")), " ")
	_ = cli.Run(args)
	outString := outStream.String()
	assert.Contains(t, outString, "Password is required")
}

func TestInstallFailureNoRepository(t *testing.T) {
	cli, outStream, _ := before()
	args := strings.Split(fmt.Sprintf("spm install -u %s -p %s", os.Getenv("USERNAME"), os.Getenv("PASSWORD")), " ")
	_ = cli.Run(args)
	outString := outStream.String()
	assert.Contains(t, outString, "Repository not specified")
}

func TestInstallFailureNoPackageYML(t *testing.T) {
	cli, outStream, _ := before()
	args := strings.Split(fmt.Sprintf("spm install -u %s -p %s -P %s", os.Getenv("USERNAME"), os.Getenv("PASSWORD"), "NOPACKAGE.yml"), " ")
	_ = cli.Run(args)
	outString := outStream.String()
	assert.Contains(t, outString, "open NOPACKAGE.yml: no such file or directory")
}

func TestInstallFailureInvalidCredentials(t *testing.T) {
	cli, outStream, _ := before()
	args := strings.Split(fmt.Sprintf("spm install %s -u hoge -p fuga", os.Getenv("REPOSITORY")), " ")
	_ = cli.Run(args)
	outString := outStream.String()
	assert.Contains(t, outString, "INVALID_LOGIN: Invalid username, password, security token; or user locked out.")
}

func TestInstallFailurePackageYmlBlank(t *testing.T) {
	cli, outStream, _ := before()
	args := strings.Split(fmt.Sprintf("spm install -u %s -p %s -P %s", os.Getenv("USERNAME"), os.Getenv("PASSWORD"), FAILURE_PACKAGE_YML_BLANK), " ")
	_ = cli.Run(args)
	outString := outStream.String()
	assert.Contains(t, outString, "Repository not specified")
}

func TestInstallFailurePackageYmlRepoBlank(t *testing.T) {
	cli, outStream, _ := before()
	args := strings.Split(fmt.Sprintf("spm install -u %s -p %s -P %s", os.Getenv("USERNAME"), os.Getenv("PASSWORD"), FAILURE_PACKAGE_YML_REPO_BLANK), " ")
	_ = cli.Run(args)
	outString := outStream.String()
	assert.Contains(t, outString, "Repository not specified")
}
<commit_msg>Mod test code for new cli interface<commit_after>package main

import (
	"bytes"
	"fmt"
	"os"
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
)

const (
	FAILURE_PACKAGE_YML_BLANK      = "./test/blank.yml"
	FAILURE_PACKAGE_YML_REPO_BLANK = "./test/repo-blank.yml"
)

func before() (*CLI, *bytes.Buffer, *bytes.Buffer) {
	outStream, errStream := new(bytes.Buffer), new(bytes.Buffer)
	cli := NewCli()
	cli.logger = NewLogger(outStream, errStream)
	return cli, outStream, errStream
}

func TestInstallSuccess(t *testing.T) {
	cli, outStream, _ := before()
	args := strings.Split(fmt.Sprintf("spm install %s -u %s -p %s", os.Getenv("REPOSITORY"), os.Getenv("USERNAME"), os.Getenv("PASSWORD")), " ")
	cli.Run(args)
	outString := outStream.String()
	assert.Contains(t, outString, fmt.Sprintf("Clone repository from https://github.com/%s (branch: %s)", os.Getenv("REPOSITORY"), "master"))
	assert.Contains(t, outString, "Check Deploy Result...")
	assert.Contains(t, outString, "Deploy is successful")
}

func TestInstallFailureNoUsername(t *testing.T) {
	cli, outStream, _ := before()
	args := strings.Split(fmt.Sprintf("spm install %s -p %s", os.Getenv("REPOSITORY"), os.Getenv("PASSWORD")), " ")
	_ = cli.Run(args)
	outString := outStream.String()
	assert.Contains(t, outString, "Username is required")
}

func TestInstallFailureNoPassword(t *testing.T) {
	cli, outStream, _ := before()
	args := strings.Split(fmt.Sprintf("spm install %s -u %s", os.Getenv("REPOSITORY"), os.Getenv("USERNAME")), " ")
	_ = cli.Run(args)
	outString := outStream.String()
	assert.Contains(t, outString, "Password is required")
}

func TestInstallFailureNoRepository(t *testing.T) {
	cli, outStream, _ := before()
	args := strings.Split(fmt.Sprintf("spm install -u %s -p %s", os.Getenv("USERNAME"), os.Getenv("PASSWORD")), " ")
	_ = cli.Run(args)
	outString := outStream.String()
	assert.Contains(t, outString, "Repository not specified")
}

func TestInstallFailureNoPackageYML(t *testing.T) {
	cli, outStream, _ := before()
	args := strings.Split(fmt.Sprintf("spm install -u %s -p %s -P %s", os.Getenv("USERNAME"), os.Getenv("PASSWORD"), "NOPACKAGE.yml"), " ")
	_ = cli.Run(args)
	outString := outStream.String()
	assert.Contains(t, outString, "open NOPACKAGE.yml: no such file or directory")
}

func TestInstallFailureInvalidCredentials(t *testing.T) {
	cli, outStream, _ := before()
	args := strings.Split(fmt.Sprintf("spm install %s -u hoge -p fuga", os.Getenv("REPOSITORY")), " ")
	_ = cli.Run(args)
	outString := outStream.String()
	assert.Contains(t, outString, "INVALID_LOGIN: Invalid username, password, security token; or user locked out.")
}

func TestInstallFailurePackageYmlBlank(t *testing.T) {
	cli, outStream, _ := before()
	args := strings.Split(fmt.Sprintf("spm install -u %s -p %s -P %s", os.Getenv("USERNAME"), os.Getenv("PASSWORD"), FAILURE_PACKAGE_YML_BLANK), " ")
	_ = cli.Run(args)
	outString := outStream.String()
	assert.Contains(t, outString, "Repository not specified")
}

func TestInstallFailurePackageYmlRepoBlank(t *testing.T) {
	cli, outStream, _ := before()
	args := strings.Split(fmt.Sprintf("spm install -u %s -p %s -P %s", os.Getenv("USERNAME"), os.Getenv("PASSWORD"), FAILURE_PACKAGE_YML_REPO_BLANK), " ")
	_ = cli.Run(args)
	outString := outStream.String()
	assert.Contains(t, outString, "Repository not specified")
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"os/signal"
	"syscall"
	"webircgateway/identd"

	"rsc.io/letsencrypt"
)

var (
	// Version - The current version of webircgateway
	Version    = "0.1.3"
	identdServ identd.Server
)

func main() {
	printVersion := flag.Bool("version", false, "Print the version")
	configFile := flag.String("config", "config.conf", "Config file location")
	runConfigTest := flag.Bool("test", false, "Just test the config file")
	flag.Parse()

	if *printVersion {
		fmt.Println(Version)
		os.Exit(0)
	}

	SetConfigFile(*configFile)
	log.Printf("Using config %s", Config.configFile)

	err := loadConfig()
	if err != nil {
		log.Printf("Config file error: %s", err.Error())
		os.Exit(1)
	}

	if *runConfigTest {
		log.Println("Config file is OK")
		os.Exit(0)
	}

	watchForSignals()
	maybeStartStaticFileServer()
	initListenerEngines()
	startServers()
	maybeStartIdentd()

	justWait := make(chan bool)
	<-justWait
}

func initListenerEngines() {
	engineConfigured := false
	for _, serverEngine := range Config.serverEngines {
		switch serverEngine {
		case "kiwiirc":
			kiwiircHTTPHandler()
			engineConfigured = true
		case "websocket":
			websocketHTTPHandler()
			engineConfigured = true
		case "sockjs":
			sockjsHTTPHandler()
			engineConfigured = true
		default:
			log.Printf("Invalid server engine: '%s'", serverEngine)
		}
	}

	if !engineConfigured {
		log.Fatal("No server engines configured")
	}
}

func maybeStartIdentd() {
	identdServ = identd.NewIdentdServer()

	if Config.identd {
		err := identdServ.Run()
		if err != nil {
			log.Printf("Error starting identd server: %s", err.Error())
		} else {
			log.Printf("Identd server started")
		}
	}
}

func maybeStartStaticFileServer() {
	if Config.webroot != "" {
		webroot := ConfigResolvePath(Config.webroot)
		log.Printf("Serving files from %s", webroot)
		http.Handle("/", http.FileServer(http.Dir(webroot)))
	}
}

func startServers() {
	// Add some general server info about this webircgateway instance
	http.HandleFunc("/webirc/", func(w http.ResponseWriter, r *http.Request) {
		out, _ := json.Marshal(map[string]interface{}{
			"name":    "webircgateway",
			"version": Version,
		})

		w.Write(out)
	})

	for _, server := range Config.servers {
		go startServer(server)
	}
}

func startServer(conf ConfigServer) {
	addr := fmt.Sprintf("%s:%d", conf.LocalAddr, conf.Port)

	if conf.TLS && conf.LetsEncryptCacheFile == "" {
		if conf.CertFile == "" || conf.KeyFile == "" {
			log.Println("'cert' and 'key' options must be set for TLS servers")
			return
		}

		tlsCert := ConfigResolvePath(conf.CertFile)
		tlsKey := ConfigResolvePath(conf.KeyFile)

		log.Printf("Listening with TLS on %s", addr)
		err := http.ListenAndServeTLS(addr, tlsCert, tlsKey, nil)
		if err != nil {
			log.Printf("Failed to listen with TLS: %s", err.Error())
		}
	} else if conf.TLS && conf.LetsEncryptCacheFile != "" {
		m := letsencrypt.Manager{}
		err := m.CacheFile(conf.LetsEncryptCacheFile)
		if err != nil {
			log.Printf("Failed to listen with letsencrypt TLS: %s", err.Error())
		}
		log.Printf("Listening with letsencrypt TLS on %s", addr)
		srv := &http.Server{
			Addr: addr,
			TLSConfig: &tls.Config{
				GetCertificate: m.GetCertificate,
			},
		}
		err = srv.ListenAndServeTLS("", "")
		log.Printf("Listening with letsencrypt failed: %s", err.Error())
	} else {
		log.Printf("Listening on %s", addr)
		err := http.ListenAndServe(addr, nil)
		log.Println(err)
	}
}

func watchForSignals() {
	c := make(chan os.Signal, 1)
	signal.Notify(c, syscall.SIGHUP)
	go func() {
		for {
			<-c
			fmt.Println("Recieved SIGHUP, reloading config file")
			loadConfig()
		}
	}()
}
<commit_msg>Version bump<commit_after>package main

import (
	"crypto/tls"
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"os/signal"
	"syscall"
	"webircgateway/identd"

	"rsc.io/letsencrypt"
)

var (
	// Version - The current version of webircgateway
	Version    = "0.2.3"
	identdServ identd.Server
)

func main() {
	printVersion := flag.Bool("version", false, "Print the version")
	configFile := flag.String("config", "config.conf", "Config file location")
	runConfigTest := flag.Bool("test", false, "Just test the config file")
	flag.Parse()

	if *printVersion {
		fmt.Println(Version)
		os.Exit(0)
	}

	SetConfigFile(*configFile)
	log.Printf("Using config %s", Config.configFile)

	err := loadConfig()
	if err != nil {
		log.Printf("Config file error: %s", err.Error())
		os.Exit(1)
	}

	if *runConfigTest {
		log.Println("Config file is OK")
		os.Exit(0)
	}

	watchForSignals()
	maybeStartStaticFileServer()
	initListenerEngines()
	startServers()
	maybeStartIdentd()

	justWait := make(chan bool)
	<-justWait
}

func initListenerEngines() {
	engineConfigured := false
	for _, serverEngine := range Config.serverEngines {
		switch serverEngine {
		case "kiwiirc":
			kiwiircHTTPHandler()
			engineConfigured = true
		case "websocket":
			websocketHTTPHandler()
			engineConfigured = true
		case "sockjs":
			sockjsHTTPHandler()
			engineConfigured = true
		default:
			log.Printf("Invalid server engine: '%s'", serverEngine)
		}
	}

	if !engineConfigured {
		log.Fatal("No server engines configured")
	}
}

func maybeStartIdentd() {
	identdServ = identd.NewIdentdServer()

	if Config.identd {
		err := identdServ.Run()
		if err != nil {
			log.Printf("Error starting identd server: %s", err.Error())
		} else {
			log.Printf("Identd server started")
		}
	}
}

func maybeStartStaticFileServer() {
	if Config.webroot != "" {
		webroot := ConfigResolvePath(Config.webroot)
		log.Printf("Serving files from %s", webroot)
		http.Handle("/", http.FileServer(http.Dir(webroot)))
	}
}

func startServers() {
	// Add some general server info about this webircgateway instance
	http.HandleFunc("/webirc/", func(w http.ResponseWriter, r *http.Request) {
		out, _ := json.Marshal(map[string]interface{}{
			"name":    "webircgateway",
			"version": Version,
		})

		w.Write(out)
	})

	for _, server := range Config.servers {
		go startServer(server)
	}
}

func startServer(conf ConfigServer) {
	addr := fmt.Sprintf("%s:%d", conf.LocalAddr, conf.Port)

	if conf.TLS && conf.LetsEncryptCacheFile == "" {
		if conf.CertFile == "" || conf.KeyFile == "" {
			log.Println("'cert' and 'key' options must be set for TLS servers")
			return
		}

		tlsCert := ConfigResolvePath(conf.CertFile)
		tlsKey := ConfigResolvePath(conf.KeyFile)

		log.Printf("Listening with TLS on %s", addr)
		err := http.ListenAndServeTLS(addr, tlsCert, tlsKey, nil)
		if err != nil {
			log.Printf("Failed to listen with TLS: %s", err.Error())
		}
	} else if conf.TLS && conf.LetsEncryptCacheFile != "" {
		m := letsencrypt.Manager{}
		err := m.CacheFile(conf.LetsEncryptCacheFile)
		if err != nil {
			log.Printf("Failed to listen with letsencrypt TLS: %s", err.Error())
		}
		log.Printf("Listening with letsencrypt TLS on %s", addr)
		srv := &http.Server{
			Addr: addr,
			TLSConfig: &tls.Config{
				GetCertificate: m.GetCertificate,
			},
		}
		err = srv.ListenAndServeTLS("", "")
		log.Printf("Listening with letsencrypt failed: %s", err.Error())
	} else {
		log.Printf("Listening on %s", addr)
		err := http.ListenAndServe(addr, nil)
		log.Println(err)
	}
}

func watchForSignals() {
	c := make(chan os.Signal, 1)
	signal.Notify(c, syscall.SIGHUP)
	go func() {
		for {
			<-c
			fmt.Println("Recieved SIGHUP, reloading config file")
			loadConfig()
		}
	}()
}
<|endoftext|>


<commit_before>package main

import "fmt"
import "strconv"
import "net/http"
import "io/ioutil"
import "net"
import "io"
import "runtime"
import "os/exec"
import "os"
import "flag"
import "strings"

var versionNum = "1.0.4"
var saveUrlContents = flag.Bool("keepweb", false, "Stores the contents of http requests in individual files")

func main() {
	fmt.Printf("Running tests, results are being written to result.txt")
	flag.Parse()
	path, err := os.Getwd()
	if err != nil {
		fmt.Printf("Fatal error: could not get working directory\r\n")
		panic(err)
	}
	outfile, err := os.Create(path + "/result.txt")
	if err != nil {
		fmt.Printf("Fatal error: could not create results file\r\n")
		panic(err)
	}
	defer outfile.Close()
	os.Stdout = outfile
	fmt.Printf("CCP connection test tool version " + versionNum + "\r\n")
	if *saveUrlContents == false {
		fmt.Printf("successful web requests will not be stored for examination, specify -keepweb=true to store them\r\n")
	}
	fmt.Printf("begin tests\r\n")
	runTests()
}

func runTests() {
	testPing()
	tcpConnect(26000)
	tcpConnect(3724)
	testPortOpen(26000)
	testPortOpen(3724)
	testLauncherURL("http://client.eveonline.com/patches/premium_patchinfoTQ_inc.txt")
	testLauncherURL("http://web.ccpgamescdn.com/launcher/tranquility/selfupdates.htm")
}

func testPortOpen(port uint64) {
	fmt.Printf("======PORT FORWARDING TEST======\r\n")
	urlStr := "http://tuq.in/tools/port.txt?port=" + strconv.FormatUint(port, 10)
	resp, err := http.Get(urlStr)
	if err != nil {
		fmt.Printf("port open check failed, could not get address\r\n")
		fmt.Println(err, "\r\n")
		return
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Printf("port open check failed, could not read response\r\n")
		fmt.Println(err, "\r\n")
		return
	}
	bodyString := string(body[:])
	fmt.Printf("The scan of port " + strconv.FormatUint(port, 10) + " returned " + bodyString + "\r\n")
}

func testLauncherURL(url string) {
	fmt.Printf("======HTTP TEST======\r\n")
	resp, err := http.Get(url)
	if err != nil {
		fmt.Printf("failed to connect to url " + url + " with error:\r\n")
		fmt.Println(err, "\r\n")
		return
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Printf("Connected, but failed to read contents of URL " + url + " with error:\r\n")
		fmt.Println(err, "\r\n")
		return
	}
	if *saveUrlContents {
		filename := "./" + cleanURL(url) + ".txt"
		outfile, err := os.Create(filename)
		if err != nil {
			fmt.Printf("could not create results file\r\n")
			panic(err)
		}
		defer outfile.Close()
		bodyString := string(body[:])
		outfile.WriteString(bodyString)
	}
	fmt.Printf("connected and read contents of " + url + " successfully\r\n")

}

func testPing() {
	fmt.Printf("======PING TEST======\r\n")
	if runtime.GOOS == "windows" {
		cmd := exec.Command("ping", "87.237.38.200")
		cmd.Stdout = os.Stdout
		err := cmd.Run()
		if err != nil {
			fmt.Printf("OS does not have ping utility\r\n")
			fmt.Println(err, "\r\n")
			return
		}
	} else if runtime.GOOS == "darwin" {
		cmd := exec.Command("ping", "-c 5", "87.237.38.200")
		cmd.Stdout = os.Stdout
		err := cmd.Run()
		if err != nil {
			fmt.Printf("OS does not have ping utility\r\n")
			fmt.Println(err, "\r\n")
			return
		}
	} else {
		fmt.Printf("unsupported OS, no ping\r\n")
	}
}

func tcpConnect(port uint64) {
	fmt.Printf("======TCP TEST======\r\n")
	fmt.Printf("tcpConnect on port " + strconv.FormatUint(port, 10) + "\r\n")
	conStr := "87.237.38.200:" + strconv.FormatUint(port, 10)
	conn, err := net.Dial("tcp", conStr)
	if err != nil {
		fmt.Printf("Error connecting:\r\n")
		fmt.Println(err, "\r\n")
		return
	}
	defer conn.Close()
	buffer := make([]byte, 0, 4096)
	_, err = conn.Read(buffer)
	if err != nil {
		if err != io.EOF {
			fmt.Printf("error reading buffer\r\n")
			fmt.Println(err, "\r\n")
			return
		}
	}
	fmt.Printf(string(buffer))
	fmt.Printf("connection successful\r\n")
}

func cleanURL(url string) string {
	holder := strings.Replace(url, ":", "-", -1)
	holder = strings.Replace(holder, "/", "", -1)
	holder = strings.Replace(holder, ".", "_", -1)
	return holder
}
<commit_msg>Sort imports<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"os/exec"
	"runtime"
	"strconv"
	"strings"
)

var versionNum = "1.0.4"
var saveUrlContents = flag.Bool("keepweb", false, "Stores the contents of http requests in individual files")

func main() {
	fmt.Printf("Running tests, results are being written to result.txt")
	flag.Parse()
	path, err := os.Getwd()
	if err != nil {
		fmt.Printf("Fatal error: could not get working directory\r\n")
		panic(err)
	}
	outfile, err := os.Create(path + "/result.txt")
	if err != nil {
		fmt.Printf("Fatal error: could not create results file\r\n")
		panic(err)
	}
	defer outfile.Close()
	os.Stdout = outfile
	fmt.Printf("CCP connection test tool version " + versionNum + "\r\n")
	if *saveUrlContents == false {
		fmt.Printf("successful web requests will not be stored for examination, specify -keepweb=true to store them\r\n")
	}
	fmt.Printf("begin tests\r\n")
	runTests()
}

func runTests() {
	testPing()
	tcpConnect(26000)
	tcpConnect(3724)
	testPortOpen(26000)
	testPortOpen(3724)
	testLauncherURL("http://client.eveonline.com/patches/premium_patchinfoTQ_inc.txt")
	testLauncherURL("http://web.ccpgamescdn.com/launcher/tranquility/selfupdates.htm")
}

func testPortOpen(port uint64) {
	fmt.Printf("======PORT FORWARDING TEST======\r\n")
	urlStr := "http://tuq.in/tools/port.txt?port=" + strconv.FormatUint(port, 10)
	resp, err := http.Get(urlStr)
	if err != nil {
		fmt.Printf("port open check failed, could not get address\r\n")
		fmt.Println(err, "\r\n")
		return
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Printf("port open check failed, could not read response\r\n")
		fmt.Println(err, "\r\n")
		return
	}
	bodyString := string(body[:])
	fmt.Printf("The scan of port " + strconv.FormatUint(port, 10) + " returned " + bodyString + "\r\n")
}

func testLauncherURL(url string) {
	fmt.Printf("======HTTP TEST======\r\n")
	resp, err := http.Get(url)
	if err != nil {
		fmt.Printf("failed to connect to url " + url + " with error:\r\n")
		fmt.Println(err, "\r\n")
		return
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Printf("Connected, but failed to read contents of URL " + url + " with error:\r\n")
		fmt.Println(err, "\r\n")
		return
	}
	if *saveUrlContents {
		filename := "./" + cleanURL(url) + ".txt"
		outfile, err := os.Create(filename)
		if err != nil {
			fmt.Printf("could not create results file\r\n")
			panic(err)
		}
		defer outfile.Close()
		bodyString := string(body[:])
		outfile.WriteString(bodyString)
	}
	fmt.Printf("connected and read contents of " + url + " successfully\r\n")

}

func testPing() {
	fmt.Printf("======PING TEST======\r\n")
	if runtime.GOOS == "windows" {
		cmd := exec.Command("ping", "87.237.38.200")
		cmd.Stdout = os.Stdout
		err := cmd.Run()
		if err != nil {
			fmt.Printf("OS does not have ping utility\r\n")
			fmt.Println(err, "\r\n")
			return
		}
	} else if runtime.GOOS == "darwin" {
		cmd := exec.Command("ping", "-c 5", "87.237.38.200")
		cmd.Stdout = os.Stdout
		err := cmd.Run()
		if err != nil {
			fmt.Printf("OS does not have ping utility\r\n")
			fmt.Println(err, "\r\n")
			return
		}
	} else {
		fmt.Printf("unsupported OS, no ping\r\n")
	}
}

func tcpConnect(port uint64) {
	fmt.Printf("======TCP TEST======\r\n")
	fmt.Printf("tcpConnect on port " + strconv.FormatUint(port, 10) + "\r\n")
	conStr := "87.237.38.200:" + strconv.FormatUint(port, 10)
	conn, err := net.Dial("tcp", conStr)
	if err != nil {
		fmt.Printf("Error connecting:\r\n")
		fmt.Println(err, "\r\n")
		return
	}
	defer conn.Close()
	buffer := make([]byte, 0, 4096)
	_, err = conn.Read(buffer)
	if err != nil {
		if err != io.EOF {
			fmt.Printf("error reading buffer\r\n")
			fmt.Println(err, "\r\n")
			return
		}
	}
	fmt.Printf(string(buffer))
	fmt.Printf("connection successful\r\n")
}

func cleanURL(url string) string {
	holder := strings.Replace(url, ":", "-", -1)
	holder = strings.Replace(holder, "/", "", -1)
	holder = strings.Replace(holder, ".", "_", -1)
	return holder
}
<|endoftext|>


<commit_before>// Copyright 2017 Mike Scherbakov
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"math"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"syscall"
	"text/tabwriter"
	"unsafe"
)

var UserProcs map[string]uint64
var TotalTasks uint64

type Limits struct {
	openFiles syscall.Rlimit
	nProc     syscall.Rlimit
}

type OutputEntry struct {
	pid          string
	cmd          string
	fds          uint64
	fdsLimit     string
	fdsPercent   float64
	nProc        uint64
	nProcLimit   string
	nProcPercent float64
}

func Prlimit(pid int, resource int, new_rlim *syscall.Rlimit, old_rlim *syscall.Rlimit) (err error) {
	// 302 is SYS_PRLIMIT64 system call. It is not exposed in Go as part of syscall, that's why we do it here.
	// Note, that this code only works on Linux x86_64
	// See details at https://groups.google.com/forum/#!topic/golang-dev/UNEHXy06O7Y
	_, _, e1 := syscall.RawSyscall6(302, uintptr(pid), uintptr(resource), uintptr(unsafe.Pointer(new_rlim)), uintptr(unsafe.Pointer(old_rlim)), 0, 0)
	if e1 != 0 {
		err = e1
	}
	return
}

func min(p1, p2 uint64) uint64 {
	if p1 < p2 {
		return p1
	}
	return p2
}

func GetLimits(pid string) (Limits, error) {
	var mylimit Limits
	var rlim syscall.Rlimit
	pidNu, _ := strconv.Atoi(pid)

	err := Prlimit(pidNu, syscall.RLIMIT_NOFILE, nil, &rlim)
	mylimit.openFiles = rlim
	if err != nil {
		return mylimit, err
	}

	// syscall.RLIMIT_NPROC is not defined, using number instead
	// See https://github.com/golang/go/issues/14854 for details
	err = Prlimit(pidNu, 6, nil, &rlim)
	mylimit.nProc = rlim
	if err != nil {
		return mylimit, err
	}
	return mylimit, nil
}

func countFiles(dir string) (uint64, error) {
	f, err := os.Open(dir)
	if err != nil {
		return 0, err
	}
	defer f.Close()
	files, err := f.Readdirnames(-1)
	if err != nil {
		return 0, err
	}
	return uint64(len(files)), nil
}

func GetFdOpen(pid string) (uint64, error) {
	return countFiles(filepath.Join("/proc", pid, "fd"))
}

func getStatus(pid string) (uid string, threads uint64, err error) {
	data, err := ioutil.ReadFile(filepath.Join("/proc", pid, "status"))
	if err != nil {
		// we can't do anything for this pid. It may not exist anymore, or we don't have enough capabilities
		return
	}
	str := string(data)

	reUid := regexp.MustCompile(`(?m:^Uid:[ \t]+([0-9]+)[ \t]+)`)
	matchedUid := reUid.FindStringSubmatch(str)
	// TODO: what if we can't parse? Need to do error-handling
	uid = matchedUid[1]

	reThreads := regexp.MustCompile(`(?m:^Threads:[ \t]+([0-9]+))`)
	matchedThreads := reThreads.FindStringSubmatch(str)
	// TODO: what if we can't parse? Need to do error-handling
	threads, err = strconv.ParseUint(matchedThreads[1], 10, 64)
	return
}

func GetNProcPerUid(pid string) (uint64, error) {
	// we actually need number of processes ran by this PID's UID
	uid, _, err := getStatus(pid)
	if err != nil {
		return 0, err
	}
	return uint64(UserProcs[uid]), nil
}

func CmdName(pid string) (string, error) {
	data, err := ioutil.ReadFile(filepath.Join("/proc", pid, "comm"))
	if err != nil {
		return "", err
	}
	return strings.TrimSuffix(string(data), "\n"), nil
}

func countProcesses(pids []string) {
	UserProcs = make(map[string]uint64)
	for _, pid := range pids {
		uid, threads, err := getStatus(pid)
		if err != nil {
			// process may no longer exist, so we just skip pid with errors
			// TODO: need to have better error handling here.
			// One of issues could be that we simply can't open any file, as we reached FD limit ourselves.
			continue
		}
		UserProcs[uid] += threads
		TotalTasks += threads
	}
}

func ProcTotalLimit() uint64 {
	data, err := ioutil.ReadFile("/proc/sys/kernel/threads-max")
	if err != nil {
		// we are in a big trouble if we can't get threads-max, so just panic right away
		panic(err)
	}
	threadsMax, err := strconv.ParseUint(strings.TrimSuffix(string(data), "\n"), 10, 64)

	if err != nil {
		panic(err)
	}
	return threadsMax
}

func FileNr() (used, max uint64) {
	data, err := ioutil.ReadFile("/proc/sys/fs/file-nr")
	if err != nil {
		panic(err)
	}
	str := strings.TrimSuffix(string(data), "\n")
	parsed := strings.Split(str, "\t")

	used, err = strconv.ParseUint(parsed[0], 10, 64)
	if err != nil {
		panic(err)
	}
	max, err = strconv.ParseUint(parsed[2], 10, 64)
	if err != nil {
		panic(err)
	}
	return
}

func FilePerProcMax() uint64 {
	data, err := ioutil.ReadFile("/proc/sys/fs/nr_open")
	if err != nil {
		panic(err)
	}
	x, err := strconv.ParseUint(strings.TrimSuffix(string(data), "\n"), 10, 64)
	if err != nil {
		panic(err)
	}
	return x
}

func main() {
	var nLines int
	if len(os.Args) == 2 && os.Args[1] == "-1" {
		nLines = -1
	} else {
		flag.IntVar(&nLines, "n", 10, "Output N most loaded processes. Use -1 to list all.")
		flag.Parse()
	}
	procs, err := filepath.Glob("/proc/[0-9]*")
	if err != nil {
		panic(err)
	}
	var pids []string
	for _, p := range procs {
		pids = append(pids, strings.Split(p, "/")[2])
	}

	// count number of processes per user
	countProcesses(pids)
	fmt.Printf("Tasks %d, system max is %d\n", TotalTasks, ProcTotalLimit())

	fileTotal, fileMax := FileNr()
	filePerProcMax := FilePerProcMax()
	fmt.Printf("File descriptors open %d, system max total is %d, system max per process is %d\n", fileTotal, fileMax, filePerProcMax)

	w := tabwriter.NewWriter(os.Stdout, 0, 0, 0, ' ', tabwriter.AlignRight)
	fmt.Fprintln(w, "PID\t FD\t FD-Rlim\t FD%\t Task\t Pr-Rlim\t Task%\t CMD\t")

	entries := []OutputEntry{}

	for _, pid := range pids {
		limits, err := GetLimits(pid)
		if err != nil {
			continue // this process may no longer exist. So let's skip it.
		}
		open, err := GetFdOpen(pid)
		if err != nil {
			continue // this process may no longer exist. So let's skip it.
		}

		fdsMaxPerProc := min(min(limits.openFiles.Cur, fileMax), filePerProcMax)
		fdsPercent := float64(open) / float64(fdsMaxPerProc) * 100.0
		if fdsPercent > 100 {
			fdsPercent = 100
		}

		fdsLimit := "-1"
		if limits.openFiles.Cur != math.MaxUint64 {
			fdsLimit = strconv.FormatUint(limits.openFiles.Cur, 10)
		}

		nProc, err := GetNProcPerUid(pid)
		if err != nil {
			continue // this process may no longer exist. So let's skip it.
		}
		nProcLimit := min(limits.nProc.Cur, ProcTotalLimit())
		pp := float64(nProc) / float64(nProcLimit) * 100.0
		if pp > 100 {
			pp = 100
		}

		// TODO: we need to check not just for uid=0, but also for CAP_SYS_RESOURCE & CAP_SYS_ADMIN
		// http://lxr.free-electrons.com/source/kernel/fork.c#L1529
		plStr := "-1"
		if limits.nProc.Cur != math.MaxUint64 {
			plStr = strconv.FormatUint(limits.nProc.Cur, 10)
		}

		cmd, err := CmdName(pid)
		if err != nil {
			continue // this process may no longer exist. So let's skip it.
		}

		entries = append(entries, OutputEntry{
			pid:          pid,
			cmd:          cmd,
			fds:          open,
			fdsLimit:     fdsLimit,
			fdsPercent:   fdsPercent,
			nProc:        nProc,
			nProcLimit:   plStr,
			nProcPercent: pp,
		})
	}

	f := func(i, j int) bool {
		return math.Max(entries[i].fdsPercent, entries[i].nProcPercent) > math.Max(entries[j].fdsPercent, entries[j].nProcPercent)
	}
	sort.Slice(entries, f)

	if nLines == -1 {
		nLines = len(entries)
	}
	for i := 0; i < nLines && i < len(entries); i++ {
		e := entries[i]
		fmt.Fprintf(w, "%s\t %d\t %s\t %2.1f\t %d\t %s\t %2.1f\t %s\t\n",
			e.pid, e.fds, e.fdsLimit, e.fdsPercent, e.nProc, e.nProcLimit, e.nProcPercent, e.cmd)
	}
	if err = w.Flush(); err != nil {
		panic(err)
	}

}
<commit_msg>Refactored: easier to understand and to add new metrics<commit_after>// Copyright 2017 Mike Scherbakov
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"math"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"syscall"
	"text/tabwriter"
	"unsafe"
)

type Metric struct {
	name string
	f    interface{}
	res  Entry
}

type Limits struct {
	openFiles syscall.Rlimit
	nProc     syscall.Rlimit
}

type Entry struct {
	v   uint64
	max uint64
}

type Boundary struct {
	by  string
	v   uint64
	max uint64
	p   float64
}

type OutputEntry struct {
	pid   string
	data  []Metric
	bound Boundary
	cmd   string
}

type Task struct {
	pid    string
	UidMap map[string]uint64
	limits Limits
}

type Tasks struct {
	pids  []Task
	total uint64
}

func Prlimit(pid int, resource int, new_rlim *syscall.Rlimit, old_rlim *syscall.Rlimit) (err error) {
	// 302 is SYS_PRLIMIT64 system call. It is not exposed in Go as part of syscall, that's why we do it here.
	// Note, that this code only works on Linux x86_64
	// See details at https://groups.google.com/forum/#!topic/golang-dev/UNEHXy06O7Y
	_, _, e1 := syscall.RawSyscall6(302, uintptr(pid), uintptr(resource), uintptr(unsafe.Pointer(new_rlim)), uintptr(unsafe.Pointer(old_rlim)), 0, 0)
	if e1 != 0 {
		err = e1
	}
	return
}

func GetLimits(pid string) (Limits, error) {
	var mylimit Limits
	var rlim syscall.Rlimit
	pidNu, _ := strconv.Atoi(pid)

	err := Prlimit(pidNu, syscall.RLIMIT_NOFILE, nil, &rlim)
	mylimit.openFiles = rlim
	if err != nil {
		return mylimit, err
	}

	// syscall.RLIMIT_NPROC is not defined, using number instead
	// See https://github.com/golang/go/issues/14854 for details
	err = Prlimit(pidNu, 6, nil, &rlim)
	mylimit.nProc = rlim
	if err != nil {
		return mylimit, err
	}
	return mylimit, nil
}

func countFiles(dir string) (uint64, error) {
	f, err := os.Open(dir)
	if err != nil {
		return 0, err
	}
	defer f.Close()
	files, err := f.Readdirnames(-1)
	if err != nil {
		return 0, err
	}
	return uint64(len(files)), nil
}

func (t Task) FdsRlim() (Entry, error) {
	v, err := countFiles(filepath.Join("/proc", t.pid, "fd"))
	if err != nil {
		return Entry{}, err // this process may no longer exist. So let's skip it.
	}
	return Entry{v, t.limits.openFiles.Cur}, nil
}

func (t Task) NprocRlim() (Entry, error) {
	uid, _, err := getStatus(t.pid)
	if err != nil {
		return Entry{}, err
	}
	return Entry{t.UidMap[uid], t.limits.nProc.Cur}, nil
}

func CalcBound(m []Metric) (b Boundary) {
	b.p = -1.0
	for i := 0; i < len(m); i++ {
		var p float64
		if m[i].res.max <= 0 {
			p = 100.0
		} else {
			p = 100.0 * float64(m[i].res.v) / float64(m[i].res.max)
		}
		if p > b.p {
			b.p = p
			b.by = m[i].name
			b.v = m[i].res.v
			b.max = m[i].res.max
		}
	}
	return
}

func getStatus(pid string) (uid string, threads uint64, err error) {
	str, err := ReadAndTrim(filepath.Join("/proc", pid, "status"))
	if err != nil {
		// we can't do anything for this pid. It may not exist anymore, or we don't have enough capabilities
		return
	}

	reUid := regexp.MustCompile(`(?m:^Uid:[ \t]+([0-9]+)[ \t]+)`)
	matchedUid := reUid.FindStringSubmatch(str)
	// TODO: what if we can't parse? Need to do error-handling
	uid = matchedUid[1]

	reThreads := regexp.MustCompile(`(?m:^Threads:[ \t]+([0-9]+))`)
	matchedThreads := reThreads.FindStringSubmatch(str)
	// TODO: what if we can't parse? Need to do error-handling
	threads, err = strconv.ParseUint(matchedThreads[1], 10, 64)
	return
}

func ProcTotalLimit() uint64 {
	str, err := ReadAndTrim("/proc/sys/kernel/threads-max")
	if err != nil {
		// we are in a big trouble if we can't get threads-max, so just panic right away
		panic(err)
	}
	threadsMax, err := strconv.ParseUint(str, 10, 64)
	if err != nil {
		panic(err)
	}
	return threadsMax
}

func TasksInit() *Tasks {
	t := new(Tasks)
	byUid := make(map[string]uint64)

	procs, err := filepath.Glob("/proc/[0-9]*")
	if err != nil {
		panic(err)
	}
	for _, p := range procs {
		pid := strings.Split(p, "/")[2]

		uid, threads, err := getStatus(pid)
		if err != nil {
			// process may no longer exist, so we just skip pid with errors
			// TODO: need to have better error handling here.
			// One of issues could be that we simply can't open any file, as we reached FD limit ourselves.
			continue
		}
		// TODO: we need to check for uid=0, CAP_SYS_RESOURCE & CAP_SYS_ADMIN
		// http://lxr.free-electrons.com/source/kernel/fork.c#L1529
		// and error handling
		l, _ := GetLimits(pid)
		byUid[uid] += threads
		t.pids = append(t.pids, Task{pid: pid, limits: l, UidMap: byUid})
		t.total += threads
	}
	return t
}

func ReadAndTrim(file string) (string, error) {
	data, err := ioutil.ReadFile(file)
	if err != nil {
		return "", err
	}
	return strings.TrimSuffix(string(data), "\n"), nil
}

func CmdName(pid string) (string, error) {
	return ReadAndTrim(filepath.Join("/proc", pid, "comm"))
}

func FileNr() (used, max uint64) {
	str, err := ReadAndTrim("/proc/sys/fs/file-nr")
	if err != nil {
		panic(err)
	}
	parsed := strings.Split(str, "\t")

	used, err = strconv.ParseUint(parsed[0], 10, 64)
	if err != nil {
		panic(err)
	}
	max, err = strconv.ParseUint(parsed[2], 10, 64)
	if err != nil {
		panic(err)
	}
	return
}

func FilePerProcMax() uint64 {
	str, err := ReadAndTrim("/proc/sys/fs/nr_open")
	if err != nil {
		panic(err)
	}
	x, err := strconv.ParseUint(str, 10, 64)
	if err != nil {
		panic(err)
	}
	return x
}

func main() {
	var nLines int
	if len(os.Args) == 2 && os.Args[1] == "-1" {
		nLines = -1
	} else {
		flag.IntVar(&nLines, "n", 10, "Output N most loaded processes. Use -1 to list all.")
		flag.Parse()
	}

	// ************** POPULATE CODE ********************
	tasks := TasksInit()
	procTotalLimit := ProcTotalLimit()
	fileTotal, fileMax := FileNr()
	filePerProcMax := FilePerProcMax()
	var out []OutputEntry
	for _, pid := range tasks.pids {
		m := []Metric{{name: "fds-rlim", f: pid.FdsRlim}, {name: "nproc-rlim", f: pid.NprocRlim}}
		for i := 0; i < len(m); i++ {
			//TODO: need error handling. What if we could not get FD limits, but got everything else?
			e, _ := m[i].f.(func() (Entry, error))()
			m[i].res = e
		}
		cmd, _ := CmdName(pid.pid)

		adds := []Metric{{name: "threads-max", res: Entry{tasks.total, procTotalLimit}},
			{name: "file-max", res: Entry{fileTotal, fileMax}},
			{name: "file-perproc-max", res: Entry{m[0].res.v, filePerProcMax}}}
		out = append(out, OutputEntry{pid.pid, m, CalcBound(append(m, adds...)), cmd})
	}

	// **************** PRINT CODE *********************
	fmt.Printf("Tasks %d, system max is %d\n", tasks.total, procTotalLimit)
	fmt.Printf("File descriptors open %d, system max total is %d, system max per process is %d\n", fileTotal, fileMax, filePerProcMax)
	w := tabwriter.NewWriter(os.Stdout, 0, 0, 0, ' ', tabwriter.AlignRight)
	fmt.Fprintln(w, "PID\t FD\t FD-RL\t TSK\t TSK-RL\t BOUND\t VAL\t MAX\t %USE\t CMD\t")

	sort.Slice(out, func(i, j int) bool { return out[i].bound.p > out[j].bound.p })
	if nLines == -1 {
		nLines = len(out)
	}
	for i := 0; i < nLines && i < len(out); i++ {
		fmt.Fprintf(w, "%s\t", out[i].pid)
		for j := 0; j < len(out[i].data); j++ {
			maxS := "-1"
			if out[i].data[j].res.max != math.MaxUint64 {
				maxS = strconv.FormatUint(out[i].data[j].res.max, 10)
			}
			fmt.Fprintf(w, "%d\t %s\t ", out[i].data[j].res.v, maxS)
		}
		fmt.Fprintf(w, "%s\t %d\t %d\t %2.1f\t ", out[i].bound.by, out[i].bound.v, out[i].bound.max, out[i].bound.p)
		fmt.Fprintf(w, "%s\t\n", out[i].cmd)
	}
	if err := w.Flush(); err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>package wallet

import (
	"fmt"
	"math"

	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"

	"github.com/NebulousLabs/bolt"
)

// updateConfirmedSet uses a consensus change to update the confirmed set of
// outputs as understood by the wallet.
func (w *Wallet) updateConfirmedSet(tx *bolt.Tx, cc modules.ConsensusChange) error {
	for _, diff := range cc.SiacoinOutputDiffs {
		// Verify that the diff is relevant to the wallet.
		_, exists := w.keys[diff.SiacoinOutput.UnlockHash]
		if !exists {
			continue
		}

		var err error
		if diff.Direction == modules.DiffApply {
			err = dbPutSiacoinOutput(tx, diff.ID, diff.SiacoinOutput)
		} else {
			err = dbDeleteSiacoinOutput(tx, diff.ID)
		}
		if err != nil {
			return err
		}
	}
	for _, diff := range cc.SiafundOutputDiffs {
		// Verify that the diff is relevant to the wallet.
		_, exists := w.keys[diff.SiafundOutput.UnlockHash]
		if !exists {
			continue
		}

		var err error
		if diff.Direction == modules.DiffApply {
			err = dbPutSiafundOutput(tx, diff.ID, diff.SiafundOutput)
		} else {
			err = dbDeleteSiafundOutput(tx, diff.ID)
		}
		if err != nil {
			return err
		}
	}
	for _, diff := range cc.SiafundPoolDiffs {
		if diff.Direction == modules.DiffApply {
			w.siafundPool = diff.Adjusted
		} else {
			w.siafundPool = diff.Previous
		}
	}
	return nil
}

// revertHistory reverts any transaction history that was destroyed by reverted
// blocks in the consensus change.
func (w *Wallet) revertHistory(cc modules.ConsensusChange) {
	for _, block := range cc.RevertedBlocks {
		// Remove any transactions that have been reverted.
		for i := len(block.Transactions) - 1; i >= 0; i-- {
			// If the transaction is relevant to the wallet, it will be the
			// most recent transaction appended to w.processedTransactions.
			// Relevance can be determined just by looking at the last element
			// of w.processedTransactions.
			txn := block.Transactions[i]
			txid := txn.ID()
			if len(w.processedTransactions) > 0 && txid == w.processedTransactions[len(w.processedTransactions)-1].TransactionID {
				w.processedTransactions = w.processedTransactions[:len(w.processedTransactions)-1]
				delete(w.processedTransactionMap, txid)
			}
		}

		// Remove the miner payout transaction if applicable.
		for _, mp := range block.MinerPayouts {
			_, exists := w.keys[mp.UnlockHash]
			if exists {
				w.processedTransactions = w.processedTransactions[:len(w.processedTransactions)-1]
				delete(w.processedTransactionMap, types.TransactionID(block.ID()))
				break
			}
		}
		w.consensusSetHeight--
	}
}

// applyHistory applies any transaction history that was introduced by the
// applied blocks.
func (w *Wallet) applyHistory(tx *bolt.Tx, applied []types.Block) error {
	for _, block := range applied {
		w.consensusSetHeight++
		// Apply the miner payout transaction if applicable.
		minerPT := modules.ProcessedTransaction{
			Transaction:           types.Transaction{},
			TransactionID:         types.TransactionID(block.ID()),
			ConfirmationHeight:    w.consensusSetHeight,
			ConfirmationTimestamp: block.Timestamp,
		}
		relevant := false
		for i, mp := range block.MinerPayouts {
			_, exists := w.keys[mp.UnlockHash]
			if exists {
				relevant = true
			}
			minerPT.Outputs = append(minerPT.Outputs, modules.ProcessedOutput{
				FundType:       types.SpecifierMinerPayout,
				MaturityHeight: w.consensusSetHeight + types.MaturityDelay,
				WalletAddress:  exists,
				RelatedAddress: mp.UnlockHash,
				Value:          mp.Value,
			})
			err := dbPutHistoricOutput(tx, types.OutputID(block.MinerPayoutID(uint64(i))), mp.Value)
			if err != nil {
				return fmt.Errorf("could not put historic output: %v", err)
			}
		}
		if relevant {
			w.processedTransactions = append(w.processedTransactions, minerPT)
			w.processedTransactionMap[minerPT.TransactionID] = &w.processedTransactions[len(w.processedTransactions)-1]
		}
		for _, txn := range block.Transactions {
			relevant := false
			pt := modules.ProcessedTransaction{
				Transaction:           txn,
				TransactionID:         txn.ID(),
				ConfirmationHeight:    w.consensusSetHeight,
				ConfirmationTimestamp: block.Timestamp,
			}

			for _, sci := range txn.SiacoinInputs {
				_, exists := w.keys[sci.UnlockConditions.UnlockHash()]
				if exists {
					relevant = true
				}
				val, err := dbGetHistoricOutput(tx, types.OutputID(sci.ParentID))
				if err != nil {
					return fmt.Errorf("could not get historic output: %v", err)
				}
				pt.Inputs = append(pt.Inputs, modules.ProcessedInput{
					FundType:       types.SpecifierSiacoinInput,
					WalletAddress:  exists,
					RelatedAddress: sci.UnlockConditions.UnlockHash(),
					Value:          val,
				})
			}

			for i, sco := range txn.SiacoinOutputs {
				_, exists := w.keys[sco.UnlockHash]
				if exists {
					relevant = true
				}
				pt.Outputs = append(pt.Outputs, modules.ProcessedOutput{
					FundType:       types.SpecifierSiacoinOutput,
					MaturityHeight: w.consensusSetHeight,
					WalletAddress:  exists,
					RelatedAddress: sco.UnlockHash,
					Value:          sco.Value,
				})
				err := dbPutHistoricOutput(tx, types.OutputID(txn.SiacoinOutputID(uint64(i))), sco.Value)
				if err != nil {
					return fmt.Errorf("could not put historic output: %v", err)
				}
			}

			for _, sfi := range txn.SiafundInputs {
				_, exists := w.keys[sfi.UnlockConditions.UnlockHash()]
				if exists {
					relevant = true
				}
				sfiValue, err := dbGetHistoricOutput(tx, types.OutputID(sfi.ParentID))
				if err != nil {
					return fmt.Errorf("could not get historic output: %v", err)
				}
				pt.Inputs = append(pt.Inputs, modules.ProcessedInput{
					FundType:       types.SpecifierSiafundInput,
					WalletAddress:  exists,
					RelatedAddress: sfi.UnlockConditions.UnlockHash(),
					Value:          sfiValue,
				})
				startVal, err := dbGetHistoricClaimStart(tx, sfi.ParentID)
				if err != nil {
					return fmt.Errorf("could not get historic claim start: %v", err)
				}
				claimValue := w.siafundPool.Sub(startVal).Mul(sfiValue)
				pt.Outputs = append(pt.Outputs, modules.ProcessedOutput{
					FundType:       types.SpecifierClaimOutput,
					MaturityHeight: w.consensusSetHeight + types.MaturityDelay,
					WalletAddress:  exists,
					RelatedAddress: sfi.ClaimUnlockHash,
					Value:          claimValue,
				})
			}

			for i, sfo := range txn.SiafundOutputs {
				_, exists := w.keys[sfo.UnlockHash]
				if exists {
					relevant = true
				}
				pt.Outputs = append(pt.Outputs, modules.ProcessedOutput{
					FundType:       types.SpecifierSiafundOutput,
					MaturityHeight: w.consensusSetHeight,
					WalletAddress:  exists,
					RelatedAddress: sfo.UnlockHash,
					Value:          sfo.Value,
				})
				id := txn.SiafundOutputID(uint64(i))
				err := dbPutHistoricOutput(tx, types.OutputID(id), sfo.Value)
				if err != nil {
					return fmt.Errorf("could not put historic output: %v", err)
				}
				err = dbPutHistoricClaimStart(tx, id, sfo.ClaimStart)
				if err != nil {
					return fmt.Errorf("could not put historic claim start: %v", err)
				}
			}

			for _, fee := range txn.MinerFees {
				pt.Outputs = append(pt.Outputs, modules.ProcessedOutput{
					FundType: types.SpecifierMinerFee,
					Value:    fee,
				})
			}
			if relevant {
				w.processedTransactions = append(w.processedTransactions, pt)
				w.processedTransactionMap[pt.TransactionID] = &w.processedTransactions[len(w.processedTransactions)-1]
			}
		}
	}

	return nil
}

// ProcessConsensusChange parses a consensus change to update the set of
// confirmed outputs known to the wallet.
func (w *Wallet) ProcessConsensusChange(cc modules.ConsensusChange) {
	if err := w.tg.Add(); err != nil {
		// The wallet should gracefully reject updates from the consensus set
		// or transaction pool that are sent after the wallet's Close method
		// has closed the wallet's ThreadGroup.
		return
	}
	defer w.tg.Done()
	w.mu.Lock()
	defer w.mu.Unlock()
	err := w.db.Update(func(tx *bolt.Tx) error {
		err := w.updateConfirmedSet(tx, cc)
		if err != nil {
			return err
		}
		w.revertHistory(cc)
		return w.applyHistory(tx, cc.AppliedBlocks)
	})
	if err != nil {
		w.log.Println("ERROR: failed to add consensus change:", err)
	}
}

// ReceiveUpdatedUnconfirmedTransactions updates the wallet's unconfirmed
// transaction set.
func (w *Wallet) ReceiveUpdatedUnconfirmedTransactions(txns []types.Transaction, _ modules.ConsensusChange) {
	if err := w.tg.Add(); err != nil {
		// Gracefully reject transactions if the wallet's Close method has
		// closed the wallet's ThreadGroup already.
		return
	}
	defer w.tg.Done()
	w.mu.Lock()
	defer w.mu.Unlock()
	err := w.db.Update(func(tx *bolt.Tx) error {
		w.unconfirmedProcessedTransactions = nil
		for _, txn := range txns {
			// To save on code complexity, relevancy is determined while building
			// up the wallet transaction.
			relevant := false
			pt := modules.ProcessedTransaction{
				Transaction:           txn,
				TransactionID:         txn.ID(),
				ConfirmationHeight:    types.BlockHeight(math.MaxUint64),
				ConfirmationTimestamp: types.Timestamp(math.MaxUint64),
			}
			for _, sci := range txn.SiacoinInputs {
				_, exists := w.keys[sci.UnlockConditions.UnlockHash()]
				if exists {
					relevant = true
				}
				val, err := dbGetHistoricOutput(tx, types.OutputID(sci.ParentID))
				if err != nil {
					return fmt.Errorf("could not get historic output: %v", err)
				}
				pt.Inputs = append(pt.Inputs, modules.ProcessedInput{
					FundType:       types.SpecifierSiacoinInput,
					WalletAddress:  exists,
					RelatedAddress: sci.UnlockConditions.UnlockHash(),
					Value:          val,
				})
			}
			for i, sco := range txn.SiacoinOutputs {
				_, exists := w.keys[sco.UnlockHash]
				if exists {
					relevant = true
				}
				pt.Outputs = append(pt.Outputs, modules.ProcessedOutput{
					FundType:       types.SpecifierSiacoinOutput,
					MaturityHeight: types.BlockHeight(math.MaxUint64),
					WalletAddress:  exists,
					RelatedAddress: sco.UnlockHash,
					Value:          sco.Value,
				})
				err := dbPutHistoricOutput(tx, types.OutputID(txn.SiacoinOutputID(uint64(i))), sco.Value)
				if err != nil {
					return fmt.Errorf("could not put historic output: %v", err)
				}
			}
			for _, fee := range txn.MinerFees {
				pt.Outputs = append(pt.Outputs, modules.ProcessedOutput{
					FundType: types.SpecifierMinerFee,
					Value:    fee,
				})
			}
			if relevant {
				w.unconfirmedProcessedTransactions = append(w.unconfirmedProcessedTransactions, pt)
			}
		}
		return nil
	})
	if err != nil {
		w.log.Println("ERROR: failed to add unconfirmed transactions:", err)
	}
}
<commit_msg>don't build ProccessedTransaction unless relevant<commit_after>package wallet

import (
	"fmt"
	"math"

	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"

	"github.com/NebulousLabs/bolt"
)

// isWalletAddress is a helper function that an UnlockHash is derived from
// one of the wallet's spendable keys.
func (w *Wallet) isWalletAddress(uh types.UnlockHash) bool {
	_, exists := w.keys[uh]
	return exists
}

// updateConfirmedSet uses a consensus change to update the confirmed set of
// outputs as understood by the wallet.
func (w *Wallet) updateConfirmedSet(tx *bolt.Tx, cc modules.ConsensusChange) error {
	for _, diff := range cc.SiacoinOutputDiffs {
		// Verify that the diff is relevant to the wallet.
		if !w.isWalletAddress(diff.SiacoinOutput.UnlockHash) {
			continue
		}

		var err error
		if diff.Direction == modules.DiffApply {
			err = dbPutSiacoinOutput(tx, diff.ID, diff.SiacoinOutput)
		} else {
			err = dbDeleteSiacoinOutput(tx, diff.ID)
		}
		if err != nil {
			return err
		}
	}
	for _, diff := range cc.SiafundOutputDiffs {
		// Verify that the diff is relevant to the wallet.
		if !w.isWalletAddress(diff.SiafundOutput.UnlockHash) {
			continue
		}

		var err error
		if diff.Direction == modules.DiffApply {
			err = dbPutSiafundOutput(tx, diff.ID, diff.SiafundOutput)
		} else {
			err = dbDeleteSiafundOutput(tx, diff.ID)
		}
		if err != nil {
			return err
		}
	}
	for _, diff := range cc.SiafundPoolDiffs {
		if diff.Direction == modules.DiffApply {
			w.siafundPool = diff.Adjusted
		} else {
			w.siafundPool = diff.Previous
		}
	}
	return nil
}

// revertHistory reverts any transaction history that was destroyed by reverted
// blocks in the consensus change.
func (w *Wallet) revertHistory(cc modules.ConsensusChange) {
	for _, block := range cc.RevertedBlocks {
		// Remove any transactions that have been reverted.
		for i := len(block.Transactions) - 1; i >= 0; i-- {
			// If the transaction is relevant to the wallet, it will be the
			// most recent transaction appended to w.processedTransactions.
			// Relevance can be determined just by looking at the last element
			// of w.processedTransactions.
			txn := block.Transactions[i]
			txid := txn.ID()
			if len(w.processedTransactions) > 0 && txid == w.processedTransactions[len(w.processedTransactions)-1].TransactionID {
				w.processedTransactions = w.processedTransactions[:len(w.processedTransactions)-1]
				delete(w.processedTransactionMap, txid)
			}
		}

		// Remove the miner payout transaction if applicable.
		for _, mp := range block.MinerPayouts {
			if w.isWalletAddress(mp.UnlockHash) {
				w.processedTransactions = w.processedTransactions[:len(w.processedTransactions)-1]
				delete(w.processedTransactionMap, types.TransactionID(block.ID()))
				break
			}
		}
		w.consensusSetHeight--
	}
}

// applyHistory applies any transaction history that was introduced by the
// applied blocks.
func (w *Wallet) applyHistory(tx *bolt.Tx, applied []types.Block) error {
	for _, block := range applied {
		w.consensusSetHeight++
		relevant := false
		for i, mp := range block.MinerPayouts {
			relevant = relevant || w.isWalletAddress(mp.UnlockHash)
			err := dbPutHistoricOutput(tx, types.OutputID(block.MinerPayoutID(uint64(i))), mp.Value)
			if err != nil {
				return fmt.Errorf("could not put historic output: %v", err)
			}
		}
		if relevant {
			// Apply the miner payout transaction if applicable.
			minerPT := modules.ProcessedTransaction{
				Transaction:           types.Transaction{},
				TransactionID:         types.TransactionID(block.ID()),
				ConfirmationHeight:    w.consensusSetHeight,
				ConfirmationTimestamp: block.Timestamp,
			}
			for _, mp := range block.MinerPayouts {
				minerPT.Outputs = append(minerPT.Outputs, modules.ProcessedOutput{
					FundType:       types.SpecifierMinerPayout,
					MaturityHeight: w.consensusSetHeight + types.MaturityDelay,
					WalletAddress:  w.isWalletAddress(mp.UnlockHash),
					RelatedAddress: mp.UnlockHash,
					Value:          mp.Value,
				})
			}
			w.processedTransactions = append(w.processedTransactions, minerPT)
			w.processedTransactionMap[minerPT.TransactionID] = &w.processedTransactions[len(w.processedTransactions)-1]
		}
		for _, txn := range block.Transactions {
			// determine if transaction is relevant
			relevant := false
			for _, sci := range txn.SiacoinInputs {
				relevant = relevant || w.isWalletAddress(sci.UnlockConditions.UnlockHash())
			}
			for i, sco := range txn.SiacoinOutputs {
				relevant = relevant || w.isWalletAddress(sco.UnlockHash)
				err := dbPutHistoricOutput(tx, types.OutputID(txn.SiacoinOutputID(uint64(i))), sco.Value)
				if err != nil {
					return fmt.Errorf("could not put historic output: %v", err)
				}
			}
			for _, sfi := range txn.SiafundInputs {
				relevant = relevant || w.isWalletAddress(sfi.UnlockConditions.UnlockHash())
			}

			for i, sfo := range txn.SiafundOutputs {
				relevant = relevant || w.isWalletAddress(sfo.UnlockHash)
				id := txn.SiafundOutputID(uint64(i))
				err := dbPutHistoricOutput(tx, types.OutputID(id), sfo.Value)
				if err != nil {
					return fmt.Errorf("could not put historic output: %v", err)
				}
				err = dbPutHistoricClaimStart(tx, id, sfo.ClaimStart)
				if err != nil {
					return fmt.Errorf("could not put historic claim start: %v", err)
				}
			}

			// only create a ProcessedTransaction if txn is relevant
			if !relevant {
				continue
			}

			pt := modules.ProcessedTransaction{
				Transaction:           txn,
				TransactionID:         txn.ID(),
				ConfirmationHeight:    w.consensusSetHeight,
				ConfirmationTimestamp: block.Timestamp,
			}

			for _, sci := range txn.SiacoinInputs {
				val, err := dbGetHistoricOutput(tx, types.OutputID(sci.ParentID))
				if err != nil {
					return fmt.Errorf("could not get historic output: %v", err)
				}
				pt.Inputs = append(pt.Inputs, modules.ProcessedInput{
					FundType:       types.SpecifierSiacoinInput,
					WalletAddress:  w.isWalletAddress(sci.UnlockConditions.UnlockHash()),
					RelatedAddress: sci.UnlockConditions.UnlockHash(),
					Value:          val,
				})
			}

			for _, sco := range txn.SiacoinOutputs {
				pt.Outputs = append(pt.Outputs, modules.ProcessedOutput{
					FundType:       types.SpecifierSiacoinOutput,
					MaturityHeight: w.consensusSetHeight,
					WalletAddress:  w.isWalletAddress(sco.UnlockHash),
					RelatedAddress: sco.UnlockHash,
					Value:          sco.Value,
				})
			}

			for _, sfi := range txn.SiafundInputs {
				sfiValue, err := dbGetHistoricOutput(tx, types.OutputID(sfi.ParentID))
				if err != nil {
					return fmt.Errorf("could not get historic output: %v", err)
				}
				pt.Inputs = append(pt.Inputs, modules.ProcessedInput{
					FundType:       types.SpecifierSiafundInput,
					WalletAddress:  w.isWalletAddress(sfi.UnlockConditions.UnlockHash()),
					RelatedAddress: sfi.UnlockConditions.UnlockHash(),
					Value:          sfiValue,
				})
				startVal, err := dbGetHistoricClaimStart(tx, sfi.ParentID)
				if err != nil {
					return fmt.Errorf("could not get historic claim start: %v", err)
				}
				claimValue := w.siafundPool.Sub(startVal).Mul(sfiValue)
				pt.Outputs = append(pt.Outputs, modules.ProcessedOutput{
					FundType:       types.SpecifierClaimOutput,
					MaturityHeight: w.consensusSetHeight + types.MaturityDelay,
					WalletAddress:  w.isWalletAddress(sfi.UnlockConditions.UnlockHash()),
					RelatedAddress: sfi.ClaimUnlockHash,
					Value:          claimValue,
				})
			}

			for _, sfo := range txn.SiafundOutputs {
				pt.Outputs = append(pt.Outputs, modules.ProcessedOutput{
					FundType:       types.SpecifierSiafundOutput,
					MaturityHeight: w.consensusSetHeight,
					WalletAddress:  w.isWalletAddress(sfo.UnlockHash),
					RelatedAddress: sfo.UnlockHash,
					Value:          sfo.Value,
				})
			}

			for _, fee := range txn.MinerFees {
				pt.Outputs = append(pt.Outputs, modules.ProcessedOutput{
					FundType: types.SpecifierMinerFee,
					Value:    fee,
				})
			}

			w.processedTransactions = append(w.processedTransactions, pt)
			w.processedTransactionMap[pt.TransactionID] = &w.processedTransactions[len(w.processedTransactions)-1]
		}
	}

	return nil
}

// ProcessConsensusChange parses a consensus change to update the set of
// confirmed outputs known to the wallet.
func (w *Wallet) ProcessConsensusChange(cc modules.ConsensusChange) {
	if err := w.tg.Add(); err != nil {
		// The wallet should gracefully reject updates from the consensus set
		// or transaction pool that are sent after the wallet's Close method
		// has closed the wallet's ThreadGroup.
		return
	}
	defer w.tg.Done()
	w.mu.Lock()
	defer w.mu.Unlock()
	err := w.db.Update(func(tx *bolt.Tx) error {
		err := w.updateConfirmedSet(tx, cc)
		if err != nil {
			return err
		}
		w.revertHistory(cc)
		return w.applyHistory(tx, cc.AppliedBlocks)
	})
	if err != nil {
		w.log.Println("ERROR: failed to add consensus change:", err)
	}
}

// ReceiveUpdatedUnconfirmedTransactions updates the wallet's unconfirmed
// transaction set.
func (w *Wallet) ReceiveUpdatedUnconfirmedTransactions(txns []types.Transaction, _ modules.ConsensusChange) {
	if err := w.tg.Add(); err != nil {
		// Gracefully reject transactions if the wallet's Close method has
		// closed the wallet's ThreadGroup already.
		return
	}
	defer w.tg.Done()

	w.mu.Lock()
	defer w.mu.Unlock()
	err := w.db.Update(func(tx *bolt.Tx) error {
		w.unconfirmedProcessedTransactions = nil
		for _, txn := range txns {
			// determine whether transaction is relevant to the wallet
			relevant := false
			for _, sci := range txn.SiacoinInputs {
				relevant = relevant || w.isWalletAddress(sci.UnlockConditions.UnlockHash())
			}
			for i, sco := range txn.SiacoinOutputs {
				relevant = relevant || w.isWalletAddress(sco.UnlockHash)
				err := dbPutHistoricOutput(tx, types.OutputID(txn.SiacoinOutputID(uint64(i))), sco.Value)
				if err != nil {
					return fmt.Errorf("could not put historic output: %v", err)
				}
			}

			// only create a ProcessedTransaction if txn is relevant
			if !relevant {
				continue
			}

			pt := modules.ProcessedTransaction{
				Transaction:           txn,
				TransactionID:         txn.ID(),
				ConfirmationHeight:    types.BlockHeight(math.MaxUint64),
				ConfirmationTimestamp: types.Timestamp(math.MaxUint64),
			}
			for _, sci := range txn.SiacoinInputs {
				val, err := dbGetHistoricOutput(tx, types.OutputID(sci.ParentID))
				if err != nil {
					return fmt.Errorf("could not get historic output: %v", err)
				}
				pt.Inputs = append(pt.Inputs, modules.ProcessedInput{
					FundType:       types.SpecifierSiacoinInput,
					WalletAddress:  w.isWalletAddress(sci.UnlockConditions.UnlockHash()),
					RelatedAddress: sci.UnlockConditions.UnlockHash(),
					Value:          val,
				})
			}
			for _, sco := range txn.SiacoinOutputs {
				pt.Outputs = append(pt.Outputs, modules.ProcessedOutput{
					FundType:       types.SpecifierSiacoinOutput,
					MaturityHeight: types.BlockHeight(math.MaxUint64),
					WalletAddress:  w.isWalletAddress(sco.UnlockHash),
					RelatedAddress: sco.UnlockHash,
					Value:          sco.Value,
				})
			}
			for _, fee := range txn.MinerFees {
				pt.Outputs = append(pt.Outputs, modules.ProcessedOutput{
					FundType: types.SpecifierMinerFee,
					Value:    fee,
				})
			}
			w.unconfirmedProcessedTransactions = append(w.unconfirmedProcessedTransactions, pt)
		}
		return nil
	})
	if err != nil {
		w.log.Println("ERROR: failed to add unconfirmed transactions:", err)
	}
}
<|endoftext|>