<commit_before>package freecache

import (
	"encoding/binary"
	"sync/atomic"
	"github.com/spaolacci/murmur3"
	"time"
)

type CacheStatus struct {
	TimeStamp,
	TimeRange,
	ItemsCount,
	ExpiredCount,
	HitCount,
	LookupCount int64
	HitRate,
	AvgLookupPerSecond,
	AvgHitPerSecond float64
}

func getCurrTimestamp() int64 {
	return int64(time.Now().Unix())
}

type Cache struct {
	segments   [256]segment
	hitCount   int64
	missCount  int64
	lastStatus CacheStatus
}

func hashFunc(data []byte) uint64 {
	return murmur3.Sum64(data)
}

// The cache size will be set to 512KB at minimum.
// If the size is set relatively large, you should call
// `debug.SetGCPercent()`, set it to a much smaller value
// to limit the memory consumption and GC pause time.
func NewCache(size int) (cache *Cache) {
	if size < 512*1024 {
		size = 512 * 1024
	}
	cache = new(Cache)
	for i := 0; i < 256; i++ {
		cache.segments[i] = newSegment(size/256, i)
	}
	cache.lastStatus = CacheStatus{TimeStamp: getCurrTimestamp()}
	return
}

// If the key is larger than 65535 or value is larger than 1/1024 of the cache size,
// the entry will not be written to the cache. expireSeconds <= 0 means no expire,
// but it can be evicted when cache is full.
func (cache *Cache) Set(key, value []byte, expireSeconds int) (err error) {
	hashVal := hashFunc(key)
	segId := hashVal & 255
	err = cache.segments[segId].set(key, value, hashVal, expireSeconds)
	return
}

// Get the value or not found error.
func (cache *Cache) Get(key []byte) (value []byte, err error) {
	hashVal := hashFunc(key)
	segId := hashVal & 255
	value, err = cache.segments[segId].get(key, hashVal)
	if err == nil {
		atomic.AddInt64(&cache.hitCount, 1)
	} else {
		atomic.AddInt64(&cache.missCount, 1)
	}
	return
}

func (cache *Cache) TTL(key []byte) (timeLeft uint32, err error) {
	hashVal := hashFunc(key)
	segId := hashVal & 255
	timeLeft, err = cache.segments[segId].ttl(key, hashVal)
	return
}

func (cache *Cache) Del(key []byte) (affected bool) {
	hashVal := hashFunc(key)
	segId := hashVal & 255
	affected = cache.segments[segId].del(key, hashVal)
	return
}

func (cache *Cache) SetInt(key int64, value []byte, expireSeconds int) (err error) {
	var bKey [8]byte
	binary.LittleEndian.PutUint64(bKey[:], uint64(key))
	return cache.Set(bKey[:], value, expireSeconds)
}

func (cache *Cache) GetInt(key int64) (value []byte, err error) {
	var bKey [8]byte
	binary.LittleEndian.PutUint64(bKey[:], uint64(key))
	return cache.Get(bKey[:])
}

func (cache *Cache) DelInt(key int64) (affected bool) {
	var bKey [8]byte
	binary.LittleEndian.PutUint64(bKey[:], uint64(key))
	return cache.Del(bKey[:])
}

func (cache *Cache) EvacuateCount() (count int64) {
	for i := 0; i < 256; i++ {
		count += atomic.LoadInt64(&cache.segments[i].totalEvacuate)
	}
	return
}

func (cache *Cache) ExpiredCount() (count int64) {
	for i := 0; i < 256; i++ {
		count += atomic.LoadInt64(&cache.segments[i].totalExpired)
	}
	return
}

func (cache *Cache) EntryCount() (entryCount int64) {
	for i := 0; i < 256; i++ {
		entryCount += atomic.LoadInt64(&cache.segments[i].entryCount)
	}
	return
}

func (cache *Cache) HitCount() int64 {
	return atomic.LoadInt64(&cache.hitCount)
}

func (cache *Cache) LookupCount() int64 {
	return atomic.LoadInt64(&cache.hitCount) + atomic.LoadInt64(&cache.missCount)
}

func (cache *Cache) HitRate() float64 {
	lookupCount := cache.LookupCount()
	if lookupCount == 0 {
		return 0
	} else {
		return float64(cache.HitCount()) / float64(lookupCount)
	}
}

func (cache *Cache) OverwriteCount() (overwriteCount int64) {
	for i := 0; i < 256; i++ {
		overwriteCount += atomic.LoadInt64(&cache.segments[i].overwrites)
	}
	return
}

func (cache *Cache) Clear() {
	for i := 0; i < 256; i++ {
		seg := cache.segments[i]
		seg.lock.Lock()
		cache.segments[i] = newSegment(len(cache.segments[i].rb.data), i)
		seg.lock.Unlock()
	}
	atomic.StoreInt64(&cache.hitCount, 0)
	atomic.StoreInt64(&cache.missCount, 0)
	cache.lastStatus.TimeStamp = getCurrTimestamp()
	cache.lastStatus.TimeRange = 0
	cache.lastStatus.ExpiredCount = 0
	cache.lastStatus.ItemsCount = 0
	cache.lastStatus.HitCount = 0
	cache.lastStatus.LookupCount = 0
	cache.lastStatus.HitRate = 0
}

func (cache *Cache) ResetStatistics() {
	atomic.StoreInt64(&cache.hitCount, 0)
	atomic.StoreInt64(&cache.missCount, 0)
	for i := 0; i < 256; i++ {
		cache.segments[i].lock.Lock()
		cache.segments[i].resetStatistics()
		cache.segments[i].lock.Unlock()
	}
	cache.lastStatus.TimeStamp = getCurrTimestamp()
	cache.lastStatus.TimeRange = 0
	cache.lastStatus.ExpiredCount = 0
	cache.lastStatus.ItemsCount = 0
	cache.lastStatus.HitCount = 0
	cache.lastStatus.LookupCount = 0
	cache.lastStatus.HitRate = 0
}

func (cache *Cache) GetStatistics() *CacheStatus {
	now := getCurrTimestamp()
	currentStatus := CacheStatus{TimeStamp: now, TimeRange: now - cache.lastStatus.TimeStamp}
	itemsCount := cache.EntryCount()
	expiredCount := cache.ExpiredCount()
	hitCount := cache.HitCount()
	lookupCount := cache.LookupCount()
	if currentStatus.TimeRange > 0 {
		currentStatus.ExpiredCount = expiredCount
		currentStatus.ItemsCount = itemsCount
		currentStatus.HitCount = hitCount - cache.lastStatus.HitCount
		currentStatus.LookupCount = lookupCount - cache.lastStatus.LookupCount
		currentStatus.AvgLookupPerSecond = float64(currentStatus.LookupCount) / float64(currentStatus.TimeRange)
		currentStatus.AvgHitPerSecond = float64(currentStatus.HitCount) / float64(currentStatus.TimeRange)
		if currentStatus.LookupCount != 0 {
			currentStatus.HitRate = float64(currentStatus.HitCount) / float64(currentStatus.LookupCount)
		} else {
			currentStatus.HitRate = 0.0
		}
		cache.lastStatus.TimeStamp = now
		cache.lastStatus.TimeRange = 0
		cache.lastStatus.ExpiredCount = expiredCount
		cache.lastStatus.ItemsCount = itemsCount
		cache.lastStatus.HitCount = hitCount
		cache.lastStatus.LookupCount = lookupCount
		cache.lastStatus.HitRate = 0

	}
	return &currentStatus
}
<commit_msg>优化统计值<commit_after>package freecache

import (
	"encoding/binary"
	"sync/atomic"
	"github.com/spaolacci/murmur3"
	"time"
)

type CacheStatus struct {
	TimeStamp,
	timeRange,
	ItemsCount,
	ExpiredCount,
	hitCount,
	lookupCount int64
	HitRate,
	AvgLookupPerSecond,
	AvgHitPerSecond float64
}

func getCurrTimestamp() int64 {
	return int64(time.Now().Unix())
}

type Cache struct {
	segments   [256]segment
	hitCount   int64
	missCount  int64
	lastStatus CacheStatus
}

func hashFunc(data []byte) uint64 {
	return murmur3.Sum64(data)
}

// The cache size will be set to 512KB at minimum.
// If the size is set relatively large, you should call
// `debug.SetGCPercent()`, set it to a much smaller value
// to limit the memory consumption and GC pause time.
func NewCache(size int) (cache *Cache) {
	if size < 512*1024 {
		size = 512 * 1024
	}
	cache = new(Cache)
	for i := 0; i < 256; i++ {
		cache.segments[i] = newSegment(size/256, i)
	}
	cache.lastStatus = CacheStatus{TimeStamp: getCurrTimestamp()}
	return
}

// If the key is larger than 65535 or value is larger than 1/1024 of the cache size,
// the entry will not be written to the cache. expireSeconds <= 0 means no expire,
// but it can be evicted when cache is full.
func (cache *Cache) Set(key, value []byte, expireSeconds int) (err error) {
	hashVal := hashFunc(key)
	segId := hashVal & 255
	err = cache.segments[segId].set(key, value, hashVal, expireSeconds)
	return
}

// Get the value or not found error.
func (cache *Cache) Get(key []byte) (value []byte, err error) {
	hashVal := hashFunc(key)
	segId := hashVal & 255
	value, err = cache.segments[segId].get(key, hashVal)
	if err == nil {
		atomic.AddInt64(&cache.hitCount, 1)
	} else {
		atomic.AddInt64(&cache.missCount, 1)
	}
	return
}

func (cache *Cache) TTL(key []byte) (timeLeft uint32, err error) {
	hashVal := hashFunc(key)
	segId := hashVal & 255
	timeLeft, err = cache.segments[segId].ttl(key, hashVal)
	return
}

func (cache *Cache) Del(key []byte) (affected bool) {
	hashVal := hashFunc(key)
	segId := hashVal & 255
	affected = cache.segments[segId].del(key, hashVal)
	return
}

func (cache *Cache) SetInt(key int64, value []byte, expireSeconds int) (err error) {
	var bKey [8]byte
	binary.LittleEndian.PutUint64(bKey[:], uint64(key))
	return cache.Set(bKey[:], value, expireSeconds)
}

func (cache *Cache) GetInt(key int64) (value []byte, err error) {
	var bKey [8]byte
	binary.LittleEndian.PutUint64(bKey[:], uint64(key))
	return cache.Get(bKey[:])
}

func (cache *Cache) DelInt(key int64) (affected bool) {
	var bKey [8]byte
	binary.LittleEndian.PutUint64(bKey[:], uint64(key))
	return cache.Del(bKey[:])
}

func (cache *Cache) EvacuateCount() (count int64) {
	for i := 0; i < 256; i++ {
		count += atomic.LoadInt64(&cache.segments[i].totalEvacuate)
	}
	return
}

func (cache *Cache) ExpiredCount() (count int64) {
	for i := 0; i < 256; i++ {
		count += atomic.LoadInt64(&cache.segments[i].totalExpired)
	}
	return
}

func (cache *Cache) EntryCount() (entryCount int64) {
	for i := 0; i < 256; i++ {
		entryCount += atomic.LoadInt64(&cache.segments[i].entryCount)
	}
	return
}

func (cache *Cache) HitCount() int64 {
	return atomic.LoadInt64(&cache.hitCount)
}

func (cache *Cache) LookupCount() int64 {
	return atomic.LoadInt64(&cache.hitCount) + atomic.LoadInt64(&cache.missCount)
}

func (cache *Cache) HitRate() float64 {
	lookupCount := cache.LookupCount()
	if lookupCount == 0 {
		return 0
	} else {
		return float64(cache.HitCount()) / float64(lookupCount)
	}
}

func (cache *Cache) OverwriteCount() (overwriteCount int64) {
	for i := 0; i < 256; i++ {
		overwriteCount += atomic.LoadInt64(&cache.segments[i].overwrites)
	}
	return
}

func (cache *Cache) Clear() {
	for i := 0; i < 256; i++ {
		seg := cache.segments[i]
		seg.lock.Lock()
		cache.segments[i] = newSegment(len(cache.segments[i].rb.data), i)
		seg.lock.Unlock()
	}
	atomic.StoreInt64(&cache.hitCount, 0)
	atomic.StoreInt64(&cache.missCount, 0)
	cache.lastStatus.TimeStamp = getCurrTimestamp()
	cache.lastStatus.timeRange = 0
	cache.lastStatus.ExpiredCount = 0
	cache.lastStatus.ItemsCount = 0
	cache.lastStatus.hitCount = 0
	cache.lastStatus.lookupCount = 0
	cache.lastStatus.HitRate = 0
}

func (cache *Cache) ResetStatistics() {
	atomic.StoreInt64(&cache.hitCount, 0)
	atomic.StoreInt64(&cache.missCount, 0)
	for i := 0; i < 256; i++ {
		cache.segments[i].lock.Lock()
		cache.segments[i].resetStatistics()
		cache.segments[i].lock.Unlock()
	}
	cache.lastStatus.TimeStamp = getCurrTimestamp()
	cache.lastStatus.timeRange = 0
	cache.lastStatus.ExpiredCount = 0
	cache.lastStatus.ItemsCount = 0
	cache.lastStatus.hitCount = 0
	cache.lastStatus.lookupCount = 0
	cache.lastStatus.HitRate = 0
}

func (cache *Cache) GetStatistics() *CacheStatus {
	now := getCurrTimestamp()
	currentStatus := CacheStatus{TimeStamp: now, timeRange: now - cache.lastStatus.TimeStamp}
	itemsCount := cache.EntryCount()
	expiredCount := cache.ExpiredCount()
	hitCount := cache.HitCount()
	lookupCount := cache.LookupCount()
	if currentStatus.timeRange > 0 {
		currentStatus.ExpiredCount = expiredCount
		currentStatus.ItemsCount = itemsCount
		currentStatus.hitCount = hitCount - cache.lastStatus.hitCount
		currentStatus.lookupCount = lookupCount - cache.lastStatus.lookupCount
		currentStatus.AvgLookupPerSecond = float64(currentStatus.lookupCount) / float64(currentStatus.timeRange)
		currentStatus.AvgHitPerSecond = float64(currentStatus.hitCount) / float64(currentStatus.timeRange)
		if currentStatus.lookupCount != 0 {
			currentStatus.HitRate = float64(currentStatus.hitCount) / float64(currentStatus.lookupCount)
		} else {
			currentStatus.HitRate = 0.0
		}
		cache.lastStatus.TimeStamp = now
		cache.lastStatus.timeRange = 0
		cache.lastStatus.ExpiredCount = expiredCount
		cache.lastStatus.ItemsCount = itemsCount
		cache.lastStatus.hitCount = hitCount
		cache.lastStatus.lookupCount = lookupCount
		cache.lastStatus.HitRate = 0

	}
	return &currentStatus
}
<|endoftext|>


<commit_before>package ec2

import (
	"fmt"
	"github.com/dynport/dgtk/cli"
	"github.com/dynport/gocli"
	"github.com/dynport/gocloud/aws/ec2"
	"io/ioutil"
	"log"
	"os"
	"sort"
	"strings"
	"time"
)

const HOURS_PER_MONTH = 365 * 24.0 / 12.0

func Register(router *cli.Router) {
	router.RegisterFunc("aws/ec2/instances/describe", DescribeInstances, "Describe ec2 instances")
	router.Register("aws/ec2/instances/run", &RunInstances{}, "Run ec2 instances")
	router.Register("aws/ec2/images/create", &CreateImage{}, "Create image from instance")
	router.Register("aws/ec2/instances/terminate", &TerminateInstances{}, "Terminate ec2 instances")
	router.Register("aws/ec2/tags/create", &CreateTags{}, "Create Tags")
	router.RegisterFunc("aws/ec2/tags/describe", DescribeTags, "Describe Tags")
	router.Register("aws/ec2/images/describe", &DescribeImages{}, "Describe ec2 Images")
	router.RegisterFunc("aws/ec2/key-pairs/describe", DescribeKeyPairs, "Describe key pairs")
	router.RegisterFunc("aws/ec2/addresses/describe", DescribeAddresses, "Describe Addresses")
	router.RegisterFunc("aws/ec2/security-groups/describe", DescribeSecurityGroups, "Describe Security Groups")
	router.RegisterFunc("aws/ec2/spot-price-history/describe", DescribeSpotPriceHistory, "Describe Spot Price History")
	router.Register("aws/ec2/prices", &Prices{Region: os.Getenv("AWS_DEFAULT_REGION")}, "EC2 Prices")
}

func client() *ec2.Client {
	return ec2.NewFromEnv()
}

// http://docs.aws.amazon.com/general/latest/gr/rande.html#ec2_region
var regionMapping = map[string]string{
	"eu-ireland": "eu-west-1",
	"eu-west":    "eu-west-1",
	"apac-tokyo": "ap-northeast-1",
	"apac-sin":   "ap-southeast-1",
	"apac-syd":   "ap-southeast-2",
}

// http://docs.aws.amazon.com/general/latest/gr/rande.html#ec2_region
var regionMappingForOd = map[string]string{
	"eu-west-1":      "eu-ireland",
	"us-east-1":      "us-east",
	"us-west-1":      "us-west",
	"ap-northeast-1": "apac-tokyo",
	"ap-southeast-1": "apac-sin",
	"ap-southeast-2": "apac-syd",
}

func normalizeRegionForOd(region string) string {
	normalized, ok := regionMappingForOd[region]
	if ok {
		return normalized
	}
	return region
}

func normalizeRegion(raw string) string {
	if v, ok := regionMapping[raw]; ok {
		return v
	}
	return raw
}
func monthlyPrice(price float64) string {
	return fmt.Sprintf("%.02f", price)
}

type DescribeImages struct {
	Canonical bool `cli:"type=opt long=canonical"`
	Self      bool `cli:"type=opt long=self"`
	Ubuntu    bool `cli:"type=opt long=ubuntu"`
	Raring    bool `cli:"type=opt long=raring"`
	Saucy     bool `cli:"type=opt long=saucy"`
}

func (a *DescribeImages) Run() error {
	log.Println("describing images")
	filter := &ec2.ImageFilter{}
	if a.Canonical {
		filter.Owner = ec2.CANONICAL_OWNER_ID
	} else if a.Self {
		filter.Owner = ec2.SELF_OWNER_ID
	}

	if a.Ubuntu {
		filter.Name = ec2.UBUNTU_PREFIX
	} else if a.Raring {
		filter.Name = ec2.UBUNTU_RARING_PREFIX
	} else if a.Saucy {
		filter.Name = ec2.UBUNTU_SAUCY_PREFIX
	}

	images, e := client().DescribeImagesWithFilter(filter)
	if e != nil {
		return e
	}
	sort.Sort(images)
	table := gocli.NewTable()
	for _, image := range images {
		table.Add(image.ImageId, image.Name, image.ImageState)
	}
	fmt.Println(table)
	return nil
}

func DescribeSpotPriceHistory() error {
	filter := &ec2.SpotPriceFilter{
		InstanceTypes:       []string{"c1.medium"},
		ProductDescriptions: []string{ec2.DESC_LINUX_UNIX},
		StartTime:           time.Now().Add(-7 * 24 * time.Hour),
	}
	prices, e := client().DescribeSpotPriceHistory(filter)
	if e != nil {
		return e
	}
	table := gocli.NewTable()
	for _, price := range prices {
		table.Add(price.InstanceType, price.ProductDescription, price.SpotPrice, price.Timestamp, price.AvailabilityZone)
	}
	fmt.Println(table)
	return nil
}

func DescribeSecurityGroups() error {
	groups, e := client().DescribeSecurityGroups()
	if e != nil {
		return e
	}
	table := gocli.NewTable()
	for _, group := range groups {
		table.Add(group.GroupId, "", "", group.GroupName)
		for _, perm := range group.IpPermissions {
			ports := ""
			if perm.FromPort > 0 {
				if perm.FromPort == perm.ToPort {
					ports += fmt.Sprintf("%d", perm.FromPort)
				} else {
					ports += fmt.Sprintf("%d-%d", perm.FromPort, perm.ToPort)
				}
			}
			groups := []string{}
			for _, group := range perm.Groups {
				groups = append(groups, group.GroupId)
			}
			if len(groups) > 0 {
				table.Add("", perm.IpProtocol, ports, strings.Join(groups, ","))
			}

			if len(perm.IpRanges) > 0 {
				table.Add("", perm.IpProtocol, ports, strings.Join(perm.IpRanges, ","))
			}
		}
	}
	fmt.Print(table)
	return nil
}

func DescribeAddresses() error {
	addresses, e := client().DescribeAddresses()
	if e != nil {
		return e
	}
	table := gocli.NewTable()
	for _, address := range addresses {
		table.Add(address.PublicIp, address.PrivateIpAddress, address.Domain, address.InstanceId)
	}
	fmt.Print(table)
	return nil
}

func DescribeKeyPairs() error {
	pairs, e := client().DescribeKeyPairs()
	if e != nil {
		return e
	}
	table := gocli.NewTable()
	for _, pair := range pairs {
		table.Add(pair.KeyName, pair.KeyFingerprint)
	}
	fmt.Print(table)
	return nil
}

type CreateImage struct {
	ImageId string `cli:"type=arg required=true"`
}

func (a *CreateImage) Run() error {
	log.Printf("creating image of instance %s", a.ImageId)
	return fmt.Errorf("implement me")
}

type RunInstances struct {
	Name             string `cli:"type=opt long=name desc='Name of the Instanze'"`
	InstanceType     string `cli:"type=opt short=t desc='Instance Type' required=true"`
	ImageId          string `cli:"type=opt short=i desc='Image Id' required=true"`
	KeyName          string `cli:"type=opt short=k desc='SSH Key' required=true"`
	SecurityGroup    string `cli:"type=opt short=g desc='Security Group'"`
	SubnetId         string `cli:"type=opt long=subnet-id desc='Subnet Id'"`
	PublicIp         bool   `cli:"type=opt long=public-ip desc='Assign Public IP'"`
	AvailabilityZone string `cli:"type=opt long=availability-zone desc='Availability Zone'"`
	UserDataFile     string `cli:"type=opt long=userdata desc='Path to file with userdata'"`
}

func (a *RunInstances) Run() error {
	config := &ec2.RunInstancesConfig{
		ImageId:          a.ImageId,
		KeyName:          a.KeyName,
		InstanceType:     a.InstanceType,
		AvailabilityZone: a.AvailabilityZone,
		SubnetId:         a.SubnetId,
	}
	if a.UserDataFile != "" {
		b, e := ioutil.ReadFile(a.UserDataFile)
		if e != nil {
			return e
		}
		config.UserData = string(b)
	}
	if a.PublicIp {
		nic := &ec2.CreateNetworkInterface{
			DeviceIndex: 0, AssociatePublicIpAddress: true, SubnetId: a.SubnetId,
		}
		if a.SecurityGroup != "" {
			nic.SecurityGroupIds = []string{a.SecurityGroup}
		}
		config.NetworkInterfaces = []*ec2.CreateNetworkInterface{nic}
	} else {
		if a.SecurityGroup != "" {
			config.SecurityGroups = []string{a.SecurityGroup}
		}
	}
	list, e := client().RunInstances(config)
	ids := []string{}
	for _, i := range list {
		ids = append(ids, i.InstanceId)
	}
	if a.Name != "" {
		log.Printf("tagging %v with %q", ids, a.Name)
		e := client().CreateTags(ids, map[string]string{"Name": a.Name})
		if e != nil {
			log.Printf("ERROR: " + e.Error())
		}
	}
	log.Printf("started instances %v", ids)
	return e
}

type TerminateInstances struct {
	InstanceIds []string `cli:"type=arg required=true"`
}

func (a *TerminateInstances) Run() error {
	return client().TerminateInstances(a.InstanceIds)
}

type CreateTags struct {
	ResourceId string `cli:"type=arg required=true"`
	Key        string `cli:"type=arg required=true"`
	Value      string `cli:"type=arg required=true"`
}

func (r *CreateTags) Run() error {
	tags := map[string]string{
		r.Key: r.Value,
	}
	return client().CreateTags([]string{r.ResourceId}, tags)
}

func DescribeTags() error {
	tags, e := client().DescribeTags()
	if e != nil {
		return e
	}
	sort.Sort(tags)
	table := gocli.NewTable()
	for _, tag := range tags {
		table.Add(tag.ResourceType, tag.ResourceId, tag.Key, tag.Value)
	}
	fmt.Println(table)
	return nil
}

func DescribeInstances() error {
	log.Print("describing ec2 instances")
	instances, e := client().DescribeInstances()
	if e != nil {
		return e
	}
	table := gocli.NewTable()
	table.Add("id", "image", "name", "state", "type", "private_ip", "ip", "az", "launched")
	for _, i := range instances {
		sgs := []string{}
		if i.InstanceStateName != "running" {
			continue
		}
		for _, g := range i.SecurityGroups {
			sgs = append(sgs, g.GroupId)
		}
		table.Add(
			i.InstanceId,
			i.ImageId,
			i.Name(),
			i.InstanceStateName,
			i.InstanceType,
			i.PrivateIpAddress,
			i.IpAddress,
			i.PlacementAvailabilityZone,
			i.LaunchTime.Format("2006-01-02T15:04"),
		)
	}
	fmt.Println(table)

	return nil
}
<commit_msg>add --detailed option for describing instances<commit_after>package ec2

import (
	"fmt"
	"github.com/dynport/dgtk/cli"
	"github.com/dynport/gocli"
	"github.com/dynport/gocloud/aws/ec2"
	"io/ioutil"
	"log"
	"os"
	"sort"
	"strings"
	"time"
)

const HOURS_PER_MONTH = 365 * 24.0 / 12.0

func Register(router *cli.Router) {
	router.Register("aws/ec2/instances/describe", &DescribeInstances{}, "Describe ec2 instances")
	router.Register("aws/ec2/instances/run", &RunInstances{}, "Run ec2 instances")
	router.Register("aws/ec2/images/create", &CreateImage{}, "Create image from instance")
	router.Register("aws/ec2/instances/terminate", &TerminateInstances{}, "Terminate ec2 instances")
	router.Register("aws/ec2/tags/create", &CreateTags{}, "Create Tags")
	router.RegisterFunc("aws/ec2/tags/describe", DescribeTags, "Describe Tags")
	router.Register("aws/ec2/images/describe", &DescribeImages{}, "Describe ec2 Images")
	router.RegisterFunc("aws/ec2/key-pairs/describe", DescribeKeyPairs, "Describe key pairs")
	router.RegisterFunc("aws/ec2/addresses/describe", DescribeAddresses, "Describe Addresses")
	router.RegisterFunc("aws/ec2/security-groups/describe", DescribeSecurityGroups, "Describe Security Groups")
	router.RegisterFunc("aws/ec2/spot-price-history/describe", DescribeSpotPriceHistory, "Describe Spot Price History")
	router.Register("aws/ec2/prices", &Prices{Region: os.Getenv("AWS_DEFAULT_REGION")}, "EC2 Prices")
}

func client() *ec2.Client {
	return ec2.NewFromEnv()
}

// http://docs.aws.amazon.com/general/latest/gr/rande.html#ec2_region
var regionMapping = map[string]string{
	"eu-ireland": "eu-west-1",
	"eu-west":    "eu-west-1",
	"apac-tokyo": "ap-northeast-1",
	"apac-sin":   "ap-southeast-1",
	"apac-syd":   "ap-southeast-2",
}

// http://docs.aws.amazon.com/general/latest/gr/rande.html#ec2_region
var regionMappingForOd = map[string]string{
	"eu-west-1":      "eu-ireland",
	"us-east-1":      "us-east",
	"us-west-1":      "us-west",
	"ap-northeast-1": "apac-tokyo",
	"ap-southeast-1": "apac-sin",
	"ap-southeast-2": "apac-syd",
}

func normalizeRegionForOd(region string) string {
	normalized, ok := regionMappingForOd[region]
	if ok {
		return normalized
	}
	return region
}

func normalizeRegion(raw string) string {
	if v, ok := regionMapping[raw]; ok {
		return v
	}
	return raw
}
func monthlyPrice(price float64) string {
	return fmt.Sprintf("%.02f", price)
}

type DescribeImages struct {
	Canonical bool `cli:"type=opt long=canonical"`
	Self      bool `cli:"type=opt long=self"`
	Ubuntu    bool `cli:"type=opt long=ubuntu"`
	Raring    bool `cli:"type=opt long=raring"`
	Saucy     bool `cli:"type=opt long=saucy"`
}

func (a *DescribeImages) Run() error {
	log.Println("describing images")
	filter := &ec2.ImageFilter{}
	if a.Canonical {
		filter.Owner = ec2.CANONICAL_OWNER_ID
	} else if a.Self {
		filter.Owner = ec2.SELF_OWNER_ID
	}

	if a.Ubuntu {
		filter.Name = ec2.UBUNTU_PREFIX
	} else if a.Raring {
		filter.Name = ec2.UBUNTU_RARING_PREFIX
	} else if a.Saucy {
		filter.Name = ec2.UBUNTU_SAUCY_PREFIX
	}

	images, e := client().DescribeImagesWithFilter(filter)
	if e != nil {
		return e
	}
	sort.Sort(images)
	table := gocli.NewTable()
	for _, image := range images {
		table.Add(image.ImageId, image.Name, image.ImageState)
	}
	fmt.Println(table)
	return nil
}

func DescribeSpotPriceHistory() error {
	filter := &ec2.SpotPriceFilter{
		InstanceTypes:       []string{"c1.medium"},
		ProductDescriptions: []string{ec2.DESC_LINUX_UNIX},
		StartTime:           time.Now().Add(-7 * 24 * time.Hour),
	}
	prices, e := client().DescribeSpotPriceHistory(filter)
	if e != nil {
		return e
	}
	table := gocli.NewTable()
	for _, price := range prices {
		table.Add(price.InstanceType, price.ProductDescription, price.SpotPrice, price.Timestamp, price.AvailabilityZone)
	}
	fmt.Println(table)
	return nil
}

func DescribeSecurityGroups() error {
	groups, e := client().DescribeSecurityGroups()
	if e != nil {
		return e
	}
	table := gocli.NewTable()
	for _, group := range groups {
		table.Add(group.GroupId, "", "", group.GroupName)
		for _, perm := range group.IpPermissions {
			ports := ""
			if perm.FromPort > 0 {
				if perm.FromPort == perm.ToPort {
					ports += fmt.Sprintf("%d", perm.FromPort)
				} else {
					ports += fmt.Sprintf("%d-%d", perm.FromPort, perm.ToPort)
				}
			}
			groups := []string{}
			for _, group := range perm.Groups {
				groups = append(groups, group.GroupId)
			}
			if len(groups) > 0 {
				table.Add("", perm.IpProtocol, ports, strings.Join(groups, ","))
			}

			if len(perm.IpRanges) > 0 {
				table.Add("", perm.IpProtocol, ports, strings.Join(perm.IpRanges, ","))
			}
		}
	}
	fmt.Print(table)
	return nil
}

func DescribeAddresses() error {
	addresses, e := client().DescribeAddresses()
	if e != nil {
		return e
	}
	table := gocli.NewTable()
	for _, address := range addresses {
		table.Add(address.PublicIp, address.PrivateIpAddress, address.Domain, address.InstanceId)
	}
	fmt.Print(table)
	return nil
}

func DescribeKeyPairs() error {
	pairs, e := client().DescribeKeyPairs()
	if e != nil {
		return e
	}
	table := gocli.NewTable()
	for _, pair := range pairs {
		table.Add(pair.KeyName, pair.KeyFingerprint)
	}
	fmt.Print(table)
	return nil
}

type CreateImage struct {
	ImageId string `cli:"type=arg required=true"`
}

func (a *CreateImage) Run() error {
	log.Printf("creating image of instance %s", a.ImageId)
	return fmt.Errorf("implement me")
}

type RunInstances struct {
	Name             string `cli:"type=opt long=name desc='Name of the Instanze'"`
	InstanceType     string `cli:"type=opt short=t desc='Instance Type' required=true"`
	ImageId          string `cli:"type=opt short=i desc='Image Id' required=true"`
	KeyName          string `cli:"type=opt short=k desc='SSH Key' required=true"`
	SecurityGroup    string `cli:"type=opt short=g desc='Security Group'"`
	SubnetId         string `cli:"type=opt long=subnet-id desc='Subnet Id'"`
	PublicIp         bool   `cli:"type=opt long=public-ip desc='Assign Public IP'"`
	AvailabilityZone string `cli:"type=opt long=availability-zone desc='Availability Zone'"`
	UserDataFile     string `cli:"type=opt long=userdata desc='Path to file with userdata'"`
}

func (a *RunInstances) Run() error {
	config := &ec2.RunInstancesConfig{
		ImageId:          a.ImageId,
		KeyName:          a.KeyName,
		InstanceType:     a.InstanceType,
		AvailabilityZone: a.AvailabilityZone,
		SubnetId:         a.SubnetId,
	}
	if a.UserDataFile != "" {
		b, e := ioutil.ReadFile(a.UserDataFile)
		if e != nil {
			return e
		}
		config.UserData = string(b)
	}
	if a.PublicIp {
		nic := &ec2.CreateNetworkInterface{
			DeviceIndex: 0, AssociatePublicIpAddress: true, SubnetId: a.SubnetId,
		}
		if a.SecurityGroup != "" {
			nic.SecurityGroupIds = []string{a.SecurityGroup}
		}
		config.NetworkInterfaces = []*ec2.CreateNetworkInterface{nic}
	} else {
		if a.SecurityGroup != "" {
			config.SecurityGroups = []string{a.SecurityGroup}
		}
	}
	list, e := client().RunInstances(config)
	ids := []string{}
	for _, i := range list {
		ids = append(ids, i.InstanceId)
	}
	if a.Name != "" {
		log.Printf("tagging %v with %q", ids, a.Name)
		e := client().CreateTags(ids, map[string]string{"Name": a.Name})
		if e != nil {
			log.Printf("ERROR: " + e.Error())
		}
	}
	log.Printf("started instances %v", ids)
	return e
}

type TerminateInstances struct {
	InstanceIds []string `cli:"type=arg required=true"`
}

func (a *TerminateInstances) Run() error {
	return client().TerminateInstances(a.InstanceIds)
}

type CreateTags struct {
	ResourceId string `cli:"type=arg required=true"`
	Key        string `cli:"type=arg required=true"`
	Value      string `cli:"type=arg required=true"`
}

func (r *CreateTags) Run() error {
	tags := map[string]string{
		r.Key: r.Value,
	}
	return client().CreateTags([]string{r.ResourceId}, tags)
}

func DescribeTags() error {
	tags, e := client().DescribeTags()
	if e != nil {
		return e
	}
	sort.Sort(tags)
	table := gocli.NewTable()
	for _, tag := range tags {
		table.Add(tag.ResourceType, tag.ResourceId, tag.Key, tag.Value)
	}
	fmt.Println(table)
	return nil
}

type DescribeInstances struct {
	Detailed bool `cli:"type=opt long=detailed"`
}

func (a *DescribeInstances) Run() error {
	log.Print("describing ec2 instances")
	instances, e := client().DescribeInstances()
	if e != nil {
		return e
	}
	if a.Detailed {
		for _, i := range instances {
			table := gocli.NewTable()
			table.Add(i.InstanceId, "Name", i.Name())
			table.Add(i.InstanceId, "ImageId", i.ImageId)
			table.Add(i.InstanceId, "InstanceType", i.InstanceType)
			table.Add(i.InstanceId, "InstanceStateName", i.InstanceStateName)
			table.Add(i.InstanceId, "Availability Zone", i.PlacementAvailabilityZone)
			table.Add(i.InstanceId, "KeyName", i.KeyName)
			table.Add(i.InstanceId, "IpAddress", i.IpAddress)
			table.Add(i.InstanceId, "PrivateIpAddress", i.PrivateIpAddress)
			table.Add(i.InstanceId, "LaunchTime", i.LaunchTime.Format("2006-01-02T15:04:05"))
			table.Add(i.InstanceId, "VpcId", i.VpcId)
			table.Add(i.InstanceId, "MonitoringState", i.MonitoringState)
			for _, group := range i.SecurityGroups {
				table.Add(i.InstanceId, "SecurityGroup", group.GroupId)
			}
			for _, tag := range i.Tags {
				table.Add(i.InstanceId, "Tag", tag.Key, tag.Value)
			}
			fmt.Println(table)
		}
		return nil
	}
	table := gocli.NewTable()
	table.Add("id", "image", "name", "state", "type", "private_ip", "ip", "az", "launched")
	for _, i := range instances {
		sgs := []string{}
		if i.InstanceStateName != "running" {
			continue
		}
		for _, g := range i.SecurityGroups {
			sgs = append(sgs, g.GroupId)
		}
		table.Add(
			i.InstanceId,
			i.ImageId,
			i.Name(),
			i.InstanceStateName,
			i.InstanceType,
			i.PrivateIpAddress,
			i.IpAddress,
			i.PlacementAvailabilityZone,
			i.LaunchTime.Format("2006-01-02T15:04"),
		)
	}
	fmt.Println(table)
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2014 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"github.com/codegangsta/cli"
	"github.com/control-center/serviced/isvcs"
	"github.com/control-center/serviced/servicedversion"

	"encoding/json"
	"fmt"
	"os"
)

// Initializer for serviced version
func (c *ServicedCli) initVersion() {
	c.app.Commands = append(c.app.Commands, cli.Command{
		Name:        "version",
		Usage:       "shows version information",
		Description: "",
		Action:      c.cmdVersion,
		Flags: []cli.Flag{
			cli.BoolFlag{
				Name:  "verbose, v",
				Usage: "Show JSON format",
			},
		},
	})
}

// serviced version
func (c *ServicedCli) cmdVersion(ctx *cli.Context) {

	var versionInfo = map[string]string{
		"Version":            servicedversion.Version,
		"GoVersion":          servicedversion.GoVersion,
		"Gitcommit":          servicedversion.Gitcommit,
		"Gitbranch":          servicedversion.Gitbranch,
		"Date":               servicedversion.Date,
		"Release":            servicedversion.Release,
		"IsvcsImage":         fmt.Sprintf("%s:%s", isvcs.IMAGE_REPO, isvcs.IMAGE_TAG),
		"IsvcsZKImage":       fmt.Sprintf("%s:%s", isvcs.ZK_IMAGE_REPO, isvcs.ZK_IMAGE_TAG),
		"IsvcsApiProxyImage": fmt.Sprintf("%s:%s", isvcs.API_KEY_PROXY_REPO, isvcs.API_KEY_PROXY_TAG),
	}

	if ctx.Bool("verbose") {
		if jsonVersion, err := json.MarshalIndent(versionInfo, " ", "  "); err != nil {
			fmt.Fprintf(os.Stderr, "failed to marshal version info: %s", err)
			os.Exit(1)
		} else {
			fmt.Println(string(jsonVersion))
		}
	} else {
		fmt.Printf("Version:    %s\n", versionInfo["Version"])
		fmt.Printf("GoVersion:  %s\n", versionInfo["GoVersion"])
		fmt.Printf("Gitcommit:  %s\n", versionInfo["Gitcommit"])
		fmt.Printf("Gitbranch:  %s\n", versionInfo["Gitbranch"])
		fmt.Printf("Date:       %s\n", versionInfo["Date"])
		fmt.Printf("Buildtag:   %s\n", versionInfo["Buildtag"])
		fmt.Printf("Release:    %s\n", versionInfo["Release"])
		images := []string{
			versionInfo["IsvcsImage"], versionInfo["IsvcsZKImage"], versionInfo["IsvcsApiProxyImage"],
		}
		fmt.Printf("IsvcsImages: %v\n", images)
	}
}
<commit_msg>Only display the api proxy image if configured for it<commit_after>// Copyright 2014 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"github.com/codegangsta/cli"
	"github.com/control-center/serviced/isvcs"
	"github.com/control-center/serviced/servicedversion"

	"encoding/json"
	"fmt"
	"os"
	"github.com/control-center/serviced/config"
)

// Initializer for serviced version
func (c *ServicedCli) initVersion() {
	c.app.Commands = append(c.app.Commands, cli.Command{
		Name:        "version",
		Usage:       "shows version information",
		Description: "",
		Action:      c.cmdVersion,
		Flags: []cli.Flag{
			cli.BoolFlag{
				Name:  "verbose, v",
				Usage: "Show JSON format",
			},
		},
	})
}

// serviced version
func (c *ServicedCli) cmdVersion(ctx *cli.Context) {

	var versionInfo = map[string]string{
		"Version":            servicedversion.Version,
		"GoVersion":          servicedversion.GoVersion,
		"Gitcommit":          servicedversion.Gitcommit,
		"Gitbranch":          servicedversion.Gitbranch,
		"Date":               servicedversion.Date,
		"Release":            servicedversion.Release,
		"IsvcsImage":         fmt.Sprintf("%s:%s", isvcs.IMAGE_REPO, isvcs.IMAGE_TAG),
		"IsvcsZKImage":       fmt.Sprintf("%s:%s", isvcs.ZK_IMAGE_REPO, isvcs.ZK_IMAGE_TAG),
	}

	// Only populate the 'IsvcsApiProxyImage' value if the api proxy is configured to start.
	if config.GetOptions().StartAPIKeyProxy {
		versionInfo["IsvcsApiProxyImage"] = fmt.Sprintf("%s:%s", isvcs.API_KEY_PROXY_REPO, isvcs.API_KEY_PROXY_TAG)
	}

	if ctx.Bool("verbose") {
		if jsonVersion, err := json.MarshalIndent(versionInfo, " ", "  "); err != nil {
			fmt.Fprintf(os.Stderr, "failed to marshal version info: %s", err)
			os.Exit(1)
		} else {
			fmt.Println(string(jsonVersion))
		}
	} else {
		fmt.Printf("Version:    %s\n", versionInfo["Version"])
		fmt.Printf("GoVersion:  %s\n", versionInfo["GoVersion"])
		fmt.Printf("Gitcommit:  %s\n", versionInfo["Gitcommit"])
		fmt.Printf("Gitbranch:  %s\n", versionInfo["Gitbranch"])
		fmt.Printf("Date:       %s\n", versionInfo["Date"])
		fmt.Printf("Buildtag:   %s\n", versionInfo["Buildtag"])
		fmt.Printf("Release:    %s\n", versionInfo["Release"])
		images := []string{
			versionInfo["IsvcsImage"], versionInfo["IsvcsZKImage"],
		}
		if image, exists := versionInfo["IsvcsApiProxyImage"]; exists {
			images = append(images, image)
		}
		fmt.Printf("IsvcsImages: %v\n", images)
	}
}
<|endoftext|>


<commit_before>//shelf is a nice servant that helps launching your favorite projects!
package main

import (
	"bufio"
	"fmt"
	"os"
	"strings"

	"github.com/clebs/shelf/core"
	"github.com/clebs/shelf/filesystem"
	"github.com/clebs/shelf/project"
)

func main() {
	args := os.Args[1:]

	if len(args) == 0 {
		help()
	} else {
		switch args[0] {
		case "add":
			add()
		case "pick":
			if len(args) < 2 {
				help("pick")
				os.Exit(2)
			}
			pick(args[1])
		case "delete":
			if len(args) < 2 {
				help("delete")
				os.Exit(2)
			}
			delete(args[1])
		case "help":
			if len(args) > 1 {
				help(args[1])
			} else {
				help()
			}
		}
	}
}

//Adds a new project to the shelf
func add() {
	p := project.Project{
		Name: ask("Project name: (use it later to pick it up)"),
		IDE: project.IDE{
			Exec:      ask("IDE executable:"),
			Workspace: ask("IDE Workspace:")},
		Repos: strings.Split(ask("Repositories:"), ","),
		Server: project.Server{
			Path: ask("Server location:"),
			Cmd:  strings.Split(ask("Server command:"), " ")}}

	filesystem.Save(p, p.Name)

	println("Your project has been added:")
	fmt.Println(p)
}

//pick allows to pick up a project from the shelf by its name
func pick(name string) {
	p := filesystem.Load(name)

	println("Loading project:")
	fmt.Println(p)
	var asyncExecutor core.AsyncExecutor

	asyncExecutor = append(asyncExecutor, core.StartIDECmd(p.IDE))
	asyncExecutor = append(asyncExecutor, core.UpdateRepositoriesCmd(p.Repos...)...)
	asyncExecutor = append(asyncExecutor, core.StartServer(p.Server))
	asyncExecutor.Run()

	var syncExecutor core.SyncExecutor
	syncExecutor = append(syncExecutor, core.UpdateRepositoriesCmd(p.Server.Path)...)
	syncExecutor.Run()
}

func delete(name string) {
	filesystem.Delete(name)
}

func help(args ...string) {
	if len(args) == 0 {
		println("Shelf is a project bootup storage. Store IDE/editor, repository and server launch configs to automatically open a working environment.\n")
		println("Usage:")
		println("    shelf command [arguments]\n")
		println("The commands are:")
		println("    add     add a new project")
		println("    pick    picks up a project from the shelf")
		println("    delete  removes a project from the shelf")
		return
	}

	switch args[0] {
	case "add":
		println("add     add a new project entering the following data:")
		println("    Project name:      name of the project, used later to find it on pick command")
		println("    IDE executable:    absolute path to the executable of your IDE, no args accepted")
		println("    IDE Workspace:     (optional) location of the workspace to open the IDE in")
		println("    Repositories:      list of comma separated repositories where the code is.")
		println("    Server location:   path to the folder where the local server is")
		println("    Server command:    command to run inside the server folder, accepts args (e.g. mvn clean install)")
	case "pick":
		println("pick     load a project previously added with the add command.")
		println("usage:   shelf pick [project name]")
	case "delete":
		println("delete   remove a project from the shelf.")
		println("usage:   shelf delete [project name]")
	}
}

var scanner = bufio.NewScanner(os.Stdin)

func ask(question string) (answer string) {
	println(question)
	scanner.Scan()
	return scanner.Text()
}
<commit_msg>Just trying new config<commit_after>//shelf is a nice servant that helps launching your favorite projects!!
package main

import (
	"bufio"
	"fmt"
	"os"
	"strings"

	"github.com/clebs/shelf/core"
	"github.com/clebs/shelf/filesystem"
	"github.com/clebs/shelf/project"
)

func main() {
	args := os.Args[1:]

	if len(args) == 0 {
		help()
	} else {
		switch args[0] {
		case "add":
			add()
		case "pick":
			if len(args) < 2 {
				help("pick")
				os.Exit(2)
			}
			pick(args[1])
		case "delete":
			if len(args) < 2 {
				help("delete")
				os.Exit(2)
			}
			delete(args[1])
		case "help":
			if len(args) > 1 {
				help(args[1])
			} else {
				help()
			}
		}
	}
}

//Adds a new project to the shelf
func add() {
	p := project.Project{
		Name: ask("Project name: (use it later to pick it up)"),
		IDE: project.IDE{
			Exec:      ask("IDE executable:"),
			Workspace: ask("IDE Workspace:")},
		Repos: strings.Split(ask("Repositories:"), ","),
		Server: project.Server{
			Path: ask("Server location:"),
			Cmd:  strings.Split(ask("Server command:"), " ")}}

	filesystem.Save(p, p.Name)

	println("Your project has been added:")
	fmt.Println(p)
}

//pick allows to pick up a project from the shelf by its name
func pick(name string) {
	p := filesystem.Load(name)

	println("Loading project:")
	fmt.Println(p)
	var asyncExecutor core.AsyncExecutor

	asyncExecutor = append(asyncExecutor, core.StartIDECmd(p.IDE))
	asyncExecutor = append(asyncExecutor, core.UpdateRepositoriesCmd(p.Repos...)...)
	asyncExecutor = append(asyncExecutor, core.StartServer(p.Server))
	asyncExecutor.Run()

	var syncExecutor core.SyncExecutor
	syncExecutor = append(syncExecutor, core.UpdateRepositoriesCmd(p.Server.Path)...)
	syncExecutor.Run()
}

func delete(name string) {
	filesystem.Delete(name)
}

func help(args ...string) {
	if len(args) == 0 {
		println("Shelf is a project bootup storage. Store IDE/editor, repository and server launch configs to automatically open a working environment.\n")
		println("Usage:")
		println("    shelf command [arguments]\n")
		println("The commands are:")
		println("    add     add a new project")
		println("    pick    picks up a project from the shelf")
		println("    delete  removes a project from the shelf")
		return
	}

	switch args[0] {
	case "add":
		println("add     add a new project entering the following data:")
		println("    Project name:      name of the project, used later to find it on pick command")
		println("    IDE executable:    absolute path to the executable of your IDE, no args accepted")
		println("    IDE Workspace:     (optional) location of the workspace to open the IDE in")
		println("    Repositories:      list of comma separated repositories where the code is.")
		println("    Server location:   path to the folder where the local server is")
		println("    Server command:    command to run inside the server folder, accepts args (e.g. mvn clean install)")
	case "pick":
		println("pick     load a project previously added with the add command.")
		println("usage:   shelf pick [project name]")
	case "delete":
		println("delete   remove a project from the shelf.")
		println("usage:   shelf delete [project name]")
	}
}

var scanner = bufio.NewScanner(os.Stdin)

func ask(question string) (answer string) {
	println(question)
	scanner.Scan()
	return scanner.Text()
}
<|endoftext|>


<commit_before>package client

import (
	"crypto/rand"
	"io"
	"math/big"
)

type Message interface {
	Write(w io.Writer, colors ColorCodes)
}

type SimpleMessage struct {
	Text string
}

func (s SimpleMessage) Write(w io.Writer, colors ColorCodes) {
	w.Write([]byte(" " + s.Text + "\r\n"))
}

// Quote is a message with an author
type Quote struct {
	Color  []byte
	Author string
	Text   string
}

func (q Quote) Write(w io.Writer, colors ColorCodes) {
	w.Write(q.Color)
	w.Write([]byte(" " + q.Author))
	w.Write(colors.Reset)
	w.Write([]byte(": " + q.Text))
	w.Write([]byte("\r\n"))
}

// Conversation displays a list of quotes
type Conversation struct {
	Quotes []Quote
}

// Write actually writes the message to the terminal
func (c Conversation) Write(w io.Writer, colors ColorCodes) {
	for _, q := range c.Quotes {
		q.Write(w, colors)
	}
	w.Write([]byte("\r\n"))
}

// LoginMessage writes a message at login
func GetMessage(w io.Writer, colors ColorCodes) {
	var messages = []Message{
		SimpleMessage{"Welcome to Kappa DB, Yo!"},
		Quote{colors.LightMagenta, "Jessy Pinkman", "Yeah, Bitch! Magnets!"},
		Quote{colors.LightMagenta, "Jessy Pinkman", "Yeah, Science!"},
		Quote{colors.LightBlue, "Saul Goodman", "Better call Saul."},
		Conversation{
			Quotes: []Quote{
				Quote{colors.LightGreen, "Walter White", "One particular element comes to mind..."},
				Quote{colors.LightMagenta, "Jessy Pinkman", "Ohhhhh... wire.."},
			},
		},
	}

	// Choose message
	index, err := rand.Int(rand.Reader, big.NewInt(int64(len(messages))))
	if err != nil {
		index = big.NewInt(0)
	}
	messages[index.Int64()].Write(w, colors)
}
<commit_msg>Update client to use common pkg<commit_after>package client

import (
	"crypto/rand"
	"io"
	"math/big"

	"github.com/subsilent/kappa/common"
)

type Message interface {
	Write(w io.Writer, colors common.ColorCodes)
}

type SimpleMessage struct {
	Text string
}

func (s SimpleMessage) Write(w io.Writer, colors common.ColorCodes) {
	w.Write([]byte(" " + s.Text + "\r\n"))
}

// Quote is a message with an author
type Quote struct {
	Color  []byte
	Author string
	Text   string
}

func (q Quote) Write(w io.Writer, colors common.ColorCodes) {
	w.Write(q.Color)
	w.Write([]byte(" " + q.Author))
	w.Write(colors.Reset)
	w.Write([]byte(": " + q.Text))
	w.Write([]byte("\r\n"))
}

// Conversation displays a list of quotes
type Conversation struct {
	Quotes []Quote
}

// Write actually writes the message to the terminal
func (c Conversation) Write(w io.Writer, colors common.ColorCodes) {
	for _, q := range c.Quotes {
		q.Write(w, colors)
	}
	w.Write([]byte("\r\n"))
}

// LoginMessage writes a message at login
func GetMessage(w io.Writer, colors common.ColorCodes) {
	var messages = []Message{
		SimpleMessage{"Welcome to Kappa DB, Yo!"},
		Quote{colors.LightMagenta, "Jessy Pinkman", "Yeah, Bitch! Magnets!"},
		Quote{colors.LightMagenta, "Jessy Pinkman", "Yeah, Science!"},
		Quote{colors.LightBlue, "Saul Goodman", "Better call Saul."},
		Conversation{
			Quotes: []Quote{
				Quote{colors.LightGreen, "Walter White", "One particular element comes to mind..."},
				Quote{colors.LightMagenta, "Jessy Pinkman", "Ohhhhh... wire.."},
			},
		},
	}

	// Choose message
	index, err := rand.Int(rand.Reader, big.NewInt(int64(len(messages))))
	if err != nil {
		index = big.NewInt(0)
	}
	messages[index.Int64()].Write(w, colors)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"runtime"

	"github.com/go-errors/errors"
	"github.com/layeh/gopher-luar"
	"github.com/mattn/go-isatty"
	"github.com/mitchellh/go-homedir"
	"github.com/yuin/gopher-lua"
	"github.com/zyedidia/clipboard"
	"github.com/zyedidia/tcell"
	"github.com/zyedidia/tcell/encoding"
)

const (
	synLinesUp           = 75  // How many lines up to look to do syntax highlighting
	synLinesDown         = 75  // How many lines down to look to do syntax highlighting
	doubleClickThreshold = 400 // How many milliseconds to wait before a second click is not a double click
	undoThreshold        = 500 // If two events are less than n milliseconds apart, undo both of them
)

var (
	// The main screen
	screen tcell.Screen

	// Object to send messages and prompts to the user
	messenger *Messenger

	// The default highlighting style
	// This simply defines the default foreground and background colors
	defStyle tcell.Style

	// Where the user's configuration is
	// This should be $XDG_CONFIG_HOME/micro
	// If $XDG_CONFIG_HOME is not set, it is ~/.config/micro
	configDir string

	// Version is the version number or commit hash
	// This should be set by the linker when compiling
	Version = "Unknown"

	// L is the lua state
	// This is the VM that runs the plugins
	L *lua.LState

	// The list of views
	tabs []*Tab
	// This is the currently open tab
	// It's just an index to the tab in the tabs array
	curTab int

	// Channel of jobs running in the background
	jobs chan JobFunction
	// Event channel
	events chan tcell.Event
)

// LoadInput determines which files should be loaded into buffers
// based on the input stored in os.Args
func LoadInput() []*Buffer {
	// There are a number of ways micro should start given its input

	// 1. If it is given a files in os.Args, it should open those

	// 2. If there is no input file and the input is not a terminal, that means
	// something is being piped in and the stdin should be opened in an
	// empty buffer

	// 3. If there is no input file and the input is a terminal, an empty buffer
	// should be opened

	var filename string
	var input []byte
	var err error
	var buffers []*Buffer

	if len(os.Args) > 1 {
		// Option 1
		// We go through each file and load it
		for i := 1; i < len(os.Args); i++ {
			filename = os.Args[i]
			// Check that the file exists
			if _, e := os.Stat(filename); e == nil {
				// If it exists we load it into a buffer
				input, err = ioutil.ReadFile(filename)
				if err != nil {
					TermMessage(err)
					continue
				}
			}
			// If the file didn't exist, input will be empty, and we'll open an empty buffer
			buffers = append(buffers, NewBuffer(input, filename))
		}
	} else if !isatty.IsTerminal(os.Stdin.Fd()) {
		// Option 2
		// The input is not a terminal, so something is being piped in
		// and we should read from stdin
		input, err = ioutil.ReadAll(os.Stdin)
		buffers = append(buffers, NewBuffer(input, filename))
	} else {
		// Option 3, just open an empty buffer
		buffers = append(buffers, NewBuffer(input, filename))
	}

	return buffers
}

// InitConfigDir finds the configuration directory for micro according to the XDG spec.
// If no directory is found, it creates one.
func InitConfigDir() {
	xdgHome := os.Getenv("XDG_CONFIG_HOME")
	if xdgHome == "" {
		// The user has not set $XDG_CONFIG_HOME so we should act like it was set to ~/.config
		home, err := homedir.Dir()
		if err != nil {
			TermMessage("Error finding your home directory\nCan't load config files")
			return
		}
		xdgHome = home + "/.config"
	}
	configDir = xdgHome + "/micro"

	if _, err := os.Stat(xdgHome); os.IsNotExist(err) {
		// If the xdgHome doesn't exist we should create it
		err = os.Mkdir(xdgHome, os.ModePerm)
		if err != nil {
			TermMessage("Error creating XDG_CONFIG_HOME directory: " + err.Error())
		}
	}

	if _, err := os.Stat(configDir); os.IsNotExist(err) {
		// If the micro specific config directory doesn't exist we should create that too
		err = os.Mkdir(configDir, os.ModePerm)
		if err != nil {
			TermMessage("Error creating configuration directory: " + err.Error())
		}
	}
}

// InitScreen creates and initializes the tcell screen
func InitScreen() {
	// Should we enable true color?
	truecolor := os.Getenv("MICRO_TRUECOLOR") == "1"

	// In order to enable true color, we have to set the TERM to `xterm-truecolor` when
	// initializing tcell, but after that, we can set the TERM back to whatever it was
	oldTerm := os.Getenv("TERM")
	if truecolor {
		os.Setenv("TERM", "xterm-truecolor")
	}

	// Initilize tcell
	var err error
	screen, err = tcell.NewScreen()
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	if err = screen.Init(); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	// Now we can put the TERM back to what it was before
	if truecolor {
		os.Setenv("TERM", oldTerm)
	}

	screen.SetStyle(defStyle)
	screen.EnableMouse()
}

// RedrawAll redraws everything -- all the views and the messenger
func RedrawAll() {
	messenger.Clear()
	for _, v := range tabs[curTab].views {
		v.Display()
	}
	DisplayTabs()
	messenger.Display()
	screen.Show()
}

// Passing -version as a flag will have micro print out the version number
var flagVersion = flag.Bool("version", false, "Show the version number")

func main() {
	flag.Parse()
	if *flagVersion {
		// If -version was passed
		fmt.Println("Micro version:", Version)
		os.Exit(0)
	}

	// Start the Lua VM for running plugins
	L = lua.NewState()
	defer L.Close()

	// Some encoding stuff in case the user isn't using UTF-8
	encoding.Register()
	tcell.SetEncodingFallback(tcell.EncodingFallbackASCII)

	// Find the user's configuration directory (probably $XDG_CONFIG_HOME/micro)
	InitConfigDir()

	// Load the user's settings
	InitSettings()
	InitCommands()
	InitBindings()

	// Load the syntax files, including the colorscheme
	LoadSyntaxFiles()

	// Load the help files
	LoadHelp()

	// Start the screen
	InitScreen()

	// This is just so if we have an error, we can exit cleanly and not completely
	// mess up the terminal being worked in
	// In other words we need to shut down tcell before the program crashes
	defer func() {
		if err := recover(); err != nil {
			screen.Fini()
			fmt.Println("Micro encountered an error:", err)
			// Print the stack trace too
			fmt.Print(errors.Wrap(err, 2).ErrorStack())
			os.Exit(1)
		}
	}()

	// Create a new messenger
	// This is used for sending the user messages in the bottom of the editor
	messenger = new(Messenger)
	messenger.history = make(map[string][]string)

	// Now we load the input
	buffers := LoadInput()
	for _, buf := range buffers {
		// For each buffer we create a new tab and place the view in that tab
		tab := NewTabFromView(NewView(buf))
		tab.SetNum(len(tabs))
		tabs = append(tabs, tab)
		// for _, t := range tabs {
		// 	for _, v := range t.views {
		// 		v.Resize(screen.Size())
		// 	}
		// }
	}

	// Load all the plugin stuff
	// We give plugins access to a bunch of variables here which could be useful to them
	L.SetGlobal("OS", luar.New(L, runtime.GOOS))
	L.SetGlobal("tabs", luar.New(L, tabs))
	L.SetGlobal("curTab", luar.New(L, curTab))
	L.SetGlobal("messenger", luar.New(L, messenger))
	L.SetGlobal("GetOption", luar.New(L, GetOption))
	L.SetGlobal("AddOption", luar.New(L, AddOption))
	L.SetGlobal("BindKey", luar.New(L, BindKey))
	L.SetGlobal("MakeCommand", luar.New(L, MakeCommand))
	L.SetGlobal("CurView", luar.New(L, CurView))
	L.SetGlobal("IsWordChar", luar.New(L, IsWordChar))

	// Used for asynchronous jobs
	L.SetGlobal("JobStart", luar.New(L, JobStart))
	L.SetGlobal("JobSend", luar.New(L, JobSend))
	L.SetGlobal("JobStop", luar.New(L, JobStop))

	LoadPlugins()

	jobs = make(chan JobFunction, 100)
	events = make(chan tcell.Event)

	// Here is the event loop which runs in a separate thread
	go func() {
		for {
			events <- screen.PollEvent()
		}
	}()

	for {
		// Display everything
		RedrawAll()

		var event tcell.Event

		// Check for new events
		select {
		case f := <-jobs:
			// If a new job has finished while running in the background we should execute the callback
			f.function(f.output, f.args...)
			continue
		case event = <-events:
		}

		switch e := event.(type) {
		case *tcell.EventMouse:
			if e.Buttons() == tcell.Button1 {
				// If the user left clicked we check a couple things
				_, h := screen.Size()
				x, y := e.Position()
				if y == h-1 && messenger.message != "" {
					// If the user clicked in the bottom bar, and there is a message down there
					// we copy it to the clipboard.
					// Often error messages are displayed down there so it can be useful to easily
					// copy the message
					clipboard.WriteAll(messenger.message)
					continue
				}

				// We loop through each view in the current tab and make sure the current view
				// it the one being clicked in
				for _, v := range tabs[curTab].views {
					if x >= v.x && x < v.x+v.width && y >= v.y && y < v.y+v.height {
						tabs[curTab].curView = v.Num
					}
				}
			}
		}

		// This function checks the mouse event for the possibility of changing the current tab
		// If the tab was changed it returns true
		if TabbarHandleMouseEvent(event) {
			continue
		}

		if searching {
			// Since searching is done in real time, we need to redraw every time
			// there is a new event in the search bar so we need a special function
			// to run instead of the standard HandleEvent.
			HandleSearchEvent(event, CurView())
		} else {
			// Send it to the view
			CurView().HandleEvent(event)
		}
	}
}
<commit_msg>Add HandleCommand and HandleShellCommand to plugin api<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"runtime"

	"github.com/go-errors/errors"
	"github.com/layeh/gopher-luar"
	"github.com/mattn/go-isatty"
	"github.com/mitchellh/go-homedir"
	"github.com/yuin/gopher-lua"
	"github.com/zyedidia/clipboard"
	"github.com/zyedidia/tcell"
	"github.com/zyedidia/tcell/encoding"
)

const (
	synLinesUp           = 75  // How many lines up to look to do syntax highlighting
	synLinesDown         = 75  // How many lines down to look to do syntax highlighting
	doubleClickThreshold = 400 // How many milliseconds to wait before a second click is not a double click
	undoThreshold        = 500 // If two events are less than n milliseconds apart, undo both of them
)

var (
	// The main screen
	screen tcell.Screen

	// Object to send messages and prompts to the user
	messenger *Messenger

	// The default highlighting style
	// This simply defines the default foreground and background colors
	defStyle tcell.Style

	// Where the user's configuration is
	// This should be $XDG_CONFIG_HOME/micro
	// If $XDG_CONFIG_HOME is not set, it is ~/.config/micro
	configDir string

	// Version is the version number or commit hash
	// This should be set by the linker when compiling
	Version = "Unknown"

	// L is the lua state
	// This is the VM that runs the plugins
	L *lua.LState

	// The list of views
	tabs []*Tab
	// This is the currently open tab
	// It's just an index to the tab in the tabs array
	curTab int

	// Channel of jobs running in the background
	jobs chan JobFunction
	// Event channel
	events chan tcell.Event
)

// LoadInput determines which files should be loaded into buffers
// based on the input stored in os.Args
func LoadInput() []*Buffer {
	// There are a number of ways micro should start given its input

	// 1. If it is given a files in os.Args, it should open those

	// 2. If there is no input file and the input is not a terminal, that means
	// something is being piped in and the stdin should be opened in an
	// empty buffer

	// 3. If there is no input file and the input is a terminal, an empty buffer
	// should be opened

	var filename string
	var input []byte
	var err error
	var buffers []*Buffer

	if len(os.Args) > 1 {
		// Option 1
		// We go through each file and load it
		for i := 1; i < len(os.Args); i++ {
			filename = os.Args[i]
			// Check that the file exists
			if _, e := os.Stat(filename); e == nil {
				// If it exists we load it into a buffer
				input, err = ioutil.ReadFile(filename)
				if err != nil {
					TermMessage(err)
					continue
				}
			}
			// If the file didn't exist, input will be empty, and we'll open an empty buffer
			buffers = append(buffers, NewBuffer(input, filename))
		}
	} else if !isatty.IsTerminal(os.Stdin.Fd()) {
		// Option 2
		// The input is not a terminal, so something is being piped in
		// and we should read from stdin
		input, err = ioutil.ReadAll(os.Stdin)
		buffers = append(buffers, NewBuffer(input, filename))
	} else {
		// Option 3, just open an empty buffer
		buffers = append(buffers, NewBuffer(input, filename))
	}

	return buffers
}

// InitConfigDir finds the configuration directory for micro according to the XDG spec.
// If no directory is found, it creates one.
func InitConfigDir() {
	xdgHome := os.Getenv("XDG_CONFIG_HOME")
	if xdgHome == "" {
		// The user has not set $XDG_CONFIG_HOME so we should act like it was set to ~/.config
		home, err := homedir.Dir()
		if err != nil {
			TermMessage("Error finding your home directory\nCan't load config files")
			return
		}
		xdgHome = home + "/.config"
	}
	configDir = xdgHome + "/micro"

	if _, err := os.Stat(xdgHome); os.IsNotExist(err) {
		// If the xdgHome doesn't exist we should create it
		err = os.Mkdir(xdgHome, os.ModePerm)
		if err != nil {
			TermMessage("Error creating XDG_CONFIG_HOME directory: " + err.Error())
		}
	}

	if _, err := os.Stat(configDir); os.IsNotExist(err) {
		// If the micro specific config directory doesn't exist we should create that too
		err = os.Mkdir(configDir, os.ModePerm)
		if err != nil {
			TermMessage("Error creating configuration directory: " + err.Error())
		}
	}
}

// InitScreen creates and initializes the tcell screen
func InitScreen() {
	// Should we enable true color?
	truecolor := os.Getenv("MICRO_TRUECOLOR") == "1"

	// In order to enable true color, we have to set the TERM to `xterm-truecolor` when
	// initializing tcell, but after that, we can set the TERM back to whatever it was
	oldTerm := os.Getenv("TERM")
	if truecolor {
		os.Setenv("TERM", "xterm-truecolor")
	}

	// Initilize tcell
	var err error
	screen, err = tcell.NewScreen()
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	if err = screen.Init(); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	// Now we can put the TERM back to what it was before
	if truecolor {
		os.Setenv("TERM", oldTerm)
	}

	screen.SetStyle(defStyle)
	screen.EnableMouse()
}

// RedrawAll redraws everything -- all the views and the messenger
func RedrawAll() {
	messenger.Clear()
	for _, v := range tabs[curTab].views {
		v.Display()
	}
	DisplayTabs()
	messenger.Display()
	screen.Show()
}

// Passing -version as a flag will have micro print out the version number
var flagVersion = flag.Bool("version", false, "Show the version number")

func main() {
	flag.Parse()
	if *flagVersion {
		// If -version was passed
		fmt.Println("Micro version:", Version)
		os.Exit(0)
	}

	// Start the Lua VM for running plugins
	L = lua.NewState()
	defer L.Close()

	// Some encoding stuff in case the user isn't using UTF-8
	encoding.Register()
	tcell.SetEncodingFallback(tcell.EncodingFallbackASCII)

	// Find the user's configuration directory (probably $XDG_CONFIG_HOME/micro)
	InitConfigDir()

	// Load the user's settings
	InitSettings()
	InitCommands()
	InitBindings()

	// Load the syntax files, including the colorscheme
	LoadSyntaxFiles()

	// Load the help files
	LoadHelp()

	// Start the screen
	InitScreen()

	// This is just so if we have an error, we can exit cleanly and not completely
	// mess up the terminal being worked in
	// In other words we need to shut down tcell before the program crashes
	defer func() {
		if err := recover(); err != nil {
			screen.Fini()
			fmt.Println("Micro encountered an error:", err)
			// Print the stack trace too
			fmt.Print(errors.Wrap(err, 2).ErrorStack())
			os.Exit(1)
		}
	}()

	// Create a new messenger
	// This is used for sending the user messages in the bottom of the editor
	messenger = new(Messenger)
	messenger.history = make(map[string][]string)

	// Now we load the input
	buffers := LoadInput()
	for _, buf := range buffers {
		// For each buffer we create a new tab and place the view in that tab
		tab := NewTabFromView(NewView(buf))
		tab.SetNum(len(tabs))
		tabs = append(tabs, tab)
		// for _, t := range tabs {
		// 	for _, v := range t.views {
		// 		v.Resize(screen.Size())
		// 	}
		// }
	}

	// Load all the plugin stuff
	// We give plugins access to a bunch of variables here which could be useful to them
	L.SetGlobal("OS", luar.New(L, runtime.GOOS))
	L.SetGlobal("tabs", luar.New(L, tabs))
	L.SetGlobal("curTab", luar.New(L, curTab))
	L.SetGlobal("messenger", luar.New(L, messenger))
	L.SetGlobal("GetOption", luar.New(L, GetOption))
	L.SetGlobal("AddOption", luar.New(L, AddOption))
	L.SetGlobal("BindKey", luar.New(L, BindKey))
	L.SetGlobal("MakeCommand", luar.New(L, MakeCommand))
	L.SetGlobal("CurView", luar.New(L, CurView))
	L.SetGlobal("IsWordChar", luar.New(L, IsWordChar))
	L.SetGlobal("HandleCommand", luar.New(L, HandleCommand))
	L.SetGlobal("HandleShellCommand", luar.New(L, HandleShellCommand))

	// Used for asynchronous jobs
	L.SetGlobal("JobStart", luar.New(L, JobStart))
	L.SetGlobal("JobSend", luar.New(L, JobSend))
	L.SetGlobal("JobStop", luar.New(L, JobStop))

	LoadPlugins()

	jobs = make(chan JobFunction, 100)
	events = make(chan tcell.Event)

	// Here is the event loop which runs in a separate thread
	go func() {
		for {
			events <- screen.PollEvent()
		}
	}()

	for {
		// Display everything
		RedrawAll()

		var event tcell.Event

		// Check for new events
		select {
		case f := <-jobs:
			// If a new job has finished while running in the background we should execute the callback
			f.function(f.output, f.args...)
			continue
		case event = <-events:
		}

		switch e := event.(type) {
		case *tcell.EventMouse:
			if e.Buttons() == tcell.Button1 {
				// If the user left clicked we check a couple things
				_, h := screen.Size()
				x, y := e.Position()
				if y == h-1 && messenger.message != "" {
					// If the user clicked in the bottom bar, and there is a message down there
					// we copy it to the clipboard.
					// Often error messages are displayed down there so it can be useful to easily
					// copy the message
					clipboard.WriteAll(messenger.message)
					continue
				}

				// We loop through each view in the current tab and make sure the current view
				// it the one being clicked in
				for _, v := range tabs[curTab].views {
					if x >= v.x && x < v.x+v.width && y >= v.y && y < v.y+v.height {
						tabs[curTab].curView = v.Num
					}
				}
			}
		}

		// This function checks the mouse event for the possibility of changing the current tab
		// If the tab was changed it returns true
		if TabbarHandleMouseEvent(event) {
			continue
		}

		if searching {
			// Since searching is done in real time, we need to redraw every time
			// there is a new event in the search bar so we need a special function
			// to run instead of the standard HandleEvent.
			HandleSearchEvent(event, CurView())
		} else {
			// Send it to the view
			CurView().HandleEvent(event)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 CodeIgnition. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"log"
	"time"

	"golang.org/x/net/context"

	"github.com/codeignition/recon/cmd/recond/config"
	"github.com/codeignition/recon/policy"
	_ "github.com/codeignition/recon/policy/handlers"
	"github.com/nats-io/nats"
)

const agentsAPIPath = "/api/agents" // agents path in the marksman server

// natsEncConn is the opened with the URL obtained from marksman.
// It is populated if the agent registers successfully.
var natsEncConn *nats.EncodedConn

// updateInterval is time.Duration that specifies the interval
// between two consecutive updates.
const updateInterval = 5 * time.Second

func main() {
	log.SetPrefix("recond: ")

	var marksmanAddr = flag.String("marksman", "http://localhost:3000", "address of the marksman server")
	flag.Parse()

	conf, err := config.Init()
	if err != nil {
		log.Fatalln(err)
	}

	// agent represents a single agent on which the recond
	// is running.
	var agent = &Agent{
		UID: conf.UID,
	}

	err = agent.register(*marksmanAddr)
	if err != nil {
		log.Fatalln(err)
	}

	defer natsEncConn.Close()

	go runStoredPolicies(conf)

	natsEncConn.Subscribe(agent.UID, func(s string) {
		fmt.Printf("Received a message: %s\n", s)
	})

	natsEncConn.Subscribe(agent.UID+"_policy", func(subj, reply string, p *policy.Policy) {
		fmt.Printf("Received a Policy: %v\n", p)
		if err := conf.AddPolicy(*p); err != nil {
			natsEncConn.Publish(reply, err.Error())
			return
		}
		if err := conf.Save(); err != nil {
			natsEncConn.Publish(reply, err.Error())
			return
		}
		events, err := p.Execute(context.TODO())
		if err != nil {
			natsEncConn.Publish(reply, err.Error())
			return
		}
		natsEncConn.Publish(reply, "policy ack") // acknowledge
		for e := range events {
			fmt.Printf("%+v\n", e)
		}
	})

	c := time.Tick(updateInterval)
	for now := range c {
		log.Println("Update sent at", now)
		agent.update()
	}
}

func runStoredPolicies(c *config.Config) {
	for _, p := range c.PolicyConfig {
		go func() {
			events, err := p.Execute(context.TODO())
			if err != nil {
				log.Fatal(err) // TODO: send to a nats errors channel
			}
			for e := range events {
				fmt.Printf("%+v\n", e)
			}
		}()
	}
}
<commit_msg>recond: publish events instead of printing<commit_after>// Copyright 2015 CodeIgnition. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"log"
	"time"

	"golang.org/x/net/context"

	"github.com/codeignition/recon/cmd/recond/config"
	"github.com/codeignition/recon/policy"
	_ "github.com/codeignition/recon/policy/handlers"
	"github.com/nats-io/nats"
)

const agentsAPIPath = "/api/agents" // agents path in the marksman server

// natsEncConn is the opened with the URL obtained from marksman.
// It is populated if the agent registers successfully.
var natsEncConn *nats.EncodedConn

// updateInterval is time.Duration that specifies the interval
// between two consecutive updates.
const updateInterval = 5 * time.Second

func main() {
	log.SetPrefix("recond: ")

	var marksmanAddr = flag.String("marksman", "http://localhost:3000", "address of the marksman server")
	flag.Parse()

	conf, err := config.Init()
	if err != nil {
		log.Fatalln(err)
	}

	// agent represents a single agent on which the recond
	// is running.
	var agent = &Agent{
		UID: conf.UID,
	}

	err = agent.register(*marksmanAddr)
	if err != nil {
		log.Fatalln(err)
	}

	defer natsEncConn.Close()

	go runStoredPolicies(conf)

	natsEncConn.Subscribe(agent.UID, func(s string) {
		fmt.Printf("Received a message: %s\n", s)
	})

	natsEncConn.Subscribe(agent.UID+"_policy", func(subj, reply string, p *policy.Policy) {
		fmt.Printf("Received a Policy: %v\n", p)
		if err := conf.AddPolicy(*p); err != nil {
			natsEncConn.Publish(reply, err.Error())
			return
		}
		if err := conf.Save(); err != nil {
			natsEncConn.Publish(reply, err.Error())
			return
		}
		events, err := p.Execute(context.TODO())
		if err != nil {
			natsEncConn.Publish(reply, err.Error())
			return
		}
		natsEncConn.Publish(reply, "policy ack") // acknowledge
		for e := range events {
			natsEncConn.Publish("policy_events", e)
		}
	})

	c := time.Tick(updateInterval)
	for now := range c {
		log.Println("Update sent at", now)
		agent.update()
	}
}

func runStoredPolicies(c *config.Config) {
	for _, p := range c.PolicyConfig {
		go func() {
			events, err := p.Execute(context.TODO())
			if err != nil {
				log.Fatal(err) // TODO: send to a nats errors channel
			}
			for e := range events {
				natsEncConn.Publish("policy_events", e)
			}
		}()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"

	"github.com/jawher/mow.cli"
	"github.com/vially/seomoz"
)

func main() {
	app := cli.App("seomoz", "Analyze URLs using SEOmoz")

	app.Spec = "[--cols=<SEOmoz COLS>] URL..."

	var (
		cols = app.IntOpt("c cols", seomoz.DefaultCols, "SEOmoz COLS")
		urls = app.StringsArg("URL", nil, "URLs to analyze")
	)

	app.Action = func() {
		client := seomoz.NewEnvClient()
		response, err := client.GetBulkURLMetrics(*urls, *cols)
		if err != nil {
			log.Fatalln(err)
		}

		for _, metrics := range response {
			fmt.Printf("%s\tLinks: %.0f\tPage Authority: %.0f\tDomain Authority: %.0f\n", metrics.URL, metrics.Links, metrics.PageAuthority, metrics.DomainAuthority)
		}
	}

	app.Run(os.Args)
}
<commit_msg>Validate API credentials are set before run<commit_after>package main

import (
	"fmt"
	"log"
	"os"

	"github.com/jawher/mow.cli"
	"github.com/vially/seomoz"
)

func main() {
	app := cli.App("seomoz", "Analyze URLs using SEOmoz")

	app.Spec = "[--cols=<SEOmoz COLS>] URL..."

	var (
		cols = app.IntOpt("c cols", seomoz.DefaultCols, "SEOmoz COLS")
		urls = app.StringsArg("URL", nil, "URLs to analyze")
	)

	app.Action = func() {
		accessID := os.Getenv("SEOMOZ_ACCESS_ID")
		secretKey := os.Getenv("SEOMOZ_SECRET_KEY")
		if accessID == "" || secretKey == "" {
			log.Fatalln("SEOmoz API credentials not configured")
		}

		client := seomoz.NewClient(accessID, secretKey)
		response, err := client.GetBulkURLMetrics(*urls, *cols)
		if err != nil {
			log.Fatalln(err)
		}

		for _, metrics := range response {
			fmt.Printf("%s\tLinks: %.0f\tPage Authority: %.0f\tDomain Authority: %.0f\n", metrics.URL, metrics.Links, metrics.PageAuthority, metrics.DomainAuthority)
		}
	}

	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package cmd_test

import (
	"errors"
	"io/ioutil"
	"strings"
	"testing"
	"time"

	"github.com/pilosa/pilosa"
	"github.com/pilosa/pilosa/cmd"
)

func TestServerHelp(t *testing.T) {
	output, err := ExecNewRootCommand(t, "server", "--help")
	if !strings.Contains(output, "Usage:") ||
		!strings.Contains(output, "Flags:") || err != nil {
		t.Fatalf("Command 'server --help' not working, err: '%v', output: '%s'", err, output)
	}
}

func TestServerConfig(t *testing.T) {
	actualDataDir, err := ioutil.TempDir("", "")
	failErr(t, err, "making data dir")
	profFile, err := ioutil.TempFile("", "")
	failErr(t, err, "making temp file")
	logFile, err := ioutil.TempFile("", "")
	failErr(t, err, "making log file")
	tests := []commandTest{
		// TEST 0
		{
			args: []string{"server", "--data-dir", actualDataDir, "--cluster.hosts", "example.com:10101,example.com:10110"},
			env:  map[string]string{"PILOSA_DATA_DIR": "/tmp/myEnvDatadir", "PILOSA_CLUSTER.POLL_INTERVAL": "3m2s"},
			cfgFileContent: `
data-dir = "/tmp/myFileDatadir"
bind = "localhost:0"

[cluster]
  poll-interval = "45s"
  replicas = 2
  hosts = [
   "localhost:19444",
   ]
`,
			validation: func() error {
				v := validator{}
				v.Check(cmd.Server.Config.DataDir, actualDataDir)
				v.Check(cmd.Server.Config.Host, "localhost:0")
				v.Check(cmd.Server.Config.Cluster.ReplicaN, 2)
				v.Check(cmd.Server.Config.Cluster.Hosts, []string{"example.com:10101", "example.com:10110"})
				v.Check(cmd.Server.Config.Cluster.PollingInterval, pilosa.Duration(time.Second*182))
				return v.Error()
			},
		},
		// TEST 1
		{
			args: []string{"server", "--anti-entropy.interval", "9m0s"},
			env:  map[string]string{"PILOSA_CLUSTER.HOSTS": "example.com:1110,example.com:1111"},
			cfgFileContent: `
bind = "localhost:0"
data-dir = "` + actualDataDir + `"
[cluster]
  hosts = [
   "localhost:19444",
   ]
[plugins]
  path = "/var/sloth"
`,
			validation: func() error {
				v := validator{}
				v.Check(cmd.Server.Config.Cluster.Hosts, []string{"example.com:1110", "example.com:1111"})
				v.Check(cmd.Server.Config.Plugins.Path, "/var/sloth")
				v.Check(cmd.Server.Config.AntiEntropy.Interval, pilosa.Duration(time.Minute*9))
				return v.Error()
			},
		},
		// TEST 2
		{
			args: []string{"server", "--log-path", logFile.Name()},
			env:  map[string]string{"PILOSA_PROFILE.CPU_TIME": "1m"},
			cfgFileContent: `
bind = "localhost:0"
data-dir = "` + actualDataDir + `"
[cluster]
  poll-interval = "2m0s"
  hosts = [
   "localhost:19444",
   ]
[anti-entropy]
  interval = "11m0s"
[profile]
  cpu = "` + profFile.Name() + `"
  cpu-time = "35s"
`,
			validation: func() error {
				v := validator{}
				v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:19444"})
				v.Check(cmd.Server.Config.Cluster.PollingInterval, pilosa.Duration(time.Minute*2))
				v.Check(cmd.Server.Config.AntiEntropy.Interval, pilosa.Duration(time.Minute*11))
				v.Check(cmd.Server.CPUProfile, profFile.Name())
				v.Check(cmd.Server.CPUTime, time.Minute)
				v.Check(cmd.Server.Config.LogPath, logFile.Name())
				if v.Error() != nil {
					return v.Error()
				}
				// confirm log file was written
				info, err := logFile.Stat()
				if err != nil || info.Size() == 0 {
					// NOTE: this test assumes that something is being written to the log
					// currently, that is relying on log: "index sync monitor initializing"
					return errors.New("Log file was not written!")
				}
				return nil
			},
		},
	}

	// run server tests
	for i, test := range tests {
		com := test.setupCommand(t)
		executed := make(chan struct{})
		var execErr error
		go func() {
			execErr = com.Execute()
			close(executed)
		}()
		select {
		case <-cmd.Server.Started:
		case <-executed:
		}
		err := cmd.Server.Close()
		failErr(t, err, "closing pilosa server command")
		<-executed
		failErr(t, execErr, "executing command")

		if err := test.validation(); err != nil {
			t.Fatalf("Failed test %d due to: %v", i, err)
		}
		test.reset()
	}
}
<commit_msg>fix server tests to bind to new node if changing the node set<commit_after>package cmd_test

import (
	"errors"
	"io/ioutil"
	"strings"
	"testing"
	"time"

	"github.com/pilosa/pilosa"
	"github.com/pilosa/pilosa/cmd"
)

func TestServerHelp(t *testing.T) {
	output, err := ExecNewRootCommand(t, "server", "--help")
	if !strings.Contains(output, "Usage:") ||
		!strings.Contains(output, "Flags:") || err != nil {
		t.Fatalf("Command 'server --help' not working, err: '%v', output: '%s'", err, output)
	}
}

func TestServerConfig(t *testing.T) {
	actualDataDir, err := ioutil.TempDir("", "")
	failErr(t, err, "making data dir")
	profFile, err := ioutil.TempFile("", "")
	failErr(t, err, "making temp file")
	logFile, err := ioutil.TempFile("", "")
	failErr(t, err, "making log file")
	tests := []commandTest{
		// TEST 0
		{
			args: []string{"server", "--data-dir", actualDataDir, "--cluster.hosts", "example.com:10111,example.com:10110", "--bind", "example.com:10111"},
			env:  map[string]string{"PILOSA_DATA_DIR": "/tmp/myEnvDatadir", "PILOSA_CLUSTER.POLL_INTERVAL": "3m2s"},
			cfgFileContent: `
	data-dir = "/tmp/myFileDatadir"
	bind = "localhost:0"

	[cluster]
		poll-interval = "45s"
		replicas = 2
		hosts = [
			"localhost:19444",
		]
	`,
			validation: func() error {
				v := validator{}
				v.Check(cmd.Server.Config.DataDir, actualDataDir)
				v.Check(cmd.Server.Config.Host, "example.com:10111")
				v.Check(cmd.Server.Config.Cluster.ReplicaN, 2)
				v.Check(cmd.Server.Config.Cluster.Hosts, []string{"example.com:10111", "example.com:10110"})
				v.Check(cmd.Server.Config.Cluster.PollingInterval, pilosa.Duration(time.Second*182))
				return v.Error()
			},
		},
		// TEST 1
		{
			args: []string{"server", "--anti-entropy.interval", "9m0s"},
			env:  map[string]string{"PILOSA_CLUSTER.HOSTS": "example.com:1110,example.com:1111", "PILOSA_BIND": "example.com:1110"},
			cfgFileContent: `
	bind = "localhost:0"
	data-dir = "` + actualDataDir + `"
	[cluster]
		hosts = [
			"localhost:19444",
		]
	[plugins]
		path = "/var/sloth"
	`,
			validation: func() error {
				v := validator{}
				v.Check(cmd.Server.Config.Cluster.Hosts, []string{"example.com:1110", "example.com:1111"})
				v.Check(cmd.Server.Config.Plugins.Path, "/var/sloth")
				v.Check(cmd.Server.Config.AntiEntropy.Interval, pilosa.Duration(time.Minute*9))
				return v.Error()
			},
		},
		// TEST 2
		{
			args: []string{"server", "--log-path", logFile.Name()},
			env:  map[string]string{"PILOSA_PROFILE.CPU_TIME": "1m"},
			cfgFileContent: `
	bind = "localhost:19444"
	data-dir = "` + actualDataDir + `"
	[cluster]
		poll-interval = "2m0s"
		hosts = [
			"localhost:19444",
		]
	[anti-entropy]
		interval = "11m0s"
	[profile]
		cpu = "` + profFile.Name() + `"
		cpu-time = "35s"
	[metric]
		service = "statsd"
		host = "127.0.0.1:8125"
	`,
			validation: func() error {
				v := validator{}
				v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:19444"})
				v.Check(cmd.Server.Config.Cluster.PollingInterval, pilosa.Duration(time.Minute*2))
				v.Check(cmd.Server.Config.AntiEntropy.Interval, pilosa.Duration(time.Minute*11))
				v.Check(cmd.Server.CPUProfile, profFile.Name())
				v.Check(cmd.Server.CPUTime, time.Minute)
				v.Check(cmd.Server.Config.LogPath, logFile.Name())
				v.Check(cmd.Server.Config.Metric.Service, "statsd")
				v.Check(cmd.Server.Config.Metric.Host, "127.0.0.1:8125")
				if v.Error() != nil {
					return v.Error()
				}
				// confirm log file was written
				info, err := logFile.Stat()
				if err != nil || info.Size() == 0 {
					// NOTE: this test assumes that something is being written to the log
					// currently, that is relying on log: "index sync monitor initializing"
					return errors.New("Log file was not written!")
				}
				return nil
			},
		},
	}

	// run server tests
	for i, test := range tests {
		com := test.setupCommand(t)
		executed := make(chan struct{})
		var execErr error
		go func() {
			execErr = com.Execute()
			close(executed)
		}()
		select {
		case <-cmd.Server.Started:
		case <-executed:
		}
		err := cmd.Server.Close()
		failErr(t, err, "closing pilosa server command")
		<-executed
		failErr(t, execErr, "executing command")

		if err := test.validation(); err != nil {
			t.Fatalf("Failed test %d due to: %v", i, err)
		}
		test.reset()
	}
}
<|endoftext|>


<commit_before>// throw stores a file in the cloud, encrypted.
package main

import (
	"crypto/sha256"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"path"
	"runtime"
	"sync"
	"time"

	"github.com/asjoyner/shade"
	"github.com/asjoyner/shade/config"
	"github.com/asjoyner/shade/drive"
	"github.com/golang/glog"
	"github.com/jpillora/backoff"

	_ "github.com/asjoyner/shade/drive/amazon"
	_ "github.com/asjoyner/shade/drive/cache"
	_ "github.com/asjoyner/shade/drive/encrypt"
	_ "github.com/asjoyner/shade/drive/google"
	_ "github.com/asjoyner/shade/drive/local"
	_ "github.com/asjoyner/shade/drive/memory"
	_ "github.com/asjoyner/shade/drive/win"
)

var (
	defaultConfig = path.Join(shade.ConfigDir(), "config.json")
	configPath    = flag.String("config", defaultConfig, "shade config file")
	numWorkers    = flag.Int("numUploaders", 20, "The number of goroutines to upload chunks in parallel.")
	maxRetries    = flag.Int("maxRetries", 10, "The number of times to try to write a chunk to persistent storage.")
	// maxChunks serves as a backstop against accidentaly uploading /dev/urandom
	// ad infinitum, and can be set to a lower value to facilitate testing memory
	// usage, etc.  You may find testdata/config.win.json helpful for this.
	maxChunks = flag.Int("maxChunks", 1000000, "The maximum number of chunks to read for a given file.")
)

type chunkToGo struct {
	chunk      shade.Chunk
	chunkbytes []byte
	manifest   *shade.File
}

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "\nusage: %s [flags] <filename> <destination filename>\n", path.Base(os.Args[0]))
		flag.PrintDefaults()
	}

	flag.Parse()
	if flag.NArg() != 2 {
		flag.Usage()
		os.Exit(2)
	}

	// read in the config
	config, err := config.Read(*configPath)
	if err != nil {
		fmt.Fprintf(os.Stderr, "could not initialize clients: %s\n", err)
		os.Exit(1)
	}

	start := time.Now()

	// initialize client
	client, err := drive.NewClient(config)
	if err != nil {
		log.Fatalf("could not initialize client: %s\n", err)
	}

	// initialize the goroutines to upload chunks
	uploadRequests := make(chan chunkToGo)
	var workers sync.WaitGroup
	workers.Add(*numWorkers)
	for w := 1; w <= *numWorkers; w++ {
		go func(reqs chan chunkToGo) {
			for r := range reqs {
				numRetries := 0
				b := &backoff.Backoff{Factor: 4}
				for {
					numRetries++
					if err := client.PutChunk(r.chunk.Sha256, r.chunkbytes, r.manifest); err != nil {
						if numRetries >= *maxRetries {
							fmt.Fprintf(os.Stderr, "chunk upload failed: %s\n", err)
							os.Exit(1)
						}
						glog.Errorf("chunk write error, will retry: %s", err)
						time.Sleep(b.Duration())
						continue
					}
					b.Reset()
					break
				}
			}
			workers.Done()
			return
		}(uploadRequests)
	}

	filename := flag.Arg(0)

	manifest := shade.NewFile(flag.Arg(1))

	fh, err := os.Open(filename)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%s\n", err)
		os.Exit(3)
	}

	var rt runtime.MemStats
	for {
		// Initialize chunk, to ensure each chunk uses a unique nonce
		chunk := shade.NewChunk()
		chunk.Index = len(manifest.Chunks)
		// Initialize chunkbytes, so it's safe for concurrent access later
		chunkbytes := make([]byte, manifest.Chunksize)

		// Read a chunk
		numBytes, err := fh.Read(chunkbytes)
		if err == io.EOF {
			break
		} else if err != nil {
			fmt.Fprintf(os.Stderr, "%s\n", err)
			os.Exit(3)
		} else if len(manifest.Chunks) >= *maxChunks {
			glog.Info("Reached the maximum number of chunks in a single file.")
			break
		}
		manifest.Filesize += int64(numBytes)

		// truncate the chunkbytes array on the last read
		if numBytes < manifest.Chunksize {
			chunkbytes = chunkbytes[:numBytes]
			manifest.LastChunksize = numBytes
		}

		a := sha256.Sum256(chunkbytes)
		chunk.Sha256 = a[:]

		manifest.Chunks = append(manifest.Chunks, chunk)

		if glog.V(3) {
			if (len(manifest.Chunks) % 10) == 0 {
				runtime.ReadMemStats(&rt)
				glog.Infof("%d chunks: %0.2f MBytes Heap, %0.2f MBytes Sys\n", len(manifest.Chunks), float64(rt.Alloc)/1024/1024, float64(rt.Sys)/1024/1024)
			}
		}
		// upload the chunk
		uploadRequests <- chunkToGo{chunk, chunkbytes, manifest}
	}
	close(uploadRequests)
	workers.Wait()

	jm, err := json.Marshal(manifest)
	if err != nil {
		fmt.Fprintf(os.Stderr, "could not marshal file manifest: %s\n", err)
		os.Exit(1)
	}
	// upload the manifest
	a := sha256.Sum256(jm)
	if err := client.PutFile(a[:], jm); err != nil {
		fmt.Fprintf(os.Stderr, "manifest upload failed: %s\n", err)
		os.Exit(1)
	}

	elapsed := time.Since(start)
	size := manifest.Filesize / 1024 / 1024
	MBps := float64(size) / float64(elapsed.Nanoseconds()) / 1000000
	fmt.Printf("Uploaded %d MB in %s at %0.2f MB/s.\n", size, elapsed, MBps)
}
<commit_msg>Reduce numUploaders default; Nb: uses lots of ram<commit_after>// throw stores a file in the cloud, encrypted.
package main

import (
	"crypto/sha256"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"path"
	"runtime"
	"sync"
	"time"

	"github.com/asjoyner/shade"
	"github.com/asjoyner/shade/config"
	"github.com/asjoyner/shade/drive"
	"github.com/golang/glog"
	"github.com/jpillora/backoff"

	_ "github.com/asjoyner/shade/drive/amazon"
	_ "github.com/asjoyner/shade/drive/cache"
	_ "github.com/asjoyner/shade/drive/encrypt"
	_ "github.com/asjoyner/shade/drive/google"
	_ "github.com/asjoyner/shade/drive/local"
	_ "github.com/asjoyner/shade/drive/memory"
	_ "github.com/asjoyner/shade/drive/win"
)

var (
	defaultConfig = path.Join(shade.ConfigDir(), "config.json")
	configPath    = flag.String("config", defaultConfig, "shade config file")
	// numUploaders has diminishing returns after about 3-4, and meaningfully
	// increases memory usage.  Setting it too high will almost certainly cause
	// OOMs when upload files of more than trivial size.
	numUploaders = flag.Int("numUploaders", 3, "The number of goroutines to upload chunks in parallel.")
	maxRetries   = flag.Int("maxRetries", 10, "The number of times to try to write a chunk to persistent storage.")
	// maxChunks serves as a backstop against accidentaly uploading /dev/urandom
	// ad infinitum, and can be set to a lower value to facilitate testing memory
	// usage, etc.  You may find testdata/config.win.json helpful for this.
	maxChunks = flag.Int("maxChunks", 1000000, "The maximum number of chunks to read for a given file.")
)

type chunkToGo struct {
	chunk      shade.Chunk
	chunkbytes []byte
	manifest   *shade.File
}

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "\nusage: %s [flags] <filename> <destination filename>\n", path.Base(os.Args[0]))
		flag.PrintDefaults()
	}

	flag.Parse()
	if flag.NArg() != 2 {
		flag.Usage()
		os.Exit(2)
	}

	// read in the config
	config, err := config.Read(*configPath)
	if err != nil {
		fmt.Fprintf(os.Stderr, "could not initialize clients: %s\n", err)
		os.Exit(1)
	}

	start := time.Now()

	// initialize client
	client, err := drive.NewClient(config)
	if err != nil {
		log.Fatalf("could not initialize client: %s\n", err)
	}

	// initialize the goroutines to upload chunks
	uploadRequests := make(chan chunkToGo)
	var workers sync.WaitGroup
	workers.Add(*numUploaders)
	for w := 1; w <= *numUploaders; w++ {
		go func(reqs chan chunkToGo) {
			for r := range reqs {
				numRetries := 0
				b := &backoff.Backoff{Factor: 4}
				for {
					numRetries++
					if err := client.PutChunk(r.chunk.Sha256, r.chunkbytes, r.manifest); err != nil {
						if numRetries >= *maxRetries {
							fmt.Fprintf(os.Stderr, "chunk upload failed: %s\n", err)
							os.Exit(1)
						}
						glog.Errorf("chunk write error, will retry: %s", err)
						time.Sleep(b.Duration())
						continue
					}
					b.Reset()
					break
				}
			}
			workers.Done()
			return
		}(uploadRequests)
	}

	filename := flag.Arg(0)

	manifest := shade.NewFile(flag.Arg(1))

	fh, err := os.Open(filename)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%s\n", err)
		os.Exit(3)
	}

	var rt runtime.MemStats
	for {
		// Initialize chunk, to ensure each chunk uses a unique nonce
		chunk := shade.NewChunk()
		chunk.Index = len(manifest.Chunks)
		// Initialize chunkbytes, so it's safe for concurrent access later
		chunkbytes := make([]byte, manifest.Chunksize)

		// Read a chunk
		numBytes, err := fh.Read(chunkbytes)
		if err == io.EOF {
			break
		} else if err != nil {
			fmt.Fprintf(os.Stderr, "%s\n", err)
			os.Exit(3)
		} else if len(manifest.Chunks) >= *maxChunks {
			glog.Info("Reached the maximum number of chunks in a single file.")
			break
		}
		manifest.Filesize += int64(numBytes)

		// truncate the chunkbytes array on the last read
		if numBytes < manifest.Chunksize {
			chunkbytes = chunkbytes[:numBytes]
			manifest.LastChunksize = numBytes
		}

		a := sha256.Sum256(chunkbytes)
		chunk.Sha256 = a[:]

		manifest.Chunks = append(manifest.Chunks, chunk)

		if glog.V(3) {
			if (len(manifest.Chunks) % 10) == 0 {
				runtime.ReadMemStats(&rt)
				glog.Infof("%d chunks: %0.2f MBytes Heap, %0.2f MBytes Sys\n", len(manifest.Chunks), float64(rt.Alloc)/1024/1024, float64(rt.Sys)/1024/1024)
			}
		}
		// upload the chunk
		uploadRequests <- chunkToGo{chunk, chunkbytes, manifest}
	}
	close(uploadRequests)
	workers.Wait()

	jm, err := json.Marshal(manifest)
	if err != nil {
		fmt.Fprintf(os.Stderr, "could not marshal file manifest: %s\n", err)
		os.Exit(1)
	}
	// upload the manifest
	a := sha256.Sum256(jm)
	if err := client.PutFile(a[:], jm); err != nil {
		fmt.Fprintf(os.Stderr, "manifest upload failed: %s\n", err)
		os.Exit(1)
	}

	elapsed := time.Since(start)
	size := manifest.Filesize / 1024 / 1024
	MBps := float64(size) / float64(elapsed.Nanoseconds()) / 1000000
	fmt.Printf("Uploaded %d MB in %s at %0.2f MB/s.\n", size, elapsed, MBps)
}
<|endoftext|>