<commit_before>package funcs

import (
	"fmt"
	"github.com/leancloud/satori/common/model"
	"github.com/toolkits/nux"
	"log"
	"strings"
	"sync"
)

var (
	diskStatsMap = make(map[string][2]*nux.DiskStats)
	dsLock       = new(sync.RWMutex)
)

func UpdateDiskStats() error {
	dsList, err := nux.ListDiskStats()
	if err != nil {
		return err
	}

	dsLock.Lock()
	defer dsLock.Unlock()
	for i := 0; i < len(dsList); i++ {
		device := dsList[i].Device
		diskStatsMap[device] = [2]*nux.DiskStats{dsList[i], diskStatsMap[device][0]}
	}
	return nil
}

func IOReadRequests(arr [2]*nux.DiskStats) uint64 {
	return arr[0].ReadRequests - arr[1].ReadRequests
}

func IOReadMerged(arr [2]*nux.DiskStats) uint64 {
	return arr[0].ReadMerged - arr[1].ReadMerged
}

func IOReadSectors(arr [2]*nux.DiskStats) uint64 {
	return arr[0].ReadSectors - arr[1].ReadSectors
}

func IOMsecRead(arr [2]*nux.DiskStats) uint64 {
	return arr[0].MsecRead - arr[1].MsecRead
}

func IOWriteRequests(arr [2]*nux.DiskStats) uint64 {
	return arr[0].WriteRequests - arr[1].WriteRequests
}

func IOWriteMerged(arr [2]*nux.DiskStats) uint64 {
	return arr[0].WriteMerged - arr[1].WriteMerged
}

func IOWriteSectors(arr [2]*nux.DiskStats) uint64 {
	return arr[0].WriteSectors - arr[1].WriteSectors
}

func IOMsecWrite(arr [2]*nux.DiskStats) uint64 {
	return arr[0].MsecWrite - arr[1].MsecWrite
}

func IOMsecTotal(arr [2]*nux.DiskStats) uint64 {
	return arr[0].MsecTotal - arr[1].MsecTotal
}

func IOMsecWeightedTotal(arr [2]*nux.DiskStats) uint64 {
	return arr[0].MsecWeightedTotal - arr[1].MsecWeightedTotal
}

func TS(arr [2]*nux.DiskStats) uint64 {
	return uint64(arr[0].TS.Sub(arr[1].TS).Nanoseconds() / 1000000)
}

func IODelta(device string, f func([2]*nux.DiskStats) uint64) uint64 {
	val, ok := diskStatsMap[device]
	if !ok {
		return 0
	}

	if val[1] == nil {
		return 0
	}
	return f(val)
}

func DiskIOMetrics() (L []*model.MetricValue) {

	dsList, err := nux.ListDiskStats()
	if err != nil {
		log.Println(err)
		return
	}

	for _, ds := range dsList {
		if !ShouldHandleDevice(ds.Device) {
			continue
		}

		device := map[string]string{
			"device": ds.Device,
		}

		L = append(L, VT("disk.io.read_requests", float64(ds.ReadRequests), device))
		L = append(L, VT("disk.io.read_merged", float64(ds.ReadMerged), device))
		L = append(L, VT("disk.io.read_sectors", float64(ds.ReadSectors), device))
		L = append(L, VT("disk.io.msec_read", float64(ds.MsecRead), device))
		L = append(L, VT("disk.io.write_requests", float64(ds.WriteRequests), device))
		L = append(L, VT("disk.io.write_merged", float64(ds.WriteMerged), device))
		L = append(L, VT("disk.io.write_sectors", float64(ds.WriteSectors), device))
		L = append(L, VT("disk.io.msec_write", float64(ds.MsecWrite), device))
		L = append(L, VT("disk.io.ios_in_progress", float64(ds.IosInProgress), device))
		L = append(L, VT("disk.io.msec_total", float64(ds.MsecTotal), device))
		L = append(L, VT("disk.io.msec_weighted_total", float64(ds.MsecWeightedTotal), device))
	}
	return
}

func IOStatsMetrics() (L []*model.MetricValue) {
	dsLock.RLock()
	defer dsLock.RUnlock()

	for device, _ := range diskStatsMap {
		if !ShouldHandleDevice(device) {
			continue
		}

		rio := IODelta(device, IOReadRequests)
		wio := IODelta(device, IOWriteRequests)
		delta_rsec := IODelta(device, IOReadSectors)
		delta_wsec := IODelta(device, IOWriteSectors)
		ruse := IODelta(device, IOMsecRead)
		wuse := IODelta(device, IOMsecWrite)
		use := IODelta(device, IOMsecTotal)
		n_io := rio + wio
		avgrq_sz := 0.0
		await := 0.0
		svctm := 0.0
		if n_io != 0 {
			avgrq_sz = float64(delta_rsec+delta_wsec) / float64(n_io)
			await = float64(ruse+wuse) / float64(n_io)
			svctm = float64(use) / float64(n_io)
		}

		duration := IODelta(device, TS)

		tags := map[string]string{
			"device": device,
		}

		L = append(L, VT("disk.io.read_bytes", float64(delta_rsec)*512.0, tags))
		L = append(L, VT("disk.io.write_bytes", float64(delta_wsec)*512.0, tags))
		L = append(L, VT("disk.io.avgrq_sz", avgrq_sz, tags))
		L = append(L, VT("disk.io.avgqu_sz", float64(IODelta(device, IOMsecWeightedTotal))/1000.0, tags))
		L = append(L, VT("disk.io.await", await, tags))
		L = append(L, VT("disk.io.svctm", svctm, tags))
		tmp := float64(use) * 100.0 / float64(duration)
		if tmp > 100.0 {
			tmp = 100.0
		}
		L = append(L, VT("disk.io.util", tmp, tags))
	}

	return
}

func ShouldHandleDevice(device string) bool {
	normal := len(device) == 3 && (strings.HasPrefix(device, "sd") || strings.HasPrefix(device, "vd"))
	aws := len(device) >= 4 && strings.HasPrefix(device, "xvd")
	return normal || aws
}
<commit_msg>Fix compliation err<commit_after>package funcs

import (
	"github.com/leancloud/satori/common/model"
	"github.com/toolkits/nux"
	"log"
	"strings"
	"sync"
)

var (
	diskStatsMap = make(map[string][2]*nux.DiskStats)
	dsLock       = new(sync.RWMutex)
)

func UpdateDiskStats() error {
	dsList, err := nux.ListDiskStats()
	if err != nil {
		return err
	}

	dsLock.Lock()
	defer dsLock.Unlock()
	for i := 0; i < len(dsList); i++ {
		device := dsList[i].Device
		diskStatsMap[device] = [2]*nux.DiskStats{dsList[i], diskStatsMap[device][0]}
	}
	return nil
}

func IOReadRequests(arr [2]*nux.DiskStats) uint64 {
	return arr[0].ReadRequests - arr[1].ReadRequests
}

func IOReadMerged(arr [2]*nux.DiskStats) uint64 {
	return arr[0].ReadMerged - arr[1].ReadMerged
}

func IOReadSectors(arr [2]*nux.DiskStats) uint64 {
	return arr[0].ReadSectors - arr[1].ReadSectors
}

func IOMsecRead(arr [2]*nux.DiskStats) uint64 {
	return arr[0].MsecRead - arr[1].MsecRead
}

func IOWriteRequests(arr [2]*nux.DiskStats) uint64 {
	return arr[0].WriteRequests - arr[1].WriteRequests
}

func IOWriteMerged(arr [2]*nux.DiskStats) uint64 {
	return arr[0].WriteMerged - arr[1].WriteMerged
}

func IOWriteSectors(arr [2]*nux.DiskStats) uint64 {
	return arr[0].WriteSectors - arr[1].WriteSectors
}

func IOMsecWrite(arr [2]*nux.DiskStats) uint64 {
	return arr[0].MsecWrite - arr[1].MsecWrite
}

func IOMsecTotal(arr [2]*nux.DiskStats) uint64 {
	return arr[0].MsecTotal - arr[1].MsecTotal
}

func IOMsecWeightedTotal(arr [2]*nux.DiskStats) uint64 {
	return arr[0].MsecWeightedTotal - arr[1].MsecWeightedTotal
}

func TS(arr [2]*nux.DiskStats) uint64 {
	return uint64(arr[0].TS.Sub(arr[1].TS).Nanoseconds() / 1000000)
}

func IODelta(device string, f func([2]*nux.DiskStats) uint64) uint64 {
	val, ok := diskStatsMap[device]
	if !ok {
		return 0
	}

	if val[1] == nil {
		return 0
	}
	return f(val)
}

func DiskIOMetrics() (L []*model.MetricValue) {

	dsList, err := nux.ListDiskStats()
	if err != nil {
		log.Println(err)
		return
	}

	for _, ds := range dsList {
		if !ShouldHandleDevice(ds.Device) {
			continue
		}

		device := map[string]string{
			"device": ds.Device,
		}

		L = append(L, VT("disk.io.read_requests", float64(ds.ReadRequests), device))
		L = append(L, VT("disk.io.read_merged", float64(ds.ReadMerged), device))
		L = append(L, VT("disk.io.read_sectors", float64(ds.ReadSectors), device))
		L = append(L, VT("disk.io.msec_read", float64(ds.MsecRead), device))
		L = append(L, VT("disk.io.write_requests", float64(ds.WriteRequests), device))
		L = append(L, VT("disk.io.write_merged", float64(ds.WriteMerged), device))
		L = append(L, VT("disk.io.write_sectors", float64(ds.WriteSectors), device))
		L = append(L, VT("disk.io.msec_write", float64(ds.MsecWrite), device))
		L = append(L, VT("disk.io.ios_in_progress", float64(ds.IosInProgress), device))
		L = append(L, VT("disk.io.msec_total", float64(ds.MsecTotal), device))
		L = append(L, VT("disk.io.msec_weighted_total", float64(ds.MsecWeightedTotal), device))
	}
	return
}

func IOStatsMetrics() (L []*model.MetricValue) {
	dsLock.RLock()
	defer dsLock.RUnlock()

	for device := range diskStatsMap {
		if !ShouldHandleDevice(device) {
			continue
		}

		rio := IODelta(device, IOReadRequests)
		wio := IODelta(device, IOWriteRequests)
		delta_rsec := IODelta(device, IOReadSectors)
		delta_wsec := IODelta(device, IOWriteSectors)
		ruse := IODelta(device, IOMsecRead)
		wuse := IODelta(device, IOMsecWrite)
		use := IODelta(device, IOMsecTotal)
		n_io := rio + wio
		avgrq_sz := 0.0
		await := 0.0
		svctm := 0.0
		if n_io != 0 {
			avgrq_sz = float64(delta_rsec+delta_wsec) / float64(n_io)
			await = float64(ruse+wuse) / float64(n_io)
			svctm = float64(use) / float64(n_io)
		}

		duration := IODelta(device, TS)

		tags := map[string]string{
			"device": device,
		}

		L = append(L, VT("disk.io.read_bytes", float64(delta_rsec)*512.0, tags))
		L = append(L, VT("disk.io.write_bytes", float64(delta_wsec)*512.0, tags))
		L = append(L, VT("disk.io.avgrq_sz", avgrq_sz, tags))
		L = append(L, VT("disk.io.avgqu_sz", float64(IODelta(device, IOMsecWeightedTotal))/1000.0, tags))
		L = append(L, VT("disk.io.await", await, tags))
		L = append(L, VT("disk.io.svctm", svctm, tags))
		tmp := float64(use) * 100.0 / float64(duration)
		if tmp > 100.0 {
			tmp = 100.0
		}
		L = append(L, VT("disk.io.util", tmp, tags))
	}

	return
}

func ShouldHandleDevice(device string) bool {
	normal := len(device) == 3 && (strings.HasPrefix(device, "sd") || strings.HasPrefix(device, "vd"))
	aws := len(device) >= 4 && strings.HasPrefix(device, "xvd")
	return normal || aws
}
<|endoftext|>


<commit_before>package simpleclient

import (
    "io/ioutil"
    "http"
    "json"
    "strings"
    "fmt"
)

type BuildClient struct {
    hostname string
    client http.Client
}

type Message struct {
    StartOn string
    Supplies string
    Requires []string
    Script string
}

func (t *Message) String() string {
    requires := "Requires:"
    for i := range t.Requires {
        requires = strings.Join([]string{requires, t.Requires[i]}, "\n\t")
    }
    return fmt.Sprintf("%s\n%s\n%s\n%s", t.StartOn, t.Supplies, requires, t.Script)
}

// TODO: read build_server URL from some type of conf file
const (
    BUILDSERVER = "http://localhost:8080"
)

/*
 BuildClient class methods
 */
// class init
func NewBuildClient(hostname string) *BuildClient {
    var client http.Client
    return &BuildClient{hostname, client}
}

// get hostname
func (t *BuildClient) GetHostname() (string) {
    return t.hostname
}

// HTTP get
func (t *BuildClient) Get(sub_url string) (string) {
    c := t.client
    r, _, err := c.Get(BUILDSERVER + "/" + sub_url)
    if err != nil {
        return "ERROR"
    }
    body, _ := ioutil.ReadAll(r.Body)
    r.Body.Close()
    var n Message
    err2 := json.Unmarshal(body, &n)
    if err2 != nil {
        return "Unmarshal Error!"
    }
    return n.String()
}

<commit_msg>removed simpleclient<commit_after><|endoftext|>


<commit_before>package staert

import (
	"encoding/base64"
	"errors"
	"fmt"
	"github.com/containous/flaeg"
	"github.com/docker/libkv"
	"github.com/docker/libkv/store"
	"github.com/mitchellh/mapstructure"
	"reflect"
	"sort"
	"strconv"
	"strings"
)

// KvSource implements Source
// It handles all mapstructure features(Squashed Embeded Sub-Structures, Maps, Pointers)
// It supports Slices (and maybe Arraies). They must be sorted in the KvStore like this :
// Key : ".../[sliceIndex]" -> Value
type KvSource struct {
	store.Store
	Prefix string // like this "prefix" (without the /)
}

// NewKvSource creates a new KvSource
func NewKvSource(backend store.Backend, addrs []string, options *store.Config, prefix string) (*KvSource, error) {
	store, err := libkv.NewStore(backend, addrs, options)
	return &KvSource{Store: store, Prefix: prefix}, err
}

// Parse uses libkv and mapstructure to fill the structure
func (kv *KvSource) Parse(cmd *flaeg.Command) (*flaeg.Command, error) {
	err := kv.LoadConfig(cmd.Config)
	if err != nil {
		return nil, err
	}
	return cmd, nil
}

// LoadConfig loads data from the KV Store into the config structure (given by reference)
func (kv *KvSource) LoadConfig(config interface{}) error {
	pairs := map[string][]byte{}
	if err := kv.ListRecursive(kv.Prefix, pairs); err != nil {
		return err
	}
	// fmt.Printf("pairs : %#v\n", pairs)
	mapstruct, err := generateMapstructure(convertPairs(pairs), kv.Prefix)
	if err != nil {
		return err
	}
	// fmt.Printf("mapstruct : %#v\n", mapstruct)
	configDecoder := &mapstructure.DecoderConfig{
		Metadata:         nil,
		Result:           config,
		WeaklyTypedInput: true,
		DecodeHook:       decodeHook,
	}
	decoder, err := mapstructure.NewDecoder(configDecoder)
	if err != nil {
		return err
	}
	if err := decoder.Decode(mapstruct); err != nil {
		return err
	}
	return nil
}

func generateMapstructure(pairs []*store.KVPair, prefix string) (map[string]interface{}, error) {
	raw := make(map[string]interface{})
	for _, p := range pairs {
		// Trim the prefix off our key first
		key := strings.TrimPrefix(strings.Trim(p.Key, "/"), strings.Trim(prefix, "/")+"/")
		raw, err := processKV(key, p.Value, raw)
		if err != nil {
			return raw, err
		}

	}
	return raw, nil
}

func processKV(key string, v []byte, raw map[string]interface{}) (map[string]interface{}, error) {
	// Determine which map we're writing the value to. We split by '/'
	// to determine any sub-maps that need to be created.
	m := raw
	children := strings.Split(key, "/")
	if len(children) > 0 {
		key = children[len(children)-1]
		children = children[:len(children)-1]
		for _, child := range children {
			if m[child] == nil {
				m[child] = make(map[string]interface{})
			}
			subm, ok := m[child].(map[string]interface{})
			if !ok {
				return nil, fmt.Errorf("child is both a data item and dir: %s", child)
			}
			m = subm
		}
	}
	m[key] = string(v)
	return raw, nil
}

func decodeHook(fromType reflect.Type, toType reflect.Type, data interface{}) (interface{}, error) {
	// TODO : Array support
	switch toType.Kind() {
	case reflect.Ptr:
		if fromType.Kind() == reflect.String {
			if data == "" {
				// default value Pointer
				return make(map[string]interface{}), nil
			}
		}
	case reflect.Slice:
		if fromType.Kind() == reflect.Map {
			// Type assertion
			dataMap, ok := data.(map[string]interface{})
			if !ok {
				return data, fmt.Errorf("input data is not a map : %#v", data)
			}
			// Sorting map
			indexes := make([]int, len(dataMap))
			i := 0
			for k := range dataMap {
				ind, err := strconv.Atoi(k)
				if err != nil {
					return dataMap, err
				}
				indexes[i] = ind
				i++
			}
			sort.Ints(indexes)
			// Building slice
			dataOutput := make([]interface{}, i)
			i = 0
			for _, k := range indexes {
				dataOutput[i] = dataMap[strconv.Itoa(k)]
				i++
			}

			return dataOutput, nil
		} else if fromType.Kind() == reflect.String {
			b, err := base64.StdEncoding.DecodeString(data.(string))
			if err != nil {
				return nil, err
			}
			return b, nil
		}
	}
	return data, nil
}

// StoreConfig stores the config into the KV Store
func (kv *KvSource) StoreConfig(config interface{}) error {
	kvMap := map[string]string{}
	if err := collateKvRecursive(reflect.ValueOf(config), kvMap, kv.Prefix); err != nil {
		return err
	}
	for k, v := range kvMap {
		var writeOptions *store.WriteOptions
		// is it a directory ?
		if strings.HasSuffix(k, "/") {
			writeOptions = &store.WriteOptions{
				IsDir: true,
			}
		}
		if err := kv.Put(k, []byte(v), writeOptions); err != nil {
			return err
		}
	}
	return nil
}

func collateKvRecursive(objValue reflect.Value, kv map[string]string, key string) error {
	name := key
	kind := objValue.Kind()
	switch kind {
	case reflect.Struct:
		for i := 0; i < objValue.NumField(); i++ {
			objType := objValue.Type()
			if objType.Field(i).Name[:1] != strings.ToUpper(objType.Field(i).Name[:1]) {
				//if unexported field
				continue
			}
			squashed := false
			if objType.Field(i).Anonymous {
				if objValue.Field(i).Kind() == reflect.Struct {
					tags := objType.Field(i).Tag
					if strings.Contains(string(tags), "squash") {
						squashed = true
					}
				}
			}
			if squashed {
				if err := collateKvRecursive(objValue.Field(i), kv, name); err != nil {
					return err
				}
			} else {
				fieldName := objType.Field(i).Name
				//useless if not empty Prefix is required ?
				if len(key) == 0 {
					name = strings.ToLower(fieldName)
				} else {
					name = key + "/" + strings.ToLower(fieldName)
				}

				if err := collateKvRecursive(objValue.Field(i), kv, name); err != nil {
					return err
				}
			}
		}

	case reflect.Ptr:
		if !objValue.IsNil() {
			// hack to avoid calling this at the beginning
			if len(kv) > 0 {
				kv[name+"/"] = ""
			}
			if err := collateKvRecursive(objValue.Elem(), kv, name); err != nil {
				return err
			}
		}
	case reflect.Map:
		for _, k := range objValue.MapKeys() {
			if k.Kind() == reflect.Struct {
				return errors.New("Struct as key not supported")
			}
			name = key + "/" + fmt.Sprint(k)
			if err := collateKvRecursive(objValue.MapIndex(k), kv, name); err != nil {
				return err
			}
		}
	case reflect.Array, reflect.Slice:
		// Byte slices get special treatment
		if objValue.Type().Elem().Kind() == reflect.Uint8 {
			kv[name] = base64.StdEncoding.EncodeToString(objValue.Bytes())
		} else {
			for i := 0; i < objValue.Len(); i++ {
				name = key + "/" + strconv.Itoa(i)
				if err := collateKvRecursive(objValue.Index(i), kv, name); err != nil {
					return err
				}
			}
		}
	case reflect.Interface, reflect.String, reflect.Bool, reflect.Int, reflect.Int8, reflect.Int16,
		reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16,
		reflect.Uint32, reflect.Uint64, reflect.Uintptr, reflect.Float32, reflect.Float64:
		if _, ok := kv[name]; ok {
			return errors.New("key already exists: " + name)
		}
		kv[name] = fmt.Sprint(objValue)

	default:
		return fmt.Errorf("Kind %s not supported", kind.String())
	}
	return nil
}

// ListRecursive lists all key value childrens under key
func (kv *KvSource) ListRecursive(key string, pairs map[string][]byte) error {
	pairsN1, err := kv.List(key)
	if err == store.ErrKeyNotFound {
		return nil
	}
	if err != nil {
		return err
	}
	if len(pairsN1) == 0 {
		pairLeaf, err := kv.Get(key)
		if err != nil {
			return err
		}
		if pairLeaf == nil {
			return nil
		}
		pairs[pairLeaf.Key] = pairLeaf.Value
		return nil
	}
	for _, p := range pairsN1 {
		err := kv.ListRecursive(p.Key, pairs)
		if err != nil {
			return err
		}
	}
	return nil
}

func convertPairs(pairs map[string][]byte) []*store.KVPair {
	slicePairs := make([]*store.KVPair, len(pairs))
	i := 0
	for k, v := range pairs {
		slicePairs[i] = &store.KVPair{
			Key:   k,
			Value: v,
		}
		i++
	}
	return slicePairs
}
<commit_msg>Add sort in kv.StoreConfig<commit_after>package staert

import (
	"encoding/base64"
	"errors"
	"fmt"
	"github.com/containous/flaeg"
	"github.com/docker/libkv"
	"github.com/docker/libkv/store"
	"github.com/mitchellh/mapstructure"
	"reflect"
	"sort"
	"strconv"
	"strings"
)

// KvSource implements Source
// It handles all mapstructure features(Squashed Embeded Sub-Structures, Maps, Pointers)
// It supports Slices (and maybe Arraies). They must be sorted in the KvStore like this :
// Key : ".../[sliceIndex]" -> Value
type KvSource struct {
	store.Store
	Prefix string // like this "prefix" (without the /)
}

// NewKvSource creates a new KvSource
func NewKvSource(backend store.Backend, addrs []string, options *store.Config, prefix string) (*KvSource, error) {
	store, err := libkv.NewStore(backend, addrs, options)
	return &KvSource{Store: store, Prefix: prefix}, err
}

// Parse uses libkv and mapstructure to fill the structure
func (kv *KvSource) Parse(cmd *flaeg.Command) (*flaeg.Command, error) {
	err := kv.LoadConfig(cmd.Config)
	if err != nil {
		return nil, err
	}
	return cmd, nil
}

// LoadConfig loads data from the KV Store into the config structure (given by reference)
func (kv *KvSource) LoadConfig(config interface{}) error {
	pairs := map[string][]byte{}
	if err := kv.ListRecursive(kv.Prefix, pairs); err != nil {
		return err
	}
	// fmt.Printf("pairs : %#v\n", pairs)
	mapstruct, err := generateMapstructure(convertPairs(pairs), kv.Prefix)
	if err != nil {
		return err
	}
	// fmt.Printf("mapstruct : %#v\n", mapstruct)
	configDecoder := &mapstructure.DecoderConfig{
		Metadata:         nil,
		Result:           config,
		WeaklyTypedInput: true,
		DecodeHook:       decodeHook,
	}
	decoder, err := mapstructure.NewDecoder(configDecoder)
	if err != nil {
		return err
	}
	if err := decoder.Decode(mapstruct); err != nil {
		return err
	}
	return nil
}

func generateMapstructure(pairs []*store.KVPair, prefix string) (map[string]interface{}, error) {
	raw := make(map[string]interface{})
	for _, p := range pairs {
		// Trim the prefix off our key first
		key := strings.TrimPrefix(strings.Trim(p.Key, "/"), strings.Trim(prefix, "/")+"/")
		raw, err := processKV(key, p.Value, raw)
		if err != nil {
			return raw, err
		}

	}
	return raw, nil
}

func processKV(key string, v []byte, raw map[string]interface{}) (map[string]interface{}, error) {
	// Determine which map we're writing the value to. We split by '/'
	// to determine any sub-maps that need to be created.
	m := raw
	children := strings.Split(key, "/")
	if len(children) > 0 {
		key = children[len(children)-1]
		children = children[:len(children)-1]
		for _, child := range children {
			if m[child] == nil {
				m[child] = make(map[string]interface{})
			}
			subm, ok := m[child].(map[string]interface{})
			if !ok {
				return nil, fmt.Errorf("child is both a data item and dir: %s", child)
			}
			m = subm
		}
	}
	m[key] = string(v)
	return raw, nil
}

func decodeHook(fromType reflect.Type, toType reflect.Type, data interface{}) (interface{}, error) {
	// TODO : Array support
	switch toType.Kind() {
	case reflect.Ptr:
		if fromType.Kind() == reflect.String {
			if data == "" {
				// default value Pointer
				return make(map[string]interface{}), nil
			}
		}
	case reflect.Slice:
		if fromType.Kind() == reflect.Map {
			// Type assertion
			dataMap, ok := data.(map[string]interface{})
			if !ok {
				return data, fmt.Errorf("input data is not a map : %#v", data)
			}
			// Sorting map
			indexes := make([]int, len(dataMap))
			i := 0
			for k := range dataMap {
				ind, err := strconv.Atoi(k)
				if err != nil {
					return dataMap, err
				}
				indexes[i] = ind
				i++
			}
			sort.Ints(indexes)
			// Building slice
			dataOutput := make([]interface{}, i)
			i = 0
			for _, k := range indexes {
				dataOutput[i] = dataMap[strconv.Itoa(k)]
				i++
			}

			return dataOutput, nil
		} else if fromType.Kind() == reflect.String {
			b, err := base64.StdEncoding.DecodeString(data.(string))
			if err != nil {
				return nil, err
			}
			return b, nil
		}
	}
	return data, nil
}

// StoreConfig stores the config into the KV Store
func (kv *KvSource) StoreConfig(config interface{}) error {
	kvMap := map[string]string{}
	if err := collateKvRecursive(reflect.ValueOf(config), kvMap, kv.Prefix); err != nil {
		return err
	}
	keys := []string{}
	for key := range kvMap {
		keys = append(keys, key)
	}
	sort.Strings(keys)
	for _, k := range keys {
		var writeOptions *store.WriteOptions
		// is it a directory ?
		if strings.HasSuffix(k, "/") {
			writeOptions = &store.WriteOptions{
				IsDir: true,
			}
		}
		if err := kv.Put(k, []byte(kvMap[k]), writeOptions); err != nil {
			return err
		}
	}
	return nil
}

func collateKvRecursive(objValue reflect.Value, kv map[string]string, key string) error {
	name := key
	kind := objValue.Kind()
	switch kind {
	case reflect.Struct:
		for i := 0; i < objValue.NumField(); i++ {
			objType := objValue.Type()
			if objType.Field(i).Name[:1] != strings.ToUpper(objType.Field(i).Name[:1]) {
				//if unexported field
				continue
			}
			squashed := false
			if objType.Field(i).Anonymous {
				if objValue.Field(i).Kind() == reflect.Struct {
					tags := objType.Field(i).Tag
					if strings.Contains(string(tags), "squash") {
						squashed = true
					}
				}
			}
			if squashed {
				if err := collateKvRecursive(objValue.Field(i), kv, name); err != nil {
					return err
				}
			} else {
				fieldName := objType.Field(i).Name
				//useless if not empty Prefix is required ?
				if len(key) == 0 {
					name = strings.ToLower(fieldName)
				} else {
					name = key + "/" + strings.ToLower(fieldName)
				}

				if err := collateKvRecursive(objValue.Field(i), kv, name); err != nil {
					return err
				}
			}
		}

	case reflect.Ptr:
		if !objValue.IsNil() {
			// hack to avoid calling this at the beginning
			if len(kv) > 0 {
				kv[name+"/"] = ""
			}
			if err := collateKvRecursive(objValue.Elem(), kv, name); err != nil {
				return err
			}
		}
	case reflect.Map:
		for _, k := range objValue.MapKeys() {
			if k.Kind() == reflect.Struct {
				return errors.New("Struct as key not supported")
			}
			name = key + "/" + fmt.Sprint(k)
			if err := collateKvRecursive(objValue.MapIndex(k), kv, name); err != nil {
				return err
			}
		}
	case reflect.Array, reflect.Slice:
		// Byte slices get special treatment
		if objValue.Type().Elem().Kind() == reflect.Uint8 {
			kv[name] = base64.StdEncoding.EncodeToString(objValue.Bytes())
		} else {
			for i := 0; i < objValue.Len(); i++ {
				name = key + "/" + strconv.Itoa(i)
				if err := collateKvRecursive(objValue.Index(i), kv, name); err != nil {
					return err
				}
			}
		}
	case reflect.Interface, reflect.String, reflect.Bool, reflect.Int, reflect.Int8, reflect.Int16,
		reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16,
		reflect.Uint32, reflect.Uint64, reflect.Uintptr, reflect.Float32, reflect.Float64:
		if _, ok := kv[name]; ok {
			return errors.New("key already exists: " + name)
		}
		kv[name] = fmt.Sprint(objValue)

	default:
		return fmt.Errorf("Kind %s not supported", kind.String())
	}
	return nil
}

// ListRecursive lists all key value childrens under key
func (kv *KvSource) ListRecursive(key string, pairs map[string][]byte) error {
	pairsN1, err := kv.List(key)
	if err == store.ErrKeyNotFound {
		return nil
	}
	if err != nil {
		return err
	}
	if len(pairsN1) == 0 {
		pairLeaf, err := kv.Get(key)
		if err != nil {
			return err
		}
		if pairLeaf == nil {
			return nil
		}
		pairs[pairLeaf.Key] = pairLeaf.Value
		return nil
	}
	for _, p := range pairsN1 {
		err := kv.ListRecursive(p.Key, pairs)
		if err != nil {
			return err
		}
	}
	return nil
}

func convertPairs(pairs map[string][]byte) []*store.KVPair {
	slicePairs := make([]*store.KVPair, len(pairs))
	i := 0
	for k, v := range pairs {
		slicePairs[i] = &store.KVPair{
			Key:   k,
			Value: v,
		}
		i++
	}
	return slicePairs
}
<|endoftext|>


<commit_before><commit_msg>Redisence: implement a function for checking a single id's status from database<commit_after><|endoftext|>


<commit_before>package gluahttp

import "github.com/yuin/gopher-lua"
import "net/http"
import "net/http/cookiejar"
import "fmt"
import "errors"
import "io/ioutil"
import "strings"

type httpModule struct {
	client *http.Client
}

type empty struct{}

func NewHttpModule() *httpModule {
	cookieJar, _ := cookiejar.New(nil)

	return &httpModule{
		client: &http.Client{
			Jar: cookieJar,
		},
	}
}

func (h *httpModule) Loader(L *lua.LState) int {
	mod := L.SetFuncs(L.NewTable(), map[string]lua.LGFunction{
		"get":           h.get,
		"delete":        h.delete,
		"head":          h.head,
		"patch":         h.patch,
		"post":          h.post,
		"put":           h.put,
		"request":       h.request,
		"request_batch": h.request_batch,
	})
	L.Push(mod)
	return 1
}

func (h *httpModule) get(L *lua.LState) int {
	return h.doRequestAndPush(L, "get", L.ToString(1), L.ToTable(2))
}

func (h *httpModule) delete(L *lua.LState) int {
	return h.doRequestAndPush(L, "delete", L.ToString(1), L.ToTable(2))
}

func (h *httpModule) head(L *lua.LState) int {
	return h.doRequestAndPush(L, "head", L.ToString(1), L.ToTable(2))
}

func (h *httpModule) patch(L *lua.LState) int {
	return h.doRequestAndPush(L, "patch", L.ToString(1), L.ToTable(2))
}

func (h *httpModule) post(L *lua.LState) int {
	return h.doRequestAndPush(L, "post", L.ToString(1), L.ToTable(2))
}

func (h *httpModule) put(L *lua.LState) int {
	return h.doRequestAndPush(L, "put", L.ToString(1), L.ToTable(2))
}

func (h *httpModule) request(L *lua.LState) int {
	return h.doRequestAndPush(L, L.ToString(1), L.ToString(2), L.ToTable(3))
}

func (h *httpModule) request_batch(L *lua.LState) int {
	requests := L.ToTable(1)
	amountRequests := requests.Len()

	errs := make([]error, amountRequests)
	responses := make([]*lua.LTable, amountRequests)
	sem := make(chan empty, amountRequests)

	i := 0

	requests.ForEach(func(_ lua.LValue, value lua.LValue) {
		requestTable := toTable(value)

		if requestTable != nil {
			method := requestTable.RawGet(lua.LNumber(1)).String()
			url := requestTable.RawGet(lua.LNumber(2)).String()
			options := toTable(requestTable.RawGet(lua.LNumber(3)))

			go func(i int, L *lua.LState, method string, url string, options *lua.LTable) {
				response, err := h.doRequest(L, method, url, options)

				if err == nil {
					errs[i] = nil
					responses[i] = response
				} else {
					errs[i] = err
					responses[i] = nil
				}

				sem <- empty{}
			}(i, L, method, url, options)
		} else {
			errs[i] = errors.New("Request must be a table")
			responses[i] = nil
			sem <- empty{}
		}

		i = i + 1
	})

	for i = 0; i < amountRequests; i++ {
		<-sem
	}

	hasErrors := false
	errorsTable := L.NewTable()
	responsesTable := L.NewTable()
	for i = 0; i < amountRequests; i++ {
		if errs[i] == nil {
			responsesTable.Append(responses[i])
			errorsTable.Append(lua.LNil)
		} else {
			responsesTable.Append(lua.LNil)
			errorsTable.Append(lua.LString(fmt.Sprintf("%s", errs[i])))
			hasErrors = true
		}
	}

	if hasErrors {
		L.Push(responsesTable)
		L.Push(errorsTable)
		return 2
	} else {
		L.Push(responsesTable)
		return 1
	}
}

func (h *httpModule) doRequest(L *lua.LState, method string, url string, options *lua.LTable) (*lua.LTable, error) {
	req, err := http.NewRequest(strings.ToUpper(method), url, nil)
	if err != nil {
		return nil, err
	}

	if options != nil {
		if reqHeaders, ok := options.RawGet(lua.LString("headers")).(*lua.LTable); ok {
			reqHeaders.ForEach(func(key lua.LValue, value lua.LValue) {
				req.Header.Set(key.String(), value.String())
			})
		}

		if reqCookies, ok := options.RawGet(lua.LString("cookies")).(*lua.LTable); ok {
			reqCookies.ForEach(func(key lua.LValue, value lua.LValue) {
				req.AddCookie(&http.Cookie{Name: key.String(), Value: value.String()})
			})
		}

		switch reqQuery := options.RawGet(lua.LString("query")).(type) {
		case *lua.LNilType:
			break

		case lua.LString:
			req.URL.RawQuery = reqQuery.String()
			break
		}

		switch reqForm := options.RawGet(lua.LString("form")).(type) {
		case *lua.LNilType:
			break

		case lua.LString:
			req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
			req.Body = ioutil.NopCloser(strings.NewReader(reqForm.String()))
			break
		}
	}

	res, err := h.client.Do(req)
	if err != nil {
		return nil, err
	}

	defer res.Body.Close()
	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, err
	}

	headers := L.NewTable()
	for key, _ := range res.Header {
		headers.RawSetString(key, lua.LString(res.Header.Get(key)))
	}

	cookies := L.NewTable()
	for _, cookie := range res.Cookies() {
		cookies.RawSetString(cookie.Name, lua.LString(cookie.Value))
	}

	response := L.NewTable()
	response.RawSetString("body", lua.LString(body))
	response.RawSetString("headers", headers)
	response.RawSetString("cookies", cookies)
	response.RawSetString("status_code", lua.LNumber(res.StatusCode))

	return response, nil
}

func (h *httpModule) doRequestAndPush(L *lua.LState, method string, url string, options *lua.LTable) int {
	response, err := h.doRequest(L, method, url, options)

	if err != nil {
		L.Push(lua.LNil)
		L.Push(lua.LString(fmt.Sprintf("%s", err)))
		return 2
	}

	L.Push(response)
	return 1
}

func toTable(v lua.LValue) *lua.LTable {
	if lv, ok := v.(*lua.LTable); ok {
		return lv
	}
	return nil
}
<commit_msg>Style guide<commit_after>package gluahttp

import "github.com/yuin/gopher-lua"
import "net/http"
import "net/http/cookiejar"
import "fmt"
import "errors"
import "io/ioutil"
import "strings"

type httpModule struct {
	client *http.Client
}

type empty struct{}

func NewHttpModule() *httpModule {
	cookieJar, _ := cookiejar.New(nil)

	return &httpModule{
		client: &http.Client{
			Jar: cookieJar,
		},
	}
}

func (h *httpModule) Loader(L *lua.LState) int {
	mod := L.SetFuncs(L.NewTable(), map[string]lua.LGFunction{
		"get":           h.get,
		"delete":        h.delete,
		"head":          h.head,
		"patch":         h.patch,
		"post":          h.post,
		"put":           h.put,
		"request":       h.request,
		"request_batch": h.requestBatch,
	})
	L.Push(mod)
	return 1
}

func (h *httpModule) get(L *lua.LState) int {
	return h.doRequestAndPush(L, "get", L.ToString(1), L.ToTable(2))
}

func (h *httpModule) delete(L *lua.LState) int {
	return h.doRequestAndPush(L, "delete", L.ToString(1), L.ToTable(2))
}

func (h *httpModule) head(L *lua.LState) int {
	return h.doRequestAndPush(L, "head", L.ToString(1), L.ToTable(2))
}

func (h *httpModule) patch(L *lua.LState) int {
	return h.doRequestAndPush(L, "patch", L.ToString(1), L.ToTable(2))
}

func (h *httpModule) post(L *lua.LState) int {
	return h.doRequestAndPush(L, "post", L.ToString(1), L.ToTable(2))
}

func (h *httpModule) put(L *lua.LState) int {
	return h.doRequestAndPush(L, "put", L.ToString(1), L.ToTable(2))
}

func (h *httpModule) request(L *lua.LState) int {
	return h.doRequestAndPush(L, L.ToString(1), L.ToString(2), L.ToTable(3))
}

func (h *httpModule) requestBatch(L *lua.LState) int {
	requests := L.ToTable(1)
	amountRequests := requests.Len()

	errs := make([]error, amountRequests)
	responses := make([]*lua.LTable, amountRequests)
	sem := make(chan empty, amountRequests)

	i := 0

	requests.ForEach(func(_ lua.LValue, value lua.LValue) {
		requestTable := toTable(value)

		if requestTable != nil {
			method := requestTable.RawGet(lua.LNumber(1)).String()
			url := requestTable.RawGet(lua.LNumber(2)).String()
			options := toTable(requestTable.RawGet(lua.LNumber(3)))

			go func(i int, L *lua.LState, method string, url string, options *lua.LTable) {
				response, err := h.doRequest(L, method, url, options)

				if err == nil {
					errs[i] = nil
					responses[i] = response
				} else {
					errs[i] = err
					responses[i] = nil
				}

				sem <- empty{}
			}(i, L, method, url, options)
		} else {
			errs[i] = errors.New("Request must be a table")
			responses[i] = nil
			sem <- empty{}
		}

		i = i + 1
	})

	for i = 0; i < amountRequests; i++ {
		<-sem
	}

	hasErrors := false
	errorsTable := L.NewTable()
	responsesTable := L.NewTable()
	for i = 0; i < amountRequests; i++ {
		if errs[i] == nil {
			responsesTable.Append(responses[i])
			errorsTable.Append(lua.LNil)
		} else {
			responsesTable.Append(lua.LNil)
			errorsTable.Append(lua.LString(fmt.Sprintf("%s", errs[i])))
			hasErrors = true
		}
	}

	if hasErrors {
		L.Push(responsesTable)
		L.Push(errorsTable)
		return 2
	} else {
		L.Push(responsesTable)
		return 1
	}
}

func (h *httpModule) doRequest(L *lua.LState, method string, url string, options *lua.LTable) (*lua.LTable, error) {
	req, err := http.NewRequest(strings.ToUpper(method), url, nil)
	if err != nil {
		return nil, err
	}

	if options != nil {
		if reqHeaders, ok := options.RawGet(lua.LString("headers")).(*lua.LTable); ok {
			reqHeaders.ForEach(func(key lua.LValue, value lua.LValue) {
				req.Header.Set(key.String(), value.String())
			})
		}

		if reqCookies, ok := options.RawGet(lua.LString("cookies")).(*lua.LTable); ok {
			reqCookies.ForEach(func(key lua.LValue, value lua.LValue) {
				req.AddCookie(&http.Cookie{Name: key.String(), Value: value.String()})
			})
		}

		switch reqQuery := options.RawGet(lua.LString("query")).(type) {
		case *lua.LNilType:
			break

		case lua.LString:
			req.URL.RawQuery = reqQuery.String()
			break
		}

		switch reqForm := options.RawGet(lua.LString("form")).(type) {
		case *lua.LNilType:
			break

		case lua.LString:
			req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
			req.Body = ioutil.NopCloser(strings.NewReader(reqForm.String()))
			break
		}
	}

	res, err := h.client.Do(req)
	if err != nil {
		return nil, err
	}

	defer res.Body.Close()
	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, err
	}

	headers := L.NewTable()
	for key, _ := range res.Header {
		headers.RawSetString(key, lua.LString(res.Header.Get(key)))
	}

	cookies := L.NewTable()
	for _, cookie := range res.Cookies() {
		cookies.RawSetString(cookie.Name, lua.LString(cookie.Value))
	}

	response := L.NewTable()
	response.RawSetString("body", lua.LString(body))
	response.RawSetString("headers", headers)
	response.RawSetString("cookies", cookies)
	response.RawSetString("status_code", lua.LNumber(res.StatusCode))

	return response, nil
}

func (h *httpModule) doRequestAndPush(L *lua.LState, method string, url string, options *lua.LTable) int {
	response, err := h.doRequest(L, method, url, options)

	if err != nil {
		L.Push(lua.LNil)
		L.Push(lua.LString(fmt.Sprintf("%s", err)))
		return 2
	}

	L.Push(response)
	return 1
}

func toTable(v lua.LValue) *lua.LTable {
	if lv, ok := v.(*lua.LTable); ok {
		return lv
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/bearbin/go-paste/pastebin"
	"github.com/codegangsta/cli"
	"io/ioutil"
	"os"
)

func main() {
	app := cli.NewApp()
	app.Usage = "get and put pastes from pastebin and other paste sites."
	app.Commands = []cli.Command{
		{
			Name:      "put",
			ShortName: "p",
			Usage:     "put a paste",
			Flags: []cli.Flag{
				cli.StringFlag{Name: "title, t", Value: "", Usage: "the title for the paste"},
			},
			Action: func(c *cli.Context) {
				var err error
				var text []byte
				if c.Args().First() == "-" {
					text, err = ioutil.ReadAll(os.Stdin)
				} else {
					text, err = ioutil.ReadFile(c.Args().First())
				}
				if err != nil {
					println("ERROR: ", err.Error())
					os.Exit(1)
				}
				code, err := pastebin.Put(string(text), c.String("title"))
				if err != nil {
					println("ERROR: ", err.Error())
				}
				println(code)
			},
		},
		{
			Name:      "get",
			ShortName: "g",
			Usage:     "get a paste",
			Action: func(c *cli.Context) {
				text, err := pastebin.Get(c.Args().First())
				if err != nil {
					println("ERROR: ", err.Error())
					os.Exit(1)
				}
				println(text)
			},
		},
	}
	app.Run(os.Args)
}
<commit_msg>More logical way of reading from stdin.<commit_after>package main

import (
	"github.com/bearbin/go-paste/pastebin"
	"github.com/codegangsta/cli"
	"io/ioutil"
	"os"
)

func main() {
	app := cli.NewApp()
	app.Usage = "get and put pastes from pastebin and other paste sites."
	app.Commands = []cli.Command{
		{
			Name:      "put",
			ShortName: "p",
			Usage:     "put a paste",
			Flags: []cli.Flag{
				cli.StringFlag{Name: "title, t", Value: "", Usage: "the title for the paste"},
			},
			Action: func(c *cli.Context) {
				var err error
				var text []byte
				if c.Args().First() == "-" || c.Args().First() == "" {
					text, err = ioutil.ReadAll(os.Stdin)
				} else {
					text, err = ioutil.ReadFile(c.Args().First())
				}
				if err != nil {
					println("ERROR: ", err.Error())
					os.Exit(1)
				}
				code, err := pastebin.Put(string(text), c.String("title"))
				if err != nil {
					println("ERROR: ", err.Error())
				}
				println(code)
			},
		},
		{
			Name:      "get",
			ShortName: "g",
			Usage:     "get a paste",
			Action: func(c *cli.Context) {
				text, err := pastebin.Get(c.Args().First())
				if err != nil {
					println("ERROR: ", err.Error())
					os.Exit(1)
				}
				println(text)
			},
		},
	}
	app.Run(os.Args)
}
<|endoftext|>


<commit_before>/*
   Copyright 2017 Shlomi Noach, GitHub Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package kv

import (
	"fmt"
	"sync"
)

type KVPair struct {
	Key   string
	Value string
}

func NewKVPair(key string, value string) *KVPair {
	return &KVPair{Key: key, Value: value}
}

func (this *KVPair) String() string {
	return fmt.Sprintf("%s:%s", this.Key, this.Value)
}

type KVStore interface {
	PutKeyValue(key string, value string) (err error)
	GetKeyValue(key string) (value string, found bool, err error)
	DistributePairs(kvPairs [](*KVPair)) (err error)
}

type KVStoreMap map[string](KVStore)

var kvMutex sync.Mutex
var kvInitOnce sync.Once
var kvStores = []KVStore{}
var kvStoresMap = make(KVStoreMap)

// InitKVStores initializes the KV stores (duh), once in the lifetime of this app.
// Configuration reload does not affect a running instance.
func InitKVStores() {
	kvMutex.Lock()
	defer kvMutex.Unlock()

	kvInitOnce.Do(func() {
		kvStoresMap = KVStoreMap{
			"internal": NewInternalKVStore(),
			"consul":   NewConsulStore(),
			"zk":       NewZkStore(),
		}
		kvStores = []KVStore{
			kvStoresMap["internal"],
			kvStoresMap["consul"],
			kvStoresMap["zk"],
		}
	})
}

func getKVStores() (stores []KVStore) {
	kvMutex.Lock()
	defer kvMutex.Unlock()

	stores = kvStores
	return stores
}

func GetValue(key string) (value string, found bool, err error) {
	for _, store := range getKVStores() {
		// It's really only the first (internal) that matters here
		return store.GetKeyValue(key)
	}
	return value, found, err
}

func PutValue(key string, value string) (err error) {
	for _, store := range getKVStores() {
		if err := store.PutKeyValue(key, value); err != nil {
			return err
		}
	}
	return nil
}

func PutKVPair(kvPair *KVPair) (err error) {
	if kvPair == nil {
		return nil
	}
	return PutValue(kvPair.Key, kvPair.Value)
}

func DistributePairs(kvPairs [](*KVPair)) (err error) {
	for _, store := range getKVStores() {
		if err := store.DistributePairs(kvPairs); err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>cleanup<commit_after>/*
   Copyright 2017 Shlomi Noach, GitHub Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package kv

import (
	"fmt"
	"sync"
)

type KVPair struct {
	Key   string
	Value string
}

func NewKVPair(key string, value string) *KVPair {
	return &KVPair{Key: key, Value: value}
}

func (this *KVPair) String() string {
	return fmt.Sprintf("%s:%s", this.Key, this.Value)
}

type KVStore interface {
	PutKeyValue(key string, value string) (err error)
	GetKeyValue(key string) (value string, found bool, err error)
	DistributePairs(kvPairs [](*KVPair)) (err error)
}

var kvMutex sync.Mutex
var kvInitOnce sync.Once
var kvStores = []KVStore{}

// InitKVStores initializes the KV stores (duh), once in the lifetime of this app.
// Configuration reload does not affect a running instance.
func InitKVStores() {
	kvMutex.Lock()
	defer kvMutex.Unlock()

	kvInitOnce.Do(func() {
		kvStores = []KVStore{
			NewInternalKVStore(),
			NewConsulStore(),
			NewZkStore(),
		}
	})
}

func getKVStores() (stores []KVStore) {
	kvMutex.Lock()
	defer kvMutex.Unlock()

	stores = kvStores
	return stores
}

func GetValue(key string) (value string, found bool, err error) {
	for _, store := range getKVStores() {
		// It's really only the first (internal) that matters here
		return store.GetKeyValue(key)
	}
	return value, found, err
}

func PutValue(key string, value string) (err error) {
	for _, store := range getKVStores() {
		if err := store.PutKeyValue(key, value); err != nil {
			return err
		}
	}
	return nil
}

func PutKVPair(kvPair *KVPair) (err error) {
	if kvPair == nil {
		return nil
	}
	return PutValue(kvPair.Key, kvPair.Value)
}

func DistributePairs(kvPairs [](*KVPair)) (err error) {
	for _, store := range getKVStores() {
		if err := store.DistributePairs(kvPairs); err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>/*
A go port of the ruby dotenv library (https://github.com/bkeepers/dotenv)

Examples/readme can be found on the github page at https://github.com/joho/godotenv

The TL;DR is that you make a .env file that looks something like

		SOME_ENV_VAR=somevalue

and then in your go code you can call

		godotenv.Load()

and all the env vars declared in .env will be avaiable through os.Getenv("SOME_ENV_VAR")
*/
package godotenv

import (
	"bufio"
	"errors"
	"os"
	"os/exec"
	"strings"
)

/*
	Call this function as close as possible to the start of your program (ideally in main)

	If you call Load without any args it will default to loading .env in the current path

	You can otherwise tell it which files to load (there can be more than one) like

		godotenv.Load("fileone", "filetwo")

	It's important to note that it WILL NOT OVERRIDE an env variable that already exists - consider the .env file to set dev vars or sensible defaults
*/
func Load(filenames ...string) (err error) {
	filenames = filenamesOrDefault(filenames)

	for _, filename := range filenames {
		err = loadFile(filename)
		if err != nil {
			return // return early on a spazout
		}
	}
	return
}

/*
  Read all env (with same file loading semantics as Load) but return values as
  a map rather than automatically writing values into env
*/
func Read(filenames ...string) (envMap map[string]string, err error) {
	filenames = filenamesOrDefault(filenames)
	envMap = make(map[string]string)

	for _, filename := range filenames {
		individualEnvMap, individualErr := readFile(filename)

		if individualErr != nil {
			err = individualErr
			return // return early on a spazout
		}

		for key, value := range individualEnvMap {
			envMap[key] = value
		}
	}

	return
}

/*
  Loads env vars from the specified filenames (empty map falls back to default)
  then executes the cmd specified.

  Simply hooks up os.Stdin/err/out to the command and calls Run()

  If you want more fine grained control over your command it's recommended
  that you use `Load()` or `Read()` and the `os/exec` package yourself.
*/
func Exec(filenames []string, cmd string, cmdArgs []string) error {
	Load(filenames...)

	command := exec.Command(cmd, cmdArgs...)
	command.Stdin = os.Stdin
	command.Stdout = os.Stdout
	command.Stderr = os.Stderr
	return command.Run()
}

func filenamesOrDefault(filenames []string) []string {
	if len(filenames) == 0 {
		return []string{".env"}
	} else {
		return filenames
	}
}

func loadFile(filename string) (err error) {
	envMap, err := readFile(filename)
	if err != nil {
		return
	}

	for key, value := range envMap {
		os.Setenv(key, value)
	}

	return
}

func readFile(filename string) (envMap map[string]string, err error) {
	file, err := os.Open(filename)
	if err != nil {
		return
	}
	defer file.Close()

	envMap = make(map[string]string)

	var lines []string
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		lines = append(lines, scanner.Text())
	}

	for _, fullLine := range lines {
		if !isIgnoredLine(fullLine) {
			key, value, err := parseLine(fullLine)

			if err == nil && os.Getenv(key) == "" {
				envMap[key] = value
			}
		}
	}
	return
}

func parseLine(line string) (key string, value string, err error) {
	if len(line) == 0 {
		err = errors.New("zero length string")
		return
	}

	// ditch the comments (but keep quoted hashes)
	if strings.Contains(line, "#") {
		segmentsBetweenHashes := strings.Split(line, "#")
		quotesAreOpen := false
		segmentsToKeep := make([]string, 0)
		for _, segment := range segmentsBetweenHashes {
			if strings.Count(segment, "\"") == 1 || strings.Count(segment, "'") == 1 {
				if quotesAreOpen {
					quotesAreOpen = false
					segmentsToKeep = append(segmentsToKeep, segment)
				} else {
					quotesAreOpen = true
				}
			}

			if len(segmentsToKeep) == 0 || quotesAreOpen {
				segmentsToKeep = append(segmentsToKeep, segment)
			}
		}

		line = strings.Join(segmentsToKeep, "#")
	}

	// now split key from value
	splitString := strings.SplitN(line, "=", 2)

	if len(splitString) != 2 {
		// try yaml mode!
		splitString = strings.SplitN(line, ":", 2)
	}

	if len(splitString) != 2 {
		err = errors.New("Can't separate key from value")
		return
	}

	// Parse the key
	key = splitString[0]
	if strings.HasPrefix(key, "export") {
		key = strings.TrimPrefix(key, "export")
	}
	key = strings.Trim(key, " ")

	// Parse the value
	value = splitString[1]
	// trim
	value = strings.Trim(value, " ")

	// check if we've got quoted values
	if strings.Count(value, "\"") == 2 || strings.Count(value, "'") == 2 {
		// pull the quotes off the edges
		value = strings.Trim(value, "\"'")

		// expand quotes
		value = strings.Replace(value, "\\\"", "\"", -1)
		// expand newlines
		value = strings.Replace(value, "\\n", "\n", -1)
	}

	return
}

func isIgnoredLine(line string) bool {
	trimmedLine := strings.Trim(line, " \n\t")
	return len(trimmedLine) == 0 || strings.HasPrefix(trimmedLine, "#")
}
<commit_msg>Add values to the envMap when reading the file.<commit_after>/*
A go port of the ruby dotenv library (https://github.com/bkeepers/dotenv)

Examples/readme can be found on the github page at https://github.com/joho/godotenv

The TL;DR is that you make a .env file that looks something like

		SOME_ENV_VAR=somevalue

and then in your go code you can call

		godotenv.Load()

and all the env vars declared in .env will be avaiable through os.Getenv("SOME_ENV_VAR")
*/
package godotenv

import (
	"bufio"
	"errors"
	"os"
	"os/exec"
	"strings"
)

/*
	Call this function as close as possible to the start of your program (ideally in main)

	If you call Load without any args it will default to loading .env in the current path

	You can otherwise tell it which files to load (there can be more than one) like

		godotenv.Load("fileone", "filetwo")

	It's important to note that it WILL NOT OVERRIDE an env variable that already exists - consider the .env file to set dev vars or sensible defaults
*/
func Load(filenames ...string) (err error) {
	filenames = filenamesOrDefault(filenames)

	for _, filename := range filenames {
		err = loadFile(filename)
		if err != nil {
			return // return early on a spazout
		}
	}
	return
}

/*
  Read all env (with same file loading semantics as Load) but return values as
  a map rather than automatically writing values into env
*/
func Read(filenames ...string) (envMap map[string]string, err error) {
	filenames = filenamesOrDefault(filenames)
	envMap = make(map[string]string)

	for _, filename := range filenames {
		individualEnvMap, individualErr := readFile(filename)

		if individualErr != nil {
			err = individualErr
			return // return early on a spazout
		}

		for key, value := range individualEnvMap {
			envMap[key] = value
		}
	}

	return
}

/*
  Loads env vars from the specified filenames (empty map falls back to default)
  then executes the cmd specified.

  Simply hooks up os.Stdin/err/out to the command and calls Run()

  If you want more fine grained control over your command it's recommended
  that you use `Load()` or `Read()` and the `os/exec` package yourself.
*/
func Exec(filenames []string, cmd string, cmdArgs []string) error {
	Load(filenames...)

	command := exec.Command(cmd, cmdArgs...)
	command.Stdin = os.Stdin
	command.Stdout = os.Stdout
	command.Stderr = os.Stderr
	return command.Run()
}

func filenamesOrDefault(filenames []string) []string {
	if len(filenames) == 0 {
		return []string{".env"}
	} else {
		return filenames
	}
}

func loadFile(filename string) (err error) {
	envMap, err := readFile(filename)
	if err != nil {
		return
	}

	for key, value := range envMap {
		if os.Getenv(key) == "" {
			os.Setenv(key, value)
		}
	}

	return
}

func readFile(filename string) (envMap map[string]string, err error) {
	file, err := os.Open(filename)
	if err != nil {
		return
	}
	defer file.Close()

	envMap = make(map[string]string)

	var lines []string
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		lines = append(lines, scanner.Text())
	}

	for _, fullLine := range lines {
		if !isIgnoredLine(fullLine) {
			key, value, err := parseLine(fullLine)

			if err == nil {
				envMap[key] = value
			}
		}
	}
	return
}

func parseLine(line string) (key string, value string, err error) {
	if len(line) == 0 {
		err = errors.New("zero length string")
		return
	}

	// ditch the comments (but keep quoted hashes)
	if strings.Contains(line, "#") {
		segmentsBetweenHashes := strings.Split(line, "#")
		quotesAreOpen := false
		segmentsToKeep := make([]string, 0)
		for _, segment := range segmentsBetweenHashes {
			if strings.Count(segment, "\"") == 1 || strings.Count(segment, "'") == 1 {
				if quotesAreOpen {
					quotesAreOpen = false
					segmentsToKeep = append(segmentsToKeep, segment)
				} else {
					quotesAreOpen = true
				}
			}

			if len(segmentsToKeep) == 0 || quotesAreOpen {
				segmentsToKeep = append(segmentsToKeep, segment)
			}
		}

		line = strings.Join(segmentsToKeep, "#")
	}

	// now split key from value
	splitString := strings.SplitN(line, "=", 2)

	if len(splitString) != 2 {
		// try yaml mode!
		splitString = strings.SplitN(line, ":", 2)
	}

	if len(splitString) != 2 {
		err = errors.New("Can't separate key from value")
		return
	}

	// Parse the key
	key = splitString[0]
	if strings.HasPrefix(key, "export") {
		key = strings.TrimPrefix(key, "export")
	}
	key = strings.Trim(key, " ")

	// Parse the value
	value = splitString[1]
	// trim
	value = strings.Trim(value, " ")

	// check if we've got quoted values
	if strings.Count(value, "\"") == 2 || strings.Count(value, "'") == 2 {
		// pull the quotes off the edges
		value = strings.Trim(value, "\"'")

		// expand quotes
		value = strings.Replace(value, "\\\"", "\"", -1)
		// expand newlines
		value = strings.Replace(value, "\\n", "\n", -1)
	}

	return
}

func isIgnoredLine(line string) bool {
	trimmedLine := strings.Trim(line, " \n\t")
	return len(trimmedLine) == 0 || strings.HasPrefix(trimmedLine, "#")
}
<|endoftext|>


<commit_before>/*
MIT License

Copyright (c) 2016 Kyriacos Kyriacou

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/

package main

import (
	"flag"
	"fmt"
	"log"
	"net/url"
	"os"
	"strconv"
	"time"

	"github.com/ChimeraCoder/anaconda"
)

// toFollow is a slice that should be populated with the users found by the search functions and
// will subsequently be used to follow each user
var toFollow []anaconda.User

// searchTerm is the search term used by the search functions
var searchTerm = flag.String("s", "", "(required) search term to find users by (i.e. gopher)")

// maxFollow is the maximum number of users that the application should follow. It is
// an upper bound since there might be a case where not enough users are found to follow
var maxFollow = flag.Int("max", 50, "(optional) max number of users to follow (hard maximum of 100 to avoid limiting by Twitter)")

// alreadyFollowing is a slice that contains all users that are already being followed.
// This is used so that the application does not attempt to follow users that are already friends
var alreadyFollowing []anaconda.User

func main() {
	api, err := newTwitterAPI()
	if err != nil {
		log.Fatal(err)
	}

	flag.Parse()
	if len(*searchTerm) == 0 {
		flag.PrintDefaults()
		os.Exit(1)
	}
	const hardMaxFollow = 100
	if *maxFollow > hardMaxFollow {
		*maxFollow = hardMaxFollow
	}

	alreadyFollowing = getAllFriends(api)

	fmt.Println("Finding users...")
	done := make(chan struct{})
	go spinner(done) // feedback for user

	found := 0
	n, err := findUsers(api)
	if err != nil {
		log.Fatal(err)
	}
	found += n
	n, err = findUsersByTweet(api)
	if err != nil {
		log.Fatal(err)
	}
	done <- struct{}{}
	found += n
	fmt.Printf("\rFound %d unique users\n", found)
	if found == 0 {
		fmt.Println("Try a broader search term next time")
		os.Exit(0)
	}

	const userURLFormat = "https://twitter.com/%s"
	fmt.Println("Following...")
	for _, user := range toFollow {
		err := followUser(api, user.Id)
		if err != nil {
			log.Print(fmt.Errorf("could not follow %s: %v\n", user.ScreenName, err))
			break // could continue, but error is most likely due to limiting by twitter and will fall through
		}
		fmt.Printf("%-40s%-s\n", user.Name, fmt.Sprintf(userURLFormat, user.ScreenName))
	}
	fmt.Println("-------------------------------------------------------------------------------")
	fmt.Println("Done!")
}

// newTwitterAPI returns a new Twitter API using keys from the environment
func newTwitterAPI() (*anaconda.TwitterApi, error) {
	keys := []string{
		"TWITTER_CONSUMER_KEY",
		"TWITTER_CONSUMER_SECRET",
		"TWITTER_ACCESS_TOKEN",
		"TWITTER_ACCESS_SECRET",
	}
	pairs := make(map[string]string, 4)
	for _, k := range keys {
		v := os.Getenv(k)
		if v == "" {
			return nil, fmt.Errorf("environment variable %q required", k)
		}
		pairs[k] = v
	}

	anaconda.SetConsumerKey(pairs["TWITTER_CONSUMER_KEY"])
	anaconda.SetConsumerSecret(pairs["TWITTER_CONSUMER_SECRET"])
	api := anaconda.NewTwitterApi(pairs["TWITTER_ACCESS_TOKEN"], pairs["TWITTER_ACCESS_SECRET"])
	_, err := api.VerifyCredentials()
	if err != nil {
		return nil, err
	}
	return api, nil
}

// spinner displays a spinner on the std output
func spinner(stop <-chan struct{}) {
	for {
		select {
		case <-stop:
			return
		default:
			for _, r := range `-\|/` {
				fmt.Printf("\r%c", r)
				time.Sleep(80 * time.Millisecond)
			}
		}
	}
}

// findUsers uses "searchTerm" to search for tweets using the users/search API
// (https://dev.twitter.com/rest/reference/get/users/search).
// Returns the number of users found
func findUsers(api *anaconda.TwitterApi) (int, error) {
	maxCount := 20
	// don't ask for more than required
	usersRequired := *maxFollow - len(toFollow)
	if maxCount > usersRequired {
		maxCount = usersRequired
	}
	if maxCount == 0 {
		return 0, nil
	}
	page := 0
	values := make(url.Values)
	values.Add("count", strconv.Itoa(maxCount))
	values.Add("include_entities", "false")

	added := 0
	for {
		values.Set("page", strconv.Itoa(page))
		resp, err := api.GetUserSearch(*searchTerm, values)
		if err != nil {
			return 0, fmt.Errorf("findUsers: %v", err)
		}
		for _, user := range resp {
			if !isFollowing(user) {
				if len(toFollow) >= *maxFollow { // check if we reached max number of people to follow
					return added, nil
				}
				toFollow = append(toFollow, user)
				added++
			}
		}
		if len(resp) != maxCount { // there are no more pages available
			break
		}
		if len(toFollow) >= *maxFollow { // got number of followers required
			break
		}
		page++
	}

	return added, nil
}

// findUsersByTweet uses "searchTerm" to search for tweets using the search/tweets API
// (https://dev.twitter.com/rest/reference/get/search/tweets).
// Returns the number of users found
func findUsersByTweet(api *anaconda.TwitterApi) (int, error) {
	values := make(url.Values)
	maxCount := 100
	// don't ask for more than required
	usersRequired := *maxFollow - len(toFollow)
	if maxCount > usersRequired {
		maxCount = usersRequired
	}
	if maxCount == 0 {
		return 0, nil
	}
	values.Add("result_type", "mixed")
	values.Add("count", strconv.Itoa(maxCount))
	values.Add("include_entities", "false")
	values.Add("lang", "en")

	added := 0
	fn := func(resp anaconda.SearchResponse) {
		for _, tweet := range resp.Statuses {
			if !isFollowing(tweet.User) {
				if len(toFollow) >= *maxFollow { // check if we reached max number of people to follow
					return
				}
				toFollow = append(toFollow, tweet.User)
				added++
			}
		}
	}

	resp, err := api.GetSearch(*searchTerm, values)
	for {
		if err != nil {
			return 0, fmt.Errorf("findUsersByTweet: %v", err)
		}
		fn(resp)
		if len(resp.Statuses) != maxCount { // no more pages
			break
		}
		if len(toFollow) >= *maxFollow { // got number of followers required
			break
		}
		resp, err = resp.GetNext(api)
	}

	return added, nil
}

// getAllFriends returns a slice of all friends of the user
func getAllFriends(api *anaconda.TwitterApi) []anaconda.User {
	var friends []anaconda.User
	ch := api.GetFriendsListAll(nil)
	for f := range ch {
		friends = append(friends, f.Friends...)
	}
	return friends
}

// isFollowing returns true if the user passed is already a friend or is
// already scheduled to be followed
func isFollowing(user anaconda.User) bool {
	for _, u := range alreadyFollowing {
		if u.Id == user.Id {
			return true
		}
	}
	for _, u := range toFollow {
		if u.Id == user.Id {
			return true
		}
	}
	return false
}

// followUser uses the "TwitterApi" passed to follow a "userID".
// If followed without error, the user's screen name is returned.
// Attempting to follow a user which is already a friend will treat
// it as a user who is not a friend
func followUser(api *anaconda.TwitterApi, userID int64) error {
	_, err := api.FollowUserId(userID, nil)
	if err != nil {
		return fmt.Errorf("follow %d: %v\n", userID, err)
	}
	return nil
}
<commit_msg>Changed success message to give new follower count<commit_after>/*
MIT License

Copyright (c) 2016 Kyriacos Kyriacou

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/

package main

import (
	"flag"
	"fmt"
	"log"
	"net/url"
	"os"
	"strconv"
	"time"

	"github.com/ChimeraCoder/anaconda"
)

// searchTerm is the search term used by the search functions
var searchTerm = flag.String("s", "", "(required) search term to find users by (i.e. gopher)")

// maxFollow is the maximum number of users that the application should follow. It is
// an upper bound since there might be a case where not enough users are found to follow
var maxFollow = flag.Int("max", 50, "(optional) max number of users to follow (hard maximum of 100 to avoid limiting by Twitter)")

// alreadyFollowing is a slice that contains all users that are already being followed.
// This is used so that the application does not attempt to follow users that are already friends
var alreadyFollowing []anaconda.User

// toFollow is a slice that should be populated with the users found by the search functions and
// will subsequently be used to follow each user
var toFollow []anaconda.User

func main() {
	api, err := newTwitterAPI()
	if err != nil {
		log.Fatal(err)
	}

	flag.Parse()
	if len(*searchTerm) == 0 {
		flag.PrintDefaults()
		os.Exit(1)
	}
	const hardMaxFollow = 100
	if *maxFollow > hardMaxFollow {
		*maxFollow = hardMaxFollow
	}

	alreadyFollowing = getAllFriends(api)

	// Start finding users

	fmt.Println("Finding users...")
	done := make(chan struct{})
	go spinner(done) // feedback for user

	found := 0
	n, err := findUsers(api)
	if err != nil {
		log.Fatal(err)
	}
	found += n
	n, err = findUsersByTweet(api)
	if err != nil {
		log.Fatal(err)
	}
	done <- struct{}{}
	found += n
	fmt.Printf("\rFound %d unique users\n", found)
	if found == 0 {
		fmt.Println("Try a broader search term next time!")
		os.Exit(0)
	}

	// Start following users

	const userURLFormat = "https://twitter.com/%s"
	fmt.Println("Following...")
	newFriends := 0
	for _, user := range toFollow {
		err := followUser(api, user.Id)
		if err != nil {
			log.Print(fmt.Errorf("could not follow %s: %v\n", user.ScreenName, err))
			break // could continue, but error is most likely due to limiting by twitter and will fall through
		}
		fmt.Printf("%-40s%-s\n", user.Name, fmt.Sprintf(userURLFormat, user.ScreenName))
		newFriends++
	}
	fmt.Println("-------------------------------------------------------------------------------")
	fmt.Printf("You are now following %d new users!\n", newFriends)
}

// newTwitterAPI returns a new Twitter API using keys from the environment
func newTwitterAPI() (*anaconda.TwitterApi, error) {
	keys := []string{
		"TWITTER_CONSUMER_KEY",
		"TWITTER_CONSUMER_SECRET",
		"TWITTER_ACCESS_TOKEN",
		"TWITTER_ACCESS_SECRET",
	}
	pairs := make(map[string]string, 4)
	for _, k := range keys {
		v := os.Getenv(k)
		if v == "" {
			return nil, fmt.Errorf("environment variable %q required", k)
		}
		pairs[k] = v
	}

	anaconda.SetConsumerKey(pairs["TWITTER_CONSUMER_KEY"])
	anaconda.SetConsumerSecret(pairs["TWITTER_CONSUMER_SECRET"])
	api := anaconda.NewTwitterApi(pairs["TWITTER_ACCESS_TOKEN"], pairs["TWITTER_ACCESS_SECRET"])
	_, err := api.VerifyCredentials()
	if err != nil {
		return nil, err
	}
	return api, nil
}

// spinner displays a spinner on the std output
func spinner(stop <-chan struct{}) {
	for {
		select {
		case <-stop:
			return
		default:
			for _, r := range `-\|/` {
				fmt.Printf("\r%c", r)
				time.Sleep(80 * time.Millisecond)
			}
		}
	}
}

// findUsers uses "searchTerm" to search for tweets using the users/search API
// (https://dev.twitter.com/rest/reference/get/users/search).
// Returns the number of users found
func findUsers(api *anaconda.TwitterApi) (int, error) {
	maxCount := 20
	// don't ask for more than required
	usersRequired := *maxFollow - len(toFollow)
	if maxCount > usersRequired {
		maxCount = usersRequired
	}
	if maxCount == 0 {
		return 0, nil
	}
	page := 0
	values := make(url.Values)
	values.Add("count", strconv.Itoa(maxCount))
	values.Add("include_entities", "false")

	added := 0
	for {
		values.Set("page", strconv.Itoa(page))
		resp, err := api.GetUserSearch(*searchTerm, values)
		if err != nil {
			return 0, fmt.Errorf("findUsers: %v", err)
		}
		for _, user := range resp {
			if !isFollowing(user) {
				if len(toFollow) >= *maxFollow { // check if we reached max number of people to follow
					return added, nil
				}
				toFollow = append(toFollow, user)
				added++
			}
		}
		if len(resp) != maxCount { // there are no more pages available
			break
		}
		if len(toFollow) >= *maxFollow { // got number of followers required
			break
		}
		page++
	}

	return added, nil
}

// findUsersByTweet uses "searchTerm" to search for tweets using the search/tweets API
// (https://dev.twitter.com/rest/reference/get/search/tweets).
// Returns the number of users found
func findUsersByTweet(api *anaconda.TwitterApi) (int, error) {
	values := make(url.Values)
	maxCount := 100
	// don't ask for more than required
	usersRequired := *maxFollow - len(toFollow)
	if maxCount > usersRequired {
		maxCount = usersRequired
	}
	if maxCount == 0 {
		return 0, nil
	}
	values.Add("result_type", "mixed")
	values.Add("count", strconv.Itoa(maxCount))
	values.Add("include_entities", "false")
	values.Add("lang", "en")

	added := 0
	fn := func(resp anaconda.SearchResponse) {
		for _, tweet := range resp.Statuses {
			if !isFollowing(tweet.User) {
				if len(toFollow) >= *maxFollow { // check if we reached max number of people to follow
					return
				}
				toFollow = append(toFollow, tweet.User)
				added++
			}
		}
	}

	resp, err := api.GetSearch(*searchTerm, values)
	for {
		if err != nil {
			return 0, fmt.Errorf("findUsersByTweet: %v", err)
		}
		fn(resp)
		if len(resp.Statuses) != maxCount { // no more pages
			break
		}
		if len(toFollow) >= *maxFollow { // got number of followers required
			break
		}
		resp, err = resp.GetNext(api)
	}

	return added, nil
}

// getAllFriends returns a slice of all friends of the user
func getAllFriends(api *anaconda.TwitterApi) []anaconda.User {
	var friends []anaconda.User
	ch := api.GetFriendsListAll(nil)
	for f := range ch {
		friends = append(friends, f.Friends...)
	}
	return friends
}

// isFollowing returns true if the user passed is already a friend or is
// already scheduled to be followed
func isFollowing(user anaconda.User) bool {
	for _, u := range alreadyFollowing {
		if u.Id == user.Id {
			return true
		}
	}
	for _, u := range toFollow {
		if u.Id == user.Id {
			return true
		}
	}
	return false
}

// followUser uses the "TwitterApi" passed to follow a "userID".
// If followed without error, the user's screen name is returned.
// Attempting to follow a user which is already a friend will treat
// it as a user who is not a friend
func followUser(api *anaconda.TwitterApi, userID int64) error {
	_, err := api.FollowUserId(userID, nil)
	if err != nil {
		return fmt.Errorf("follow %d: %v\n", userID, err)
	}
	return nil
}
<|endoftext|>


<commit_before>/*
package goptions implements a flexible parser for command line options.

Key targets were the support for both long and short flag versions, mutually
exclusive flags, and verbs. Flags and their corresponding variables are defined
by the tags in a (possibly anonymous) struct.

    var options struct {
    	Name string `goptions:"-n, --name"`
    	Force bool `goptions:"-f, --force"`
    	Verbosity int `goptions:"-v, --verbose"`
    }

Short flags can be combined (e.g. `-nfv`). Long flags take their value after a
separating space. The equals notation (`--long-flag=value`) is NOT supported
right now.

Every member of the struct which is supposed to catch a command line value
has to have a "goptions" tag. The contains the short and long flag names for this
member but can additionally specify any of these options below.

    obligatory        - Flag must be specified. Otherwise an error will be returned
                        when Parse() is called.
    description='...' - Set the description for this particular flag. Will be
                        used by the HelpFunc.
    mutexgroup='...'  - Add this flag to a MutexGroup. Only one flag of the
                        ones sharing a MutexGroup can be set. Otherwise an error
                        will be returned when Parse() is called. If one flag in a
                        MutexGroup is `obligatory` one flag of the group must be
                        specified. A flag can be in multiple MutexGroups at once.

Depending on the type of the struct member, additional options might become available:

    Type: *os.File
        The given string is interpreted as a path to a file. If the string is "-"
        os.Stdin or os.Stdout will be used. os.Stdin will be returned, if the
        `rdonly` flag was set. os.Stdout will be returned, if `wronly` was set.
    Available options:
        Any combination of create, append, rdonly, wronly, rdwr,
        excl, sync, trunc and perm can be specified and correspond directly with
        the combination of the homonymous flags in the os package.

    Type: *net.TCPAddr
        The given string is interpreted as a tcp address. It is passed to
        net.ResolvTCPAddr() with "tcp" as the network type identifier.

    Type: *net/url.URL
        The given string is parsed by net/url.Parse()

If a member is a slice type, multiple definitions of the flags are possible. For each
specification the underlying type will be used.

goptions also has support for verbs. Each verb accepts its own set of flags which
take exactly the same tag format as global options. For an usage example of verbs
see the PrintHelp() example.
*/
package goptions

import (
	"fmt"
	"os"
	"path/filepath"
)

const (
	VERSION = "2.2.0"
)

var (
	globalFlagSet *FlagSet
)

// ParseAndFail is a convenience function to parse os.Args[1:] and print
// the help if an error occurs. This should cover 90% of this library's
// applications.
func ParseAndFail(v interface{}) {
	err := Parse(v)
	if err != nil {
		errCode := 0
		if err != ErrHelpRequest {
			errCode = 1
			fmt.Printf("Error: %s\n", err)
		}
		PrintHelp()
		os.Exit(errCode)
	}
}

// Parse parses the command-line flags from os.Args[1:].
func Parse(v interface{}) error {
	globalFlagSet = NewFlagSet(filepath.Base(os.Args[0]), v)
	return globalFlagSet.Parse(os.Args[1:])
}

// PrintHelp renders the default help to os.Stderr.
func PrintHelp() {
	if globalFlagSet == nil {
		panic("Must call Parse() before PrintHelp()")
	}
	globalFlagSet.PrintHelp(os.Stderr)
}
<commit_msg>Add documentation<commit_after>/*
package goptions implements a flexible parser for command line options.

Key targets were the support for both long and short flag versions, mutually
exclusive flags, and verbs. Flags and their corresponding variables are defined
by the tags in a (possibly anonymous) struct.

    var options struct {
    	Name string `goptions:"-n, --name"`
    	Force bool `goptions:"-f, --force"`
    	Verbosity int `goptions:"-v, --verbose"`
    }

Short flags can be combined (e.g. `-nfv`). Long flags take their value after a
separating space. The equals notation (`--long-flag=value`) is NOT supported
right now.

Every member of the struct which is supposed to catch a command line value
has to have a "goptions" tag. The contains the short and long flag names for this
member but can additionally specify any of these options below.

    obligatory        - Flag must be specified. Otherwise an error will be returned
                        when Parse() is called.
    description='...' - Set the description for this particular flag. Will be
                        used by the HelpFunc.
    mutexgroup='...'  - Add this flag to a MutexGroup. Only one flag of the
                        ones sharing a MutexGroup can be set. Otherwise an error
                        will be returned when Parse() is called. If one flag in a
                        MutexGroup is `obligatory` one flag of the group must be
                        specified. A flag can be in multiple MutexGroups at once.

Depending on the type of the struct member, additional options might become available:

    Type: *os.File
        The given string is interpreted as a path to a file. If the string is "-"
        os.Stdin or os.Stdout will be used. os.Stdin will be returned, if the
        `rdonly` flag was set. os.Stdout will be returned, if `wronly` was set.
    Available options:
        Any combination of create, append, rdonly, wronly, rdwr,
        excl, sync, trunc and perm can be specified and correspond directly with
        the combination of the homonymous flags in the os package.

    Type: *net.TCPAddr
        The given string is interpreted as a tcp address. It is passed to
        net.ResolvTCPAddr() with "tcp" as the network type identifier.

    Type: *net/url.URL
        The given string is parsed by net/url.Parse()

    Type: time.Duration
        The given string is parsed by time.ParseDuration()

If a member is a slice type, multiple definitions of the flags are possible. For each
specification the underlying type will be used.

goptions also has support for verbs. Each verb accepts its own set of flags which
take exactly the same tag format as global options. For an usage example of verbs
see the PrintHelp() example.
*/
package goptions

import (
	"fmt"
	"os"
	"path/filepath"
)

const (
	VERSION = "2.2.0"
)

var (
	globalFlagSet *FlagSet
)

// ParseAndFail is a convenience function to parse os.Args[1:] and print
// the help if an error occurs. This should cover 90% of this library's
// applications.
func ParseAndFail(v interface{}) {
	err := Parse(v)
	if err != nil {
		errCode := 0
		if err != ErrHelpRequest {
			errCode = 1
			fmt.Printf("Error: %s\n", err)
		}
		PrintHelp()
		os.Exit(errCode)
	}
}

// Parse parses the command-line flags from os.Args[1:].
func Parse(v interface{}) error {
	globalFlagSet = NewFlagSet(filepath.Base(os.Args[0]), v)
	return globalFlagSet.Parse(os.Args[1:])
}

// PrintHelp renders the default help to os.Stderr.
func PrintHelp() {
	if globalFlagSet == nil {
		panic("Must call Parse() before PrintHelp()")
	}
	globalFlagSet.PrintHelp(os.Stderr)
}
<|endoftext|>


<commit_before>package goreport

import (
	"github.com/CapillarySoftware/gostat/protoStat"
	nano "github.com/op/go-nanomsg"
	// "strings"
	"fmt"
	"sync"
	"time"
)

var asyncQ chan *protoStat.ProtoStat
var confQ chan config
var wg sync.WaitGroup

func init() {
	wg.Add(1)
	asyncQ = make(chan *protoStat.ProtoStat, 1000)
	confQ = make(chan config, 1) //block on config
	go asyncProcess(asyncQ, confQ)
}

// Reporter used by clients
type Reporter struct {
	async chan *protoStat.ProtoStat
	conf  chan config
}

//Simple push socket to send external
type push struct {
	socket *nano.PushSocket
}

// Configuration for the background thread
type config struct {
	timeout int
	url     string
}

//Connect to the remote server
func (this *push) connect(url *string) (err error) {
	if nil != this.socket {
		this.socket.Close()
	}
	this.socket, err = nano.NewPushSocket()
	if nil != err {
		return
	}
	_, err = this.socket.Connect(*url)
	return
}

//Close the push socket
func (this *push) Close() {
	if nil != this.socket {
		this.socket.Close()
	}
}

//Set the timeout for the send socket
func (this *push) SetTimeout(millis time.Duration) {
	this.socket.SetSendTimeout(millis * time.Millisecond)
}

//Create a new push socket, only for internal use
func newPush(url *string, timeout time.Duration) (p push, err error) {
	err = p.connect(url)
	p.SetTimeout(timeout)
	return
}

//async background thread that processing all stats
func asyncProcess(q <-chan *protoStat.ProtoStat, conf <-chan config) {
	c := <-conf
	if c.url == "" {
		fmt.Println("Failed to get valid configuration maybe ReporterConfig not called?")
		wg.Done()
		return
	}
	sendQ, err := newPush(&c.url, time.Duration(c.timeout))
	if nil != err {
		wg.Done()
		return
	}
	stats := make(map[string]*protoStat.ProtoStat)
	reportInterval := make(chan bool, 1)
	go func() {
		for {
			time.Sleep(5 * time.Second)
			reportInterval <- true
		}
	}()
main:
	for {
		select {

		case c = <-conf:
			// fmt.Println(c)
			sendQ.Close()
			sendQ, err = newPush(&c.url, time.Duration(c.timeout))
			if nil != err {
				fmt.Println("Failed to reconfigure queue")
				break main
			}
		case m := <-q:
			if nil == m {
				break main
			}
			updateMap(stats, m)
		case _ = <-reportInterval:
			// fmt.Println("Time to report :", report)
			// fmt.Println(stats)
			if len(stats) > 0 {
				err = sendQ.sendStats(stats)
				stats = make(map[string]*protoStat.ProtoStat)
				if nil != err {
					fmt.Println("Failed to send stats: ", err)
				}
			}
		}
	}

	//cleanup anything still on the queue
	for m := range q {
		if nil == m {
			break
		}
		updateMap(stats, m)
	}
	err = sendQ.sendStats(stats)
	if nil != err {
		fmt.Println(err)
	}

	fmt.Println("Finished bg thread")
	wg.Done()
}

//update map with new data
func updateMap(stats map[string]*protoStat.ProtoStat, stat *protoStat.ProtoStat) {
	// fmt.Println("map: ", stats)
	ck := stat.GetKey() + stat.GetIndexKey()
	oldStat, ok := stats[ck]
	if !ok {
		stats[ck] = stat
	} else {
		v := oldStat.GetValue() + stat.GetValue()
		oldStat.Value = &v
	}
}

func (this *push) sendStats(stats map[string]*protoStat.ProtoStat) (err error) {
	var s []*protoStat.ProtoStat
	// fmt.Println(stats)
	pStats := new(protoStat.ProtoStats)
	for _, v := range stats {
		s = append(s, v)
	}
	// fmt.Println(s)
	pStats.Stats = s
	fmt.Println(pStats)
	bytes, err := pStats.Marshal()
	if nil != err {
		return
	}
	_, err = this.socket.Send(bytes, 0) //blocking
	return
}

//New reporter that reports at 5 second intervals
func ReporterConfig(url string, timeout int) {
	c := config{timeout: timeout, url: url}
	confQ <- c
	return
}

func NewReporter() (r Reporter) {
	r = Reporter{async: asyncQ, conf: confQ}
	return
}

//Add a basic key value stat
func (this *Reporter) AddStat(key string, value float64) {
	stat := protoStat.ProtoStat{Key: &key, Value: &value}
	this.async <- &stat
}

//Add multiple stats into the same graph
func (this *Reporter) AddStatWIndex(key string, value float64, indexKey string) {
	stat := protoStat.ProtoStat{Key: &key, Value: &value, IndexKey: &indexKey}
	this.async <- &stat
}

//Close the reporter and the background thread
func (this *Reporter) Close() {
	close(this.async)
	close(this.conf)
	fmt.Println("Closed queue, waiting for cleanup")
	wg.Wait()
}
<commit_msg>add time to stats that get reported<commit_after>package goreport

import (
	"github.com/CapillarySoftware/gostat/protoStat"
	nano "github.com/op/go-nanomsg"
	// "strings"
	"fmt"
	"sync"
	"time"
)

var asyncQ chan *protoStat.ProtoStat
var confQ chan config
var wg sync.WaitGroup

func init() {
	wg.Add(1)
	asyncQ = make(chan *protoStat.ProtoStat, 1000)
	confQ = make(chan config, 1) //block on config
	go asyncProcess(asyncQ, confQ)
}

// Reporter used by clients
type Reporter struct {
	async chan *protoStat.ProtoStat
	conf  chan config
}

//Simple push socket to send external
type push struct {
	socket *nano.PushSocket
}

// Configuration for the background thread
type config struct {
	timeout int
	url     string
}

//Connect to the remote server
func (this *push) connect(url *string) (err error) {
	if nil != this.socket {
		this.socket.Close()
	}
	this.socket, err = nano.NewPushSocket()
	if nil != err {
		return
	}
	_, err = this.socket.Connect(*url)
	return
}

//Close the push socket
func (this *push) Close() {
	if nil != this.socket {
		this.socket.Close()
	}
}

//Set the timeout for the send socket
func (this *push) SetTimeout(millis time.Duration) {
	this.socket.SetSendTimeout(millis * time.Millisecond)
}

//Create a new push socket, only for internal use
func newPush(url *string, timeout time.Duration) (p push, err error) {
	err = p.connect(url)
	p.SetTimeout(timeout)
	return
}

//async background thread that processing all stats
func asyncProcess(q <-chan *protoStat.ProtoStat, conf <-chan config) {
	c := <-conf
	if c.url == "" {
		fmt.Println("Failed to get valid configuration maybe ReporterConfig not called?")
		wg.Done()
		return
	}
	sendQ, err := newPush(&c.url, time.Duration(c.timeout))
	if nil != err {
		wg.Done()
		return
	}
	stats := make(map[string]*protoStat.ProtoStat)
	reportInterval := make(chan bool, 1)
	go func() {
		for {
			time.Sleep(5 * time.Second)
			reportInterval <- true
		}
	}()
main:
	for {
		select {

		case c = <-conf:
			// fmt.Println(c)
			sendQ.Close()
			sendQ, err = newPush(&c.url, time.Duration(c.timeout))
			if nil != err {
				fmt.Println("Failed to reconfigure queue")
				break main
			}
		case m := <-q:
			if nil == m {
				break main
			}
			updateMap(stats, m)
		case _ = <-reportInterval:
			// fmt.Println("Time to report :", report)
			// fmt.Println(stats)
			if len(stats) > 0 {
				err = sendQ.sendStats(stats)
				stats = make(map[string]*protoStat.ProtoStat)
				if nil != err {
					fmt.Println("Failed to send stats: ", err)
				}
			}
		}
	}

	//cleanup anything still on the queue
	for m := range q {
		if nil == m {
			break
		}
		updateMap(stats, m)
	}
	err = sendQ.sendStats(stats)
	if nil != err {
		fmt.Println(err)
	}

	fmt.Println("Finished bg thread")
	wg.Done()
}

//update map with new data
func updateMap(stats map[string]*protoStat.ProtoStat, stat *protoStat.ProtoStat) {
	// fmt.Println("map: ", stats)
	ck := stat.GetKey() + stat.GetIndexKey()
	oldStat, ok := stats[ck]
	if !ok {
		stats[ck] = stat
	} else {
		v := oldStat.GetValue() + stat.GetValue()
		oldStat.Value = &v
	}
}

func (this *push) sendStats(stats map[string]*protoStat.ProtoStat) (err error) {
	var s []*protoStat.ProtoStat
	// fmt.Println(stats)
	pStats := new(protoStat.ProtoStats)
	for _, v := range stats {
		s = append(s, v)
	}
	// fmt.Println(s)
	pStats.Stats = s
	now := time.Now().UTC().UnixNano()
	pStats.TimeNano = &now

	fmt.Println(pStats)
	bytes, err := pStats.Marshal()
	if nil != err {
		return
	}
	_, err = this.socket.Send(bytes, 0) //blocking
	return
}

//New reporter that reports at 5 second intervals
func ReporterConfig(url string, timeout int) {
	c := config{timeout: timeout, url: url}
	confQ <- c
	return
}

func NewReporter() (r Reporter) {
	r = Reporter{async: asyncQ, conf: confQ}
	return
}

func (this *Reporter) AddRepeatedStat(key string) {

}
func (this *Reporter) AddRepeatedStatWIndex(key string, indexKey string) {

}

//Add a basic key value stat
func (this *Reporter) AddStat(key string, value float64) {
	stat := protoStat.ProtoStat{Key: &key, Value: &value}
	this.async <- &stat
}

//Add multiple stats into the same graph
func (this *Reporter) AddStatWIndex(key string, value float64, indexKey string) {
	stat := protoStat.ProtoStat{Key: &key, Value: &value, IndexKey: &indexKey}
	this.async <- &stat
}

//Close the reporter and the background thread
func (this *Reporter) Close() {
	close(this.async)
	close(this.conf)
	fmt.Println("Closed queue, waiting for cleanup")
	wg.Wait()
}
<|endoftext|>