<commit_before>/*  lMDQ is a MDQ server that caches metadata locally so it's local clients can lookup
    pre-checked metadata and not depend on a working connection to a remote MDQ server.

    It uses SQLite as it's datastore and allows lookup by either entityID or Location.
    The latter is used by WAYF for it's mass hosting services BIRK and KRIB.

    It can also be used as a library for just looking up metadata inside a go program.

    Or a client can use the SQLite database directly using the following query:

		"select e.md, e.hash from entity e, lookup l where l.hash = $1 and l.entity_id_fk = e.id and e.validuntil >= $2"

    where $1 is the lowercase hex sha1 of the entityID or location without the {sha1} prefix
    $2 is the current epoch.

    to-do:
        √ caching interface
          invalidate cache ???
*/

package lMDQ

import (
	"crypto/sha1"
	"database/sql"
	"encoding/hex"
	"errors"
	// 	_ "github.com/mattn/go-sqlite3" for handling sqlite3
	_ "github.com/mattn/go-sqlite3"
	"github.com/wayf-dk/gosaml"
	"github.com/wayf-dk/goxml"
	"sort"
	"strings"
	"sync"
	"time"
)

type (
	// EntityRec refers entity info
	EntityRec struct {
		id       int64
		entityid string
		hash     string
	}
	// MDQ refers to metadata query
	MDQ struct {
		db    *sql.DB
		stmt  *sql.Stmt
		Path  string
		Cache map[string]*MdXp
		Lock  sync.Mutex
		Table string
	}
	// MdXp refers to check validity
	MdXp struct {
		*goxml.Xp
		created time.Time
	}
)

var (
	cacheduration = time.Minute * 60
	// MetaDataNotFoundError refers to error
	MetaDataNotFoundError = errors.New("Metadata not found")
)

// Valid refers to check the validity of metadata
func (xp *MdXp) Valid(duration time.Duration) bool {
	since := time.Since(xp.created)
	//log.Println(since, duration, since  < duration)
	return since < duration
}

// Open refers to open metadata file
func (mdq *MDQ) Open() (err error) {
	mdq.Lock.Lock()
	defer mdq.Lock.Unlock()
	mdq.Cache = make(map[string]*MdXp)
	if mdq.db != nil {
	    mdq.db.Close()
	}
	mdq.db, err = sql.Open("sqlite3", mdq.Path)
	if err != nil {
		return
	}
	// This is supposed to be a very smart wayf do to prefix search - keep an eye on whether a 10 char prefix ie. using 40 bits is enough
	mdq.stmt, err = mdq.db.Prepare("select e.md md from entity_" + mdq.Table + " e, lookup_" + mdq.Table + " l where ? < l.hash||'z' and l.hash||'z' <= ? and l.entity_id_fk = e.id")
	if err != nil {
		return
	}
	return
}

// MDQ looks up an entity using the supplied feed and key.
// The key can be an entityID or a location, optionally in {sha1} format
// It returns a non nil err if the entity is not found
// and the metadata and a hash/etag over the content if it is.
// The hash can be used to decide if a cached dom object is still valid,
// This might be an optimization as the database lookup is much faster that the parsing.
func (mdq *MDQ) MDQ(key string) (xp *goxml.Xp, err error) {
	return mdq.dbget(key, true)
}

func (mdq *MDQ) dbget(key string, cache bool) (xp *goxml.Xp, err error) {
	mdq.Lock.Lock()
	defer mdq.Lock.Unlock()

	k := key
	if strings.HasPrefix(key, "{sha1}") {
		key = key[6:]
	} else {
		hash := sha1.Sum([]byte(key))
		key = hex.EncodeToString(append(hash[:]))
	}
	key = key[:10] // only use the first 10 chars for key
	cachedxp := mdq.Cache[key]
	if cachedxp != nil && cachedxp.Valid(cacheduration) {
		xp = cachedxp.Xp.CpXp()
		return
	}

	var xml []byte
	err = mdq.stmt.QueryRow(key, key+"z").Scan(&xml)
	switch {
	case err == sql.ErrNoRows:
		err = goxml.Wrap(MetaDataNotFoundError, "err:Metadata not found", "key:"+k, "table:"+mdq.Table)
		return
	case err != nil:
		return
	default:
		md := gosaml.Inflate(xml)
		xp = goxml.NewXp(md)
	}
	if cache {
		mdxp := new(MdXp)
		mdxp.Xp = xp
		mdxp.created = time.Now()
		mdq.Cache[key] = mdxp
	}
	return
}

// MDQFilter refers Filtering by xpath for testing purposes
func (mdq *MDQ) MDQFilter(xpathfilter string) (xp *goxml.Xp, numberOfEntities int, err error) {
	recs, err := mdq.getEntityList()
	if err != nil {
		return
	}

	// get the entities into an ordered slice
	index := make([]string, len(recs))
	i := 0
	for k := range recs {
		index[i] = k
		i++
	}
	sort.Strings(index)

	//log.Println(xpathfilter)
	xp = goxml.NewXpFromString(`<md:EntitiesDescriptor xmlns:md="urn:oasis:names:tc:SAML:2.0:metadata" />`)

	root, _ := xp.Doc.DocumentElement()
	for _, entityID := range index {
		ent, _ := mdq.dbget(entityID, false)

		if xpathfilter == "" || len(ent.Query(nil, xpathfilter)) > 0 {
			entity, _ := ent.Doc.DocumentElement()
			root.AddChild(xp.CopyNode(entity, 1))
			numberOfEntities++
		}
	}
	return
}

// getEntityList returns a map keyed by entityIDs for the
// current entities in the database
func (mdq *MDQ) getEntityList() (entities map[string]EntityRec, err error) {

	entities = make(map[string]EntityRec)
	var rows *sql.Rows
	rows, err = mdq.db.Query("select id, entityid, hash from entity_" + mdq.Table)
	if err != nil {
		return
	}
	defer rows.Close()
	for rows.Next() {
		var rec EntityRec
		if err = rows.Scan(&rec.id, &rec.entityid, &rec.hash); err != nil {
			return
		}
		entities[rec.entityid] = rec
	}
	if err = rows.Err(); err != nil { // no reason to actually check err, but if we later forget ...
		return
	}
	return
}
<commit_msg>go fmt<commit_after>/*  lMDQ is a MDQ server that caches metadata locally so it's local clients can lookup
    pre-checked metadata and not depend on a working connection to a remote MDQ server.

    It uses SQLite as it's datastore and allows lookup by either entityID or Location.
    The latter is used by WAYF for it's mass hosting services BIRK and KRIB.

    It can also be used as a library for just looking up metadata inside a go program.

    Or a client can use the SQLite database directly using the following query:

		"select e.md, e.hash from entity e, lookup l where l.hash = $1 and l.entity_id_fk = e.id and e.validuntil >= $2"

    where $1 is the lowercase hex sha1 of the entityID or location without the {sha1} prefix
    $2 is the current epoch.

    to-do:
        √ caching interface
          invalidate cache ???
*/

package lMDQ

import (
	"crypto/sha1"
	"database/sql"
	"encoding/hex"
	"errors"
	// 	_ "github.com/mattn/go-sqlite3" for handling sqlite3
	_ "github.com/mattn/go-sqlite3"
	"github.com/wayf-dk/gosaml"
	"github.com/wayf-dk/goxml"
	"sort"
	"strings"
	"sync"
	"time"
)

type (
	// EntityRec refers entity info
	EntityRec struct {
		id       int64
		entityid string
		hash     string
	}
	// MDQ refers to metadata query
	MDQ struct {
		db    *sql.DB
		stmt  *sql.Stmt
		Path  string
		Cache map[string]*MdXp
		Lock  sync.Mutex
		Table string
	}
	// MdXp refers to check validity
	MdXp struct {
		*goxml.Xp
		created time.Time
	}
)

var (
	cacheduration = time.Minute * 60
	// MetaDataNotFoundError refers to error
	MetaDataNotFoundError = errors.New("Metadata not found")
)

// Valid refers to check the validity of metadata
func (xp *MdXp) Valid(duration time.Duration) bool {
	since := time.Since(xp.created)
	//log.Println(since, duration, since  < duration)
	return since < duration
}

// Open refers to open metadata file
func (mdq *MDQ) Open() (err error) {
	mdq.Lock.Lock()
	defer mdq.Lock.Unlock()
	mdq.Cache = make(map[string]*MdXp)
	if mdq.db != nil {
		mdq.db.Close()
	}
	mdq.db, err = sql.Open("sqlite3", mdq.Path)
	if err != nil {
		return
	}
	// This is supposed to be a very smart wayf do to prefix search - keep an eye on whether a 10 char prefix ie. using 40 bits is enough
	mdq.stmt, err = mdq.db.Prepare("select e.md md from entity_" + mdq.Table + " e, lookup_" + mdq.Table + " l where ? < l.hash||'z' and l.hash||'z' <= ? and l.entity_id_fk = e.id")
	if err != nil {
		return
	}
	return
}

// MDQ looks up an entity using the supplied feed and key.
// The key can be an entityID or a location, optionally in {sha1} format
// It returns a non nil err if the entity is not found
// and the metadata and a hash/etag over the content if it is.
// The hash can be used to decide if a cached dom object is still valid,
// This might be an optimization as the database lookup is much faster that the parsing.
func (mdq *MDQ) MDQ(key string) (xp *goxml.Xp, err error) {
	return mdq.dbget(key, true)
}

func (mdq *MDQ) dbget(key string, cache bool) (xp *goxml.Xp, err error) {
	mdq.Lock.Lock()
	defer mdq.Lock.Unlock()

	k := key
	if strings.HasPrefix(key, "{sha1}") {
		key = key[6:]
	} else {
		hash := sha1.Sum([]byte(key))
		key = hex.EncodeToString(append(hash[:]))
	}
	key = key[:10] // only use the first 10 chars for key
	cachedxp := mdq.Cache[key]
	if cachedxp != nil && cachedxp.Valid(cacheduration) {
		xp = cachedxp.Xp.CpXp()
		return
	}

	var xml []byte
	err = mdq.stmt.QueryRow(key, key+"z").Scan(&xml)
	switch {
	case err == sql.ErrNoRows:
		err = goxml.Wrap(MetaDataNotFoundError, "err:Metadata not found", "key:"+k, "table:"+mdq.Table)
		return
	case err != nil:
		return
	default:
		md := gosaml.Inflate(xml)
		xp = goxml.NewXp(md)
	}
	if cache {
		mdxp := new(MdXp)
		mdxp.Xp = xp
		mdxp.created = time.Now()
		mdq.Cache[key] = mdxp
	}
	return
}

// MDQFilter refers Filtering by xpath for testing purposes
func (mdq *MDQ) MDQFilter(xpathfilter string) (xp *goxml.Xp, numberOfEntities int, err error) {
	recs, err := mdq.getEntityList()
	if err != nil {
		return
	}

	// get the entities into an ordered slice
	index := make([]string, len(recs))
	i := 0
	for k := range recs {
		index[i] = k
		i++
	}
	sort.Strings(index)

	//log.Println(xpathfilter)
	xp = goxml.NewXpFromString(`<md:EntitiesDescriptor xmlns:md="urn:oasis:names:tc:SAML:2.0:metadata" />`)

	root, _ := xp.Doc.DocumentElement()
	for _, entityID := range index {
		ent, _ := mdq.dbget(entityID, false)

		if xpathfilter == "" || len(ent.Query(nil, xpathfilter)) > 0 {
			entity, _ := ent.Doc.DocumentElement()
			root.AddChild(xp.CopyNode(entity, 1))
			numberOfEntities++
		}
	}
	return
}

// getEntityList returns a map keyed by entityIDs for the
// current entities in the database
func (mdq *MDQ) getEntityList() (entities map[string]EntityRec, err error) {

	entities = make(map[string]EntityRec)
	var rows *sql.Rows
	rows, err = mdq.db.Query("select id, entityid, hash from entity_" + mdq.Table)
	if err != nil {
		return
	}
	defer rows.Close()
	for rows.Next() {
		var rec EntityRec
		if err = rows.Scan(&rec.id, &rec.entityid, &rec.hash); err != nil {
			return
		}
		entities[rec.entityid] = rec
	}
	if err = rows.Err(); err != nil { // no reason to actually check err, but if we later forget ...
		return
	}
	return
}
<|endoftext|>


<commit_before>// Package dms3mail configuration structures and variables
//
package dms3mail

import "github.com/richbl/go-distributed-motion-s3/dms3libs"

// mailConfig contains dms3mail configuration settings read from TOML file
var mailConfig *structEmailSettings

// motion mail configuration parameters
type structEmailSettings struct {
	Filename     string
	FileLocation string
	Email        *structEmail
	SMTP         *structSMTP
	Logging      *dms3libs.StructLogging
}

// email composition parameters
type structEmail struct {
	To   string
	From string
}

// SMTP mailer parameters
type structSMTP struct {
	Address            string
	Port               int
	Domain             string
	Username           string
	Password           string
	Authentication     string
	EnableStartTLSAuto bool
}

// event details to be sent via email
type structEventDetails struct {
	eventMedia  string
	eventDate   string
	eventChange string
	clientName  string
}

// HTML tokens used in email template
type emailTemplateElements struct {
	Header string
	Event  string
	Date   string
	Client string
	Change string
	Footer string
}
<commit_msg>Removed unused values from external GoMail package<commit_after>// Package dms3mail configuration structures and variables
//
package dms3mail

import "github.com/richbl/go-distributed-motion-s3/dms3libs"

// mailConfig contains dms3mail configuration settings read from TOML file
var mailConfig *structEmailSettings

// motion mail configuration parameters
type structEmailSettings struct {
	Filename     string
	FileLocation string
	Email        *structEmail
	SMTP         *structSMTP
	Logging      *dms3libs.StructLogging
}

// email composition parameters
type structEmail struct {
	To   string
	From string
}

// SMTP mailer parameters
type structSMTP struct {
	Address  string
	Port     int
	Username string
	Password string
}

// event details to be sent via email
type structEventDetails struct {
	eventMedia  string
	eventDate   string
	eventChange string
	clientName  string
}

// HTML tokens used in email template
type emailTemplateElements struct {
	Header string
	Event  string
	Date   string
	Client string
	Change string
	Footer string
}
<|endoftext|>


<commit_before>// Copyright 2014 Manu Martinez-Almeida.  All rights reserved.
// Use of this source code is governed by a MIT style
// license that can be found in the LICENSE file.

package binding

import (
	"encoding/json"
	"encoding/xml"
	"errors"
	"net/http"
	"reflect"
	"strconv"
	"strings"
)

type (
	Binding interface {
		Bind(*http.Request, interface{}) error
	}

	// JSON binding
	jsonBinding struct{}

	// XML binding
	xmlBinding struct{}

	// // form binding
	formBinding struct{}
)

var (
	JSON = jsonBinding{}
	XML  = xmlBinding{}
	Form = formBinding{} // todo
)

func (_ jsonBinding) Bind(req *http.Request, obj interface{}) error {
	decoder := json.NewDecoder(req.Body)
	if err := decoder.Decode(obj); err == nil {
		return Validate(obj)
	} else {
		return err
	}
}

func (_ xmlBinding) Bind(req *http.Request, obj interface{}) error {
	decoder := xml.NewDecoder(req.Body)
	if err := decoder.Decode(obj); err == nil {
		return Validate(obj)
	} else {
		return err
	}
}

func (_ formBinding) Bind(req *http.Request, obj interface{}) error {
	if err := req.ParseForm(); err != nil {
		return err
	}
	if err := mapForm(obj, req.Form); err != nil {
		return err
	}
	return Validate(obj)
}

func mapForm(ptr interface{}, form map[string][]string) error {
	typ := reflect.TypeOf(ptr).Elem()
	formStruct := reflect.ValueOf(ptr).Elem()
	for i := 0; i < typ.NumField(); i++ {
		typeField := typ.Field(i)
		if inputFieldName := typeField.Tag.Get("form"); inputFieldName != "" {
			structField := formStruct.Field(i)
			if !structField.CanSet() {
				continue
			}

			inputValue, exists := form[inputFieldName]
			if !exists {
				continue
			}
			numElems := len(inputValue)
			if structField.Kind() == reflect.Slice && numElems > 0 {
				sliceOf := structField.Type().Elem().Kind()
				slice := reflect.MakeSlice(structField.Type(), numElems, numElems)
				for i := 0; i < numElems; i++ {
					if err := setWithProperType(sliceOf, inputValue[i], slice.Index(i)); err != nil {
						return err
					}
				}
				formStruct.Elem().Field(i).Set(slice)
			} else {
				if err := setWithProperType(typeField.Type.Kind(), inputValue[0], structField); err != nil {
					return err
				}
			}
		}
	}
	return nil
}

func setWithProperType(valueKind reflect.Kind, val string, structField reflect.Value) error {
	switch valueKind {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		if val == "" {
			val = "0"
		}
		intVal, err := strconv.Atoi(val)
		if err != nil {
			return err
		} else {
			structField.SetInt(int64(intVal))
		}
	case reflect.Bool:
		if val == "" {
			val = "false"
		}
		boolVal, err := strconv.ParseBool(val)
		if err != nil {
			return err
		} else {
			structField.SetBool(boolVal)
		}
	case reflect.Float32:
		if val == "" {
			val = "0.0"
		}
		floatVal, err := strconv.ParseFloat(val, 32)
		if err != nil {
			return err
		} else {
			structField.SetFloat(floatVal)
		}
	case reflect.Float64:
		if val == "" {
			val = "0.0"
		}
		floatVal, err := strconv.ParseFloat(val, 64)
		if err != nil {
			return err
		} else {
			structField.SetFloat(floatVal)
		}
	case reflect.String:
		structField.SetString(val)
	}
	return nil
}

// Don't pass in pointers to bind to. Can lead to bugs. See:
// https://github.com/codegangsta/martini-contrib/issues/40
// https://github.com/codegangsta/martini-contrib/pull/34#issuecomment-29683659
func ensureNotPointer(obj interface{}) {
	if reflect.TypeOf(obj).Kind() == reflect.Ptr {
		panic("Pointers are not accepted as binding models")
	}
}

func Validate(obj interface{}, parents ...string) error {
	typ := reflect.TypeOf(obj)
	val := reflect.ValueOf(obj)

	if typ.Kind() == reflect.Ptr {
		typ = typ.Elem()
		val = val.Elem()
	}

	switch typ.Kind() {
	case reflect.Struct:
		for i := 0; i < typ.NumField(); i++ {
			field := typ.Field(i)

			// Allow ignored fields in the struct
			if field.Tag.Get("form") == "-" {
				continue
			}

			fieldValue := val.Field(i).Interface()
			zero := reflect.Zero(field.Type).Interface()

			if strings.Index(field.Tag.Get("binding"), "required") > -1 {
				fieldType := field.Type.Kind()
				if fieldType == reflect.Struct {
					if reflect.DeepEqual(zero, fieldValue) {
						return errors.New("Required " + field.Name)
					}
					err := Validate(fieldValue, field.Name)
					if err != nil {
						return err
					}
				} else if reflect.DeepEqual(zero, fieldValue) {
					if len(parents) > 0 {
						return errors.New("Required " + field.Name + " on " + parents[0])
					} else {
						return errors.New("Required " + field.Name)
					}
				} else if fieldType == reflect.Slice && field.Type.Elem().Kind() == reflect.Struct {
					err := Validate(fieldValue)
					if err != nil {
						return err
					}
				}
			}
		}
	case reflect.Slice:
		for i := 0; i < val.Len(); i++ {
			fieldValue := val.Index(i).Interface()
			err := Validate(fieldValue)
			if err != nil {
				return err
			}
		}
	default:
		return nil
	}
	return nil
}
<commit_msg>Add validating sub structures<commit_after>// Copyright 2014 Manu Martinez-Almeida.  All rights reserved.
// Use of this source code is governed by a MIT style
// license that can be found in the LICENSE file.

package binding

import (
	"encoding/json"
	"encoding/xml"
	"errors"
	"net/http"
	"reflect"
	"strconv"
	"strings"
)

type (
	Binding interface {
		Bind(*http.Request, interface{}) error
	}

	// JSON binding
	jsonBinding struct{}

	// XML binding
	xmlBinding struct{}

	// // form binding
	formBinding struct{}
)

var (
	JSON = jsonBinding{}
	XML  = xmlBinding{}
	Form = formBinding{} // todo
)

func (_ jsonBinding) Bind(req *http.Request, obj interface{}) error {
	decoder := json.NewDecoder(req.Body)
	if err := decoder.Decode(obj); err == nil {
		return Validate(obj)
	} else {
		return err
	}
}

func (_ xmlBinding) Bind(req *http.Request, obj interface{}) error {
	decoder := xml.NewDecoder(req.Body)
	if err := decoder.Decode(obj); err == nil {
		return Validate(obj)
	} else {
		return err
	}
}

func (_ formBinding) Bind(req *http.Request, obj interface{}) error {
	if err := req.ParseForm(); err != nil {
		return err
	}
	if err := mapForm(obj, req.Form); err != nil {
		return err
	}
	return Validate(obj)
}

func mapForm(ptr interface{}, form map[string][]string) error {
	typ := reflect.TypeOf(ptr).Elem()
	formStruct := reflect.ValueOf(ptr).Elem()
	for i := 0; i < typ.NumField(); i++ {
		typeField := typ.Field(i)
		if inputFieldName := typeField.Tag.Get("form"); inputFieldName != "" {
			structField := formStruct.Field(i)
			if !structField.CanSet() {
				continue
			}

			inputValue, exists := form[inputFieldName]
			if !exists {
				continue
			}
			numElems := len(inputValue)
			if structField.Kind() == reflect.Slice && numElems > 0 {
				sliceOf := structField.Type().Elem().Kind()
				slice := reflect.MakeSlice(structField.Type(), numElems, numElems)
				for i := 0; i < numElems; i++ {
					if err := setWithProperType(sliceOf, inputValue[i], slice.Index(i)); err != nil {
						return err
					}
				}
				formStruct.Elem().Field(i).Set(slice)
			} else {
				if err := setWithProperType(typeField.Type.Kind(), inputValue[0], structField); err != nil {
					return err
				}
			}
		}
	}
	return nil
}

func setWithProperType(valueKind reflect.Kind, val string, structField reflect.Value) error {
	switch valueKind {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		if val == "" {
			val = "0"
		}
		intVal, err := strconv.Atoi(val)
		if err != nil {
			return err
		} else {
			structField.SetInt(int64(intVal))
		}
	case reflect.Bool:
		if val == "" {
			val = "false"
		}
		boolVal, err := strconv.ParseBool(val)
		if err != nil {
			return err
		} else {
			structField.SetBool(boolVal)
		}
	case reflect.Float32:
		if val == "" {
			val = "0.0"
		}
		floatVal, err := strconv.ParseFloat(val, 32)
		if err != nil {
			return err
		} else {
			structField.SetFloat(floatVal)
		}
	case reflect.Float64:
		if val == "" {
			val = "0.0"
		}
		floatVal, err := strconv.ParseFloat(val, 64)
		if err != nil {
			return err
		} else {
			structField.SetFloat(floatVal)
		}
	case reflect.String:
		structField.SetString(val)
	}
	return nil
}

// Don't pass in pointers to bind to. Can lead to bugs. See:
// https://github.com/codegangsta/martini-contrib/issues/40
// https://github.com/codegangsta/martini-contrib/pull/34#issuecomment-29683659
func ensureNotPointer(obj interface{}) {
	if reflect.TypeOf(obj).Kind() == reflect.Ptr {
		panic("Pointers are not accepted as binding models")
	}
}

func Validate(obj interface{}, parents ...string) error {
	typ := reflect.TypeOf(obj)
	val := reflect.ValueOf(obj)

	if typ.Kind() == reflect.Ptr {
		typ = typ.Elem()
		val = val.Elem()
	}

	switch typ.Kind() {
	case reflect.Struct:
		for i := 0; i < typ.NumField(); i++ {
			field := typ.Field(i)

			// Allow ignored fields in the struct
			if field.Tag.Get("form") == "-" {
				continue
			}

			fieldValue := val.Field(i).Interface()
			zero := reflect.Zero(field.Type).Interface()

			if strings.Index(field.Tag.Get("binding"), "required") > -1 {
				fieldType := field.Type.Kind()
				if fieldType == reflect.Struct {
					if reflect.DeepEqual(zero, fieldValue) {
						return errors.New("Required " + field.Name)
					}
					err := Validate(fieldValue, field.Name)
					if err != nil {
						return err
					}
				} else if reflect.DeepEqual(zero, fieldValue) {
					if len(parents) > 0 {
						return errors.New("Required " + field.Name + " on " + parents[0])
					} else {
						return errors.New("Required " + field.Name)
					}
				} else if fieldType == reflect.Slice && field.Type.Elem().Kind() == reflect.Struct {
					err := Validate(fieldValue)
					if err != nil {
						return err
					}
				}
			} else {
				fieldType := field.Type.Kind()
				if fieldType == reflect.Struct {
					if reflect.DeepEqual(zero, fieldValue) {
						continue
					}
					err := Validate(fieldValue, field.Name)
					if err != nil {
						return err
					}
				}
			}
		}
	case reflect.Slice:
		for i := 0; i < val.Len(); i++ {
			fieldValue := val.Index(i).Interface()
			err := Validate(fieldValue)
			if err != nil {
				return err
			}
		}
	default:
		return nil
	}
	return nil
}
<|endoftext|>


<commit_before>package goketo

import (
	"encoding/json"
	"fmt"
	"net/url"
	"strconv"
	"strings"

	"github.com/Sirupsen/logrus"
)

// Convert all to Interfaces for re-usability
// Add fmt.Sprintf("%v:%v", host, port) to build strings
// apiResponse common api response structure
type apiResponse struct {
	RequestID string `json:"requestId"`
	Success   bool   `json:"success"`
	Next      string `json:"nextPageToken,omitempty"`
	More      bool   `json:"moreResult,omitempty"`
	Errors    []struct {
		Code    string `json:"code"`
		Message string `json:"message"`
	} `json:"errors,omitempty"`
}

// LeadResponse response from list request
type LeadResponse struct {
	apiResponse
	Result json.RawMessage `json:"result,omitempty"`
}

// LeadResult default result struct as part of the lead - can be customized to allow greater fields
type LeadResult struct {
	ID        int    `json:"id"`
	FirstName string `json:"firstName"`
	LastName  string `json:"lastName"`
	Email     string `json:"email"`
	Created   string `json:"createdAt"`
	Updated   string `json:"updatedAt"`
}

// LeadRequest builds a request for data retrieval
type LeadRequest struct {
	ID     int    // List ID
	Next   string // Next page Token
	Fields string
}

// LeadUpdate builds the data for an update
type LeadUpdate struct {
	Action string          `json:"action"` // createOnly - updateOnly - createOrUpdate(default request) - createDuplicate
	Lookup string          `json:"lookupField"`
	Input  json.RawMessage `json:"input"`
}

// LeadUpdateResponse data format for update response
type LeadUpdateResponse struct {
	ID      string             `json:"requestId"`
	Success bool               `json:"success"`
	Result  []LeadUpdateResult `json:"result,omitempty"`
	Error   []LeadError        `json:"errors,omitempty"`
}

// LeadUpdateResult holds result for all updates
type LeadUpdateResult struct {
	ID     int    `json:"id"`
	Status string `json:"status"`
}

// LeadError shows the error code and message for response
type LeadError struct {
	Code    string `json:"code"`
	Message string `json:"message"`
}

// LeadFieldResponse response for all fields
type LeadFieldResponse struct {
	client *Client
	apiResponse
	Result []LeadField `json:"result"`
}

// LeadField describes all possible fields for Leads
type LeadField struct {
	ID     int    `json:"id"`
	Name   string `json:"displayName"`
	Type   string `json:"dataType"`
	Length int    `json:"length"`
	Rest   struct {
		Name     string `json:"name"`
		ReadOnly bool   `json:"readOnly"`
	} `json:"rest"`
	Soap struct {
		Name     string `json:"name"`
		ReadOnly bool   `json:"readOnly"`
	} `json:"soap"`
}

// DeletedLeadResponse response of Deleted lead request
type DeletedLeadResponse struct {
	*LeadResponse
	Result []DeletedLead `json:"result"`
}

// DeletedLead result
type DeletedLead struct {
	ID         int      `json:"id"`
	LeadID     int      `json:"leadId"`
	Date       string   `json:"activityDate"`
	TypeID     int      `json:"activityTypeId"`
	PrimaryID  int      `json:"primaryAttributeValueId"`
	PrimaryVal string   `json:"primaryAttributeValue"`
	Attributes []string `json:"attributes"`
}

// Leads Get leads by list Id
func Leads(req Requester, leadReq *LeadRequest) (leads *LeadResponse, err error) {
	urlQuery := url.Values{}
	if leadReq.Next != "" {
		urlQuery.Set("nextPageToken", leadReq.Next)
	}
	if len(leadReq.Fields) > 0 {
		urlQuery.Set("fields", strings.Join(strings.Fields(leadReq.Fields), ""))
	}
	url := fmt.Sprintf("list/%s/leads.json?%s", strconv.Itoa(leadReq.ID), urlQuery.Encode())
	logrus.Debug("Get: ", url)
	body, err := req.Get(url)
	if err != nil {
		return nil, err
	}
	logrus.Debug("Body: ", string(body))
	err = json.Unmarshal(body, &leads)
	return leads, err
}

// Lead Get lead by Id - aka member by ID
func Lead(req Requester, leadReq *LeadRequest) (lead *LeadResponse, err error) {
	urlQuery := url.Values{}
	if len(leadReq.Fields) > 0 {
		urlQuery.Set("fields", strings.Join(strings.Fields(leadReq.Fields), ""))
	}
	url := fmt.Sprintf("lead/%s.json?%s", strconv.Itoa(leadReq.ID), urlQuery.Encode())
	logrus.Debug("Get: ", url)
	body, err := req.Get(url)
	if err != nil {
		return
	}
	logrus.Debug("Body: ", string(body))
	err = json.Unmarshal(body, &lead)
	return lead, err
}

// UpdateLeads post update of data for a lead
func UpdateLeads(req Requester, update *LeadUpdate) ([]byte, error) {
	data, err := json.Marshal(update)
	if err != nil {
		return nil, err
	}
	body, err := req.Post("leads.json", data)
	return body, err
}

// LeadFields return all fields and the data type of a lead object
func LeadFields(req Requester) (fields *LeadFieldResponse, err error) {
	body, err := req.Get("leads/describe.json")
	err = json.Unmarshal(body, &fields)
	return fields, err
}

// DeletedLeads returns a list of leads that were deleted
func DeletedLeads(req Requester, leadReq *LeadRequest) (deletedLeads *DeletedLeadResponse, err error) {
	urlQuery := url.Values{}
	if leadReq.Next != "" {
		urlQuery.Set("&nextPageToken", leadReq.Next)
	}
	url := fmt.Sprintf("activities/deletedleads.json?%s", urlQuery.Encode())
	logrus.Debug("Get: ", url)
	body, err := req.Get(url)
	err = json.Unmarshal(body, &deletedLeads)
	return deletedLeads, err
}
<commit_msg>Added leads filter support<commit_after>package goketo

import (
	"encoding/json"
	"fmt"
	"net/url"
	"strconv"
	"strings"

	"github.com/Sirupsen/logrus"
)

// Convert all to Interfaces for re-usability
// Add fmt.Sprintf("%v:%v", host, port) to build strings
// apiResponse common api response structure
type apiResponse struct {
	RequestID string `json:"requestId"`
	Success   bool   `json:"success"`
	Next      string `json:"nextPageToken,omitempty"`
	More      bool   `json:"moreResult,omitempty"`
	Errors    []struct {
		Code    string `json:"code"`
		Message string `json:"message"`
	} `json:"errors,omitempty"`
}

// LeadResponse response from list request
type LeadResponse struct {
	apiResponse
	Result json.RawMessage `json:"result,omitempty"`
}

// LeadResult default result struct as part of the lead - can be customized to allow greater fields
type LeadResult struct {
	ID        int    `json:"id"`
	FirstName string `json:"firstName"`
	LastName  string `json:"lastName"`
	Email     string `json:"email"`
	Created   string `json:"createdAt"`
	Updated   string `json:"updatedAt"`
}

// LeadRequest builds a request for data retrieval
type LeadRequest struct {
	ID     int    // List ID
	Next   string // Next page Token
	Fields string
}

// LeadUpdate builds the data for an update
type LeadUpdate struct {
	Action string          `json:"action"` // createOnly - updateOnly - createOrUpdate(default request) - createDuplicate
	Lookup string          `json:"lookupField"`
	Input  json.RawMessage `json:"input"`
}

// LeadUpdateResponse data format for update response
type LeadUpdateResponse struct {
	ID      string             `json:"requestId"`
	Success bool               `json:"success"`
	Result  []LeadUpdateResult `json:"result,omitempty"`
	Error   []LeadError        `json:"errors,omitempty"`
}

// LeadUpdateResult holds result for all updates
type LeadUpdateResult struct {
	ID     int    `json:"id"`
	Status string `json:"status"`
}

// LeadError shows the error code and message for response
type LeadError struct {
	Code    string `json:"code"`
	Message string `json:"message"`
}

// LeadFieldResponse response for all fields
type LeadFieldResponse struct {
	client *Client
	apiResponse
	Result []LeadField `json:"result"`
}

// LeadField describes all possible fields for Leads
type LeadField struct {
	ID     int    `json:"id"`
	Name   string `json:"displayName"`
	Type   string `json:"dataType"`
	Length int    `json:"length"`
	Rest   struct {
		Name     string `json:"name"`
		ReadOnly bool   `json:"readOnly"`
	} `json:"rest"`
	Soap struct {
		Name     string `json:"name"`
		ReadOnly bool   `json:"readOnly"`
	} `json:"soap"`
}

// DeletedLeadResponse response of Deleted lead request
type DeletedLeadResponse struct {
	*LeadResponse
	Result []DeletedLead `json:"result"`
}

// DeletedLead result
type DeletedLead struct {
	ID         int      `json:"id"`
	LeadID     int      `json:"leadId"`
	Date       string   `json:"activityDate"`
	TypeID     int      `json:"activityTypeId"`
	PrimaryID  int      `json:"primaryAttributeValueId"`
	PrimaryVal string   `json:"primaryAttributeValue"`
	Attributes []string `json:"attributes"`
}

// Leads Get leads by list Id
func Leads(req Requester, leadReq *LeadRequest) (leads *LeadResponse, err error) {
	urlQuery := url.Values{}
	if leadReq.Next != "" {
		urlQuery.Set("nextPageToken", leadReq.Next)
	}
	if len(leadReq.Fields) > 0 {
		// join fields that are separate by space ' ' instead of ','
		urlQuery.Set("fields", strings.Join(strings.Fields(leadReq.Fields), ","))
	}
	url := fmt.Sprintf("list/%s/leads.json?%s", strconv.Itoa(leadReq.ID), urlQuery.Encode())
	logrus.Debug("Get: ", url)
	body, err := req.Get(url)
	if err != nil {
		return nil, err
	}
	logrus.Debug("Body: ", string(body))
	err = json.Unmarshal(body, &leads)
	return leads, err
}

// Lead Get lead by Id - aka member by ID
func Lead(req Requester, leadReq *LeadRequest) (lead *LeadResponse, err error) {
	urlQuery := url.Values{}
	if len(leadReq.Fields) > 0 {
		// join fields that are separate by space ' ' instead of ','
		urlQuery.Set("fields", strings.Join(strings.Fields(leadReq.Fields), ","))
	}
	url := fmt.Sprintf("lead/%s.json?%s", strconv.Itoa(leadReq.ID), urlQuery.Encode())
	logrus.Debug("Get: ", url)
	body, err := req.Get(url)
	if err != nil {
		return
	}
	logrus.Debug("Body: ", string(body))
	err = json.Unmarshal(body, &lead)
	return lead, err
}

// LeadsFilter Get leads by filter Type
// Common filter types:
//  - id
//  - cookies
//  - email
//  - twitterId
//  - facebookId
//  - linkedInId
//  - sfdcAccountId
//  - sfdcContactId
//  - sfdcLeadId
//  - sfdcLeadOwnerId
//  - sfdcOpptyId
func LeadsFilter(req Requester, leadReq *LeadRequest, filterType string, filterValues []string) (leads *LeadResponse, err error) {
	urlQuery := url.Values{}
	urlQuery.Set("filterType", filterType)
	if leadReq.Next != "" {
		urlQuery.Set("nextPageToken", leadReq.Next)
	}
	if len(leadReq.Fields) > 0 {
		// join fields that are separate by space ' ' instead of ','
		urlQuery.Set("fields", strings.Join(strings.Fields(leadReq.Fields), ","))
	}
	if len(filterValues) > 0 {
		urlQuery.Set("filterValues", strings.Join(filterValues, ","))
	}
	url := fmt.Sprintf("leads.json?%s", urlQuery.Encode())
	logrus.Debug("Get: ", url)
	body, err := req.Get(url)
	if err != nil {
		return nil, err
	}
	logrus.Debug("Body: ", string(body))
	err = json.Unmarshal(body, &leads)
	return leads, err
}

// UpdateLeads post update of data for a lead
func UpdateLeads(req Requester, update *LeadUpdate) ([]byte, error) {
	data, err := json.Marshal(update)
	if err != nil {
		return nil, err
	}
	body, err := req.Post("leads.json", data)
	return body, err
}

// LeadFields return all fields and the data type of a lead object
func LeadFields(req Requester) (fields *LeadFieldResponse, err error) {
	body, err := req.Get("leads/describe.json")
	err = json.Unmarshal(body, &fields)
	return fields, err
}

// DeletedLeads returns a list of leads that were deleted
func DeletedLeads(req Requester, leadReq *LeadRequest) (deletedLeads *DeletedLeadResponse, err error) {
	urlQuery := url.Values{}
	if leadReq.Next != "" {
		urlQuery.Set("&nextPageToken", leadReq.Next)
	}
	url := fmt.Sprintf("activities/deletedleads.json?%s", urlQuery.Encode())
	logrus.Debug("Get: ", url)
	body, err := req.Get(url)
	err = json.Unmarshal(body, &deletedLeads)
	return deletedLeads, err
}
<|endoftext|>


<commit_before>// Copyright 2014 Manu Martinez-Almeida.  All rights reserved.
// Use of this source code is governed by a MIT style
// license that can be found in the LICENSE file.

package binding

import (
	"encoding/json"
	"encoding/xml"
	"errors"
	"net/http"
	"reflect"
	"strconv"
	"strings"
)

type (
	Binding interface {
		Bind(*http.Request, interface{}) error
	}

	// JSON binding
	jsonBinding struct{}

	// XML binding
	xmlBinding struct{}

	// // form binding
	formBinding struct{}
)

var (
	JSON = jsonBinding{}
	XML  = xmlBinding{}
	Form = formBinding{} // todo
)

func (_ jsonBinding) Bind(req *http.Request, obj interface{}) error {
	decoder := json.NewDecoder(req.Body)
	if err := decoder.Decode(obj); err == nil {
		return Validate(obj)
	} else {
		return err
	}
}

func (_ xmlBinding) Bind(req *http.Request, obj interface{}) error {
	decoder := xml.NewDecoder(req.Body)
	if err := decoder.Decode(obj); err == nil {
		return Validate(obj)
	} else {
		return err
	}
}

func (_ formBinding) Bind(req *http.Request, obj interface{}) error {
	if err := req.ParseForm(); err != nil {
		return err
	}
	if err := mapForm(obj, req.Form); err != nil {
		return err
	}
	return Validate(obj)
}

func mapForm(ptr interface{}, form map[string][]string) error {
	typ := reflect.TypeOf(ptr).Elem()
	formStruct := reflect.ValueOf(ptr).Elem()
	for i := 0; i < typ.NumField(); i++ {
		typeField := typ.Field(i)
		if inputFieldName := typeField.Tag.Get("form"); inputFieldName != "" {
			structField := formStruct.Field(i)
			if !structField.CanSet() {
				continue
			}

			inputValue, exists := form[inputFieldName]
			if !exists {
				continue
			}
			numElems := len(inputValue)
			if structField.Kind() == reflect.Slice && numElems > 0 {
				sliceOf := structField.Type().Elem().Kind()
				slice := reflect.MakeSlice(structField.Type(), numElems, numElems)
				for i := 0; i < numElems; i++ {
					if err := setWithProperType(sliceOf, inputValue[i], slice.Index(i)); err != nil {
						return err
					}
				}
				formStruct.Field(i).Set(slice)
			} else {
				if err := setWithProperType(typeField.Type.Kind(), inputValue[0], structField); err != nil {
					return err
				}
			}
		}
	}
	return nil
}

func setWithProperType(valueKind reflect.Kind, val string, structField reflect.Value) error {
	switch valueKind {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		if val == "" {
			val = "0"
		}
		intVal, err := strconv.Atoi(val)
		if err != nil {
			return err
		} else {
			structField.SetInt(int64(intVal))
		}
	case reflect.Bool:
		if val == "" {
			val = "false"
		}
		boolVal, err := strconv.ParseBool(val)
		if err != nil {
			return err
		} else {
			structField.SetBool(boolVal)
		}
	case reflect.Float32:
		if val == "" {
			val = "0.0"
		}
		floatVal, err := strconv.ParseFloat(val, 32)
		if err != nil {
			return err
		} else {
			structField.SetFloat(floatVal)
		}
	case reflect.Float64:
		if val == "" {
			val = "0.0"
		}
		floatVal, err := strconv.ParseFloat(val, 64)
		if err != nil {
			return err
		} else {
			structField.SetFloat(floatVal)
		}
	case reflect.String:
		structField.SetString(val)
	}
	return nil
}

// Don't pass in pointers to bind to. Can lead to bugs. See:
// https://github.com/codegangsta/martini-contrib/issues/40
// https://github.com/codegangsta/martini-contrib/pull/34#issuecomment-29683659
func ensureNotPointer(obj interface{}) {
	if reflect.TypeOf(obj).Kind() == reflect.Ptr {
		panic("Pointers are not accepted as binding models")
	}
}

func Validate(obj interface{}) error {
	typ := reflect.TypeOf(obj)
	val := reflect.ValueOf(obj)

	if typ.Kind() == reflect.Ptr {
		typ = typ.Elem()
		val = val.Elem()
	}

	switch typ.Kind() {
	case reflect.Struct:
		for i := 0; i < typ.NumField(); i++ {
			field := typ.Field(i)

			// Allow ignored and unexported fields in the struct
			if field.Tag.Get("form") == "-" || field.PkgPath != "" {
				continue
			}

			fieldValue := val.Field(i).Interface()
			zero := reflect.Zero(field.Type).Interface()

			if strings.Index(field.Tag.Get("binding"), "required") > -1 {
				fieldType := field.Type.Kind()
				if fieldType == reflect.Struct {
					err := Validate(fieldValue)
					if err != nil {
						return err
					}
				} else if reflect.DeepEqual(zero, fieldValue) {
					return errors.New("Required " + field.Name)
				} else if fieldType == reflect.Slice && field.Type.Elem().Kind() == reflect.Struct {
					err := Validate(fieldValue)
					if err != nil {
						return err
					}
				}
			}
		}
	case reflect.Slice:
		for i := 0; i < val.Len(); i++ {
			fieldValue := val.Index(i).Interface()
			err := Validate(fieldValue)
			if err != nil {
				return err
			}
		}
	default:
		return nil
	}
	return nil
}
<commit_msg>Fix unexported field detection<commit_after>// Copyright 2014 Manu Martinez-Almeida.  All rights reserved.
// Use of this source code is governed by a MIT style
// license that can be found in the LICENSE file.

package binding

import (
	"encoding/json"
	"encoding/xml"
	"errors"
	"net/http"
	"reflect"
	"strconv"
	"strings"
)

type (
	Binding interface {
		Bind(*http.Request, interface{}) error
	}

	// JSON binding
	jsonBinding struct{}

	// XML binding
	xmlBinding struct{}

	// // form binding
	formBinding struct{}
)

var (
	JSON = jsonBinding{}
	XML  = xmlBinding{}
	Form = formBinding{} // todo
)

func (_ jsonBinding) Bind(req *http.Request, obj interface{}) error {
	decoder := json.NewDecoder(req.Body)
	if err := decoder.Decode(obj); err == nil {
		return Validate(obj)
	} else {
		return err
	}
}

func (_ xmlBinding) Bind(req *http.Request, obj interface{}) error {
	decoder := xml.NewDecoder(req.Body)
	if err := decoder.Decode(obj); err == nil {
		return Validate(obj)
	} else {
		return err
	}
}

func (_ formBinding) Bind(req *http.Request, obj interface{}) error {
	if err := req.ParseForm(); err != nil {
		return err
	}
	if err := mapForm(obj, req.Form); err != nil {
		return err
	}
	return Validate(obj)
}

func mapForm(ptr interface{}, form map[string][]string) error {
	typ := reflect.TypeOf(ptr).Elem()
	formStruct := reflect.ValueOf(ptr).Elem()
	for i := 0; i < typ.NumField(); i++ {
		typeField := typ.Field(i)
		if inputFieldName := typeField.Tag.Get("form"); inputFieldName != "" {
			structField := formStruct.Field(i)
			if !structField.CanSet() {
				continue
			}

			inputValue, exists := form[inputFieldName]
			if !exists {
				continue
			}
			numElems := len(inputValue)
			if structField.Kind() == reflect.Slice && numElems > 0 {
				sliceOf := structField.Type().Elem().Kind()
				slice := reflect.MakeSlice(structField.Type(), numElems, numElems)
				for i := 0; i < numElems; i++ {
					if err := setWithProperType(sliceOf, inputValue[i], slice.Index(i)); err != nil {
						return err
					}
				}
				formStruct.Field(i).Set(slice)
			} else {
				if err := setWithProperType(typeField.Type.Kind(), inputValue[0], structField); err != nil {
					return err
				}
			}
		}
	}
	return nil
}

func setWithProperType(valueKind reflect.Kind, val string, structField reflect.Value) error {
	switch valueKind {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		if val == "" {
			val = "0"
		}
		intVal, err := strconv.Atoi(val)
		if err != nil {
			return err
		} else {
			structField.SetInt(int64(intVal))
		}
	case reflect.Bool:
		if val == "" {
			val = "false"
		}
		boolVal, err := strconv.ParseBool(val)
		if err != nil {
			return err
		} else {
			structField.SetBool(boolVal)
		}
	case reflect.Float32:
		if val == "" {
			val = "0.0"
		}
		floatVal, err := strconv.ParseFloat(val, 32)
		if err != nil {
			return err
		} else {
			structField.SetFloat(floatVal)
		}
	case reflect.Float64:
		if val == "" {
			val = "0.0"
		}
		floatVal, err := strconv.ParseFloat(val, 64)
		if err != nil {
			return err
		} else {
			structField.SetFloat(floatVal)
		}
	case reflect.String:
		structField.SetString(val)
	}
	return nil
}

// Don't pass in pointers to bind to. Can lead to bugs. See:
// https://github.com/codegangsta/martini-contrib/issues/40
// https://github.com/codegangsta/martini-contrib/pull/34#issuecomment-29683659
func ensureNotPointer(obj interface{}) {
	if reflect.TypeOf(obj).Kind() == reflect.Ptr {
		panic("Pointers are not accepted as binding models")
	}
}

func Validate(obj interface{}) error {
	typ := reflect.TypeOf(obj)
	val := reflect.ValueOf(obj)

	if typ.Kind() == reflect.Ptr {
		typ = typ.Elem()
		val = val.Elem()
	}

	switch typ.Kind() {
	case reflect.Struct:
		for i := 0; i < typ.NumField(); i++ {
			field := typ.Field(i)

			// Allow ignored and unexported fields in the struct
			if len(field.PkgPath) > 0 || field.Tag.Get("form") == "-" {
				continue
			}

			fieldValue := val.Field(i).Interface()
			zero := reflect.Zero(field.Type).Interface()

			if strings.Index(field.Tag.Get("binding"), "required") > -1 {
				fieldType := field.Type.Kind()
				if fieldType == reflect.Struct {
					err := Validate(fieldValue)
					if err != nil {
						return err
					}
				} else if reflect.DeepEqual(zero, fieldValue) {
					return errors.New("Required " + field.Name)
				} else if fieldType == reflect.Slice && field.Type.Elem().Kind() == reflect.Struct {
					err := Validate(fieldValue)
					if err != nil {
						return err
					}
				}
			}
		}
	case reflect.Slice:
		for i := 0; i < val.Len(); i++ {
			fieldValue := val.Index(i).Interface()
			err := Validate(fieldValue)
			if err != nil {
				return err
			}
		}
	default:
		return nil
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import "fmt"

var cmdList = &Command{
	UsageLine: "list",
	Short:     "list all of the available gitignore templates",
	Long: `
Lists all of the available gitignore templates from
https://github.com/github/gitignore.

Example usage:

gitignorer list

  `,
}

func init() {
	cmdList.Run = runList
}

func runList(cmd *Command, args []string) {
	templates, _, err := client.Gitignores.List()
	if err != nil {
		fmt.Println(err)
		return
	}

	for _, t := range *templates {
		fmt.Println(t)
	}
}
<commit_msg>Remove extra spaces in long description for list command.<commit_after>package main

import "fmt"

var cmdList = &Command{
	UsageLine: "list",
	Short:     "list all of the available gitignore templates",
	Long: `
Lists all of the available gitignore templates from
https://github.com/github/gitignore.

Example usage:

gitignorer list

`,
}

func init() {
	cmdList.Run = runList
}

func runList(cmd *Command, args []string) {
	templates, _, err := client.Gitignores.List()
	if err != nil {
		fmt.Println(err)
		return
	}

	for _, t := range *templates {
		fmt.Println(t)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"strings"
)

var listCmd = &Command{
	Name:    "list",
	Summary: "List recognized repositories",
	Help: `The list command scans all available packages and collects information about
their repositories.  By default, each repository is listed along with its
dependencies and contained packages.

The -f option takes a template as a format.  The data passed into the
template invocation is an (rx/repo) RepoMap, and the default format is:

` + ind2sp(listTemplate),
}

var listFormat = listCmd.Flag.String("f", "", "List output format")

func listFunc(cmd *Command, args ...string) {
	switch len(args) {
	case 0:
		args = append(args, "all")
	case 1:
	default:
		cmd.BadArgs("too many arguments")
	}

	// Scan before accessing Repos
	if err := Scan(); err != nil {
		cmd.Fatalf("scan: %s", err)
	}

	if *listFormat != "" {
		render(stdout, *listFormat, Repos)
		return
	}

	render(stdout, listTemplate, Repos)
}

func init() {
	listCmd.Run = listFunc
}

var listTemplate = `{{range .}}Repository ({{.VCS}}) {{printf "%q" .Path}}:
	Dependencies:{{range .RepoDeps}}
		{{.}}{{end}}
	Packages:{{range .Packages}}
		{{.ImportPath}}{{end}}

{{end}}`

func ind2sp(s string) string {
	lines := strings.Split(s, "\n")
	for i := range lines {
		lines[i] = "  " + lines[i]
	}
	return strings.Join(lines, "\n")
}
<commit_msg>add --long output format<commit_after>package main

import (
	"strings"
)

var listCmd = &Command{
	Name:    "list",
	Summary: "List recognized repositories",
	Help: `The list command scans all available packages and collects information about
their repositories.  By default, each repository is listed along with its
dependencies and contained packages.

The -f option takes a template as a format.  The data passed into the
template invocation is an (rx/repo) RepoMap, and the default format is:

` + ind2sp(listTemplate) + `

If you specify --long, the format will be:

` + ind2sp(listTemplateLong),
}

var (
	listFormat = listCmd.Flag.String("f", "", "List output format")
	listLong = listCmd.Flag.Bool("long", false, "Use long output format")
)

func listFunc(cmd *Command, args ...string) {
	switch len(args) {
	case 0:
		args = append(args, "all")
	case 1:
	default:
		cmd.BadArgs("too many arguments")
	}

	// Scan before accessing Repos
	if err := Scan(); err != nil {
		cmd.Fatalf("scan: %s", err)
	}

	switch {
	case *listFormat != "":
		render(stdout, *listFormat, Repos)
	case *listLong:
		render(stdout, listTemplateLong, Repos)
	default:
		render(stdout, listTemplate, Repos)
	}
}

func init() {
	listCmd.Run = listFunc
}

var (
	listTemplate = `{{range .}}{{.Path}}:{{range .Packages}} {{.Name}}{{end}}
{{end}}`

	listTemplateLong =`{{range .}}Repository ({{.VCS}}) {{printf "%q" .Path}}:
	Dependencies:{{range .RepoDeps}}
		{{.}}{{end}}
	Packages:{{range .Packages}}
		{{.ImportPath}}{{end}}

{{end}}`
)

func ind2sp(s string) string {
	lines := strings.Split(s, "\n")
	for i := range lines {
		lines[i] = "  " + lines[i]
	}
	return strings.Join(lines, "\n")
}
<|endoftext|>


<commit_before>package jsh

import (
	"encoding/json"
	"fmt"
	"net/http"
)

// List is just a wrapper around an object array that implements Sendable
type List []*Object

/*
Validate ensures that List is JSON API compatible.
*/
func (list List) Validate(r *http.Request, response bool) *Error {
	for _, object := range list {
		err := object.Validate(r, response)
		if err != nil {
			return err
		}
	}

	return nil
}

/*
UnmarshalJSON allows us to manually decode a list via the json.Unmarshaler
interface.
*/
func (list *List) UnmarshalJSON(rawData []byte) error {
	// Create a sub-type here so when we call Unmarshal below, we don't recursively
	// call this function over and over
	type MarshalList List

	// if our "List" is a single object, modify the JSON to make it into a list
	// by wrapping with "[ ]"
	if rawData[0] == '{' {
		rawData = []byte(fmt.Sprintf("[%s]", rawData))
	}

	newList := MarshalList{}

	err := json.Unmarshal(rawData, &newList)
	if err != nil {
		return err
	}

	convertedList := List(newList)
	*list = convertedList

	return nil
}
<commit_msg>Supporting dynamic data response for single object<commit_after>package jsh

import (
	"encoding/json"
	"fmt"
	"net/http"
)

// List is just a wrapper around an object array that implements Sendable
type List []*Object

/*
Validate ensures that List is JSON API compatible.
*/
func (list List) Validate(r *http.Request, response bool) *Error {
	for _, object := range list {
		err := object.Validate(r, response)
		if err != nil {
			return err
		}
	}

	return nil
}

/*
UnmarshalJSON allows us to manually decode a list via the json.Unmarshaler
interface.
*/
func (list *List) UnmarshalJSON(rawData []byte) error {
	// Create a sub-type here so when we call Unmarshal below, we don't recursively
	// call this function over and over
	type UnmarshalList List

	// if our "List" is a single object, modify the JSON to make it into a list
	// by wrapping with "[ ]"
	if rawData[0] == '{' {
		rawData = []byte(fmt.Sprintf("[%s]", rawData))
	}

	newList := UnmarshalList{}

	err := json.Unmarshal(rawData, &newList)
	if err != nil {
		return err
	}

	convertedList := List(newList)
	*list = convertedList

	return nil
}

/*
MarshalJSON returns a top level object for the "data" attribute if a single object. In
all other cases returns a JSON encoded list for "data".
*/
func (list List) MarshalJSON() ([]byte, error) {
	// avoid stack overflow by using this subtype for marshaling
	type MarshalList List
	marshalList := MarshalList(list)
	count := len(marshalList)

	switch {
	case count == 1:
		return json.Marshal(marshalList[0])
	default:
		return json.Marshal(marshalList)
	}
}
<|endoftext|>


<commit_before>package gps

import "sort"

// Lock represents data from a lock file (or however the implementing tool
// chooses to store it) at a particular version that is relevant to the
// satisfiability solving process.
//
// In general, the information produced by gps on finding a successful
// solution is all that would be necessary to constitute a lock file, though
// tools can include whatever other information they want in their storage.
type Lock interface {
	// Indicates the version of the solver used to generate this lock data
	//SolverVersion() string

	// The hash of inputs to gps that resulted in this lock data
	InputHash() []byte

	// Projects returns the list of LockedProjects contained in the lock data.
	Projects() []LockedProject
}

// LockedProject is a single project entry from a lock file. It expresses the
// project's name, one or both of version and underlying revision, the network
// URI for accessing it, the path at which it should be placed within a vendor
// directory, and the packages that are used in it.
type LockedProject struct {
	pi   ProjectIdentifier
	v    UnpairedVersion
	r    Revision
	pkgs []string
}

// SimpleLock is a helper for tools to easily describe lock data when they know
// that no hash, or other complex information, is available.
type SimpleLock []LockedProject

var _ Lock = SimpleLock{}

// InputHash always returns an empty string for SimpleLock. This makes it useless
// as a stable lock to be written to disk, but still useful for some ephemeral
// purposes.
func (SimpleLock) InputHash() []byte {
	return nil
}

// Projects returns the entire contents of the SimpleLock.
func (l SimpleLock) Projects() []LockedProject {
	return l
}

// NewLockedProject creates a new LockedProject struct with a given
// ProjectIdentifier (name and optional upstream source URL), version. and list
// of packages required from the project.
//
// Note that passing a nil version will cause a panic. This is a correctness
// measure to ensure that the solver is never exposed to a version-less lock
// entry. Such a case would be meaningless - the solver would have no choice but
// to simply dismiss that project. By creating a hard failure case via panic
// instead, we are trying to avoid inflicting the resulting pain on the user by
// instead forcing a decision on the Analyzer implementation.
func NewLockedProject(id ProjectIdentifier, v Version, pkgs []string) LockedProject {
	if v == nil {
		panic("must provide a non-nil version to create a LockedProject")
	}

	lp := LockedProject{
		pi:   id,
		pkgs: pkgs,
	}

	switch tv := v.(type) {
	case Revision:
		lp.r = tv
	case branchVersion:
		lp.v = tv
	case semVersion:
		lp.v = tv
	case plainVersion:
		lp.v = tv
	case versionPair:
		lp.r = tv.r
		lp.v = tv.v
	}

	return lp
}

// Ident returns the identifier describing the project. This includes both the
// local name (the root name by which the project is referenced in import paths)
// and the network name, where the upstream source lives.
func (lp LockedProject) Ident() ProjectIdentifier {
	return lp.pi
}

// Version assembles together whatever version and/or revision data is
// available into a single Version.
func (lp LockedProject) Version() Version {
	if lp.r == "" {
		return lp.v
	}

	if lp.v == nil {
		return lp.r
	}

	return lp.v.Is(lp.r)
}

func (lp LockedProject) toAtom() atomWithPackages {
	pa := atom{
		id: lp.Ident(),
	}

	if lp.v == nil {
		pa.v = lp.r
	} else if lp.r != "" {
		pa.v = lp.v.Is(lp.r)
	} else {
		pa.v = lp.v
	}

	return atomWithPackages{
		a:  pa,
		pl: lp.pkgs,
	}
}

type safeLock struct {
	h []byte
	p []LockedProject
}

func (sl safeLock) InputHash() []byte {
	return sl.h
}

func (sl safeLock) Projects() []LockedProject {
	return sl.p
}

// prepLock ensures a lock is prepared and safe for use by the solver. This is
// mostly about defensively ensuring that no outside routine can modify the lock
// while the solver is in-flight.
//
// This is achieved by copying the lock's data into a new safeLock.
func prepLock(l Lock) Lock {
	pl := l.Projects()

	rl := safeLock{
		h: l.InputHash(),
		p: make([]LockedProject, len(pl)),
	}
	copy(rl.p, pl)

	return rl
}

// SortLockedProjects sorts a slice of LockedProject in alphabetical order by
// ProjectRoot.
func SortLockedProjects(lps []LockedProject) {
	sort.Stable(lpsorter(lps))
}

type lpsorter []LockedProject

func (lps lpsorter) Swap(i, j int) {
	lps[i], lps[j] = lps[j], lps[i]
}

func (lps lpsorter) Len() int {
	return len(lps)
}

func (lps lpsorter) Less(i, j int) bool {
	return lps[i].pi.ProjectRoot < lps[j].pi.ProjectRoot
}
<commit_msg>Add method to return packages from LockedProject<commit_after>package gps

import "sort"

// Lock represents data from a lock file (or however the implementing tool
// chooses to store it) at a particular version that is relevant to the
// satisfiability solving process.
//
// In general, the information produced by gps on finding a successful
// solution is all that would be necessary to constitute a lock file, though
// tools can include whatever other information they want in their storage.
type Lock interface {
	// Indicates the version of the solver used to generate this lock data
	//SolverVersion() string

	// The hash of inputs to gps that resulted in this lock data
	InputHash() []byte

	// Projects returns the list of LockedProjects contained in the lock data.
	Projects() []LockedProject
}

// LockedProject is a single project entry from a lock file. It expresses the
// project's name, one or both of version and underlying revision, the network
// URI for accessing it, the path at which it should be placed within a vendor
// directory, and the packages that are used in it.
type LockedProject struct {
	pi   ProjectIdentifier
	v    UnpairedVersion
	r    Revision
	pkgs []string
}

// SimpleLock is a helper for tools to easily describe lock data when they know
// that no hash, or other complex information, is available.
type SimpleLock []LockedProject

var _ Lock = SimpleLock{}

// InputHash always returns an empty string for SimpleLock. This makes it useless
// as a stable lock to be written to disk, but still useful for some ephemeral
// purposes.
func (SimpleLock) InputHash() []byte {
	return nil
}

// Projects returns the entire contents of the SimpleLock.
func (l SimpleLock) Projects() []LockedProject {
	return l
}

// NewLockedProject creates a new LockedProject struct with a given
// ProjectIdentifier (name and optional upstream source URL), version. and list
// of packages required from the project.
//
// Note that passing a nil version will cause a panic. This is a correctness
// measure to ensure that the solver is never exposed to a version-less lock
// entry. Such a case would be meaningless - the solver would have no choice but
// to simply dismiss that project. By creating a hard failure case via panic
// instead, we are trying to avoid inflicting the resulting pain on the user by
// instead forcing a decision on the Analyzer implementation.
func NewLockedProject(id ProjectIdentifier, v Version, pkgs []string) LockedProject {
	if v == nil {
		panic("must provide a non-nil version to create a LockedProject")
	}

	lp := LockedProject{
		pi:   id,
		pkgs: pkgs,
	}

	switch tv := v.(type) {
	case Revision:
		lp.r = tv
	case branchVersion:
		lp.v = tv
	case semVersion:
		lp.v = tv
	case plainVersion:
		lp.v = tv
	case versionPair:
		lp.r = tv.r
		lp.v = tv.v
	}

	return lp
}

// Ident returns the identifier describing the project. This includes both the
// local name (the root name by which the project is referenced in import paths)
// and the network name, where the upstream source lives.
func (lp LockedProject) Ident() ProjectIdentifier {
	return lp.pi
}

// Version assembles together whatever version and/or revision data is
// available into a single Version.
func (lp LockedProject) Version() Version {
	if lp.r == "" {
		return lp.v
	}

	if lp.v == nil {
		return lp.r
	}

	return lp.v.Is(lp.r)
}

// Packages returns the list of packages from within the LockedProject that are
// actually used in the import graph. Some caveats:
//
//  * The names given are relative to the root import path for the project. If
//    the root package itself is imported, it's represented as ".".
//  * Just because a package path isn't included in this list doesn't mean it's
//    safe to remove - it could contain C files, or other assets, that can't be
//    safely removed.
//  * The slice is not a copy. If you need to modify it, copy it first.
func (lp LockedProject) Packages() []string {
	return lp.pkgs
}

func (lp LockedProject) toAtom() atomWithPackages {
	pa := atom{
		id: lp.Ident(),
	}

	if lp.v == nil {
		pa.v = lp.r
	} else if lp.r != "" {
		pa.v = lp.v.Is(lp.r)
	} else {
		pa.v = lp.v
	}

	return atomWithPackages{
		a:  pa,
		pl: lp.pkgs,
	}
}

type safeLock struct {
	h []byte
	p []LockedProject
}

func (sl safeLock) InputHash() []byte {
	return sl.h
}

func (sl safeLock) Projects() []LockedProject {
	return sl.p
}

// prepLock ensures a lock is prepared and safe for use by the solver. This is
// mostly about defensively ensuring that no outside routine can modify the lock
// while the solver is in-flight.
//
// This is achieved by copying the lock's data into a new safeLock.
func prepLock(l Lock) Lock {
	pl := l.Projects()

	rl := safeLock{
		h: l.InputHash(),
		p: make([]LockedProject, len(pl)),
	}
	copy(rl.p, pl)

	return rl
}

// SortLockedProjects sorts a slice of LockedProject in alphabetical order by
// ProjectRoot.
func SortLockedProjects(lps []LockedProject) {
	sort.Stable(lpsorter(lps))
}

type lpsorter []LockedProject

func (lps lpsorter) Swap(i, j int) {
	lps[i], lps[j] = lps[j], lps[i]
}

func (lps lpsorter) Len() int {
	return len(lps)
}

func (lps lpsorter) Less(i, j int) bool {
	return lps[i].pi.ProjectRoot < lps[j].pi.ProjectRoot
}
<|endoftext|>


<commit_before>package sync

import (
	"fmt"
	"log"
	"time"

	"github.com/svdberg/syncmysport-runkeeper/Godeps/_workspace/src/github.com/newrelic/go-agent"
	dm "github.com/svdberg/syncmysport-runkeeper/datamodel"
	rk "github.com/svdberg/syncmysport-runkeeper/runkeeper"
	stv "github.com/svdberg/syncmysport-runkeeper/strava"
)

type Syncer interface {
	Sync() (int, int, error)
}

/*
 * Returns the set of activities that are in RunKeeper, but not in Strava.
 * So if the set of Runkeeper activites is A, and the set of Strava activities is B,
 * this function calculates B\A.
 */
func CalculateRKDifference(rkActivities dm.ActivitySet, stvActivities dm.ActivitySet) *dm.ActivitySet {
	return rkActivities.ApproxSubtract(stvActivities)
}

type SyncTask struct {
	StravaToken       string `json:"stv_token"`
	RunkeeperToken    string `json:"rk_token"`
	LastSeenTimestamp int    `json:"last_seen_ts"`
	Uid               int64  `json:"id"`
	Environment       string `json:"environment"`
}

func CreateSyncTask(rkToken string, stvToken string, lastSeenTS int, environment string) *SyncTask {
	return &SyncTask{stvToken, rkToken, lastSeenTS, -1, environment}
}

/*
 * return the Total difference and the number of Activites created
 */
func (st SyncTask) Sync(stvClient stv.StravaClientInt, rkClient rk.RunkeeperCientInt, txn newrelic.Transaction) (int, int, error) {
	//get activities from strava
	//normalize time to the start of the day, because Runkeeper only supports days as offset, not timestamps
	tsOfStartOfDay := calculateTsAtStartOfDay(st.LastSeenTimestamp)
	segment := newrelic.Segment{}
	segment.Name = "strava-retrieve-activities"
	segment.StartTime = newrelic.StartSegmentNow(txn)
	activities, err := stvClient.GetSTVActivitiesSince(tsOfStartOfDay)
	segment.End()
	if err != nil {
		log.Printf("Error: %e while retrieving Strava activitites since %s, aborting this run", err, time.Unix(int64(st.LastSeenTimestamp), 0))
		return 0, 0, err
	}
	stvDetailedActivities := dm.NewActivitySet()
	for _, actSummary := range activities {
		//get Detailed Actv
		detailedAct, _ := stvClient.GetSTVDetailedActivity(actSummary.Id)

		//get associated Streams
		timeStream, err := stvClient.GetSTVActivityStream(actSummary.Id, "Time")
		if err != nil {
			log.Printf("Error while retrieving time series from Strava: %s", err)
			return 0, 0, err
		}

		//Optional streams
		// var locStream, hrStream, altStream *stv.StreamSet
		locStream, err := stvClient.GetSTVActivityStream(actSummary.Id, "GPS")
		if err != nil {
			log.Printf("Error while reading lat/long for activity id %d: %e", actSummary.Id, err)
		}
		hrStream, err := stvClient.GetSTVActivityStream(actSummary.Id, "Heartrate")
		if err != nil {
			log.Printf("Error while reading Heartrate for activity id %d: %e", actSummary.Id, err)
		}
		altStream, err := stvClient.GetSTVActivityStream(actSummary.Id, "Altitude")
		if err != nil {
			log.Printf("Error while reading Altitude for activity id %d: %e", actSummary.Id, err)
		}

		stvDetailedActivities.Add(*stv.ConvertToActivity(detailedAct, timeStream, locStream, hrStream, altStream))
	}
	log.Printf("Got %d items from Strava", stvDetailedActivities.NumElements())
	for i := 0; i < stvDetailedActivities.NumElements(); i++ {
		log.Printf("Strava Activity: %s", stvDetailedActivities.Get(i))
	}
	//stvDetailedActivities should be in UTC

	//get activities from runkeeper
	segment = newrelic.Segment{}
	segment.Name = "runkeeper-retrieve-activities"
	segment.StartTime = newrelic.StartSegmentNow(txn)
	rkActivitiesOverview, err := rkClient.GetRKActivitiesSince(st.LastSeenTimestamp)
	rkDetailActivities := rkClient.EnrichRKActivities(rkActivitiesOverview)
	segment.End()
	//log.Printf("rk detail activities: %s", rkDetailActivities)

	rkActivities := dm.NewActivitySet()
	for _, item := range rkDetailActivities {
		rkActivities.Add(*rk.ConvertToActivity(&item))
	}
	if err != nil {
		log.Printf("%s", err)
	}
	log.Printf("Got %d items from RunKeeper", rkActivities.NumElements())
	// rkActivities are in UTC (or should be)

	for i := 0; i < rkActivities.NumElements(); i++ {
		log.Printf("Runkeeper Activity: %s", rkActivities.Get(i))
	}

	//caclulate difference
	itemsToSyncToRk := stvDetailedActivities.ApproxSubtract(rkActivities)
	log.Printf("Difference between Runkeeper and Strava is %d items", itemsToSyncToRk.NumElements())

	//write to runkeeper
	segment = newrelic.Segment{}
	segment.Name = "runkeeper-write-activities"
	segment.StartTime = newrelic.StartSegmentNow(txn)
	totalItemsCreated := 0
	for i := 0; i < itemsToSyncToRk.NumElements(); i++ {
		log.Printf("Now storing item %s to RunKeeper", itemsToSyncToRk.Get(i))
		var (
			uri string
			err error
		)
		if st.Environment == "Prod" {
			uri, err = rkClient.PostActivity(rk.ConvertToRkActivity(itemsToSyncToRk.Get(i)))
		} else {
			log.Print("Assuming DEBUG/TEST mode, not actually writing to Runkeeper")
			err = nil
			uri = "fake_uri"
		}

		if err != nil {
			log.Printf("Something failed during the write to Runkeeper: %s", err)
			return itemsToSyncToRk.NumElements(), totalItemsCreated, err
		}
		if uri != "" {
			log.Printf("URI of activity: %s", uri)
			totalItemsCreated++
		}
	}
	segment.End()
	return itemsToSyncToRk.NumElements(), totalItemsCreated, nil
}

func calculateTsAtStartOfDay(timestamp int) int {
	timeAtTimestamp := time.Unix(int64(timestamp), 0).UTC()
	year, month, day := timeAtTimestamp.Date()
	//	Mon Jan 2 15:04:05 -0700 MST 2006
	ts, err := time.Parse("2006-1-2 15:04:05 MST", fmt.Sprintf("%d-%d-%d 00:00:00 UTC", year, month, day))
	if err != nil {
		log.Printf("Error parsing time from y/m/d to ts of current day: %s", err)
	}
	ts = ts.Add(time.Duration(1) * time.Minute)
	return int(ts.Unix())
}
<commit_msg>Added logging to trace GPS data missing<commit_after>package sync

import (
	"fmt"
	"log"
	"time"

	"github.com/svdberg/syncmysport-runkeeper/Godeps/_workspace/src/github.com/newrelic/go-agent"
	dm "github.com/svdberg/syncmysport-runkeeper/datamodel"
	rk "github.com/svdberg/syncmysport-runkeeper/runkeeper"
	stv "github.com/svdberg/syncmysport-runkeeper/strava"
)

type Syncer interface {
	Sync() (int, int, error)
}

/*
 * Returns the set of activities that are in RunKeeper, but not in Strava.
 * So if the set of Runkeeper activites is A, and the set of Strava activities is B,
 * this function calculates B\A.
 */
func CalculateRKDifference(rkActivities dm.ActivitySet, stvActivities dm.ActivitySet) *dm.ActivitySet {
	return rkActivities.ApproxSubtract(stvActivities)
}

type SyncTask struct {
	StravaToken       string `json:"stv_token"`
	RunkeeperToken    string `json:"rk_token"`
	LastSeenTimestamp int    `json:"last_seen_ts"`
	Uid               int64  `json:"id"`
	Environment       string `json:"environment"`
}

func CreateSyncTask(rkToken string, stvToken string, lastSeenTS int, environment string) *SyncTask {
	return &SyncTask{stvToken, rkToken, lastSeenTS, -1, environment}
}

/*
 * return the Total difference and the number of Activites created
 */
func (st SyncTask) Sync(stvClient stv.StravaClientInt, rkClient rk.RunkeeperCientInt, txn newrelic.Transaction) (int, int, error) {
	//get activities from strava
	//normalize time to the start of the day, because Runkeeper only supports days as offset, not timestamps
	tsOfStartOfDay := calculateTsAtStartOfDay(st.LastSeenTimestamp)
	segment := newrelic.Segment{}
	segment.Name = "strava-retrieve-activities"
	segment.StartTime = newrelic.StartSegmentNow(txn)
	activities, err := stvClient.GetSTVActivitiesSince(tsOfStartOfDay)
	segment.End()
	if err != nil {
		log.Printf("Error: %e while retrieving Strava activitites since %s, aborting this run", err, time.Unix(int64(st.LastSeenTimestamp), 0))
		return 0, 0, err
	}
	stvDetailedActivities := dm.NewActivitySet()
	for _, actSummary := range activities {
		//get Detailed Actv
		detailedAct, _ := stvClient.GetSTVDetailedActivity(actSummary.Id)

		//get associated Streams
		timeStream, err := stvClient.GetSTVActivityStream(actSummary.Id, "Time")
		if err != nil {
			log.Printf("Error while retrieving time series from Strava: %s", err)
			return 0, 0, err
		}
		log.Printf("timeStream: %s", timeStream)

		//Optional streams
		// var locStream, hrStream, altStream *stv.StreamSet
		locStream, err := stvClient.GetSTVActivityStream(actSummary.Id, "GPS")
		if err != nil {
			log.Printf("Error while reading lat/long for activity id %d: %e", actSummary.Id, err)
		}
		log.Printf("locStream: %s", locStream)
		hrStream, err := stvClient.GetSTVActivityStream(actSummary.Id, "Heartrate")
		if err != nil {
			log.Printf("Error while reading Heartrate for activity id %d: %e", actSummary.Id, err)
		}

		altStream, err := stvClient.GetSTVActivityStream(actSummary.Id, "Altitude")
		if err != nil {
			log.Printf("Error while reading Altitude for activity id %d: %e", actSummary.Id, err)
		}

		stvDetailedActivities.Add(*stv.ConvertToActivity(detailedAct, timeStream, locStream, hrStream, altStream))
	}
	log.Printf("Got %d items from Strava", stvDetailedActivities.NumElements())
	for i := 0; i < stvDetailedActivities.NumElements(); i++ {
		log.Printf("Strava Activity: %s", stvDetailedActivities.Get(i))
	}
	//stvDetailedActivities should be in UTC

	//get activities from runkeeper
	segment = newrelic.Segment{}
	segment.Name = "runkeeper-retrieve-activities"
	segment.StartTime = newrelic.StartSegmentNow(txn)
	rkActivitiesOverview, err := rkClient.GetRKActivitiesSince(st.LastSeenTimestamp)
	rkDetailActivities := rkClient.EnrichRKActivities(rkActivitiesOverview)
	segment.End()
	//log.Printf("rk detail activities: %s", rkDetailActivities)

	rkActivities := dm.NewActivitySet()
	for _, item := range rkDetailActivities {
		rkActivities.Add(*rk.ConvertToActivity(&item))
	}
	if err != nil {
		log.Printf("%s", err)
	}
	log.Printf("Got %d items from RunKeeper", rkActivities.NumElements())
	// rkActivities are in UTC (or should be)

	for i := 0; i < rkActivities.NumElements(); i++ {
		log.Printf("Runkeeper Activity: %s", rkActivities.Get(i))
	}

	//caclulate difference
	itemsToSyncToRk := stvDetailedActivities.ApproxSubtract(rkActivities)
	log.Printf("Difference between Runkeeper and Strava is %d items", itemsToSyncToRk.NumElements())

	//write to runkeeper
	segment = newrelic.Segment{}
	segment.Name = "runkeeper-write-activities"
	segment.StartTime = newrelic.StartSegmentNow(txn)
	totalItemsCreated := 0
	for i := 0; i < itemsToSyncToRk.NumElements(); i++ {
		log.Printf("Now storing item %s to RunKeeper", itemsToSyncToRk.Get(i))
		var (
			uri string
			err error
		)
		if st.Environment == "Prod" {
			uri, err = rkClient.PostActivity(rk.ConvertToRkActivity(itemsToSyncToRk.Get(i)))
		} else {
			log.Print("Assuming DEBUG/TEST mode, not actually writing to Runkeeper")
			err = nil
			uri = "fake_uri"
		}

		if err != nil {
			log.Printf("Something failed during the write to Runkeeper: %s", err)
			return itemsToSyncToRk.NumElements(), totalItemsCreated, err
		}
		if uri != "" {
			log.Printf("URI of activity: %s", uri)
			totalItemsCreated++
		}
	}
	segment.End()
	return itemsToSyncToRk.NumElements(), totalItemsCreated, nil
}

func calculateTsAtStartOfDay(timestamp int) int {
	timeAtTimestamp := time.Unix(int64(timestamp), 0).UTC()
	year, month, day := timeAtTimestamp.Date()
	//	Mon Jan 2 15:04:05 -0700 MST 2006
	ts, err := time.Parse("2006-1-2 15:04:05 MST", fmt.Sprintf("%d-%d-%d 00:00:00 UTC", year, month, day))
	if err != nil {
		log.Printf("Error parsing time from y/m/d to ts of current day: %s", err)
	}
	ts = ts.Add(time.Duration(1) * time.Minute)
	return int(ts.Unix())
}
<|endoftext|>


<commit_before>package goselenium

import "time"

type waitResponse struct {
	s bool
	e error
}

// Until represents a function that will be continuously repeated until it
// succeeds or a timeout is reached.
type Until func(w WebDriver) (bool, error)

// UntilElementPresent attempts to locate an element on the page. It is
// determined as existing if the state is 'Success' and the error is nil.
func UntilElementPresent(by By, sleep time.Duration) Until {
	return func(w WebDriver) (bool, error) {
		time.Sleep(sleep)

		el, err := w.FindElement(by)
		return err == nil && el != nil, err
	}
}

func (s *seleniumWebDriver) Wait(u Until, timeout time.Duration) (bool, error) {
	response := make(chan *waitResponse, 1)

	// Loops continuously until the timeout is reached or the result is a
	// success.
	go func() {
		for {
			s, e := u(s)
			if e == nil && s {
				response <- &waitResponse{s: s, e: e}
			}
		}
	}()

	select {
	case r := <-response:
		return r.s, r.e
	case <-time.After(timeout):
		return false, WaitTimeoutError
	}
}
<commit_msg>Remove useless comment<commit_after>package goselenium

import "time"

type waitResponse struct {
	s bool
	e error
}

// Until represents a function that will be continuously repeated until it
// succeeds or a timeout is reached.
type Until func(w WebDriver) (bool, error)

// UntilElementPresent attempts to locate an element on the page. It is
// determined as existing if the state is 'Success' and the error is nil.
func UntilElementPresent(by By, sleep time.Duration) Until {
	return func(w WebDriver) (bool, error) {
		time.Sleep(sleep)

		el, err := w.FindElement(by)
		return err == nil && el != nil, err
	}
}

func (s *seleniumWebDriver) Wait(u Until, timeout time.Duration) (bool, error) {
	response := make(chan *waitResponse, 1)

	go func() {
		for {
			s, e := u(s)
			if e == nil && s {
				response <- &waitResponse{s: s, e: e}
			}
		}
	}()

	select {
	case r := <-response:
		return r.s, r.e
	case <-time.After(timeout):
		return false, WaitTimeoutError
	}
}
<|endoftext|>


<commit_before>package tsdb

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"

	"github.com/influxdb/influxdb/influxql"
)

func NewStore(path string) *Store {
	opts := NewEngineOptions()
	opts.Config = NewConfig()

	return &Store{
		path:          path,
		EngineOptions: opts,
		Logger:        log.New(os.Stderr, "[store] ", log.LstdFlags),
	}
}

var (
	ErrShardNotFound = fmt.Errorf("shard not found")
)

type Store struct {
	mu   sync.RWMutex
	path string

	databaseIndexes map[string]*DatabaseIndex
	shards          map[uint64]*Shard

	EngineOptions EngineOptions
	Logger        *log.Logger
	closing       chan struct{}
}

// Path returns the store's root path.
func (s *Store) Path() string { return s.path }

// DatabaseIndexN returns the number of databases indicies in the store.
func (s *Store) DatabaseIndexN() int {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return len(s.databaseIndexes)
}

// Shard returns a shard by id.
func (s *Store) Shard(id uint64) *Shard {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.shards[id]
}

// ShardN returns the number of shard in the store.
func (s *Store) ShardN() int {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return len(s.shards)
}

func (s *Store) CreateShard(database, retentionPolicy string, shardID uint64) error {
	s.mu.Lock()
	defer s.mu.Unlock()

	select {
	case <-s.closing:
		return fmt.Errorf("closing")
	default:
	}

	// shard already exists
	if _, ok := s.shards[shardID]; ok {
		return nil
	}

	// created the db and retention policy dirs if they don't exist
	if err := os.MkdirAll(filepath.Join(s.path, database, retentionPolicy), 0700); err != nil {
		return err
	}

	// create the WAL directory
	walPath := filepath.Join(s.EngineOptions.Config.WALDir, database, retentionPolicy, fmt.Sprintf("%d", shardID))
	if err := os.MkdirAll(walPath, 0700); err != nil {
		return err
	}

	// create the database index if it does not exist
	db, ok := s.databaseIndexes[database]
	if !ok {
		db = NewDatabaseIndex()
		s.databaseIndexes[database] = db
	}

	shardPath := filepath.Join(s.path, database, retentionPolicy, strconv.FormatUint(shardID, 10))
	shard := NewShard(shardID, db, shardPath, walPath, s.EngineOptions)
	if err := shard.Open(); err != nil {
		return err
	}

	s.shards[shardID] = shard

	return nil
}

// DeleteShard removes a shard from disk.
func (s *Store) DeleteShard(shardID uint64) error {
	s.mu.Lock()
	defer s.mu.Unlock()

	// ensure shard exists
	sh, ok := s.shards[shardID]
	if !ok {
		return nil
	}

	if err := sh.Close(); err != nil {
		return err
	}

	if err := os.Remove(sh.path); err != nil {
		return err
	}

	if err := os.RemoveAll(sh.walPath); err != nil {
		return err
	}

	delete(s.shards, shardID)

	return nil
}

// DeleteDatabase will close all shards associated with a database and remove the directory and files from disk.
func (s *Store) DeleteDatabase(name string, shardIDs []uint64) error {
	s.mu.Lock()
	defer s.mu.Unlock()
	for _, id := range shardIDs {
		shard := s.shards[id]
		if shard != nil {
			shard.Close()
		}
	}
	if err := os.RemoveAll(filepath.Join(s.path, name)); err != nil {
		return err
	}
	if err := os.RemoveAll(filepath.Join(s.EngineOptions.Config.WALDir, name)); err != nil {
		return err
	}
	delete(s.databaseIndexes, name)
	return nil
}

// ShardIDs returns a slice of all ShardIDs under management.
func (s *Store) ShardIDs() []uint64 {
	ids := make([]uint64, 0, len(s.shards))
	for i, _ := range s.shards {
		ids = append(ids, i)
	}
	return ids
}

func (s *Store) ValidateAggregateFieldsInStatement(shardID uint64, measurementName string, stmt *influxql.SelectStatement) error {
	s.mu.RLock()
	shard := s.shards[shardID]
	s.mu.RUnlock()
	if shard == nil {
		return ErrShardNotFound
	}
	return shard.ValidateAggregateFieldsInStatement(measurementName, stmt)
}

func (s *Store) DatabaseIndex(name string) *DatabaseIndex {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.databaseIndexes[name]
}

// Databases returns all the databases in the indexes
func (s *Store) Databases() []string {
	s.mu.RLock()
	defer s.mu.RUnlock()
	databases := []string{}
	for db := range s.databaseIndexes {
		databases = append(databases, db)
	}
	return databases
}

func (s *Store) Measurement(database, name string) *Measurement {
	s.mu.RLock()
	db := s.databaseIndexes[database]
	s.mu.RUnlock()
	if db == nil {
		return nil
	}
	return db.Measurement(name)
}

// DiskSize returns the size of all the shard files in bytes.  This size does not include the WAL size.
func (s *Store) DiskSize() (int64, error) {
	s.mu.RLock()
	defer s.mu.RUnlock()
	var size int64
	for _, shardID := range s.ShardIDs() {
		shard := s.Shard(shardID)
		sz, err := shard.DiskSize()
		if err != nil {
			return 0, err
		}
		size += sz
	}
	return size, nil
}

// deleteSeries loops through the local shards and deletes the series data and metadata for the passed in series keys
func (s *Store) deleteSeries(keys []string) error {
	s.mu.RLock()
	defer s.mu.RUnlock()
	for _, sh := range s.shards {
		if err := sh.DeleteSeries(keys); err != nil {
			return err
		}
	}
	return nil
}

// deleteMeasurement loops through the local shards and removes the measurement field encodings from each shard
func (s *Store) deleteMeasurement(name string, seriesKeys []string) error {
	s.mu.RLock()
	defer s.mu.RUnlock()
	for _, sh := range s.shards {
		if err := sh.DeleteMeasurement(name, seriesKeys); err != nil {
			return err
		}
	}
	return nil
}

func (s *Store) loadIndexes() error {
	dbs, err := ioutil.ReadDir(s.path)
	if err != nil {
		return err
	}
	for _, db := range dbs {
		if !db.IsDir() {
			s.Logger.Printf("Skipping database dir: %s. Not a directory", db.Name())
			continue
		}
		s.databaseIndexes[db.Name()] = NewDatabaseIndex()
	}
	return nil
}

func (s *Store) loadShards() error {
	// loop through the current database indexes
	for db := range s.databaseIndexes {
		rps, err := ioutil.ReadDir(filepath.Join(s.path, db))
		if err != nil {
			return err
		}

		for _, rp := range rps {
			// retention policies should be directories.  Skip anything that is not a dir.
			if !rp.IsDir() {
				s.Logger.Printf("Skipping retention policy dir: %s. Not a directory", rp.Name())
				continue
			}

			shards, err := ioutil.ReadDir(filepath.Join(s.path, db, rp.Name()))
			if err != nil {
				return err
			}
			for _, sh := range shards {
				path := filepath.Join(s.path, db, rp.Name(), sh.Name())
				walPath := filepath.Join(s.EngineOptions.Config.WALDir, db, rp.Name(), sh.Name())

				// Shard file names are numeric shardIDs
				shardID, err := strconv.ParseUint(sh.Name(), 10, 64)
				if err != nil {
					s.Logger.Printf("Skipping shard: %s. Not a valid path", rp.Name())
					continue
				}

				shard := NewShard(shardID, s.databaseIndexes[db], path, walPath, s.EngineOptions)
				err = shard.Open()
				if err != nil {
					return fmt.Errorf("failed to open shard %d: %s", shardID, err)
				}
				s.shards[shardID] = shard
			}
		}
	}
	return nil

}

func (s *Store) Open() error {
	s.mu.Lock()
	defer s.mu.Unlock()

	s.closing = make(chan struct{})

	s.shards = map[uint64]*Shard{}
	s.databaseIndexes = map[string]*DatabaseIndex{}

	s.Logger.Printf("Using data dir: %v", s.Path())

	// Create directory.
	if err := os.MkdirAll(s.path, 0777); err != nil {
		return err
	}

	// TODO: Start AE for Node
	if err := s.loadIndexes(); err != nil {
		return err
	}

	if err := s.loadShards(); err != nil {
		return err
	}

	return nil
}

func (s *Store) WriteToShard(shardID uint64, points []Point) error {
	s.mu.RLock()
	defer s.mu.RUnlock()
	sh, ok := s.shards[shardID]
	if !ok {
		return ErrShardNotFound
	}

	return sh.WritePoints(points)
}

func (s *Store) CreateMapper(shardID uint64, stmt influxql.Statement, chunkSize int) (Mapper, error) {
	shard := s.Shard(shardID)

	switch st := stmt.(type) {
	case *influxql.SelectStatement:
		return NewSelectMapper(shard, st, chunkSize), nil
	case *influxql.ShowMeasurementsStatement:
		return NewShowMeasurementsMapper(shard, st, chunkSize), nil
	default:
		return nil, fmt.Errorf("can't create mapper for statement type: %v", st)
	}
}

func (s *Store) Close() error {
	s.mu.Lock()
	defer s.mu.Unlock()

	for _, sh := range s.shards {
		if err := sh.Close(); err != nil {
			return err
		}
	}
	close(s.closing)
	s.closing = nil
	s.shards = nil
	s.databaseIndexes = nil

	return nil
}

// IsRetryable returns true if this error is temporary and could be retried
func IsRetryable(err error) bool {
	if err == nil {
		return true
	}

	if strings.Contains(err.Error(), "field type conflict") {
		return false
	}
	return true
}
<commit_msg>Don't close nil channel<commit_after>package tsdb

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"

	"github.com/influxdb/influxdb/influxql"
)

func NewStore(path string) *Store {
	opts := NewEngineOptions()
	opts.Config = NewConfig()

	return &Store{
		path:          path,
		EngineOptions: opts,
		Logger:        log.New(os.Stderr, "[store] ", log.LstdFlags),
	}
}

var (
	ErrShardNotFound = fmt.Errorf("shard not found")
)

type Store struct {
	mu   sync.RWMutex
	path string

	databaseIndexes map[string]*DatabaseIndex
	shards          map[uint64]*Shard

	EngineOptions EngineOptions
	Logger        *log.Logger
	closing       chan struct{}
}

// Path returns the store's root path.
func (s *Store) Path() string { return s.path }

// DatabaseIndexN returns the number of databases indicies in the store.
func (s *Store) DatabaseIndexN() int {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return len(s.databaseIndexes)
}

// Shard returns a shard by id.
func (s *Store) Shard(id uint64) *Shard {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.shards[id]
}

// ShardN returns the number of shard in the store.
func (s *Store) ShardN() int {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return len(s.shards)
}

func (s *Store) CreateShard(database, retentionPolicy string, shardID uint64) error {
	s.mu.Lock()
	defer s.mu.Unlock()

	select {
	case <-s.closing:
		return fmt.Errorf("closing")
	default:
	}

	// shard already exists
	if _, ok := s.shards[shardID]; ok {
		return nil
	}

	// created the db and retention policy dirs if they don't exist
	if err := os.MkdirAll(filepath.Join(s.path, database, retentionPolicy), 0700); err != nil {
		return err
	}

	// create the WAL directory
	walPath := filepath.Join(s.EngineOptions.Config.WALDir, database, retentionPolicy, fmt.Sprintf("%d", shardID))
	if err := os.MkdirAll(walPath, 0700); err != nil {
		return err
	}

	// create the database index if it does not exist
	db, ok := s.databaseIndexes[database]
	if !ok {
		db = NewDatabaseIndex()
		s.databaseIndexes[database] = db
	}

	shardPath := filepath.Join(s.path, database, retentionPolicy, strconv.FormatUint(shardID, 10))
	shard := NewShard(shardID, db, shardPath, walPath, s.EngineOptions)
	if err := shard.Open(); err != nil {
		return err
	}

	s.shards[shardID] = shard

	return nil
}

// DeleteShard removes a shard from disk.
func (s *Store) DeleteShard(shardID uint64) error {
	s.mu.Lock()
	defer s.mu.Unlock()

	// ensure shard exists
	sh, ok := s.shards[shardID]
	if !ok {
		return nil
	}

	if err := sh.Close(); err != nil {
		return err
	}

	if err := os.Remove(sh.path); err != nil {
		return err
	}

	if err := os.RemoveAll(sh.walPath); err != nil {
		return err
	}

	delete(s.shards, shardID)

	return nil
}

// DeleteDatabase will close all shards associated with a database and remove the directory and files from disk.
func (s *Store) DeleteDatabase(name string, shardIDs []uint64) error {
	s.mu.Lock()
	defer s.mu.Unlock()
	for _, id := range shardIDs {
		shard := s.shards[id]
		if shard != nil {
			shard.Close()
		}
	}
	if err := os.RemoveAll(filepath.Join(s.path, name)); err != nil {
		return err
	}
	if err := os.RemoveAll(filepath.Join(s.EngineOptions.Config.WALDir, name)); err != nil {
		return err
	}
	delete(s.databaseIndexes, name)
	return nil
}

// ShardIDs returns a slice of all ShardIDs under management.
func (s *Store) ShardIDs() []uint64 {
	ids := make([]uint64, 0, len(s.shards))
	for i, _ := range s.shards {
		ids = append(ids, i)
	}
	return ids
}

func (s *Store) ValidateAggregateFieldsInStatement(shardID uint64, measurementName string, stmt *influxql.SelectStatement) error {
	s.mu.RLock()
	shard := s.shards[shardID]
	s.mu.RUnlock()
	if shard == nil {
		return ErrShardNotFound
	}
	return shard.ValidateAggregateFieldsInStatement(measurementName, stmt)
}

func (s *Store) DatabaseIndex(name string) *DatabaseIndex {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.databaseIndexes[name]
}

// Databases returns all the databases in the indexes
func (s *Store) Databases() []string {
	s.mu.RLock()
	defer s.mu.RUnlock()
	databases := []string{}
	for db := range s.databaseIndexes {
		databases = append(databases, db)
	}
	return databases
}

func (s *Store) Measurement(database, name string) *Measurement {
	s.mu.RLock()
	db := s.databaseIndexes[database]
	s.mu.RUnlock()
	if db == nil {
		return nil
	}
	return db.Measurement(name)
}

// DiskSize returns the size of all the shard files in bytes.  This size does not include the WAL size.
func (s *Store) DiskSize() (int64, error) {
	s.mu.RLock()
	defer s.mu.RUnlock()
	var size int64
	for _, shardID := range s.ShardIDs() {
		shard := s.Shard(shardID)
		sz, err := shard.DiskSize()
		if err != nil {
			return 0, err
		}
		size += sz
	}
	return size, nil
}

// deleteSeries loops through the local shards and deletes the series data and metadata for the passed in series keys
func (s *Store) deleteSeries(keys []string) error {
	s.mu.RLock()
	defer s.mu.RUnlock()
	for _, sh := range s.shards {
		if err := sh.DeleteSeries(keys); err != nil {
			return err
		}
	}
	return nil
}

// deleteMeasurement loops through the local shards and removes the measurement field encodings from each shard
func (s *Store) deleteMeasurement(name string, seriesKeys []string) error {
	s.mu.RLock()
	defer s.mu.RUnlock()
	for _, sh := range s.shards {
		if err := sh.DeleteMeasurement(name, seriesKeys); err != nil {
			return err
		}
	}
	return nil
}

func (s *Store) loadIndexes() error {
	dbs, err := ioutil.ReadDir(s.path)
	if err != nil {
		return err
	}
	for _, db := range dbs {
		if !db.IsDir() {
			s.Logger.Printf("Skipping database dir: %s. Not a directory", db.Name())
			continue
		}
		s.databaseIndexes[db.Name()] = NewDatabaseIndex()
	}
	return nil
}

func (s *Store) loadShards() error {
	// loop through the current database indexes
	for db := range s.databaseIndexes {
		rps, err := ioutil.ReadDir(filepath.Join(s.path, db))
		if err != nil {
			return err
		}

		for _, rp := range rps {
			// retention policies should be directories.  Skip anything that is not a dir.
			if !rp.IsDir() {
				s.Logger.Printf("Skipping retention policy dir: %s. Not a directory", rp.Name())
				continue
			}

			shards, err := ioutil.ReadDir(filepath.Join(s.path, db, rp.Name()))
			if err != nil {
				return err
			}
			for _, sh := range shards {
				path := filepath.Join(s.path, db, rp.Name(), sh.Name())
				walPath := filepath.Join(s.EngineOptions.Config.WALDir, db, rp.Name(), sh.Name())

				// Shard file names are numeric shardIDs
				shardID, err := strconv.ParseUint(sh.Name(), 10, 64)
				if err != nil {
					s.Logger.Printf("Skipping shard: %s. Not a valid path", rp.Name())
					continue
				}

				shard := NewShard(shardID, s.databaseIndexes[db], path, walPath, s.EngineOptions)
				err = shard.Open()
				if err != nil {
					return fmt.Errorf("failed to open shard %d: %s", shardID, err)
				}
				s.shards[shardID] = shard
			}
		}
	}
	return nil

}

func (s *Store) Open() error {
	s.mu.Lock()
	defer s.mu.Unlock()

	s.closing = make(chan struct{})

	s.shards = map[uint64]*Shard{}
	s.databaseIndexes = map[string]*DatabaseIndex{}

	s.Logger.Printf("Using data dir: %v", s.Path())

	// Create directory.
	if err := os.MkdirAll(s.path, 0777); err != nil {
		return err
	}

	// TODO: Start AE for Node
	if err := s.loadIndexes(); err != nil {
		return err
	}

	if err := s.loadShards(); err != nil {
		return err
	}

	return nil
}

func (s *Store) WriteToShard(shardID uint64, points []Point) error {
	s.mu.RLock()
	defer s.mu.RUnlock()
	sh, ok := s.shards[shardID]
	if !ok {
		return ErrShardNotFound
	}

	return sh.WritePoints(points)
}

func (s *Store) CreateMapper(shardID uint64, stmt influxql.Statement, chunkSize int) (Mapper, error) {
	shard := s.Shard(shardID)

	switch st := stmt.(type) {
	case *influxql.SelectStatement:
		return NewSelectMapper(shard, st, chunkSize), nil
	case *influxql.ShowMeasurementsStatement:
		return NewShowMeasurementsMapper(shard, st, chunkSize), nil
	default:
		return nil, fmt.Errorf("can't create mapper for statement type: %v", st)
	}
}

func (s *Store) Close() error {
	s.mu.Lock()
	defer s.mu.Unlock()

	for _, sh := range s.shards {
		if err := sh.Close(); err != nil {
			return err
		}
	}
	if s.closing != nil {
		close(s.closing)
	}
	s.closing = nil
	s.shards = nil
	s.databaseIndexes = nil

	return nil
}

// IsRetryable returns true if this error is temporary and could be retried
func IsRetryable(err error) bool {
	if err == nil {
		return true
	}

	if strings.Contains(err.Error(), "field type conflict") {
		return false
	}
	return true
}
<|endoftext|>


<commit_before>package pelicantun

import (
	"crypto/rand"
	"fmt"
)

const KeyLen = 149

const randByteCount = 32

func RandBytes(n int) []byte {

	b := make([]byte, n)
	_, err := rand.Read(b)
	if err != nil {
		panic(fmt.Errorf("error on rand.Read: '%s'", err))
	}

	return b
}

func GenPelicanKey() []byte {

	key := RandBytes(2 * randByteCount)

	hmac := Sha256HMAC(key[:randByteCount], key[randByteCount:2*randByteCount])

	//fmt.Printf("in GenPelicanKey, hmac = '%x'\n", hmac)

	signed_key := make([]byte, len(key)+len(hmac))
	//fmt.Printf("\n\n GenPelicanKey, signed_key is len %d\n", len(signed_key))
	copy(signed_key, key)
	copy(signed_key[2*randByteCount:], hmac)

	//fmt.Printf("before alpha, signed_key = '%x'\n", signed_key)

	alpha_signed_key := EncodeBytesBase36(signed_key)

	sz := len(alpha_signed_key)
	if sz > KeyLen {
		panic(fmt.Sprintf("key too long: %d but expected", sz, KeyLen))
	}
	if sz < KeyLen {
		panic(fmt.Sprintf("key too short: %d but expected", sz, KeyLen))
	}

	//fmt.Printf("\n\n GenPelicanKey, alpha_signed_key is len %d\n", len(alpha_signed_key))
	return alpha_signed_key
}

func IsLegitPelicanKey(alpha_signed_key []byte) bool {
	_, signed_key, err := Base36toBigInt(alpha_signed_key)
	if err != nil {
		// TODO: no longer constant time with early return. Side channel vuln.
		return false
	}

	if len(signed_key) != 97 {
		// TODO: no longer constant time with early return. Side channel vuln.
		return false
	}
	signed_key = signed_key[1:]

	//fmt.Printf("after un-alpha, signed_key = '%x'\n", signed_key)

	hmac := signed_key[2*randByteCount:]

	//fmt.Printf("after un-alpha, expected hmac = '%x'\n", hmac)

	return CheckSha256HMAC(signed_key[:randByteCount], hmac, signed_key[randByteCount:2*randByteCount])
}
<commit_msg>sanity check<commit_after>package pelicantun

import (
	"crypto/rand"
	"fmt"
)

const KeyLen = 149

const randByteCount = 32

func RandBytes(n int) []byte {

	b := make([]byte, n)
	_, err := rand.Read(b)
	if err != nil {
		panic(fmt.Errorf("error on rand.Read: '%s'", err))
	}

	return b
}

func GenPelicanKey() []byte {

	key := RandBytes(2 * randByteCount)

	hmac := Sha256HMAC(key[:randByteCount], key[randByteCount:2*randByteCount])

	//fmt.Printf("in GenPelicanKey, hmac = '%x'\n", hmac)

	signed_key := make([]byte, len(key)+len(hmac))
	//fmt.Printf("\n\n GenPelicanKey, signed_key is len %d\n", len(signed_key))
	copy(signed_key, key)
	copy(signed_key[2*randByteCount:], hmac)

	//fmt.Printf("before alpha, signed_key = '%x'\n", signed_key)

	alpha_signed_key := EncodeBytesBase36(signed_key)

	sz := len(alpha_signed_key)
	if sz > KeyLen {
		panic(fmt.Sprintf("key too long: %d but expected", sz, KeyLen))
	}
	if sz < KeyLen {
		panic(fmt.Sprintf("key too short: %d but expected", sz, KeyLen))
	}

	if !IsLegitPelicanKey(alpha_signed_key) {
		panic("alpha_signed_key not passing the IsLegitPelicanKey() test")
	}

	//fmt.Printf("\n\n GenPelicanKey, alpha_signed_key is len %d\n", len(alpha_signed_key))
	return alpha_signed_key
}

func IsLegitPelicanKey(alpha_signed_key []byte) bool {
	_, signed_key, err := Base36toBigInt(alpha_signed_key)
	if err != nil {
		// TODO: no longer constant time with early return. Side channel vuln.
		return false
	}

	if len(signed_key) != 97 {
		// TODO: no longer constant time with early return. Side channel vuln.
		return false
	}
	signed_key = signed_key[1:]

	//fmt.Printf("after un-alpha, signed_key = '%x'\n", signed_key)

	hmac := signed_key[2*randByteCount:]

	//fmt.Printf("after un-alpha, expected hmac = '%x'\n", hmac)

	return CheckSha256HMAC(signed_key[:randByteCount], hmac, signed_key[randByteCount:2*randByteCount])
}
<|endoftext|>