<commit_before>package admin

import (
	"fmt"
	"reflect"
	"strconv"
	"strings"
	"time"

	"github.com/jinzhu/gorm"
	"github.com/jinzhu/now"
	"github.com/qor/inflection"
	"github.com/qor/qor"
	"github.com/qor/qor/resource"
	"github.com/qor/qor/roles"
	"github.com/qor/qor/utils"
)

type Resource struct {
	resource.Resource
	admin         *Admin
	Config        *Config
	Metas         []*Meta
	actions       []*Action
	scopes        []*Scope
	filters       map[string]*Filter
	searchAttrs   *[]string
	sortableAttrs *[]string
	indexAttrs    []string
	newAttrs      []string
	editAttrs     []string
	showAttrs     []string
	cachedMetas   *map[string][]*Meta
	SearchHandler func(keyword string, context *qor.Context) *gorm.DB
}

func (res *Resource) Meta(meta *Meta) {
	if res.GetMeta(meta.Name) != nil {
		utils.ExitWithMsg("Duplicated meta %v defined for resource %v", meta.Name, res.Name)
	}

	meta.base = res
	meta.updateMeta()
	res.Metas = append(res.Metas, meta)
}

func (res Resource) GetAdmin() *Admin {
	return res.admin
}

func (res Resource) ToParam() string {
	if res.Config.Singleton == true {
		return utils.ToParamString(inflection.Singular(res.Name))
	}
	return utils.ToParamString(inflection.Plural(res.Name))
}

func (res Resource) UseTheme(theme string) []string {
	if res.Config != nil {
		res.Config.Themes = append(res.Config.Themes, theme)
		return res.Config.Themes
	}
	return []string{}
}

func (res *Resource) convertObjectToMap(context *Context, value interface{}, kind string) interface{} {
	reflectValue := reflect.Indirect(reflect.ValueOf(value))
	switch reflectValue.Kind() {
	case reflect.Slice:
		values := []interface{}{}
		for i := 0; i < reflectValue.Len(); i++ {
			values = append(values, res.convertObjectToMap(context, reflectValue.Index(i).Interface(), kind))
		}
		return values
	case reflect.Struct:
		var metas []*Meta
		if kind == "index" {
			metas = res.indexMetas()
		} else if kind == "show" {
			metas = res.showMetas()
		}

		values := map[string]interface{}{}
		for _, meta := range metas {
			if meta.HasPermission(roles.Read, context.Context) {
				if valuer := meta.GetValuer(); valuer != nil {
					value := valuer(value, context.Context)
					if meta.Resource != nil {
						value = meta.Resource.(*Resource).convertObjectToMap(context, value, kind)
					}
					values[meta.GetName()] = value
				}
			}
		}
		return values
	default:
		panic(fmt.Sprintf("Can't convert %v (%v) to map", reflectValue, reflectValue.Kind()))
	}
}

func (res *Resource) Decode(context *qor.Context, value interface{}) error {
	return resource.Decode(context, value, res)
}

func (res *Resource) allAttrs() []string {
	var attrs []string
	scope := &gorm.Scope{Value: res.Value}

Fields:
	for _, field := range scope.GetModelStruct().StructFields {
		for _, meta := range res.Metas {
			if field.Name == meta.Alias {
				attrs = append(attrs, meta.Name)
				continue Fields
			}
		}

		if field.IsForeignKey {
			continue
		}

		for _, value := range []string{"CreatedAt", "UpdatedAt", "DeletedAt"} {
			if value == field.Name {
				continue Fields
			}
		}

		if (field.IsNormal || field.Relationship != nil) && !field.IsIgnored {
			attrs = append(attrs, field.Name)
		}
	}

MetaIncluded:
	for _, meta := range res.Metas {
		for _, attr := range attrs {
			if attr == meta.Alias || attr == meta.Name {
				continue MetaIncluded
			}
		}
		attrs = append(attrs, meta.Name)
	}

	return attrs
}

func (res *Resource) getAttrs(attrs []string) []string {
	if len(attrs) == 0 {
		return res.allAttrs()
	} else {
		var onlyExcludeAttrs = true
		for _, attr := range attrs {
			if !strings.HasPrefix(attr, "-") {
				onlyExcludeAttrs = false
				break
			}
		}
		if onlyExcludeAttrs {
			return append(res.allAttrs(), attrs...)
		}
		return attrs
	}
}

func (res *Resource) IndexAttrs(columns ...string) []string {
	if len(columns) > 0 {
		res.indexAttrs = columns
	}
	return res.getAttrs(res.indexAttrs)
}

func (res *Resource) NewAttrs(columns ...string) []string {
	if len(columns) > 0 {
		res.newAttrs = columns
	}
	return res.getAttrs(res.newAttrs)
}

func (res *Resource) EditAttrs(columns ...string) []string {
	if len(columns) > 0 {
		res.editAttrs = columns
	}
	return res.getAttrs(res.editAttrs)
}

func (res *Resource) ShowAttrs(columns ...string) []string {
	if len(columns) > 0 {
		res.showAttrs = columns
	}
	return res.getAttrs(res.showAttrs)
}

func (res *Resource) SortableAttrs(columns ...string) []string {
	if len(columns) != 0 || res.sortableAttrs == nil {
		if len(columns) == 0 {
			columns = res.indexAttrs
		}
		res.sortableAttrs = &[]string{}
		scope := res.GetAdmin().Config.DB.NewScope(res.Value)
		for _, column := range columns {
			if field, ok := scope.FieldByName(column); ok && field.DBName != "" {
				attrs := append(*res.sortableAttrs, column)
				res.sortableAttrs = &attrs
			}
		}
	}
	return *res.sortableAttrs
}

func (res *Resource) SearchAttrs(columns ...string) []string {
	if len(columns) != 0 || res.searchAttrs == nil {
		if len(columns) == 0 {
			columns = res.indexAttrs
		}

		res.searchAttrs = &columns
		res.SearchHandler = func(keyword string, context *qor.Context) *gorm.DB {
			db := context.GetDB()
			var conditions []string
			var keywords []interface{}
			scope := db.NewScope(res.Value)

			for _, column := range columns {
				if field, ok := scope.FieldByName(column); ok && field.IsNormal {
					switch field.Field.Kind() {
					case reflect.String:
						conditions = append(conditions, fmt.Sprintf("upper(%v) like upper(?)", scope.Quote(field.DBName)))
						keywords = append(keywords, "%"+keyword+"%")
					case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
						if _, err := strconv.Atoi(keyword); err == nil {
							conditions = append(conditions, fmt.Sprintf("%v = ?", scope.Quote(field.DBName)))
							keywords = append(keywords, keyword)
						}
					case reflect.Float32, reflect.Float64:
						if _, err := strconv.ParseFloat(keyword, 64); err == nil {
							conditions = append(conditions, fmt.Sprintf("%v = ?", scope.Quote(field.DBName)))
							keywords = append(keywords, keyword)
						}
					case reflect.Struct:
						// time ?
						if _, ok := field.Field.Interface().(time.Time); ok {
							if parsedTime, err := now.Parse(keyword); err == nil {
								conditions = append(conditions, fmt.Sprintf("%v = ?", scope.Quote(field.DBName)))
								keywords = append(keywords, parsedTime)
							}
						}
					case reflect.Ptr:
						// time ?
						if _, ok := field.Field.Interface().(*time.Time); ok {
							if parsedTime, err := now.Parse(keyword); err == nil {
								conditions = append(conditions, fmt.Sprintf("%v = ?", scope.Quote(field.DBName)))
								keywords = append(keywords, parsedTime)
							}
						}
					default:
						conditions = append(conditions, fmt.Sprintf("%v = ?", scope.Quote(field.DBName)))
						keywords = append(keywords, keyword)
					}
				}
			}

			if len(conditions) > 0 {
				return context.GetDB().Where(strings.Join(conditions, " OR "), keywords...)
			} else {
				return context.GetDB()
			}
		}
	}

	return *res.searchAttrs
}

func (res *Resource) getCachedMetas(cacheKey string, fc func() []resource.Metaor) []*Meta {
	if res.cachedMetas == nil {
		res.cachedMetas = &map[string][]*Meta{}
	}

	if values, ok := (*res.cachedMetas)[cacheKey]; ok {
		return values
	} else {
		values := fc()
		var metas []*Meta
		for _, value := range values {
			metas = append(metas, value.(*Meta))
		}
		(*res.cachedMetas)[cacheKey] = metas
		return metas
	}
}

func (res *Resource) GetMetas(attrs []string) []resource.Metaor {
	if len(attrs) == 0 {
		attrs = res.allAttrs()
	}
	var showAttrs, ignoredAttrs []string
	for _, attr := range attrs {
		if strings.HasPrefix(attr, "-") {
			ignoredAttrs = append(ignoredAttrs, strings.TrimLeft(attr, "-"))
		} else {
			showAttrs = append(showAttrs, attr)
		}
	}

	primaryKey := res.PrimaryFieldName()

	metas := []resource.Metaor{}
Attrs:
	for _, attr := range showAttrs {
		for _, a := range ignoredAttrs {
			if attr == a {
				continue Attrs
			}
		}

		var meta *Meta
		for _, m := range res.Metas {
			if m.GetName() == attr {
				meta = m
				break
			}
		}

		if meta == nil {
			meta = &Meta{}
			meta.Name = attr
			meta.base = res
			if attr == primaryKey {
				meta.Type = "hidden"
			}
			meta.updateMeta()
		}

		metas = append(metas, meta)
	}

	return metas
}

func (res *Resource) GetMeta(name string) *Meta {
	for _, meta := range res.Metas {
		if meta.Name == name || meta.GetFieldName() == name {
			return meta
		}
	}
	return nil
}

func (res *Resource) indexMetas() []*Meta {
	return res.getCachedMetas("index_metas", func() []resource.Metaor {
		return res.GetMetas(res.IndexAttrs())
	})
}

func (res *Resource) newMetas() []*Meta {
	return res.getCachedMetas("new_metas", func() []resource.Metaor {
		return res.GetMetas(res.NewAttrs())
	})
}

func (res *Resource) editMetas() []*Meta {
	return res.getCachedMetas("edit_metas", func() []resource.Metaor {
		return res.GetMetas(res.EditAttrs())
	})
}

func (res *Resource) showMetas() []*Meta {
	return res.getCachedMetas("show_metas", func() []resource.Metaor {
		return res.GetMetas(res.ShowAttrs())
	})
}

func (res *Resource) allMetas() []*Meta {
	return res.getCachedMetas("all_metas", func() []resource.Metaor {
		return res.GetMetas([]string{})
	})
}

func (res *Resource) allowedMetas(attrs []*Meta, context *Context, roles ...roles.PermissionMode) []*Meta {
	var metas = []*Meta{}
	for _, meta := range attrs {
		for _, role := range roles {
			if meta.HasPermission(role, context.Context) {
				metas = append(metas, meta)
				break
			}
		}
	}
	return metas
}

func (res *Resource) HasPermission(mode roles.PermissionMode, context *qor.Context) bool {
	if res == nil || res.Config == nil || res.Config.Permission == nil {
		return true
	}
	return res.Config.Permission.HasPermission(mode, context.Roles...)
}
<commit_msg>Improve search handler to handle bool<commit_after>package admin

import (
	"fmt"
	"reflect"
	"strconv"
	"strings"
	"time"

	"github.com/jinzhu/gorm"
	"github.com/jinzhu/now"
	"github.com/qor/inflection"
	"github.com/qor/qor"
	"github.com/qor/qor/resource"
	"github.com/qor/qor/roles"
	"github.com/qor/qor/utils"
)

type Resource struct {
	resource.Resource
	admin         *Admin
	Config        *Config
	Metas         []*Meta
	actions       []*Action
	scopes        []*Scope
	filters       map[string]*Filter
	searchAttrs   *[]string
	sortableAttrs *[]string
	indexAttrs    []string
	newAttrs      []string
	editAttrs     []string
	showAttrs     []string
	cachedMetas   *map[string][]*Meta
	SearchHandler func(keyword string, context *qor.Context) *gorm.DB
}

func (res *Resource) Meta(meta *Meta) {
	if res.GetMeta(meta.Name) != nil {
		utils.ExitWithMsg("Duplicated meta %v defined for resource %v", meta.Name, res.Name)
	}

	meta.base = res
	meta.updateMeta()
	res.Metas = append(res.Metas, meta)
}

func (res Resource) GetAdmin() *Admin {
	return res.admin
}

func (res Resource) ToParam() string {
	if res.Config.Singleton == true {
		return utils.ToParamString(inflection.Singular(res.Name))
	}
	return utils.ToParamString(inflection.Plural(res.Name))
}

func (res Resource) UseTheme(theme string) []string {
	if res.Config != nil {
		res.Config.Themes = append(res.Config.Themes, theme)
		return res.Config.Themes
	}
	return []string{}
}

func (res *Resource) convertObjectToMap(context *Context, value interface{}, kind string) interface{} {
	reflectValue := reflect.Indirect(reflect.ValueOf(value))
	switch reflectValue.Kind() {
	case reflect.Slice:
		values := []interface{}{}
		for i := 0; i < reflectValue.Len(); i++ {
			values = append(values, res.convertObjectToMap(context, reflectValue.Index(i).Interface(), kind))
		}
		return values
	case reflect.Struct:
		var metas []*Meta
		if kind == "index" {
			metas = res.indexMetas()
		} else if kind == "show" {
			metas = res.showMetas()
		}

		values := map[string]interface{}{}
		for _, meta := range metas {
			if meta.HasPermission(roles.Read, context.Context) {
				if valuer := meta.GetValuer(); valuer != nil {
					value := valuer(value, context.Context)
					if meta.Resource != nil {
						value = meta.Resource.(*Resource).convertObjectToMap(context, value, kind)
					}
					values[meta.GetName()] = value
				}
			}
		}
		return values
	default:
		panic(fmt.Sprintf("Can't convert %v (%v) to map", reflectValue, reflectValue.Kind()))
	}
}

func (res *Resource) Decode(context *qor.Context, value interface{}) error {
	return resource.Decode(context, value, res)
}

func (res *Resource) allAttrs() []string {
	var attrs []string
	scope := &gorm.Scope{Value: res.Value}

Fields:
	for _, field := range scope.GetModelStruct().StructFields {
		for _, meta := range res.Metas {
			if field.Name == meta.Alias {
				attrs = append(attrs, meta.Name)
				continue Fields
			}
		}

		if field.IsForeignKey {
			continue
		}

		for _, value := range []string{"CreatedAt", "UpdatedAt", "DeletedAt"} {
			if value == field.Name {
				continue Fields
			}
		}

		if (field.IsNormal || field.Relationship != nil) && !field.IsIgnored {
			attrs = append(attrs, field.Name)
		}
	}

MetaIncluded:
	for _, meta := range res.Metas {
		for _, attr := range attrs {
			if attr == meta.Alias || attr == meta.Name {
				continue MetaIncluded
			}
		}
		attrs = append(attrs, meta.Name)
	}

	return attrs
}

func (res *Resource) getAttrs(attrs []string) []string {
	if len(attrs) == 0 {
		return res.allAttrs()
	} else {
		var onlyExcludeAttrs = true
		for _, attr := range attrs {
			if !strings.HasPrefix(attr, "-") {
				onlyExcludeAttrs = false
				break
			}
		}
		if onlyExcludeAttrs {
			return append(res.allAttrs(), attrs...)
		}
		return attrs
	}
}

func (res *Resource) IndexAttrs(columns ...string) []string {
	if len(columns) > 0 {
		res.indexAttrs = columns
	}
	return res.getAttrs(res.indexAttrs)
}

func (res *Resource) NewAttrs(columns ...string) []string {
	if len(columns) > 0 {
		res.newAttrs = columns
	}
	return res.getAttrs(res.newAttrs)
}

func (res *Resource) EditAttrs(columns ...string) []string {
	if len(columns) > 0 {
		res.editAttrs = columns
	}
	return res.getAttrs(res.editAttrs)
}

func (res *Resource) ShowAttrs(columns ...string) []string {
	if len(columns) > 0 {
		res.showAttrs = columns
	}
	return res.getAttrs(res.showAttrs)
}

func (res *Resource) SortableAttrs(columns ...string) []string {
	if len(columns) != 0 || res.sortableAttrs == nil {
		if len(columns) == 0 {
			columns = res.indexAttrs
		}
		res.sortableAttrs = &[]string{}
		scope := res.GetAdmin().Config.DB.NewScope(res.Value)
		for _, column := range columns {
			if field, ok := scope.FieldByName(column); ok && field.DBName != "" {
				attrs := append(*res.sortableAttrs, column)
				res.sortableAttrs = &attrs
			}
		}
	}
	return *res.sortableAttrs
}

func (res *Resource) SearchAttrs(columns ...string) []string {
	if len(columns) != 0 || res.searchAttrs == nil {
		if len(columns) == 0 {
			columns = res.indexAttrs
		}

		res.searchAttrs = &columns
		res.SearchHandler = func(keyword string, context *qor.Context) *gorm.DB {
			db := context.GetDB()
			var conditions []string
			var keywords []interface{}
			scope := db.NewScope(res.Value)

			for _, column := range columns {
				if field, ok := scope.FieldByName(column); ok && field.IsNormal {
					switch field.Field.Kind() {
					case reflect.String:
						conditions = append(conditions, fmt.Sprintf("upper(%v) like upper(?)", scope.Quote(field.DBName)))
						keywords = append(keywords, "%"+keyword+"%")
					case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
						if _, err := strconv.Atoi(keyword); err == nil {
							conditions = append(conditions, fmt.Sprintf("%v = ?", scope.Quote(field.DBName)))
							keywords = append(keywords, keyword)
						}
					case reflect.Float32, reflect.Float64:
						if _, err := strconv.ParseFloat(keyword, 64); err == nil {
							conditions = append(conditions, fmt.Sprintf("%v = ?", scope.Quote(field.DBName)))
							keywords = append(keywords, keyword)
						}
					case reflect.Bool:
						if value, err := strconv.ParseBool(keyword); err == nil {
							conditions = append(conditions, fmt.Sprintf("%v = ?", scope.Quote(field.DBName)))
							keywords = append(keywords, value)
						}
					case reflect.Struct:
						// time ?
						if _, ok := field.Field.Interface().(time.Time); ok {
							if parsedTime, err := now.Parse(keyword); err == nil {
								conditions = append(conditions, fmt.Sprintf("%v = ?", scope.Quote(field.DBName)))
								keywords = append(keywords, parsedTime)
							}
						}
					case reflect.Ptr:
						// time ?
						if _, ok := field.Field.Interface().(*time.Time); ok {
							if parsedTime, err := now.Parse(keyword); err == nil {
								conditions = append(conditions, fmt.Sprintf("%v = ?", scope.Quote(field.DBName)))
								keywords = append(keywords, parsedTime)
							}
						}
					default:
						conditions = append(conditions, fmt.Sprintf("%v = ?", scope.Quote(field.DBName)))
						keywords = append(keywords, keyword)
					}
				}
			}

			if len(conditions) > 0 {
				return context.GetDB().Where(strings.Join(conditions, " OR "), keywords...)
			} else {
				return context.GetDB()
			}
		}
	}

	return *res.searchAttrs
}

func (res *Resource) getCachedMetas(cacheKey string, fc func() []resource.Metaor) []*Meta {
	if res.cachedMetas == nil {
		res.cachedMetas = &map[string][]*Meta{}
	}

	if values, ok := (*res.cachedMetas)[cacheKey]; ok {
		return values
	} else {
		values := fc()
		var metas []*Meta
		for _, value := range values {
			metas = append(metas, value.(*Meta))
		}
		(*res.cachedMetas)[cacheKey] = metas
		return metas
	}
}

func (res *Resource) GetMetas(attrs []string) []resource.Metaor {
	if len(attrs) == 0 {
		attrs = res.allAttrs()
	}
	var showAttrs, ignoredAttrs []string
	for _, attr := range attrs {
		if strings.HasPrefix(attr, "-") {
			ignoredAttrs = append(ignoredAttrs, strings.TrimLeft(attr, "-"))
		} else {
			showAttrs = append(showAttrs, attr)
		}
	}

	primaryKey := res.PrimaryFieldName()

	metas := []resource.Metaor{}
Attrs:
	for _, attr := range showAttrs {
		for _, a := range ignoredAttrs {
			if attr == a {
				continue Attrs
			}
		}

		var meta *Meta
		for _, m := range res.Metas {
			if m.GetName() == attr {
				meta = m
				break
			}
		}

		if meta == nil {
			meta = &Meta{}
			meta.Name = attr
			meta.base = res
			if attr == primaryKey {
				meta.Type = "hidden"
			}
			meta.updateMeta()
		}

		metas = append(metas, meta)
	}

	return metas
}

func (res *Resource) GetMeta(name string) *Meta {
	for _, meta := range res.Metas {
		if meta.Name == name || meta.GetFieldName() == name {
			return meta
		}
	}
	return nil
}

func (res *Resource) indexMetas() []*Meta {
	return res.getCachedMetas("index_metas", func() []resource.Metaor {
		return res.GetMetas(res.IndexAttrs())
	})
}

func (res *Resource) newMetas() []*Meta {
	return res.getCachedMetas("new_metas", func() []resource.Metaor {
		return res.GetMetas(res.NewAttrs())
	})
}

func (res *Resource) editMetas() []*Meta {
	return res.getCachedMetas("edit_metas", func() []resource.Metaor {
		return res.GetMetas(res.EditAttrs())
	})
}

func (res *Resource) showMetas() []*Meta {
	return res.getCachedMetas("show_metas", func() []resource.Metaor {
		return res.GetMetas(res.ShowAttrs())
	})
}

func (res *Resource) allMetas() []*Meta {
	return res.getCachedMetas("all_metas", func() []resource.Metaor {
		return res.GetMetas([]string{})
	})
}

func (res *Resource) allowedMetas(attrs []*Meta, context *Context, roles ...roles.PermissionMode) []*Meta {
	var metas = []*Meta{}
	for _, meta := range attrs {
		for _, role := range roles {
			if meta.HasPermission(role, context.Context) {
				metas = append(metas, meta)
				break
			}
		}
	}
	return metas
}

func (res *Resource) HasPermission(mode roles.PermissionMode, context *qor.Context) bool {
	if res == nil || res.Config == nil || res.Config.Permission == nil {
		return true
	}
	return res.Config.Permission.HasPermission(mode, context.Roles...)
}
<|endoftext|>


<commit_before>package resource

import (
	"reflect"

	"github.com/jinzhu/gorm"
	"github.com/qor/qor"
	"github.com/qor/roles"
)

// Resourcer interface
type Resourcer interface {
	GetResource() *Resource
	GetMetas([]string) []Metaor
	CallFindMany(interface{}, *qor.Context) error
	CallFindOne(interface{}, *MetaValues, *qor.Context) error
	CallSave(interface{}, *qor.Context) error
	CallDelete(interface{}, *qor.Context) error
	NewSlice() interface{}
	NewStruct() interface{}
}

// ConfigureResourceBeforeInitializeInterface if a struct implemented this interface, it will be called before everything when create a resource with the struct
type ConfigureResourceBeforeInitializeInterface interface {
	ConfigureQorResourceBeforeInitialize(Resourcer)
}

// ConfigureResourceInterface if a struct implemented this interface, it will be called after configured by user
type ConfigureResourceInterface interface {
	ConfigureQorResource(Resourcer)
}

// Resource is a struct that including basic definition of qor resource
type Resource struct {
	Name            string
	Value           interface{}
	FindManyHandler func(interface{}, *qor.Context) error
	FindOneHandler  func(interface{}, *MetaValues, *qor.Context) error
	SaveHandler     func(interface{}, *qor.Context) error
	DeleteHandler   func(interface{}, *qor.Context) error
	Permission      *roles.Permission
	validators      []func(interface{}, *MetaValues, *qor.Context) error
	processors      []func(interface{}, *MetaValues, *qor.Context) error
	primaryField    *gorm.Field
}

// New initialize qor resource
func New(value interface{}) *Resource {
	name := reflect.Indirect(reflect.ValueOf(value)).Type().Name()
	res := &Resource{Value: value, Name: name}
	res.FindOneHandler = res.findOneHandler
	res.FindManyHandler = res.findManyHandler
	res.SaveHandler = res.saveHandler
	res.DeleteHandler = res.deleteHandler

	return res
}

// GetResource return itself to match interface `Resourcer`
func (res *Resource) GetResource() *Resource {
	return res
}

// AddValidator add validator to resource, it will invoked when creating, updating, and will rollback the change if validator return any error
func (res *Resource) AddValidator(fc func(interface{}, *MetaValues, *qor.Context) error) {
	res.validators = append(res.validators, fc)
}

// AddProcessor add processor to resource, it is used to process data before creating, updating, will rollback the change if it return any error
func (res *Resource) AddProcessor(fc func(interface{}, *MetaValues, *qor.Context) error) {
	res.processors = append(res.processors, fc)
}

// NewStruct initialize a struct for the Resource
func (res *Resource) NewStruct() interface{} {
	return reflect.New(reflect.Indirect(reflect.ValueOf(res.Value)).Type()).Interface()
}

// NewSlice initialize a slice of struct for the Resource
func (res *Resource) NewSlice() interface{} {
	sliceType := reflect.SliceOf(reflect.TypeOf(res.Value))
	slice := reflect.MakeSlice(sliceType, 0, 0)
	slicePtr := reflect.New(sliceType)
	slicePtr.Elem().Set(slice)
	return slicePtr.Interface()
}

// GetMetas get defined metas, to match interface `Resourcer`
func (res *Resource) GetMetas([]string) []Metaor {
	panic("not defined")
}

// HasPermission check permission of resource
func (res *Resource) HasPermission(mode roles.PermissionMode, context *qor.Context) bool {
	if res == nil || res.Permission == nil {
		return true
	}
	return res.Permission.HasPermission(mode, context.Roles...)
}

// PrimaryField return gorm's primary field
func (res *Resource) PrimaryField() *gorm.Field {
	if res.primaryField == nil {
		scope := gorm.Scope{Value: res.Value}
		res.primaryField = scope.PrimaryField()
	}
	return res.primaryField
}

// PrimaryDBName return db column name of the resource's primary field
func (res *Resource) PrimaryDBName() (name string) {
	field := res.PrimaryField()
	if field != nil {
		name = field.DBName
	}
	return
}

// PrimaryFieldName return struct column name of the resource's primary field
func (res *Resource) PrimaryFieldName() (name string) {
	field := res.PrimaryField()
	if field != nil {
		name = field.Name
	}
	return
}
<commit_msg>Refactor Resource Name generating<commit_after>package resource

import (
	"reflect"

	"github.com/jinzhu/gorm"
	"github.com/qor/qor"
	"github.com/qor/qor/utils"
	"github.com/qor/roles"
)

// Resourcer interface
type Resourcer interface {
	GetResource() *Resource
	GetMetas([]string) []Metaor
	CallFindMany(interface{}, *qor.Context) error
	CallFindOne(interface{}, *MetaValues, *qor.Context) error
	CallSave(interface{}, *qor.Context) error
	CallDelete(interface{}, *qor.Context) error
	NewSlice() interface{}
	NewStruct() interface{}
}

// ConfigureResourceBeforeInitializeInterface if a struct implemented this interface, it will be called before everything when create a resource with the struct
type ConfigureResourceBeforeInitializeInterface interface {
	ConfigureQorResourceBeforeInitialize(Resourcer)
}

// ConfigureResourceInterface if a struct implemented this interface, it will be called after configured by user
type ConfigureResourceInterface interface {
	ConfigureQorResource(Resourcer)
}

// Resource is a struct that including basic definition of qor resource
type Resource struct {
	Name            string
	Value           interface{}
	FindManyHandler func(interface{}, *qor.Context) error
	FindOneHandler  func(interface{}, *MetaValues, *qor.Context) error
	SaveHandler     func(interface{}, *qor.Context) error
	DeleteHandler   func(interface{}, *qor.Context) error
	Permission      *roles.Permission
	validators      []func(interface{}, *MetaValues, *qor.Context) error
	processors      []func(interface{}, *MetaValues, *qor.Context) error
	primaryField    *gorm.Field
}

// New initialize qor resource
func New(value interface{}) *Resource {
	var (
		name = utils.HumanizeString(utils.ModelType(value).Name())
		res  = &Resource{Value: value, Name: name}
	)

	res.FindOneHandler = res.findOneHandler
	res.FindManyHandler = res.findManyHandler
	res.SaveHandler = res.saveHandler
	res.DeleteHandler = res.deleteHandler
	return res
}

// GetResource return itself to match interface `Resourcer`
func (res *Resource) GetResource() *Resource {
	return res
}

// AddValidator add validator to resource, it will invoked when creating, updating, and will rollback the change if validator return any error
func (res *Resource) AddValidator(fc func(interface{}, *MetaValues, *qor.Context) error) {
	res.validators = append(res.validators, fc)
}

// AddProcessor add processor to resource, it is used to process data before creating, updating, will rollback the change if it return any error
func (res *Resource) AddProcessor(fc func(interface{}, *MetaValues, *qor.Context) error) {
	res.processors = append(res.processors, fc)
}

// NewStruct initialize a struct for the Resource
func (res *Resource) NewStruct() interface{} {
	return reflect.New(reflect.Indirect(reflect.ValueOf(res.Value)).Type()).Interface()
}

// NewSlice initialize a slice of struct for the Resource
func (res *Resource) NewSlice() interface{} {
	sliceType := reflect.SliceOf(reflect.TypeOf(res.Value))
	slice := reflect.MakeSlice(sliceType, 0, 0)
	slicePtr := reflect.New(sliceType)
	slicePtr.Elem().Set(slice)
	return slicePtr.Interface()
}

// GetMetas get defined metas, to match interface `Resourcer`
func (res *Resource) GetMetas([]string) []Metaor {
	panic("not defined")
}

// HasPermission check permission of resource
func (res *Resource) HasPermission(mode roles.PermissionMode, context *qor.Context) bool {
	if res == nil || res.Permission == nil {
		return true
	}
	return res.Permission.HasPermission(mode, context.Roles...)
}

// PrimaryField return gorm's primary field
func (res *Resource) PrimaryField() *gorm.Field {
	if res.primaryField == nil {
		scope := gorm.Scope{Value: res.Value}
		res.primaryField = scope.PrimaryField()
	}
	return res.primaryField
}

// PrimaryDBName return db column name of the resource's primary field
func (res *Resource) PrimaryDBName() (name string) {
	field := res.PrimaryField()
	if field != nil {
		name = field.DBName
	}
	return
}

// PrimaryFieldName return struct column name of the resource's primary field
func (res *Resource) PrimaryFieldName() (name string) {
	field := res.PrimaryField()
	if field != nil {
		name = field.Name
	}
	return
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"strings"
	"sync"
	"testing"
	"time"

	. "launchpad.net/gocheck"
)

// ----------- Gocheck config ----------------

func Test(t *testing.T) { TestingT(t) }

type GostSuite struct{}

var _ = Suite(&GostSuite{})

// ----------- Setup ----------------

var (
	rec                          = &recorder{}
	testUDPConn                  *net.UDPConn
	testForwardListener          net.Listener
	when                         = time.Time{}
	aggregateFlushChan           = make(chan time.Time)
	aggregateForwardedFlushChan  = make(chan time.Time)
	aggregateForwardingFlushChan = make(chan time.Time)
	flushers                     int
)

func (*GostSuite) SetUpSuite(c *C) {
	// Stub out control functions
	now = func() time.Time { return when }
	aggregateFlushTicker = func() <-chan time.Time { return aggregateFlushChan }
	aggregateForwardedFlushTicker = func() <-chan time.Time { return aggregateForwardedFlushChan }
	aggregateForwardingFlushTicker = func() <-chan time.Time { return aggregateForwardingFlushChan }

	// Start workers
	go flush()
	go aggregate()
	// Don't want to see bad output lines (we could capture and inspect these if we want to test them later).
	log.SetOutput(ioutil.Discard)

	var err error
	testForwardListener, err = net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		c.Fatal(err)
	}
	forwardingIncoming = make(chan *Stat, incomingQueueSize)
	go flushForwarding()
	go aggregateForwarding()
	go aggregateForwarded()
	go forwardServer(testForwardListener)

	// Bind to any port
	if err := debugServer.Start(0); err != nil {
		c.Fatal(err)
	}

	udp, err := net.ResolveUDPAddr("udp", "127.0.0.1:0")
	if err != nil {
		c.Fatal(err)
	}
	testUDPConn, err = net.ListenUDP("udp", udp)
	if err != nil {
		c.Fatal(err)
	}
	go clientServer(testUDPConn)
}

func (*GostSuite) TearDownSuite(c *C) {
	testUDPConn.Close()
	testForwardListener.Close()
}

func (*GostSuite) SetUpTest(c *C) {
	rec.Start()
	conf = &Conf{
		GraphiteAddr:             rec.Addr,
		ForwardingAddr:           testForwardListener.Addr().String(),
		ForwardedNamespace:       "global",
		ClearStatsBetweenFlushes: true,
		FlushIntervalMS:          2000, // Fake
		Namespace:                "com.example",
	}
}

func (*GostSuite) TearDownTest(c *C) {
	rec.Close()
}

type graphiteValue struct {
	Value     float64
	Timestamp time.Time
}

type graphiteMessage struct {
	Raw    []byte
	Parsed map[string]*graphiteValue
}

func parseRawGraphiteMessage(raw []byte) *graphiteMessage {
	parsed := make(map[string]*graphiteValue)
	for _, line := range strings.Split(string(raw), "\n") {
		line = strings.TrimSpace(line)
		if len(line) == 0 {
			continue
		}
		key := ""
		value := &graphiteValue{}
		var unixTime int64
		if _, err := fmt.Sscanf(line, "%s %f %d", &key, &value.Value, &unixTime); err != nil {
			panic("Not a graphite message: " + line)
		}
		value.Timestamp = time.Unix(unixTime, 0)
		parsed[key] = value
	}
	return &graphiteMessage{raw, parsed}
}

type recorder struct {
	W        *sync.WaitGroup
	Addr     string
	Listener net.Listener
	messages chan []byte
}

// waitForMessage flushes the aggregator and returns the collected messages.
func (r *recorder) waitForMessage() *graphiteMessage {
	// Ensure the aggregator has time to collect all the messages we've sent in.
	time.Sleep(time.Millisecond)
	aggregateFlushChan <- when
	return parseRawGraphiteMessage(<-r.messages)
}

func (r *recorder) Start() {
	r.W = &sync.WaitGroup{}
	// Listen on a free port
	listener, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		panic(err)
	}
	r.Listener = listener
	r.Addr = listener.Addr().String()
	r.messages = make(chan []byte)
	go func() {
		for {
			c, err := r.Listener.Accept()
			if err != nil {
				return
			}
			go r.Handle(c)
		}
	}()
}

func (r *recorder) Handle(c net.Conn) {
	r.W.Add(1)
	defer r.W.Done()
	msg, err := ioutil.ReadAll(c)
	if err != nil {
		return
	}
	r.messages <- msg
}

func (r *recorder) Close() {
	r.Listener.Close()
	r.W.Wait()
}

func sendGostMessages(c *C, msgs ...string) {
	conn, err := net.Dial("udp", testUDPConn.LocalAddr().String())
	if err != nil {
		c.Fatal(err)
	}
	defer conn.Close()
	for _, msg := range msgs {
		conn.Write([]byte(msg))
	}
}

// approx is a gocheck checker for approximate equality of floats.
// Note that approximate equality of floats is a fraught topic. This is a very naive comparison.
type approxChecker struct {
	*CheckerInfo
}

var approx = &approxChecker{
	&CheckerInfo{Name: "approx", Params: []string{"obtained", "expected"}},
}

func (c *approxChecker) Check(params []interface{}, names []string) (result bool, error string) {
	f1, ok1 := params[0].(float64)
	f2, ok2 := params[1].(float64)
	if !(ok1 && ok2) {
		return false, "must compare float64s"
	}
	if f1 == f2 {
		return true, ""
	} else if f1 > f2 {
		f1, f2 = f2, f1
	}
	delta := (f2 - f1) / f1
	if delta < 0.0001 { // Accept f2 up to 0.01% greater than f1
		return true, ""
	}
	return false, ""
}

type testCase struct {
	Key   string
	Value float64
}

func checkAllApprox(c *C, tests []testCase) {
	msg := rec.waitForMessage()
	for _, test := range tests {
		key := "com.example." + test.Key
		parsed := msg.Parsed[key]
		if parsed == nil {
			fmt.Println(string(msg.Raw))
			c.Fatal("No graphite key found: ", key)
		}
		c.Check(parsed.Value, approx, test.Value)
	}
}

// ----------- Tests ----------------

func (*GostSuite) TestCounters(c *C) {
	sendGostMessages(c, "foobar:3|c", "foobar:5|c", "baz:2|c|@0.1", "baz:4|c|@0.1")
	checkAllApprox(c, []testCase{
		{"foobar.count", 8.0},
		{"foobar.rate", 4.0},
		{"baz.count", 60.0},
		{"baz.rate", 30.0},
	})
}

func (*GostSuite) TestTimers(c *C) {
	sendGostMessages(c, "foobar:100|ms", "foobar:100|ms", "foobar:400|ms", "baz:500|ms")
	checkAllApprox(c, []testCase{
		{"foobar.timer.count", 3.0},
		{"foobar.timer.rate", 1.5},
		{"foobar.timer.min", 100.0},
		{"foobar.timer.max", 400.0},
		{"baz.timer.count", 1.0},
	})
}

func (*GostSuite) TestGauges(c *C) {
	sendGostMessages(c, "foobar:3|g")
	// Hack to ensure that the first foobar message gets processed before the second
	// TODO: find a better way
	time.Sleep(time.Millisecond)
	sendGostMessages(c, "foobar:4|g", "baz:1|g")

	checkAllApprox(c, []testCase{
		{"foobar.gauge", 4.0},
		{"baz.gauge", 1.0},
	})
}

func (*GostSuite) TestSets(c *C) {
	sendGostMessages(c, "foobar:123|s", "foobar:234|s", "foobar:123|s", "baz:456|s")
	checkAllApprox(c, []testCase{
		{"foobar.set", 2.0},
		{"baz.set", 1.0},
	})
}

func (*GostSuite) TestMetaStats(c *C) {
	sendGostMessages(c, "foobar:2|c", "foobar:3|g", "foobar:asdf|s")
	sendGostMessages(c, "baz:300|g")
	sendGostMessages(c, "baz:300|asdfasdf")
	checkAllApprox(c, []testCase{
		{"gost.bad_messages_seen.count", 2.0},
		{"gost.packets_received.count", 5.0},
		// foobar/count (2), foobar/gauge (1), baz/gauge (1), gost.packets_received/count (2),
		// gost.bad_messages_seen/count (2), gost.distinct_metrics_flushed/gauge (1)
		// total = 9
		{"gost.distinct_metrics_flushed.gauge", 9.0},
	})
}

func (*GostSuite) TestWithStatClearing(c *C) {
	conf.ClearStatsBetweenFlushes = true
	sendGostMessages(c, "a:1|c")
	sendGostMessages(c, "b:2|ms")
	sendGostMessages(c, "c:3|g")
	sendGostMessages(c, "d:4|s")
	rec.waitForMessage()
	sendGostMessages(c, "foobar:2|c")
	msg := rec.waitForMessage()
	for _, key := range []string{"a", "b", "c", "d"} {
		c.Check(msg.Parsed["com.example."+key+".count"], IsNil)
	}
	c.Check(msg.Parsed["com.example.foobar.count"].Value, approx, 2.0)
}

func (*GostSuite) TestWithoutStatClearing(c *C) {
	conf.ClearStatsBetweenFlushes = false
	sendGostMessages(c, "a:1|c", "b:2|ms", "c:3|g", "d:4|s")
	time.Sleep(time.Millisecond)
	aggregateFlushChan <- when
	<-rec.messages
	sendGostMessages(c, "foobar:2|c")
	time.Sleep(time.Millisecond)
	aggregateFlushChan <- when
	msg := parseRawGraphiteMessage(<-rec.messages)
	c.Check(msg.Parsed["com.example.a.count"].Value, approx, 0.0)
	c.Check(msg.Parsed["com.example.a.rate"].Value, approx, 0.0)
	c.Check(msg.Parsed["com.example.b.timer.count"], IsNil)
	c.Check(msg.Parsed["com.example.c.gauge"].Value, approx, 3.0)
	c.Check(msg.Parsed["com.example.d.set"].Value, approx, 0.0)
	c.Check(msg.Parsed["com.example.foobar.count"].Value, approx, 2.0)
}

func (*GostSuite) TestSanitization(c *C) {
	allChars := []byte{}
	for i := 33; i <= 126; i++ {
		c := byte(i)
		switch c {
		case '*', '/', ':', '<', '>', '[', ']', '{', '}':
			continue
		}
		allChars = append(allChars, c)
	}
	sendGostMessages(c,
		fmt.Sprintf("%s:1|c", allChars),
		"föo\tbar:1|c",  // Non-printable or non-ascii is removed
		"foo bar:1|c",   // spaces changed to _
		"foo/bar:1|c",   // / changed to -
		"rem*ove1:1|c",  // * removed
		"<remove2>:1|c", // < > removed
		"[remove3]:1|c", // [ ] removed
		"{remove4}:1|c", // { } removed
	)
	checkAllApprox(c, []testCase{
		{fmt.Sprintf("%s.count", allChars), 1.0},
		{"fobar.count", 1.0},
		{"foo_bar.count", 1.0},
		{"foo-bar.count", 1.0},
		{"remove1.count", 1.0},
		{"remove2.count", 1.0},
		{"remove3.count", 1.0},
		{"remove4.count", 1.0},
	})
}

func (*GostSuite) TestForwardedKeyParsing(c *C) {
	forwardingEnabled = true
	sendGostMessages(c, "f|foo:1|c", "f|f|bar:1|c", "f||baz:1|c", "quf|ux:1|c")
	checkAllApprox(c, []testCase{
		{"quf|ux.count", 1.0},
	})

	// Push the forwarded messages through
	aggregateForwardingFlushChan <- when
	time.Sleep(time.Millisecond)
	aggregateForwardedFlushChan <- when
	msg := parseRawGraphiteMessage(<-rec.messages)
	c.Check(msg.Parsed["global.foo.count"].Value, approx, 1.0)
	c.Check(msg.Parsed["global.f|bar.count"].Value, approx, 1.0)
	c.Check(msg.Parsed["global.|baz.count"].Value, approx, 1.0)
}

func (*GostSuite) TestSampleRates(c *C) {
	sendGostMessages(c, "a:1|c|@0.1", "b:1|c|@1.0", "c:1|c|@3.0", "d:1|c|@0.0", "e:1|c|@-0.5")
	msg := rec.waitForMessage()
	c.Check(msg.Parsed["com.example.a.count"].Value, approx, 10.0)
	c.Check(msg.Parsed["com.example.b.count"].Value, approx, 1.0)
	for _, key := range []string{"c", "d", "e"} {
		c.Check(msg.Parsed["com.example."+key+".count"], IsNil)
	}
}

func (*GostSuite) TestMultilineMessage(c *C) {
	sendGostMessages(c, "foobar:3|c\nfoobar:5|c\nbaz:200|g")
	checkAllApprox(c, []testCase{
		{"foobar.count", 8.0},
		{"foobar.rate", 4.0},
		{"baz.gauge", 200.0},
		{"gost.packets_received.count", 1.0},
		{"gost.packets_received.rate", 0.5},
	})
}
<commit_msg>Test for checking that forwarding doesn't happen when disabled<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"strings"
	"sync"
	"testing"
	"time"

	. "launchpad.net/gocheck"
)

// ----------- Gocheck config ----------------

func Test(t *testing.T) { TestingT(t) }

type GostSuite struct{}

var _ = Suite(&GostSuite{})

// ----------- Setup ----------------

var (
	rec                          = &recorder{}
	testUDPConn                  *net.UDPConn
	testForwardListener          net.Listener
	when                         = time.Time{}
	aggregateFlushChan           = make(chan time.Time)
	aggregateForwardedFlushChan  = make(chan time.Time)
	aggregateForwardingFlushChan = make(chan time.Time)
	flushers                     int
)

func (*GostSuite) SetUpSuite(c *C) {
	// Stub out control functions
	now = func() time.Time { return when }
	aggregateFlushTicker = func() <-chan time.Time { return aggregateFlushChan }
	aggregateForwardedFlushTicker = func() <-chan time.Time { return aggregateForwardedFlushChan }
	aggregateForwardingFlushTicker = func() <-chan time.Time { return aggregateForwardingFlushChan }

	// Start workers
	go flush()
	go aggregate()
	// Don't want to see bad output lines (we could capture and inspect these if we want to test them later).
	log.SetOutput(ioutil.Discard)

	var err error
	testForwardListener, err = net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		c.Fatal(err)
	}
	forwardingIncoming = make(chan *Stat, incomingQueueSize)
	go flushForwarding()
	go aggregateForwarding()
	go aggregateForwarded()
	go forwardServer(testForwardListener)

	// Bind to any port
	if err := debugServer.Start(0); err != nil {
		c.Fatal(err)
	}

	udp, err := net.ResolveUDPAddr("udp", "127.0.0.1:0")
	if err != nil {
		c.Fatal(err)
	}
	testUDPConn, err = net.ListenUDP("udp", udp)
	if err != nil {
		c.Fatal(err)
	}
	go clientServer(testUDPConn)
}

func (*GostSuite) TearDownSuite(c *C) {
	testUDPConn.Close()
	testForwardListener.Close()
}

func (*GostSuite) SetUpTest(c *C) {
	rec.Start()
	conf = &Conf{
		GraphiteAddr:             rec.Addr,
		ForwardingAddr:           testForwardListener.Addr().String(),
		ForwardedNamespace:       "global",
		ClearStatsBetweenFlushes: true,
		FlushIntervalMS:          2000, // Fake
		Namespace:                "com.example",
	}
}

func (*GostSuite) TearDownTest(c *C) {
	rec.Close()
}

type graphiteValue struct {
	Value     float64
	Timestamp time.Time
}

type graphiteMessage struct {
	Raw    []byte
	Parsed map[string]*graphiteValue
}

func parseRawGraphiteMessage(raw []byte) *graphiteMessage {
	parsed := make(map[string]*graphiteValue)
	for _, line := range strings.Split(string(raw), "\n") {
		line = strings.TrimSpace(line)
		if len(line) == 0 {
			continue
		}
		key := ""
		value := &graphiteValue{}
		var unixTime int64
		if _, err := fmt.Sscanf(line, "%s %f %d", &key, &value.Value, &unixTime); err != nil {
			panic("Not a graphite message: " + line)
		}
		value.Timestamp = time.Unix(unixTime, 0)
		parsed[key] = value
	}
	return &graphiteMessage{raw, parsed}
}

type recorder struct {
	W        *sync.WaitGroup
	Addr     string
	Listener net.Listener
	messages chan []byte
}

// waitForMessage flushes the aggregator and returns the collected messages.
func (r *recorder) waitForMessage() *graphiteMessage {
	// Ensure the aggregator has time to collect all the messages we've sent in.
	time.Sleep(time.Millisecond)
	aggregateFlushChan <- when
	return parseRawGraphiteMessage(<-r.messages)
}

func (r *recorder) Start() {
	r.W = &sync.WaitGroup{}
	// Listen on a free port
	listener, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		panic(err)
	}
	r.Listener = listener
	r.Addr = listener.Addr().String()
	r.messages = make(chan []byte)
	go func() {
		for {
			c, err := r.Listener.Accept()
			if err != nil {
				return
			}
			go r.Handle(c)
		}
	}()
}

func (r *recorder) Handle(c net.Conn) {
	r.W.Add(1)
	defer r.W.Done()
	msg, err := ioutil.ReadAll(c)
	if err != nil {
		return
	}
	r.messages <- msg
}

func (r *recorder) Close() {
	r.Listener.Close()
	r.W.Wait()
}

func sendGostMessages(c *C, msgs ...string) {
	conn, err := net.Dial("udp", testUDPConn.LocalAddr().String())
	if err != nil {
		c.Fatal(err)
	}
	defer conn.Close()
	for _, msg := range msgs {
		conn.Write([]byte(msg))
	}
}

// approx is a gocheck checker for approximate equality of floats.
// Note that approximate equality of floats is a fraught topic. This is a very naive comparison.
type approxChecker struct {
	*CheckerInfo
}

var approx = &approxChecker{
	&CheckerInfo{Name: "approx", Params: []string{"obtained", "expected"}},
}

func (c *approxChecker) Check(params []interface{}, names []string) (result bool, error string) {
	f1, ok1 := params[0].(float64)
	f2, ok2 := params[1].(float64)
	if !(ok1 && ok2) {
		return false, "must compare float64s"
	}
	if f1 == f2 {
		return true, ""
	} else if f1 > f2 {
		f1, f2 = f2, f1
	}
	delta := (f2 - f1) / f1
	if delta < 0.0001 { // Accept f2 up to 0.01% greater than f1
		return true, ""
	}
	return false, ""
}

type testCase struct {
	Key   string
	Value float64
}

func checkAllApprox(c *C, tests []testCase) {
	msg := rec.waitForMessage()
	for _, test := range tests {
		key := "com.example." + test.Key
		parsed := msg.Parsed[key]
		if parsed == nil {
			fmt.Println(string(msg.Raw))
			c.Fatal("No graphite key found: ", key)
		}
		c.Check(parsed.Value, approx, test.Value)
	}
}

// ----------- Tests ----------------

func (*GostSuite) TestCounters(c *C) {
	sendGostMessages(c, "foobar:3|c", "foobar:5|c", "baz:2|c|@0.1", "baz:4|c|@0.1")
	checkAllApprox(c, []testCase{
		{"foobar.count", 8.0},
		{"foobar.rate", 4.0},
		{"baz.count", 60.0},
		{"baz.rate", 30.0},
	})
}

func (*GostSuite) TestTimers(c *C) {
	sendGostMessages(c, "foobar:100|ms", "foobar:100|ms", "foobar:400|ms", "baz:500|ms")
	checkAllApprox(c, []testCase{
		{"foobar.timer.count", 3.0},
		{"foobar.timer.rate", 1.5},
		{"foobar.timer.min", 100.0},
		{"foobar.timer.max", 400.0},
		{"baz.timer.count", 1.0},
	})
}

func (*GostSuite) TestGauges(c *C) {
	sendGostMessages(c, "foobar:3|g")
	// Hack to ensure that the first foobar message gets processed before the second
	// TODO: find a better way
	time.Sleep(time.Millisecond)
	sendGostMessages(c, "foobar:4|g", "baz:1|g")

	checkAllApprox(c, []testCase{
		{"foobar.gauge", 4.0},
		{"baz.gauge", 1.0},
	})
}

func (*GostSuite) TestSets(c *C) {
	sendGostMessages(c, "foobar:123|s", "foobar:234|s", "foobar:123|s", "baz:456|s")
	checkAllApprox(c, []testCase{
		{"foobar.set", 2.0},
		{"baz.set", 1.0},
	})
}

func (*GostSuite) TestMetaStats(c *C) {
	sendGostMessages(c, "foobar:2|c", "foobar:3|g", "foobar:asdf|s")
	sendGostMessages(c, "baz:300|g")
	sendGostMessages(c, "baz:300|asdfasdf")
	checkAllApprox(c, []testCase{
		{"gost.bad_messages_seen.count", 2.0},
		{"gost.packets_received.count", 5.0},
		// foobar/count (2), foobar/gauge (1), baz/gauge (1), gost.packets_received/count (2),
		// gost.bad_messages_seen/count (2), gost.distinct_metrics_flushed/gauge (1)
		// total = 9
		{"gost.distinct_metrics_flushed.gauge", 9.0},
	})
}

func (*GostSuite) TestWithStatClearing(c *C) {
	conf.ClearStatsBetweenFlushes = true
	sendGostMessages(c, "a:1|c")
	sendGostMessages(c, "b:2|ms")
	sendGostMessages(c, "c:3|g")
	sendGostMessages(c, "d:4|s")
	rec.waitForMessage()
	sendGostMessages(c, "foobar:2|c")
	msg := rec.waitForMessage()
	for _, key := range []string{"a", "b", "c", "d"} {
		c.Check(msg.Parsed["com.example."+key+".count"], IsNil)
	}
	c.Check(msg.Parsed["com.example.foobar.count"].Value, approx, 2.0)
}

func (*GostSuite) TestWithoutStatClearing(c *C) {
	conf.ClearStatsBetweenFlushes = false
	sendGostMessages(c, "a:1|c", "b:2|ms", "c:3|g", "d:4|s")
	time.Sleep(time.Millisecond)
	aggregateFlushChan <- when
	<-rec.messages
	sendGostMessages(c, "foobar:2|c")
	time.Sleep(time.Millisecond)
	aggregateFlushChan <- when
	msg := parseRawGraphiteMessage(<-rec.messages)
	c.Check(msg.Parsed["com.example.a.count"].Value, approx, 0.0)
	c.Check(msg.Parsed["com.example.a.rate"].Value, approx, 0.0)
	c.Check(msg.Parsed["com.example.b.timer.count"], IsNil)
	c.Check(msg.Parsed["com.example.c.gauge"].Value, approx, 3.0)
	c.Check(msg.Parsed["com.example.d.set"].Value, approx, 0.0)
	c.Check(msg.Parsed["com.example.foobar.count"].Value, approx, 2.0)
}

func (*GostSuite) TestSanitization(c *C) {
	allChars := []byte{}
	for i := 33; i <= 126; i++ {
		c := byte(i)
		switch c {
		case '*', '/', ':', '<', '>', '[', ']', '{', '}':
			continue
		}
		allChars = append(allChars, c)
	}
	sendGostMessages(c,
		fmt.Sprintf("%s:1|c", allChars),
		"föo\tbar:1|c",  // Non-printable or non-ascii is removed
		"foo bar:1|c",   // spaces changed to _
		"foo/bar:1|c",   // / changed to -
		"rem*ove1:1|c",  // * removed
		"<remove2>:1|c", // < > removed
		"[remove3]:1|c", // [ ] removed
		"{remove4}:1|c", // { } removed
	)
	checkAllApprox(c, []testCase{
		{fmt.Sprintf("%s.count", allChars), 1.0},
		{"fobar.count", 1.0},
		{"foo_bar.count", 1.0},
		{"foo-bar.count", 1.0},
		{"remove1.count", 1.0},
		{"remove2.count", 1.0},
		{"remove3.count", 1.0},
		{"remove4.count", 1.0},
	})
}

func (*GostSuite) TestForwardedKeyParsing(c *C) {
	forwardingEnabled = true
	sendGostMessages(c, "f|foo:1|c", "f|f|bar:1|c", "f||baz:1|c", "quf|ux:1|c")
	checkAllApprox(c, []testCase{
		{"quf|ux.count", 1.0},
	})

	// Push the forwarded messages through
	aggregateForwardingFlushChan <- when
	time.Sleep(time.Millisecond)
	aggregateForwardedFlushChan <- when
	msg := parseRawGraphiteMessage(<-rec.messages)
	c.Check(msg.Parsed["global.foo.count"].Value, approx, 1.0)
	c.Check(msg.Parsed["global.f|bar.count"].Value, approx, 1.0)
	c.Check(msg.Parsed["global.|baz.count"].Value, approx, 1.0)
}

func (*GostSuite) TestNoForwarding(c *C) {
	forwardingEnabled = false
	sendGostMessages(c, "f|foo:1|c")
	checkAllApprox(c, []testCase{
		{"f|foo.count", 1.0},
	})
}

func (*GostSuite) TestSampleRates(c *C) {
	sendGostMessages(c, "a:1|c|@0.1", "b:1|c|@1.0", "c:1|c|@3.0", "d:1|c|@0.0", "e:1|c|@-0.5")
	msg := rec.waitForMessage()
	c.Check(msg.Parsed["com.example.a.count"].Value, approx, 10.0)
	c.Check(msg.Parsed["com.example.b.count"].Value, approx, 1.0)
	for _, key := range []string{"c", "d", "e"} {
		c.Check(msg.Parsed["com.example."+key+".count"], IsNil)
	}
}

func (*GostSuite) TestMultilineMessage(c *C) {
	sendGostMessages(c, "foobar:3|c\nfoobar:5|c\nbaz:200|g")
	checkAllApprox(c, []testCase{
		{"foobar.count", 8.0},
		{"foobar.rate", 4.0},
		{"baz.gauge", 200.0},
		{"gost.packets_received.count", 1.0},
		{"gost.packets_received.rate", 0.5},
	})
}
<|endoftext|>


<commit_before>package newcot

import (
	"math/rand"
	"testing"
	"unsafe"

	"github.com/ready-steady/support/assert"
)

func TestHashTap(t *testing.T) {
	const (
		capacity = 10
	)

	hash := newHash(2, capacity)

	assert.Equal(hash.tap([]uint64{4, 2}), false, t)
	assert.Equal(hash.tap([]uint64{4, 2}), true, t)

	keys := make([]string, 0, capacity)
	for k, _ := range hash.mapping {
		keys = append(keys, k)
	}

	assert.Equal(len(keys), 1, t)

	if isLittleEndian() {
		assert.Equal(keys[0],
			"\x04\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00", t)
	} else {
		assert.Equal(keys[0],
			"\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x02", t)
	}
}

func isLittleEndian() bool {
	var x uint32 = 0x01020304
	return *(*byte)(unsafe.Pointer(&x)) == 0x04
}

func BenchmarkHashTap(b *testing.B) {
	const (
		dimensionCount = 20
		parentCount    = 1000

		depth    = dimensionCount
		capacity = 2 * parentCount * dimensionCount
	)

	generator := rand.New(rand.NewSource(0))
	data := make([]uint64, 2*parentCount*dimensionCount)
	for _, i := range data {
		data[i] = uint64(generator.Int63())
	}

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		hash := newHash(depth, capacity)
		for j := 0; j < parentCount; j++ {
			hash.tap(data[2*j*depth : (2*j+1)*depth])
		}
	}
}
<commit_msg>Adjusted Hash’s benchmark once again<commit_after>package newcot

import (
	"math/rand"
	"testing"
	"unsafe"

	"github.com/ready-steady/support/assert"
)

func TestHashTap(t *testing.T) {
	const (
		capacity = 10
	)

	hash := newHash(2, capacity)

	assert.Equal(hash.tap([]uint64{4, 2}), false, t)
	assert.Equal(hash.tap([]uint64{4, 2}), true, t)

	keys := make([]string, 0, capacity)
	for k, _ := range hash.mapping {
		keys = append(keys, k)
	}

	assert.Equal(len(keys), 1, t)

	if isLittleEndian() {
		assert.Equal(keys[0],
			"\x04\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00", t)
	} else {
		assert.Equal(keys[0],
			"\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x02", t)
	}
}

func isLittleEndian() bool {
	var x uint32 = 0x01020304
	return *(*byte)(unsafe.Pointer(&x)) == 0x04
}

func BenchmarkHashTap(b *testing.B) {
	const (
		dimensionCount = 20
		parentCount    = 1000

		depth    = dimensionCount
		capacity = 2 * parentCount * dimensionCount
	)

	generator := rand.New(rand.NewSource(0))
	data := make([]uint64, 2*parentCount*dimensionCount)
	for _, i := range data {
		data[i] = uint64(generator.Int63())
	}

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		hash := newHash(depth, capacity)
		for j := 0; j < parentCount; j++ {
			hash.tap(data[(2*j+0)*depth : (2*j+1)*depth])
			hash.tap(data[(2*j+1)*depth : (2*j+2)*depth])
		}
	}
}
<|endoftext|>


<commit_before>package agent

import (
	"fmt"
	"github.com/golang/glog"
	"github.com/romana/core/common"
	"github.com/romana/core/pkg/util/firewall"
)

// addPolicy is a placeholder. TODO
func (a *Agent) addPolicy(input interface{}, ctx common.RestContext) (interface{}, error) {
	//	policy := input.(*common.Policy)
	return nil, nil
}

// deletePolicy is a placeholder. TODO
func (a *Agent) deletePolicy(input interface{}, ctx common.RestContext) (interface{}, error) {
	//	policyId := ctx.PathVariables["policyID"]
	return nil, nil
}

// listPolicies is a placeholder. TODO.
func (a *Agent) listPolicies(input interface{}, ctx common.RestContext) (interface{}, error) {
	return nil, nil
}

// statusHandler reports operational statistics.
func (a *Agent) statusHandler(input interface{}, ctx common.RestContext) (interface{}, error) {
	fw, err := firewall.NewFirewall(a.Helper.Executor, a.store, a.networkConfig)
	if err != nil {
		return nil, err
	}

	iptablesRules, err := fw.ListRules()
	if err != nil {
		return nil, err
	}
	return iptablesRules, nil
}

// k8sPodDownHandler cleans up after pod deleted.
func (a *Agent) k8sPodDownHandler(input interface{}, ctx common.RestContext) (interface{}, error) {
	glog.Infoln("Agent: Entering k8sPodDownHandler()")
	netReq := input.(*NetworkRequest)
	netif := netReq.NetIf

	fw, err := firewall.NewFirewall(a.Helper.Executor, a.store, a.networkConfig)
	if err != nil {
		return nil, err
	}

	err = fw.Cleanup(netif)
	if err != nil {
		return nil, err
	}

	// Spawn new thread to process the request
	glog.Infof("Agent: Got request for network configuration: %v\n", netReq)

	return "OK", nil
}

// k8sPodUpHandler handles HTTP requests for endpoints provisioning.
func (a *Agent) k8sPodUpHandler(input interface{}, ctx common.RestContext) (interface{}, error) {
	glog.Infof("Agent: Entering k8sPodUpHandler()")
	netReq := input.(*NetworkRequest)

	glog.Infof("Agent: Got request for network configuration: %v\n", netReq)
	// Spawn new thread to process the request

	// TODO don't know if fork-bombs are possible in go but if they are this
	// need to be refactored as buffered channel with fixed pool of workers
	go a.k8sPodUpHandle(*netReq)

	// TODO I wonder if this should actually return something like a
	// link to a status of this request which will later get updated
	// with success or failure -- Greg.
	return "OK", nil
}

// index handles HTTP requests for endpoints provisioning.
// Currently tested with Romana ML2 driver.
// TODO index should be reserved for an actual index, while this function
// need to be renamed as interfaceHandler and need to respond on it's own url.
func (a *Agent) index(input interface{}, ctx common.RestContext) (interface{}, error) {
	// Parse out NetIf form the request
	netif := input.(*NetIf)

	glog.Infof("Got interface: Name %s, IP %s Mac %s\n", netif.Name, netif.IP, netif.Mac)
	// Spawn new thread to process the request

	// TODO don't know if fork-bombs are possible in go but if they are this
	// need to be refactored as buffered channel with fixed pool of workers
	go a.interfaceHandle(*netif)

	// TODO I wonder if this should actually return something like a
	// link to a status of this request which will later get updated
	// with success or failure -- Greg.
	return "OK", nil
}

// k8sPodUpHandle does a number of operations on given endpoint to ensure
// it's connected:
// 1. Ensures interface is ready
// 2. Creates ip route pointing new interface
// 3. Provisions firewall rules
func (a *Agent) k8sPodUpHandle(netReq NetworkRequest) error {
	glog.Info("Agent: Entering k8sPodUpHandle()")

	netif := netReq.NetIf
	if netif.Name == "" {
		return agentErrorString("Agent: Interface name required")
	}
	if !a.Helper.waitForIface(netif.Name) {
		// TODO should we resubmit failed interface in queue for later
		// retry ? ... considering openstack will give up as well after
		// timeout
		msg := fmt.Sprintf("Requested interface not available in time - %s", netif.Name)
		glog.Infoln("Agent: ", msg)
		return agentErrorString(msg)
	}

	glog.Info("Agent: creating endpoint routes")
	if err := a.Helper.ensureRouteToEndpoint(&netif); err != nil {
		glog.Error(agentError(err))
		return agentError(err)
	}

	glog.Info("Agent: provisioning firewall")
	fw, err := firewall.NewFirewall(a.Helper.Executor, a.store, a.networkConfig)
	if err != nil {
		glog.Error(agentError(err))
		return agentError(err)
	}

	if err1 := fw.Init(netif); err1 != nil {
		glog.Error(agentError(err))
		return agentError(err)
	}

	metadata := fw.Metadata()
	chainNames := metadata["chains"].([]string)
	hostAddr := a.networkConfig.RomanaGW()
	hostMask, _ := a.networkConfig.RomanaGWMask().Size()

	// Default firewall rules for Kubernetes
	// Allow ICMP, DHCP and SSH between host and instances.
	var defaultRules []firewall.FirewallRule

	inboundChain := chainNames[firewall.InputChainIndex]
	inboundRule := firewall.NewFirewallRule()
	inboundRule.SetBody(fmt.Sprintf("%s %s", inboundChain, "-p tcp --sport 22 -j ACCEPT"))
	defaultRules = append(defaultRules, inboundRule)

	inboundRule = firewall.NewFirewallRule()
	inboundRule.SetBody(fmt.Sprintf("%s %s", inboundChain, "-p icmp --icmp-type 0 -j ACCEPT"))
	defaultRules = append(defaultRules, inboundRule)

	inboundRule = firewall.NewFirewallRule()
	inboundRule.SetBody(fmt.Sprintf("%s -d %s/%d %s", inboundChain, hostAddr, hostMask, "-j DROP"))
	defaultRules = append(defaultRules, inboundRule)

	outboundChain := chainNames[firewall.OutputChainIndex]
	outboundRule := firewall.NewFirewallRule()
	outboundRule.SetBody(fmt.Sprintf("%s %s", outboundChain, "-p tcp --dport 22 -j ACCEPT"))
	defaultRules = append(defaultRules, outboundRule)

	forwardInChain := chainNames[firewall.ForwardInChainIndex]
	forwardInRule := firewall.NewFirewallRule()
	forwardInRule.SetBody(fmt.Sprintf("%s %s", forwardInChain, "-m comment --comment Outgoing -j ACCEPT"))
	defaultRules = append(defaultRules, forwardInRule)

	forwardOutChain := chainNames[firewall.ForwardOutChainIndex]
	forwardOutRule := firewall.NewFirewallRule()
	forwardOutRule.SetBody(fmt.Sprintf("%s %s", forwardOutChain, "-m state --state RELATED,ESTABLISHED -j ACCEPT"))
	defaultRules = append(defaultRules, forwardOutRule)

	fw.SetDefaultRules(defaultRules)

	if err := fw.ProvisionEndpoint(); err != nil {
		glog.Error(agentError(err))
		return agentError(err)
	}

	glog.Info("Agent: All good", netif)
	return nil
}

// interfaceHandle does a number of operations on given endpoint to ensure
// it's connected:
// 1. Ensures interface is ready
// 2. Checks if DHCP is running
// 3. Creates ip route pointing new interface
// 4. Provisions static DHCP lease for new interface
// 5. Provisions firewall rules
func (a *Agent) interfaceHandle(netif NetIf) error {
	glog.Info("Agent: processing request to provision new interface")
	if !a.Helper.waitForIface(netif.Name) {
		// TODO should we resubmit failed interface in queue for later
		// retry ? ... considering oenstack will give up as well after
		// timeout
		return agentErrorString(fmt.Sprintf("Requested interface not available in time - %s", netif.Name))
	}

	// dhcpPid is only needed here for fail fast check
	// will try to poll the pid again in provisionLease
	glog.Info("Agent: checking if DHCP is running")
	_, err := a.Helper.DhcpPid()
	if err != nil {
		glog.Error(agentError(err))
		return agentError(err)
	}
	glog.Info("Agent: creating endpoint routes")
	if err := a.Helper.ensureRouteToEndpoint(&netif); err != nil {
		glog.Error(agentError(err))
		return agentError(err)
	}
	glog.Info("Agent: provisioning DHCP")
	if err := a.leaseFile.provisionLease(&netif); err != nil {
		glog.Error(agentError(err))
		return agentError(err)
	}

	glog.Info("Agent: provisioning firewall")
	fw, err := firewall.NewFirewall(a.Helper.Executor, a.store, a.networkConfig)
	if err != nil {
		glog.Error(agentError(err))
		return agentError(err)
	}

	if err1 := fw.Init(netif); err1 != nil {
		glog.Error(agentError(err))
		return agentError(err)
	}

	metadata := fw.Metadata()
	chainNames := metadata["chains"].([]string)
	hostAddr := a.networkConfig.RomanaGW()
	hostMask, _ := a.networkConfig.RomanaGWMask().Size()

	// Default firewall rules for OpenStack
	// Allow ICMP, DHCP and SSH between host and instances.
	inboundChain := chainNames[firewall.InputChainIndex]
	inboundRule := firewall.NewFirewallRule()
	inboundRule.SetBody(fmt.Sprintf("%s %s", inboundChain, "-d 255.255.255.255/32 -p udp -m udp --sport 68 --dport 67 -j ACCEPT"))
	inboundRule.SetBody(fmt.Sprintf("%s %s", inboundChain, "-p tcp --sport 22 -j ACCEPT"))
	inboundRule.SetBody(fmt.Sprintf("%s %s", inboundChain, "-p icmp --icmp-type 0 -j ACCEPT"))
	inboundRule.SetBody(fmt.Sprintf("%s -d %s/%d %s", inboundChain, hostAddr, hostMask, "-j DROP"))

	outboundChain := chainNames[firewall.OutputChainIndex]
	outboundRule := firewall.NewFirewallRule()
	outboundRule.SetBody(fmt.Sprintf("%s -s %s/32 -p udp -m udp --sport 67 --dport 68 -j ACCEPT", outboundChain, hostAddr))
	outboundRule.SetBody(fmt.Sprintf("%s %s", outboundChain, "-p tcp --dport 22 -j ACCEPT"))

	forwardInChain := chainNames[firewall.ForwardInChainIndex]
	forwardInRule := firewall.NewFirewallRule()
	forwardInRule.SetBody(fmt.Sprintf("%s %s", forwardInChain, "-m comment --comment Outgoing -j ACCEPT"))

	forwardOutChain := chainNames[firewall.ForwardOutChainIndex]
	forwardOutRule := firewall.NewFirewallRule()
	forwardOutRule.SetBody(fmt.Sprintf("%s %s", forwardOutChain, "-m state --state RELATED,ESTABLISHED -j ACCEPT"))

	fw.SetDefaultRules([]firewall.FirewallRule{inboundRule, outboundRule, forwardInRule, forwardOutRule})

	if err := fw.ProvisionEndpoint(); err != nil {
		glog.Error(agentError(err))
		return agentError(err)
	}
	if err := fw.ProvisionEndpoint(); err != nil {
		glog.Error(agentError(err))
		return agentError(err)
	}

	glog.Info("All good", netif)
	return nil
}
<commit_msg>Firewall interface for defining default rules (21)<commit_after>package agent

import (
	"fmt"
	"github.com/golang/glog"
	"github.com/romana/core/common"
	"github.com/romana/core/pkg/util/firewall"
)

// addPolicy is a placeholder. TODO
func (a *Agent) addPolicy(input interface{}, ctx common.RestContext) (interface{}, error) {
	//	policy := input.(*common.Policy)
	return nil, nil
}

// deletePolicy is a placeholder. TODO
func (a *Agent) deletePolicy(input interface{}, ctx common.RestContext) (interface{}, error) {
	//	policyId := ctx.PathVariables["policyID"]
	return nil, nil
}

// listPolicies is a placeholder. TODO.
func (a *Agent) listPolicies(input interface{}, ctx common.RestContext) (interface{}, error) {
	return nil, nil
}

// statusHandler reports operational statistics.
func (a *Agent) statusHandler(input interface{}, ctx common.RestContext) (interface{}, error) {
	fw, err := firewall.NewFirewall(a.Helper.Executor, a.store, a.networkConfig)
	if err != nil {
		return nil, err
	}

	iptablesRules, err := fw.ListRules()
	if err != nil {
		return nil, err
	}
	return iptablesRules, nil
}

// k8sPodDownHandler cleans up after pod deleted.
func (a *Agent) k8sPodDownHandler(input interface{}, ctx common.RestContext) (interface{}, error) {
	glog.Infoln("Agent: Entering k8sPodDownHandler()")
	netReq := input.(*NetworkRequest)
	netif := netReq.NetIf

	fw, err := firewall.NewFirewall(a.Helper.Executor, a.store, a.networkConfig)
	if err != nil {
		return nil, err
	}

	err = fw.Cleanup(netif)
	if err != nil {
		return nil, err
	}

	// Spawn new thread to process the request
	glog.Infof("Agent: Got request for network configuration: %v\n", netReq)

	return "OK", nil
}

// k8sPodUpHandler handles HTTP requests for endpoints provisioning.
func (a *Agent) k8sPodUpHandler(input interface{}, ctx common.RestContext) (interface{}, error) {
	glog.Infof("Agent: Entering k8sPodUpHandler()")
	netReq := input.(*NetworkRequest)

	glog.Infof("Agent: Got request for network configuration: %v\n", netReq)
	// Spawn new thread to process the request

	// TODO don't know if fork-bombs are possible in go but if they are this
	// need to be refactored as buffered channel with fixed pool of workers
	go a.k8sPodUpHandle(*netReq)

	// TODO I wonder if this should actually return something like a
	// link to a status of this request which will later get updated
	// with success or failure -- Greg.
	return "OK", nil
}

// index handles HTTP requests for endpoints provisioning.
// Currently tested with Romana ML2 driver.
// TODO index should be reserved for an actual index, while this function
// need to be renamed as interfaceHandler and need to respond on it's own url.
func (a *Agent) index(input interface{}, ctx common.RestContext) (interface{}, error) {
	// Parse out NetIf form the request
	netif := input.(*NetIf)

	glog.Infof("Got interface: Name %s, IP %s Mac %s\n", netif.Name, netif.IP, netif.Mac)
	// Spawn new thread to process the request

	// TODO don't know if fork-bombs are possible in go but if they are this
	// need to be refactored as buffered channel with fixed pool of workers
	go a.interfaceHandle(*netif)

	// TODO I wonder if this should actually return something like a
	// link to a status of this request which will later get updated
	// with success or failure -- Greg.
	return "OK", nil
}

// k8sPodUpHandle does a number of operations on given endpoint to ensure
// it's connected:
// 1. Ensures interface is ready
// 2. Creates ip route pointing new interface
// 3. Provisions firewall rules
func (a *Agent) k8sPodUpHandle(netReq NetworkRequest) error {
	glog.Info("Agent: Entering k8sPodUpHandle()")

	netif := netReq.NetIf
	if netif.Name == "" {
		return agentErrorString("Agent: Interface name required")
	}
	if !a.Helper.waitForIface(netif.Name) {
		// TODO should we resubmit failed interface in queue for later
		// retry ? ... considering openstack will give up as well after
		// timeout
		msg := fmt.Sprintf("Requested interface not available in time - %s", netif.Name)
		glog.Infoln("Agent: ", msg)
		return agentErrorString(msg)
	}

	glog.Info("Agent: creating endpoint routes")
	if err := a.Helper.ensureRouteToEndpoint(&netif); err != nil {
		glog.Error(agentError(err))
		return agentError(err)
	}

	glog.Info("Agent: provisioning firewall")
	fw, err := firewall.NewFirewall(a.Helper.Executor, a.store, a.networkConfig)
	if err != nil {
		glog.Error(agentError(err))
		return agentError(err)
	}

	if err1 := fw.Init(netif); err1 != nil {
		glog.Error(agentError(err))
		return agentError(err)
	}

	metadata := fw.Metadata()
	chainNames := metadata["chains"].([]string)
	hostAddr := a.networkConfig.RomanaGW()
	hostMask, _ := a.networkConfig.RomanaGWMask().Size()

	// Default firewall rules for Kubernetes
	// Allow ICMP, DHCP and SSH between host and instances.
	var defaultRules []firewall.FirewallRule

	// ProvisionEndpoint applies default rules in reverse order
	// so DROP goes first
	inboundRule = firewall.NewFirewallRule()
	inboundRule.SetBody(fmt.Sprintf("%s -d %s/%d %s", inboundChain, hostAddr, hostMask, "-j DROP"))
	defaultRules = append(defaultRules, inboundRule)

	inboundChain := chainNames[firewall.InputChainIndex]
	inboundRule := firewall.NewFirewallRule()
	inboundRule.SetBody(fmt.Sprintf("%s %s", inboundChain, "-p tcp --sport 22 -j ACCEPT"))
	defaultRules = append(defaultRules, inboundRule)

	inboundRule = firewall.NewFirewallRule()
	inboundRule.SetBody(fmt.Sprintf("%s %s", inboundChain, "-p icmp --icmp-type 0 -j ACCEPT"))
	defaultRules = append(defaultRules, inboundRule)

	outboundChain := chainNames[firewall.OutputChainIndex]
	outboundRule := firewall.NewFirewallRule()
	outboundRule.SetBody(fmt.Sprintf("%s %s", outboundChain, "-p tcp --dport 22 -j ACCEPT"))
	defaultRules = append(defaultRules, outboundRule)

	forwardInChain := chainNames[firewall.ForwardInChainIndex]
	forwardInRule := firewall.NewFirewallRule()
	forwardInRule.SetBody(fmt.Sprintf("%s %s", forwardInChain, "-m comment --comment Outgoing -j ACCEPT"))
	defaultRules = append(defaultRules, forwardInRule)

	forwardOutChain := chainNames[firewall.ForwardOutChainIndex]
	forwardOutRule := firewall.NewFirewallRule()
	forwardOutRule.SetBody(fmt.Sprintf("%s %s", forwardOutChain, "-m state --state RELATED,ESTABLISHED -j ACCEPT"))
	defaultRules = append(defaultRules, forwardOutRule)

	fw.SetDefaultRules(defaultRules)

	if err := fw.ProvisionEndpoint(); err != nil {
		glog.Error(agentError(err))
		return agentError(err)
	}

	glog.Info("Agent: All good", netif)
	return nil
}

// interfaceHandle does a number of operations on given endpoint to ensure
// it's connected:
// 1. Ensures interface is ready
// 2. Checks if DHCP is running
// 3. Creates ip route pointing new interface
// 4. Provisions static DHCP lease for new interface
// 5. Provisions firewall rules
func (a *Agent) interfaceHandle(netif NetIf) error {
	glog.Info("Agent: processing request to provision new interface")
	if !a.Helper.waitForIface(netif.Name) {
		// TODO should we resubmit failed interface in queue for later
		// retry ? ... considering oenstack will give up as well after
		// timeout
		return agentErrorString(fmt.Sprintf("Requested interface not available in time - %s", netif.Name))
	}

	// dhcpPid is only needed here for fail fast check
	// will try to poll the pid again in provisionLease
	glog.Info("Agent: checking if DHCP is running")
	_, err := a.Helper.DhcpPid()
	if err != nil {
		glog.Error(agentError(err))
		return agentError(err)
	}
	glog.Info("Agent: creating endpoint routes")
	if err := a.Helper.ensureRouteToEndpoint(&netif); err != nil {
		glog.Error(agentError(err))
		return agentError(err)
	}
	glog.Info("Agent: provisioning DHCP")
	if err := a.leaseFile.provisionLease(&netif); err != nil {
		glog.Error(agentError(err))
		return agentError(err)
	}

	glog.Info("Agent: provisioning firewall")
	fw, err := firewall.NewFirewall(a.Helper.Executor, a.store, a.networkConfig)
	if err != nil {
		glog.Error(agentError(err))
		return agentError(err)
	}

	if err1 := fw.Init(netif); err1 != nil {
		glog.Error(agentError(err))
		return agentError(err)
	}

	metadata := fw.Metadata()
	chainNames := metadata["chains"].([]string)
	hostAddr := a.networkConfig.RomanaGW()
	hostMask, _ := a.networkConfig.RomanaGWMask().Size()

	// Default firewall rules for OpenStack
	// Allow ICMP, DHCP and SSH between host and instances.
	inboundChain := chainNames[firewall.InputChainIndex]
	inboundRule := firewall.NewFirewallRule()
	inboundRule.SetBody(fmt.Sprintf("%s %s", inboundChain, "-d 255.255.255.255/32 -p udp -m udp --sport 68 --dport 67 -j ACCEPT"))
	inboundRule.SetBody(fmt.Sprintf("%s %s", inboundChain, "-p tcp --sport 22 -j ACCEPT"))
	inboundRule.SetBody(fmt.Sprintf("%s %s", inboundChain, "-p icmp --icmp-type 0 -j ACCEPT"))
	inboundRule.SetBody(fmt.Sprintf("%s -d %s/%d %s", inboundChain, hostAddr, hostMask, "-j DROP"))

	outboundChain := chainNames[firewall.OutputChainIndex]
	outboundRule := firewall.NewFirewallRule()
	outboundRule.SetBody(fmt.Sprintf("%s -s %s/32 -p udp -m udp --sport 67 --dport 68 -j ACCEPT", outboundChain, hostAddr))
	outboundRule.SetBody(fmt.Sprintf("%s %s", outboundChain, "-p tcp --dport 22 -j ACCEPT"))

	forwardInChain := chainNames[firewall.ForwardInChainIndex]
	forwardInRule := firewall.NewFirewallRule()
	forwardInRule.SetBody(fmt.Sprintf("%s %s", forwardInChain, "-m comment --comment Outgoing -j ACCEPT"))

	forwardOutChain := chainNames[firewall.ForwardOutChainIndex]
	forwardOutRule := firewall.NewFirewallRule()
	forwardOutRule.SetBody(fmt.Sprintf("%s %s", forwardOutChain, "-m state --state RELATED,ESTABLISHED -j ACCEPT"))

	fw.SetDefaultRules([]firewall.FirewallRule{inboundRule, outboundRule, forwardInRule, forwardOutRule})

	if err := fw.ProvisionEndpoint(); err != nil {
		glog.Error(agentError(err))
		return agentError(err)
	}
	if err := fw.ProvisionEndpoint(); err != nil {
		glog.Error(agentError(err))
		return agentError(err)
	}

	glog.Info("All good", netif)
	return nil
}
<|endoftext|>


<commit_before>package mcts

import (
	"golang.org/x/net/context"

	"github.com/nelhage/taktician/ai"
	"github.com/nelhage/taktician/tak"
)

func UniformRandomPolicy(ctx context.Context,
	m *MonteCarloAI,
	p *tak.Position, alloc *tak.Position) *tak.Position {
	moves := p.AllMoves(nil)
	var next *tak.Position
	for {
		r := m.r.Int31n(int32(len(moves)))
		m := moves[r]
		var e error
		if next, e = p.MovePreallocated(&m, alloc); e == nil {
			break
		}
		moves[0], moves[r] = moves[r], moves[0]
		moves = moves[1:]
	}
	return next
}

func NewMinimaxPolicy(cfg *MCTSConfig, depth int) PolicyFunc {
	mm := ai.NewMinimax(ai.MinimaxConfig{
		Size:    cfg.Size,
		NoTable: true,
		Depth:   depth,
		Seed:    cfg.Seed,
	})
	return func(ctx context.Context,
		m *MonteCarloAI,
		p *tak.Position, next *tak.Position) *tak.Position {
		move := mm.GetMove(ctx, p)
		next, _ = p.MovePreallocated(&move, next)
		return next
	}
}

func EvalWeightedPolicy(ctx context.Context,
	mc *MonteCarloAI,
	p *tak.Position, alloc *tak.Position) *tak.Position {
	var buf [500]tak.Move
	moves := p.AllMoves(buf[:])
	var best tak.Move
	var sum int64
	for _, m := range moves {
		child, e := p.MovePreallocated(&m, alloc)
		if e != nil {
			continue
		}
		w := mc.eval(&mc.c, child)
		if w > ai.WinThreshold {
			return child
		}
		w += 1000
		if w <= 0 {
			w = 1
		}
		sum += w
		if mc.r.Int63n(sum) < w {
			best = m
		}
	}
	next, _ := p.MovePreallocated(&best, alloc)
	return next
}
<commit_msg>base policy off current value<commit_after>package mcts

import (
	"golang.org/x/net/context"

	"github.com/nelhage/taktician/ai"
	"github.com/nelhage/taktician/tak"
)

func UniformRandomPolicy(ctx context.Context,
	m *MonteCarloAI,
	p *tak.Position, alloc *tak.Position) *tak.Position {
	moves := p.AllMoves(nil)
	var next *tak.Position
	for {
		r := m.r.Int31n(int32(len(moves)))
		m := moves[r]
		var e error
		if next, e = p.MovePreallocated(&m, alloc); e == nil {
			break
		}
		moves[0], moves[r] = moves[r], moves[0]
		moves = moves[1:]
	}
	return next
}

func NewMinimaxPolicy(cfg *MCTSConfig, depth int) PolicyFunc {
	mm := ai.NewMinimax(ai.MinimaxConfig{
		Size:    cfg.Size,
		NoTable: true,
		Depth:   depth,
		Seed:    cfg.Seed,
	})
	return func(ctx context.Context,
		m *MonteCarloAI,
		p *tak.Position, next *tak.Position) *tak.Position {
		move := mm.GetMove(ctx, p)
		next, _ = p.MovePreallocated(&move, next)
		return next
	}
}

func EvalWeightedPolicy(ctx context.Context,
	mc *MonteCarloAI,
	p *tak.Position, alloc *tak.Position) *tak.Position {
	var buf [500]tak.Move
	moves := p.AllMoves(buf[:])
	var best tak.Move
	var sum int64
	base := mc.eval(&mc.c, p) - 500
	for _, m := range moves {
		child, e := p.MovePreallocated(&m, alloc)
		if e != nil {
			continue
		}
		w := mc.eval(&mc.c, child)
		if w > ai.WinThreshold {
			return child
		}
		w -= base
		if w <= 0 {
			w = 1
		}
		sum += w
		if mc.r.Int63n(sum) < w {
			best = m
		}
	}
	next, _ := p.MovePreallocated(&best, alloc)
	return next
}
<|endoftext|>


<commit_before>package eth

import (
	"encoding/json"

	"github.com/ethereum/go-ethereum/common"
)

func WritePeers(path string, addresses []string) {
	if len(addresses) > 0 {
		data, _ := json.MarshalIndent(addresses, "", "    ")
		common.WriteFile(path, data)
	}
}

func ReadPeers(path string) (ips []string, err error) {
	var data string
	data, err = common.ReadAllFile(path)
	if err != nil {
		json.Unmarshal([]byte(data), &ips)
	}
	return
}
<commit_msg>eth: remove disused peer_util.go<commit_after><|endoftext|>


<commit_before>package dynect

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
	"time"
)

const (
	DynAPIPrefix = "https://api.dynect.net/REST"
)

var (
	PollingInterval  = 1 * time.Second
	ErrPromotedToJob = errors.New("promoted to job")
	ErrRateLimited   = errors.New("too many requests")
)

// handleJobRedirect overrides the net/http.DefaultClient's redirection policy
// function.
//
// This function will set the Content-Type, and Auth-Token headers, so that we
// don't get an error back from the API.
func handleJobRedirect(req *http.Request, via []*http.Request) error {
	// Set the Content-Type header.
	req.Header.Set("Content-Type", "application/json")

	// Now, try and divine the Auth-Token header's value from previous
	// requests.
	for _, r := range via {
		if authHdr := r.Header.Get("Auth-Token"); authHdr != "" {
			req.Header.Set("Auth-Token", authHdr)
			return nil
		}
	}
	return fmt.Errorf("failed to set Auth-Token header from previous requests")
}

// A client for use with DynECT's REST API.
type Client struct {
	Token        string
	CustomerName string
	transport    *http.Transport
	verbose      bool
}

// Creates a new Httpclient.
func NewClient(customerName string) *Client {
	return &Client{
		CustomerName: customerName,
		transport:    &http.Transport{Proxy: http.ProxyFromEnvironment},
	}
}

// Enable, or disable verbose output from the client.
//
// This will enable (or disable) logging messages that explain what the client
// is about to do, like the endpoint it is about to make a request to. If the
// request fails with an unexpected HTTP response code, then the response body
// will be logged out, as well.
func (c *Client) Verbose(p bool) {
	c.verbose = p
}

// Establishes a new session with the DynECT API.
func (c *Client) Login(username, password string) error {
	var req = LoginBlock{
		Username:     username,
		Password:     password,
		CustomerName: c.CustomerName}

	var resp LoginResponse

	err := c.Do("POST", "Session", req, &resp)
	if err != nil {
		return err
	}

	c.Token = resp.Data.Token
	return nil
}

func (c *Client) LoggedIn() bool {
	return len(c.Token) > 0
}

func (c *Client) Logout() error {
	return c.Do("DELETE", "Session", nil, nil)
}

// newRequest creates a new *http.Request, and sets the following headers:
//

//
Auth-Token

//
Content-Type

//

func (c *Client) newRequest(method, urlStr string, data []byte) (*http.Request, error) {
	var r *http.Request
	var err error

	if data != nil {
		r, err = http.NewRequest(method, urlStr, bytes.NewReader(data))
	} else {
		r, err = http.NewRequest(method, urlStr, nil)
	}

	r.Header.Set("Auth-Token", c.Token)
	r.Header.Set("Content-Type", "application/json")

	return r, err
}

func (c *Client) Do(method, endpoint string, requestData, responseData interface{}) error {
	// Throw an error if the user tries to make a request if the client is
	// logged out/unauthenticated, but make an exemption for when the
	// caller is trying to log in.
	if !c.LoggedIn() && method != "POST" && endpoint != "Session" {
		return errors.New("Will not perform request; client is closed")
	}

	var err error

	// Marshal the request data into a byte slice.
	if c.verbose {
		log.Println("dynect: marshaling request data")
	}
	var js []byte
	if requestData != nil {
		js, err = json.Marshal(requestData)
	} else {
		js = []byte("")
	}
	if err != nil {
		return err
	}

	urlStr := fmt.Sprintf("%s/%s", DynAPIPrefix, endpoint)

	// Create a new http.Request.
	req, err := c.newRequest(method, urlStr, js)
	if err != nil {
		return err
	}

	if c.verbose {
		log.Printf("Making %s request to %q", method, urlStr)
	}

	var resp *http.Response
	resp, err = c.transport.RoundTrip(req)

	if err != nil {
		if c.verbose {
			respBody, _ := ioutil.ReadAll(resp.Body)
			log.Printf("%s", string(respBody))
		}
		return err
	}
	defer resp.Body.Close()

	switch resp.StatusCode {
	case 200:
		if resp.ContentLength == 0 {
			// Zero-length content body?
			log.Println("dynect: warning: zero-length response body; skipping decoding of response")
			return nil
		}

		//dec := json.NewDecoder(resp.Body)
		text, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return fmt.Errorf("Could not read response body")
		}
		if err := json.Unmarshal(text, &responseData); err != nil {
			return fmt.Errorf("Error unmarshalling response:", err)
		}

		return nil

	case 307:
		// Handle the temporary redirect, which should point to a
		// /REST/Jobs endpoint.
		loc := resp.Header.Get("Location")
		log.Println("dynect: request is taking too long to complete: redirecting to", loc)

		// Going in to this blind, the documentation says that it will
		// return a URI when promoting a long-running request to a
		// job.
		//
		// Since a URL is technically a URI, we should do some checks
		// on the returned URI to sanitize it, and make sure that it is
		// in the format we would like it to be.
		if strings.HasPrefix(loc, "/REST/") {
			loc = strings.TrimLeft(loc, "/REST/")
		}
		if !strings.HasPrefix(loc, DynAPIPrefix) {
			loc = fmt.Sprintf("%s/%s", DynAPIPrefix, loc)
		}

		log.Println("Fetching location:", loc)

		// Generate a new request.
		req, err := c.newRequest("GET", loc, nil)
		if err != nil {
			return err
		}

		var jobData JobData

		// Poll the API endpoint, until we get a response back.
		for {
			select {
			case <-time.After(PollingInterval):
				resp, err := c.transport.RoundTrip(req)
				if err != nil {
					return err
				}
				defer resp.Body.Close()

				text, err := ioutil.ReadAll(resp.Body)
				//log.Println(string(text))
				if err != nil {
					return fmt.Errorf("Could not read response body:", err)
				}
				if err := json.Unmarshal(text, &jobData); err != nil {
					return fmt.Errorf("failed to decode job response body:", err)
				}

				// Check to see the status of the job.
				//
				// If it is "incomplete", loop around again.
				//
				// Should the job's status be "success", then
				// return the data, business-as-usual.
				//
				// TODO(nesv): Figure out what to do in the
				// event of a "failure" job status.

				switch jobData.Status {
				case "incomplete":
					continue
				case "success":
					if err := json.Unmarshal(text, &responseData); err != nil {
						return fmt.Errorf("failed to decode response body:", err)
					}
					return nil
				case "failure":
					return fmt.Errorf("request failed: %v", jobData.Messages)
				}
			}
		}

		return nil

	case 429:
		return ErrRateLimited
	}

	// If we got here, this means that the client does not know how to
	// interpret the response, and it should just error out.
	reason, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return fmt.Errorf("failed to read in response body")
	}
	return fmt.Errorf("server responded with %v: %v",
		resp.Status,
		string(reason))
}
<commit_msg>Check for errors when readings the response body<commit_after>package dynect

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
	"time"
)

const (
	DynAPIPrefix = "https://api.dynect.net/REST"
)

var (
	PollingInterval  = 1 * time.Second
	ErrPromotedToJob = errors.New("promoted to job")
	ErrRateLimited   = errors.New("too many requests")
)

// handleJobRedirect overrides the net/http.DefaultClient's redirection policy
// function.
//
// This function will set the Content-Type, and Auth-Token headers, so that we
// don't get an error back from the API.
func handleJobRedirect(req *http.Request, via []*http.Request) error {
	// Set the Content-Type header.
	req.Header.Set("Content-Type", "application/json")

	// Now, try and divine the Auth-Token header's value from previous
	// requests.
	for _, r := range via {
		if authHdr := r.Header.Get("Auth-Token"); authHdr != "" {
			req.Header.Set("Auth-Token", authHdr)
			return nil
		}
	}
	return fmt.Errorf("failed to set Auth-Token header from previous requests")
}

// A client for use with DynECT's REST API.
type Client struct {
	Token        string
	CustomerName string
	transport    *http.Transport
	verbose      bool
}

// Creates a new Httpclient.
func NewClient(customerName string) *Client {
	return &Client{
		CustomerName: customerName,
		transport:    &http.Transport{Proxy: http.ProxyFromEnvironment},
	}
}

// Enable, or disable verbose output from the client.
//
// This will enable (or disable) logging messages that explain what the client
// is about to do, like the endpoint it is about to make a request to. If the
// request fails with an unexpected HTTP response code, then the response body
// will be logged out, as well.
func (c *Client) Verbose(p bool) {
	c.verbose = p
}

// Establishes a new session with the DynECT API.
func (c *Client) Login(username, password string) error {
	var req = LoginBlock{
		Username:     username,
		Password:     password,
		CustomerName: c.CustomerName}

	var resp LoginResponse

	err := c.Do("POST", "Session", req, &resp)
	if err != nil {
		return err
	}

	c.Token = resp.Data.Token
	return nil
}

func (c *Client) LoggedIn() bool {
	return len(c.Token) > 0
}

func (c *Client) Logout() error {
	return c.Do("DELETE", "Session", nil, nil)
}

// newRequest creates a new *http.Request, and sets the following headers:
//

//
Auth-Token

//
Content-Type

//

func (c *Client) newRequest(method, urlStr string, data []byte) (*http.Request, error) {
	var r *http.Request
	var err error

	if data != nil {
		r, err = http.NewRequest(method, urlStr, bytes.NewReader(data))
	} else {
		r, err = http.NewRequest(method, urlStr, nil)
	}

	r.Header.Set("Auth-Token", c.Token)
	r.Header.Set("Content-Type", "application/json")

	return r, err
}

func (c *Client) Do(method, endpoint string, requestData, responseData interface{}) error {
	// Throw an error if the user tries to make a request if the client is
	// logged out/unauthenticated, but make an exemption for when the
	// caller is trying to log in.
	if !c.LoggedIn() && method != "POST" && endpoint != "Session" {
		return errors.New("Will not perform request; client is closed")
	}

	var err error

	// Marshal the request data into a byte slice.
	if c.verbose {
		log.Println("dynect: marshaling request data")
	}
	var js []byte
	if requestData != nil {
		js, err = json.Marshal(requestData)
	} else {
		js = []byte("")
	}
	if err != nil {
		return err
	}

	urlStr := fmt.Sprintf("%s/%s", DynAPIPrefix, endpoint)

	// Create a new http.Request.
	req, err := c.newRequest(method, urlStr, js)
	if err != nil {
		return err
	}

	if c.verbose {
		log.Printf("Making %s request to %q", method, urlStr)
	}

	var resp *http.Response
	resp, err = c.transport.RoundTrip(req)

	if err != nil {
		if c.verbose {
			respBody, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				return err
			}
			log.Printf("%s", string(respBody))
		}
		return err
	}
	defer resp.Body.Close()

	switch resp.StatusCode {
	case 200:
		if resp.ContentLength == 0 {
			// Zero-length content body?
			log.Println("dynect: warning: zero-length response body; skipping decoding of response")
			return nil
		}

		//dec := json.NewDecoder(resp.Body)
		text, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return fmt.Errorf("Could not read response body")
		}
		if err := json.Unmarshal(text, &responseData); err != nil {
			return fmt.Errorf("Error unmarshalling response:", err)
		}

		return nil

	case 307:
		// Handle the temporary redirect, which should point to a
		// /REST/Jobs endpoint.
		loc := resp.Header.Get("Location")
		log.Println("dynect: request is taking too long to complete: redirecting to", loc)

		// Going in to this blind, the documentation says that it will
		// return a URI when promoting a long-running request to a
		// job.
		//
		// Since a URL is technically a URI, we should do some checks
		// on the returned URI to sanitize it, and make sure that it is
		// in the format we would like it to be.
		if strings.HasPrefix(loc, "/REST/") {
			loc = strings.TrimLeft(loc, "/REST/")
		}
		if !strings.HasPrefix(loc, DynAPIPrefix) {
			loc = fmt.Sprintf("%s/%s", DynAPIPrefix, loc)
		}

		log.Println("Fetching location:", loc)

		// Generate a new request.
		req, err := c.newRequest("GET", loc, nil)
		if err != nil {
			return err
		}

		var jobData JobData

		// Poll the API endpoint, until we get a response back.
		for {
			select {
			case <-time.After(PollingInterval):
				resp, err := c.transport.RoundTrip(req)
				if err != nil {
					return err
				}
				defer resp.Body.Close()

				text, err := ioutil.ReadAll(resp.Body)
				//log.Println(string(text))
				if err != nil {
					return fmt.Errorf("Could not read response body:", err)
				}
				if err := json.Unmarshal(text, &jobData); err != nil {
					return fmt.Errorf("failed to decode job response body:", err)
				}

				// Check to see the status of the job.
				//
				// If it is "incomplete", loop around again.
				//
				// Should the job's status be "success", then
				// return the data, business-as-usual.
				//
				// TODO(nesv): Figure out what to do in the
				// event of a "failure" job status.

				switch jobData.Status {
				case "incomplete":
					continue
				case "success":
					if err := json.Unmarshal(text, &responseData); err != nil {
						return fmt.Errorf("failed to decode response body:", err)
					}
					return nil
				case "failure":
					return fmt.Errorf("request failed: %v", jobData.Messages)
				}
			}
		}

		return nil

	case 429:
		return ErrRateLimited
	}

	// If we got here, this means that the client does not know how to
	// interpret the response, and it should just error out.
	reason, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return fmt.Errorf("failed to read in response body")
	}
	return fmt.Errorf("server responded with %v: %v",
		resp.Status,
		string(reason))
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"

	"github.com/tantalum73/Go-APNS"
)

func main() {
	m := goapns.NewMessage().Title("Title").Body("A Test notification :)").Sound("Default")
	m.Custom("key", "val")
	c, err := goapns.NewConnection("../../../../Push Test Push Cert.p12", "PasswortdesZertifikates")
	if err != nil {
		fmt.Println("Error loading cert :(")
	} else {
		c.Development()
	}

	//fmt.Println(m)
	//iPad: 791660155ff167aa766730228fd33f4b0f22d83087448d106ef0a717ef5b2407
	//iPhone: 428dc1d681e576f69f337cd0061b1cdd8da9b76daab39203fa649c26187722c0
	tokens := []string{"791660155ff167aa766730228fd33f4b0f22d83087448d106ef0a717ef5b2407",
		"428dc1d681e576f69f337cd0061b1cdd8da9b76daab39203fa649c26187722c0"}
	ch := make(chan goapns.Response, len(tokens))
	//c.Push(m, tokens, ch)
	fmt.Println(m.JSONstring())

	for response := range ch {
		//fmt.Printf("\nReceived response: %v\n", response)
		if !response.Sent() {
			fmt.Printf("\nThere was an error sending to device %v : %v\n Timestamp: %v, == Zero %v\n\n", response.Token, response.Error, response.Timestamp(), response.Timestamp().IsZero())
		} else {
			fmt.Printf("\nPush successful for token: %v\n", response.Token)
		}

	}
}
<commit_msg>Updated Device Token<commit_after>package main

import (
	"fmt"

	"github.com/tantalum73/Go-APNS"
)

func main() {
	m := goapns.NewMessage().Title("Title").Body("A Test notification :)").Sound("Default").Badge(42)
	m.Custom("key", "val")
	c, err := goapns.NewConnection("../../../../Push Test Push Cert.p12", "PasswortdesZertifikates")
	if err != nil {
		fmt.Println("Erroror loading cert :(")
	} else {
		c.Development()
	}

	//fmt.Println(m)
	//iPad: a26f0000c05286ee6f31756b1b9d05b4a37ad512fabbe266dd21357b376f0e0e
	//iPhone: 428dc1d681e576f69f337cd0061b1cdd8da9b76daab39203fa649c26187722c0
	tokens := []string{"a26f0000c05286ee6f31756b1b9d05b4a37ad512fabbe266dd21357b376f0e0e",
		"428dc1d681e576f69f337cd0061b1cdd8da9b76daab39203fa649c26187722c0"}
	ch := make(chan goapns.Response, len(tokens))
	c.Push(m, tokens, ch)
	fmt.Println(m.JSONstring())

	for response := range ch {
		//fmt.Printf("\nReceived response: %v\n", response)
		if !response.Sent() {
			fmt.Printf("\nThere was an error sending to device %v : %v\n Timestamp: %v, == Zero %v\n\n", response.Token, response.Erroror, response.Timestamp(), response.Timestamp().IsZero())
		} else {
			fmt.Printf("\nPush successful for token: %v\n", response.Token)
		}

	}
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/kataras/iris"
	"github.com/plimble/gosock"
)

func main() {
	s := iris.New()

	ws := gosock.NewServer()

	ws.On("echo", func(ss *gosock.Socket, data []byte) {
		ss.Emit("echo", data)
	})

	s.Get("/socket", ws.WebHandler(nil))

	s.Listen(":9999")
}
<commit_msg>update<commit_after>package main

import (
	"github.com/kataras/iris"
	"github.com/plimble/gosock"
)

func main() {
	s := iris.New()

	ws := gosock.NewServer()

	ws.On("echo", func(ss *gosock.Socket, data []byte) {
		ss.Emit("echo", data)
	})

	s.Get("/socket", ws.WebHandler(nil, nil))

	s.Listen(":9999")
}
<|endoftext|>


<commit_before>package main

import (
  "bytes"
  "encoding/binary"
  //"crypto/tls"
  "log"
  //"time"
  "compress/zlib"
)

func init() {
}

func connect(config *NetworkConfig) {

}

func Publishv1(input chan []*FileEvent,
               registrar chan []*FileEvent,
               config *NetworkConfig) {
  var zbuf, packbuf bytes.Buffer
  socket := connect(config)
  for events := range input {

    for event := range events {
    }

    // Compress it
    // A new zlib writer  is used for every payload of events so that any
    // individual payload can be decompressed alone.
    // TODO(sissel): Make compression level tunable
    compressor, _ := zlib.NewWriterLevel(&buffer, 3)
    buffer.Truncate(0)
    _, err := compressor.Write(data)
    err = compressor.Flush()
    compressor.Close()

    // Loop forever trying to send.
    // This will cause reconnects/etc on failures automatically
    for {
      err = socket.Send(nonce, zmq.SNDMORE)
      if err != nil {
        continue // send failed, retry!
      }
      err = socket.Send(ciphertext, 0)
      if err != nil {
        continue // send failed, retry!
      }

      data, err = socket.Recv(0)
      // TODO(sissel): Figure out acknowledgement protocol? If any?
      if err == nil {
        break // success!
      }
    }

    // Tell the registrar that we've successfully sent these events
    registrar <- events
  } /* for each event payload */
} // Publish
<commit_msg>finish implementing lumberjack protocol in lumberjack2 (test works against logstash 1.1.13)<commit_after>package main

import (
  "bytes"
  "encoding/binary"
  "encoding/pem"
  "crypto/tls"
  "crypto/x509"
  "net"
  "io"
  "os"
  "io/ioutil"
  "log"
  "time"
  "compress/zlib"
  "strconv"
)

var hostname string
func init() {
  log.Printf("publisher init\n")
  hostname, _ = os.Hostname()

}

func Publishv1(input chan []*FileEvent,
               registrar chan []*FileEvent,
               config *NetworkConfig) {
  var buffer bytes.Buffer
  var socket *tls.Conn
  var sequence uint32
  var err error

  socket = connect(config)
  defer socket.Close()

  for events := range input {
    buffer.Truncate(0)
    compressor, _ := zlib.NewWriterLevel(&buffer, 3)

    for _, event := range events {
      sequence += 1
      writeDataFrame(event, sequence, compressor)
    }
    compressor.Flush()
    compressor.Close()

    compressed_payload := buffer.Bytes()

    // Send buffer until we're successful...
    oops := func(err error) {
      log.Printf("Socket error, will reconnect: %s\n", err)
      time.Sleep(1 * time.Second)
      socket.Close()
      socket = connect(config)
    }

    SendPayload: for {
      // Abort if our whole request takes longer than the configured
      // network timeout.
      socket.SetDeadline(time.Now().Add(config.Timeout))

      // Set the window size to the length of this payload in events.
      _, err = socket.Write([]byte("1W"))
      if err != nil { oops(err); continue }
      binary.Write(socket, binary.BigEndian, uint32(len(events)))
      if err != nil { oops(err); continue }

      // Write compressed frame
      socket.Write([]byte("1C"))
      if err != nil { oops(err); continue }
      binary.Write(socket, binary.BigEndian, uint32(len(compressed_payload)))
      if err != nil { oops(err); continue }
      _, err = socket.Write(compressed_payload)
      if err != nil { oops(err); continue }

      // read ack
      response := make([]byte, 0, 6)
      ackbytes := 0
      for ackbytes != 6 {
        n, err := socket.Read(response[len(response):cap(response)])
        if err != nil {
          log.Printf("Read error looking for ack: %s\n", err)
          socket.Close()
          socket = connect(config)
          continue SendPayload // retry sending on new connection
        } else {
          ackbytes += n
        }
      }

      // TODO(sissel): verify ack
      // Success, stop trying to send the payload.
      break
    }

    // Tell the registrar that we've successfully sent these events
    registrar <- events
  } /* for each event payload */
} // Publish

func connect(config *NetworkConfig) (socket *tls.Conn) {
  var tlsconfig tls.Config

  if len(config.SSLCertificate) > 0 && len(config.SSLKey) > 0 {
    log.Printf("Loading client ssl certificate: %s and %s\n",
               config.SSLCertificate, config.SSLKey)
    cert, err := tls.LoadX509KeyPair(config.SSLCertificate, config.SSLKey)
    if err != nil {
      log.Fatalf("Failed loading client ssl certificate: %s\n", err)
    }
    tlsconfig.Certificates = []tls.Certificate{cert}
  }

  if len(config.SSLCA) > 0 {
    log.Printf("Setting trusted CA from file: %s\n", config.SSLCA)
    tlsconfig.RootCAs = x509.NewCertPool()

    pemdata, err := ioutil.ReadFile(config.SSLCA)
    if err != nil { log.Fatalf("Failure reading CA certificate: %s\n", err) }

    block, _ := pem.Decode(pemdata)
    if block == nil {
      log.Fatalf("Failed to decode PEM data, is %s a valid cert?\n", config.SSLCA)
    }
    if block.Type != "CERTIFICATE" {
      log.Fatalf("This is not a certificate file: %s\n", config.SSLCA)
    }

    cert, err := x509.ParseCertificate(block.Bytes)
    if err != nil {
      log.Fatalf("Failed to parse a certificate: %s\n", config.SSLCA)
    }
    tlsconfig.RootCAs.AddCert(cert)
  }

  address := "127.0.0.1:3333"
  for {
    tcpsocket, err := net.DialTimeout("tcp", address, config.Timeout)
    if err != nil {
      log.Printf("Failure connecting to %s: %s\n", address, err)
      time.Sleep(1 * time.Second)
      continue
    }

    socket = tls.Client(tcpsocket, &tlsconfig)
    socket.SetDeadline(time.Now().Add(config.Timeout))
    err = socket.Handshake()
    if err != nil {
      log.Printf("Failed to tls handshake with %s %s\n", address, err)
      time.Sleep(1 * time.Second)
      socket.Close()
      continue
    }

    log.Printf("Connected to %s\n", address)

    // connected, let's rock and roll.
    return
  }
  return
}

func writeDataFrame(event *FileEvent, sequence uint32, output io.Writer) {
  //log.Printf("event: %s\n", *event.Text)
  // header, "1D"
  output.Write([]byte("1D"))
  // sequence number
  binary.Write(output, binary.BigEndian, uint32(sequence))
  // 'pair' count
  binary.Write(output, binary.BigEndian, uint32(len(*event.Fields) + 3))

  writeKV("file", *event.Source, output)
  writeKV("offset", strconv.FormatInt(event.Offset, 10), output)
  writeKV("line", *event.Text, output)
  for k, v := range(*event.Fields) {
    writeKV(k, v, output)
  }
}

func writeKV(key string, value string, output io.Writer) {
  //log.Printf("kv: %d/%s %d/%s\n", len(key), key, len(value), value)
  binary.Write(output, binary.BigEndian, uint32(len(key)))
  output.Write([]byte(key))
  binary.Write(output, binary.BigEndian, uint32(len(value)))
  output.Write([]byte(value))
}
<|endoftext|>