<commit_before>package node

import (
	"fmt"
	"github.com/MG-RAST/Shock/shock-server/conf"
	e "github.com/MG-RAST/Shock/shock-server/errors"
	"github.com/MG-RAST/Shock/shock-server/logger"
	"github.com/MG-RAST/Shock/shock-server/node"
	"github.com/MG-RAST/Shock/shock-server/request"
	"github.com/MG-RAST/Shock/shock-server/responder"
	"github.com/MG-RAST/Shock/shock-server/user"
	"github.com/MG-RAST/Shock/shock-server/util"
	"github.com/MG-RAST/golib/go-uuid/uuid"
	"github.com/MG-RAST/golib/mgo/bson"
	"github.com/MG-RAST/golib/stretchr/goweb/context"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"time"
)

var (
	RangeRegex = regexp.MustCompile(`^([\[\]]{1})(.*);(.*)([\]\[]{1})$`)
)

const (
	longDateForm  = "2006-01-02T15:04:05-07:00"
	shortDateForm = "2006-01-02"
)

// GET: /node
// To do:
// - Iterate node queries
func (cr *NodeController) ReadMany(ctx context.Context) error {
	u, err := request.Authenticate(ctx.HttpRequest())
	if err != nil && err.Error() != e.NoAuth {
		return request.AuthError(err, ctx)
	}

	// Gather query params
	query := ctx.HttpRequest().URL.Query()

	// Setup query and nodes objects
	// Note: query is composed of 3 sub-query objects:
	// 1) qPerm - user permissions (system-defined)
	// 2) qOpts - query options (user-defined)
	// 3) qAcls - ACL queries (user-defined)
	q := bson.M{}
	qPerm := bson.M{}
	qOpts := bson.M{}
	qAcls := bson.M{}
	nodes := node.Nodes{}

	if u != nil {
		// Skip this part if user is an admin
		if !u.Admin {
			qPerm["$or"] = []bson.M{bson.M{"acl.read": "public"}, bson.M{"acl.read": u.Uuid}, bson.M{"acl.owner": u.Uuid}}
		}
	} else {
		// User is anonymous
		if conf.ANON_READ {
			// select on only nodes that are publicly readable
			qPerm["acl.read"] = "public"
		} else {
			return responder.RespondWithError(ctx, http.StatusUnauthorized, e.NoAuth)
		}
	}

	// bson.M is a convenient alias for a map[string]interface{} map, useful for dealing with BSON in a native way.
	var OptsMArray []bson.M

	// default sort field and direction (can only be changed with querynode operator, not query operator)
	order := "created_on"
	direction := "-"

	// Gather params to make db query. Do not include the following list.
	if _, ok := query["query"]; ok {
		paramlist := map[string]int{"limit": 1, "offset": 1, "query": 1}
		for key := range query {
			if _, found := paramlist[key]; !found {
				keyStr := fmt.Sprintf("attributes.%s", key)
				for _, value := range query[key] {
					if value != "" {
						OptsMArray = append(OptsMArray, parseOption(keyStr, value))
					} else {
						OptsMArray = append(OptsMArray, bson.M{keyStr: map[string]bool{"$exists": true}})
					}
				}
			}
		}
	} else if _, ok := query["querynode"]; ok {
		paramlist := map[string]int{"limit": 1, "offset": 1, "querynode": 1, "order": 1, "direction": 1, "owner": 1, "read": 1, "write": 1, "delete": 1, "public_owner": 1, "public_read": 1, "public_write": 1, "public_delete": 1}
		for key := range query {
			if _, found := paramlist[key]; !found {
				for _, value := range query[key] {
					if value != "" {
						OptsMArray = append(OptsMArray, parseOption(key, value))
					} else {
						OptsMArray = append(OptsMArray, bson.M{key: map[string]bool{"$exists": true}})
					}
				}
			}
		}
		if _, ok := query["order"]; ok {
			order = query.Get("order")
		}
		if _, ok := query["direction"]; ok {
			if query.Get("direction") == "asc" {
				direction = ""
			}
		}
	}

	if len(OptsMArray) > 0 {
		qOpts["$and"] = OptsMArray
	}

	// bson.M is a convenient alias for a map[string]interface{} map, useful for dealing with BSON in a native way.
	var AclsMArray []bson.M

	// Allowing user to query based on ACL's with a comma-separated list of users.
	// Restricting ACL queries to just the querynode operation.
	// Users can be written as a username or a UUID.
	if _, qok := query["querynode"]; qok {
		for _, atype := range []string{"owner", "read", "write", "delete"} {
			if _, ok := query[atype]; ok {
				users := strings.Split(query.Get(atype), ",")
				for _, v := range users {
					if uuid.Parse(v) != nil {
						AclsMArray = append(AclsMArray, bson.M{"acl." + atype: v})
					} else {
						u := user.User{Username: v}
						if err := u.SetMongoInfo(); err != nil {
							err_msg := "err " + err.Error()
							logger.Error(err_msg)
							return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg)
						}
						AclsMArray = append(AclsMArray, bson.M{"acl." + atype: u.Uuid})
					}
				}
			}
		}
		// Allowing users to query based on whether ACL is public
		for _, atype := range []string{"owner", "read", "write", "delete"} {
			if _, ok := query["public_"+atype]; ok {
				AclsMArray = append(AclsMArray, bson.M{"acl." + atype: "public"})
			}
		}
	}

	if len(AclsMArray) > 0 {
		qAcls["$and"] = AclsMArray
	}

	// Combine permissions query with query parameters and ACL query into one AND clause
	q["$and"] = []bson.M{qPerm, qOpts, qAcls}

	// defaults
	limit := 25
	offset := 0
	if _, ok := query["limit"]; ok {
		limit = util.ToInt(query.Get("limit"))
	}
	if _, ok := query["offset"]; ok {
		offset = util.ToInt(query.Get("offset"))
	}

	// Get nodes from db
	order = direction + order
	count, err := nodes.GetPaginated(q, limit, offset, order)
	if err != nil {
		err_msg := "err " + err.Error()
		logger.Error(err_msg)
		return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg)
	}
	return responder.RespondWithPaginatedData(ctx, nodes, limit, offset, count)
}

func parseOption(key string, value string) bson.M {
	not := false
	// If value starts with ! then set flag to encapsulate query with $not operator and
	//  remove ! character from the beginning of the value string
	if value[0] == '!' {
		value = value[1:]
		not = true
	}

	// Parsing query option into bson.M query object
	opt := bson.M{}

	// Only one of the following conditions can be met at a time
	// mongodb doesn't allow for negating the entire query so the logic
	//   has to be written for each case below when the not flag is set.
	if numValue, err := strconv.Atoi(value); err == nil {
		// numeric values
		if not {
			opt = bson.M{"$and": []bson.M{bson.M{key: bson.M{"$ne": value}}, bson.M{key: bson.M{"$ne": numValue}}}}
		} else {
			opt = bson.M{"$or": []bson.M{bson.M{key: value}, bson.M{key: numValue}}}
		}
	} else if value == "null" {
		// value is "null" => nil
		if not {
			opt = bson.M{"$and": []bson.M{bson.M{key: bson.M{"$ne": value}}, bson.M{key: bson.M{"$ne": nil}}}}
		} else {
			opt = bson.M{"$or": []bson.M{bson.M{key: value}, bson.M{key: nil}}}
		}
	} else if matches := RangeRegex.FindStringSubmatch(value); len(matches) > 0 {
		// value matches the regex for a range
		lowerBound := bson.M{}
		upperBound := bson.M{}
		var val1 interface{} = matches[2]
		var val2 interface{} = matches[3]
		parseTypedValue(&val1)
		parseTypedValue(&val2)
		if not {
			if matches[1] == "[" {
				lowerBound = bson.M{key: bson.M{"$lt": val1}}
			} else {
				lowerBound = bson.M{key: bson.M{"$lte": val1}}
			}
			if matches[4] == "]" {
				upperBound = bson.M{key: bson.M{"$gt": val2}}
			} else {
				upperBound = bson.M{key: bson.M{"$gte": val2}}
			}
			opt = bson.M{"$or": []bson.M{lowerBound, upperBound}}
		} else {
			if matches[1] == "[" {
				lowerBound = bson.M{key: bson.M{"$gte": val1}}
			} else {
				lowerBound = bson.M{key: bson.M{"$gt": val1}}
			}
			if matches[4] == "]" {
				upperBound = bson.M{key: bson.M{"$lte": val2}}
			} else {
				upperBound = bson.M{key: bson.M{"$lt": val2}}
			}
			opt = bson.M{"$and": []bson.M{lowerBound, upperBound}}
		}
	} else if string(value[0]) == "*" || string(value[len(value)-1]) == "*" {
		// value starts or ends with wildcard, or both
		// Note: The $not operator could probably be used for some of these queries but
		//       the $not operator does not support operations with the $regex operator
		//       thus I have built the opposite regexes below for the "not" option.
		if not {
			if string(value[0]) != "*" {
				value = value[0 : len(value)-1]
				opt = bson.M{key: bson.M{"$regex": "^(?!" + value + ").*$"}}
			} else if string(value[len(value)-1]) != "*" {
				value = value[1:]
				opt = bson.M{key: bson.M{"$regex": "^.*(?<!" + value + ")$"}}
			} else {
				value = value[1 : len(value)-1]
				opt = bson.M{key: bson.M{"$regex": "^((?!" + value + ").)*$"}}
			}
		} else {
			if string(value[0]) != "*" {
				value = value[0 : len(value)-1]
				opt = bson.M{key: bson.M{"$regex": "^" + value}}
			} else if string(value[len(value)-1]) != "*" {
				value = value[1:]
				opt = bson.M{key: bson.M{"$regex": value + "$"}}
			} else {
				value = value[1 : len(value)-1]
				opt = bson.M{key: bson.M{"$regex": value}}
			}
		}
	} else {
		if not {
			opt = bson.M{key: bson.M{"$ne": value}}
		} else {
			opt = bson.M{key: value}
		}
	}
	return opt
}

func parseTypedValue(i *interface{}) {
	if val, err := strconv.Atoi((*i).(string)); err == nil {
		*i = val
	} else if t, err := time.Parse(longDateForm, (*i).(string)); err == nil {
		*i = t
	} else if t, err := time.Parse(shortDateForm, (*i).(string)); err == nil {
		*i = t
	}
	return
}
<commit_msg>throw error on missing query keyword<commit_after>package node

import (
	"fmt"
	"github.com/MG-RAST/Shock/shock-server/conf"
	e "github.com/MG-RAST/Shock/shock-server/errors"
	"github.com/MG-RAST/Shock/shock-server/logger"
	"github.com/MG-RAST/Shock/shock-server/node"
	"github.com/MG-RAST/Shock/shock-server/request"
	"github.com/MG-RAST/Shock/shock-server/responder"
	"github.com/MG-RAST/Shock/shock-server/user"
	"github.com/MG-RAST/Shock/shock-server/util"
	"github.com/MG-RAST/golib/go-uuid/uuid"
	"github.com/MG-RAST/golib/mgo/bson"
	"github.com/MG-RAST/golib/stretchr/goweb/context"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"time"
)

var (
	RangeRegex = regexp.MustCompile(`^([\[\]]{1})(.*);(.*)([\]\[]{1})$`)
)

const (
	longDateForm  = "2006-01-02T15:04:05-07:00"
	shortDateForm = "2006-01-02"
)

// GET: /node
// To do:
// - Iterate node queries
func (cr *NodeController) ReadMany(ctx context.Context) error {
	u, err := request.Authenticate(ctx.HttpRequest())
	if err != nil && err.Error() != e.NoAuth {
		return request.AuthError(err, ctx)
	}

	// Gather query params
	query := ctx.HttpRequest().URL.Query()

	// Setup query and nodes objects
	// Note: query is composed of 3 sub-query objects:
	// 1) qPerm - user permissions (system-defined)
	// 2) qOpts - query options (user-defined)
	// 3) qAcls - ACL queries (user-defined)
	q := bson.M{}
	qPerm := bson.M{}
	qOpts := bson.M{}
	qAcls := bson.M{}
	nodes := node.Nodes{}

	if u != nil {
		// Skip this part if user is an admin
		if !u.Admin {
			qPerm["$or"] = []bson.M{bson.M{"acl.read": "public"}, bson.M{"acl.read": u.Uuid}, bson.M{"acl.owner": u.Uuid}}
		}
	} else {
		// User is anonymous
		if conf.ANON_READ {
			// select on only nodes that are publicly readable
			qPerm["acl.read"] = "public"
		} else {
			return responder.RespondWithError(ctx, http.StatusUnauthorized, e.NoAuth)
		}
	}

	// bson.M is a convenient alias for a map[string]interface{} map, useful for dealing with BSON in a native way.
	var OptsMArray []bson.M

	// default sort field and direction (can only be changed with querynode operator, not query operator)
	order := "created_on"
	direction := "-"

	// Gather params to make db query. Do not include the following list.
	if _, ok := query["query"]; ok {
		paramlist := map[string]int{"limit": 1, "offset": 1, "query": 1}
		for key := range query {
			if _, found := paramlist[key]; !found {
				keyStr := fmt.Sprintf("attributes.%s", key)
				for _, value := range query[key] {
					if value != "" {
						OptsMArray = append(OptsMArray, parseOption(keyStr, value))
					} else {
						OptsMArray = append(OptsMArray, bson.M{keyStr: map[string]bool{"$exists": true}})
					}
				}
			}
		}
	} else if _, ok := query["querynode"]; ok {
		paramlist := map[string]int{"limit": 1, "offset": 1, "querynode": 1, "order": 1, "direction": 1, "owner": 1, "read": 1, "write": 1, "delete": 1, "public_owner": 1, "public_read": 1, "public_write": 1, "public_delete": 1}
		for key := range query {
			if _, found := paramlist[key]; !found {
				for _, value := range query[key] {
					if value != "" {
						OptsMArray = append(OptsMArray, parseOption(key, value))
					} else {
						OptsMArray = append(OptsMArray, bson.M{key: map[string]bool{"$exists": true}})
					}
				}
			}
		}
		if _, ok := query["order"]; ok {
			order = query.Get("order")
		}
		if _, ok := query["direction"]; ok {
			if query.Get("direction") == "asc" {
				direction = ""
			}
		}
	} else {
		// return error if no query type is used
		return responder.RespondWithError(ctx, http.StatusBadRequest, "no query type specified, use one of: query, querynode")
	}

	if len(OptsMArray) > 0 {
		qOpts["$and"] = OptsMArray
	}

	// bson.M is a convenient alias for a map[string]interface{} map, useful for dealing with BSON in a native way.
	var AclsMArray []bson.M

	// Allowing user to query based on ACL's with a comma-separated list of users.
	// Restricting ACL queries to just the querynode operation.
	// Users can be written as a username or a UUID.
	if _, qok := query["querynode"]; qok {
		for _, atype := range []string{"owner", "read", "write", "delete"} {
			if _, ok := query[atype]; ok {
				users := strings.Split(query.Get(atype), ",")
				for _, v := range users {
					if uuid.Parse(v) != nil {
						AclsMArray = append(AclsMArray, bson.M{"acl." + atype: v})
					} else {
						u := user.User{Username: v}
						if err := u.SetMongoInfo(); err != nil {
							err_msg := "err " + err.Error()
							logger.Error(err_msg)
							return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg)
						}
						AclsMArray = append(AclsMArray, bson.M{"acl." + atype: u.Uuid})
					}
				}
			}
		}
		// Allowing users to query based on whether ACL is public
		for _, atype := range []string{"owner", "read", "write", "delete"} {
			if _, ok := query["public_"+atype]; ok {
				AclsMArray = append(AclsMArray, bson.M{"acl." + atype: "public"})
			}
		}
	}

	if len(AclsMArray) > 0 {
		qAcls["$and"] = AclsMArray
	}

	// Combine permissions query with query parameters and ACL query into one AND clause
	q["$and"] = []bson.M{qPerm, qOpts, qAcls}

	// defaults
	limit := 25
	offset := 0
	if _, ok := query["limit"]; ok {
		limit = util.ToInt(query.Get("limit"))
	}
	if _, ok := query["offset"]; ok {
		offset = util.ToInt(query.Get("offset"))
	}

	// Get nodes from db
	order = direction + order
	count, err := nodes.GetPaginated(q, limit, offset, order)
	if err != nil {
		err_msg := "err " + err.Error()
		logger.Error(err_msg)
		return responder.RespondWithError(ctx, http.StatusBadRequest, err_msg)
	}
	return responder.RespondWithPaginatedData(ctx, nodes, limit, offset, count)
}

func parseOption(key string, value string) bson.M {
	not := false
	// If value starts with ! then set flag to encapsulate query with $not operator and
	//  remove ! character from the beginning of the value string
	if value[0] == '!' {
		value = value[1:]
		not = true
	}

	// Parsing query option into bson.M query object
	opt := bson.M{}

	// Only one of the following conditions can be met at a time
	// mongodb doesn't allow for negating the entire query so the logic
	//   has to be written for each case below when the not flag is set.
	if numValue, err := strconv.Atoi(value); err == nil {
		// numeric values
		if not {
			opt = bson.M{"$and": []bson.M{bson.M{key: bson.M{"$ne": value}}, bson.M{key: bson.M{"$ne": numValue}}}}
		} else {
			opt = bson.M{"$or": []bson.M{bson.M{key: value}, bson.M{key: numValue}}}
		}
	} else if value == "null" {
		// value is "null" => nil
		if not {
			opt = bson.M{"$and": []bson.M{bson.M{key: bson.M{"$ne": value}}, bson.M{key: bson.M{"$ne": nil}}}}
		} else {
			opt = bson.M{"$or": []bson.M{bson.M{key: value}, bson.M{key: nil}}}
		}
	} else if matches := RangeRegex.FindStringSubmatch(value); len(matches) > 0 {
		// value matches the regex for a range
		lowerBound := bson.M{}
		upperBound := bson.M{}
		var val1 interface{} = matches[2]
		var val2 interface{} = matches[3]
		parseTypedValue(&val1)
		parseTypedValue(&val2)
		if not {
			if matches[1] == "[" {
				lowerBound = bson.M{key: bson.M{"$lt": val1}}
			} else {
				lowerBound = bson.M{key: bson.M{"$lte": val1}}
			}
			if matches[4] == "]" {
				upperBound = bson.M{key: bson.M{"$gt": val2}}
			} else {
				upperBound = bson.M{key: bson.M{"$gte": val2}}
			}
			opt = bson.M{"$or": []bson.M{lowerBound, upperBound}}
		} else {
			if matches[1] == "[" {
				lowerBound = bson.M{key: bson.M{"$gte": val1}}
			} else {
				lowerBound = bson.M{key: bson.M{"$gt": val1}}
			}
			if matches[4] == "]" {
				upperBound = bson.M{key: bson.M{"$lte": val2}}
			} else {
				upperBound = bson.M{key: bson.M{"$lt": val2}}
			}
			opt = bson.M{"$and": []bson.M{lowerBound, upperBound}}
		}
	} else if string(value[0]) == "*" || string(value[len(value)-1]) == "*" {
		// value starts or ends with wildcard, or both
		// Note: The $not operator could probably be used for some of these queries but
		//       the $not operator does not support operations with the $regex operator
		//       thus I have built the opposite regexes below for the "not" option.
		if not {
			if string(value[0]) != "*" {
				value = value[0 : len(value)-1]
				opt = bson.M{key: bson.M{"$regex": "^(?!" + value + ").*$"}}
			} else if string(value[len(value)-1]) != "*" {
				value = value[1:]
				opt = bson.M{key: bson.M{"$regex": "^.*(?<!" + value + ")$"}}
			} else {
				value = value[1 : len(value)-1]
				opt = bson.M{key: bson.M{"$regex": "^((?!" + value + ").)*$"}}
			}
		} else {
			if string(value[0]) != "*" {
				value = value[0 : len(value)-1]
				opt = bson.M{key: bson.M{"$regex": "^" + value}}
			} else if string(value[len(value)-1]) != "*" {
				value = value[1:]
				opt = bson.M{key: bson.M{"$regex": value + "$"}}
			} else {
				value = value[1 : len(value)-1]
				opt = bson.M{key: bson.M{"$regex": value}}
			}
		}
	} else {
		if not {
			opt = bson.M{key: bson.M{"$ne": value}}
		} else {
			opt = bson.M{key: value}
		}
	}
	return opt
}

func parseTypedValue(i *interface{}) {
	if val, err := strconv.Atoi((*i).(string)); err == nil {
		*i = val
	} else if t, err := time.Parse(longDateForm, (*i).(string)); err == nil {
		*i = t
	} else if t, err := time.Parse(shortDateForm, (*i).(string)); err == nil {
		*i = t
	}
	return
}
<|endoftext|>


<commit_before>package display

import (
	"github.com/zyedidia/micro/v2/internal/buffer"
	"github.com/zyedidia/micro/v2/internal/config"
	"github.com/zyedidia/micro/v2/internal/screen"
	"github.com/zyedidia/micro/v2/internal/util"
	"github.com/zyedidia/micro/v2/internal/views"
)

type UIWindow struct {
	root *views.Node
}

func NewUIWindow(n *views.Node) *UIWindow {
	uw := new(UIWindow)
	uw.root = n
	return uw
}

func (w *UIWindow) drawNode(n *views.Node) {
	cs := n.Children()
	dividerStyle := config.DefStyle
	if style, ok := config.Colorscheme["divider"]; ok {
		dividerStyle = style
	}

	divchars := config.GetGlobalOption("divchars").(string)
	if util.CharacterCountInString(divchars) != 2 {
		divchars = "|-"
	}

	divchar, combc, _ := util.DecodeCharacterInString(divchars)

	divreverse := config.GetGlobalOption("divreverse").(bool)
	if divreverse {
		dividerStyle = dividerStyle.Reverse(true)
	}

	for i, c := range cs {
		if c.IsLeaf() && c.Kind == views.STVert {
			if i != len(cs)-1 {
				for h := 0; h < c.H; h++ {
					screen.SetContent(c.X+c.W, c.Y+h, divchar, combc, dividerStyle)
				}
			}
		} else {
			w.drawNode(c)
		}
	}
}

func (w *UIWindow) Display() {
	w.drawNode(w.root)
}

func (w *UIWindow) GetMouseSplitID(vloc buffer.Loc) uint64 {
	var mouseLoc func(*views.Node) uint64
	mouseLoc = func(n *views.Node) uint64 {
		cs := n.Children()
		for i, c := range cs {
			if c.Kind == views.STVert {
				if i != len(cs)-1 {
					if vloc.X == c.X+c.W && vloc.Y >= c.Y && vloc.Y < c.Y+c.H {
						return c.ID()
					}
				}
			} else if c.Kind == views.STHoriz {
				if i != len(cs)-1 {
					if vloc.Y == c.Y+c.H-1 && vloc.X >= c.X && vloc.X < c.X+c.W {
						return c.ID()
					}
				}
			}
		}
		for _, c := range cs {
			m := mouseLoc(c)
			if m != 0 {
				return m
			}
		}
		return 0
	}
	return mouseLoc(w.root)
}
func (w *UIWindow) Resize(width, height int) {}
func (w *UIWindow) SetActive(b bool)         {}
<commit_msg>Fix erased vertical dividing line (#1810)<commit_after>package display

import (
	"github.com/zyedidia/micro/v2/internal/buffer"
	"github.com/zyedidia/micro/v2/internal/config"
	"github.com/zyedidia/micro/v2/internal/screen"
	"github.com/zyedidia/micro/v2/internal/util"
	"github.com/zyedidia/micro/v2/internal/views"
)

type UIWindow struct {
	root *views.Node
}

func NewUIWindow(n *views.Node) *UIWindow {
	uw := new(UIWindow)
	uw.root = n
	return uw
}

func (w *UIWindow) drawNode(n *views.Node) {
	cs := n.Children()
	dividerStyle := config.DefStyle
	if style, ok := config.Colorscheme["divider"]; ok {
		dividerStyle = style
	}

	divchars := config.GetGlobalOption("divchars").(string)
	if util.CharacterCountInString(divchars) != 2 {
		divchars = "|-"
	}

	divchar, combc, _ := util.DecodeCharacterInString(divchars)

	divreverse := config.GetGlobalOption("divreverse").(bool)
	if divreverse {
		dividerStyle = dividerStyle.Reverse(true)
	}

	for i, c := range cs {
		if c.Kind == views.STVert {
			if i != len(cs)-1 {
				for h := 0; h < c.H; h++ {
					screen.SetContent(c.X+c.W, c.Y+h, divchar, combc, dividerStyle)
				}
			}
		}
		w.drawNode(c)
	}
}

func (w *UIWindow) Display() {
	w.drawNode(w.root)
}

func (w *UIWindow) GetMouseSplitID(vloc buffer.Loc) uint64 {
	var mouseLoc func(*views.Node) uint64
	mouseLoc = func(n *views.Node) uint64 {
		cs := n.Children()
		for i, c := range cs {
			if c.Kind == views.STVert {
				if i != len(cs)-1 {
					if vloc.X == c.X+c.W && vloc.Y >= c.Y && vloc.Y < c.Y+c.H {
						return c.ID()
					}
				}
			} else if c.Kind == views.STHoriz {
				if i != len(cs)-1 {
					if vloc.Y == c.Y+c.H-1 && vloc.X >= c.X && vloc.X < c.X+c.W {
						return c.ID()
					}
				}
			}
		}
		for _, c := range cs {
			m := mouseLoc(c)
			if m != 0 {
				return m
			}
		}
		return 0
	}
	return mouseLoc(w.root)
}
func (w *UIWindow) Resize(width, height int) {}
func (w *UIWindow) SetActive(b bool)         {}
<|endoftext|>


<commit_before>// Copyright (c) 2016 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package marshaled

import (
	"io"
	"log"
	"sort"
	"strings"
	"sync/atomic"

	"github.com/uber-go/gwr/source"
)

// NOTE: This approach is perhaps overfit to the json module's marshalling
// mindset.  A better interface (for performance) would work by passing a
// writer to the specific encoder, rather than a []byte-returning Marshal
// function.  This would be possible perhaps using something like
// io.MultiWriter.

// DataSource wraps a format-agnostic data source and provides one or
// more formats for it.
//
// DataSource implements:
// - DataSource to satisfy DataSources and low level protocols
// - ItemDataSource so that higher level protocols may add their own framing
// - GenericDataWatcher inwardly to the wrapped GenericDataSource
type DataSource struct {
	// TODO: better to have alternate implementations for each combination
	// rather than one with these nil checks
	source      source.GenericDataSource
	getSource   source.GetableDataSource
	watchSource source.WatchableDataSource
	watiSource  source.WatchInitableDataSource
	actiSource  source.ActivateWatchableDataSource

	formats     map[string]source.GenericDataFormat
	formatNames []string
	watchers    map[string]*marshaledWatcher
	active      uint32
	itemChan    chan interface{}
	itemsChan   chan []interface{}
}

// NewDataSource creates a DataSource for a given format-agnostic data source
// and a map of marshalers
func NewDataSource(
	src source.GenericDataSource,
	formats map[string]source.GenericDataFormat,
) *DataSource {
	if formats == nil {
		formats = make(map[string]source.GenericDataFormat)
	}

	// source-defined formats
	if fmtsrc, ok := src.(source.GenericDataSourceFormats); ok {
		fmts := fmtsrc.Formats()
		for name, fmt := range fmts {
			formats[name] = fmt
		}
	}

	// standard json protocol
	if formats["json"] == nil {
		formats["json"] = LDJSONMarshal
	}

	// convenience templated text protocol
	if txtsrc, ok := src.(source.TextTemplatedSource); ok && formats["text"] == nil {
		if tt := txtsrc.TextTemplate(); tt != nil && formats["text"] == nil {
			formats["text"] = NewTemplatedMarshal(tt)
		}
	}

	ds := &DataSource{
		source:   src,
		formats:  formats,
		watchers: make(map[string]*marshaledWatcher, len(formats)),
	}
	ds.getSource, _ = src.(source.GetableDataSource)
	ds.watchSource, _ = src.(source.WatchableDataSource)
	ds.watiSource, _ = src.(source.WatchInitableDataSource)
	ds.actiSource, _ = src.(source.ActivateWatchableDataSource)
	for name, format := range formats {
		ds.formatNames = append(ds.formatNames, name)
		ds.watchers[name] = newMarshaledWatcher(ds, format)
	}
	sort.Strings(ds.formatNames)

	if ds.watchSource != nil {
		ds.watchSource.SetWatcher(ds)
	}

	return ds
}

// Active returns true if there are any active watchers, false otherwise.  If
// Active returns false, so will any calls to HandleItem and HandleItems.
func (mds *DataSource) Active() bool {
	return atomic.LoadUint32(&mds.active) != 0
}

// Name passes through the GenericDataSource.Name()
func (mds *DataSource) Name() string {
	return mds.source.Name()
}

// Formats returns the list of supported format names.
func (mds *DataSource) Formats() []string {
	return mds.formatNames
}

// Attrs returns arbitrary description information about the data source.
func (mds *DataSource) Attrs() map[string]interface{} {
	// TODO: support per-format Attrs?
	// TODO: any support for per-source Attrs?
	return nil
}

// Get marshals data source's Get data to the writer
func (mds *DataSource) Get(formatName string, w io.Writer) error {
	if mds.getSource == nil {
		return source.ErrNotGetable
	}
	format, ok := mds.formats[strings.ToLower(formatName)]
	if !ok {
		return source.ErrUnsupportedFormat
	}
	data := mds.getSource.Get()
	buf, err := format.MarshalGet(data)
	if err != nil {
		log.Printf("get marshaling error %v", err)
		return err
	}
	_, err = w.Write(buf)
	return err
}

// Watch marshals any data source GetInit data to the writer, and then
// retains a reference to the writer so that any future agnostic data source
// Watch(emit)'ed data gets marshaled to it as well
func (mds *DataSource) Watch(formatName string, w io.Writer) error {
	if mds.watchSource == nil {
		return source.ErrNotWatchable
	}
	watcher, ok := mds.watchers[strings.ToLower(formatName)]
	if !ok {
		return source.ErrUnsupportedFormat
	}
	if err := watcher.init(w); err != nil {
		return err
	}
	return mds.startWatching()
}

// WatchItems marshals any data source GetInit data as a single item to the
// ItemWatcher's HandleItem method.  The watcher is then retained and future
// items are marshaled to its HandleItem method.
func (mds *DataSource) WatchItems(formatName string, iw source.ItemWatcher) error {
	if mds.watchSource == nil {
		return source.ErrNotWatchable
	}
	watcher, ok := mds.watchers[strings.ToLower(formatName)]
	if !ok {
		return source.ErrUnsupportedFormat
	}
	if err := watcher.initItems(iw); err != nil {
		return err
	}
	return mds.startWatching()
}

func (mds *DataSource) startWatching() error {
	// TODO: we could optimize the only-one-format-being-watched case
	if !atomic.CompareAndSwapUint32(&mds.active, 0, 1) {
		return nil
	}
	// TODO: tune size
	mds.itemChan = make(chan interface{}, 100)
	mds.itemsChan = make(chan []interface{}, 100)
	go mds.processItemChan()
	if mds.actiSource != nil {
		mds.actiSource.Activate()
	}
	return nil
}

func (mds *DataSource) stopWatching() {
	if !atomic.CompareAndSwapUint32(&mds.active, 1, 0) {
		return
	}
	for _, watcher := range mds.watchers {
		watcher.Close()
	}
}

func (mds *DataSource) processItemChan() {
	for mds.Active() {
		any := false

		select {
		case item := <-mds.itemChan:
			for _, watcher := range mds.watchers {
				if watcher.emit(item) {
					any = true
				}
			}

		case items := <-mds.itemsChan:
			for _, watcher := range mds.watchers {
				if watcher.emitBatch(items) {
					any = true
				}
			}
		}

		if !any {
			mds.stopWatching()
		}
	}
	mds.itemChan = nil
	mds.itemsChan = nil
}

// HandleItem implements GenericDataWatcher.HandleItem by passing the item to
// all current marshaledWatchers.
func (mds *DataSource) HandleItem(item interface{}) bool {
	if !mds.Active() {
		return false
	}
	select {
	case mds.itemChan <- item:
		return true
	default:
		mds.stopWatching()
		return false
	}
}

// HandleItems implements GenericDataWatcher.HandleItems by passing the batch
// to all current marshaledWatchers.
func (mds *DataSource) HandleItems(items []interface{}) bool {
	if !mds.Active() {
		return false
	}
	select {
	case mds.itemsChan <- items:
		return true
	default:
		mds.stopWatching()
		return false
	}
}
<commit_msg>Make marshaled data source channel parameters tunable (#4)<commit_after>// Copyright (c) 2016 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package marshaled

import (
	"io"
	"log"
	"sort"
	"strings"
	"sync/atomic"
	"time"

	"github.com/uber-go/gwr/source"
)

// NOTE: This approach is perhaps overfit to the json module's marshalling
// mindset.  A better interface (for performance) would work by passing a
// writer to the specific encoder, rather than a []byte-returning Marshal
// function.  This would be possible perhaps using something like
// io.MultiWriter.

// DataSource wraps a format-agnostic data source and provides one or
// more formats for it.
//
// DataSource implements:
// - DataSource to satisfy DataSources and low level protocols
// - ItemDataSource so that higher level protocols may add their own framing
// - GenericDataWatcher inwardly to the wrapped GenericDataSource
type DataSource struct {
	// TODO: better to have alternate implementations for each combination
	// rather than one with these nil checks
	source      source.GenericDataSource
	getSource   source.GetableDataSource
	watchSource source.WatchableDataSource
	watiSource  source.WatchInitableDataSource
	actiSource  source.ActivateWatchableDataSource

	formats     map[string]source.GenericDataFormat
	formatNames []string
	watchers    map[string]*marshaledWatcher
	active      uint32
	itemChan    chan interface{}
	itemsChan   chan []interface{}
	maxItems    int
	maxBatches  int
	maxWait     time.Duration
}

// NewDataSource creates a DataSource for a given format-agnostic data source
// and a map of marshalers
func NewDataSource(
	src source.GenericDataSource,
	formats map[string]source.GenericDataFormat,
) *DataSource {
	if formats == nil {
		formats = make(map[string]source.GenericDataFormat)
	}

	// source-defined formats
	if fmtsrc, ok := src.(source.GenericDataSourceFormats); ok {
		fmts := fmtsrc.Formats()
		for name, fmt := range fmts {
			formats[name] = fmt
		}
	}

	// standard json protocol
	if formats["json"] == nil {
		formats["json"] = LDJSONMarshal
	}

	// convenience templated text protocol
	if txtsrc, ok := src.(source.TextTemplatedSource); ok && formats["text"] == nil {
		if tt := txtsrc.TextTemplate(); tt != nil && formats["text"] == nil {
			formats["text"] = NewTemplatedMarshal(tt)
		}
	}

	ds := &DataSource{
		source:   src,
		formats:  formats,
		watchers: make(map[string]*marshaledWatcher, len(formats)),
		// TODO: tunable
		maxItems:   100,
		maxBatches: 100,
		maxWait:    100 * time.Microsecond,
	}
	ds.getSource, _ = src.(source.GetableDataSource)
	ds.watchSource, _ = src.(source.WatchableDataSource)
	ds.watiSource, _ = src.(source.WatchInitableDataSource)
	ds.actiSource, _ = src.(source.ActivateWatchableDataSource)
	for name, format := range formats {
		ds.formatNames = append(ds.formatNames, name)
		ds.watchers[name] = newMarshaledWatcher(ds, format)
	}
	sort.Strings(ds.formatNames)

	if ds.watchSource != nil {
		ds.watchSource.SetWatcher(ds)
	}

	return ds
}

// Active returns true if there are any active watchers, false otherwise.  If
// Active returns false, so will any calls to HandleItem and HandleItems.
func (mds *DataSource) Active() bool {
	return atomic.LoadUint32(&mds.active) != 0
}

// Name passes through the GenericDataSource.Name()
func (mds *DataSource) Name() string {
	return mds.source.Name()
}

// Formats returns the list of supported format names.
func (mds *DataSource) Formats() []string {
	return mds.formatNames
}

// Attrs returns arbitrary description information about the data source.
func (mds *DataSource) Attrs() map[string]interface{} {
	// TODO: support per-format Attrs?
	// TODO: any support for per-source Attrs?
	return nil
}

// Get marshals data source's Get data to the writer
func (mds *DataSource) Get(formatName string, w io.Writer) error {
	if mds.getSource == nil {
		return source.ErrNotGetable
	}
	format, ok := mds.formats[strings.ToLower(formatName)]
	if !ok {
		return source.ErrUnsupportedFormat
	}
	data := mds.getSource.Get()
	buf, err := format.MarshalGet(data)
	if err != nil {
		log.Printf("get marshaling error %v", err)
		return err
	}
	_, err = w.Write(buf)
	return err
}

// Watch marshals any data source GetInit data to the writer, and then
// retains a reference to the writer so that any future agnostic data source
// Watch(emit)'ed data gets marshaled to it as well
func (mds *DataSource) Watch(formatName string, w io.Writer) error {
	if mds.watchSource == nil {
		return source.ErrNotWatchable
	}
	watcher, ok := mds.watchers[strings.ToLower(formatName)]
	if !ok {
		return source.ErrUnsupportedFormat
	}
	if err := watcher.init(w); err != nil {
		return err
	}
	return mds.startWatching()
}

// WatchItems marshals any data source GetInit data as a single item to the
// ItemWatcher's HandleItem method.  The watcher is then retained and future
// items are marshaled to its HandleItem method.
func (mds *DataSource) WatchItems(formatName string, iw source.ItemWatcher) error {
	if mds.watchSource == nil {
		return source.ErrNotWatchable
	}
	watcher, ok := mds.watchers[strings.ToLower(formatName)]
	if !ok {
		return source.ErrUnsupportedFormat
	}
	if err := watcher.initItems(iw); err != nil {
		return err
	}
	return mds.startWatching()
}

func (mds *DataSource) startWatching() error {
	// TODO: we could optimize the only-one-format-being-watched case
	if !atomic.CompareAndSwapUint32(&mds.active, 0, 1) {
		return nil
	}
	mds.itemChan = make(chan interface{}, mds.maxItems)
	mds.itemsChan = make(chan []interface{}, mds.maxBatches)
	go mds.processItemChan()
	if mds.actiSource != nil {
		mds.actiSource.Activate()
	}
	return nil
}

func (mds *DataSource) stopWatching() {
	if !atomic.CompareAndSwapUint32(&mds.active, 1, 0) {
		return
	}
	for _, watcher := range mds.watchers {
		watcher.Close()
	}
}

func (mds *DataSource) processItemChan() {
	for mds.Active() {
		any := false

		select {
		case item := <-mds.itemChan:
			for _, watcher := range mds.watchers {
				if watcher.emit(item) {
					any = true
				}
			}

		case items := <-mds.itemsChan:
			for _, watcher := range mds.watchers {
				if watcher.emitBatch(items) {
					any = true
				}
			}
		}

		if !any {
			mds.stopWatching()
		}
	}
	mds.itemChan = nil
	mds.itemsChan = nil
}

// HandleItem implements GenericDataWatcher.HandleItem by passing the item to
// all current marshaledWatchers.
func (mds *DataSource) HandleItem(item interface{}) bool {
	if !mds.Active() {
		return false
	}
	select {
	case mds.itemChan <- item:
		return true
	case <-time.After(mds.maxWait):
		mds.stopWatching()
		return false
	}
}

// HandleItems implements GenericDataWatcher.HandleItems by passing the batch
// to all current marshaledWatchers.
func (mds *DataSource) HandleItems(items []interface{}) bool {
	if !mds.Active() {
		return false
	}
	select {
	case mds.itemsChan <- items:
		return true
	case <-time.After(mds.maxWait):
		mds.stopWatching()
		return false
	}
}
<|endoftext|>


<commit_before>package search

import (
	"fmt"
	. "github.com/balzaczyy/golucene/core/search"
	"math"
	"math/rand"
	"sync"
)

// search/RandomSimilarityProvider.java

var allSims = func() []Similarity {
	var ans []Similarity
	ans = append(ans, NewDefaultSimilarity())
	// ans = append(ans, newBM25Similarity())
	// for _, basicModel := range BASIC_MODELS {
	// 	for _, afterEffect := range AFTER_EFFECTS {
	// 		for _, normalization := range NORMALIZATIONS {
	// 			ans = append(ans, newDFRSimilarity(basicModel, afterEffect, normalization))
	// 		}
	// 	}
	// }
	// for _, distribution := range DISTRIBUTIONS {
	// 	for _, lambda := range LAMBDAS {
	// 		for _, normalization := range NORMALIZATIONS {
	// 			ans = append(ans, newIBSimilarity(ditribution, lambda, normalization))
	// 		}
	// 	}
	// }
	// ans = append(ans, newLMJelinekMercerSimilarity(0.1))
	// ans = append(ans, newLMJelinekMercerSimilarity(0.7))
	return ans
}()

/*
Similarity implementation that randomizes Similarity implementations
per-field.

The choices are 'sticky', so the selected algorithm is ways used for
the same field.
*/
type RandomSimilarityProvider struct {
	*PerFieldSimilarityWrapper
	sync.Locker
	defaultSim       *DefaultSimilarity
	knownSims        []Similarity
	previousMappings map[string]Similarity
	perFieldSeed     int
	coordType        int // 0 = no coord, 1 = coord, 2 = crazy coord
	shouldQueryNorm  bool
}

func NewRandomSimilarityProvider(r *rand.Rand) *RandomSimilarityProvider {
	sims := make([]Similarity, len(allSims))
	for i, v := range r.Perm(len(allSims)) {
		sims[i] = allSims[v]
		assert(sims[i] != nil)
	}
	ans := &RandomSimilarityProvider{
		Locker:           &sync.Mutex{},
		defaultSim:       NewDefaultSimilarity(),
		previousMappings: make(map[string]Similarity),
		perFieldSeed:     r.Int(),
		coordType:        r.Intn(3),
		shouldQueryNorm:  r.Intn(2) == 0,
		knownSims:        sims,
	}
	ans.PerFieldSimilarityWrapper = NewPerFieldSimilarityWrapper(ans)
	return ans
}

func (rp *RandomSimilarityProvider) QueryNorm(valueForNormalization float32) float32 {
	panic("not implemented yet")
}

const primeRK = 16777619

/* simple string hash used by Go strings package */
func hashstr(sep string) int {
	hash := uint32(0)
	for i := 0; i < len(sep); i++ {
		hash = hash*primeRK + uint32(sep[i])
	}
	return int(hash)
}

func (p *RandomSimilarityProvider) Get(name string) Similarity {
	p.Lock()
	defer p.Unlock()
	sim, ok := p.previousMappings[name]
	if !ok {
		hash := int(math.Abs(math.Pow(float64(p.perFieldSeed), float64(hashstr(name)))))
		sim = p.knownSims[hash%len(p.knownSims)]
		p.previousMappings[name] = sim
	}
	assert(sim != nil)
	return sim
}

func assert(ok bool) {
	assert2(ok, "assert fail")
}

func assert2(ok bool, msg string, args ...interface{}) {
	if !ok {
		panic(fmt.Sprintf(msg, args...))
	}
}

func (rp *RandomSimilarityProvider) String() string {
	rp.Lock() // synchronized
	defer rp.Unlock()
	var coordMethod string
	switch rp.coordType {
	case 0:
		coordMethod = "no"
	case 1:
		coordMethod = "yes"
	default:
		coordMethod = "crazy"
	}
	return fmt.Sprintf("RandomSimilarityProvider(queryNorm=%v,coord=%v): %v",
		rp.shouldQueryNorm, coordMethod, rp.previousMappings)
}
<commit_msg>implement RandomSimilarityProvider.QueryNorm()<commit_after>package search

import (
	"fmt"
	. "github.com/balzaczyy/golucene/core/search"
	"math"
	"math/rand"
	"sync"
)

// search/RandomSimilarityProvider.java

var allSims = func() []Similarity {
	var ans []Similarity
	ans = append(ans, NewDefaultSimilarity())
	// ans = append(ans, newBM25Similarity())
	// for _, basicModel := range BASIC_MODELS {
	// 	for _, afterEffect := range AFTER_EFFECTS {
	// 		for _, normalization := range NORMALIZATIONS {
	// 			ans = append(ans, newDFRSimilarity(basicModel, afterEffect, normalization))
	// 		}
	// 	}
	// }
	// for _, distribution := range DISTRIBUTIONS {
	// 	for _, lambda := range LAMBDAS {
	// 		for _, normalization := range NORMALIZATIONS {
	// 			ans = append(ans, newIBSimilarity(ditribution, lambda, normalization))
	// 		}
	// 	}
	// }
	// ans = append(ans, newLMJelinekMercerSimilarity(0.1))
	// ans = append(ans, newLMJelinekMercerSimilarity(0.7))
	return ans
}()

/*
Similarity implementation that randomizes Similarity implementations
per-field.

The choices are 'sticky', so the selected algorithm is ways used for
the same field.
*/
type RandomSimilarityProvider struct {
	*PerFieldSimilarityWrapper
	sync.Locker
	defaultSim       *DefaultSimilarity
	knownSims        []Similarity
	previousMappings map[string]Similarity
	perFieldSeed     int
	coordType        int // 0 = no coord, 1 = coord, 2 = crazy coord
	shouldQueryNorm  bool
}

func NewRandomSimilarityProvider(r *rand.Rand) *RandomSimilarityProvider {
	sims := make([]Similarity, len(allSims))
	for i, v := range r.Perm(len(allSims)) {
		sims[i] = allSims[v]
		assert(sims[i] != nil)
	}
	ans := &RandomSimilarityProvider{
		Locker:           &sync.Mutex{},
		defaultSim:       NewDefaultSimilarity(),
		previousMappings: make(map[string]Similarity),
		perFieldSeed:     r.Int(),
		coordType:        r.Intn(3),
		shouldQueryNorm:  r.Intn(2) == 0,
		knownSims:        sims,
	}
	ans.PerFieldSimilarityWrapper = NewPerFieldSimilarityWrapper(ans)
	return ans
}

func (rp *RandomSimilarityProvider) QueryNorm(sumOfSquaredWeights float32) float32 {
	if rp.shouldQueryNorm {
		return rp.defaultSim.QueryNorm(sumOfSquaredWeights)
	}
	return 1.0
}

const primeRK = 16777619

/* simple string hash used by Go strings package */
func hashstr(sep string) int {
	hash := uint32(0)
	for i := 0; i < len(sep); i++ {
		hash = hash*primeRK + uint32(sep[i])
	}
	return int(hash)
}

func (p *RandomSimilarityProvider) Get(name string) Similarity {
	p.Lock()
	defer p.Unlock()
	sim, ok := p.previousMappings[name]
	if !ok {
		hash := int(math.Abs(math.Pow(float64(p.perFieldSeed), float64(hashstr(name)))))
		sim = p.knownSims[hash%len(p.knownSims)]
		p.previousMappings[name] = sim
	}
	assert(sim != nil)
	return sim
}

func assert(ok bool) {
	assert2(ok, "assert fail")
}

func assert2(ok bool, msg string, args ...interface{}) {
	if !ok {
		panic(fmt.Sprintf(msg, args...))
	}
}

func (rp *RandomSimilarityProvider) String() string {
	rp.Lock() // synchronized
	defer rp.Unlock()
	var coordMethod string
	switch rp.coordType {
	case 0:
		coordMethod = "no"
	case 1:
		coordMethod = "yes"
	default:
		coordMethod = "crazy"
	}
	return fmt.Sprintf("RandomSimilarityProvider(queryNorm=%v,coord=%v): %v",
		rp.shouldQueryNorm, coordMethod, rp.previousMappings)
}
<|endoftext|>


<commit_before>// The matrix package contains various utilities for dealing with raw matrices.
// The interface is loosely based on the NumPy package in Python.
//
// The key interfaces here are:
//
// * NDArray – A multidimensional array, with dense and (2D-only) sparse
//   implementations.
// * Matrix – A two-dimensional array, with various methods only available when
//   working in two dimensions.  A two-dimensional NDArray can be
//   trivially converted to the Matrix type by calling arr.M().
//
// When possible, function implementations take advantage of matrix sparsity.
// For instance, MProd(), the matrix multiplication function, performs the
// minimum amount of work required based on the types of its arguments.
//
// Certain linear algebra methods, particularly in the Matrix interface, rely
// on BLAS. In order to use it, you will need to register an appropriate engine.
// See the documentation at https://github.com/gonum/blas for details. You can
// register a default (native Go) engine by calling InitDefaultBlas().
// If you see a panic message like
// "mat64: no blas engine registered: call Register()"
// then you need to register a BLAS engine. If you don't see this error, then
// you probably don't need to worry about it.
package matrix

import (
	"fmt"
	"math/rand"
	"time"
)

func init() {
	rand.Seed(time.Now().UnixNano())
}

// ArraySparsity indicates the representation type of the matrix
type ArraySparsity int

const (
	DenseArray ArraySparsity = iota
	SparseCooMatrix
	SparseDiagMatrix
)

// A NDArray is an n-dimensional array of numbers which can be manipulated in
// various ways. Concrete implementations can differ; for instance, sparse
// and dense representations are possible.
type NDArray interface {

	// Return the element-wise sum of this array and one or more others
	Add(others ...NDArray) NDArray

	// Returns true if and only if all items are nonzero
	All() bool

	// Returns true if f is true for all array elements
	AllF(f func(v float64) bool) bool

	// Returns true if f is true for all pairs of array elements in the same position
	AllF2(f func(v1, v2 float64) bool, other NDArray) bool

	// Returns true if and only if any item is nonzero
	Any() bool

	// Returns true if f is true for any array element
	AnyF(f func(v float64) bool) bool

	// Returns true if f is true for any pair of array elements in the same position
	AnyF2(f func(v1, v2 float64) bool, other NDArray) bool

	// Return the result of applying a function to all elements
	Apply(f func(float64) float64) NDArray

	// Get the matrix data as a flattened 1D array; sparse matrices will make
	// a copy first.
	Array() []float64

	// Create a new array by concatenating this with another array along the
	// specified axis. The array shapes must be equal along all other axes.
	// It is legal to add a new axis.
	Concat(axis int, others ...NDArray) NDArray

	// Returns a duplicate of this array
	Copy() NDArray

	// Counts the number of nonzero elements in the array
	CountNonzero() int

	// Returns a dense copy of the array
	Dense() NDArray

	// Return the element-wise quotient of this array and one or more others.
	// This function defines 0 / 0 = 0, so it's useful for sparse arrays.
	Div(others ...NDArray) NDArray

	// Returns true if and only if all elements in the two arrays are equal
	Equal(other NDArray) bool

	// Set all array elements to the given value
	Fill(value float64)

	// Get an array element in a flattened verison of this array
	FlatItem(index int) float64

	// Set an array element in a flattened version of this array
	FlatItemSet(value float64, index int)

	// Return an iterator over populated matrix entries
	FlatIter() FlatNDArrayIterator

	// Get an array element
	Item(index ...int) float64

	// Return the result of adding a scalar value to each array element
	ItemAdd(value float64) NDArray

	// Return the result of dividing each array element by a scalar value
	ItemDiv(value float64) NDArray

	// Return the reuslt of multiplying each array element by a scalar value
	ItemProd(value float64) NDArray

	// Return the result of subtracting a scalar value from each array element
	ItemSub(value float64) NDArray

	// Set an array element
	ItemSet(value float64, index ...int)

	// Return an iterator over populated matrix entries
	Iter() CoordNDArrayIterator

	// Returns the array as a matrix. This is only possible for 1D and 2D arrays;
	// 1D arrays of length n are converted into n x 1 vectors.
	M() Matrix

	// Get the value of the largest array element
	Max() float64

	// Get the value of the smallest array element
	Min() float64

	// Return the element-wise product of this array and one or more others
	Prod(others ...NDArray) NDArray

	// The number of dimensions in the matrix
	NDim() int

	// Return a copy of the array, normalized to sum to 1
	Normalize() NDArray

	// Get a 1D copy of the array, in 'C' order: rightmost axes change fastest
	Ravel() NDArray

	// A slice giving the size of all array dimensions
	Shape() []int

	// The total number of elements in the matrix
	Size() int

	// Get an array containing a rectangular slice of this array.
	// `from` and `to` should both have one index per axis. The indices
	// in `from` and `to` define the first and just-past-last indices you wish
	// to select along each axis. Negative indexing is supported: when slicing,
	// index -1 refers to the item just past the last and -arr.Size() refers to
	// the first element.
	Slice(from []int, to []int) NDArray

	// Ask whether the matrix has a sparse representation (useful for optimization)
	Sparsity() ArraySparsity

	// Return the element-wise difference of this array and one or more others
	Sub(others ...NDArray) NDArray

	// Return the sum of all array elements
	Sum() float64

	// Return the same matrix, but with axes transposed. The same data is used,
	// for speed and memory efficiency. Use Copy() to create a new array.
	// A 1D array is unchanged; create a 2D analog to rotate a vector.
	T() NDArray
}

// Create an array from literal data
func A(shape []int, values ...float64) NDArray {
	size := 1
	for _, sz := range shape {
		size *= sz
	}
	if len(values) != size {
		panic(fmt.Sprintf("Expected %d array elements but got %d", size, len(values)))
	}
	array := &DenseF64Array{
		shape: shape,
		array: make([]float64, len(values)),
	}
	copy(array.array[:], values[:])
	return array
}

// Create a 1D array
func A1(values ...float64) NDArray {
	return A([]int{len(values)}, values...)
}

// Create a 2D array
func A2(values ...[]float64) NDArray {
	array := &DenseF64Array{
		shape: []int{len(values), len(values[0])},
		array: make([]float64, len(values)*len(values[0])),
	}
	for i0 := 0; i0 < array.shape[0]; i0++ {
		if len(values[i0]) != array.shape[1] {
			panic(fmt.Sprintf("A2 got inconsistent array lengths %d and %d", array.shape[1], len(values[i0])))
		}
		for i1 := 0; i1 < array.shape[1]; i1++ {
			array.ItemSet(values[i0][i1], i0, i1)
		}
	}
	return array
}

// Create an NDArray of float64 values, initialized to zero
func Dense(size ...int) NDArray {
	totalSize := 1
	for _, sz := range size {
		totalSize *= sz
	}
	return &DenseF64Array{
		shape: size,
		array: make([]float64, totalSize),
	}
}

// Create an NDArray of float64 values, initialized to value
func WithValue(value float64, size ...int) NDArray {
	array := Dense(size...)
	array.Fill(value)
	return array
}

// Create an NDArray of float64 values, initialized to zero
func Zeros(size ...int) NDArray {
	return Dense(size...)
}

// Create an NDArray of float64 values, initialized to one
func Ones(size ...int) NDArray {
	return WithValue(1.0, size...)
}

// Create a dense NDArray of float64 values, initialized to uniformly random
// values in [0, 1).
func Rand(size ...int) NDArray {
	array := Dense(size...)

	max := array.Size()
	for i := 0; i < max; i++ {
		array.FlatItemSet(rand.Float64(), i)
	}

	return array
}

// Create a dense NDArray of float64 values, initialized to random values in
// [-math.MaxFloat64, +math.MaxFloat64] distributed on the standard Normal
// distribution.
func RandN(size ...int) NDArray {
	array := Dense(size...)

	max := array.Size()
	for i := 0; i < max; i++ {
		array.FlatItemSet(rand.NormFloat64(), i)
	}

	return array
}
<commit_msg>fixed godoc<commit_after>// The matrix package contains various utilities for dealing with raw matrices.
// The interface is loosely based on the NumPy package in Python.
//
// The key interfaces here are:
//
// NDArray – A multidimensional array, with dense and (2D-only) sparse
// implementations.
//
// Matrix – A two-dimensional array, with various methods only available when
// working in two dimensions.  A two-dimensional NDArray can be
// trivially converted to the Matrix type by calling arr.M().
//
// When possible, function implementations take advantage of matrix sparsity.
// For instance, MProd(), the matrix multiplication function, performs the
// minimum amount of work required based on the types of its arguments.
//
// Certain linear algebra methods, particularly in the Matrix interface, rely
// on BLAS. In order to use it, you will need to register an appropriate engine.
// See the documentation at https://github.com/gonum/blas for details. You can
// register a default (native Go) engine by calling InitDefaultBlas().
// If you see a panic message like
// "mat64: no blas engine registered: call Register()"
// then you need to register a BLAS engine. If you don't see this error, then
// you probably don't need to worry about it.
package matrix

import (
	"fmt"
	"math/rand"
	"time"
)

func init() {
	rand.Seed(time.Now().UnixNano())
}

// ArraySparsity indicates the representation type of the matrix
type ArraySparsity int

const (
	DenseArray ArraySparsity = iota
	SparseCooMatrix
	SparseDiagMatrix
)

// A NDArray is an n-dimensional array of numbers which can be manipulated in
// various ways. Concrete implementations can differ; for instance, sparse
// and dense representations are possible.
type NDArray interface {

	// Return the element-wise sum of this array and one or more others
	Add(others ...NDArray) NDArray

	// Returns true if and only if all items are nonzero
	All() bool

	// Returns true if f is true for all array elements
	AllF(f func(v float64) bool) bool

	// Returns true if f is true for all pairs of array elements in the same position
	AllF2(f func(v1, v2 float64) bool, other NDArray) bool

	// Returns true if and only if any item is nonzero
	Any() bool

	// Returns true if f is true for any array element
	AnyF(f func(v float64) bool) bool

	// Returns true if f is true for any pair of array elements in the same position
	AnyF2(f func(v1, v2 float64) bool, other NDArray) bool

	// Return the result of applying a function to all elements
	Apply(f func(float64) float64) NDArray

	// Get the matrix data as a flattened 1D array; sparse matrices will make
	// a copy first.
	Array() []float64

	// Create a new array by concatenating this with another array along the
	// specified axis. The array shapes must be equal along all other axes.
	// It is legal to add a new axis.
	Concat(axis int, others ...NDArray) NDArray

	// Returns a duplicate of this array
	Copy() NDArray

	// Counts the number of nonzero elements in the array
	CountNonzero() int

	// Returns a dense copy of the array
	Dense() NDArray

	// Return the element-wise quotient of this array and one or more others.
	// This function defines 0 / 0 = 0, so it's useful for sparse arrays.
	Div(others ...NDArray) NDArray

	// Returns true if and only if all elements in the two arrays are equal
	Equal(other NDArray) bool

	// Set all array elements to the given value
	Fill(value float64)

	// Get an array element in a flattened verison of this array
	FlatItem(index int) float64

	// Set an array element in a flattened version of this array
	FlatItemSet(value float64, index int)

	// Return an iterator over populated matrix entries
	FlatIter() FlatNDArrayIterator

	// Get an array element
	Item(index ...int) float64

	// Return the result of adding a scalar value to each array element
	ItemAdd(value float64) NDArray

	// Return the result of dividing each array element by a scalar value
	ItemDiv(value float64) NDArray

	// Return the reuslt of multiplying each array element by a scalar value
	ItemProd(value float64) NDArray

	// Return the result of subtracting a scalar value from each array element
	ItemSub(value float64) NDArray

	// Set an array element
	ItemSet(value float64, index ...int)

	// Return an iterator over populated matrix entries
	Iter() CoordNDArrayIterator

	// Returns the array as a matrix. This is only possible for 1D and 2D arrays;
	// 1D arrays of length n are converted into n x 1 vectors.
	M() Matrix

	// Get the value of the largest array element
	Max() float64

	// Get the value of the smallest array element
	Min() float64

	// Return the element-wise product of this array and one or more others
	Prod(others ...NDArray) NDArray

	// The number of dimensions in the matrix
	NDim() int

	// Return a copy of the array, normalized to sum to 1
	Normalize() NDArray

	// Get a 1D copy of the array, in 'C' order: rightmost axes change fastest
	Ravel() NDArray

	// A slice giving the size of all array dimensions
	Shape() []int

	// The total number of elements in the matrix
	Size() int

	// Get an array containing a rectangular slice of this array.
	// `from` and `to` should both have one index per axis. The indices
	// in `from` and `to` define the first and just-past-last indices you wish
	// to select along each axis. Negative indexing is supported: when slicing,
	// index -1 refers to the item just past the last and -arr.Size() refers to
	// the first element.
	Slice(from []int, to []int) NDArray

	// Ask whether the matrix has a sparse representation (useful for optimization)
	Sparsity() ArraySparsity

	// Return the element-wise difference of this array and one or more others
	Sub(others ...NDArray) NDArray

	// Return the sum of all array elements
	Sum() float64

	// Return the same matrix, but with axes transposed. The same data is used,
	// for speed and memory efficiency. Use Copy() to create a new array.
	// A 1D array is unchanged; create a 2D analog to rotate a vector.
	T() NDArray
}

// Create an array from literal data
func A(shape []int, values ...float64) NDArray {
	size := 1
	for _, sz := range shape {
		size *= sz
	}
	if len(values) != size {
		panic(fmt.Sprintf("Expected %d array elements but got %d", size, len(values)))
	}
	array := &DenseF64Array{
		shape: shape,
		array: make([]float64, len(values)),
	}
	copy(array.array[:], values[:])
	return array
}

// Create a 1D array
func A1(values ...float64) NDArray {
	return A([]int{len(values)}, values...)
}

// Create a 2D array
func A2(values ...[]float64) NDArray {
	array := &DenseF64Array{
		shape: []int{len(values), len(values[0])},
		array: make([]float64, len(values)*len(values[0])),
	}
	for i0 := 0; i0 < array.shape[0]; i0++ {
		if len(values[i0]) != array.shape[1] {
			panic(fmt.Sprintf("A2 got inconsistent array lengths %d and %d", array.shape[1], len(values[i0])))
		}
		for i1 := 0; i1 < array.shape[1]; i1++ {
			array.ItemSet(values[i0][i1], i0, i1)
		}
	}
	return array
}

// Create an NDArray of float64 values, initialized to zero
func Dense(size ...int) NDArray {
	totalSize := 1
	for _, sz := range size {
		totalSize *= sz
	}
	return &DenseF64Array{
		shape: size,
		array: make([]float64, totalSize),
	}
}

// Create an NDArray of float64 values, initialized to value
func WithValue(value float64, size ...int) NDArray {
	array := Dense(size...)
	array.Fill(value)
	return array
}

// Create an NDArray of float64 values, initialized to zero
func Zeros(size ...int) NDArray {
	return Dense(size...)
}

// Create an NDArray of float64 values, initialized to one
func Ones(size ...int) NDArray {
	return WithValue(1.0, size...)
}

// Create a dense NDArray of float64 values, initialized to uniformly random
// values in [0, 1).
func Rand(size ...int) NDArray {
	array := Dense(size...)

	max := array.Size()
	for i := 0; i < max; i++ {
		array.FlatItemSet(rand.Float64(), i)
	}

	return array
}

// Create a dense NDArray of float64 values, initialized to random values in
// [-math.MaxFloat64, +math.MaxFloat64] distributed on the standard Normal
// distribution.
func RandN(size ...int) NDArray {
	array := Dense(size...)

	max := array.Size()
	for i := 0; i < max; i++ {
		array.FlatItemSet(rand.NormFloat64(), i)
	}

	return array
}
<|endoftext|>


<commit_before>package main

import (
    "flags"
    "fmt"
    "path/filepath"
)

func dumper(c chan string) filepath.WalkFunc {

    return func(path string, info os.FileInfo, err error) error {
        c <- string
        return nil
    }

}

func main() {

    filenames := make(chan string, 3333)
    var root string

    f := dumper(filenames)

    flags.parse()
    if len(flags.Argv) == 0 {
        root = "."
    } else {
        root = flags.Argv[0]
    }

    go func() {
        filepath.Walk(root, f)
        close(filenames)
    }()

    count := 0

    for filename := range filenames {
        count += 1
    }

    fmt.Printf("%d files found.\n", count)

}
<commit_msg>parse command-line variables<commit_after>package main

import (
    "flag"
    "fmt"
    "log"
    "os"
    "path/filepath"
    "milo/utils"
    "strings"
)

var root string
var extensions []string
var pattern string

// getNames creates a filepath.WalkFunc suitable for passing to
// filepath.Walk which passes the filenames found into a channel.
func getNames(c chan string) filepath.WalkFunc {
    return func(path string, info os.FileInfo, err error) error {
        if info.Mode().IsRegular() {
            c <- path
        }
        return nil
    }
}

// init parses the command-line arguments into the values
// used to execute. There should be a pattern at the very
// least. Optionally, a path (defaulting to "."), and
// file extensions to search may be provided.
func init() {
    flag.Parse()
    args := flag.Args()
    if len(args) == 0 {
        log.Fatalf("No arguments passed.")
    }
    args = getExts(args)
    args = getRoot(args)
}

// getExts sets the extensions global variable,
// removes any extension arguments from args,
// and returns args for further processing.
func getExts(args []string) []string {
    var unused []string
    for _, val := range args {
        if strings.HasPrefix(val, "--"){
            extensions = append(extensions, val)
        } else {
            unused = append(unused, val)
        }
    }
    return unused
}

// getRoot finds a valid directory in the command-line
// args, sets it to the global "root" variable, and
// returns the remaining arguments.
func getRoot(args []string) []string {
    var unused []string
    for _, val := range args {
        if utils.IsDir(val) {
            if root != "" {
                log.Fatalf("Too many directory arguments\n")
            } else {
                root = val
            }
        } else {
            unused = append(unused, val)
        }
    }
    if root == "" {
        root = "."
    }
    return unused
}

func main() {

    filenames := make(chan string, 3333)

    // Make a function containing this channel.
    f := getNames(filenames)

    go func() {
        filepath.Walk(root, f)
        close(filenames)
    }()

    count := 0

    for _ = range filenames {
        count += 1
    }

    fmt.Printf("%d files found.\n", count)

}
<|endoftext|>


<commit_before>package main

import (
	"code.google.com/p/go.tools/go/types"
	"flag"
	"fmt"
	"go/ast"
	"go/build"
	"go/parser"
	"go/token"
	"os"
	"path/filepath"
)

var (
	interfacesFrom string
	typesFrom      string
	reverse        bool
)

func init() {
	flag.StringVar(&interfacesFrom, "interfaces", "std", "Comma-separated list of Which packages to scan for interfaces. Defaults to std.")
	flag.StringVar(&typesFrom, "types", "", "Comma-separated list of packages whose types to check for implemented interfaces.")
	flag.BoolVar(&reverse, "reverse", false, "Print 'implemented by' as opposed to 'implements' relations.")

	flag.Parse()
}

type Interface struct {
	Name       string
	Underlying *types.Interface
	Obj        types.Object
}

type Type struct {
	Object   types.Object
	TypeName *types.TypeName
	Pointer  *types.Pointer
}

// getInterfaces extracts all the interfaces from the objects we
// parsed.
func getInterfaces(typs []Type) []Interface {
	var interfaces []Interface

	for _, typ := range typs {
		// Only types, not variables/constants
		// Only interfaces
		if iface, ok := typ.TypeName.Type().Underlying().(*types.Interface); ok {
			interfaces = append(interfaces, Interface{typ.Object.Name(), iface, typ.Object})
		}
	}

	return interfaces
}

func parseFile(fset *token.FileSet, fileName string) (f *ast.File, err error) {
	astFile, err := parser.ParseFile(fset, fileName, nil, 0)
	if err != nil {
		return f, fmt.Errorf("could not parse: %s", err)
	}

	return astFile, nil
}

type Context struct {
	allImports map[string]*types.Package
	context    types.Context
}

func NewContext() *Context {
	ctx := &Context{
		allImports: make(map[string]*types.Package),
	}

	ctx.context = types.Context{
		Import: ctx.importer,
	}

	return ctx
}

func (ctx *Context) importer(imports map[string]*types.Package, path string) (pkg *types.Package, err error) {
	// types.Importer does not seem to be designed for recursive
	// parsing like we're doing here. Specifically, each nested import
	// will maintain its own imports map. This will lead to duplicate
	// imports and in turn packages, which will lead to funny errors
	// such as "cannot pass argument ip (variable of type net.IP) to
	// variable of type net.IP"
	//
	// To work around this, we keep a global imports map, allImports,
	// to which we add all nested imports, and which we use as the
	// cache, instead of imports.
	//
	// Since all nested imports will also use this importer, there
	// should be no way to end up with duplicate imports.

	// We first try to use GcImport directly. This has the downside of
	// using possibly out-of-date packages, but it has the upside of
	// not having to parse most of the Go standard library.

	buildPkg, buildErr := build.Import(path, ".", 0)
	// If we found no build dir, assume we're dealing with installed
	// but no source. If we found a build dir, only use GcImport if
	// it's in GOROOT. This way we always use up-to-date code for
	// normal packages but avoid parsing the standard library.
	if (buildErr == nil && buildPkg.Goroot) || buildErr != nil {
		pkg, err = types.GcImport(ctx.allImports, path)
		if err == nil {
			// We don't use imports, but per API we have to add the package.
			imports[pkg.Path()] = pkg
			ctx.allImports[pkg.Path()] = pkg
			return pkg, nil
		}
	}

	// See if we already imported this package
	if pkg = ctx.allImports[path]; pkg != nil && pkg.Complete() {
		return pkg, nil
	}

	// allImports failed, try to use go/build
	if buildErr != nil {
		return nil, buildErr
	}

	// TODO check if the .a file is up to date and use it instead
	fileSet := token.NewFileSet()

	isGoFile := func(d os.FileInfo) bool {
		allFiles := make([]string, 0, len(buildPkg.GoFiles)+len(buildPkg.CgoFiles))
		allFiles = append(allFiles, buildPkg.GoFiles...)
		allFiles = append(allFiles, buildPkg.CgoFiles...)

		for _, file := range allFiles {
			if file == d.Name() {
				return true
			}
		}
		return false
	}
	pkgs, err := parser.ParseDir(fileSet, buildPkg.Dir, isGoFile, 0)
	if err != nil {
		return nil, err
	}

	delete(pkgs, "documentation")
	var astPkg *ast.Package
	var name string
	for name, astPkg = range pkgs {
		// Use the first non-main package, or the only package we
		// found.
		//
		// NOTE(dh) I can't think of a reason why there should be
		// multiple packages in a single directory, but ParseDir
		// accommodates for that possibility.
		if len(pkgs) == 1 || name != "main" {
			break
		}
	}

	if astPkg == nil {
		return nil, fmt.Errorf("can't find import: %s", name)
	}

	var ff []*ast.File
	for _, f := range astPkg.Files {
		ff = append(ff, f)
	}

	context := types.Context{
		Import: ctx.importer,
	}

	pkg, err = context.Check(name, fileSet, ff...)
	if err != nil {
		return pkg, err
	}
	if !pkg.Complete() {
		pkg = types.NewPackage(pkg.Pos(), pkg.Path(), pkg.Name(), pkg.Scope(), pkg.Imports(), true)
	}

	imports[path] = pkg
	ctx.allImports[path] = pkg
	return pkg, nil
}

func (ctx *Context) getTypes(paths ...string) ([]Type, []error) {
	var errors []error
	var typs []Type

	for _, path := range paths {
		buildPkg, err := build.Import(path, ".", 0)
		if err != nil {
			errors = append(errors, fmt.Errorf("Couldn't import %s: %s", path, err))
			continue
		}
		fset := token.NewFileSet()
		var astFiles []*ast.File
		var pkg *types.Package
		if buildPkg.Goroot {
			// TODO what if the compiled package in GoRoot is
			// outdated?
			pkg, err = types.GcImport(ctx.allImports, path)
			if err != nil {
				errors = append(errors, fmt.Errorf("Couldn't import %s: %s", path, err))
				continue
			}
		} else {
			if len(buildPkg.GoFiles) == 0 {
				errors = append(errors, fmt.Errorf("Couldn't parse %s: No go files", path))
				continue
			}
			for _, file := range buildPkg.GoFiles {
				astFile, err := parseFile(fset, filepath.Join(buildPkg.Dir, file))
				if err != nil {
					errors = append(errors, fmt.Errorf("Couldn't parse %s: %s", err))
					continue
				}
				astFiles = append(astFiles, astFile)
			}

			pkg, err = check(ctx, astFiles[0].Name.Name, fset, astFiles)
			if err != nil {
				errors = append(errors, fmt.Errorf("Couldn't parse %s: %s\n", path, err))
				continue
			}
		}

		scope := pkg.Scope()
		for i := 0; i < scope.NumEntries(); i++ {
			obj := scope.At(i)

			// Only types, not variables/constants
			if typ, ok := obj.(*types.TypeName); ok {
				typs = append(typs, Type{
					Object:   obj,
					TypeName: typ,
					Pointer:  types.NewPointer(typ.Type()),
				})
			}

		}
	}
	return typs, errors
}

func check(ctx *Context, name string, fset *token.FileSet, astFiles []*ast.File) (pkg *types.Package, err error) {
	return ctx.context.Check(name, fset, astFiles...)
}

func listErrors(errors []error) {
	for _, err := range errors {
		fmt.Println(err)
	}
}

func doesImplement(typ types.Type, iface *types.Interface) bool {
	fnc, _ := types.MissingMethod(typ, iface)
	return fnc == nil
}

func listImplementedInterfaces(universe, toCheck []Type) {
	interfaces := getInterfaces(universe)

	for _, typ := range toCheck {
		var implements []Interface
		var implementsPointer []Interface
		for _, iface := range interfaces {
			if iface.Underlying.NumMethods() == 0 {
				// Everything implements empty interfaces, skip those
				continue
			}

			if typ.Object.Pkg() == iface.Obj.Pkg() && typ.Object.Name() == iface.Name {
				// An interface will always implement itself, so skip those
				continue
			}

			if doesImplement(typ.Object.Type(), iface.Underlying) {
				implements = append(implements, iface)
			}

			if _, ok := typ.TypeName.Type().Underlying().(*types.Interface); !ok {
				if doesImplement(typ.Pointer.Underlying(), iface.Underlying) {
					implementsPointer = append(implementsPointer, iface)
				}
			}
		}

		if len(implements) > 0 {
			fmt.Printf("%s.%s implements...\n", typ.TypeName.Pkg().Path(), typ.Object.Name())
			for _, iface := range implements {
				fmt.Printf("\t%s.%s\n", iface.Obj.Pkg().Path(), iface.Name)
			}
		}
		// TODO DRY
		if len(implementsPointer) > 0 {
			fmt.Printf("*%s.%s implements...\n", typ.TypeName.Pkg().Path(), typ.Object.Name())
			for _, iface := range implementsPointer {
				fmt.Printf("\t%s.%s\n", iface.Obj.Pkg().Path(), iface.Name)
			}
		}
	}
}

func listImplementers(universe, toCheck []Type) {
	interfaces := getInterfaces(universe)

	for _, iface := range interfaces {
		var implementedBy []string
		for _, typ := range toCheck {
			if iface.Underlying.NumMethods() == 0 {
				// Everything implements empty interfaces, skip those
				continue
			}

			if typ.Object.Pkg() == iface.Obj.Pkg() && typ.Object.Name() == iface.Name {
				// An interface will always implement itself, so skip those
				continue
			}

			if doesImplement(typ.Object.Type(), iface.Underlying) {
				implementedBy = append(implementedBy, fmt.Sprintf("%s.%s", typ.TypeName.Pkg().Path(), typ.Object.Name()))
			}

			if _, ok := typ.TypeName.Type().Underlying().(*types.Interface); !ok {
				if doesImplement(typ.Pointer.Underlying(), iface.Underlying) {
					implementedBy = append(implementedBy, fmt.Sprintf("*%s.%s", typ.TypeName.Pkg().Path(), typ.Object.Name()))
				}
			}
		}

		if len(implementedBy) > 0 {
			fmt.Printf("%s.%s is implemented by...\n", iface.Obj.Pkg().Name(), iface.Name)
			for _, s := range implementedBy {
				fmt.Printf("\t%s\n", s)
			}
		}
	}
}

func main() {
	if typesFrom == "" {
		flag.Usage()
		os.Exit(1)
	}

	ctx := NewContext()
	universe, errs := ctx.getTypes(matchPackages(interfacesFrom)...)
	listErrors(errs)
	toCheck, errs := ctx.getTypes(matchPackages(typesFrom)...)
	listErrors(errs)

	if reverse {
		listImplementers(universe, toCheck)
	} else {
		listImplementedInterfaces(universe, toCheck)
	}
}
<commit_msg>improve help messages<commit_after>package main

import (
	"code.google.com/p/go.tools/go/types"
	"flag"
	"fmt"
	"go/ast"
	"go/build"
	"go/parser"
	"go/token"
	"os"
	"path/filepath"
)

var (
	interfacesFrom string
	typesFrom      string
	reverse        bool
	printHelp      bool
)

func init() {
	flag.StringVar(&interfacesFrom, "interfaces", "std", "Comma-separated list of which packages to scan for interfaces. Defaults to std.")
	flag.StringVar(&typesFrom, "types", "", "Comma-separated list of packages whose types to check for implemented interfaces. Required.")
	flag.BoolVar(&reverse, "reverse", false, "Print 'implemented by' as opposed to 'implements' relations.")
	flag.BoolVar(&printHelp, "help", false, "Print a help text and exit.")

	flag.Parse()
}

type Interface struct {
	Name       string
	Underlying *types.Interface
	Obj        types.Object
}

type Type struct {
	Object   types.Object
	TypeName *types.TypeName
	Pointer  *types.Pointer
}

// getInterfaces extracts all the interfaces from the objects we
// parsed.
func getInterfaces(typs []Type) []Interface {
	var interfaces []Interface

	for _, typ := range typs {
		// Only types, not variables/constants
		// Only interfaces
		if iface, ok := typ.TypeName.Type().Underlying().(*types.Interface); ok {
			interfaces = append(interfaces, Interface{typ.Object.Name(), iface, typ.Object})
		}
	}

	return interfaces
}

func parseFile(fset *token.FileSet, fileName string) (f *ast.File, err error) {
	astFile, err := parser.ParseFile(fset, fileName, nil, 0)
	if err != nil {
		return f, fmt.Errorf("could not parse: %s", err)
	}

	return astFile, nil
}

type Context struct {
	allImports map[string]*types.Package
	context    types.Context
}

func NewContext() *Context {
	ctx := &Context{
		allImports: make(map[string]*types.Package),
	}

	ctx.context = types.Context{
		Import: ctx.importer,
	}

	return ctx
}

func (ctx *Context) importer(imports map[string]*types.Package, path string) (pkg *types.Package, err error) {
	// types.Importer does not seem to be designed for recursive
	// parsing like we're doing here. Specifically, each nested import
	// will maintain its own imports map. This will lead to duplicate
	// imports and in turn packages, which will lead to funny errors
	// such as "cannot pass argument ip (variable of type net.IP) to
	// variable of type net.IP"
	//
	// To work around this, we keep a global imports map, allImports,
	// to which we add all nested imports, and which we use as the
	// cache, instead of imports.
	//
	// Since all nested imports will also use this importer, there
	// should be no way to end up with duplicate imports.

	// We first try to use GcImport directly. This has the downside of
	// using possibly out-of-date packages, but it has the upside of
	// not having to parse most of the Go standard library.

	buildPkg, buildErr := build.Import(path, ".", 0)
	// If we found no build dir, assume we're dealing with installed
	// but no source. If we found a build dir, only use GcImport if
	// it's in GOROOT. This way we always use up-to-date code for
	// normal packages but avoid parsing the standard library.
	if (buildErr == nil && buildPkg.Goroot) || buildErr != nil {
		pkg, err = types.GcImport(ctx.allImports, path)
		if err == nil {
			// We don't use imports, but per API we have to add the package.
			imports[pkg.Path()] = pkg
			ctx.allImports[pkg.Path()] = pkg
			return pkg, nil
		}
	}

	// See if we already imported this package
	if pkg = ctx.allImports[path]; pkg != nil && pkg.Complete() {
		return pkg, nil
	}

	// allImports failed, try to use go/build
	if buildErr != nil {
		return nil, buildErr
	}

	// TODO check if the .a file is up to date and use it instead
	fileSet := token.NewFileSet()

	isGoFile := func(d os.FileInfo) bool {
		allFiles := make([]string, 0, len(buildPkg.GoFiles)+len(buildPkg.CgoFiles))
		allFiles = append(allFiles, buildPkg.GoFiles...)
		allFiles = append(allFiles, buildPkg.CgoFiles...)

		for _, file := range allFiles {
			if file == d.Name() {
				return true
			}
		}
		return false
	}
	pkgs, err := parser.ParseDir(fileSet, buildPkg.Dir, isGoFile, 0)
	if err != nil {
		return nil, err
	}

	delete(pkgs, "documentation")
	var astPkg *ast.Package
	var name string
	for name, astPkg = range pkgs {
		// Use the first non-main package, or the only package we
		// found.
		//
		// NOTE(dh) I can't think of a reason why there should be
		// multiple packages in a single directory, but ParseDir
		// accommodates for that possibility.
		if len(pkgs) == 1 || name != "main" {
			break
		}
	}

	if astPkg == nil {
		return nil, fmt.Errorf("can't find import: %s", name)
	}

	var ff []*ast.File
	for _, f := range astPkg.Files {
		ff = append(ff, f)
	}

	context := types.Context{
		Import: ctx.importer,
	}

	pkg, err = context.Check(name, fileSet, ff...)
	if err != nil {
		return pkg, err
	}
	if !pkg.Complete() {
		pkg = types.NewPackage(pkg.Pos(), pkg.Path(), pkg.Name(), pkg.Scope(), pkg.Imports(), true)
	}

	imports[path] = pkg
	ctx.allImports[path] = pkg
	return pkg, nil
}

func (ctx *Context) getTypes(paths ...string) ([]Type, []error) {
	var errors []error
	var typs []Type

	for _, path := range paths {
		buildPkg, err := build.Import(path, ".", 0)
		if err != nil {
			errors = append(errors, fmt.Errorf("Couldn't import %s: %s", path, err))
			continue
		}
		fset := token.NewFileSet()
		var astFiles []*ast.File
		var pkg *types.Package
		if buildPkg.Goroot {
			// TODO what if the compiled package in GoRoot is
			// outdated?
			pkg, err = types.GcImport(ctx.allImports, path)
			if err != nil {
				errors = append(errors, fmt.Errorf("Couldn't import %s: %s", path, err))
				continue
			}
		} else {
			if len(buildPkg.GoFiles) == 0 {
				errors = append(errors, fmt.Errorf("Couldn't parse %s: No go files", path))
				continue
			}
			for _, file := range buildPkg.GoFiles {
				astFile, err := parseFile(fset, filepath.Join(buildPkg.Dir, file))
				if err != nil {
					errors = append(errors, fmt.Errorf("Couldn't parse %s: %s", err))
					continue
				}
				astFiles = append(astFiles, astFile)
			}

			pkg, err = check(ctx, astFiles[0].Name.Name, fset, astFiles)
			if err != nil {
				errors = append(errors, fmt.Errorf("Couldn't parse %s: %s\n", path, err))
				continue
			}
		}

		scope := pkg.Scope()
		for i := 0; i < scope.NumEntries(); i++ {
			obj := scope.At(i)

			// Only types, not variables/constants
			if typ, ok := obj.(*types.TypeName); ok {
				typs = append(typs, Type{
					Object:   obj,
					TypeName: typ,
					Pointer:  types.NewPointer(typ.Type()),
				})
			}

		}
	}
	return typs, errors
}

func check(ctx *Context, name string, fset *token.FileSet, astFiles []*ast.File) (pkg *types.Package, err error) {
	return ctx.context.Check(name, fset, astFiles...)
}

func listErrors(errors []error) {
	for _, err := range errors {
		fmt.Println(err)
	}
}

func doesImplement(typ types.Type, iface *types.Interface) bool {
	fnc, _ := types.MissingMethod(typ, iface)
	return fnc == nil
}

func listImplementedInterfaces(universe, toCheck []Type) {
	interfaces := getInterfaces(universe)

	for _, typ := range toCheck {
		var implements []Interface
		var implementsPointer []Interface
		for _, iface := range interfaces {
			if iface.Underlying.NumMethods() == 0 {
				// Everything implements empty interfaces, skip those
				continue
			}

			if typ.Object.Pkg() == iface.Obj.Pkg() && typ.Object.Name() == iface.Name {
				// An interface will always implement itself, so skip those
				continue
			}

			if doesImplement(typ.Object.Type(), iface.Underlying) {
				implements = append(implements, iface)
			}

			if _, ok := typ.TypeName.Type().Underlying().(*types.Interface); !ok {
				if doesImplement(typ.Pointer.Underlying(), iface.Underlying) {
					implementsPointer = append(implementsPointer, iface)
				}
			}
		}

		if len(implements) > 0 {
			fmt.Printf("%s.%s implements...\n", typ.TypeName.Pkg().Path(), typ.Object.Name())
			for _, iface := range implements {
				fmt.Printf("\t%s.%s\n", iface.Obj.Pkg().Path(), iface.Name)
			}
		}
		// TODO DRY
		if len(implementsPointer) > 0 {
			fmt.Printf("*%s.%s implements...\n", typ.TypeName.Pkg().Path(), typ.Object.Name())
			for _, iface := range implementsPointer {
				fmt.Printf("\t%s.%s\n", iface.Obj.Pkg().Path(), iface.Name)
			}
		}
	}
}

func listImplementers(universe, toCheck []Type) {
	interfaces := getInterfaces(universe)

	for _, iface := range interfaces {
		var implementedBy []string
		for _, typ := range toCheck {
			if iface.Underlying.NumMethods() == 0 {
				// Everything implements empty interfaces, skip those
				continue
			}

			if typ.Object.Pkg() == iface.Obj.Pkg() && typ.Object.Name() == iface.Name {
				// An interface will always implement itself, so skip those
				continue
			}

			if doesImplement(typ.Object.Type(), iface.Underlying) {
				implementedBy = append(implementedBy, fmt.Sprintf("%s.%s", typ.TypeName.Pkg().Path(), typ.Object.Name()))
			}

			if _, ok := typ.TypeName.Type().Underlying().(*types.Interface); !ok {
				if doesImplement(typ.Pointer.Underlying(), iface.Underlying) {
					implementedBy = append(implementedBy, fmt.Sprintf("*%s.%s", typ.TypeName.Pkg().Path(), typ.Object.Name()))
				}
			}
		}

		if len(implementedBy) > 0 {
			fmt.Printf("%s.%s is implemented by...\n", iface.Obj.Pkg().Name(), iface.Name)
			for _, s := range implementedBy {
				fmt.Printf("\t%s\n", s)
			}
		}
	}
}

func main() {
	if printHelp {
		flag.Usage()
		fmt.Fprintln(os.Stderr)
		fmt.Fprintln(os.Stderr,
			`implements is a tool that will tell you which types implement which
interfaces, or alternatively by which types interfaces are
implemented.

You use it by specifying a set of packages to scan for interfaces and
another set of packages to scan for types. The two sets can but don't
have to overlap.

When specifying packages, "std" will stand for all of the standard
library. Also, the "..." pattern as understood by the go tool is
supported as well.

By default, implements will iterate all types and list the interfaces
they implement. By supplying the -reverse flag, however, it will
iterate all interfaces and list the types that implement the
interfaces.

Example: For all interfaces in the fmt package you want to know the
types in the standard library that implement them:

    implements -interfaces fmt -types std -reverse

Another example: For all types in your own package you want to know
which interfaces from the standard library they implement:

    implements -interfaces std -types my/own/package`)

		os.Exit(0)
	}

	if typesFrom == "" {
		flag.Usage()
		os.Exit(1)
	}

	ctx := NewContext()
	universe, errs := ctx.getTypes(matchPackages(interfacesFrom)...)
	listErrors(errs)
	toCheck, errs := ctx.getTypes(matchPackages(typesFrom)...)
	listErrors(errs)

	if reverse {
		listImplementers(universe, toCheck)
	} else {
		listImplementedInterfaces(universe, toCheck)
	}
}
<|endoftext|>


<commit_before>package jocko

import (
	"fmt"

	"github.com/travisjeffery/jocko/log"
	"github.com/travisjeffery/jocko/protocol"
)

// Client is used to request other brokers.
type client interface {
	FetchMessages(clientID string, fetchRequest *protocol.FetchRequest) (*protocol.FetchResponses, error)
	CreateTopics(clientID string, createRequest *protocol.CreateTopicRequests) (*protocol.CreateTopicsResponse, error)
	LeaderAndISR(clientID string, request *protocol.LeaderAndISRRequest) (*protocol.LeaderAndISRResponse, error)
	// others
}

// Replicator fetches from the partition's leader producing to itself the follower, thereby replicating the partition.
type Replicator struct {
	config              ReplicatorConfig
	logger              log.Logger
	replica             *Replica
	clientID            string
	minBytes            int32
	fetchSize           int32
	maxWaitTime         int32
	highwaterMarkOffset int64
	offset              int64
	msgs                chan []byte
	done                chan struct{}
	leader              client
}

type ReplicatorConfig struct {
	MinBytes    int32
	MaxWaitTime int32
}

// NewReplicator returns a new replicator instance.
func NewReplicator(config ReplicatorConfig, replica *Replica, leader client, logger log.Logger) *Replicator {
	r := &Replicator{
		config:   config,
		logger:   logger,
		replica:  replica,
		clientID: fmt.Sprintf("Replicator-%d", replica.BrokerID),
		leader:   leader,
		done:     make(chan struct{}, 2),
		msgs:     make(chan []byte, 2),
	}
	return r
}

func (r *Replicator) Replicate() {
	go r.fetchMessages()
	go r.appendMessages()
}

func (r *Replicator) fetchMessages() {
	for {
		select {
		case <-r.done:
			return
		default:
			fetchRequest := &protocol.FetchRequest{
				ReplicaID:   r.replica.BrokerID,
				MaxWaitTime: r.maxWaitTime,
				MinBytes:    r.minBytes,
				Topics: []*protocol.FetchTopic{{
					Topic: r.replica.Partition.Topic,
					Partitions: []*protocol.FetchPartition{{
						Partition:   r.replica.Partition.ID,
						FetchOffset: r.offset,
					}},
				}},
			}
			fetchResponse, err := r.leader.FetchMessages(r.clientID, fetchRequest)
			// TODO: probably shouldn't panic. just let this replica fall out of ISR.
			if err != nil {
				r.logger.Error("failed to fetch messages", log.Error("error", err))
				continue
			}
			for _, resp := range fetchResponse.Responses {
				for _, p := range resp.PartitionResponses {
					offset := int64(protocol.Encoding.Uint64(p.RecordSet[:8])) + 1
					if offset > r.offset {
						r.msgs <- p.RecordSet
						r.highwaterMarkOffset = p.HighWatermark
						r.offset = offset
					}
				}
			}
		}
	}
}

func (r *Replicator) appendMessages() {
	for {
		select {
		case <-r.done:
			return
		case msg := <-r.msgs:
			_, err := r.replica.Log.Append(msg)
			if err != nil {
				panic(err)
			}
		}
	}
}

// Close the replicator object when we are no longer following
func (r *Replicator) Close() error {
	close(r.done)
	return nil
}
<commit_msg>replicator: handle bad fetch requests<commit_after>package jocko

import (
	"fmt"

	"github.com/travisjeffery/jocko/log"
	"github.com/travisjeffery/jocko/protocol"
)

// Client is used to request other brokers.
type client interface {
	FetchMessages(clientID string, fetchRequest *protocol.FetchRequest) (*protocol.FetchResponses, error)
	CreateTopics(clientID string, createRequest *protocol.CreateTopicRequests) (*protocol.CreateTopicsResponse, error)
	LeaderAndISR(clientID string, request *protocol.LeaderAndISRRequest) (*protocol.LeaderAndISRResponse, error)
	// others
}

// Replicator fetches from the partition's leader producing to itself the follower, thereby replicating the partition.
type Replicator struct {
	config              ReplicatorConfig
	logger              log.Logger
	replica             *Replica
	clientID            string
	minBytes            int32
	fetchSize           int32
	maxWaitTime         int32
	highwaterMarkOffset int64
	offset              int64
	msgs                chan []byte
	done                chan struct{}
	leader              client
}

type ReplicatorConfig struct {
	MinBytes    int32
	MaxWaitTime int32
}

// NewReplicator returns a new replicator instance.
func NewReplicator(config ReplicatorConfig, replica *Replica, leader client, logger log.Logger) *Replicator {
	r := &Replicator{
		config:   config,
		logger:   logger,
		replica:  replica,
		clientID: fmt.Sprintf("Replicator-%d", replica.BrokerID),
		leader:   leader,
		done:     make(chan struct{}, 2),
		msgs:     make(chan []byte, 2),
	}
	return r
}

func (r *Replicator) Replicate() {
	go r.fetchMessages()
	go r.appendMessages()
}

func (r *Replicator) fetchMessages() {
	for {
		select {
		case <-r.done:
			return
		default:
			fetchRequest := &protocol.FetchRequest{
				ReplicaID:   r.replica.BrokerID,
				MaxWaitTime: r.maxWaitTime,
				MinBytes:    r.minBytes,
				Topics: []*protocol.FetchTopic{{
					Topic: r.replica.Partition.Topic,
					Partitions: []*protocol.FetchPartition{{
						Partition:   r.replica.Partition.ID,
						FetchOffset: r.offset,
					}},
				}},
			}
			fetchResponse, err := r.leader.FetchMessages(r.clientID, fetchRequest)
			// TODO: probably shouldn't panic. just let this replica fall out of ISR.
			if err != nil {
				r.logger.Error("failed to fetch messages", log.Error("error", err))
				continue
			}
			for _, resp := range fetchResponse.Responses {
				for _, p := range resp.PartitionResponses {
					if p.ErrorCode != protocol.ErrNone.Code() {
						r.logger.Error("partition response error", log.Int16("error code", p.ErrorCode), log.Any("response", p))
						continue
					}
					if p.RecordSet == nil {
						r.logger.Debug("replicator: fetch messages: record set is nil")
						continue
					}
					offset := int64(protocol.Encoding.Uint64(p.RecordSet[:8]))
					if offset > r.offset {
						r.msgs <- p.RecordSet
						r.highwaterMarkOffset = p.HighWatermark
						r.offset = offset
					}
				}
			}
		}
	}
}

func (r *Replicator) appendMessages() {
	for {
		select {
		case <-r.done:
			return
		case msg := <-r.msgs:
			_, err := r.replica.Log.Append(msg)
			if err != nil {
				panic(err)
			}
		}
	}
}

// Close the replicator object when we are no longer following
func (r *Replicator) Close() error {
	close(r.done)
	return nil
}
<|endoftext|>