<commit_before>package main

import (
	"context"
	"errors"
	"io"
	"strings"

	neo "github.com/johnnadratowski/golang-neo4j-bolt-driver"
)

var DefaultMaxConn = 20

type service struct {
	url  string
	pool neo.DriverPool
}

func (s *service) Match(cxt context.Context, vocab, pattern string, res chan<- *Class) error {
	if vocab == "" {
		return errors.New("vocab cannot be empty")
	}

	if pattern == "" {
		return errors.New("pattern cannot be empty")
	}

	// Case insensitive.
	pattern = strings.ToLower(pattern)

	conn, err := s.pool.OpenPool()
	if err != nil {
		return err
	}
	defer conn.Close()

	stmt, err := conn.PrepareNeo(`
		MATCH (v:Vocabulary {id: {vocab}})<-[:classOf]-(c:Class)
		WHERE lower(c.label) =~ {pattern}
			OR any(syn in c.synonyms where lower(syn) =~ {pattern})
		RETURN c.id, v.id, c.label, c.code, c.synonyms
	`)
	if err != nil {
		return err
	}

	rows, err := stmt.QueryNeo(map[string]interface{}{
		"pattern": pattern,
		"vocab":   vocab,
	})
	if err != nil {
		return err
	}
	defer rows.Close()

	for {
		vals, _, err := rows.NextNeo()
		if err == io.EOF {
			break
		} else if err != nil {
			return err
		}

		c := &Class{
			ID:    vals[0].(string),
			Vocab: vals[1].(string),
			Label: vals[2].(string),
			Code:  vals[3].(string),
		}

		if vals[4] != nil {
			for _, v := range vals[4].([]interface{}) {
				c.Synonyms = append(c.Synonyms, v.(string))
			}
		}

		select {
		case <-cxt.Done():
			return nil

		case res <- c:
		}
	}

	return nil
}

// Get a class by code.
func (s *service) Get(vocab, code string) (*Class, error) {
	if vocab == "" {
		return nil, errors.New("vocab cannot be empty")
	}

	if code == "" {
		return nil, errors.New("code cannot be empty")
	}

	conn, err := s.pool.OpenPool()
	if err != nil {
		return nil, err
	}
	defer conn.Close()

	stmt, err := conn.PrepareNeo(`
		MATCH (v:Vocabulary {id: {vocab}})<-[:classOf]-(c:Class {code: {code}})
		RETURN c.id, v.id, c.label, c.code, c.synonyms
		LIMIT 1
	`)
	if err != nil {
		return nil, err
	}

	rows, err := stmt.QueryNeo(map[string]interface{}{
		"vocab": vocab,
		"code":  code,
	})
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	vals, _, err := rows.NextNeo()
	if err == io.EOF {
		return nil, nil
	} else if err != nil {
		return nil, err
	}

	c := &Class{
		ID:    vals[0].(string),
		Vocab: vals[1].(string),
		Label: vals[2].(string),
		Code:  vals[3].(string),
	}

	if vals[4] != nil {
		for _, v := range vals[4].([]interface{}) {
			c.Synonyms = append(c.Synonyms, v.(string))
		}
	}

	return c, nil
}

// Checks whether a set of codes exist.
func (s *service) Validate(vocab string, codes []string) ([]bool, error) {
	if vocab == "" {
		return nil, errors.New("vocab cannot be empty")
	}

	if len(codes) == 0 {
		return nil, nil
	}

	conn, err := s.pool.OpenPool()
	if err != nil {
		return nil, err
	}
	defer conn.Close()

	stmt, err := conn.PrepareNeo(`
		MATCH (:Vocabulary {id: {vocab}})<-[:classOf]-(c:Class {code: {code}})
		RETURN 1
		LIMIT 1
	`)
	if err != nil {
		return nil, err
	}

	params := map[string]interface{}{
		"code":  "",
		"vocab": vocab,
	}

	bools := make([]bool, len(codes))

	for i, code := range codes {
		params["code"] = code

		rows, err := stmt.QueryNeo(params)
		if err != nil {
			return nil, err
		}

		_, _, err = rows.NextNeo()
		rows.Close()

		if err == io.EOF {
			continue
		} else if err != nil {
			return nil, err
		}

		bools[i] = true
	}

	return bools, nil
}

func (s *service) traversal(cxt context.Context, query, vocab, code string, res chan<- *Class) error {
	if vocab == "" {
		return errors.New("vocab cannot be empty")
	}

	if code == "" {
		return errors.New("code must be specified")
	}

	conn, err := s.pool.OpenPool()
	if err != nil {
		return err
	}
	defer conn.Close()

	stmt, err := conn.PrepareNeo(query)
	if err != nil {
		return err
	}

	rows, err := stmt.QueryNeo(map[string]interface{}{
		"code":  code,
		"vocab": vocab,
	})
	if err != nil {
		return err
	}
	defer rows.Close()

	for {
		vals, _, err := rows.NextNeo()
		if err == io.EOF {
			break
		} else if err != nil {
			return err
		}

		c := &Class{
			ID:    vals[0].(string),
			Vocab: vals[1].(string),
			Label: vals[2].(string),
			Code:  vals[3].(string),
		}

		if vals[4] != nil {
			for _, v := range vals[4].([]interface{}) {
				c.Synonyms = append(c.Synonyms, v.(string))
			}
		}

		select {
		case <-cxt.Done():
			return nil

		case res <- c:
		}
	}

	return nil
}

// Parents returns all parents of a class.
func (s *service) Parents(cxt context.Context, vocab, code string, res chan<- *Class) error {
	query := `
		MATCH (v:Vocabulary {id: {vocab}})<-[:classOf]-(:Class {code: {code}})-[:subClassOf]->(c:Class)
		RETURN c.id, v.id, c.label, c.code, c.synonyms
	`
	return s.traversal(cxt, query, vocab, code, res)
}

// Children returns all children of a class.
func (s *service) Children(cxt context.Context, vocab, code string, res chan<- *Class) error {
	query := `
		MATCH (v:Vocabulary {id: {vocab}})<-[:classOf]-(:Class {code: {code}})<-[:subClassOf]-(c:Class)
		RETURN c.id, v.id, c.label, c.code, c.synonyms
	`
	return s.traversal(cxt, query, vocab, code, res)
}

// Get all ancestors of this class.
func (s *service) Ancestors(cxt context.Context, vocab, code string, res chan<- *Class) error {
	query := `
		MATCH (v:Vocabulary {id: {vocab}})<-[:classOf]-(:Class {code: {code}})-[:subClassOf*1..]->(c:Class)
		RETURN c.id, v.id, c.label, c.code, c.synonyms
	`
	return s.traversal(cxt, query, vocab, code, res)
}

// Get all descendants of this class.
func (s *service) Descendants(cxt context.Context, vocab, code string, res chan<- *Class) error {
	query := `
		MATCH (v:Vocabulary {id: {vocab}})<-[:classOf]-(:Class {code: {code}})<-[:subClassOf*1..]-(c:Class)
		RETURN c.id, v.id, c.label, c.code, c.synonyms
	`
	return s.traversal(cxt, query, vocab, code, res)
}

// Flatten takes a set of codes and returns all the codes themselves with
// all descendants. The use case if for matching
func (s *service) Flatten(cxt context.Context, vocab string, codes []string, res chan<- *Class) error {
	query := `
			MATCH (v:Vocabulary {id: {vocab}})<-[:classOf]-(:Class {code: {code}})<-[:subClassOf*0..]-(c:Class)
			RETURN c.id, v.id, c.label, c.code, c.synonyms
		`
	for _, code := range codes {
		if err := s.traversal(cxt, query, vocab, code, res); err != nil {
			return err
		}
	}

	return nil
}

func NewService(url string, maxconn int) (Service, error) {
	if maxconn <= 0 {
		maxconn = DefaultMaxConn
	}

	if !strings.HasPrefix(url, "bolt://") {
		url = "bolt://" + url
	}

	pool, err := neo.NewDriverPool(url, maxconn)
	if err != nil {
		return nil, err
	}

	return &service{
		url:  url,
		pool: pool,
	}, nil
}
<commit_msg>Pass params to traversal method<commit_after>package main

import (
	"context"
	"errors"
	"io"
	"strings"

	neo "github.com/johnnadratowski/golang-neo4j-bolt-driver"
)

var DefaultMaxConn = 20

type service struct {
	url  string
	pool neo.DriverPool
}

func (s *service) Match(cxt context.Context, vocab, pattern string, res chan<- *Class) error {
	if vocab == "" {
		return errors.New("vocab cannot be empty")
	}

	if pattern == "" {
		return errors.New("pattern cannot be empty")
	}

	query := `
		MATCH (v:Vocabulary {id: {vocab}})<-[:classOf]-(c:Class)
		WHERE lower(c.label) =~ {pattern}
			OR any(syn in c.synonyms where lower(syn) =~ {pattern})
		RETURN c.id, v.id, c.label, c.code, c.synonyms
	`

	// Case insensitive.
	pattern = strings.ToLower(pattern)

	params := map[string]interface{}{
		"pattern": pattern,
		"vocab":   vocab,
	}

	return s.traversal(cxt, query, params, res)
}

// Get a class by code.
func (s *service) Get(vocab, code string) (*Class, error) {
	if vocab == "" {
		return nil, errors.New("vocab cannot be empty")
	}

	if code == "" {
		return nil, errors.New("code cannot be empty")
	}

	conn, err := s.pool.OpenPool()
	if err != nil {
		return nil, err
	}
	defer conn.Close()

	stmt, err := conn.PrepareNeo(`
		MATCH (v:Vocabulary {id: {vocab}})<-[:classOf]-(c:Class {code: {code}})
		RETURN c.id, v.id, c.label, c.code, c.synonyms
		LIMIT 1
	`)
	if err != nil {
		return nil, err
	}

	rows, err := stmt.QueryNeo(map[string]interface{}{
		"vocab": vocab,
		"code":  code,
	})
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	vals, _, err := rows.NextNeo()
	if err == io.EOF {
		return nil, nil
	} else if err != nil {
		return nil, err
	}

	c := &Class{
		ID:    vals[0].(string),
		Vocab: vals[1].(string),
		Label: vals[2].(string),
		Code:  vals[3].(string),
	}

	if vals[4] != nil {
		for _, v := range vals[4].([]interface{}) {
			c.Synonyms = append(c.Synonyms, v.(string))
		}
	}

	return c, nil
}

// Checks whether a set of codes exist.
func (s *service) Validate(vocab string, codes []string) ([]bool, error) {
	if vocab == "" {
		return nil, errors.New("vocab cannot be empty")
	}

	if len(codes) == 0 {
		return nil, nil
	}

	conn, err := s.pool.OpenPool()
	if err != nil {
		return nil, err
	}
	defer conn.Close()

	stmt, err := conn.PrepareNeo(`
		MATCH (:Vocabulary {id: {vocab}})<-[:classOf]-(c:Class {code: {code}})
		RETURN 1
		LIMIT 1
	`)
	if err != nil {
		return nil, err
	}

	params := map[string]interface{}{
		"code":  "",
		"vocab": vocab,
	}

	bools := make([]bool, len(codes))

	for i, code := range codes {
		params["code"] = code

		rows, err := stmt.QueryNeo(params)
		if err != nil {
			return nil, err
		}

		_, _, err = rows.NextNeo()
		rows.Close()

		if err == io.EOF {
			continue
		} else if err != nil {
			return nil, err
		}

		bools[i] = true
	}

	return bools, nil
}

// traversal executes a query and populates a channel of classes that match.
func (s *service) traversal(cxt context.Context, query string, params map[string]interface{}, res chan<- *Class) error {
	conn, err := s.pool.OpenPool()
	if err != nil {
		return err
	}
	defer conn.Close()

	stmt, err := conn.PrepareNeo(query)
	if err != nil {
		return err
	}

	rows, err := stmt.QueryNeo(params)
	if err != nil {
		return err
	}
	defer rows.Close()

	for {
		vals, _, err := rows.NextNeo()
		if err == io.EOF {
			break
		} else if err != nil {
			return err
		}

		c := &Class{
			ID:    vals[0].(string),
			Vocab: vals[1].(string),
			Label: vals[2].(string),
			Code:  vals[3].(string),
		}

		if vals[4] != nil {
			for _, v := range vals[4].([]interface{}) {
				c.Synonyms = append(c.Synonyms, v.(string))
			}
		}

		select {
		case <-cxt.Done():
			return nil

		case res <- c:
		}
	}

	return nil
}

// Parents returns all parents of a class.
func (s *service) Parents(cxt context.Context, vocab, code string, res chan<- *Class) error {
	if vocab == "" {
		return errors.New("vocab cannot be empty")
	}

	if code == "" {
		return errors.New("code must be specified")
	}

	query := `
		MATCH (v:Vocabulary {id: {vocab}})<-[:classOf]-(:Class {code: {code}})-[:subClassOf]->(c:Class)
		RETURN c.id, v.id, c.label, c.code, c.synonyms
	`

	params := map[string]interface{}{
		"vocab": vocab,
		"code":  code,
	}

	return s.traversal(cxt, query, params, res)
}

// Children returns all children of a class.
func (s *service) Children(cxt context.Context, vocab, code string, res chan<- *Class) error {
	if vocab == "" {
		return errors.New("vocab cannot be empty")
	}

	if code == "" {
		return errors.New("code must be specified")
	}

	query := `
		MATCH (v:Vocabulary {id: {vocab}})<-[:classOf]-(:Class {code: {code}})<-[:subClassOf]-(c:Class)
		RETURN c.id, v.id, c.label, c.code, c.synonyms
	`

	params := map[string]interface{}{
		"vocab": vocab,
		"code":  code,
	}

	return s.traversal(cxt, query, params, res)
}

// Get all ancestors of this class.
func (s *service) Ancestors(cxt context.Context, vocab, code string, res chan<- *Class) error {
	if vocab == "" {
		return errors.New("vocab cannot be empty")
	}

	if code == "" {
		return errors.New("code must be specified")
	}

	query := `
		MATCH (v:Vocabulary {id: {vocab}})<-[:classOf]-(:Class {code: {code}})-[:subClassOf*1..]->(c:Class)
		RETURN c.id, v.id, c.label, c.code, c.synonyms
	`

	params := map[string]interface{}{
		"vocab": vocab,
		"code":  code,
	}

	return s.traversal(cxt, query, params, res)
}

// Get all descendants of this class.
func (s *service) Descendants(cxt context.Context, vocab, code string, res chan<- *Class) error {
	if vocab == "" {
		return errors.New("vocab cannot be empty")
	}

	if code == "" {
		return errors.New("code must be specified")
	}

	query := `
		MATCH (v:Vocabulary {id: {vocab}})<-[:classOf]-(:Class {code: {code}})<-[:subClassOf*1..]-(c:Class)
		RETURN c.id, v.id, c.label, c.code, c.synonyms
	`

	params := map[string]interface{}{
		"vocab": vocab,
		"code":  code,
	}

	return s.traversal(cxt, query, params, res)
}

// Flatten takes a set of codes and returns all the codes themselves with
// all descendants. The use case if for matching
func (s *service) Flatten(cxt context.Context, vocab string, codes []string, res chan<- *Class) error {
	if vocab == "" {
		return errors.New("vocab cannot be empty")
	}

	if len(codes) == 0 {
		return errors.New("at least one code must be specified")
	}

	query := `
		MATCH (v:Vocabulary {id: {vocab}})<-[:classOf]-(:Class {code: {code}})<-[:subClassOf*0..]-(c:Class)
		RETURN c.id, v.id, c.label, c.code, c.synonyms
	`

	for _, code := range codes {
		params := map[string]interface{}{
			"vocab": vocab,
			"code":  code,
		}
		if err := s.traversal(cxt, query, params, res); err != nil {
			return err
		}
	}

	return nil
}

func NewService(url string, maxconn int) (Service, error) {
	if maxconn <= 0 {
		maxconn = DefaultMaxConn
	}

	if !strings.HasPrefix(url, "bolt://") {
		url = "bolt://" + url
	}

	pool, err := neo.NewDriverPool(url, maxconn)
	if err != nil {
		return nil, err
	}

	return &service{
		url:  url,
		pool: pool,
	}, nil
}
<|endoftext|>


<commit_before>package manta

// Required for CWorld to work:
// ----------------------------
// - CHandle< CBaseEntity >
// - CBodyComponent
// - CNetworkedQuantizedFloat
// - CGameSceneNodeHandle
// - CStrongHandle< InfoForResourceTypeCTextureBase >
// - CStrongHandle< InfoForResourceTypeCModel >
// - CUtlStringToken
// - CUtlVector< CAnimationLayer >
// - CEntityIdentity*
// - CUtlSymbolLarge
// - CPhysicsComponent
// - CRenderComponent
//
// - Color
// - QAngle
// - HSequence
// - Vector
// - SolidType_t
// - SurroundingBoundsType_t
// - MoveCollide_t
// - MoveType_t
// - gender_t
// - RenderMode_t
// - RenderFx_t
//
// - bool
// - uint8
// - uint16
// - uint32
// - uint64
// - int8
// - int32
// - float32
//
// - float32[24]
// - CStrongHandle< InfoForResourceTypeIMaterial2 >[6]

// Type for a decoder function
type DecodeFcn func(*reader) interface{}

// Type for an array decoder function
type DecodeArrayFcn func(*reader) interface{}

// PropertySerializer interface
type PropertySerializer struct {
	Decode      DecodeFcn
	DecodeArray DecodeArrayFcn
	IsArray     bool
	Length      uint32
}

// Contains a list of available property serializers
type PropertySerializerTable struct {
	Serializers map[string]*PropertySerializer
}

// Returns a table containing all know property serializers
func GetDefaultPropertySerializerTable() *PropertySerializerTable {
	// Init table
	tbl := &PropertySerializerTable{}
	tbl.Serializers = make(map[string]*PropertySerializer)

	// Append default serializers/decoders
	// For now, only arrays are added

	tbl.Serializers["float32[24]"] = &PropertySerializer{
		nil, nil, true, 24,
	}

	tbl.Serializers["CStrongHandle< InfoForResourceTypeIMaterial2 >[6]"] = &PropertySerializer{
		nil, nil, true, 6,
	}

	return tbl
}

// Returns a serializer by name
func (pst *PropertySerializerTable) GetPropertySerializerByName(name string) *PropertySerializer {
	// This function should panic at some point
	return pst.Serializers[name]
}
<commit_msg>Added default serializer to return for GetPropertySerializerByName<commit_after>package manta

// Required for CWorld to work:
// ----------------------------
// - CHandle< CBaseEntity >
// - CBodyComponent
// - CNetworkedQuantizedFloat
// - CGameSceneNodeHandle
// - CStrongHandle< InfoForResourceTypeCTextureBase >
// - CStrongHandle< InfoForResourceTypeCModel >
// - CUtlStringToken
// - CUtlVector< CAnimationLayer >
// - CEntityIdentity*
// - CUtlSymbolLarge
// - CPhysicsComponent
// - CRenderComponent
//
// - Color
// - QAngle
// - HSequence
// - Vector
// - SolidType_t
// - SurroundingBoundsType_t
// - MoveCollide_t
// - MoveType_t
// - gender_t
// - RenderMode_t
// - RenderFx_t
//
// - bool
// - uint8
// - uint16
// - uint32
// - uint64
// - int8
// - int32
// - float32
//
// - float32[24]
// - CStrongHandle< InfoForResourceTypeIMaterial2 >[6]

// Type for a decoder function
type DecodeFcn func(*reader) interface{}

// Type for an array decoder function
type DecodeArrayFcn func(*reader) interface{}

// PropertySerializer interface
type PropertySerializer struct {
	Decode      DecodeFcn
	DecodeArray DecodeArrayFcn
	IsArray     bool
	Length      uint32
}

// Contains a list of available property serializers
type PropertySerializerTable struct {
	Serializers map[string]*PropertySerializer
}

// Returns a table containing all know property serializers
func GetDefaultPropertySerializerTable() *PropertySerializerTable {
	// Init table
	tbl := &PropertySerializerTable{}
	tbl.Serializers = make(map[string]*PropertySerializer)

	// Append default serializers/decoders
	// For now, only arrays are added

	tbl.Serializers["float32[24]"] = &PropertySerializer{
		nil, nil, true, 24,
	}

	tbl.Serializers["CStrongHandle< InfoForResourceTypeIMaterial2 >[6]"] = &PropertySerializer{
		nil, nil, true, 6,
	}

	return tbl
}

// Returns a serializer by name
func (pst *PropertySerializerTable) GetPropertySerializerByName(name string) *PropertySerializer {
	ser := pst.Serializers[name]

	if ser == nil {
		// This function should panic at some point
		return &PropertySerializer{
			nil, nil, false, 0,
		}
	}

	return ser
}
<|endoftext|>


<commit_before>// Package protobuf contains a codec to encode and decode entities in Protocol Buffer
package protobuf

import (
	"errors"

	"github.com/asdine/storm/codec/json"
	"github.com/golang/protobuf/proto"
)

const name = "protobuf"

// More details on Protocol Buffers https://github.com/golang/protobuf
var (
	Codec                       = new(protobufCodec)
	errNotProtocolBufferMessage = errors.New("value isn't a Protocol Buffers Message")
)

type protobufCodec int

// Encode value with protocol buffer.
// If type isn't a Protocol buffer Message, gob encoder will be used instead.
func (c protobufCodec) Marshal(v interface{}) ([]byte, error) {
	message, ok := v.(proto.Message)
	if !ok {
		// toBytes() may need to encode non-protobuf type, if that occurs use json
		return json.Codec.Marshal(v)
	}
	return proto.Marshal(message)
}

func (c protobufCodec) Unmarshal(b []byte, v interface{}) error {
	message, ok := v.(proto.Message)
	if !ok {
		// toBytes() may have encoded non-protobuf type, if that occurs use json
		return json.Codec.Unmarshal(b, v)
	}
	return proto.Unmarshal(b, message)
}

func (c protobufCodec) Name() string {
	return name
}
<commit_msg>fixed comment to reflect fallback is json <commit_after>// Package protobuf contains a codec to encode and decode entities in Protocol Buffer
package protobuf

import (
	"errors"

	"github.com/asdine/storm/codec/json"
	"github.com/golang/protobuf/proto"
)

const name = "protobuf"

// More details on Protocol Buffers https://github.com/golang/protobuf
var (
	Codec                       = new(protobufCodec)
	errNotProtocolBufferMessage = errors.New("value isn't a Protocol Buffers Message")
)

type protobufCodec int

// Encode value with protocol buffer.
// If type isn't a Protocol buffer Message, json encoder will be used instead.
func (c protobufCodec) Marshal(v interface{}) ([]byte, error) {
	message, ok := v.(proto.Message)
	if !ok {
		// toBytes() may need to encode non-protobuf type, if that occurs use json
		return json.Codec.Marshal(v)
	}
	return proto.Marshal(message)
}

func (c protobufCodec) Unmarshal(b []byte, v interface{}) error {
	message, ok := v.(proto.Message)
	if !ok {
		// toBytes() may have encoded non-protobuf type, if that occurs use json
		return json.Codec.Unmarshal(b, v)
	}
	return proto.Unmarshal(b, message)
}

func (c protobufCodec) Name() string {
	return name
}
<|endoftext|>


<commit_before>// Copyright 2010  The "go-linoise" Authors
//
// Use of this source code is governed by the Simplified BSD License
// that can be found in the LICENSE file.
//
// This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
// OR CONDITIONS OF ANY KIND, either express or implied. See the License
// for more details.

/* Important: linoise sets tty in 'raw mode' so there is to use CR+LF (\r\n) at
writing.
*/

package linoise

import (
	"bufio"
	"fmt"
	"os"
	"strings"

	"github.com/kless/go-term/term"
)

// Values by default for prompts.
var (
	PS1 = "linoise$ "
	PS2 = "> "
)

// Input / Output
var (
	input  *os.File = os.Stdin
	output *os.File = os.Stdout
)

// === Type
// ===

// Represents a line.
type Line struct {
	useHistory bool
	ps1Len     int      // Primary prompt size
	ps1        string   // Primary prompt
	ps2        string   // Command continuations
	*buffer             // Text buffer
	hist       *history // History file
}

// Gets a line type using the primary prompt by default. Sets the TTY raw mode.
func NewLine(hist *history) *Line {
	term.MakeRaw()

	buf := newBuffer(len(PS1))
	buf.insertRunes([]int(PS1))

	return &Line{
		hasHistory(hist),
		len(PS1),
		PS1,
		PS2,
		buf,
		hist,
	}
}

// Gets a line type using the given prompt as primary. Sets the TTY raw mode.
// 'ansiLen' is the length of ANSI codes that the prompt could have.
func NewLinePrompt(prompt string, ansiLen int, hist *history) *Line {
	term.MakeRaw()

	buf := newBuffer(len(prompt) - ansiLen)
	buf.insertRunes([]int(prompt))

	return &Line{
		hasHistory(hist),
		len(prompt) - ansiLen,
		prompt,
		PS2,
		buf,
		hist,
	}
}

// Restores terminal settings so it is disabled the raw mode.
func (ln *Line) RestoreTerm() {
	term.RestoreTerm()
}

// Tests if it has an history file.
func hasHistory(h *history) bool {
	if h == nil {
		return false
	}
	return true
}

// === Output
// ===

// Prints the primary prompt.
func (ln *Line) prompt() (err os.Error) {
	if lines, err = ln.end(); err != nil {
		return err
	}

	for lines > 0 {
		if _, err = output.Write(delLine_cursorUp); err != nil {
			return OutputError(err.String())
		}
		lines--
	}

	if _, err = output.Write(delLine_CR); err != nil {
		return OutputError(err.String())
	}
	if _, err = fmt.Fprint(output, ln.ps1); err != nil {
		return OutputError(err.String())
	}

	ln.pos, ln.size = ln.ps1Len, ln.ps1Len
	return
}

// === Get
// ===

// Reads charactes from input to write them to output, allowing line editing.
// The errors that could return are to indicate if Ctrl-D was pressed, and for
// both input / output errors.
func (ln *Line) Read() (line string, err os.Error) {
	var anotherLine []int  // For lines got from history.
	var isHistoryUsed bool // If the history has been accessed.

	in := bufio.NewReader(input) // Read input.
	seq := make([]byte, 2)       // For escape sequences.
	seq2 := make([]byte, 2)      // Extended escape sequences.

	for {
		rune, _, err := in.ReadRune()
		if err != nil {
			return "", InputError(err.String())
		}

		switch rune {
		default:
			if err = ln.insertRune(rune); err != nil {
				return "", err
			}
			continue

		case 13: // enter
			line = ln.toString()

			if ln.useHistory {
				ln.hist.Add(line)
			}

			if _, err = output.Write(_CR_LF); err != nil {
				return "", OutputError(err.String())
			}

			return strings.TrimSpace(line), nil

		case 127, 8: // backspace, Ctrl-h
			if err = ln.deletePrev(); err != nil {
				return "", err
			}
			continue

		case 9: // horizontal tab
			// TODO: disabled by now
			continue

		case 3: // Ctrl-c
			if err = ln.insertRunes(ctrlC); err != nil {
				return "", err
			}
			if _, err = output.Write(_CR_LF); err != nil {
				return "", OutputError(err.String())
			}
			if err = ln.prompt(); err != nil {
				return "", err
			}

			continue

		case 4: // Ctrl-d
			if err = ln.insertRunes(ctrlD); err != nil {
				return "", err
			}
			if _, err = output.Write(_CR_LF); err != nil {
				return "", OutputError(err.String())
			}

			return "", ErrCtrlD

		// Escape sequence
		case _ESC:
			if _, err = in.Read(seq); err != nil {
				return "", InputError(err.String())
			}
			//fmt.Print(" >", seq) //!!! For DEBUG

			if seq[0] == _L_BRACKET {
				switch seq[1] {
				case 68:
					goto _leftArrow
				case 67:
					goto _rightArrow
				case 65, 66: // Up, Down
					goto _upDownArrow
				}

				// Extended escape.
				if seq[1] > 48 && seq[1] < 55 {
					if _, err = in.Read(seq2); err != nil {
						return "", InputError(err.String())
					}
					//fmt.Print(" >>", seq2) //!!! For DEBUG

					// TODO: doesn't works
					if seq[1] == 51 && seq2[0] == 126 { // Delete
						if err = ln.delete(); err != nil {
							return "", err
						}
					}
				}
				continue
			}

			if seq[0] == 79 {
				switch seq[1] {
				case 72: // Home
					goto _start
				case 70: // End
					goto _end
				}
			}
			continue

		case 20: // Ctrl-t, swap actual character by the previous one.
			if err = ln.swap(); err != nil {
				return "", err
			}
			continue

		case 21: // Ctrl+u, delete the whole line.
			goto _deleteLine

		case 11: // Ctrl+k, delete from current to end of line.
			if err = ln.deleteRight(); err != nil {
				return "", err
			}
			continue

		case 1: // Ctrl+a, go to the start of the line.
			goto _start

		case 5: // Ctrl+e, go to the end of the line.
			goto _end

		case 2: // Ctrl-b
			goto _leftArrow

		case 6: // Ctrl-f
			goto _rightArrow

		case 16: // Ctrl-p
			seq[1] = 65
			goto _upDownArrow

		case 14: // Ctrl-n
			seq[1] = 66
			goto _upDownArrow
		}

	_upDownArrow: // Up and down arrow: history
		if !ln.useHistory {
			continue
		}

		// Up
		if seq[1] == 65 {
			anotherLine, err = ln.hist.Prev()
			// Down
		} else {
			anotherLine, err = ln.hist.Next()
		}
		if err != nil {
			continue
		}

		// Update the current history entry before to overwrite it with
		// the next one.
		// TODO: it has to be removed before of to be saved the history
		if !isHistoryUsed {
			ln.hist.Add(ln.toString())
		}
		isHistoryUsed = true

		ln.grow(len(anotherLine))
		ln.size = len(anotherLine)
		copy(ln.data[0:], anotherLine)

		if err = ln.refresh(); err != nil {
			return "", err
		}
		continue

	_leftArrow:
		if err = ln.backward(); err != nil {
			return "", err
		}
		continue

	_rightArrow:
		if err = ln.forward(); err != nil {
			return "", err
		}
		continue

	_start:
		if err = ln.start(); err != nil {
			return "", err
		}
		continue

	_end:
		if _, err = ln.end(); err != nil {
			return "", err
		}
		continue

	_deleteLine:
		if err = ln.prompt(); err != nil {
			return "", err
		}
		continue
	}
	return
}

<commit_msg>Use a field non-anonymous for *buffer in type Line<commit_after>// Copyright 2010  The "go-linoise" Authors
//
// Use of this source code is governed by the Simplified BSD License
// that can be found in the LICENSE file.
//
// This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
// OR CONDITIONS OF ANY KIND, either express or implied. See the License
// for more details.

/* Important: linoise sets tty in 'raw mode' so there is to use CR+LF (\r\n) at
writing.
*/

package linoise

import (
	"bufio"
	"fmt"
	"os"
	"strings"

	"github.com/kless/go-term/term"
)

// Values by default for prompts.
var (
	PS1 = "linoise$ "
	PS2 = "> "
)

// Input / Output
var (
	input  *os.File = os.Stdin
	output *os.File = os.Stdout
)

// === Type
// ===

// Represents a line.
type Line struct {
	useHistory bool
	ps1Len     int      // Primary prompt size
	ps1        string   // Primary prompt
	ps2        string   // Command continuations
	buf        *buffer  // Text buffer
	hist       *history // History file
}

// Gets a line type using the primary prompt by default. Sets the TTY raw mode.
func NewLine(hist *history) *Line {
	term.MakeRaw()

	buf := newBuffer(len(PS1))
	buf.insertRunes([]int(PS1))

	return &Line{
		hasHistory(hist),
		len(PS1),
		PS1,
		PS2,
		buf,
		hist,
	}
}

// Gets a line type using the given prompt as primary. Sets the TTY raw mode.
// 'ansiLen' is the length of ANSI codes that the prompt could have.
func NewLinePrompt(prompt string, ansiLen int, hist *history) *Line {
	term.MakeRaw()

	buf := newBuffer(len(prompt) - ansiLen)
	buf.insertRunes([]int(prompt))

	return &Line{
		hasHistory(hist),
		len(prompt) - ansiLen,
		prompt,
		PS2,
		buf,
		hist,
	}
}

// Restores terminal settings so it is disabled the raw mode.
func (ln *Line) RestoreTerm() {
	term.RestoreTerm()
}

// Tests if it has an history file.
func hasHistory(h *history) bool {
	if h == nil {
		return false
	}
	return true
}

// === Output
// ===

// Prints the primary prompt.
func (ln *Line) prompt() (err os.Error) {
	if lines, err = ln.buf.end(); err != nil {
		return err
	}

	for lines > 0 {
		if _, err = output.Write(delLine_cursorUp); err != nil {
			return OutputError(err.String())
		}
		lines--
	}

	if _, err = output.Write(delLine_CR); err != nil {
		return OutputError(err.String())
	}
	if _, err = fmt.Fprint(output, ln.ps1); err != nil {
		return OutputError(err.String())
	}

	ln.buf.pos, ln.buf.size = ln.ps1Len, ln.ps1Len
	return
}

// === Get
// ===

// Reads charactes from input to write them to output, allowing line editing.
// The errors that could return are to indicate if Ctrl-D was pressed, and for
// both input / output errors.
func (ln *Line) Read() (line string, err os.Error) {
	var anotherLine []int  // For lines got from history.
	var isHistoryUsed bool // If the history has been accessed.

	in := bufio.NewReader(input) // Read input.
	seq := make([]byte, 2)       // For escape sequences.
	seq2 := make([]byte, 2)      // Extended escape sequences.

	for {
		rune, _, err := in.ReadRune()
		if err != nil {
			return "", InputError(err.String())
		}

		switch rune {
		default:
			if err = ln.buf.insertRune(rune); err != nil {
				return "", err
			}
			continue

		case 13: // enter
			line = ln.buf.toString()

			if ln.useHistory {
				ln.hist.Add(line)
			}

			if _, err = output.Write(_CR_LF); err != nil {
				return "", OutputError(err.String())
			}

			return strings.TrimSpace(line), nil

		case 127, 8: // backspace, Ctrl-h
			if err = ln.buf.deletePrev(); err != nil {
				return "", err
			}
			continue

		case 9: // horizontal tab
			// TODO: disabled by now
			continue

		case 3: // Ctrl-c
			if err = ln.buf.insertRunes(ctrlC); err != nil {
				return "", err
			}
			if _, err = output.Write(_CR_LF); err != nil {
				return "", OutputError(err.String())
			}
			if err = ln.prompt(); err != nil {
				return "", err
			}

			continue

		case 4: // Ctrl-d
			if err = ln.buf.insertRunes(ctrlD); err != nil {
				return "", err
			}
			if _, err = output.Write(_CR_LF); err != nil {
				return "", OutputError(err.String())
			}

			return "", ErrCtrlD

		// Escape sequence
		case _ESC:
			if _, err = in.Read(seq); err != nil {
				return "", InputError(err.String())
			}
			//fmt.Print(" >", seq) //!!! For DEBUG

			if seq[0] == _L_BRACKET {
				switch seq[1] {
				case 68:
					goto _leftArrow
				case 67:
					goto _rightArrow
				case 65, 66: // Up, Down
					goto _upDownArrow
				}

				// Extended escape.
				if seq[1] > 48 && seq[1] < 55 {
					if _, err = in.Read(seq2); err != nil {
						return "", InputError(err.String())
					}
					//fmt.Print(" >>", seq2) //!!! For DEBUG

					// TODO: doesn't works
					if seq[1] == 51 && seq2[0] == 126 { // Delete
						if err = ln.buf.delete(); err != nil {
							return "", err
						}
					}
				}
				continue
			}

			if seq[0] == 79 {
				switch seq[1] {
				case 72: // Home
					goto _start
				case 70: // End
					goto _end
				}
			}
			continue

		case 20: // Ctrl-t, swap actual character by the previous one.
			if err = ln.buf.swap(); err != nil {
				return "", err
			}
			continue

		case 21: // Ctrl+u, delete the whole line.
			goto _deleteLine

		case 11: // Ctrl+k, delete from current to end of line.
			if err = ln.buf.deleteRight(); err != nil {
				return "", err
			}
			continue

		case 1: // Ctrl+a, go to the start of the line.
			goto _start

		case 5: // Ctrl+e, go to the end of the line.
			goto _end

		case 2: // Ctrl-b
			goto _leftArrow

		case 6: // Ctrl-f
			goto _rightArrow

		case 16: // Ctrl-p
			seq[1] = 65
			goto _upDownArrow

		case 14: // Ctrl-n
			seq[1] = 66
			goto _upDownArrow
		}

	_upDownArrow: // Up and down arrow: history
		if !ln.useHistory {
			continue
		}

		// Up
		if seq[1] == 65 {
			anotherLine, err = ln.hist.Prev()
			// Down
		} else {
			anotherLine, err = ln.hist.Next()
		}
		if err != nil {
			continue
		}

		// Update the current history entry before to overwrite it with
		// the next one.
		// TODO: it has to be removed before of to be saved the history
		if !isHistoryUsed {
			ln.hist.Add(ln.buf.toString())
		}
		isHistoryUsed = true

		ln.buf.grow(len(anotherLine))
		ln.buf.size = len(anotherLine)
		copy(ln.buf.data[0:], anotherLine)

		if err = ln.buf.refresh(); err != nil {
			return "", err
		}
		continue

	_leftArrow:
		if err = ln.buf.backward(); err != nil {
			return "", err
		}
		continue

	_rightArrow:
		if err = ln.buf.forward(); err != nil {
			return "", err
		}
		continue

	_start:
		if err = ln.buf.start(); err != nil {
			return "", err
		}
		continue

	_end:
		if _, err = ln.buf.end(); err != nil {
			return "", err
		}
		continue

	_deleteLine:
		if err = ln.prompt(); err != nil {
			return "", err
		}
		continue
	}
	return
}

<|endoftext|>


<commit_before>// Copyright 2015 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build !nosystemd

package collector

import (
	"fmt"

	"github.com/coreos/go-systemd/dbus"
	"github.com/prometheus/client_golang/prometheus"
)

type systemdCollector struct {
	unitDesc          *prometheus.Desc
	systemRunningDesc *prometheus.Desc
}

var unitStatesName = []string{"active", "activating", "deactivating", "inactive", "failed"}

func init() {
	Factories["systemd"] = NewSystemdCollector
}

// Takes a prometheus registry and returns a new Collector exposing
// systemd statistics.
func NewSystemdCollector() (Collector, error) {
	const subsystem = "systemd"

	unitDesc := prometheus.NewDesc(
		prometheus.BuildFQName(Namespace, subsystem, "unit_state"),
		"Systemd unit", []string{"name", "state"}, nil,
	)
	systemRunningDesc := prometheus.NewDesc(
		prometheus.BuildFQName(Namespace, subsystem, "system_running"),
		"Whether the system is operational (see 'systemctl is-system-running')",
		nil, nil,
	)

	return &systemdCollector{
		unitDesc:          unitDesc,
		systemRunningDesc: systemRunningDesc,
	}, nil
}

func (c *systemdCollector) Update(ch chan<- prometheus.Metric) (err error) {
	units, err := c.listUnits()
	if err != nil {
		return fmt.Errorf("couldn't get units states: %s", err)
	}
	c.collectUnitStatusMetrics(ch, units)

	systemState, err := c.getSystemState()
	if err != nil {
		return fmt.Errorf("couldn't get system state: %s", err)
	}
	c.collectSystemState(ch, systemState)

	return nil
}

func (c *systemdCollector) collectUnitStatusMetrics(ch chan<- prometheus.Metric, units []dbus.UnitStatus) {
	for _, unit := range units {
		for _, stateName := range unitStatesName {
			isActive := 0.0
			if stateName == unit.ActiveState {
				isActive = 1.0
			}
			ch <- prometheus.MustNewConstMetric(
				c.unitDesc, prometheus.GaugeValue, isActive,
				unit.Name, stateName)
		}
	}
}

func (c *systemdCollector) collectSystemState(ch chan<- prometheus.Metric, systemState string) {
	isSystemRunning := 0.0
	if systemState == `"running"` {
		isSystemRunning = 1.0
	}
	ch <- prometheus.MustNewConstMetric(c.systemRunningDesc, prometheus.GaugeValue, isSystemRunning)
}

func (c *systemdCollector) listUnits() ([]dbus.UnitStatus, error) {
	conn, err := dbus.New()
	if err != nil {
		return nil, fmt.Errorf("couldn't get dbus connection: %s", err)
	}
	units, err := conn.ListUnits()
	conn.Close()
	return units, err
}

func (c *systemdCollector) getSystemState() (state string, err error) {
	conn, err := dbus.New()
	if err != nil {
		return "", fmt.Errorf("couldn't get dbus connection: %s", err)
	}
	state, err = conn.GetManagerProperty("SystemState")
	conn.Close()
	return state, err
}
<commit_msg>systemd-collector: support private/direct connections without dbus<commit_after>// Copyright 2015 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build !nosystemd

package collector

import (
	"flag"
	"fmt"

	"github.com/coreos/go-systemd/dbus"
	"github.com/prometheus/client_golang/prometheus"
)

type systemdCollector struct {
	unitDesc          *prometheus.Desc
	systemRunningDesc *prometheus.Desc
}

var unitStatesName = []string{"active", "activating", "deactivating", "inactive", "failed"}

var (
	systemdPrivate = flag.Bool(
		"collector.systemd.private",
		false,
		"Establish a private, direct connection to systemd without dbus.",
	)
)

func init() {
	Factories["systemd"] = NewSystemdCollector
}

// Takes a prometheus registry and returns a new Collector exposing
// systemd statistics.
func NewSystemdCollector() (Collector, error) {
	const subsystem = "systemd"

	unitDesc := prometheus.NewDesc(
		prometheus.BuildFQName(Namespace, subsystem, "unit_state"),
		"Systemd unit", []string{"name", "state"}, nil,
	)
	systemRunningDesc := prometheus.NewDesc(
		prometheus.BuildFQName(Namespace, subsystem, "system_running"),
		"Whether the system is operational (see 'systemctl is-system-running')",
		nil, nil,
	)

	return &systemdCollector{
		unitDesc:          unitDesc,
		systemRunningDesc: systemRunningDesc,
	}, nil
}

func (c *systemdCollector) Update(ch chan<- prometheus.Metric) (err error) {
	units, err := c.listUnits()
	if err != nil {
		return fmt.Errorf("couldn't get units states: %s", err)
	}
	c.collectUnitStatusMetrics(ch, units)

	systemState, err := c.getSystemState()
	if err != nil {
		return fmt.Errorf("couldn't get system state: %s", err)
	}
	c.collectSystemState(ch, systemState)

	return nil
}

func (c *systemdCollector) collectUnitStatusMetrics(ch chan<- prometheus.Metric, units []dbus.UnitStatus) {
	for _, unit := range units {
		for _, stateName := range unitStatesName {
			isActive := 0.0
			if stateName == unit.ActiveState {
				isActive = 1.0
			}
			ch <- prometheus.MustNewConstMetric(
				c.unitDesc, prometheus.GaugeValue, isActive,
				unit.Name, stateName)
		}
	}
}

func (c *systemdCollector) collectSystemState(ch chan<- prometheus.Metric, systemState string) {
	isSystemRunning := 0.0
	if systemState == `"running"` {
		isSystemRunning = 1.0
	}
	ch <- prometheus.MustNewConstMetric(c.systemRunningDesc, prometheus.GaugeValue, isSystemRunning)
}

func (c *systemdCollector) newDbus() (*dbus.Conn, error) {
	if *systemdPrivate {
		return dbus.NewSystemdConnection()
	}
	return dbus.New()
}

func (c *systemdCollector) listUnits() ([]dbus.UnitStatus, error) {
	conn, err := c.newDbus()
	if err != nil {
		return nil, fmt.Errorf("couldn't get dbus connection: %s", err)
	}
	units, err := conn.ListUnits()
	conn.Close()
	return units, err
}

func (c *systemdCollector) getSystemState() (state string, err error) {
	conn, err := c.newDbus()
	if err != nil {
		return "", fmt.Errorf("couldn't get dbus connection: %s", err)
	}
	state, err = conn.GetManagerProperty("SystemState")
	conn.Close()
	return state, err
}
<|endoftext|>


<commit_before>// +build OMIT
package main

import (
	"fmt"

	"veyron.io/veyron/veyron/lib/signals"
	_ "veyron.io/veyron/veyron/profiles"
	sflag "veyron.io/veyron/veyron/security/flag"
	"veyron.io/veyron/veyron2/ipc"
	"veyron.io/veyron/veyron2/rt"

	"pingpong"
)

type pongd struct{}

func (f *pongd) Ping(_ ipc.ServerContext, message string) (result string, err error) {
	fmt.Println(message)
	return "PONG", nil
}

func main() {
	r := rt.Init()
	log := r.Logger()
	s, err := r.NewServer()
	if err != nil {
		log.Fatal("failure creating server: ", err)
	}
	log.Info("Waiting for ping")

	serverPong := pingpong.NewServerPingPong(&pongd{})

	if endpoint, err := s.Listen("tcp", "127.0.0.1:0"); err == nil {
		fmt.Printf("Listening at: %v\n", endpoint)
	} else {
		log.Fatal("error listening to service: ", err)
	}

	if err := s.Serve("pingpong", ipc.LeafDispatcher(serverPong, sflag.NewAuthorizerOrDie())); err != nil {
		log.Fatal("error serving service: ", err)
	}

	// Wait forever.
	<-signals.ShutdownOnSignals()
}
<commit_msg>pong/pong.go: Fix Listen() argument.<commit_after>// +build OMIT
package main

import (
	"fmt"

	"veyron.io/veyron/veyron/lib/signals"
	"veyron.io/veyron/veyron/profiles"
	sflag "veyron.io/veyron/veyron/security/flag"
	"veyron.io/veyron/veyron2/ipc"
	"veyron.io/veyron/veyron2/rt"

	"pingpong"
)

type pongd struct{}

func (f *pongd) Ping(_ ipc.ServerContext, message string) (result string, err error) {
	fmt.Println(message)
	return "PONG", nil
}

func main() {
	r := rt.Init()
	log := r.Logger()
	s, err := r.NewServer()
	if err != nil {
		log.Fatal("failure creating server: ", err)
	}
	log.Info("Waiting for ping")

	serverPong := pingpong.NewServerPingPong(&pongd{})

	if endpoint, err := s.Listen(profiles.LocalListenSpec); err == nil {
		fmt.Printf("Listening at: %v\n", endpoint)
	} else {
		log.Fatal("error listening to service: ", err)
	}

	if err := s.Serve("pingpong", ipc.LeafDispatcher(serverPong, sflag.NewAuthorizerOrDie())); err != nil {
		log.Fatal("error serving service: ", err)
	}

	// Wait forever.
	<-signals.ShutdownOnSignals()
}
<|endoftext|>


<commit_before>package views

import (
	"net/http"
	"time"

	"github.com/antihax/evedata/services/vanguard"
	"github.com/antihax/evedata/services/vanguard/models"
)

func init() {
	vanguard.AddRoute("GET", "/locatorResponses",
		func(w http.ResponseWriter, r *http.Request) {
			renderTemplate(w,
				"locatorResponses.html",
				time.Hour*24*31,
				newPage(r, "Locator Responses"))
		})
	vanguard.AddAuthRoute("GET", "/U/locatorResponses", apiGetLocatorResponses)
}

func apiGetLocatorResponses(w http.ResponseWriter, r *http.Request) {
	s := vanguard.SessionFromContext(r.Context())

	// Get the sessions main characterID
	characterID, ok := s.Values["characterID"].(int32)
	if !ok {
		httpErrCode(w, nil, http.StatusUnauthorized)
		return
	}

	v, err := models.GetLocatorResponses(characterID)
	if err != nil {
		httpErr(w, err)
		return
	}

	renderJSON(w, v, 0)

	if err = s.Save(r, w); err != nil {
		httpErr(w, err)
		return
	}
}
<commit_msg>cache for a short while<commit_after>package views

import (
	"net/http"
	"time"

	"github.com/antihax/evedata/services/vanguard"
	"github.com/antihax/evedata/services/vanguard/models"
)

func init() {
	vanguard.AddRoute("GET", "/locatorResponses",
		func(w http.ResponseWriter, r *http.Request) {
			renderTemplate(w,
				"locatorResponses.html",
				time.Hour*24*31,
				newPage(r, "Locator Responses"))
		})
	vanguard.AddAuthRoute("GET", "/U/locatorResponses", apiGetLocatorResponses)
}

func apiGetLocatorResponses(w http.ResponseWriter, r *http.Request) {
	s := vanguard.SessionFromContext(r.Context())

	// Get the sessions main characterID
	characterID, ok := s.Values["characterID"].(int32)
	if !ok {
		httpErrCode(w, nil, http.StatusUnauthorized)
		return
	}

	v, err := models.GetLocatorResponses(characterID)
	if err != nil {
		httpErr(w, err)
		return
	}

	renderJSON(w, v, 5)

	if err = s.Save(r, w); err != nil {
		httpErr(w, err)
		return
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package binloginfo

import (
	"context"
	"regexp"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/pingcap/errors"
	"github.com/pingcap/parser/terror"
	"github.com/pingcap/tidb-tools/tidb-binlog/node"
	pumpcli "github.com/pingcap/tidb-tools/tidb-binlog/pump_client"
	"github.com/pingcap/tidb/kv"
	"github.com/pingcap/tidb/metrics"
	"github.com/pingcap/tidb/sessionctx"
	"github.com/pingcap/tidb/util/logutil"
	binlog "github.com/pingcap/tipb/go-binlog"
	"go.uber.org/zap"
	"google.golang.org/grpc"
)

func init() {
	grpc.EnableTracing = false
}

// pumpsClient is the client to write binlog, it is opened on server start and never close,
// shared by all sessions.
var pumpsClient *pumpcli.PumpsClient
var pumpsClientLock sync.RWMutex

// BinlogInfo contains binlog data and binlog client.
type BinlogInfo struct {
	Data   *binlog.Binlog
	Client *pumpcli.PumpsClient
}

// GetPumpsClient gets the pumps client instance.
func GetPumpsClient() *pumpcli.PumpsClient {
	pumpsClientLock.RLock()
	client := pumpsClient
	pumpsClientLock.RUnlock()
	return client
}

// SetPumpsClient sets the pumps client instance.
func SetPumpsClient(client *pumpcli.PumpsClient) {
	pumpsClientLock.Lock()
	pumpsClient = client
	pumpsClientLock.Unlock()
}

// GetPrewriteValue gets binlog prewrite value in the context.
func GetPrewriteValue(ctx sessionctx.Context, createIfNotExists bool) *binlog.PrewriteValue {
	vars := ctx.GetSessionVars()
	v, ok := vars.TxnCtx.Binlog.(*binlog.PrewriteValue)
	if !ok && createIfNotExists {
		schemaVer := ctx.GetSessionVars().TxnCtx.SchemaVersion
		v = &binlog.PrewriteValue{SchemaVersion: schemaVer}
		vars.TxnCtx.Binlog = v
	}
	return v
}

var skipBinlog uint32
var ignoreError uint32

// DisableSkipBinlogFlag disable the skipBinlog flag.
func DisableSkipBinlogFlag() {
	atomic.StoreUint32(&skipBinlog, 0)
	logutil.Logger(context.Background()).Warn("[binloginfo] disable the skipBinlog flag")
}

// SetIgnoreError sets the ignoreError flag, this function called when TiDB start
// up and find config.Binlog.IgnoreError is true.
func SetIgnoreError(on bool) {
	if on {
		atomic.StoreUint32(&ignoreError, 1)
	} else {
		atomic.StoreUint32(&ignoreError, 0)
	}
}

// WriteBinlog writes a binlog to Pump.
func (info *BinlogInfo) WriteBinlog(clusterID uint64) error {
	skip := atomic.LoadUint32(&skipBinlog)
	if skip > 0 {
		metrics.CriticalErrorCounter.Add(1)
		return nil
	}

	if info.Client == nil {
		return errors.New("pumps client is nil")
	}

	// it will retry in PumpsClient if write binlog fail.
	err := info.Client.WriteBinlog(info.Data)
	if err != nil {
		logutil.Logger(context.Background()).Error("write binlog failed", zap.Error(err))
		if atomic.LoadUint32(&ignoreError) == 1 {
			logutil.Logger(context.Background()).Error("write binlog fail but error ignored")
			metrics.CriticalErrorCounter.Add(1)
			// If error happens once, we'll stop writing binlog.
			atomic.CompareAndSwapUint32(&skipBinlog, skip, skip+1)
			return nil
		}

		if strings.Contains(err.Error(), "received message larger than max") {
			// This kind of error is not critical, return directly.
			return errors.Errorf("binlog data is too large (%s)", err.Error())
		}

		return terror.ErrCritical.GenWithStackByArgs(err)
	}

	return nil
}

// SetDDLBinlog sets DDL binlog in the kv.Transaction.
func SetDDLBinlog(client *pumpcli.PumpsClient, txn kv.Transaction, jobID int64, ddlQuery string) {
	if client == nil {
		return
	}

	ddlQuery = addSpecialComment(ddlQuery)
	info := &BinlogInfo{
		Data: &binlog.Binlog{
			Tp:       binlog.BinlogType_Prewrite,
			DdlJobId: jobID,
			DdlQuery: []byte(ddlQuery),
		},
		Client: client,
	}
	txn.SetOption(kv.BinlogInfo, info)
}

const specialPrefix = `/*!90000 `

func addSpecialComment(ddlQuery string) string {
	if strings.Contains(ddlQuery, specialPrefix) {
		return ddlQuery
	}
	upperQuery := strings.ToUpper(ddlQuery)
	reg, err := regexp.Compile(`SHARD_ROW_ID_BITS\s*=\s*\d+`)
	terror.Log(err)
	loc := reg.FindStringIndex(upperQuery)
	if len(loc) < 2 {
		return ddlQuery
	}
	return ddlQuery[:loc[0]] + specialPrefix + ddlQuery[loc[0]:loc[1]] + ` */` + ddlQuery[loc[1]:]
}

// MockPumpsClient creates a PumpsClient, used for test.
func MockPumpsClient(client binlog.PumpClient) *pumpcli.PumpsClient {
	nodeID := "pump-1"
	pump := &pumpcli.PumpStatus{
		Status: node.Status{
			NodeID: nodeID,
			State:  node.Online,
		},
		Client: client,
	}

	pumpInfos := &pumpcli.PumpInfos{
		Pumps:            make(map[string]*pumpcli.PumpStatus),
		AvaliablePumps:   make(map[string]*pumpcli.PumpStatus),
		UnAvaliablePumps: make(map[string]*pumpcli.PumpStatus),
	}
	pumpInfos.Pumps[nodeID] = pump
	pumpInfos.AvaliablePumps[nodeID] = pump

	pCli := &pumpcli.PumpsClient{
		ClusterID:          1,
		Pumps:              pumpInfos,
		Selector:           pumpcli.NewSelector(pumpcli.Range),
		BinlogWriteTimeout: time.Second,
	}
	pCli.Selector.SetPumps([]*pumpcli.PumpStatus{pump})

	return pCli
}
<commit_msg>binloginfo: Optimize addSpecialComment by reusing compiled regexp (#10502)<commit_after>// Copyright 2016 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package binloginfo

import (
	"context"
	"regexp"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/pingcap/errors"
	"github.com/pingcap/parser/terror"
	"github.com/pingcap/tidb-tools/tidb-binlog/node"
	pumpcli "github.com/pingcap/tidb-tools/tidb-binlog/pump_client"
	"github.com/pingcap/tidb/kv"
	"github.com/pingcap/tidb/metrics"
	"github.com/pingcap/tidb/sessionctx"
	"github.com/pingcap/tidb/util/logutil"
	binlog "github.com/pingcap/tipb/go-binlog"
	"go.uber.org/zap"
	"google.golang.org/grpc"
)

func init() {
	grpc.EnableTracing = false
}

// pumpsClient is the client to write binlog, it is opened on server start and never close,
// shared by all sessions.
var pumpsClient *pumpcli.PumpsClient
var pumpsClientLock sync.RWMutex
var shardPat = regexp.MustCompile(`SHARD_ROW_ID_BITS\s*=\s*\d+`)

// BinlogInfo contains binlog data and binlog client.
type BinlogInfo struct {
	Data   *binlog.Binlog
	Client *pumpcli.PumpsClient
}

// GetPumpsClient gets the pumps client instance.
func GetPumpsClient() *pumpcli.PumpsClient {
	pumpsClientLock.RLock()
	client := pumpsClient
	pumpsClientLock.RUnlock()
	return client
}

// SetPumpsClient sets the pumps client instance.
func SetPumpsClient(client *pumpcli.PumpsClient) {
	pumpsClientLock.Lock()
	pumpsClient = client
	pumpsClientLock.Unlock()
}

// GetPrewriteValue gets binlog prewrite value in the context.
func GetPrewriteValue(ctx sessionctx.Context, createIfNotExists bool) *binlog.PrewriteValue {
	vars := ctx.GetSessionVars()
	v, ok := vars.TxnCtx.Binlog.(*binlog.PrewriteValue)
	if !ok && createIfNotExists {
		schemaVer := ctx.GetSessionVars().TxnCtx.SchemaVersion
		v = &binlog.PrewriteValue{SchemaVersion: schemaVer}
		vars.TxnCtx.Binlog = v
	}
	return v
}

var skipBinlog uint32
var ignoreError uint32

// DisableSkipBinlogFlag disable the skipBinlog flag.
func DisableSkipBinlogFlag() {
	atomic.StoreUint32(&skipBinlog, 0)
	logutil.Logger(context.Background()).Warn("[binloginfo] disable the skipBinlog flag")
}

// SetIgnoreError sets the ignoreError flag, this function called when TiDB start
// up and find config.Binlog.IgnoreError is true.
func SetIgnoreError(on bool) {
	if on {
		atomic.StoreUint32(&ignoreError, 1)
	} else {
		atomic.StoreUint32(&ignoreError, 0)
	}
}

// WriteBinlog writes a binlog to Pump.
func (info *BinlogInfo) WriteBinlog(clusterID uint64) error {
	skip := atomic.LoadUint32(&skipBinlog)
	if skip > 0 {
		metrics.CriticalErrorCounter.Add(1)
		return nil
	}

	if info.Client == nil {
		return errors.New("pumps client is nil")
	}

	// it will retry in PumpsClient if write binlog fail.
	err := info.Client.WriteBinlog(info.Data)
	if err != nil {
		logutil.Logger(context.Background()).Error("write binlog failed", zap.Error(err))
		if atomic.LoadUint32(&ignoreError) == 1 {
			logutil.Logger(context.Background()).Error("write binlog fail but error ignored")
			metrics.CriticalErrorCounter.Add(1)
			// If error happens once, we'll stop writing binlog.
			atomic.CompareAndSwapUint32(&skipBinlog, skip, skip+1)
			return nil
		}

		if strings.Contains(err.Error(), "received message larger than max") {
			// This kind of error is not critical, return directly.
			return errors.Errorf("binlog data is too large (%s)", err.Error())
		}

		return terror.ErrCritical.GenWithStackByArgs(err)
	}

	return nil
}

// SetDDLBinlog sets DDL binlog in the kv.Transaction.
func SetDDLBinlog(client *pumpcli.PumpsClient, txn kv.Transaction, jobID int64, ddlQuery string) {
	if client == nil {
		return
	}

	ddlQuery = addSpecialComment(ddlQuery)
	info := &BinlogInfo{
		Data: &binlog.Binlog{
			Tp:       binlog.BinlogType_Prewrite,
			DdlJobId: jobID,
			DdlQuery: []byte(ddlQuery),
		},
		Client: client,
	}
	txn.SetOption(kv.BinlogInfo, info)
}

const specialPrefix = `/*!90000 `

func addSpecialComment(ddlQuery string) string {
	if strings.Contains(ddlQuery, specialPrefix) {
		return ddlQuery
	}
	loc := shardPat.FindStringIndex(strings.ToUpper(ddlQuery))
	if loc == nil {
		return ddlQuery
	}
	return ddlQuery[:loc[0]] + specialPrefix + ddlQuery[loc[0]:loc[1]] + ` */` + ddlQuery[loc[1]:]
}

// MockPumpsClient creates a PumpsClient, used for test.
func MockPumpsClient(client binlog.PumpClient) *pumpcli.PumpsClient {
	nodeID := "pump-1"
	pump := &pumpcli.PumpStatus{
		Status: node.Status{
			NodeID: nodeID,
			State:  node.Online,
		},
		Client: client,
	}

	pumpInfos := &pumpcli.PumpInfos{
		Pumps:            make(map[string]*pumpcli.PumpStatus),
		AvaliablePumps:   make(map[string]*pumpcli.PumpStatus),
		UnAvaliablePumps: make(map[string]*pumpcli.PumpStatus),
	}
	pumpInfos.Pumps[nodeID] = pump
	pumpInfos.AvaliablePumps[nodeID] = pump

	pCli := &pumpcli.PumpsClient{
		ClusterID:          1,
		Pumps:              pumpInfos,
		Selector:           pumpcli.NewSelector(pumpcli.Range),
		BinlogWriteTimeout: time.Second,
	}
	pCli.Selector.SetPumps([]*pumpcli.PumpStatus{pump})

	return pCli
}
<|endoftext|>


<commit_before>package proxyquerier

import (
	"context"
	"time"

	"github.com/jacksontj/promxy/config"
	"github.com/jacksontj/promxy/promclient"
	"github.com/jacksontj/promxy/servergroup"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/prometheus/pkg/labels"
	"github.com/prometheus/prometheus/storage"
	"github.com/sirupsen/logrus"
)

var (
	proxyQuerierSummary = prometheus.NewSummaryVec(prometheus.SummaryOpts{
		Name: "proxy_querier_request",
		Help: "Summary of proxyquerier calls to downstreams",
	}, []string{"host", "call", "status"})
)

func init() {
	prometheus.MustRegister(proxyQuerierSummary)
}

type ProxyQuerier struct {
	Ctx          context.Context
	Start        time.Time
	End          time.Time
	ServerGroups servergroup.ServerGroups

	Cfg *proxyconfig.PromxyConfig
}

// Select returns a set of series that matches the given label matchers.
func (h *ProxyQuerier) Select(selectParams *storage.SelectParams, matchers ...*labels.Matcher) (storage.SeriesSet, error) {
	start := time.Now()
	defer func() {
		logrus.WithFields(logrus.Fields{
			"selectParams": selectParams,
			"matchers":     matchers,
			"took":         time.Now().Sub(start),
		}).Debug("Select")
	}()

	result, err := h.ServerGroups.GetValue(h.Ctx, h.Start, h.End, matchers)
	if err != nil {
		return nil, err
	}

	iterators := promclient.IteratorsForValue(result)

	series := make([]storage.Series, len(iterators))
	for i, iterator := range iterators {
		series[i] = &Series{iterator}
	}

	return NewSeriesSet(series), nil
}

// LabelValues returns all potential values for a label name.
func (h *ProxyQuerier) LabelValues(name string) ([]string, error) {
	start := time.Now()
	defer func() {
		logrus.WithFields(logrus.Fields{
			"name": name,
			"took": time.Now().Sub(start),
		}).Debug("LabelValues")
	}()

	result, err := h.ServerGroups.GetValuesForLabelName(h.Ctx, "/api/v1/label/"+string(name)+"/values")
	if err != nil {
		return nil, err
	}

	ret := make([]string, len(result.Data))
	for i, r := range result.Data {
		ret[i] = string(r)
	}

	return ret, nil
}

// Close closes the querier. Behavior for subsequent calls to Querier methods
// is undefined.
func (h *ProxyQuerier) Close() error { return nil }
<commit_msg>Use new offset selectParam<commit_after>package proxyquerier

import (
	"context"
	"time"

	"github.com/jacksontj/promxy/config"
	"github.com/jacksontj/promxy/promclient"
	"github.com/jacksontj/promxy/servergroup"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/prometheus/pkg/labels"
	"github.com/prometheus/prometheus/storage"
	"github.com/sirupsen/logrus"
)

var (
	proxyQuerierSummary = prometheus.NewSummaryVec(prometheus.SummaryOpts{
		Name: "proxy_querier_request",
		Help: "Summary of proxyquerier calls to downstreams",
	}, []string{"host", "call", "status"})
)

func init() {
	prometheus.MustRegister(proxyQuerierSummary)
}

type ProxyQuerier struct {
	Ctx          context.Context
	Start        time.Time
	End          time.Time
	ServerGroups servergroup.ServerGroups

	Cfg *proxyconfig.PromxyConfig
}

// Select returns a set of series that matches the given label matchers.
func (h *ProxyQuerier) Select(selectParams *storage.SelectParams, matchers ...*labels.Matcher) (storage.SeriesSet, error) {
	start := time.Now()
	defer func() {
		logrus.WithFields(logrus.Fields{
			"selectParams": selectParams,
			"matchers":     matchers,
			"took":         time.Now().Sub(start),
		}).Debug("Select")
	}()

	end := h.End
	if selectParams.Offset > 0 {
		end = end.Add(time.Duration(-selectParams.Offset) * time.Millisecond)
	}

	result, err := h.ServerGroups.GetValue(h.Ctx, h.Start, end, matchers)
	if err != nil {
		return nil, err
	}

	iterators := promclient.IteratorsForValue(result)

	series := make([]storage.Series, len(iterators))
	for i, iterator := range iterators {
		series[i] = &Series{iterator}
	}

	return NewSeriesSet(series), nil
}

// LabelValues returns all potential values for a label name.
func (h *ProxyQuerier) LabelValues(name string) ([]string, error) {
	start := time.Now()
	defer func() {
		logrus.WithFields(logrus.Fields{
			"name": name,
			"took": time.Now().Sub(start),
		}).Debug("LabelValues")
	}()

	result, err := h.ServerGroups.GetValuesForLabelName(h.Ctx, "/api/v1/label/"+string(name)+"/values")
	if err != nil {
		return nil, err
	}

	ret := make([]string, len(result.Data))
	for i, r := range result.Data {
		ret[i] = string(r)
	}

	return ret, nil
}

// Close closes the querier. Behavior for subsequent calls to Querier methods
// is undefined.
func (h *ProxyQuerier) Close() error { return nil }
<|endoftext|>


<commit_before>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package testing

import (
	"bufio"
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	stdtesting "testing"
	"time"

	"labix.org/v2/mgo"
	gc "launchpad.net/gocheck"

	"launchpad.net/juju-core/cert"
	"launchpad.net/juju-core/log"
	"launchpad.net/juju-core/utils"
)

var (
	// MgoServer is a shared mongo server used by tests.
	MgoServer = &MgoInstance{}
)

type MgoInstance struct {
	// Addr holds the address of the shared MongoDB server set up by
	// MgoTestPackage.
	Addr string

	// MgoPort holds the port used by the shared MongoDB server.
	Port int

	// Server holds the running MongoDB command.
	Server *exec.Cmd

	// Exited receives a value when the mongodb server exits.
	Exited <-chan struct{}

	// Dir holds the directory that MongoDB is running in.
	Dir string

	// params is a list of additional parameters that will be passed to
	// the mongod application
	Params []string
}

// We specify a timeout to mgo.Dial, to prevent
// mongod failures hanging the tests.
const mgoDialTimeout = 15 * time.Second

// MgoSuite is a suite that deletes all content from the shared MongoDB
// server at the end of every test and supplies a connection to the shared
// MongoDB server.
type MgoSuite struct {
	Session *mgo.Session
}

// startMgoServer starts a MongoDB server in a temporary directory.
func (inst *MgoInstance) Start() error {
	dbdir, err := ioutil.TempDir("", "test-mgo")
	if err != nil {
		return err
	}
	pemPath := filepath.Join(dbdir, "server.pem")
	err = ioutil.WriteFile(pemPath, []byte(ServerCert+ServerKey), 0600)
	if err != nil {
		return fmt.Errorf("cannot write cert/key PEM: %v", err)
	}
	inst.Port = FindTCPPort()
	inst.Addr = fmt.Sprintf("localhost:%d", inst.Port)
	inst.Dir = dbdir
	if err := inst.runMgoServer(); err != nil {
		inst.Addr = ""
		inst.Port = 0
		os.RemoveAll(inst.Dir)
		inst.Dir = ""
		return err
	}

	// wait until it's running
	deadline := time.Now().Add(time.Second * 10)
	for {
		err := inst.ping()
		if err == nil || time.Now().After(deadline) {
			return err
		}
	}
}

func (inst *MgoInstance) ping() error {
	session := inst.MgoDialDirect()
	defer session.Close()
	return session.Ping()
}

// runMgoServer runs the MongoDB server at the
// address and directory already configured.
func (inst *MgoInstance) runMgoServer() error {
	if inst.Server != nil {
		panic("mongo server is already running")
	}
	mgoport := strconv.Itoa(inst.Port)
	mgoargs := []string{
		"--auth",
		"--dbpath", inst.Dir,
		"--sslOnNormalPorts",
		"--sslPEMKeyFile", filepath.Join(inst.Dir, "server.pem"),
		"--sslPEMKeyPassword", "ignored",
		"--bind_ip", "localhost",
		"--port", mgoport,
		"--nssize", "1",
		"--noprealloc",
		"--smallfiles",
		"--nojournal",
		"--nounixsocket",
	}
	if inst.Params != nil {
		mgoargs = append(mgoargs, inst.Params...)
	}
	server := exec.Command("mongod", mgoargs...)
	out, err := server.StdoutPipe()
	if err != nil {
		return err
	}
	server.Stderr = server.Stdout
	exited := make(chan struct{})
	go func() {
		lines := readLines(out, 20)
		err := server.Wait()
		exitErr, _ := err.(*exec.ExitError)
		if err == nil || exitErr != nil && exitErr.Exited() {
			// mongodb has exited without being killed, so print the
			// last few lines of its log output.
			for _, line := range lines {
				log.Infof("mongod: %s", line)
			}
		}
		close(exited)
	}()
	inst.Exited = exited
	if err := server.Start(); err != nil {
		return err
	}
	inst.Server = server

	return nil
}

func (inst *MgoInstance) mgoKill() {
	inst.Server.Process.Kill()
	<-inst.Exited
	inst.Server = nil
	inst.Exited = nil
}

func (inst *MgoInstance) Destroy() {
	if inst.Server != nil {
		inst.mgoKill()
		os.RemoveAll(inst.Dir)
		inst.Addr, inst.Dir = "", ""
	}
}

// MgoRestart restarts the mongo server, useful for
// testing what happens when a state server goes down.
func (inst *MgoInstance) MgoRestart() {
	inst.mgoKill()
	if err := inst.Start(); err != nil {
		panic(err)
	}
}

// MgoTestPackage should be called to register the tests for any package that
// requires a MongoDB server.
func MgoTestPackage(t *stdtesting.T) {
	if err := MgoServer.Start(); err != nil {
		t.Fatal(err)
	}
	defer MgoServer.Destroy()
	gc.TestingT(t)
}

func (s *MgoSuite) SetUpSuite(c *gc.C) {
	if MgoServer.Addr == "" {
		panic("MgoSuite tests must be run with MgoTestPackage")
	}
	mgo.SetStats(true)
	// Make tests that use password authentication faster.
	utils.FastInsecureHash = true
}

// readLines reads lines from the given reader and returns
// the last n non-empty lines, ignoring empty lines.
func readLines(r io.Reader, n int) []string {
	br := bufio.NewReader(r)
	lines := make([]string, n)
	i := 0
	for {
		line, err := br.ReadString('\n')
		if line = strings.TrimRight(line, "\n"); line != "" {
			lines[i%n] = line
			i++
		}
		if err != nil {
			break
		}
	}
	final := make([]string, 0, n+1)
	if i > n {
		final = append(final, fmt.Sprintf("[%d lines omitted]", i-n))
	}
	for j := 0; j < n; j++ {
		if line := lines[(j+i)%n]; line != "" {
			final = append(final, line)
		}
	}
	return final
}

func (s *MgoSuite) TearDownSuite(c *gc.C) {
	utils.FastInsecureHash = false
}

// MgoDial returns a new connection to the shared MongoDB server.
func (inst *MgoInstance) MgoDial() *mgo.Session {
	return inst.dial(false)
}

// MgoDialDirect returns a new direct connection to the shared MongoDB server. This
// must be used if you're connecting to a replicaset that hasn't been initiated
// yet.
func (inst *MgoInstance) MgoDialDirect() *mgo.Session {
	return inst.dial(true)
}

func (inst *MgoInstance) dial(direct bool) *mgo.Session {
	pool := x509.NewCertPool()
	xcert, err := cert.ParseCert([]byte(CACert))
	if err != nil {
		panic(err)
	}
	pool.AddCert(xcert)
	tlsConfig := &tls.Config{
		RootCAs:    pool,
		ServerName: "anything",
	}
	session, err := mgo.DialWithInfo(&mgo.DialInfo{
		Direct: direct,
		Addrs:  []string{inst.Addr},
		Dial: func(addr net.Addr) (net.Conn, error) {
			return tls.Dial("tcp", addr.String(), tlsConfig)
		},
		Timeout: mgoDialTimeout,
	})
	if err != nil {
		panic(err)
	}
	return session
}

func (s *MgoSuite) SetUpTest(c *gc.C) {
	mgo.ResetStats()
	s.Session = MgoServer.MgoDial()
}

// MgoReset deletes all content from the shared MongoDB server.
func (inst *MgoInstance) MgoReset() {
	session := inst.MgoDial()
	defer session.Close()

	dbnames, ok := resetAdminPasswordAndFetchDBNames(session)
	if ok {
		log.Infof("MgoReset successfully reset admin password")
	} else {
		// We restart it to regain access.  This should only
		// happen when tests fail.
		log.Noticef("testing: restarting MongoDB server after unauthorized access")
		inst.Destroy()
		if err := inst.Start(); err != nil {
			panic(err)
		}
		return
	}
	for _, name := range dbnames {
		switch name {
		case "admin", "local", "config":
		default:
			if err := session.DB(name).DropDatabase(); err != nil {
				panic(fmt.Errorf("Cannot drop MongoDB database %v: %v", name, err))
			}
		}
	}
}

// resetAdminPasswordAndFetchDBNames logs into the database with a
// plausible password and returns all the database's db names. We need
// to try several passwords because we don't know what state the mongo
// server is in when MgoReset is called. If the test has set a custom
// password, we're out of luck, but if they are using
// DefaultStatePassword, we can succeed.
func resetAdminPasswordAndFetchDBNames(session *mgo.Session) ([]string, bool) {
	// First try with no password
	dbnames, err := session.DatabaseNames()
	if err == nil {
		return dbnames, true
	}
	if !isUnauthorized(err) {
		panic(err)
	}
	// Then try the two most likely passwords in turn.
	for _, password := range []string{
		DefaultMongoPassword,
		utils.UserPasswordHash(DefaultMongoPassword, utils.CompatSalt),
	} {
		admin := session.DB("admin")
		if err := admin.Login("admin", password); err != nil {
			log.Infof("failed to log in with password %q", password)
			continue
		}
		dbnames, err := session.DatabaseNames()
		if err == nil {
			if err := admin.RemoveUser("admin"); err != nil {
				panic(err)
			}
			return dbnames, true
		}
		if !isUnauthorized(err) {
			panic(err)
		}
		log.Infof("unauthorized access when getting database names; password %q", password)
	}
	return nil, false
}

// isUnauthorized is a copy of the same function in state/open.go.
func isUnauthorized(err error) bool {
	if err == nil {
		return false
	}
	// Some unauthorized access errors have no error code,
	// just a simple error string.
	if err.Error() == "auth fails" {
		return true
	}
	if err, ok := err.(*mgo.QueryError); ok {
		return err.Code == 10057 ||
			err.Message == "need to login" ||
			err.Message == "unauthorized"
	}
	return false
}

func (s *MgoSuite) TearDownTest(c *gc.C) {
	MgoServer.MgoReset()
	s.Session.Close()
	for i := 0; ; i++ {
		stats := mgo.GetStats()
		if stats.SocketsInUse == 0 && stats.SocketsAlive == 0 {
			break
		}
		if i == 20 {
			c.Fatal("Test left sockets in a dirty state")
		}
		c.Logf("Waiting for sockets to die: %d in use, %d alive", stats.SocketsInUse, stats.SocketsAlive)
		time.Sleep(500 * time.Millisecond)
	}
}

// FindTCPPort finds an unused TCP port and returns it.
// Use of this function has an inherent race condition - another
// process may claim the port before we try to use it.
// We hope that the probability is small enough during
// testing to be negligible.
func FindTCPPort() int {
	l, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		panic(err)
	}
	l.Close()
	return l.Addr().(*net.TCPAddr).Port
}
<commit_msg>dial is the answer you seek<commit_after>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package testing

import (
	"bufio"
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	stdtesting "testing"
	"time"

	"labix.org/v2/mgo"
	gc "launchpad.net/gocheck"

	"launchpad.net/juju-core/cert"
	"launchpad.net/juju-core/log"
	"launchpad.net/juju-core/utils"
)

var (
	// MgoServer is a shared mongo server used by tests.
	MgoServer = &MgoInstance{}
)

type MgoInstance struct {
	// Addr holds the address of the shared MongoDB server set up by
	// MgoTestPackage.
	Addr string

	// MgoPort holds the port used by the shared MongoDB server.
	Port int

	// Server holds the running MongoDB command.
	Server *exec.Cmd

	// Exited receives a value when the mongodb server exits.
	Exited <-chan struct{}

	// Dir holds the directory that MongoDB is running in.
	Dir string

	// params is a list of additional parameters that will be passed to
	// the mongod application
	Params []string
}

// We specify a timeout to mgo.Dial, to prevent
// mongod failures hanging the tests.
const mgoDialTimeout = 15 * time.Second

// MgoSuite is a suite that deletes all content from the shared MongoDB
// server at the end of every test and supplies a connection to the shared
// MongoDB server.
type MgoSuite struct {
	Session *mgo.Session
}

// startMgoServer starts a MongoDB server in a temporary directory.
func (inst *MgoInstance) Start() error {
	dbdir, err := ioutil.TempDir("", "test-mgo")
	if err != nil {
		return err
	}
	pemPath := filepath.Join(dbdir, "server.pem")
	err = ioutil.WriteFile(pemPath, []byte(ServerCert+ServerKey), 0600)
	if err != nil {
		return fmt.Errorf("cannot write cert/key PEM: %v", err)
	}
	inst.Port = FindTCPPort()
	inst.Addr = fmt.Sprintf("localhost:%d", inst.Port)
	inst.Dir = dbdir
	if err := inst.runMgoServer(); err != nil {
		inst.Addr = ""
		inst.Port = 0
		os.RemoveAll(inst.Dir)
		inst.Dir = ""
		return err
	}

	// by dialing right now, we'll wait until it's running
	session := inst.MgoDialDirect()
	session.Close()
	return nil
}

// runMgoServer runs the MongoDB server at the
// address and directory already configured.
func (inst *MgoInstance) runMgoServer() error {
	if inst.Server != nil {
		panic("mongo server is already running")
	}
	mgoport := strconv.Itoa(inst.Port)
	mgoargs := []string{
		"--auth",
		"--dbpath", inst.Dir,
		"--sslOnNormalPorts",
		"--sslPEMKeyFile", filepath.Join(inst.Dir, "server.pem"),
		"--sslPEMKeyPassword", "ignored",
		"--bind_ip", "localhost",
		"--port", mgoport,
		"--nssize", "1",
		"--noprealloc",
		"--smallfiles",
		"--nojournal",
		"--nounixsocket",
	}
	if inst.Params != nil {
		mgoargs = append(mgoargs, inst.Params...)
	}
	server := exec.Command("mongod", mgoargs...)
	out, err := server.StdoutPipe()
	if err != nil {
		return err
	}
	server.Stderr = server.Stdout
	exited := make(chan struct{})
	go func() {
		lines := readLines(out, 20)
		err := server.Wait()
		exitErr, _ := err.(*exec.ExitError)
		if err == nil || exitErr != nil && exitErr.Exited() {
			// mongodb has exited without being killed, so print the
			// last few lines of its log output.
			for _, line := range lines {
				log.Infof("mongod: %s", line)
			}
		}
		close(exited)
	}()
	inst.Exited = exited
	if err := server.Start(); err != nil {
		return err
	}
	inst.Server = server

	return nil
}

func (inst *MgoInstance) mgoKill() {
	inst.Server.Process.Kill()
	<-inst.Exited
	inst.Server = nil
	inst.Exited = nil
}

func (inst *MgoInstance) Destroy() {
	if inst.Server != nil {
		inst.mgoKill()
		os.RemoveAll(inst.Dir)
		inst.Addr, inst.Dir = "", ""
	}
}

// MgoRestart restarts the mongo server, useful for
// testing what happens when a state server goes down.
func (inst *MgoInstance) MgoRestart() {
	inst.mgoKill()
	if err := inst.Start(); err != nil {
		panic(err)
	}
}

// MgoTestPackage should be called to register the tests for any package that
// requires a MongoDB server.
func MgoTestPackage(t *stdtesting.T) {
	if err := MgoServer.Start(); err != nil {
		t.Fatal(err)
	}
	defer MgoServer.Destroy()
	gc.TestingT(t)
}

func (s *MgoSuite) SetUpSuite(c *gc.C) {
	if MgoServer.Addr == "" {
		panic("MgoSuite tests must be run with MgoTestPackage")
	}
	mgo.SetStats(true)
	// Make tests that use password authentication faster.
	utils.FastInsecureHash = true
}

// readLines reads lines from the given reader and returns
// the last n non-empty lines, ignoring empty lines.
func readLines(r io.Reader, n int) []string {
	br := bufio.NewReader(r)
	lines := make([]string, n)
	i := 0
	for {
		line, err := br.ReadString('\n')
		if line = strings.TrimRight(line, "\n"); line != "" {
			lines[i%n] = line
			i++
		}
		if err != nil {
			break
		}
	}
	final := make([]string, 0, n+1)
	if i > n {
		final = append(final, fmt.Sprintf("[%d lines omitted]", i-n))
	}
	for j := 0; j < n; j++ {
		if line := lines[(j+i)%n]; line != "" {
			final = append(final, line)
		}
	}
	return final
}

func (s *MgoSuite) TearDownSuite(c *gc.C) {
	utils.FastInsecureHash = false
}

// MgoDial returns a new connection to the shared MongoDB server.
func (inst *MgoInstance) MgoDial() *mgo.Session {
	return inst.dial(false)
}

// MgoDialDirect returns a new direct connection to the shared MongoDB server. This
// must be used if you're connecting to a replicaset that hasn't been initiated
// yet.
func (inst *MgoInstance) MgoDialDirect() *mgo.Session {
	return inst.dial(true)
}

func (inst *MgoInstance) dial(direct bool) *mgo.Session {
	pool := x509.NewCertPool()
	xcert, err := cert.ParseCert([]byte(CACert))
	if err != nil {
		panic(err)
	}
	pool.AddCert(xcert)
	tlsConfig := &tls.Config{
		RootCAs:    pool,
		ServerName: "anything",
	}
	session, err := mgo.DialWithInfo(&mgo.DialInfo{
		Direct: direct,
		Addrs:  []string{inst.Addr},
		Dial: func(addr net.Addr) (net.Conn, error) {
			return tls.Dial("tcp", addr.String(), tlsConfig)
		},
		Timeout: mgoDialTimeout,
	})
	if err != nil {
		panic(err)
	}
	return session
}

func (s *MgoSuite) SetUpTest(c *gc.C) {
	mgo.ResetStats()
	s.Session = MgoServer.MgoDial()
}

// MgoReset deletes all content from the shared MongoDB server.
func (inst *MgoInstance) MgoReset() {
	session := inst.MgoDial()
	defer session.Close()

	dbnames, ok := resetAdminPasswordAndFetchDBNames(session)
	if ok {
		log.Infof("MgoReset successfully reset admin password")
	} else {
		// We restart it to regain access.  This should only
		// happen when tests fail.
		log.Noticef("testing: restarting MongoDB server after unauthorized access")
		inst.Destroy()
		if err := inst.Start(); err != nil {
			panic(err)
		}
		return
	}
	for _, name := range dbnames {
		switch name {
		case "admin", "local", "config":
		default:
			if err := session.DB(name).DropDatabase(); err != nil {
				panic(fmt.Errorf("Cannot drop MongoDB database %v: %v", name, err))
			}
		}
	}
}

// resetAdminPasswordAndFetchDBNames logs into the database with a
// plausible password and returns all the database's db names. We need
// to try several passwords because we don't know what state the mongo
// server is in when MgoReset is called. If the test has set a custom
// password, we're out of luck, but if they are using
// DefaultStatePassword, we can succeed.
func resetAdminPasswordAndFetchDBNames(session *mgo.Session) ([]string, bool) {
	// First try with no password
	dbnames, err := session.DatabaseNames()
	if err == nil {
		return dbnames, true
	}
	if !isUnauthorized(err) {
		panic(err)
	}
	// Then try the two most likely passwords in turn.
	for _, password := range []string{
		DefaultMongoPassword,
		utils.UserPasswordHash(DefaultMongoPassword, utils.CompatSalt),
	} {
		admin := session.DB("admin")
		if err := admin.Login("admin", password); err != nil {
			log.Infof("failed to log in with password %q", password)
			continue
		}
		dbnames, err := session.DatabaseNames()
		if err == nil {
			if err := admin.RemoveUser("admin"); err != nil {
				panic(err)
			}
			return dbnames, true
		}
		if !isUnauthorized(err) {
			panic(err)
		}
		log.Infof("unauthorized access when getting database names; password %q", password)
	}
	return nil, false
}

// isUnauthorized is a copy of the same function in state/open.go.
func isUnauthorized(err error) bool {
	if err == nil {
		return false
	}
	// Some unauthorized access errors have no error code,
	// just a simple error string.
	if err.Error() == "auth fails" {
		return true
	}
	if err, ok := err.(*mgo.QueryError); ok {
		return err.Code == 10057 ||
			err.Message == "need to login" ||
			err.Message == "unauthorized"
	}
	return false
}

func (s *MgoSuite) TearDownTest(c *gc.C) {
	MgoServer.MgoReset()
	s.Session.Close()
	for i := 0; ; i++ {
		stats := mgo.GetStats()
		if stats.SocketsInUse == 0 && stats.SocketsAlive == 0 {
			break
		}
		if i == 20 {
			c.Fatal("Test left sockets in a dirty state")
		}
		c.Logf("Waiting for sockets to die: %d in use, %d alive", stats.SocketsInUse, stats.SocketsAlive)
		time.Sleep(500 * time.Millisecond)
	}
}

// FindTCPPort finds an unused TCP port and returns it.
// Use of this function has an inherent race condition - another
// process may claim the port before we try to use it.
// We hope that the probability is small enough during
// testing to be negligible.
func FindTCPPort() int {
	l, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		panic(err)
	}
	l.Close()
	return l.Addr().(*net.TCPAddr).Port
}
<|endoftext|>


<commit_before>package ma

import (
	"yap/alg/graph"
	"yap/nlp/format/lex"
	. "yap/nlp/types"
	"yap/util"

	"fmt"
	"log"
	"regexp"
	"strings"
)

const ESTIMATED_MORPHS_PER_TOKEN = 5

type BGULex struct {
	MaxPrefixLen int
	Prefixes     map[string][]BasicMorphemes

	Lex map[string][]BasicMorphemes

	Files []string
	Stats *AnalyzeStats

	AlwaysNNP bool
}

var (
	PUNCT = map[string]string{
		":":   "yyCLN",
		",":   "yyCM",
		"-":   "yyDASH",
		".":   "yyDOT",
		"...": "yyELPS",
		"!":   "yyEXCL",
		"(":   "yyLRB",
		"?":   "yyQM",
		")":   "yyRRB",
		";":   "yySCLN",
		"\"":  "yyQUOT",
	}
	OOVMSRS = []string{
		"NNP-",
		"NNP-gen=F|gen=M|num=S",
		"NNP-gen=M|num=S",
		"NNP-gen=F|num=S",
		"NN-gen=M|num=P|num=S",
		"NN-gen=M|num=S",
		"NN-gen=F|num=S",
		"NN-gen=M|num=P",
		"NN-gen=F|num=P",
	}
	REGEX = []struct {
		RE  *regexp.Regexp
		POS string
	}{
		{regexp.MustCompile("^\\d+(\\.\\d+)?$|^\\d{1,3}(,\\d{3})*(\\.\\d+)?$"), "CD"},
		{regexp.MustCompile("\\d"), "NCD"},
	}
	_ MorphologicalAnalyzer = &BGULex{}
)

func (l *BGULex) loadTokens(file, format string) {
	tokens, err := lex.ReadFile(file, format)
	if err != nil {
		panic(fmt.Sprintf("Failed to load %v: %v", file, err))
	}
	var m map[string][]BasicMorphemes
	if format == "prefix" {
		l.Prefixes = make(map[string][]BasicMorphemes, len(tokens))
		m = l.Prefixes
	} else if format == "lexicon" {
		l.Lex = make(map[string][]BasicMorphemes, len(tokens))
		m = l.Lex
	}
	log.Println("Found", len(tokens), "tokens in lexicon file:", file)
	for _, token := range tokens {
		if cur, exists := m[token.Token]; exists {
			m[token.Token] = append(cur, token.Morphemes...)
		} else {
			m[token.Token] = token.Morphemes
		}
	}
}

func (l *BGULex) LoadPrefixes(file string) {
	l.loadTokens(file, "prefix")
	l.MaxPrefixLen = 0
	for _, morphs := range l.Prefixes {
		if l.MaxPrefixLen < len(morphs) {
			l.MaxPrefixLen = len(morphs)
		}
	}
	log.Println("Loaded", len(l.Prefixes), "prefixes from lexicon")
}

func (l *BGULex) LoadLex(file string, nnpnofeats bool) {
	lex.ADD_NNP_NO_FEATS = nnpnofeats
	l.loadTokens(file, "lexicon")
	log.Println("Loaded", len(l.Lex), "tokens from lexicon")
}

func makeMorphWithPOS(input, lemma, POS string) []BasicMorphemes {
	return []BasicMorphemes{BasicMorphemes([]*Morpheme{
		&Morpheme{
			BasicDirectedEdge: graph.BasicDirectedEdge{0, 0, 1},
			Form:              input,
			Lemma:             lemma,
			CPOS:              POS,
			POS:               POS,
			FeatureStr:        "",
		},
	})}
}

func (l *BGULex) AddOOVAnalysis(lat *Lattice, prefix BasicMorphemes, hostStr string, numToken int) {
	for _, msr := range OOVMSRS {
		// if logAnalyze {
		// 	log.Println("Adding msr", msr)
		// }
		msrsplit := strings.Split(msr, "-")
		newMorph := []BasicMorphemes{BasicMorphemes([]*Morpheme{
			&Morpheme{
				BasicDirectedEdge: graph.BasicDirectedEdge{0, 0, 1},
				Form:              hostStr,
				Lemma:             hostStr,
				CPOS:              msrsplit[0],
				POS:               msrsplit[0],
				FeatureStr:        msrsplit[1],
			},
		})}
		lat.AddAnalysis(prefix, newMorph, numToken)
	}
}

func (l *BGULex) OOVAnalysis(input string) []BasicMorphemes {
	retval := make([]*Morpheme, 0, len(OOVMSRS))
	for i, msr := range OOVMSRS {
		msrsplit := strings.Split(msr, "-")
		retval = append(retval, &Morpheme{
			BasicDirectedEdge: graph.BasicDirectedEdge{i, 0, 1},
			Form:              input,
			Lemma:             input,
			CPOS:              msrsplit[0],
			POS:               msrsplit[0],
			FeatureStr:        msrsplit[1],
		})
	}
	return []BasicMorphemes{BasicMorphemes(retval)}
}

func checkRegexes(input string) ([]BasicMorphemes, bool) {
	for _, curRegex := range REGEX {
		if curRegex.RE.MatchString(input) {
			return makeMorphWithPOS(input, "", curRegex.POS), true
		}
	}
	return nil, false
}

var logAnalyze bool = false

func (l *BGULex) OOVForLen(lat *Lattice, input string, startingNode, numToken, prefixLen int) bool {
	var (
		found   bool
		hostStr string
	)
	if len(input) < prefixLen*2 {
		return found
	}
	prefixLat, prefixExists := l.Prefixes[input[0:prefixLen*2]]
	// log.Println("\tPrefixes", input[0:prefixLen*2], prefixExists)
	if prefixExists {
		hostStr = input[2*prefixLen:]
		if len(hostStr) > 2 {
			// Always add NNP hosts for len(hosts)>1 (unicode = 2 runes)
			for _, prefix := range prefixLat {
				l.AddOOVAnalysis(lat, prefix, hostStr, numToken)
				// lat.AddAnalysis(prefix, l.OOVAnalysis(hostStr), numToken)
			}
		}
	}
	return found
}

func (l *BGULex) analyzeTokenForLen(lat *Lattice, input string, startingNode, numToken, prefixLen int) bool {
	var (
		found, hostExists bool
		hostLat           []BasicMorphemes
		hostStr           string
	)
	if len(input) < prefixLen*2 {
		return found
	}
	prefixLat, prefixExists := l.Prefixes[input[0:prefixLen*2]]
	// log.Println("\tPrefixes", input[0:prefixLen*2], prefixExists)
	if prefixExists {
		hostStr = input[2*prefixLen:]
		if l.AlwaysNNP {
			if len(hostStr) > 2 {
				// Always add NNP hosts for len(hosts)>1 (unicode = 2 runes)
				for _, prefix := range prefixLat {
					l.AddOOVAnalysis(lat, prefix, hostStr, numToken)
					// lat.AddAnalysis(prefix, l.OOVAnalysis(hostStr), numToken)
				}
			}
		}
		hostLat, hostExists = l.Lex[hostStr]
		if !hostExists {
			hostLat, hostExists = checkRegexes(hostStr)
		}
		// log.Println("\tHosts", input[2*prefixLen:], hostExists)
		if hostExists {
			for _, prefix := range prefixLat {
				// log.Println("\t\tAdding", prefix, hostLat)
				lat.AddAnalysis(prefix, hostLat, numToken)
			}
			found = true
		}
	}
	return found
}

func (l *BGULex) AnalyzeToken(input string, startingNode, numToken int) (*Lattice, interface{}) {
	if logAnalyze {
		log.Println("Analyzing token", numToken, "starting at", startingNode)
	}
	lat := &Lattice{
		Token:     Token(input),
		Morphemes: make(Morphemes, 0, ESTIMATED_MORPHS_PER_TOKEN),
		Next:      make(map[int][]int, ESTIMATED_MORPHS_PER_TOKEN),
		BottomId:  startingNode,
		TopId:     startingNode,
	}
	lat.Next[0] = make([]int, 0, 1)
	var (
		hostLat               []BasicMorphemes
		hostExists, anyExists bool
	)
	if punctVal, exists := PUNCT[input]; exists {
		m := &Morpheme{
			BasicDirectedEdge: graph.BasicDirectedEdge{0, 0, 0},
			Form:              input,
			CPOS:              punctVal,
			POS:               punctVal,
		}
		basics := []BasicMorphemes{BasicMorphemes{m}}
		lat.AddAnalysis(nil, basics, numToken)
		return lat, nil
	}
	if l.AlwaysNNP {
		l.AddOOVAnalysis(lat, nil, input, numToken)
		// oovLat := l.OOVAnalysis(input)
		// lat.AddAnalysis(nil, oovLat, numToken)
	}
	hostLat, hostExists = l.Lex[input]
	if !hostExists {
		hostLat, hostExists = checkRegexes(input)
	}
	if hostExists {
		if logAnalyze {
			log.Println("\tPrefix 0")
		}
		lat.AddAnalysis(nil, hostLat, numToken)
		anyExists = true
	} else {
		if !l.AlwaysNNP {
			l.AddOOVAnalysis(lat, nil, input, numToken)
			// oovLat := l.OOVAnalysis(input)
			// lat.AddAnalysis(nil, oovLat, numToken)
		}
	}
	for i := 1; i < util.Min(l.MaxPrefixLen, len(input)); i++ {
		if logAnalyze {
			log.Println("\ti is", i)
		}
		found := l.analyzeTokenForLen(lat, input, startingNode, numToken, i)
		anyExists = anyExists || found
	}
	if !anyExists {
		// if logAnalyze {
		log.Println("Token", numToken, "is OOV:", input)
		for i := 1; i < util.Min(l.MaxPrefixLen, len(input)); i++ {
			if logAnalyze {
				log.Println("\ti is", i)
			}
			_ = l.OOVForLen(lat, input, startingNode, numToken, i)
		}
		// }
		if l.Stats != nil {
			l.Stats.OOVTokens++
			l.Stats.AddOOVToken(input)
		}
	}
	lat.Optimize()
	return lat, nil
}

func (l *BGULex) Analyze(input []string) (LatticeSentence, interface{}) {
	retval := make(LatticeSentence, len(input))
	var (
		lat     *Lattice
		curNode int
	)
	for i, token := range input {
		if l.Stats != nil {
			l.Stats.TotalTokens++
			l.Stats.AddToken(token)
		}
		lat, _ = l.AnalyzeToken(token, curNode, i)
		curNode = lat.Top()
		// log.Println("New top is", curNode)
		retval[i] = *lat
	}
	return retval, nil
}
<commit_msg>Fix off-by-one token index<commit_after>package ma

import (
	"yap/alg/graph"
	"yap/nlp/format/lex"
	. "yap/nlp/types"
	"yap/util"

	"fmt"
	"log"
	"regexp"
	"strings"
)

const ESTIMATED_MORPHS_PER_TOKEN = 5

type BGULex struct {
	MaxPrefixLen int
	Prefixes     map[string][]BasicMorphemes

	Lex map[string][]BasicMorphemes

	Files []string
	Stats *AnalyzeStats

	AlwaysNNP bool
}

var (
	PUNCT = map[string]string{
		":":   "yyCLN",
		",":   "yyCM",
		"-":   "yyDASH",
		".":   "yyDOT",
		"...": "yyELPS",
		"!":   "yyEXCL",
		"(":   "yyLRB",
		"?":   "yyQM",
		")":   "yyRRB",
		";":   "yySCLN",
		"\"":  "yyQUOT",
	}
	OOVMSRS = []string{
		"NNP-",
		"NNP-gen=F|gen=M|num=S",
		"NNP-gen=M|num=S",
		"NNP-gen=F|num=S",
		"NN-gen=M|num=P|num=S",
		"NN-gen=M|num=S",
		"NN-gen=F|num=S",
		"NN-gen=M|num=P",
		"NN-gen=F|num=P",
	}
	REGEX = []struct {
		RE  *regexp.Regexp
		POS string
	}{
		{regexp.MustCompile("^\\d+(\\.\\d+)?$|^\\d{1,3}(,\\d{3})*(\\.\\d+)?$"), "CD"},
		{regexp.MustCompile("\\d"), "NCD"},
	}
	_ MorphologicalAnalyzer = &BGULex{}
)

func (l *BGULex) loadTokens(file, format string) {
	tokens, err := lex.ReadFile(file, format)
	if err != nil {
		panic(fmt.Sprintf("Failed to load %v: %v", file, err))
	}
	var m map[string][]BasicMorphemes
	if format == "prefix" {
		l.Prefixes = make(map[string][]BasicMorphemes, len(tokens))
		m = l.Prefixes
	} else if format == "lexicon" {
		l.Lex = make(map[string][]BasicMorphemes, len(tokens))
		m = l.Lex
	}
	log.Println("Found", len(tokens), "tokens in lexicon file:", file)
	for _, token := range tokens {
		if cur, exists := m[token.Token]; exists {
			m[token.Token] = append(cur, token.Morphemes...)
		} else {
			m[token.Token] = token.Morphemes
		}
	}
}

func (l *BGULex) LoadPrefixes(file string) {
	l.loadTokens(file, "prefix")
	l.MaxPrefixLen = 0
	for _, morphs := range l.Prefixes {
		if l.MaxPrefixLen < len(morphs) {
			l.MaxPrefixLen = len(morphs)
		}
	}
	log.Println("Loaded", len(l.Prefixes), "prefixes from lexicon")
}

func (l *BGULex) LoadLex(file string, nnpnofeats bool) {
	lex.ADD_NNP_NO_FEATS = nnpnofeats
	l.loadTokens(file, "lexicon")
	log.Println("Loaded", len(l.Lex), "tokens from lexicon")
}

func makeMorphWithPOS(input, lemma, POS string) []BasicMorphemes {
	return []BasicMorphemes{BasicMorphemes([]*Morpheme{
		&Morpheme{
			BasicDirectedEdge: graph.BasicDirectedEdge{0, 0, 1},
			Form:              input,
			Lemma:             lemma,
			CPOS:              POS,
			POS:               POS,
			FeatureStr:        "",
		},
	})}
}

func (l *BGULex) AddOOVAnalysis(lat *Lattice, prefix BasicMorphemes, hostStr string, numToken int) {
	for _, msr := range OOVMSRS {
		// if logAnalyze {
		// 	log.Println("Adding msr", msr)
		// }
		msrsplit := strings.Split(msr, "-")
		newMorph := []BasicMorphemes{BasicMorphemes([]*Morpheme{
			&Morpheme{
				BasicDirectedEdge: graph.BasicDirectedEdge{0, 0, 1},
				Form:              hostStr,
				Lemma:             hostStr,
				CPOS:              msrsplit[0],
				POS:               msrsplit[0],
				FeatureStr:        msrsplit[1],
			},
		})}
		lat.AddAnalysis(prefix, newMorph, numToken)
	}
}

func (l *BGULex) OOVAnalysis(input string) []BasicMorphemes {
	retval := make([]*Morpheme, 0, len(OOVMSRS))
	for i, msr := range OOVMSRS {
		msrsplit := strings.Split(msr, "-")
		retval = append(retval, &Morpheme{
			BasicDirectedEdge: graph.BasicDirectedEdge{i, 0, 1},
			Form:              input,
			Lemma:             input,
			CPOS:              msrsplit[0],
			POS:               msrsplit[0],
			FeatureStr:        msrsplit[1],
		})
	}
	return []BasicMorphemes{BasicMorphemes(retval)}
}

func checkRegexes(input string) ([]BasicMorphemes, bool) {
	for _, curRegex := range REGEX {
		if curRegex.RE.MatchString(input) {
			return makeMorphWithPOS(input, "", curRegex.POS), true
		}
	}
	return nil, false
}

var logAnalyze bool = false

func (l *BGULex) OOVForLen(lat *Lattice, input string, startingNode, numToken, prefixLen int) bool {
	var (
		found   bool
		hostStr string
	)
	if len(input) < prefixLen*2 {
		return found
	}
	prefixLat, prefixExists := l.Prefixes[input[0:prefixLen*2]]
	// log.Println("\tPrefixes", input[0:prefixLen*2], prefixExists)
	if prefixExists {
		hostStr = input[2*prefixLen:]
		if len(hostStr) > 2 {
			// Always add NNP hosts for len(hosts)>1 (unicode = 2 runes)
			for _, prefix := range prefixLat {
				l.AddOOVAnalysis(lat, prefix, hostStr, numToken)
				// lat.AddAnalysis(prefix, l.OOVAnalysis(hostStr), numToken)
			}
		}
	}
	return found
}

func (l *BGULex) analyzeTokenForLen(lat *Lattice, input string, startingNode, numToken, prefixLen int) bool {
	var (
		found, hostExists bool
		hostLat           []BasicMorphemes
		hostStr           string
	)
	if len(input) < prefixLen*2 {
		return found
	}
	prefixLat, prefixExists := l.Prefixes[input[0:prefixLen*2]]
	// log.Println("\tPrefixes", input[0:prefixLen*2], prefixExists)
	if prefixExists {
		hostStr = input[2*prefixLen:]
		if l.AlwaysNNP {
			if len(hostStr) > 2 {
				// Always add NNP hosts for len(hosts)>1 (unicode = 2 runes)
				for _, prefix := range prefixLat {
					l.AddOOVAnalysis(lat, prefix, hostStr, numToken)
					// lat.AddAnalysis(prefix, l.OOVAnalysis(hostStr), numToken)
				}
			}
		}
		hostLat, hostExists = l.Lex[hostStr]
		if !hostExists {
			hostLat, hostExists = checkRegexes(hostStr)
		}
		// log.Println("\tHosts", input[2*prefixLen:], hostExists)
		if hostExists {
			for _, prefix := range prefixLat {
				// log.Println("\t\tAdding", prefix, hostLat)
				lat.AddAnalysis(prefix, hostLat, numToken)
			}
			found = true
		}
	}
	return found
}

func (l *BGULex) AnalyzeToken(input string, startingNode, indexToken int) (*Lattice, interface{}) {
	numToken := indexToken + 1
	if logAnalyze {
		log.Println("Analyzing token", numToken, "starting at", startingNode)
	}
	lat := &Lattice{
		Token:     Token(input),
		Morphemes: make(Morphemes, 0, ESTIMATED_MORPHS_PER_TOKEN),
		Next:      make(map[int][]int, ESTIMATED_MORPHS_PER_TOKEN),
		BottomId:  startingNode,
		TopId:     startingNode,
	}
	lat.Next[0] = make([]int, 0, 1)
	var (
		hostLat               []BasicMorphemes
		hostExists, anyExists bool
	)
	if punctVal, exists := PUNCT[input]; exists {
		m := &Morpheme{
			BasicDirectedEdge: graph.BasicDirectedEdge{0, 0, 0},
			Form:              input,
			CPOS:              punctVal,
			POS:               punctVal,
		}
		basics := []BasicMorphemes{BasicMorphemes{m}}
		lat.AddAnalysis(nil, basics, numToken)
		return lat, nil
	}
	if l.AlwaysNNP {
		l.AddOOVAnalysis(lat, nil, input, numToken)
		// oovLat := l.OOVAnalysis(input)
		// lat.AddAnalysis(nil, oovLat, numToken)
	}
	hostLat, hostExists = l.Lex[input]
	if !hostExists {
		hostLat, hostExists = checkRegexes(input)
	}
	if hostExists {
		if logAnalyze {
			log.Println("\tPrefix 0")
		}
		lat.AddAnalysis(nil, hostLat, numToken)
		anyExists = true
	} else {
		if !l.AlwaysNNP {
			l.AddOOVAnalysis(lat, nil, input, numToken)
			// oovLat := l.OOVAnalysis(input)
			// lat.AddAnalysis(nil, oovLat, numToken)
		}
	}
	for i := 1; i < util.Min(l.MaxPrefixLen, len(input)); i++ {
		if logAnalyze {
			log.Println("\ti is", i)
		}
		found := l.analyzeTokenForLen(lat, input, startingNode, numToken, i)
		anyExists = anyExists || found
	}
	if !anyExists {
		// if logAnalyze {
		log.Println("Token", numToken, "is OOV:", input)
		for i := 1; i < util.Min(l.MaxPrefixLen, len(input)); i++ {
			if logAnalyze {
				log.Println("\ti is", i)
			}
			_ = l.OOVForLen(lat, input, startingNode, numToken, i)
		}
		// }
		if l.Stats != nil {
			l.Stats.OOVTokens++
			l.Stats.AddOOVToken(input)
		}
	}
	lat.Optimize()
	return lat, nil
}

func (l *BGULex) Analyze(input []string) (LatticeSentence, interface{}) {
	retval := make(LatticeSentence, len(input))
	var (
		lat     *Lattice
		curNode int
	)
	for i, token := range input {
		if l.Stats != nil {
			l.Stats.TotalTokens++
			l.Stats.AddToken(token)
		}
		lat, _ = l.AnalyzeToken(token, curNode, i)
		curNode = lat.Top()
		// log.Println("New top is", curNode)
		retval[i] = *lat
	}
	return retval, nil
}
<|endoftext|>


<commit_before>// Copyright 2013 M-Lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package rtt

import (
	"code.google.com/p/google-api-go-client/bigquery/v2"
	"net"
	"strconv"
	"time"
)

const (
	dateFormat = "2006-01-02"
	timeFormat = "2006-01-02 15:04:05"
)

// bqRow is an intermediate data structure used to make data from BigQuery more
// accessible in the data processing and storing stage.
type bqRow struct {
	lastUpdated        time.Time
	serverIP, clientIP net.IP
	rtt                float64
}

// bqRows is a list of bqRow
type bqRows []*bqRow

// simplifyBQResponse takes BigQuery response rows and converts the string
// interface values into appropriate types. For example, rtt string is parsed
// into float64.
func simplifyBQResponse(rows []*bigquery.TableRow) bqRows {
	data := make(bqRows, 0, len(rows))

	var newRow *bqRow
	var rtt float64
	var lastUpdatedInt int64
	var err error

	for _, row := range rows {
		newRow = &bqRow{
			serverIP: net.ParseIP(row.F[1].V.(string)),
			clientIP: net.ParseIP(row.F[2].V.(string)),
		}
		rtt, err = strconv.ParseFloat(row.F[3].V.(string), 64)
		if err != nil {
			continue
		}
		lastUpdatedInt, err = strconv.ParseInt(row.F[0].V.(string), 10, 64)
		if err != nil {
			continue
		}
		newRow.rtt = rtt
		newRow.lastUpdated = time.Unix(lastUpdatedInt, 0)
		data = append(data, newRow)
	}
	return data
}
<commit_msg>Make simplifyBQResponse more terse and check ParseIP.<commit_after>// Copyright 2013 M-Lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package rtt

import (
	"code.google.com/p/google-api-go-client/bigquery/v2"
	"net"
	"strconv"
	"time"
)

const (
	dateFormat = "2006-01-02"
	timeFormat = "2006-01-02 15:04:05"
)

// bqRow is an intermediate data structure used to make data from BigQuery more
// accessible in the data processing and storing stage.
type bqRow struct {
	lastUpdated        time.Time
	serverIP, clientIP net.IP
	rtt                float64
}

// bqRows is a list of bqRow
type bqRows []*bqRow

// simplifyBQResponse takes BigQuery response rows and converts the string
// interface values into appropriate types. For example, rtt string is parsed
// into float64.
func simplifyBQResponse(rows []*bigquery.TableRow) bqRows {
	data := make(bqRows, 0, len(rows))

	var newRow *bqRow
	var lastUpdatedInt int64
	var err error

	for _, row := range rows {
		newRow = &bqRow{}
		newRow.serverIP = net.ParseIP(row.F[1].V.(string))
		if newRow.serverIP == nil {
			continue
		}
		newRow.clientIP = net.ParseIP(row.F[2].V.(string))
		if newRow.clientIP == nil {
			continue
		}
		newRow.rtt, err = strconv.ParseFloat(row.F[3].V.(string), 64)
		if err != nil {
			continue
		}
		lastUpdatedInt, err = strconv.ParseInt(row.F[0].V.(string), 10, 64)
		if err != nil {
			continue
		}
		newRow.lastUpdated = time.Unix(lastUpdatedInt, 0)
		data = append(data, newRow)
	}
	return data
}
<|endoftext|>