<commit_before>package funk

import (
	"fmt"
	"math/rand"
	"reflect"
)

// Chunk creates an array of elements split into groups with the length of size.
// If array can't be split evenly, the final chunk will be
// the remaining element.
func Chunk(arr interface{}, size int) interface{} {
	if !IsIteratee(arr) {
		panic("First parameter must be neither array nor slice")
	}

	arrValue := reflect.ValueOf(arr)

	arrType := arrValue.Type()

	resultSliceType := reflect.SliceOf(arrType)

	// Initialize final result slice which will contains slice
	resultSlice := reflect.MakeSlice(resultSliceType, 0, 0)

	itemType := arrType.Elem()

	var itemSlice reflect.Value

	itemSliceType := reflect.SliceOf(itemType)

	length := arrValue.Len()

	for i := 0; i < length; i++ {
		if i%size == 0 || i == 0 {
			if itemSlice.Kind() != reflect.Invalid {
				resultSlice = reflect.Append(resultSlice, itemSlice)
			}

			itemSlice = reflect.MakeSlice(itemSliceType, 0, 0)
		}

		itemSlice = reflect.Append(itemSlice, arrValue.Index(i))

		if i == length-1 {
			resultSlice = reflect.Append(resultSlice, itemSlice)
		}
	}

	return resultSlice.Interface()
}

// ToMap transforms a slice of instances to a Map.
// []*Foo => Map<int, *Foo>
func ToMap(in interface{}, pivot string) interface{} {
	value := reflect.ValueOf(in)

	// input value must be a slice
	if value.Kind() != reflect.Slice {
		panic(fmt.Sprintf("%v must be a slice", in))
	}

	inType := value.Type()

	structType := inType.Elem()

	// retrieve the struct in the slice to deduce key type
	if structType.Kind() == reflect.Ptr {
		structType = structType.Elem()
	}

	field, _ := structType.FieldByName(pivot)

	// value of the map will be the input type
	collectionType := reflect.MapOf(field.Type, inType.Elem())

	// create a map from scratch
	collection := reflect.MakeMap(collectionType)

	for i := 0; i < value.Len(); i++ {
		instance := value.Index(i)
		var field reflect.Value

		if instance.Kind() == reflect.Ptr {
			field = instance.Elem().FieldByName(pivot)
		} else {
			field = instance.FieldByName(pivot)
		}

		collection.SetMapIndex(field, instance)
	}

	return collection.Interface()
}

func mapSlice(arrValue reflect.Value, funcValue reflect.Value) interface{} {
	funcType := funcValue.Type()

	if funcType.NumIn() != 1 || funcType.NumOut() == 0 {
		panic("Map function with an array must have one parameter and must return at least one parameter")
	}

	arrElemType := arrValue.Type().Elem()

	// Checking whether element type is convertible to function's first argument's type.
	if !arrElemType.ConvertibleTo(funcType.In(0)) {
		panic("Map function's argument is not compatible with type of array.")
	}

	if funcType.NumOut() == 1 {
		// Get slice type corresponding to function's return value's type.
		resultSliceType := reflect.SliceOf(funcType.Out(0))

		// MakeSlice takes a slice kind type, and makes a slice.
		resultSlice := reflect.MakeSlice(resultSliceType, 0, 0)

		for i := 0; i < arrValue.Len(); i++ {
			result := funcValue.Call([]reflect.Value{arrValue.Index(i)})[0]

			resultSlice = reflect.Append(resultSlice, result)
		}

		return resultSlice.Interface()
	}

	if funcType.NumOut() == 2 {
		// value of the map will be the input type
		collectionType := reflect.MapOf(funcType.Out(0), funcType.Out(1))

		// create a map from scratch
		collection := reflect.MakeMap(collectionType)

		for i := 0; i < arrValue.Len(); i++ {
			results := funcValue.Call([]reflect.Value{arrValue.Index(i)})

			collection.SetMapIndex(results[0], results[1])
		}

		return collection.Interface()
	}

	return nil
}

func mapMap(arrValue reflect.Value, funcValue reflect.Value) interface{} {
	funcType := funcValue.Type()

	if funcType.NumIn() != 2 {
		panic("Map function with an array must have one parameter")
	}

	// Only one returned parameter, should be a slice
	if funcType.NumOut() == 1 {
		// Get slice type corresponding to function's return value's type.
		resultSliceType := reflect.SliceOf(funcType.Out(0))

		// MakeSlice takes a slice kind type, and makes a slice.
		resultSlice := reflect.MakeSlice(resultSliceType, 0, 0)

		for _, key := range arrValue.MapKeys() {
			results := funcValue.Call([]reflect.Value{key, arrValue.MapIndex(key)})

			result := results[0]

			resultSlice = reflect.Append(resultSlice, result)
		}

		return resultSlice.Interface()
	}

	// two parameters, should be a map
	if funcType.NumOut() == 2 {
		// value of the map will be the input type
		collectionType := reflect.MapOf(funcType.Out(0), funcType.Out(1))

		// create a map from scratch
		collection := reflect.MakeMap(collectionType)

		for _, key := range arrValue.MapKeys() {
			results := funcValue.Call([]reflect.Value{key, arrValue.MapIndex(key)})

			collection.SetMapIndex(results[0], results[1])

		}

		return collection.Interface()
	}

	return nil
}

// Map manipulates an iteratee and transforms it to another type.
func Map(arr interface{}, mapFunc interface{}) interface{} {
	if !IsIteratee(arr) {
		panic("First parameter must be an iteratee")
	}

	if !IsFunction(mapFunc) {
		panic("Second argument must be function")
	}

	var (
		funcValue = reflect.ValueOf(mapFunc)
		arrValue  = reflect.ValueOf(arr)
		arrType   = arrValue.Type()
	)

	kind := arrType.Kind()

	if kind == reflect.Slice || kind == reflect.Array {
		return mapSlice(arrValue, funcValue)
	}

	if kind == reflect.Map {
		return mapMap(arrValue, funcValue)
	}

	panic(fmt.Sprintf("Type %s is not supported by Map", arrType.String()))
}

// FlattenDeep recursively flattens array.
func FlattenDeep(out interface{}) interface{} {
	return flattenDeep(reflect.ValueOf(out)).Interface()
}

func flattenDeep(value reflect.Value) reflect.Value {
	sliceType := sliceElem(value.Type())

	resultSlice := reflect.MakeSlice(reflect.SliceOf(sliceType), 0, 0)

	return flatten(value, resultSlice)
}

func flatten(value reflect.Value, result reflect.Value) reflect.Value {
	length := value.Len()

	for i := 0; i < length; i++ {
		item := value.Index(i)
		kind := item.Kind()

		if kind == reflect.Slice || kind == reflect.Array {
			result = flatten(item, result)
		} else {
			result = reflect.Append(result, item)
		}
	}

	return result
}

// Shuffle creates an array of shuffled values
func Shuffle(in interface{}) interface{} {
	value := reflect.ValueOf(in)
	valueType := value.Type()

	kind := value.Kind()

	if kind == reflect.Array || kind == reflect.Slice {
		length := value.Len()

		resultSlice := makeSlice(value, length)

		for i, v := range rand.Perm(length) {
			resultSlice.Index(i).Set(value.Index(v))
		}

		return resultSlice.Interface()
	}

	panic(fmt.Sprintf("Type %s is not supported by Shuffle", valueType.String()))
}

// Reverse transforms an array the first element will become the last,
// the second element will become the second to last, etc.
func Reverse(in interface{}) interface{} {
	value := reflect.ValueOf(in)
	valueType := value.Type()

	kind := value.Kind()

	if kind == reflect.String {
		return ReverseString(in.(string))
	}

	if kind == reflect.Array || kind == reflect.Slice {
		length := value.Len()

		resultSlice := makeSlice(value, length)

		j := 0
		for i := length - 1; i >= 0; i-- {
			resultSlice.Index(j).Set(value.Index(i))
			j++
		}

		return resultSlice.Interface()
	}

	panic(fmt.Sprintf("Type %s is not supported by Reverse", valueType.String()))
}

// Uniq creates an array with unique values.
func Uniq(in interface{}) interface{} {
	value := reflect.ValueOf(in)
	valueType := value.Type()

	kind := value.Kind()

	if kind == reflect.Array || kind == reflect.Slice {
		length := value.Len()

		seen := make(map[interface{}]bool, length)
		j := 0

		for i := 0; i < length; i++ {
			val := value.Index(i)
			v := val.Interface()

			if _, ok := seen[v]; ok {
				continue
			}

			seen[v] = true
			value.Index(j).Set(val)
			j++
		}

		return value.Slice(0, j).Interface()
	}

	panic(fmt.Sprintf("Type %s is not supported by Uniq", valueType.String()))
}
<commit_msg>Panic if the map function doesn't return one or two parameters.<commit_after>package funk

import (
	"fmt"
	"math/rand"
	"reflect"
)

// Chunk creates an array of elements split into groups with the length of size.
// If array can't be split evenly, the final chunk will be
// the remaining element.
func Chunk(arr interface{}, size int) interface{} {
	if !IsIteratee(arr) {
		panic("First parameter must be neither array nor slice")
	}

	arrValue := reflect.ValueOf(arr)

	arrType := arrValue.Type()

	resultSliceType := reflect.SliceOf(arrType)

	// Initialize final result slice which will contains slice
	resultSlice := reflect.MakeSlice(resultSliceType, 0, 0)

	itemType := arrType.Elem()

	var itemSlice reflect.Value

	itemSliceType := reflect.SliceOf(itemType)

	length := arrValue.Len()

	for i := 0; i < length; i++ {
		if i%size == 0 || i == 0 {
			if itemSlice.Kind() != reflect.Invalid {
				resultSlice = reflect.Append(resultSlice, itemSlice)
			}

			itemSlice = reflect.MakeSlice(itemSliceType, 0, 0)
		}

		itemSlice = reflect.Append(itemSlice, arrValue.Index(i))

		if i == length-1 {
			resultSlice = reflect.Append(resultSlice, itemSlice)
		}
	}

	return resultSlice.Interface()
}

// ToMap transforms a slice of instances to a Map.
// []*Foo => Map<int, *Foo>
func ToMap(in interface{}, pivot string) interface{} {
	value := reflect.ValueOf(in)

	// input value must be a slice
	if value.Kind() != reflect.Slice {
		panic(fmt.Sprintf("%v must be a slice", in))
	}

	inType := value.Type()

	structType := inType.Elem()

	// retrieve the struct in the slice to deduce key type
	if structType.Kind() == reflect.Ptr {
		structType = structType.Elem()
	}

	field, _ := structType.FieldByName(pivot)

	// value of the map will be the input type
	collectionType := reflect.MapOf(field.Type, inType.Elem())

	// create a map from scratch
	collection := reflect.MakeMap(collectionType)

	for i := 0; i < value.Len(); i++ {
		instance := value.Index(i)
		var field reflect.Value

		if instance.Kind() == reflect.Ptr {
			field = instance.Elem().FieldByName(pivot)
		} else {
			field = instance.FieldByName(pivot)
		}

		collection.SetMapIndex(field, instance)
	}

	return collection.Interface()
}

func mapSlice(arrValue reflect.Value, funcValue reflect.Value) interface{} {
	funcType := funcValue.Type()

	if funcType.NumIn() != 1 || funcType.NumOut() == 0 || funcType.NumOut() > 2 {
		panic("Map function with an array must have one parameter and must return one or two parameters")
	}

	arrElemType := arrValue.Type().Elem()

	// Checking whether element type is convertible to function's first argument's type.
	if !arrElemType.ConvertibleTo(funcType.In(0)) {
		panic("Map function's argument is not compatible with type of array.")
	}

	if funcType.NumOut() == 1 {
		// Get slice type corresponding to function's return value's type.
		resultSliceType := reflect.SliceOf(funcType.Out(0))

		// MakeSlice takes a slice kind type, and makes a slice.
		resultSlice := reflect.MakeSlice(resultSliceType, 0, 0)

		for i := 0; i < arrValue.Len(); i++ {
			result := funcValue.Call([]reflect.Value{arrValue.Index(i)})[0]

			resultSlice = reflect.Append(resultSlice, result)
		}

		return resultSlice.Interface()
	}

	if funcType.NumOut() == 2 {
		// value of the map will be the input type
		collectionType := reflect.MapOf(funcType.Out(0), funcType.Out(1))

		// create a map from scratch
		collection := reflect.MakeMap(collectionType)

		for i := 0; i < arrValue.Len(); i++ {
			results := funcValue.Call([]reflect.Value{arrValue.Index(i)})

			collection.SetMapIndex(results[0], results[1])
		}

		return collection.Interface()
	}

	return nil
}

func mapMap(arrValue reflect.Value, funcValue reflect.Value) interface{} {
	funcType := funcValue.Type()

	if funcType.NumIn() != 2 || funcType.NumOut() == 0 || funcType.NumOut() > 2 {
		panic("Map function with an map must have one parameter and must return one or two parameters")
	}

	// Only one returned parameter, should be a slice
	if funcType.NumOut() == 1 {
		// Get slice type corresponding to function's return value's type.
		resultSliceType := reflect.SliceOf(funcType.Out(0))

		// MakeSlice takes a slice kind type, and makes a slice.
		resultSlice := reflect.MakeSlice(resultSliceType, 0, 0)

		for _, key := range arrValue.MapKeys() {
			results := funcValue.Call([]reflect.Value{key, arrValue.MapIndex(key)})

			result := results[0]

			resultSlice = reflect.Append(resultSlice, result)
		}

		return resultSlice.Interface()
	}

	// two parameters, should be a map
	if funcType.NumOut() == 2 {
		// value of the map will be the input type
		collectionType := reflect.MapOf(funcType.Out(0), funcType.Out(1))

		// create a map from scratch
		collection := reflect.MakeMap(collectionType)

		for _, key := range arrValue.MapKeys() {
			results := funcValue.Call([]reflect.Value{key, arrValue.MapIndex(key)})

			collection.SetMapIndex(results[0], results[1])

		}

		return collection.Interface()
	}

	return nil
}

// Map manipulates an iteratee and transforms it to another type.
func Map(arr interface{}, mapFunc interface{}) interface{} {
	if !IsIteratee(arr) {
		panic("First parameter must be an iteratee")
	}

	if !IsFunction(mapFunc) {
		panic("Second argument must be function")
	}

	var (
		funcValue = reflect.ValueOf(mapFunc)
		arrValue  = reflect.ValueOf(arr)
		arrType   = arrValue.Type()
	)

	kind := arrType.Kind()

	if kind == reflect.Slice || kind == reflect.Array {
		return mapSlice(arrValue, funcValue)
	}

	if kind == reflect.Map {
		return mapMap(arrValue, funcValue)
	}

	panic(fmt.Sprintf("Type %s is not supported by Map", arrType.String()))
}

// FlattenDeep recursively flattens array.
func FlattenDeep(out interface{}) interface{} {
	return flattenDeep(reflect.ValueOf(out)).Interface()
}

func flattenDeep(value reflect.Value) reflect.Value {
	sliceType := sliceElem(value.Type())

	resultSlice := reflect.MakeSlice(reflect.SliceOf(sliceType), 0, 0)

	return flatten(value, resultSlice)
}

func flatten(value reflect.Value, result reflect.Value) reflect.Value {
	length := value.Len()

	for i := 0; i < length; i++ {
		item := value.Index(i)
		kind := item.Kind()

		if kind == reflect.Slice || kind == reflect.Array {
			result = flatten(item, result)
		} else {
			result = reflect.Append(result, item)
		}
	}

	return result
}

// Shuffle creates an array of shuffled values
func Shuffle(in interface{}) interface{} {
	value := reflect.ValueOf(in)
	valueType := value.Type()

	kind := value.Kind()

	if kind == reflect.Array || kind == reflect.Slice {
		length := value.Len()

		resultSlice := makeSlice(value, length)

		for i, v := range rand.Perm(length) {
			resultSlice.Index(i).Set(value.Index(v))
		}

		return resultSlice.Interface()
	}

	panic(fmt.Sprintf("Type %s is not supported by Shuffle", valueType.String()))
}

// Reverse transforms an array the first element will become the last,
// the second element will become the second to last, etc.
func Reverse(in interface{}) interface{} {
	value := reflect.ValueOf(in)
	valueType := value.Type()

	kind := value.Kind()

	if kind == reflect.String {
		return ReverseString(in.(string))
	}

	if kind == reflect.Array || kind == reflect.Slice {
		length := value.Len()

		resultSlice := makeSlice(value, length)

		j := 0
		for i := length - 1; i >= 0; i-- {
			resultSlice.Index(j).Set(value.Index(i))
			j++
		}

		return resultSlice.Interface()
	}

	panic(fmt.Sprintf("Type %s is not supported by Reverse", valueType.String()))
}

// Uniq creates an array with unique values.
func Uniq(in interface{}) interface{} {
	value := reflect.ValueOf(in)
	valueType := value.Type()

	kind := value.Kind()

	if kind == reflect.Array || kind == reflect.Slice {
		length := value.Len()

		seen := make(map[interface{}]bool, length)
		j := 0

		for i := 0; i < length; i++ {
			val := value.Index(i)
			v := val.Interface()

			if _, ok := seen[v]; ok {
				continue
			}

			seen[v] = true
			value.Index(j).Set(val)
			j++
		}

		return value.Slice(0, j).Interface()
	}

	panic(fmt.Sprintf("Type %s is not supported by Uniq", valueType.String()))
}
<|endoftext|>


<commit_before>package main

import (
	"strings"
	"unicode"

	log "github.com/Sirupsen/logrus"
	"github.com/parnurzeal/gorequest"
)

type translateRequest struct {
	Text string `json:"text"`
	From string `json:"from"`
	To   string `json:"to"`
}

type translateResponse struct {
	Text            string `json:"text"`
	From            string `json:"from"`
	To              string `json:"to"`
	TranslationText string `json:"translationText"`
}

func translateString(text string) (string, error) {
	if !shouldTranslateText(text) {
		return text, nil
	}

	var response translateResponse

	request := translateRequest{
		From: "ja",
		To:   "en",
		Text: text,
	}

	resp, reply, errs := gorequest.New().Post("http://127.0.0.1:3000/api/translate").
		Type("json").SendStruct(&request).EndStruct(&response)
	for _, err := range errs {
		log.WithFields(log.Fields{
			"response": resp,
			"reply":    reply,
		}).Error(err)

		return "", err
	}

	out := response.TranslationText

	if len(out) < 1 {
		log.Warnf("Translator returned empty string, replacing with original text %q", text)
		out = text
	}

	out = cleanTranslation(out)

	return out, nil
}

func cleanTranslation(text string) string {
	// Removes any rune that isn't printable or a space
	isValid := func(r rune) rune {
		if !unicode.IsPrint(r) && !unicode.IsSpace(r) {
			return -1
		}

		return r
	}

	text = strings.Map(isValid, text)

	// Repeated whitespace
	text = replaceRegex(text, `\s{2,}`, " ")

	// ー ー ー ー
	text = replaceRegex(text, `\s+((\s+)?[-―ー]){2,}`, " ー")

	text = replaceRegex(text, `((\s+)?っ)+`, "")

	return text
}
<commit_msg>Only clean translator output<commit_after>package main

import (
	"strings"
	"unicode"

	log "github.com/Sirupsen/logrus"
	"github.com/parnurzeal/gorequest"
)

type translateRequest struct {
	Text string `json:"text"`
	From string `json:"from"`
	To   string `json:"to"`
}

type translateResponse struct {
	Text            string `json:"text"`
	From            string `json:"from"`
	To              string `json:"to"`
	TranslationText string `json:"translationText"`
}

func translateString(text string) (string, error) {
	if !shouldTranslateText(text) {
		return text, nil
	}

	var response translateResponse

	request := translateRequest{
		From: "ja",
		To:   "en",
		Text: text,
	}

	resp, reply, errs := gorequest.New().Post("http://127.0.0.1:3000/api/translate").
		Type("json").SendStruct(&request).EndStruct(&response)
	for _, err := range errs {
		log.WithFields(log.Fields{
			"response": resp,
			"reply":    reply,
		}).Error(err)

		return "", err
	}

	out := response.TranslationText

	if len(out) < 1 {
		log.Warnf("Translator returned empty string, replacing with original text %q", text)
		out = text
	} else {
		out = cleanTranslation(out)
	}

	return out, nil
}

func cleanTranslation(text string) string {
	// Removes any rune that isn't printable or a space
	isValid := func(r rune) rune {
		if !unicode.IsPrint(r) && !unicode.IsSpace(r) {
			return -1
		}

		return r
	}

	text = strings.Map(isValid, text)

	// Repeated whitespace
	text = replaceRegex(text, `\s{2,}`, " ")

	// ー ー ー ー
	text = replaceRegex(text, `\s+((\s+)?[-―ー]){2,}`, " ー")

	text = replaceRegex(text, `((\s+)?っ)+`, "")

	return text
}
<|endoftext|>


<commit_before>package irma

import (
	"bytes"
	"context"
	"encoding/hex"
	"encoding/json"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/go-errors/errors"
	"github.com/hashicorp/go-retryablehttp"
	"github.com/privacybydesign/gabi"
	"github.com/privacybydesign/gabi/revocation"
	sseclient "github.com/sietseringers/go-sse"
	"github.com/sirupsen/logrus"
	prefixed "github.com/x-cray/logrus-prefixed-formatter"

	"github.com/privacybydesign/irmago/internal/common"
	"github.com/privacybydesign/irmago/internal/disable_sigpipe"
)

// HTTPTransport sends and receives JSON messages to a HTTP server.
type HTTPTransport struct {
	Server     string
	Binary     bool
	ForceHTTPS bool
	client     *retryablehttp.Client
	headers    http.Header
}

var HTTPHeaders = map[string]http.Header{}

// Logger is used for logging. If not set, init() will initialize it to logrus.StandardLogger().
var Logger *logrus.Logger

var transportlogger *log.Logger

func init() {
	logger := logrus.New()
	logger.SetFormatter(&prefixed.TextFormatter{
		DisableColors:   true,
		FullTimestamp:   true,
		TimestampFormat: "15:04:05.000000",
	})
	SetLogger(logger)
}

func SetLogger(logger *logrus.Logger) {
	Logger = logger
	gabi.Logger = Logger
	common.Logger = Logger
	revocation.Logger = Logger
	sseclient.Logger = log.New(Logger.WithField("type", "sseclient").WriterLevel(logrus.TraceLevel), "", 0)
}

// NewHTTPTransport returns a new HTTPTransport.
func NewHTTPTransport(serverURL string, forceHTTPS bool) *HTTPTransport {
	if Logger.IsLevelEnabled(logrus.TraceLevel) {
		transportlogger = log.New(Logger.WriterLevel(logrus.TraceLevel), "transport: ", 0)
	} else {
		transportlogger = log.New(ioutil.Discard, "", 0)
	}

	if serverURL != "" && !strings.HasSuffix(serverURL, "/") {
		serverURL += "/"
	}

	// Create a transport that dials with a SIGPIPE handler (which is only active on iOS)
	var innerTransport http.Transport

	innerTransport.Dial = func(network, addr string) (c net.Conn, err error) {
		c, err = net.Dial(network, addr)
		if err != nil {
			return c, err
		}
		if err = disable_sigpipe.DisableSigPipe(c); err != nil {
			return c, err
		}
		return c, nil
	}

	client := &retryablehttp.Client{
		Logger:       transportlogger,
		RetryWaitMin: 100 * time.Millisecond,
		RetryWaitMax: 200 * time.Millisecond,
		RetryMax:     2,
		Backoff:      retryablehttp.DefaultBackoff,
		CheckRetry: func(ctx context.Context, resp *http.Response, err error) (bool, error) {
			// Don't retry on 5xx (which retryablehttp does by default)
			return err != nil || resp.StatusCode == 0, err
		},
		HTTPClient: &http.Client{
			Timeout:   time.Second * 3,
			Transport: &innerTransport,
		},
	}

	var host string
	u, err := url.Parse(serverURL)
	if err != nil {
		Logger.Warnf("failed to parse URL %s: %s", serverURL, err.Error())
	} else {
		host = u.Host
	}
	headers := HTTPHeaders[host].Clone()
	if headers == nil {
		headers = http.Header{}
	}
	return &HTTPTransport{
		Server:     serverURL,
		ForceHTTPS: forceHTTPS,
		headers:    headers,
		client:     client,
	}
}

func (transport *HTTPTransport) marshal(o interface{}) ([]byte, error) {
	if transport.Binary {
		return MarshalBinary(o)
	}
	return json.Marshal(o)
}

func (transport *HTTPTransport) unmarshal(data []byte, dst interface{}) error {
	if transport.Binary {
		return UnmarshalBinary(data, dst)
	}
	return json.Unmarshal(data, dst)
}

func (transport *HTTPTransport) unmarshalValidate(data []byte, dst interface{}) error {
	if transport.Binary {
		return UnmarshalValidateBinary(data, dst)
	}
	return UnmarshalValidate(data, dst)
}

func (transport *HTTPTransport) log(prefix string, message interface{}, binary bool) {
	if !Logger.IsLevelEnabled(logrus.TraceLevel) {
		return // do nothing if nothing would be printed anyway
	}
	var str string
	switch s := message.(type) {
	case []byte:
		str = string(s)
	case string:
		str = s
	default:
		tmp, _ := json.Marshal(message)
		str = string(tmp)
		binary = false
	}
	if !binary {
		Logger.Tracef("transport: %s: %s", prefix, str)
	} else {
		Logger.Tracef("transport: %s (hex): %s", prefix, hex.EncodeToString([]byte(str)))
	}
}

// SetHeader sets a header to be sent in requests.
func (transport *HTTPTransport) SetHeader(name, val string) {
	transport.headers.Set(name, val)
}

func (transport *HTTPTransport) request(
	url string, method string, reader io.Reader, contenttype string,
) (response *http.Response, err error) {
	var req retryablehttp.Request
	u := transport.Server + url
	if common.ForceHTTPS && transport.ForceHTTPS && !strings.HasPrefix(u, "https") {
		return nil, &SessionError{ErrorType: ErrorHTTPS, Err: errors.New("remote server does not use https")}
	}
	req.Request, err = http.NewRequest(method, u, reader)
	if err != nil {
		return nil, &SessionError{ErrorType: ErrorTransport, Err: err}
	}
	req.Header = transport.headers.Clone()
	if req.Header.Get("User-agent") == "" {
		req.Header.Set("User-Agent", "irmago")
	}
	if reader != nil && contenttype != "" {
		req.Header.Set("Content-Type", contenttype)
	}
	res, err := transport.client.Do(&req)
	if err != nil {
		return nil, &SessionError{ErrorType: ErrorTransport, Err: err}
	}
	return res, nil
}

func (transport *HTTPTransport) jsonRequest(url string, method string, result interface{}, object interface{}) error {
	if method != http.MethodPost && method != http.MethodGet && method != http.MethodDelete {
		panic("Unsupported HTTP method " + method)
	}
	if method == http.MethodGet && object != nil {
		panic("Cannot GET and also post an object")
	}

	var reader io.Reader
	var contenttype string
	if object != nil {
		switch o := object.(type) {
		case []byte:
			transport.log("body", o, true)
			contenttype = "application/octet-stream"
			reader = bytes.NewBuffer(o)
		case string:
			transport.log("body", o, false)
			contenttype = "text/plain; charset=UTF-8"
			reader = bytes.NewBuffer([]byte(o))
		default:
			marshaled, err := transport.marshal(object)
			if err != nil {
				return &SessionError{ErrorType: ErrorSerialization, Err: err}
			}
			transport.log("body", string(marshaled), transport.Binary)
			if transport.Binary {
				contenttype = "application/octet-stream"
			} else {
				contenttype = "application/json; charset=UTF-8"
			}
			reader = bytes.NewBuffer(marshaled)
		}
	}

	res, err := transport.request(url, method, reader, contenttype)
	if err != nil {
		return err
	}
	if method == http.MethodDelete {
		return nil
	}

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return &SessionError{ErrorType: ErrorServerResponse, Err: err, RemoteStatus: res.StatusCode}
	}
	if res.StatusCode != 200 {
		apierr := &RemoteError{}
		err = transport.unmarshal(body, apierr)
		if err != nil || apierr.ErrorName == "" { // Not an ApiErrorMessage
			return &SessionError{ErrorType: ErrorServerResponse, Err: err, RemoteStatus: res.StatusCode}
		}
		transport.log("error", apierr, false)
		return &SessionError{ErrorType: ErrorApi, RemoteStatus: res.StatusCode, RemoteError: apierr}
	}

	transport.log("response", body, transport.Binary)
	if result == nil { // caller doesn't care about server response
		return nil
	}
	if _, resultstr := result.(*string); resultstr {
		*result.(*string) = string(body)
	} else {
		err = transport.unmarshalValidate(body, result)
		if err != nil {
			return &SessionError{ErrorType: ErrorServerResponse, Err: err, RemoteStatus: res.StatusCode}
		}
	}

	return nil
}

func (transport *HTTPTransport) GetBytes(url string) ([]byte, error) {
	res, err := transport.request(url, http.MethodGet, nil, "")
	if err != nil {
		return nil, &SessionError{ErrorType: ErrorTransport, Err: err}
	}

	if res.StatusCode != 200 {
		return nil, &SessionError{ErrorType: ErrorServerResponse, RemoteStatus: res.StatusCode}
	}
	b, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, &SessionError{ErrorType: ErrorServerResponse, Err: err, RemoteStatus: res.StatusCode}
	}
	return b, nil
}

// Post sends the object to the server and parses its response into result.
func (transport *HTTPTransport) Post(url string, result interface{}, object interface{}) error {
	return transport.jsonRequest(url, http.MethodPost, result, object)
}

// Get performs a GET request and parses the server's response into result.
func (transport *HTTPTransport) Get(url string, result interface{}) error {
	return transport.jsonRequest(url, http.MethodGet, result, nil)
}

// Delete performs a DELETE.
func (transport *HTTPTransport) Delete() {
	_ = transport.jsonRequest("", http.MethodDelete, nil, nil)
}
<commit_msg>Added support for HTTP status code 204 when doing JSON requests<commit_after>package irma

import (
	"bytes"
	"context"
	"encoding/hex"
	"encoding/json"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/go-errors/errors"
	"github.com/hashicorp/go-retryablehttp"
	"github.com/privacybydesign/gabi"
	"github.com/privacybydesign/gabi/revocation"
	sseclient "github.com/sietseringers/go-sse"
	"github.com/sirupsen/logrus"
	prefixed "github.com/x-cray/logrus-prefixed-formatter"

	"github.com/privacybydesign/irmago/internal/common"
	"github.com/privacybydesign/irmago/internal/disable_sigpipe"
)

// HTTPTransport sends and receives JSON messages to a HTTP server.
type HTTPTransport struct {
	Server     string
	Binary     bool
	ForceHTTPS bool
	client     *retryablehttp.Client
	headers    http.Header
}

var HTTPHeaders = map[string]http.Header{}

// Logger is used for logging. If not set, init() will initialize it to logrus.StandardLogger().
var Logger *logrus.Logger

var transportlogger *log.Logger

func init() {
	logger := logrus.New()
	logger.SetFormatter(&prefixed.TextFormatter{
		DisableColors:   true,
		FullTimestamp:   true,
		TimestampFormat: "15:04:05.000000",
	})
	SetLogger(logger)
}

func SetLogger(logger *logrus.Logger) {
	Logger = logger
	gabi.Logger = Logger
	common.Logger = Logger
	revocation.Logger = Logger
	sseclient.Logger = log.New(Logger.WithField("type", "sseclient").WriterLevel(logrus.TraceLevel), "", 0)
}

// NewHTTPTransport returns a new HTTPTransport.
func NewHTTPTransport(serverURL string, forceHTTPS bool) *HTTPTransport {
	if Logger.IsLevelEnabled(logrus.TraceLevel) {
		transportlogger = log.New(Logger.WriterLevel(logrus.TraceLevel), "transport: ", 0)
	} else {
		transportlogger = log.New(ioutil.Discard, "", 0)
	}

	if serverURL != "" && !strings.HasSuffix(serverURL, "/") {
		serverURL += "/"
	}

	// Create a transport that dials with a SIGPIPE handler (which is only active on iOS)
	var innerTransport http.Transport

	innerTransport.Dial = func(network, addr string) (c net.Conn, err error) {
		c, err = net.Dial(network, addr)
		if err != nil {
			return c, err
		}
		if err = disable_sigpipe.DisableSigPipe(c); err != nil {
			return c, err
		}
		return c, nil
	}

	client := &retryablehttp.Client{
		Logger:       transportlogger,
		RetryWaitMin: 100 * time.Millisecond,
		RetryWaitMax: 200 * time.Millisecond,
		RetryMax:     2,
		Backoff:      retryablehttp.DefaultBackoff,
		CheckRetry: func(ctx context.Context, resp *http.Response, err error) (bool, error) {
			// Don't retry on 5xx (which retryablehttp does by default)
			return err != nil || resp.StatusCode == 0, err
		},
		HTTPClient: &http.Client{
			Timeout:   time.Second * 3,
			Transport: &innerTransport,
		},
	}

	var host string
	u, err := url.Parse(serverURL)
	if err != nil {
		Logger.Warnf("failed to parse URL %s: %s", serverURL, err.Error())
	} else {
		host = u.Host
	}
	headers := HTTPHeaders[host].Clone()
	if headers == nil {
		headers = http.Header{}
	}
	return &HTTPTransport{
		Server:     serverURL,
		ForceHTTPS: forceHTTPS,
		headers:    headers,
		client:     client,
	}
}

func (transport *HTTPTransport) marshal(o interface{}) ([]byte, error) {
	if transport.Binary {
		return MarshalBinary(o)
	}
	return json.Marshal(o)
}

func (transport *HTTPTransport) unmarshal(data []byte, dst interface{}) error {
	if transport.Binary {
		return UnmarshalBinary(data, dst)
	}
	return json.Unmarshal(data, dst)
}

func (transport *HTTPTransport) unmarshalValidate(data []byte, dst interface{}) error {
	if transport.Binary {
		return UnmarshalValidateBinary(data, dst)
	}
	return UnmarshalValidate(data, dst)
}

func (transport *HTTPTransport) log(prefix string, message interface{}, binary bool) {
	if !Logger.IsLevelEnabled(logrus.TraceLevel) {
		return // do nothing if nothing would be printed anyway
	}
	var str string
	switch s := message.(type) {
	case []byte:
		str = string(s)
	case string:
		str = s
	default:
		tmp, _ := json.Marshal(message)
		str = string(tmp)
		binary = false
	}
	if !binary {
		Logger.Tracef("transport: %s: %s", prefix, str)
	} else {
		Logger.Tracef("transport: %s (hex): %s", prefix, hex.EncodeToString([]byte(str)))
	}
}

// SetHeader sets a header to be sent in requests.
func (transport *HTTPTransport) SetHeader(name, val string) {
	transport.headers.Set(name, val)
}

func (transport *HTTPTransport) request(
	url string, method string, reader io.Reader, contenttype string,
) (response *http.Response, err error) {
	var req retryablehttp.Request
	u := transport.Server + url
	if common.ForceHTTPS && transport.ForceHTTPS && !strings.HasPrefix(u, "https") {
		return nil, &SessionError{ErrorType: ErrorHTTPS, Err: errors.New("remote server does not use https")}
	}
	req.Request, err = http.NewRequest(method, u, reader)
	if err != nil {
		return nil, &SessionError{ErrorType: ErrorTransport, Err: err}
	}
	req.Header = transport.headers.Clone()
	if req.Header.Get("User-agent") == "" {
		req.Header.Set("User-Agent", "irmago")
	}
	if reader != nil && contenttype != "" {
		req.Header.Set("Content-Type", contenttype)
	}
	res, err := transport.client.Do(&req)
	if err != nil {
		return nil, &SessionError{ErrorType: ErrorTransport, Err: err}
	}
	return res, nil
}

func (transport *HTTPTransport) jsonRequest(url string, method string, result interface{}, object interface{}) error {
	if method != http.MethodPost && method != http.MethodGet && method != http.MethodDelete {
		panic("Unsupported HTTP method " + method)
	}
	if method == http.MethodGet && object != nil {
		panic("Cannot GET and also post an object")
	}

	var reader io.Reader
	var contenttype string
	if object != nil {
		switch o := object.(type) {
		case []byte:
			transport.log("body", o, true)
			contenttype = "application/octet-stream"
			reader = bytes.NewBuffer(o)
		case string:
			transport.log("body", o, false)
			contenttype = "text/plain; charset=UTF-8"
			reader = bytes.NewBuffer([]byte(o))
		default:
			marshaled, err := transport.marshal(object)
			if err != nil {
				return &SessionError{ErrorType: ErrorSerialization, Err: err}
			}
			transport.log("body", string(marshaled), transport.Binary)
			if transport.Binary {
				contenttype = "application/octet-stream"
			} else {
				contenttype = "application/json; charset=UTF-8"
			}
			reader = bytes.NewBuffer(marshaled)
		}
	}

	res, err := transport.request(url, method, reader, contenttype)
	if err != nil {
		return err
	}
	if method == http.MethodDelete {
		return nil
	}

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return &SessionError{ErrorType: ErrorServerResponse, Err: err, RemoteStatus: res.StatusCode}
	}
	if res.StatusCode == http.StatusNoContent {
		if result != nil {
			return &SessionError{
				ErrorType:    ErrorServerResponse,
				Err:          errors.New("'204 No Content' received, but result was expected"),
				RemoteStatus: res.StatusCode,
			}
		}
	} else if res.StatusCode != http.StatusOK {
		apierr := &RemoteError{}
		err = transport.unmarshal(body, apierr)
		if err != nil || apierr.ErrorName == "" { // Not an ApiErrorMessage
			return &SessionError{ErrorType: ErrorServerResponse, Err: err, RemoteStatus: res.StatusCode}
		}
		transport.log("error", apierr, false)
		return &SessionError{ErrorType: ErrorApi, RemoteStatus: res.StatusCode, RemoteError: apierr}
	}

	transport.log("response", body, transport.Binary)
	if result == nil { // caller doesn't care about server response
		return nil
	}
	if _, resultstr := result.(*string); resultstr {
		*result.(*string) = string(body)
	} else {
		err = transport.unmarshalValidate(body, result)
		if err != nil {
			return &SessionError{ErrorType: ErrorServerResponse, Err: err, RemoteStatus: res.StatusCode}
		}
	}

	return nil
}

func (transport *HTTPTransport) GetBytes(url string) ([]byte, error) {
	res, err := transport.request(url, http.MethodGet, nil, "")
	if err != nil {
		return nil, &SessionError{ErrorType: ErrorTransport, Err: err}
	}

	if res.StatusCode != 200 {
		return nil, &SessionError{ErrorType: ErrorServerResponse, RemoteStatus: res.StatusCode}
	}
	b, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, &SessionError{ErrorType: ErrorServerResponse, Err: err, RemoteStatus: res.StatusCode}
	}
	return b, nil
}

// Post sends the object to the server and parses its response into result.
func (transport *HTTPTransport) Post(url string, result interface{}, object interface{}) error {
	return transport.jsonRequest(url, http.MethodPost, result, object)
}

// Get performs a GET request and parses the server's response into result.
func (transport *HTTPTransport) Get(url string, result interface{}) error {
	return transport.jsonRequest(url, http.MethodGet, result, nil)
}

// Delete performs a DELETE.
func (transport *HTTPTransport) Delete() {
	_ = transport.jsonRequest("", http.MethodDelete, nil, nil)
}
<|endoftext|>


<commit_before>package sudoku

import (
	"math"
)

//1.0 is a no op. 0.0 to 1.0 is increase goodness; 1.0 and above is decraase good
type probabilityTweak float64

type probabilityTwiddler func(*SolveStep, []*SolveStep, []*CompoundSolveStep, *Grid) probabilityTweak

type probabilityTwiddlerItem struct {
	f    probabilityTwiddler
	name string
}

//twiddlers is the list of all of the twiddlers we should apply to change the
//probability distribution of possibilities at each step. They capture biases
//that humans have about which cells to focus on (which is separate from
//Technique.humanLikelihood, since that is about how common a technique in
//general, not in a specific context.)
var twiddlers []probabilityTwiddlerItem

func init() {
	//twiddlers is not a map because a) we need to attach more info anyway,
	//and b) we want a stable ordering.
	twiddlers = []probabilityTwiddlerItem{
		{
			f:    twiddleHumanLikelihood,
			name: "Human Likelihood",
		},
		{
			f:    twiddleChainedSteps,
			name: "Chained Steps",
		},
		{
			f:    twiddleCommonNumbers,
			name: "Common Numbers",
		},
		{
			f:    twiddlePointingTargetOverlap,
			name: "Pointing Target Overlap",
		},
	}
}

//twiddlePointingTargetOverlap twiddles based on how much the targetcells
//overlap with the pointingcells of the proposed step. This tries to capture
//the fact that for cull steps in particular, we want to heavily incentivize
//steps that directly reduce possibilities in the next round of steps. This is
//conceptually similar to ChainSimilarity, but more targeted.
func twiddlePointingTargetOverlap(currentStep *SolveStep, inProgressCompoundStep []*SolveStep, pastSteps []*CompoundSolveStep, grid *Grid) probabilityTweak {
	if len(inProgressCompoundStep) == 0 {
		return 1.0
	}
	lastStep := inProgressCompoundStep[len(inProgressCompoundStep)-1]

	if currentStep == nil {
		return 1.0
	}

	//We're going to look for two kinds of overlap: targetCell to targetCell,
	//and targetCell to PointerCell, because some techniques want one or the
	//other (do any want both?). We'll use the higher overlap.

	//Compute Target --> Pointer overlap

	targetPointerUnion := currentStep.PointerCells.Union(lastStep.TargetCells)
	targetPointerIntersection := currentStep.PointerCells.Intersection(lastStep.TargetCells)

	targetPointerOverlap := float64(len(targetPointerIntersection)) / float64(len(targetPointerUnion))

	if math.IsNaN(targetPointerOverlap) {
		targetPointerOverlap = 0.0
	}

	//Compute Target --> Target overlap

	targetTargetUnion := currentStep.TargetCells.Union(lastStep.TargetCells)
	targetTargetIntersection := currentStep.TargetCells.Intersection(lastStep.TargetCells)

	targetTargetOverlap := float64(len(targetTargetIntersection)) / float64(len(targetTargetUnion))

	if math.IsNaN(targetTargetOverlap) {
		targetTargetOverlap = 0.0
	}

	//Pick the larger overlap to go with.

	overlap := targetPointerOverlap

	if targetTargetOverlap > targetPointerOverlap {
		overlap = targetTargetOverlap
		//TargetTargetOverlap is slightly better than targetPointer overlap.
		//This number will be flipped in the next step, so bigger is better.
		overlap *= 1.1
	}

	//The more overlap, the better, at an increasing rate. And the smaller the
	//output, the better the twiddle is.

	flippedOverlap := 1.0 - overlap

	//A value of 0--if there's perfect overlap--is nonsense. It's too strong!
	if flippedOverlap == 0.0 {
		flippedOverlap = 0.001
	}

	//Squaring the flipped overlap will accelerate small ones.
	return probabilityTweak(flippedOverlap * flippedOverlap)

}

//twiddleTechniqueWeight is a fundamental twiddler based on the
//HumanLikeliehood of the current technique. In fact, it's so fundamental that
//it's arguably not even a twiddler at all.
func twiddleHumanLikelihood(currentStep *SolveStep, inProgressCompoundStep []*SolveStep, pastSteps []*CompoundSolveStep, grid *Grid) probabilityTweak {
	if currentStep == nil {
		return 1.0
	}
	return probabilityTweak(currentStep.HumanLikelihood())
}

//twiddleCommonNumbers will twiddle up steps whose TargetNumbers are over-
//represented in the grid (but not DIM). This captures that humans, in
//practice, will often choose to look for cells to fill for a number that is
//represented more in the grid, since they're more likely to be constrained by
//neighorbors with the same number.
func twiddleCommonNumbers(currentStep *SolveStep, inProgressCompoundStep []*SolveStep, pastSteps []*CompoundSolveStep, grid *Grid) probabilityTweak {

	//Skip steps that aren't fill or fill multiple
	if !currentStep.Technique.IsFill() || len(currentStep.TargetNums) > 1 {
		return 1.0
	}

	keyNum := currentStep.TargetNums[0]

	count := 0

	for _, cell := range grid.Cells() {
		if cell.Number() == keyNum {
			count++
		}
	}

	if count == 0 || count == DIM {
		count = 1
	}

	//TODO: figure out if this should be on a curve.
	return probabilityTweak(count)

}

//This function will tweak weights quite a bit to make it more likely that we will pick a subsequent step that
// is 'related' to the cells modified in the last step. For example, if the
// last step had targetCells that shared a row, then a step with
//target cells in that same row will be more likely this step. This captures the fact that humans, in practice,
//will have 'chains' of steps that are all related.
func twiddleChainedSteps(currentStep *SolveStep, inProgressCompoundStep []*SolveStep, pastSteps []*CompoundSolveStep, grid *Grid) probabilityTweak {

	var lastModifiedCells CellSlice

	if len(inProgressCompoundStep) > 0 {
		lastModifiedCells = inProgressCompoundStep[len(inProgressCompoundStep)-1].TargetCells
	} else if len(pastSteps) > 0 {
		lastCompoundStep := pastSteps[len(pastSteps)-1]
		if lastCompoundStep.FillStep != nil {
			lastModifiedCells = lastCompoundStep.FillStep.TargetCells
		}
	}

	if lastModifiedCells == nil {
		return 1.0
	}

	//Tweak every weight by how related they are.
	//Remember: these are INVERTED weights, so tweaking them down is BETTER.

	//TODO: consider attentuating the effect of this; chaining is nice but shouldn't totally change the calculation for hard techniques.
	//It turns out that we probably want to STRENGTHEN the effect.
	//Logically we should be attenuating Dissimilarity here, but for some reason the math.Pow(dissimilairty, 10) doesn't actually
	//appear to work here, which is maddening.

	similarity := currentStep.TargetCells.chainSimilarity(lastModifiedCells)
	//Make sure that similarity is higher than 1 so raising 2 to this power will make it go up.
	similarity *= 10

	return probabilityTweak(math.Pow(10, similarity))

}
<commit_msg>Removed two TODOs that are now captured in #276.<commit_after>package sudoku

import (
	"math"
)

//1.0 is a no op. 0.0 to 1.0 is increase goodness; 1.0 and above is decraase good
type probabilityTweak float64

type probabilityTwiddler func(*SolveStep, []*SolveStep, []*CompoundSolveStep, *Grid) probabilityTweak

type probabilityTwiddlerItem struct {
	f    probabilityTwiddler
	name string
}

//twiddlers is the list of all of the twiddlers we should apply to change the
//probability distribution of possibilities at each step. They capture biases
//that humans have about which cells to focus on (which is separate from
//Technique.humanLikelihood, since that is about how common a technique in
//general, not in a specific context.)
var twiddlers []probabilityTwiddlerItem

func init() {
	//twiddlers is not a map because a) we need to attach more info anyway,
	//and b) we want a stable ordering.
	twiddlers = []probabilityTwiddlerItem{
		{
			f:    twiddleHumanLikelihood,
			name: "Human Likelihood",
		},
		{
			f:    twiddleChainedSteps,
			name: "Chained Steps",
		},
		{
			f:    twiddleCommonNumbers,
			name: "Common Numbers",
		},
		{
			f:    twiddlePointingTargetOverlap,
			name: "Pointing Target Overlap",
		},
	}
}

//twiddlePointingTargetOverlap twiddles based on how much the targetcells
//overlap with the pointingcells of the proposed step. This tries to capture
//the fact that for cull steps in particular, we want to heavily incentivize
//steps that directly reduce possibilities in the next round of steps. This is
//conceptually similar to ChainSimilarity, but more targeted.
func twiddlePointingTargetOverlap(currentStep *SolveStep, inProgressCompoundStep []*SolveStep, pastSteps []*CompoundSolveStep, grid *Grid) probabilityTweak {
	if len(inProgressCompoundStep) == 0 {
		return 1.0
	}
	lastStep := inProgressCompoundStep[len(inProgressCompoundStep)-1]

	if currentStep == nil {
		return 1.0
	}

	//We're going to look for two kinds of overlap: targetCell to targetCell,
	//and targetCell to PointerCell, because some techniques want one or the
	//other (do any want both?). We'll use the higher overlap.

	//Compute Target --> Pointer overlap

	targetPointerUnion := currentStep.PointerCells.Union(lastStep.TargetCells)
	targetPointerIntersection := currentStep.PointerCells.Intersection(lastStep.TargetCells)

	targetPointerOverlap := float64(len(targetPointerIntersection)) / float64(len(targetPointerUnion))

	if math.IsNaN(targetPointerOverlap) {
		targetPointerOverlap = 0.0
	}

	//Compute Target --> Target overlap

	targetTargetUnion := currentStep.TargetCells.Union(lastStep.TargetCells)
	targetTargetIntersection := currentStep.TargetCells.Intersection(lastStep.TargetCells)

	targetTargetOverlap := float64(len(targetTargetIntersection)) / float64(len(targetTargetUnion))

	if math.IsNaN(targetTargetOverlap) {
		targetTargetOverlap = 0.0
	}

	//Pick the larger overlap to go with.

	overlap := targetPointerOverlap

	if targetTargetOverlap > targetPointerOverlap {
		overlap = targetTargetOverlap
		//TargetTargetOverlap is slightly better than targetPointer overlap.
		//This number will be flipped in the next step, so bigger is better.
		overlap *= 1.1
	}

	//The more overlap, the better, at an increasing rate. And the smaller the
	//output, the better the twiddle is.

	flippedOverlap := 1.0 - overlap

	//A value of 0--if there's perfect overlap--is nonsense. It's too strong!
	if flippedOverlap == 0.0 {
		flippedOverlap = 0.001
	}

	//Squaring the flipped overlap will accelerate small ones.
	return probabilityTweak(flippedOverlap * flippedOverlap)

}

//twiddleTechniqueWeight is a fundamental twiddler based on the
//HumanLikeliehood of the current technique. In fact, it's so fundamental that
//it's arguably not even a twiddler at all.
func twiddleHumanLikelihood(currentStep *SolveStep, inProgressCompoundStep []*SolveStep, pastSteps []*CompoundSolveStep, grid *Grid) probabilityTweak {
	if currentStep == nil {
		return 1.0
	}
	return probabilityTweak(currentStep.HumanLikelihood())
}

//twiddleCommonNumbers will twiddle up steps whose TargetNumbers are over-
//represented in the grid (but not DIM). This captures that humans, in
//practice, will often choose to look for cells to fill for a number that is
//represented more in the grid, since they're more likely to be constrained by
//neighorbors with the same number.
func twiddleCommonNumbers(currentStep *SolveStep, inProgressCompoundStep []*SolveStep, pastSteps []*CompoundSolveStep, grid *Grid) probabilityTweak {

	//Skip steps that aren't fill or fill multiple
	if !currentStep.Technique.IsFill() || len(currentStep.TargetNums) > 1 {
		return 1.0
	}

	keyNum := currentStep.TargetNums[0]

	count := 0

	for _, cell := range grid.Cells() {
		if cell.Number() == keyNum {
			count++
		}
	}

	if count == 0 || count == DIM {
		count = 1
	}

	return probabilityTweak(count)

}

//This function will tweak weights quite a bit to make it more likely that we will pick a subsequent step that
// is 'related' to the cells modified in the last step. For example, if the
// last step had targetCells that shared a row, then a step with
//target cells in that same row will be more likely this step. This captures the fact that humans, in practice,
//will have 'chains' of steps that are all related.
func twiddleChainedSteps(currentStep *SolveStep, inProgressCompoundStep []*SolveStep, pastSteps []*CompoundSolveStep, grid *Grid) probabilityTweak {

	var lastModifiedCells CellSlice

	if len(inProgressCompoundStep) > 0 {
		lastModifiedCells = inProgressCompoundStep[len(inProgressCompoundStep)-1].TargetCells
	} else if len(pastSteps) > 0 {
		lastCompoundStep := pastSteps[len(pastSteps)-1]
		if lastCompoundStep.FillStep != nil {
			lastModifiedCells = lastCompoundStep.FillStep.TargetCells
		}
	}

	if lastModifiedCells == nil {
		return 1.0
	}

	//Tweak every weight by how related they are.
	//Remember: these are INVERTED weights, so tweaking them down is BETTER.

	//Logically we should be attenuating Dissimilarity here, but for some reason the math.Pow(dissimilairty, 10) doesn't actually
	//appear to work here, which is maddening.

	similarity := currentStep.TargetCells.chainSimilarity(lastModifiedCells)
	//Make sure that similarity is higher than 1 so raising 2 to this power will make it go up.
	similarity *= 10

	return probabilityTweak(math.Pow(10, similarity))

}
<|endoftext|>


<commit_before>package slugviewer

import (
	"errors"
	"flag"
	"fmt"
	"strings"
	"io/ioutil"
	"time"
)

import (
	"manhattan/commands"
	pb "code.google.com/p/goprotobuf/proto"
	tp "tritium/proto"
)

const CmdName = "slugviewer"

type SlugViewCmd struct {
	options *Options
	flags   *flag.FlagSet
}

func New() *SlugViewCmd {
	cmd := SlugViewCmd{}
	cmd.flags = flag.NewFlagSet(CmdName, flag.ContinueOnError)
	cmd.options = &Options{}

	cmd.options.SetDefaults()
	cmd.options.SetupFlags(cmd.flags)
	return &cmd
}

func (cmd *SlugViewCmd) Name() string {
	return CmdName
}

func (cmd *SlugViewCmd) Description() string {
	return "Opens up a slug for viewing."
}

func (cmd *SlugViewCmd) PrintUsage() {
	fmt.Println("Usage:")
	fmt.Println("moov " + CmdName + " <slug>")
	fmt.Println()
	fmt.Println("Flag Options:")
	cmd.flags.PrintDefaults()
}

func (cmd *SlugViewCmd) Execute(args []string) (err error) {
	args, err = commands.ParseFlags(cmd.flags, args)
	if err != nil {
		if err == flag.ErrHelp {
			return nil
		}
		return err
	}
	if len(args) > 0 {
		cmd.options.SlugLoc = args[0]
	}
	err = cmd.options.Reconcile()
	if err != nil {
		return commands.MakeHelpError(err)
	}

	var data []byte

	data, err = ioutil.ReadFile(cmd.options.SlugLoc)
	if err != nil {
		return errors.New("Problem opening the slug for reading.")
	}

	duration := time.Duration(0)
	slug := &tp.Slug{}
	for i := 0; i < 1000; i++ {
		startTime := time.Now()
		err = pb.Unmarshal(data, slug)
		duration += time.Now().Sub(startTime)
		if err != nil {
			return err
		}
	}

	if !cmd.options.TimeOnly {
		printSlug(slug, 0)
	}
	fmt.Printf("Duration of unmarshaling: %v\n", duration/time.Duration(100))

	return nil
}

func printIndent(str string, indLvl int, rest ...interface{}) {
	fmt.Printf(strings.Repeat("  ", indLvl)+str+"\n", rest...)
}

func printSlug(s *tp.Slug, indLvl int) {
	printIndent("Slug:", indLvl)
	indLvl += 1
	printIndent("Name -> %v", indLvl, s.GetName())
	printIndent("Version -> %v", indLvl, s.GetVersion())
	printIndent("Rewriter Rules:", indLvl)
	for ind, item := range s.GetRrules() {
		printIndent("Rrule[%d] -> %v", indLvl+1, ind, item)
	}
	printIndent("SSL Whitelist:", indLvl)
	for ind, item := range s.GetSslWhitelist() {
		printIndent("whitelist[%d] -> %v", indLvl+1, ind, item)
	}
	printIndent("Credentials -> %v", indLvl, s.GetCredentials())
	printIndent("Transforms:",indLvl)
	for ind, item := range s.GetTransformers() {
		printIndent("transform[%d]:", indLvl+1, ind)
		printTransform(item, indLvl+2)
	}
}

func printTransform(t *tp.Transform, indLvl int) {
	printIndent("Script Objects:", indLvl)
	for ind, item := range t.GetObjects() {
		printIndent("Script Obj[%d]:", indLvl+1, ind)
		printScriptObject(item, indLvl+2)
	}
	printIndent("Package:", indLvl)
	printPackage(t.GetPkg(), indLvl+1)
}

func printPackage(p *tp.Package, indLvl int) {
	printIndent("Name -> %v", indLvl, p.GetName())
	printIndent("Path -> %v", indLvl, p.GetPath())
	printIndent("Dependencies:", indLvl)
	for ind, item := range p.GetDependencies() {
		printIndent("Dependecny[%v] -> %v", indLvl+1, ind, item)
	}
	printIndent("Types:", indLvl)
	for ind, item := range p.GetTypes() {
		printIndent("Type[%v] -> %v", indLvl+1, ind, item)
	}
	printIndent("Functions:", indLvl)
	for ind, item := range p.GetFunctions() {
		printIndent("Function[%v]:", indLvl+1, ind)
		printFunction(item, indLvl+2)
	}
}

func printFunction(f *tp.Function, indLvl int) {
	printIndent("Name -> %v", indLvl, f.GetName())
	printIndent("Description -> %v", indLvl, f.GetDescription())
	printIndent("Filename -> %v", indLvl, f.GetFilename())
	printIndent("Line Number -> %v", indLvl, f.GetLineNumber())
	printIndent("Namespace -> %v", indLvl, f.GetNamespace())
	printIndent("Scope Type Id -> %v", indLvl, f.GetScopeTypeId())
	printIndent("Scope Type -> %v", indLvl, f.GetScopeType())
	printIndent("Return Type Id -> %v", indLvl, f.GetReturnTypeId())
	printIndent("Return Type -> %v", indLvl, f.GetReturnType())
	printIndent("Opens Type Id -> %v", indLvl, f.GetOpensTypeId())
	printIndent("Opens Type -> %v", indLvl, f.GetOpensType())
	printIndent("BuiltIn -> %v", indLvl, f.GetBuiltIn())
	printIndent("Arguments:", indLvl)
	for ind, item := range f.GetArgs() {
		printIndent("Argument[%d] -> %v", indLvl, ind, item)
	}
	printIndent("Instruction:", indLvl)
	printInstruction(f.GetInstruction(), indLvl+1)
}

func printScriptObject(so *tp.ScriptObject, indLvl int) {
	printIndent("Name -> %v", indLvl, so.GetName())
	printIndent("Scope Type -> %v", indLvl, so.GetScopeTypeId())
	printIndent("Linked -> %v", indLvl, so.GetLinked())
	printIndent("Module -> %v", indLvl, so.GetModule())
	printIndent("Root:", indLvl)
	printInstruction(so.GetRoot(), indLvl+1)
}

func printInstruction(i *tp.Instruction, indLvl int) {
	printIndent("Type -> %v", indLvl, i.GetType())
	printIndent("Value -> %v", indLvl, i.GetValue())
	printIndent("ObjectId -> %v", indLvl, i.GetObjectId())
	printIndent("Function Id -> %v", indLvl, i.GetFunctionId())
	printIndent("Line Number -> %v", indLvl, i.GetLineNumber())
	printIndent("Yield Type Id -> %v", indLvl, i.GetYieldTypeId())
	printIndent("Is Valid -> %v", indLvl, i.GetIsValid())
	printIndent("Namespace -> %v", indLvl, i.GetNamespace())
	printIndent("Type Qualifier -> %v", indLvl, i.GetTypeQualifier())
	//printIndent("Is User Called -> %v", indLvl, i.GetIsUserCalled()) //doesn't have an accessor for some reason
	printIndent("Children:", indLvl)
	for ind, item := range i.GetChildren() {
		printIndent("Child[%d]:", indLvl+1, ind)
		printInstruction(item, indLvl+2)
	}
	printIndent("Arguments:", indLvl)
	for ind, item := range i.GetArguments() {
		printIndent("Argument[%d]:", indLvl+1, ind)
		printInstruction(item, indLvl+2)
	}
}

<commit_msg>expose active layer names in slugviewer<commit_after>package slugviewer

import (
	"errors"
	"flag"
	"fmt"
	"strings"
	"io/ioutil"
	"time"
)

import (
	"manhattan/commands"
	pb "code.google.com/p/goprotobuf/proto"
	tp "tritium/proto"
)

const CmdName = "slugviewer"

type SlugViewCmd struct {
	options *Options
	flags   *flag.FlagSet
}

func New() *SlugViewCmd {
	cmd := SlugViewCmd{}
	cmd.flags = flag.NewFlagSet(CmdName, flag.ContinueOnError)
	cmd.options = &Options{}

	cmd.options.SetDefaults()
	cmd.options.SetupFlags(cmd.flags)
	return &cmd
}

func (cmd *SlugViewCmd) Name() string {
	return CmdName
}

func (cmd *SlugViewCmd) Description() string {
	return "Opens up a slug for viewing."
}

func (cmd *SlugViewCmd) PrintUsage() {
	fmt.Println("Usage:")
	fmt.Println("moov " + CmdName + " <slug>")
	fmt.Println()
	fmt.Println("Flag Options:")
	cmd.flags.PrintDefaults()
}

func (cmd *SlugViewCmd) Execute(args []string) (err error) {
	args, err = commands.ParseFlags(cmd.flags, args)
	if err != nil {
		if err == flag.ErrHelp {
			return nil
		}
		return err
	}
	if len(args) > 0 {
		cmd.options.SlugLoc = args[0]
	}
	err = cmd.options.Reconcile()
	if err != nil {
		return commands.MakeHelpError(err)
	}

	var data []byte

	data, err = ioutil.ReadFile(cmd.options.SlugLoc)
	if err != nil {
		return errors.New("Problem opening the slug for reading.")
	}

	duration := time.Duration(0)
	slug := &tp.Slug{}
	for i := 0; i < 1000; i++ {
		startTime := time.Now()
		err = pb.Unmarshal(data, slug)
		duration += time.Now().Sub(startTime)
		if err != nil {
			return err
		}
	}

	if !cmd.options.TimeOnly {
		printSlug(slug, 0)
	}
	fmt.Printf("Duration of unmarshaling: %v\n", duration/time.Duration(100))

	return nil
}

func printIndent(str string, indLvl int, rest ...interface{}) {
	fmt.Printf(strings.Repeat("  ", indLvl)+str+"\n", rest...)
}

func printSlug(s *tp.Slug, indLvl int) {
	printIndent("Slug:", indLvl)
	indLvl += 1
	printIndent("Name -> %v", indLvl, s.GetName())
	printIndent("Version -> %v", indLvl, s.GetVersion())
	printIndent("Active Layer(s) -> %v", indLvl, s.GetActiveLayers())
	printIndent("Rewriter Rules:", indLvl)
	for ind, item := range s.GetRrules() {
		printIndent("Rrule[%d] -> %v", indLvl+1, ind, item)
	}
	printIndent("SSL Whitelist:", indLvl)
	for ind, item := range s.GetSslWhitelist() {
		printIndent("whitelist[%d] -> %v", indLvl+1, ind, item)
	}
	printIndent("Credentials -> %v", indLvl, s.GetCredentials())
	printIndent("Transforms:",indLvl)
	for ind, item := range s.GetTransformers() {
		printIndent("transform[%d]:", indLvl+1, ind)
		printTransform(item, indLvl+2)
	}
}

func printTransform(t *tp.Transform, indLvl int) {
	printIndent("Script Objects:", indLvl)
	for ind, item := range t.GetObjects() {
		printIndent("Script Obj[%d]:", indLvl+1, ind)
		printScriptObject(item, indLvl+2)
	}
	printIndent("Package:", indLvl)
	printPackage(t.GetPkg(), indLvl+1)
}

func printPackage(p *tp.Package, indLvl int) {
	printIndent("Name -> %v", indLvl, p.GetName())
	printIndent("Path -> %v", indLvl, p.GetPath())
	printIndent("Dependencies:", indLvl)
	for ind, item := range p.GetDependencies() {
		printIndent("Dependecny[%v] -> %v", indLvl+1, ind, item)
	}
	printIndent("Types:", indLvl)
	for ind, item := range p.GetTypes() {
		printIndent("Type[%v] -> %v", indLvl+1, ind, item)
	}
	printIndent("Functions:", indLvl)
	for ind, item := range p.GetFunctions() {
		printIndent("Function[%v]:", indLvl+1, ind)
		printFunction(item, indLvl+2)
	}
}

func printFunction(f *tp.Function, indLvl int) {
	printIndent("Name -> %v", indLvl, f.GetName())
	printIndent("Description -> %v", indLvl, f.GetDescription())
	printIndent("Filename -> %v", indLvl, f.GetFilename())
	printIndent("Line Number -> %v", indLvl, f.GetLineNumber())
	printIndent("Namespace -> %v", indLvl, f.GetNamespace())
	printIndent("Scope Type Id -> %v", indLvl, f.GetScopeTypeId())
	printIndent("Scope Type -> %v", indLvl, f.GetScopeType())
	printIndent("Return Type Id -> %v", indLvl, f.GetReturnTypeId())
	printIndent("Return Type -> %v", indLvl, f.GetReturnType())
	printIndent("Opens Type Id -> %v", indLvl, f.GetOpensTypeId())
	printIndent("Opens Type -> %v", indLvl, f.GetOpensType())
	printIndent("BuiltIn -> %v", indLvl, f.GetBuiltIn())
	printIndent("Arguments:", indLvl)
	for ind, item := range f.GetArgs() {
		printIndent("Argument[%d] -> %v", indLvl, ind, item)
	}
	printIndent("Instruction:", indLvl)
	printInstruction(f.GetInstruction(), indLvl+1)
}

func printScriptObject(so *tp.ScriptObject, indLvl int) {
	printIndent("Name -> %v", indLvl, so.GetName())
	printIndent("Scope Type -> %v", indLvl, so.GetScopeTypeId())
	printIndent("Linked -> %v", indLvl, so.GetLinked())
	printIndent("Module -> %v", indLvl, so.GetModule())
	printIndent("Root:", indLvl)
	printInstruction(so.GetRoot(), indLvl+1)
}

func printInstruction(i *tp.Instruction, indLvl int) {
	printIndent("Type -> %v", indLvl, i.GetType())
	printIndent("Value -> %v", indLvl, i.GetValue())
	printIndent("ObjectId -> %v", indLvl, i.GetObjectId())
	printIndent("Function Id -> %v", indLvl, i.GetFunctionId())
	printIndent("Line Number -> %v", indLvl, i.GetLineNumber())
	printIndent("Yield Type Id -> %v", indLvl, i.GetYieldTypeId())
	printIndent("Is Valid -> %v", indLvl, i.GetIsValid())
	printIndent("Namespace -> %v", indLvl, i.GetNamespace())
	printIndent("Type Qualifier -> %v", indLvl, i.GetTypeQualifier())
	//printIndent("Is User Called -> %v", indLvl, i.GetIsUserCalled()) //doesn't have an accessor for some reason
	printIndent("Children:", indLvl)
	for ind, item := range i.GetChildren() {
		printIndent("Child[%d]:", indLvl+1, ind)
		printInstruction(item, indLvl+2)
	}
	printIndent("Arguments:", indLvl)
	for ind, item := range i.GetArguments() {
		printIndent("Argument[%d]:", indLvl+1, ind)
		printInstruction(item, indLvl+2)
	}
}

<|endoftext|>


<commit_before>// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pubsub

import (
	"errors"
	"fmt"
	"sync/atomic"
	"testing"
	"time"

	"golang.org/x/net/context"
	"golang.org/x/sync/errgroup"
)

func TestFlowControllerCancel(t *testing.T) {
	// Test canceling a flow controller's context.
	t.Parallel()
	fc := newFlowController(3, 10)
	if err := fc.acquire(context.Background(), 5); err != nil {
		t.Fatal(err)
	}
	// Experiment: a context that times out should always return an error.
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Millisecond)
	defer cancel()
	if err := fc.acquire(ctx, 6); err != context.DeadlineExceeded {
		t.Fatalf("got %v, expected DeadlineExceeded", err)
	}
	// Control: a context that is not done should always return nil.
	go func() {
		time.Sleep(5 * time.Millisecond)
		fc.release(5)
	}()
	if err := fc.acquire(context.Background(), 6); err != nil {
		t.Errorf("got %v, expected nil", err)
	}
}

func TestFlowControllerLargeRequest(t *testing.T) {
	// Large requests succeed, consuming the entire allotment.
	t.Parallel()
	fc := newFlowController(3, 10)
	err := fc.acquire(context.Background(), 11)
	if err != nil {
		t.Fatal(err)
	}
}

func TestFlowControllerNoStarve(t *testing.T) {
	// A large request won't starve, because the flowController is
	// (best-effort) FIFO.
	t.Parallel()
	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
	defer cancel()
	fc := newFlowController(10, 10)
	first := make(chan int)
	for i := 0; i < 20; i++ {
		go func() {
			for {
				if err := fc.acquire(ctx, 1); err != nil {
					if err != context.Canceled {
						t.Error(err)
					}
					return
				}
				select {
				case first <- 1:
				default:
				}
				fc.release(1)
			}
		}()
	}
	<-first // Wait until the flowController's state is non-zero.
	if err := fc.acquire(ctx, 11); err != nil {
		t.Errorf("got %v, want nil", err)
	}
}

func TestFlowControllerSaturation(t *testing.T) {
	t.Parallel()
	const (
		maxCount = 6
		maxSize  = 10
	)
	for _, test := range []struct {
		acquireSize         int
		wantCount, wantSize int64
	}{
		{
			// Many small acquires cause the flow controller to reach its max count.
			acquireSize: 1,
			wantCount:   6,
			wantSize:    6,
		},
		{
			// Five acquires of size 2 will cause the flow controller to reach its max size,
			// but not its max count.
			acquireSize: 2,
			wantCount:   5,
			wantSize:    10,
		},
		{
			// If the requests are the right size (relatively prime to maxSize),
			// the flow controller will not saturate on size. (In this case, not on count either.)
			acquireSize: 3,
			wantCount:   3,
			wantSize:    9,
		},
	} {
		fc := newFlowController(maxCount, maxSize)
		// Atomically track flow controller state.
		var curCount, curSize int64
		success := errors.New("")
		// Time out if wantSize or wantCount is never reached.
		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
		defer cancel()
		g, ctx := errgroup.WithContext(ctx)
		for i := 0; i < 10; i++ {
			g.Go(func() error {
				var hitCount, hitSize bool
				// Run at least until we hit the expected values, and at least
				// for enough iterations to exceed them if the flow controller
				// is broken.
				for i := 0; i < 100 || !hitCount || !hitSize; i++ {
					select {
					case <-ctx.Done():
						return ctx.Err()
					default:
					}
					if err := fc.acquire(ctx, test.acquireSize); err != nil {
						return err
					}
					c := atomic.AddInt64(&curCount, 1)
					if c > test.wantCount {
						return fmt.Errorf("count %d exceeds want %d", c, test.wantCount)
					}
					if c == test.wantCount {
						hitCount = true
					}
					s := atomic.AddInt64(&curSize, int64(test.acquireSize))
					if s > test.wantSize {
						return fmt.Errorf("size %d exceeds want %d", s, test.wantSize)
					}
					if s == test.wantSize {
						hitSize = true
					}
					time.Sleep(5 * time.Millisecond) // Let other goroutines make progress.
					if atomic.AddInt64(&curCount, -1) < 0 {
						return errors.New("negative count")
					}
					if atomic.AddInt64(&curSize, -int64(test.acquireSize)) < 0 {
						return errors.New("negative size")
					}
					fc.release(test.acquireSize)
				}
				return success
			})
		}
		if err := g.Wait(); err != success {
			t.Errorf("%+v: %v", test, err)
			continue
		}
	}
}

func TestFlowControllerTryAcquire(t *testing.T) {
	fc := newFlowController(3, 10)

	// Successfully tryAcquire 4 bytes.
	if !fc.tryAcquire(4) {
		t.Error("got false, wanted true")
	}

	// Fail to tryAcquire 7 bytes.
	if fc.tryAcquire(7) {
		t.Error("got true, wanted false")
	}

	// Successfully tryAcquire 6 byte.
	if !fc.tryAcquire(6) {
		t.Error("got false, wanted true")
	}
}

func TestFlowControllerUnboundedCount(t *testing.T) {
	ctx := context.Background()
	fc := newFlowController(0, 10)

	// Successfully acquire 4 bytes.
	if err := fc.acquire(ctx, 4); err != nil {
		t.Errorf("got %v, wanted no error")
	}

	// Successfully tryAcquire 4 bytes.
	if !fc.tryAcquire(4) {
		t.Error("got false, wanted true")
	}

	// Fail to tryAcquire 3 bytes.
	if fc.tryAcquire(3) {
		t.Error("got true, wanted false")
	}
}

func TestFlowControllerUnboundedBytes(t *testing.T) {
	ctx := context.Background()
	fc := newFlowController(2, 0)

	// Successfully acquire 4GB.
	if err := fc.acquire(ctx, 4e9); err != nil {
		t.Errorf("got %v, wanted no error")
	}

	// Successfully tryAcquire 4GB bytes.
	if !fc.tryAcquire(4e9) {
		t.Error("got false, wanted true")
	}

	// Fail to tryAcquire a third message.
	if fc.tryAcquire(3) {
		t.Error("got true, wanted false")
	}
}
<commit_msg>pubsub: fix missing args in format<commit_after>// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pubsub

import (
	"errors"
	"fmt"
	"sync/atomic"
	"testing"
	"time"

	"golang.org/x/net/context"
	"golang.org/x/sync/errgroup"
)

func TestFlowControllerCancel(t *testing.T) {
	// Test canceling a flow controller's context.
	t.Parallel()
	fc := newFlowController(3, 10)
	if err := fc.acquire(context.Background(), 5); err != nil {
		t.Fatal(err)
	}
	// Experiment: a context that times out should always return an error.
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Millisecond)
	defer cancel()
	if err := fc.acquire(ctx, 6); err != context.DeadlineExceeded {
		t.Fatalf("got %v, expected DeadlineExceeded", err)
	}
	// Control: a context that is not done should always return nil.
	go func() {
		time.Sleep(5 * time.Millisecond)
		fc.release(5)
	}()
	if err := fc.acquire(context.Background(), 6); err != nil {
		t.Errorf("got %v, expected nil", err)
	}
}

func TestFlowControllerLargeRequest(t *testing.T) {
	// Large requests succeed, consuming the entire allotment.
	t.Parallel()
	fc := newFlowController(3, 10)
	err := fc.acquire(context.Background(), 11)
	if err != nil {
		t.Fatal(err)
	}
}

func TestFlowControllerNoStarve(t *testing.T) {
	// A large request won't starve, because the flowController is
	// (best-effort) FIFO.
	t.Parallel()
	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
	defer cancel()
	fc := newFlowController(10, 10)
	first := make(chan int)
	for i := 0; i < 20; i++ {
		go func() {
			for {
				if err := fc.acquire(ctx, 1); err != nil {
					if err != context.Canceled {
						t.Error(err)
					}
					return
				}
				select {
				case first <- 1:
				default:
				}
				fc.release(1)
			}
		}()
	}
	<-first // Wait until the flowController's state is non-zero.
	if err := fc.acquire(ctx, 11); err != nil {
		t.Errorf("got %v, want nil", err)
	}
}

func TestFlowControllerSaturation(t *testing.T) {
	t.Parallel()
	const (
		maxCount = 6
		maxSize  = 10
	)
	for _, test := range []struct {
		acquireSize         int
		wantCount, wantSize int64
	}{
		{
			// Many small acquires cause the flow controller to reach its max count.
			acquireSize: 1,
			wantCount:   6,
			wantSize:    6,
		},
		{
			// Five acquires of size 2 will cause the flow controller to reach its max size,
			// but not its max count.
			acquireSize: 2,
			wantCount:   5,
			wantSize:    10,
		},
		{
			// If the requests are the right size (relatively prime to maxSize),
			// the flow controller will not saturate on size. (In this case, not on count either.)
			acquireSize: 3,
			wantCount:   3,
			wantSize:    9,
		},
	} {
		fc := newFlowController(maxCount, maxSize)
		// Atomically track flow controller state.
		var curCount, curSize int64
		success := errors.New("")
		// Time out if wantSize or wantCount is never reached.
		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
		defer cancel()
		g, ctx := errgroup.WithContext(ctx)
		for i := 0; i < 10; i++ {
			g.Go(func() error {
				var hitCount, hitSize bool
				// Run at least until we hit the expected values, and at least
				// for enough iterations to exceed them if the flow controller
				// is broken.
				for i := 0; i < 100 || !hitCount || !hitSize; i++ {
					select {
					case <-ctx.Done():
						return ctx.Err()
					default:
					}
					if err := fc.acquire(ctx, test.acquireSize); err != nil {
						return err
					}
					c := atomic.AddInt64(&curCount, 1)
					if c > test.wantCount {
						return fmt.Errorf("count %d exceeds want %d", c, test.wantCount)
					}
					if c == test.wantCount {
						hitCount = true
					}
					s := atomic.AddInt64(&curSize, int64(test.acquireSize))
					if s > test.wantSize {
						return fmt.Errorf("size %d exceeds want %d", s, test.wantSize)
					}
					if s == test.wantSize {
						hitSize = true
					}
					time.Sleep(5 * time.Millisecond) // Let other goroutines make progress.
					if atomic.AddInt64(&curCount, -1) < 0 {
						return errors.New("negative count")
					}
					if atomic.AddInt64(&curSize, -int64(test.acquireSize)) < 0 {
						return errors.New("negative size")
					}
					fc.release(test.acquireSize)
				}
				return success
			})
		}
		if err := g.Wait(); err != success {
			t.Errorf("%+v: %v", test, err)
			continue
		}
	}
}

func TestFlowControllerTryAcquire(t *testing.T) {
	fc := newFlowController(3, 10)

	// Successfully tryAcquire 4 bytes.
	if !fc.tryAcquire(4) {
		t.Error("got false, wanted true")
	}

	// Fail to tryAcquire 7 bytes.
	if fc.tryAcquire(7) {
		t.Error("got true, wanted false")
	}

	// Successfully tryAcquire 6 byte.
	if !fc.tryAcquire(6) {
		t.Error("got false, wanted true")
	}
}

func TestFlowControllerUnboundedCount(t *testing.T) {
	ctx := context.Background()
	fc := newFlowController(0, 10)

	// Successfully acquire 4 bytes.
	if err := fc.acquire(ctx, 4); err != nil {
		t.Errorf("got %v, wanted no error", err)
	}

	// Successfully tryAcquire 4 bytes.
	if !fc.tryAcquire(4) {
		t.Error("got false, wanted true")
	}

	// Fail to tryAcquire 3 bytes.
	if fc.tryAcquire(3) {
		t.Error("got true, wanted false")
	}
}

func TestFlowControllerUnboundedBytes(t *testing.T) {
	ctx := context.Background()
	fc := newFlowController(2, 0)

	// Successfully acquire 4GB.
	if err := fc.acquire(ctx, 4e9); err != nil {
		t.Errorf("got %v, wanted no error", err)
	}

	// Successfully tryAcquire 4GB bytes.
	if !fc.tryAcquire(4e9) {
		t.Error("got false, wanted true")
	}

	// Fail to tryAcquire a third message.
	if fc.tryAcquire(3) {
		t.Error("got true, wanted false")
	}
}
<|endoftext|>


<commit_before>package raftgorums_test

import (
	"io/ioutil"
	"math/rand"
	"os"
	"reflect"
	"testing"

	"github.com/relab/raft/commonpb"
	"github.com/relab/raft/raftgorums"
)

func newFileStorage(t testing.TB, overwrite bool, filepath ...string) (fs *raftgorums.FileStorage, path string, cleanup func()) {
	var dbfile string

	if len(filepath) < 1 {
		file, err := ioutil.TempFile("", "bolt")

		if err != nil {
			t.Fatal(err)
		}

		dbfile = file.Name()
	} else {
		dbfile = filepath[0]
	}

	storage, err := raftgorums.NewFileStorage(dbfile, overwrite)

	if err != nil {
		t.Fatal(err)
	}

	return storage, dbfile, func() {
		if err := os.Remove(dbfile); err != nil {
			t.Fatal(err)
		}
	}
}

func TestNewFileStorage(t *testing.T) {
	recover := false

	// Create storage on path.
	_, path, _ := newFileStorage(t, !recover)

	// Recover from path, where file exists.
	_, _, cleanup2 := newFileStorage(t, recover, path)
	cleanup2()

	// Recover from path, where file doesn't exist.
	newFileStorage(t, recover, path)

	// Overwrite path, where file exists.
	_, _, cleanup3 := newFileStorage(t, !recover, path)
	cleanup3()

	// Overwrite path, where file doesn't exist.
	_, _, cleanup4 := newFileStorage(t, !recover, path)
	cleanup4()

	if _, err := os.Stat(path); err == nil {
		t.Errorf("got %s exists, want %s removed", path, path)
	}
}

func TestFileStorageStoreValue(t *testing.T) {
	var storage raftgorums.Storage
	storage, _, cleanup := newFileStorage(t, true)
	defer cleanup()

	var expected uint64 = 5

	err := storage.Set(raftgorums.KeyTerm, expected)

	if err != nil {
		t.Fatal(err)
	}

	got, err := storage.Get(raftgorums.KeyTerm)

	if err != nil {
		t.Fatal(err)
	}

	if !reflect.DeepEqual(got, expected) {
		t.Errorf("got %+v, want %+v", got, expected)
	}
}

func TestFileStorageStoreEntry(t *testing.T) {
	var storage raftgorums.Storage
	storage, _, cleanup := newFileStorage(t, true)
	defer cleanup()

	expected := &commonpb.Entry{Term: 5}

	err := storage.StoreEntries([]*commonpb.Entry{expected})

	if err != nil {
		t.Fatal(err)
	}

	got, err := storage.GetEntry(0)

	if err != nil {
		t.Fatal(err)
	}

	if !reflect.DeepEqual(got, expected) {
		t.Errorf("got %+v, want %+v", got, expected)
	}
}

func BenchmarkSnapshot(b *testing.B) {
	storage, _, cleanup := newFileStorage(b, true, "benchsnap.bolt")
	defer cleanup()

	// 200kb.
	data := make([]byte, 200000)
	rand.Read(data)

	snapshot := &commonpb.Snapshot{Data: data}

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		if err := storage.SetSnapshot(snapshot); err != nil {
			b.Error(err)
		}
	}
}
<commit_msg>raftgorums/filestorage_test.go: Benchmark StoreEntries<commit_after>package raftgorums_test

import (
	"fmt"
	"io/ioutil"
	"math"
	"math/rand"
	"os"
	"reflect"
	"testing"

	"github.com/relab/raft/commonpb"
	"github.com/relab/raft/raftgorums"
)

func newFileStorage(t testing.TB, overwrite bool, filepath ...string) (fs *raftgorums.FileStorage, path string, cleanup func()) {
	var dbfile string

	if len(filepath) < 1 {
		file, err := ioutil.TempFile("", "bolt")

		if err != nil {
			t.Fatal(err)
		}

		dbfile = file.Name()
	} else {
		dbfile = filepath[0]
	}

	storage, err := raftgorums.NewFileStorage(dbfile, overwrite)

	if err != nil {
		t.Fatal(err)
	}

	return storage, dbfile, func() {
		if err := os.Remove(dbfile); err != nil {
			t.Fatal(err)
		}
	}
}

func TestNewFileStorage(t *testing.T) {
	recover := false

	// Create storage on path.
	_, path, _ := newFileStorage(t, !recover)

	// Recover from path, where file exists.
	_, _, cleanup2 := newFileStorage(t, recover, path)
	cleanup2()

	// Recover from path, where file doesn't exist.
	newFileStorage(t, recover, path)

	// Overwrite path, where file exists.
	_, _, cleanup3 := newFileStorage(t, !recover, path)
	cleanup3()

	// Overwrite path, where file doesn't exist.
	_, _, cleanup4 := newFileStorage(t, !recover, path)
	cleanup4()

	if _, err := os.Stat(path); err == nil {
		t.Errorf("got %s exists, want %s removed", path, path)
	}
}

func TestFileStorageStoreValue(t *testing.T) {
	var storage raftgorums.Storage
	storage, _, cleanup := newFileStorage(t, true)
	defer cleanup()

	var expected uint64 = 5

	err := storage.Set(raftgorums.KeyTerm, expected)

	if err != nil {
		t.Fatal(err)
	}

	got, err := storage.Get(raftgorums.KeyTerm)

	if err != nil {
		t.Fatal(err)
	}

	if !reflect.DeepEqual(got, expected) {
		t.Errorf("got %+v, want %+v", got, expected)
	}
}

func TestFileStorageStoreEntry(t *testing.T) {
	var storage raftgorums.Storage
	storage, _, cleanup := newFileStorage(t, true)
	defer cleanup()

	expected := &commonpb.Entry{Term: 5}

	err := storage.StoreEntries([]*commonpb.Entry{expected})

	if err != nil {
		t.Fatal(err)
	}

	got, err := storage.GetEntry(0)

	if err != nil {
		t.Fatal(err)
	}

	if !reflect.DeepEqual(got, expected) {
		t.Errorf("got %+v, want %+v", got, expected)
	}
}

func BenchmarkSnapshot(b *testing.B) {
	storage, _, cleanup := newFileStorage(b, true, "benchsnap.bolt")
	defer cleanup()

	// 200kb.
	data := make([]byte, 200000)
	rand.Read(data)

	snapshot := &commonpb.Snapshot{Data: data}

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		if err := storage.SetSnapshot(snapshot); err != nil {
			b.Error(err)
		}
	}
}

func BenchmarkThroughput(b *testing.B) {
	rand.Seed(500)

	type benchmark struct {
		name           string
		numEntries     int
		payloadInBytes int
	}

	var benchmarks []benchmark

	for i := 0; i < 5; i++ {
		for _, payload := range []int{10, 50, 100, 200, 500, 1000} {
			numEntries := int(math.Pow(10, float64(i)))
			name := fmt.Sprintf("%d entries with %d bytes", numEntries, payload)
			benchmarks = append(benchmarks, benchmark{
				name,
				numEntries,
				payload,
			})
		}
	}

	for _, bm := range benchmarks {
		storage, _, cleanup := newFileStorage(b, true, "benchthroughput.bolt")
		entries := make([]*commonpb.Entry, bm.numEntries)

		for i := 0; i < bm.numEntries; i++ {
			b := make([]byte, bm.payloadInBytes)
			rand.Read(b)
			entries[i] = &commonpb.Entry{Data: b}
		}

		b.Run(bm.name, func(b *testing.B) {
			for i := 0; i < b.N; i++ {
				if err := storage.StoreEntries(entries); err != nil {
					b.Error(err)
				}
			}
		})

		cleanup()
	}
}
<|endoftext|>


<commit_before>// package ui provides the UI components of Goed.
package ui

import (
	"fmt"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"

	"github.com/tcolar/goed/actions"
	"github.com/tcolar/goed/backend"
	"github.com/tcolar/goed/core"
	"github.com/tcolar/goed/event"
)

var _ core.Editable = (*Editor)(nil)

// Editor is goed's main Editor pane (singleton)
type Editor struct {
	Cmdbar      *Cmdbar
	config      *core.Config
	Statusbar   *Statusbar
	Fg, Bg      core.Style
	theme       *core.Theme
	Cols        []*Col
	curViewId   int64
	CurCol      *Col
	cmdOn       bool
	term        core.Term
	views       map[int64]*View
	fileWatcher *event.FileWatcher
}

func NewEditor(term core.Term, config *core.Config) *Editor {
	return &Editor{
		term:        term,
		config:      config,
		views:       map[int64]*View{},
		fileWatcher: event.NewFileWatcher(),
	}
}

// Editor with Mock terminal for testing
func NewMockEditor() *Editor {
	return &Editor{
		term:   core.NewMockTerm(),
		config: core.LoadConfig("config.toml"),
		views:  map[int64]*View{},
	}
}

func (e *Editor) Dispatch(action core.Action) {
	core.Bus.Dispatch(action)
}

func (e *Editor) Commandbar() core.Commander {
	return e.Cmdbar
}

func (e *Editor) Quit() {
	if e.fileWatcher != nil {
		e.fileWatcher.Stop()
	}
	event.Shutdown()
	e.term.Close()
	os.Exit(0)
}

// Start starts-up the editor
func (e *Editor) Start(locs []string) {
	err := e.term.Init()
	if err != nil {
		panic(err)
	}

	e.term.SetExtendedColors(core.Colors == 256)
	e.theme, err = core.ReadTheme(core.FindResource(path.Join("themes", e.config.Theme)))
	if err != nil {
		panic(err)
	}
	e.Fg = e.theme.Fg
	e.Bg = e.theme.Bg

	h, w := e.term.Size()
	e.Cmdbar = &Cmdbar{}
	e.Cmdbar.SetBounds(0, 0, 0, w)
	e.Statusbar = &Statusbar{}
	e.Statusbar.SetBounds(h-1, 0, h-1, w)
	dirs := []string{}
	files := []string{}
	for _, loc := range locs {
		if stat, err := os.Stat(loc); err == nil && stat.IsDir() {
			dirs = append(dirs, loc)
		} else {
			files = append(files, loc)
		}
	}
	if len(dirs) == 0 {
		if len(files) > 0 {
			dirs = []string{path.Dir(locs[0])}
		} else {
			dirs = []string{"."}
		}
	}
	e.Cols = append(e.Cols, &Col{WidthRatio: 1.0})
	ratio := 1.0 / float64(len(dirs))
	for _, dir := range dirs {
		view := e.NewView(dir)
		view.HeightRatio = ratio
		e.Cols[0].Views = append(e.Cols[0].Views, view.Id())
		e.Open(dir, view.Id(), "", true)
	}
	e.CurCol = e.Cols[0]
	e.curViewId = e.CurCol.Views[0]
	if len(files) > 0 {
		e.CurCol.WidthRatio = 0.2
		c := &Col{WidthRatio: 0.8}
		ratio := 1.0 / float64(len(files))
		for _, f := range files {
			view := e.NewView(f)
			view.HeightRatio = ratio
			c.Views = append(c.Views, view.Id())
			e.Open(f, view.Id(), "", true)
		}
		e.Cols = append(e.Cols, c)
		e.CurCol = c
		e.curViewId = c.Views[0]
	}

	actions.Ar.EdResize(e.term.Size())

	actions.Ar.EdRender()

	if e.fileWatcher != nil {
		go e.fileWatcher.Start()
	}

	go core.Bus.Start()

	go e.autoScroller()

	go event.Listen()

	e.term.Listen()
}

// Open opens a given location in the editor (in the given view)
// or new view if viewId < 0
func (e *Editor) Open(loc string, viewId int64, rel string, create bool) (int64, error) {
	loc = strings.TrimSpace(loc)
	rel = strings.TrimSpace(rel)
	if len(rel) > 0 && !strings.HasPrefix(loc, string(os.PathSeparator)) {
		loc = path.Join(rel, loc)
	}
	// make it absolute
	loc, err := filepath.Abs(loc)
	if err != nil {
		return -1, err
	}
	stat, err := os.Stat(loc)
	newFile := false
	if os.IsNotExist(err) {
		if !create {
			return -1, err
		}
		newFile = true
	}
	title := filepath.Base(loc)
	if !newFile && stat.IsDir() {
		loc += string(os.PathSeparator)
		title += string(os.PathSeparator)
	}
	nv := false
	var view core.Viewable
	if viewId < 0 {
		view = e.NewFileView(loc)
		nv = true
	} else {
		view = e.ViewById(viewId)
	}
	view.Reset()
	view.SetTitle(title)
	if newFile || !stat.IsDir() {
		err = e.openFile(loc, view)
	} else {
		err = e.openDir(loc, view)
	}
	if err != nil {
		return -1, err
	}
	if nv {
		if stat != nil && stat.IsDir() {
			e.AddDirViewSmart(viewCast(view))
		} else {
			e.InsertViewSmart(viewCast(view))
		}
	}
	view.Reset()
	view.SetWorkDir(filepath.Dir(loc))
	return view.Id(), nil
}

// OpenDir opens a directory listing
func (e *Editor) openDir(loc string, view core.Viewable) error {
	v := viewCast(view)
	if v == nil {
		return fmt.Errorf("No such view")
	}
	v.highlighter = &TermHighlighter{}
	args := append([]string{"ls"}, core.OsLsArgs...)
	title := filepath.Base(loc) + "/"
	backend, err := backend.NewMemBackendCmd(args, loc, view.Id(), &title, true)
	if err != nil {
		return err
	}
	actions.Ar.ViewSetType(v.Id(), core.ViewTypeDirListing)
	view.SetBackend(backend)
	e.SetStatus(fmt.Sprintf("%v", view.WorkDir()))
	return nil
}

// OpenFile opens a file in the editor
func (e *Editor) openFile(loc string, view core.Viewable) error {
	b, err := backend.NewFileBackend(loc, view.Id())
	if err != nil {
		return err
	}
	view.SetBackend(b)
	e.SetStatus(fmt.Sprintf("%v  [%d]", view.WorkDir(), view.Id()))
	view.SetDirty(false)
	e.ViewActivate(view.Id())
	return nil
}

func (e *Editor) Render() {
	e.TermFB(e.theme.Fg, e.theme.Bg)
	e.term.Clear(e.Bg, e.Bg)

	for _, c := range e.Cols {
		for _, v := range c.Views {
			e.ViewById(v).Render()
		}
	}

	// cursor
	v := viewCast(e.CurView())
	cc, cl := v.CurCol(), v.CurLine()
	c, _, _ := v.CurChar()
	// With some terminals & color schemes the cursor might be "invisible" if we are at a
	// location with no text (ie: end of line)
	// so in that case put as space there to cause the cursor to appear.
	var car = ' '
	if c != nil {
		car = *c
	}
	// Note the terminal inverts the colors where the cursor is
	// this is why this statement might appear "backward"
	e.TermFB(e.theme.BgCursor, e.theme.FgCursor)
	y1, x1, _, _ := v.Bounds()
	e.TermChar(cl+y1-v.offy+2, cc+x1-v.offx+2, car)
	e.TermFB(e.theme.Fg, e.theme.Bg)

	e.Cmdbar.Render()
	e.Statusbar.Render()

	e.TermFlush()
}

func (e *Editor) SetStatusErr(s string) {
	if e.Statusbar == nil {
		return
	}
	e.Statusbar.msg = s
	e.Statusbar.isErr = true
	e.Statusbar.Render()
}

func (e *Editor) SetStatus(s string) {
	if e.Statusbar == nil {
		return
	}
	e.Statusbar.msg = s
	e.Statusbar.msg = s
	e.Statusbar.isErr = false
	e.Statusbar.Render()
}

func (e *Editor) Config() core.Config {
	return *e.config
}

func (e *Editor) Theme() *core.Theme {
	return e.theme
}

func (e *Editor) CurView() core.Viewable {
	v, found := e.views[e.curViewId]
	if !found {
		return e.views[e.Cols[0].Views[0]]
	}
	return v
}

func (e *Editor) CurViewId() int64 {
	return e.curViewId
}

func (e *Editor) SetCursor(y, x int) {
	e.term.SetCursor(x, y)
}

func (e *Editor) CmdOn() bool {
	return e.cmdOn
}

func (e *Editor) SetCmdOn(v bool) {
	e.cmdOn = v
}

func (e *Editor) TermFlush() {
	e.term.Flush()
}

// true if ok to quit
func (e *Editor) QuitCheck() bool {
	for _, c := range e.Cols {
		for _, vi := range c.Views {
			v, found := e.views[vi]
			if found && !v.canClose() {
				return false
			}
		}
	}
	return true
}

func (e *Editor) FileEvent(op core.FileOp, loc string) {
	for _, v := range e.views {
		v.fileEvent(op, loc)
	}
}

func (e *Editor) StartTermView(args []string) int64 {
	vid := exec(args, true)
	v := viewCast(core.Ed.ViewById(vid))
	if v == nil || v.backend == nil {
		return -1
	}
	// source the goed shell script onc eterminal has launched
	b := v.backend.(*backend.BackendCmd)
	ext := ".sh"
	if os.Getenv("SHELL") == "rc" {
		ext = ".rc"
	}
	if os.Getenv("SHELL") == "fish" {
		ext = ".fish"
	}
	cmd := ". $HOME/.goed/default/actions/goed" +
		fmt.Sprintf("%s %d %d\n", ext, core.InstanceId, v.Id())
	go func(cmd string) {
		started := b.WaitRunning(time.Minute)
		if !started {
			return
		}
		end := time.Now().Add(time.Hour).Unix()
		for time.Now().Unix() < end {
			if !b.SubCmdRunning() {
				b.SendBytes([]byte(cmd))
				return
			}
			time.Sleep(50 * time.Millisecond)
		}
	}(cmd)
	return vid
}

// Handle selection auto scrolling of views
func (e *Editor) autoScroller() {
	action := autoScrollAction{}
	pause := 200 * time.Millisecond
	for {
		core.Bus.Dispatch(action)
		time.Sleep(pause)
	}
}

type autoScrollAction struct {
}

func (e autoScrollAction) Run() {
	v := viewCast(core.Ed.ViewById(core.Ed.CurViewId()))
	if v == nil {
		return
	}
	x, y := v.autoScrollX, v.autoScrollY
	if x == 0 && y == 0 {
		return
	}
	if len(v.selections) == 0 {
		return
	}
	s := v.selections[0]
	ln := v.CurLine()
	v.offx += x
	v.offy += y
	if y > 0 {
		s.LineTo += y
	} else {
		s.LineFrom += y
	}
	if x > 0 {
		s.ColTo += x
	} else {
		s.ColFrom += x
	}
	// handle scroll / selection "overflows"
	lnLen := v.LineLen(v.Slice(), ln)
	if v.offy >= v.LineCount()-v.LastViewLine() {
		v.offy = v.LineCount() - v.LastViewLine()
	}
	if v.offy < 0 {
		v.offy = 0
	}
	if v.offx > lnLen-v.LastViewCol() {
		v.offx = lnLen - v.LastViewCol()
	}
	if v.offx < 0 {
		v.offx = 0
	}
	if s.LineFrom < 0 {
		s.LineFrom = 0
	} else if s.LineFrom > v.LineCount() {
		s.LineFrom = v.LineCount()
	}
	if s.LineTo < 0 {
		s.LineTo = 0
	} else if s.LineTo > v.LineCount() {
		s.LineTo = v.LineCount()
	}
	if s.ColFrom < 0 {
		s.ColFrom = 0
	} else if s.ColFrom > lnLen {
		s.ColFrom = lnLen
	}
	if s.ColTo < 0 {
		s.ColTo = 0
	} else if s.ColTo > lnLen {
		s.ColTo = lnLen
	}
	s.Normalize()
	v.selections = []core.Selection{
		s,
	}
	core.Ed.Render()
}

// TODO: Do away with those ugly assertions
func viewCast(v core.Viewable) *View {
	if v == nil {
		return nil
	}
	return v.(*View)
}

func widgetCast(w Renderer) *View {
	if w == nil {
		return nil
	}
	if v, ok := w.(*View); ok {
		return v
	}
	return nil
}
<commit_msg>For OpenFile, if already opened in a view, reuse it<commit_after>// package ui provides the UI components of Goed.
package ui

import (
	"fmt"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"

	"github.com/tcolar/goed/actions"
	"github.com/tcolar/goed/backend"
	"github.com/tcolar/goed/core"
	"github.com/tcolar/goed/event"
)

var _ core.Editable = (*Editor)(nil)

// Editor is goed's main Editor pane (singleton)
type Editor struct {
	Cmdbar      *Cmdbar
	config      *core.Config
	Statusbar   *Statusbar
	Fg, Bg      core.Style
	theme       *core.Theme
	Cols        []*Col
	curViewId   int64
	CurCol      *Col
	cmdOn       bool
	term        core.Term
	views       map[int64]*View
	fileWatcher *event.FileWatcher
}

func NewEditor(term core.Term, config *core.Config) *Editor {
	return &Editor{
		term:        term,
		config:      config,
		views:       map[int64]*View{},
		fileWatcher: event.NewFileWatcher(),
	}
}

// Editor with Mock terminal for testing
func NewMockEditor() *Editor {
	return &Editor{
		term:   core.NewMockTerm(),
		config: core.LoadConfig("config.toml"),
		views:  map[int64]*View{},
	}
}

func (e *Editor) Dispatch(action core.Action) {
	core.Bus.Dispatch(action)
}

func (e *Editor) Commandbar() core.Commander {
	return e.Cmdbar
}

func (e *Editor) Quit() {
	if e.fileWatcher != nil {
		e.fileWatcher.Stop()
	}
	event.Shutdown()
	e.term.Close()
	os.Exit(0)
}

// Start starts-up the editor
func (e *Editor) Start(locs []string) {
	err := e.term.Init()
	if err != nil {
		panic(err)
	}

	e.term.SetExtendedColors(core.Colors == 256)
	e.theme, err = core.ReadTheme(core.FindResource(path.Join("themes", e.config.Theme)))
	if err != nil {
		panic(err)
	}
	e.Fg = e.theme.Fg
	e.Bg = e.theme.Bg

	h, w := e.term.Size()
	e.Cmdbar = &Cmdbar{}
	e.Cmdbar.SetBounds(0, 0, 0, w)
	e.Statusbar = &Statusbar{}
	e.Statusbar.SetBounds(h-1, 0, h-1, w)
	dirs := []string{}
	files := []string{}
	for _, loc := range locs {
		if stat, err := os.Stat(loc); err == nil && stat.IsDir() {
			dirs = append(dirs, loc)
		} else {
			files = append(files, loc)
		}
	}
	if len(dirs) == 0 {
		if len(files) > 0 {
			dirs = []string{path.Dir(locs[0])}
		} else {
			dirs = []string{"."}
		}
	}
	e.Cols = append(e.Cols, &Col{WidthRatio: 1.0})
	ratio := 1.0 / float64(len(dirs))
	for _, dir := range dirs {
		view := e.NewView(dir)
		view.HeightRatio = ratio
		e.Cols[0].Views = append(e.Cols[0].Views, view.Id())
		e.Open(dir, view.Id(), "", true)
	}
	e.CurCol = e.Cols[0]
	e.curViewId = e.CurCol.Views[0]
	if len(files) > 0 {
		e.CurCol.WidthRatio = 0.2
		c := &Col{WidthRatio: 0.8}
		ratio := 1.0 / float64(len(files))
		for _, f := range files {
			view := e.NewView(f)
			view.HeightRatio = ratio
			c.Views = append(c.Views, view.Id())
			e.Open(f, view.Id(), "", true)
		}
		e.Cols = append(e.Cols, c)
		e.CurCol = c
		e.curViewId = c.Views[0]
	}

	actions.Ar.EdResize(e.term.Size())

	actions.Ar.EdRender()

	if e.fileWatcher != nil {
		go e.fileWatcher.Start()
	}

	go core.Bus.Start()

	go e.autoScroller()

	go event.Listen()

	e.term.Listen()
}

// Open opens a given location in the editor (in the given view)
// or new view if viewId <= 0
func (e *Editor) Open(loc string, viewId int64, rel string, create bool) (int64, error) {
	loc = strings.TrimSpace(loc)
	rel = strings.TrimSpace(rel)
	if len(rel) > 0 && !strings.HasPrefix(loc, string(os.PathSeparator)) {
		loc = path.Join(rel, loc)
	}
	// make it absolute
	loc, err := filepath.Abs(loc)
	if err != nil {
		return -1, err
	}
	stat, err := os.Stat(loc)
	newFile := false
	if os.IsNotExist(err) {
		if !create {
			return -1, err
		}
		newFile = true
	}
	title := filepath.Base(loc)
	if !newFile && stat.IsDir() {
		loc += string(os.PathSeparator)
		title += string(os.PathSeparator)
	}
	nv := false
	var view core.Viewable
	if viewId <= 0 { // if already have a view for that path, use it
		views := e.ViewsByLoc(loc)
		if len(views) > 0 {
			viewId = views[0]
		}
	}
	if viewId <= 0 {
		view = e.NewFileView(loc)
		nv = true
	} else {
		view = e.ViewById(viewId)
	}
	view.Reset()
	view.SetTitle(title)
	if newFile || !stat.IsDir() {
		err = e.openFile(loc, view)
	} else {
		err = e.openDir(loc, view)
	}
	if err != nil {
		return -1, err
	}
	if nv {
		if stat != nil && stat.IsDir() {
			e.AddDirViewSmart(viewCast(view))
		} else {
			e.InsertViewSmart(viewCast(view))
		}
	}
	view.Reset()
	view.SetWorkDir(filepath.Dir(loc))
	return view.Id(), nil
}

// OpenDir opens a directory listing
func (e *Editor) openDir(loc string, view core.Viewable) error {
	v := viewCast(view)
	if v == nil {
		return fmt.Errorf("No such view")
	}
	v.highlighter = &TermHighlighter{}
	args := append([]string{"ls"}, core.OsLsArgs...)
	title := filepath.Base(loc) + "/"
	backend, err := backend.NewMemBackendCmd(args, loc, view.Id(), &title, true)
	if err != nil {
		return err
	}
	actions.Ar.ViewSetType(v.Id(), core.ViewTypeDirListing)
	view.SetBackend(backend)
	e.SetStatus(fmt.Sprintf("%v", view.WorkDir()))
	return nil
}

// OpenFile opens a file in the editor
func (e *Editor) openFile(loc string, view core.Viewable) error {
	b, err := backend.NewFileBackend(loc, view.Id())
	if err != nil {
		return err
	}
	view.SetBackend(b)
	e.SetStatus(fmt.Sprintf("%v  [%d]", view.WorkDir(), view.Id()))
	view.SetDirty(false)
	e.ViewActivate(view.Id())
	return nil
}

func (e *Editor) Render() {
	e.TermFB(e.theme.Fg, e.theme.Bg)
	e.term.Clear(e.Bg, e.Bg)

	for _, c := range e.Cols {
		for _, v := range c.Views {
			e.ViewById(v).Render()
		}
	}

	// cursor
	v := viewCast(e.CurView())
	cc, cl := v.CurCol(), v.CurLine()
	c, _, _ := v.CurChar()
	// With some terminals & color schemes the cursor might be "invisible" if we are at a
	// location with no text (ie: end of line)
	// so in that case put as space there to cause the cursor to appear.
	var car = ' '
	if c != nil {
		car = *c
	}
	// Note the terminal inverts the colors where the cursor is
	// this is why this statement might appear "backward"
	e.TermFB(e.theme.BgCursor, e.theme.FgCursor)
	y1, x1, _, _ := v.Bounds()
	e.TermChar(cl+y1-v.offy+2, cc+x1-v.offx+2, car)
	e.TermFB(e.theme.Fg, e.theme.Bg)

	e.Cmdbar.Render()
	e.Statusbar.Render()

	e.TermFlush()
}

func (e *Editor) SetStatusErr(s string) {
	if e.Statusbar == nil {
		return
	}
	e.Statusbar.msg = s
	e.Statusbar.isErr = true
	e.Statusbar.Render()
}

func (e *Editor) SetStatus(s string) {
	if e.Statusbar == nil {
		return
	}
	e.Statusbar.msg = s
	e.Statusbar.msg = s
	e.Statusbar.isErr = false
	e.Statusbar.Render()
}

func (e *Editor) Config() core.Config {
	return *e.config
}

func (e *Editor) Theme() *core.Theme {
	return e.theme
}

func (e *Editor) CurView() core.Viewable {
	v, found := e.views[e.curViewId]
	if !found {
		return e.views[e.Cols[0].Views[0]]
	}
	return v
}

func (e *Editor) CurViewId() int64 {
	return e.curViewId
}

func (e *Editor) SetCursor(y, x int) {
	e.term.SetCursor(x, y)
}

func (e *Editor) CmdOn() bool {
	return e.cmdOn
}

func (e *Editor) SetCmdOn(v bool) {
	e.cmdOn = v
}

func (e *Editor) TermFlush() {
	e.term.Flush()
}

// true if ok to quit
func (e *Editor) QuitCheck() bool {
	for _, c := range e.Cols {
		for _, vi := range c.Views {
			v, found := e.views[vi]
			if found && !v.canClose() {
				return false
			}
		}
	}
	return true
}

func (e *Editor) FileEvent(op core.FileOp, loc string) {
	for _, v := range e.views {
		v.fileEvent(op, loc)
	}
}

func (e *Editor) StartTermView(args []string) int64 {
	vid := exec(args, true)
	v := viewCast(core.Ed.ViewById(vid))
	if v == nil || v.backend == nil {
		return -1
	}
	// source the goed shell script onc eterminal has launched
	b := v.backend.(*backend.BackendCmd)
	ext := ".sh"
	if os.Getenv("SHELL") == "rc" {
		ext = ".rc"
	}
	if os.Getenv("SHELL") == "fish" {
		ext = ".fish"
	}
	cmd := ". $HOME/.goed/default/actions/goed" +
		fmt.Sprintf("%s %d %d\n", ext, core.InstanceId, v.Id())
	go func(cmd string) {
		started := b.WaitRunning(time.Minute)
		if !started {
			return
		}
		end := time.Now().Add(time.Hour).Unix()
		for time.Now().Unix() < end {
			if !b.SubCmdRunning() {
				b.SendBytes([]byte(cmd))
				return
			}
			time.Sleep(50 * time.Millisecond)
		}
	}(cmd)
	return vid
}

// Handle selection auto scrolling of views
func (e *Editor) autoScroller() {
	action := autoScrollAction{}
	pause := 200 * time.Millisecond
	for {
		core.Bus.Dispatch(action)
		time.Sleep(pause)
	}
}

type autoScrollAction struct {
}

func (e autoScrollAction) Run() {
	v := viewCast(core.Ed.ViewById(core.Ed.CurViewId()))
	if v == nil {
		return
	}
	x, y := v.autoScrollX, v.autoScrollY
	if x == 0 && y == 0 {
		return
	}
	if len(v.selections) == 0 {
		return
	}
	s := v.selections[0]
	ln := v.CurLine()
	v.offx += x
	v.offy += y
	if y > 0 {
		s.LineTo += y
	} else {
		s.LineFrom += y
	}
	if x > 0 {
		s.ColTo += x
	} else {
		s.ColFrom += x
	}
	// handle scroll / selection "overflows"
	lnLen := v.LineLen(v.Slice(), ln)
	if v.offy >= v.LineCount()-v.LastViewLine() {
		v.offy = v.LineCount() - v.LastViewLine()
	}
	if v.offy < 0 {
		v.offy = 0
	}
	if v.offx > lnLen-v.LastViewCol() {
		v.offx = lnLen - v.LastViewCol()
	}
	if v.offx < 0 {
		v.offx = 0
	}
	if s.LineFrom < 0 {
		s.LineFrom = 0
	} else if s.LineFrom > v.LineCount() {
		s.LineFrom = v.LineCount()
	}
	if s.LineTo < 0 {
		s.LineTo = 0
	} else if s.LineTo > v.LineCount() {
		s.LineTo = v.LineCount()
	}
	if s.ColFrom < 0 {
		s.ColFrom = 0
	} else if s.ColFrom > lnLen {
		s.ColFrom = lnLen
	}
	if s.ColTo < 0 {
		s.ColTo = 0
	} else if s.ColTo > lnLen {
		s.ColTo = lnLen
	}
	s.Normalize()
	v.selections = []core.Selection{
		s,
	}
	core.Ed.Render()
}

// TODO: Do away with those ugly assertions
func viewCast(v core.Viewable) *View {
	if v == nil {
		return nil
	}
	return v.(*View)
}

func widgetCast(w Renderer) *View {
	if w == nil {
		return nil
	}
	if v, ok := w.(*View); ok {
		return v
	}
	return nil
}
<|endoftext|>